diff --git a/.github/workflows/build-deploy-linux-cuda-11.0.yml b/.github/workflows/build-deploy-linux-cuda-11.0.yml index 98ca27951..061c16afc 100644 --- a/.github/workflows/build-deploy-linux-cuda-11.0.yml +++ b/.github/workflows/build-deploy-linux-cuda-11.0.yml @@ -64,6 +64,6 @@ jobs: nvcc --version sudo apt-get autoremove sudo apt-get clean - mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Dlibnd4j.chip=cuda -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Pcuda clean --batch-mode deploy -DskipTests + mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Dlibnd4j.chip=cuda -pl ":nd4j-cuda-11.2,:deeplearning4j-cuda-11.2,:libnd4j" --also-make -Pcuda clean --batch-mode deploy -DskipTests diff --git a/.github/workflows/build-deploy-windows-cuda-11.0.yml b/.github/workflows/build-deploy-windows-cuda-11.0.yml index 4b847aa4f..2f75ee74f 100644 --- a/.github/workflows/build-deploy-windows-cuda-11.0.yml +++ b/.github/workflows/build-deploy-windows-cuda-11.0.yml @@ -54,6 +54,6 @@ jobs: dir "%CUDA_PATH%\lib" set "PATH=C:\msys64\usr\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\lib\x64;%PATH%" echo "Running cuda build" - mvn -Possrh -Djavacpp.platform=windows-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Djavacpp.platform=windows-x86_64 -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda -Pcuda clean --batch-mode deploy -DskipTests + mvn -Possrh -Djavacpp.platform=windows-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Djavacpp.platform=windows-x86_64 -pl ":nd4j-cuda-11.2,:deeplearning4j-cuda-11.2,:libnd4j" --also-make -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda -Pcuda clean --batch-mode deploy -DskipTests diff --git a/.github/workflows/build-deploy-windows.yml b/.github/workflows/build-deploy-windows.yml index 6612c6465..5ef668698 100644 --- a/.github/workflows/build-deploy-windows.yml +++ b/.github/workflows/build-deploy-windows.yml @@ -16,7 +16,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} windows-x86_64: needs: pre-ci - runs-on: windows-2016 + runs-on: windows-2019 steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.8.0 diff --git a/.github/workflows/cpu-integration-tests.yaml b/.github/workflows/cpu-integration-tests.yaml index bba0e345d..2128c363b 100644 --- a/.github/workflows/cpu-integration-tests.yaml +++ b/.github/workflows/cpu-integration-tests.yaml @@ -31,7 +31,7 @@ jobs: protoc --version cd dl4j-test-resources-master && mvn clean install -DskipTests && cd .. export OMP_NUM_THREADS=1 - mvn -Pintegration-tests -Ptestresources -Dlibnd4j.buildthreads=1 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test + mvn -Pintegration-tests -Ptestresources -Dlibnd4j.buildthreads=1 -Ptest-nd4j-native -Dlibnd4j.chip=cpu clean test windows-x86_64: runs-on: windows-2019 @@ -39,12 +39,12 @@ jobs: - uses: actions/checkout@v2 - uses: ./.github/actions/msys2-base-setup - uses: ./.github/actions/download-dl4j-test-resources-windows - - name: Run testsLossOpValidation + - name: Run tests shell: cmd run: | set "PATH=C:\msys64\usr\bin;%PATH%" export OMP_NUM_THREADS=1 - mvn -DskipTestResourceEnforcement=true -Pintegration-tests -Ptestresources -Dlibnd4j.buildthreads=1 -Dlibnd4j.build="Debug" -Djavacpp.platform=windows-x86_64 -libnd4j.platform=windows-x86_64 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test + mvn -DskipTestResourceEnforcement=true -Pintegration-tests -Ptestresources -Dlibnd4j.buildthreads=1 -Dlibnd4j.build="Debug" -Djavacpp.platform=windows-x86_64 -libnd4j.platform=windows-x86_64 -Ptest-nd4j-native -Dlibnd4j.chip=cpu clean test @@ -60,5 +60,5 @@ jobs: run: | brew install unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python export OMP_NUM_THREADS=1 - mvn -Pintegration-tests -Dlibnd4j.build="Debug" -Dlibnd4j.buildthreads=1 -Ptestresources -Djavacpp.platform=macosx-x86_64 -libnd4j.platform=macosx-x86_64 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test + mvn -Pintegration-tests -Dlibnd4j.build="Debug" -Dlibnd4j.buildthreads=1 -Ptestresources -Djavacpp.platform=macosx-x86_64 -libnd4j.platform=macosx-x86_64 -Ptest-nd4j-native -Dlibnd4j.chip=cpu clean test diff --git a/.github/workflows/cpu-sanity-check-tests.yaml b/.github/workflows/cpu-sanity-check-tests.yaml index e116885c8..2737672bc 100644 --- a/.github/workflows/cpu-sanity-check-tests.yaml +++ b/.github/workflows/cpu-sanity-check-tests.yaml @@ -31,7 +31,7 @@ jobs: protoc --version cd dl4j-test-resources-master && mvn clean install -DskipTests && cd .. export OMP_NUM_THREADS=1 - mvn -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -DexcludedGroups="long-running-tests,large-resources" -Dlibnd4j.buildthreads=1 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test --fail-never + mvn -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -Dlibnd4j.buildthreads=1 -Ptest-nd4j-native -Dlibnd4j.chip=cpu clean test windows-x86_64: runs-on: windows-2019 @@ -44,7 +44,7 @@ jobs: run: | set "PATH=C:\msys64\usr\bin;%PATH%" export OMP_NUM_THREADS=1 - mvn -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -DexcludedGroups="long-running-tests,large-resources" -DskipTestResourceEnforcement=true -Ptestresources -Dlibnd4j.buildthreads=1 -Dlibnd4j.build="Debug" -Djavacpp.platform=windows-x86_64 -libnd4j.platform=windows-x86_64 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test --fail-never + mvn -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -DskipTestResourceEnforcement=true -Ptestresources -Dlibnd4j.buildthreads=1 -Dlibnd4j.build="Debug" -Djavacpp.platform=windows-x86_64 -libnd4j.platform=windows-x86_64 -Ptest-nd4j-native -Dlibnd4j.chip=cpu clean test @@ -60,5 +60,5 @@ jobs: run: | brew install unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python export OMP_NUM_THREADS=1 - mvn -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -Dlibnd4j.build="Debug" -Dlibnd4j.buildthreads=1 -Ptestresources -Djavacpp.platform=macosx-x86_64 -libnd4j.platform=macosx-x86_64 -Pnd4j-tests-cpu -Dlibnd4j.chip=cpu clean test + mvn -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -Dlibnd4j.build="Debug" -Dlibnd4j.buildthreads=1 -Ptestresources -Djavacpp.platform=macosx-x86_64 -libnd4j.platform=macosx-x86_64 -Ptest-nd4j-native -Dlibnd4j.chip=cpu clean test diff --git a/.github/workflows/run-cpu-integration-tests-self-hosted.yml b/.github/workflows/run-cpu-integration-tests-self-hosted.yml deleted file mode 100644 index b18938ade..000000000 --- a/.github/workflows/run-cpu-integration-tests-self-hosted.yml +++ /dev/null @@ -1,29 +0,0 @@ -on: - workflow_dispatch: -jobs: - linux-x86_64: - runs-on: [self-hosted] - steps: - - uses: AutoModality/action-clean@v1 - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.8.0 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@v2 - - uses: ./.github/actions/download-dl4j-test-resources-linux - - name: Run cpu tests - shell: bash - env: - DEBIAN_FRONTEND: noninteractive - run: | - export PATH="/opt/protobuf/bin:/usr/local/cuda-11.2/bin:$PATH" - nvcc --version - mvn --version - cmake --version - protoc --version - export OMP_NUM_THREADS=1 - mkdir -p ${GITHUB_WORKSPACE}/resources - mkdir -p ${GITHUB_WORKSPACE}/cache - mvn -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -DexcludedGroups="long-running-tests, large-resources, distributed-systems" -DskipTestResourceEnforcement=true -Ptestresources -Pintegration-tests -Pnd4j-tests-cpu clean test --fail-never - mvn -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -Dgroups="long-running-tests, large-resources, distributed-systems" -Ptestresources -Pnd4j-tests-cpu -Dtest.offheap.size=14g -Dtest.heap.size=6g -Dsurefire.parallel.forcedTimeout=500 -Dsurefire.parallel.timeout=500 -Dsurefire.timeout=200 -Dsurefire.exitTimeout=500 test --fail-never -rf :nd4j - diff --git a/.github/workflows/run-cpu-tests-sanity-checks.yml b/.github/workflows/run-cpu-tests-sanity-checks.yml index 874abb234..47202170c 100644 --- a/.github/workflows/run-cpu-tests-sanity-checks.yml +++ b/.github/workflows/run-cpu-tests-sanity-checks.yml @@ -34,5 +34,5 @@ jobs: cmake --version protoc --version export OMP_NUM_THREADS=1 - mvn -DskipTestResourceEnforcement=true -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -DexcludedGroups="long-running-tests,large-resources" -Pnd4j-tests-cpu --also-make clean test --fail-never + mvn -DskipTestResourceEnforcement=true -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-native,:samediff-import,:libnd4j" -Ptest-nd4j-native --also-make clean test diff --git a/.github/workflows/run-gpu-integration-tests-self-hosted.yml b/.github/workflows/run-gpu-integration-tests-self-hosted.yml deleted file mode 100644 index caeb13de3..000000000 --- a/.github/workflows/run-gpu-integration-tests-self-hosted.yml +++ /dev/null @@ -1,56 +0,0 @@ -on: - workflow_dispatch: -jobs: - # Wait for up to a minute for previous run to complete, abort if not done by then - pre-ci: - runs-on: self-hosted - timeout-minutes: 1 - steps: - - name: 'Block Concurrent Executions' - uses: softprops/turnstyle@v1 - with: - poll-interval-seconds: 10 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - linux-x86_64: - needs: pre-ci - runs-on: [self-hosted] - steps: - - uses: AutoModality/action-clean@v1 - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.8.0 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@v2 - - uses: ./.github/actions/download-dl4j-test-resources-linux - - name: Run gpu tests - shell: bash - env: - DEBIAN_FRONTEND: noninteractive - run: | - export PATH="/opt/protobuf/bin:/usr/local/cuda-11/bin:$PATH" - nvcc --version - mvn --version - cmake --version - protoc --version - export OMP_NUM_THREADS=1 - mkdir -p ${GITHUB_WORKSPACE}/resources - mkdir -p ${GITHUB_WORKSPACE}/cache - export CUDA_VISIBLE_DEVICES=0 - echo "Running tests for cuda 11.0" - export PATH="/opt/protobuf/bin:/usr/local/cuda-11.2/bin:$PATH" - mvn -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cuda -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Pcuda clean --batch-mode install -DskipTests - mvn -Djunit.jupiter.execution.parallel.enabled=false -Dtest.offheap.size=6g -Pcuda -Dlibnd4j.chip=cuda -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -DexcludedGroups="long-running-tests, large-resources, distributed-systems" -DskipTestResourceEnforcement=true -Ptestresources -Pintegration-tests -Pnd4j-tests-cuda clean test --fail-never -rf :nd4j - #mvn -Pcuda -Dlibnd4j.chip=cuda -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -Dgroups="long-running-tests, large-resources, distributed-systems" -Ptestresources -Pnd4j-tests-cuda -Dtest.offheap.size=14g -Dtest.heap.size=6g -Dsurefire.parallel.forcedTimeout=200 -Dsurefire.parallel.timeout=200 -Dsurefire.timeout=200 -Dsurefire.exitTimeout=200 test --fail-never -rf :nd4j - echo "Running tests for cuda 11.2" - ${GITHUB_WORKSPACE}/change-cuda-versions.sh 11.2 - echo "Changed cuda to 11.2" - export PATH="/opt/protobuf/bin:/usr/local/cuda-11.2/bin:$PATH" - echo "Updated path for 11.2" - echo "Installing jars for 11.2" - mvn -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cuda -pl ":nd4j-cuda-11.2,:deeplearning4j-cuda-11.2,:libnd4j" --also-make -Pcuda clean --batch-mode install -DskipTests - echo "Installed jars for 11.2, running smaller tests for cuda 11.2" - mvn -Djunit.jupiter.execution.parallel.enabled=false -Dtest.offheap.size=4g -Pcuda -Dlibnd4j.chip=cuda -Dlibnd4j.chip=cuda -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -DexcludedGroups="long-running-tests, large-resources, distributed-systems" -DskipTestResourceEnforcement=true -Ptestresources -Pintegration-tests -Pnd4j-tests-cuda clean test --fail-never -rf :nd4j - #echo "Running larger for cuda 11.2" - #mvn -Pcuda -Dlibnd4j.chip=cuda -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -Dgroups="long-running-tests, large-resources, distributed-systems" -Ptestresources -Pnd4j-tests-cuda -Dtest.offheap.size=14g -Dtest.heap.size=6g -Dsurefire.parallel.forcedTimeout=200 -Dsurefire.parallel.timeout=200 -Dsurefire.timeout=200 -Dsurefire.exitTimeout=200 test --fail-never -rf :nd4j - diff --git a/.github/workflows/run-gpu-tests-sanity-checks.yml b/.github/workflows/run-gpu-tests-sanity-checks.yml index 8e5ee37b7..96ebf3364 100644 --- a/.github/workflows/run-gpu-tests-sanity-checks.yml +++ b/.github/workflows/run-gpu-tests-sanity-checks.yml @@ -35,5 +35,5 @@ jobs: protoc --version bash ./change-cuda-versions.sh 11.2 export OMP_NUM_THREADS=1 - mvn -DskipTestResourceEnforcement=true -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-cuda-11.2,:samediff-import,:libnd4j" -Dlibnd4j.helper=cudnn -Ptest-nd4j-cuda --also-make -Dlibnd4j.chip=cuda clean test --fail-never + mvn -DskipTestResourceEnforcement=true -Ptestresources -pl ":deeplearning4j-modelimport,:deeplearning4j-core,:nd4j-cuda-11.2,:samediff-import,:libnd4j" -Dlibnd4j.compute="5.0 5.2 5.3 6.0 8.0" -Ptest-nd4j-cuda --also-make -Dlibnd4j.chip=cuda clean test diff --git a/.gitignore b/.gitignore index 750bdc186..09430be6d 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,6 @@ release.properties *.iml *.prefs *.dylib -lib/ .vs/ .vscode/ nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/bin @@ -75,9 +74,11 @@ nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativebla *.orig #libnd4j cmake -libnd4j/cmake* +bruai4j-native-common/cmake* #vim *.swp -*.dll \ No newline at end of file +*.dll +/bruai4j-native/bruai4j-native-common/blasbuild/ +/bruai4j-native/bruai4j-native-common/build/ diff --git a/ADRs/0002-ONNX_Runtime.md b/ADRs/0002-ONNX_Runtime.md index c3b843652..bb22d0cec 100644 --- a/ADRs/0002-ONNX_Runtime.md +++ b/ADRs/0002-ONNX_Runtime.md @@ -1,7 +1,7 @@ # Onnx runtime module ## Status -Implemented +Proposed Proposed by: Adam Gibson (23-09-2020) diff --git a/ADRs/0003-Import_IR.md b/ADRs/0003-Import_IR.md index 7f471b97d..eef7a789a 100644 --- a/ADRs/0003-Import_IR.md +++ b/ADRs/0003-Import_IR.md @@ -2,7 +2,7 @@ ## Status -Implemented +Proposed Proposed by: Adam Gibson (28-09-2020) diff --git a/ADRs/0003-NdArray_Strides_ArmCompute.md b/ADRs/0003-NdArray_Strides_ArmCompute.md index 0fb153d68..02e3b2a34 100644 --- a/ADRs/0003-NdArray_Strides_ArmCompute.md +++ b/ADRs/0003-NdArray_Strides_ArmCompute.md @@ -1,8 +1,9 @@ + # Libnd4j NdArray padded buffers, strides for Arm_Compute Library wrapper ## Status -Implemented +PROPOSED Proposed by: Abdelrauf (23/09/2020) diff --git a/ADRs/0004-Mapping_IR.md b/ADRs/0004-Mapping_IR.md index ab6b64a74..b62eba532 100644 --- a/ADRs/0004-Mapping_IR.md +++ b/ADRs/0004-Mapping_IR.md @@ -1,7 +1,7 @@ # Import IR ## Status -Implemented +Proposed Proposed by: Adam Gibson (28-09-2020) diff --git a/ADRs/0005-Interpreter.md b/ADRs/0005-Interpreter.md index db57fbf79..6e2cc44d1 100644 --- a/ADRs/0005-Interpreter.md +++ b/ADRs/0005-Interpreter.md @@ -1,7 +1,7 @@ # Interpreter ## Status -Rejected +Proposed Proposed by: Adam Gibson (28-09-2020) diff --git a/ADRs/0006 - Test architecture.md b/ADRs/0006 - Test architecture.md deleted file mode 100644 index 5ed86ffcb..000000000 --- a/ADRs/0006 - Test architecture.md +++ /dev/null @@ -1,77 +0,0 @@ -# Junit 5 tag usage - -## Status -Proposed - -Proposed by: Adam Gibson (21-03-2021) - -Discussed with: N/A - -## Context -DL4J was a junit 4 based code based for testing. -It's now based on junit 5's jupiter API, which has support for [Tags](https://junit.org/junit5/docs/5.0.1/api/org/junit/jupiter/api/Tag.html). - -DL4j's code base has a number of different kinds of tests that fall in to several categories: -1. Long and flaky involving distributed systems (spark, parameter-server) -2. Code that requires large downloads, but runs quickly -3. Quick tests that test basic functionality -4. Comprehensive integration tests that test several parts of a code base - -Due to the variety of behaviors across different tests, it's hard to tell what's actually needed -for running and validating whether changes work against such a complex test base. - -Much of the time, most of the tests aren't related to a given change. -Often times, quick sanity checks are all that's needed in order to make sure a change works. - -A common set of tags is used to filter which tests are needed to run when. -This allows us to retain complex integration tests and run them on a set schedule -to catch regressions while allowing a defined subset of tests to run for a quick feedback loop. - - - - -## Decision - -A few kinds of tags exist: -1. Time based: long-time,short-time -2. Network based: has-download -3. Distributed systems: spark, multi-threaded -4. Functional cross-cutting concerns: multi module tests, similar functionality (excludes time based) -5. Platform specific tests that can vary on different hardware: cpu, gpu -6. JVM crash: (jvm-crash) Tests with native code can crash the JVM for tests. It's useful to be able to turn those off when debugging.: jvm-crash -7. RNG: (rng) for RNG related tests -8. Samediff:(samediff) samediff related tests -9. Training related functionality -10. long-running-tests: The longer running tests that take a longer execution time -11. large-resources: tests requiring a large amount of ram/cpu (>= 2g up to 16g) - - -New maven properties for maven surefire: -test.offheap.size: tunes off heap size for javacpp -test.heap.size: tunes heap size of test jvms - - -Auto tuning the number of CPU cores for tests relative to the number of CPUs present - - - -## Consequences -### Advantages -* Ability to sort through and filter tests based on different running environments - -* Ability to reason about test suites as a whole dynamically across modules - -* Avoid the need to define test suites - -* Ability to define groups of tags based in profiles - -* Ability to dynamically filter tests from the maven command line - - -### Disadvantages - -* Documentation and maintenance burden needing to know what tags do what - -* Test maintenance for newcomers who may not know how to tag tests - - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 0a25d9775..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,51 +0,0 @@ -# Contributing to Deeplearning4j - -Thanks for your interest in DL4J. Our goal is to bring fast, open-source deep learning to all JVM-based communities. - - -## Getting Started - -Deeplearning4j's [open issues are here](https://github.com/eclipse/deeplearning4j/issues). In time, we'll tag issues that would make a good first pull request for new contributors. An easy way to get started helping the project is to *file an issue*. You can do that on the Deeplearning4j issues page by clicking on the green button at the right. Issues can include bugs to fix, features to add, or documentation that looks outdated. - -Note that you will need to [build dl4j from source](https://deeplearning4j.org/docs/latest/deeplearning4j-build-from-source) - -For some tips on contributing to open source, this [post is helpful](https://smartbear.com/blog/test-and-monitor/14-ways-to-contribute-to-open-source-without-being/). - -## Contributions - -Deeplearning4j welcomes contributions from everyone. - -Contributions to Deeplearning4j should be made in the form of GitHub pull requests. Each pull request will -be reviewed by a core contributor (someone with permission to land patches) and either landed in the -main tree or given feedback for changes that would be required. - -## Pull Request Checklist - -- Branch from the master branch and, if needed, rebase to the current master - branch before submitting your pull request. If it doesn't merge cleanly with - master you may be asked to rebase your changes. - -- Commits should be as small as possible, while ensuring that each commit is - correct independently (i.e., each commit should compile and pass tests). - -- Don't put submodule updates in your pull request unless they are to landed - commits. - -- If your patch is not getting reviewed or you need a specific person to review - it, you can @-reply a reviewer asking for a review in the pull request or a - comment. - -- Work-in-progress pull requests are welcome. Please prefix them with `[WIP]` to tell the continuous integration (CI) backend not to run tests/checks on them (until that tag is removed and another commit is pushed up). - -- Add tests relevant to the fixed bug or new feature. - -## Conduct & License - -We follow the [Rust Code of Conduct](http://www.rust-lang.org/conduct.html). - -All code in this repository is released under the Apache Software Foundation License, 2.0, and by contributing to this repository, you agree to release that contribution under that same license. - - -## Eclipse Contributor Agreement and Commit Signing - -Please see the following page for details: [https://deeplearning4j.org/eclipse-contributors](https://deeplearning4j.org/eclipse-contributors) \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 74503dbea..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,29 +0,0 @@ - - /* ****************************************************************************** - * - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * See the NOTICE file distributed with this work for additional - * information regarding copyright ownership. - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -#!groovy - -/* - To redefine some job/run parameters, - please provide arguments to jenkinsBuilder step. - Example: jenkinsBuilder platforms: [] - */ - -jenkinsBuilder() - diff --git a/LICENSE b/LICENSE index 77fac477a..7a4a3ea24 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ -Apache License + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -178,7 +179,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +187,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -198,187 +199,4 @@ Apache License distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. - -########################## - -Keras code - -Auto-generated documentation: https://github.com/deeplearning4j/deeplearning4j/blob/master/docs/doc_generator.py - -COPYRIGHT - -All contributions by François Chollet: -Copyright (c) 2015 - 2018, François Chollet. -All rights reserved. - -All contributions by Google: -Copyright (c) 2015 - 2018, Google, Inc. -All rights reserved. - -All contributions by Microsoft: -Copyright (c) 2017 - 2018, Microsoft, Inc. -All rights reserved. - -All other contributions: -Copyright (c) 2015 - 2018, the respective contributors. -All rights reserved. - -Each contributor holds copyright over their respective contributions. -The project versioning (Git) records all such contribution source information. - -The MIT License (MIT) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -########################## - -OpenCSV Code - -CSVParser: https://github.com/deeplearning4j/deeplearning4j/blob/master/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/SerializableCSVParser.java - -Apache 2.0 License - -All contributions by Bytecode Pty Ltd. -Copyright 2005 Bytecode Pty Ltd. -All rights reserved. - - -########################## - -Aeron Code - -Modifed Code: nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronUtil.java - -Copyright 2014 - 2016 Real Logic Ltd. All rights reserved. - -Apache License, Version 2.0 - - -########################## - -cnpy Code - -Forked Code: libnd4j/include/cnpy/ - -The MIT License - -Copyright (c) Carl Rogers, 2011 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -########################## - -Protocol Buffers Code - -Codebase: nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/google/protobuf/ - -Protocol Buffers - Google's data interchange format -Copyright 2008 Google Inc. All rights reserved. -https://developers.google.com/protocol-buffers/ - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -########################## - -ONNX Code - -Protocol Buffers: nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/ - -Copyright (c) Facebook Inc. and Microsoft Corporation. All rights reserved. - -Licensed under the MIT license. - - -########################## - -TensorFlow Code - -Protocol Buffers: nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/ -Operations: libnd4j/include/ops/declarable/generic/parity_ops/ - -Copyright 2015-2017 The TensorFlow Authors. All rights reserved. - -Apache License, Version 2.0 - - -########################## - -Ansj Code - -Codebase: deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp-chinese/src/main/java/org/ansj/ -Resources: deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp-chinese/src/main/resources/ - -Copyright 2011-2016 ansj_seg. All rights reserved. - -Apache License, Version 2.0 - - -########################## - -Kuromoji Code - -Codebase: deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp-japanese/src/main/java/com/atilika/kuromoji/ - -Copyright (c) 2010-2015 Atilika Inc. and contributors. All rights reserved. - -Apache License, Version 2.0 + limitations under the License. \ No newline at end of file diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..96214742d --- /dev/null +++ b/NOTICE @@ -0,0 +1,23 @@ +Brutex Network Deeplearning4j +Copyright 2021 Brutex Network Contributors + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software developed by +* Brian Rosenberger. Copyright (C) 2021 Brian Rosenberger. + +This product includes software developed at +* Eclipse Deeplearning4j (Apache 2.0). Copyright 2020-2021 Eclipse Deeplearning4j Contributors + +This product includes software developed at +* Skymind Inc (Apache 2.0). Copyright (C) 2015-2018 Skymind Inc. + +This product includes software developed at +* Konduit KK (Apache 2.0). Copyright (C) 2020. + +This product includes software from the Tensorflow Project (Apache 2.0). +* Copyright (C) 2015-2018 Tensorflow Authors. + +This product includes software from the Onnx Project project (Apache 2.0). +* Copyright (C) 2020 Onnx Contributors (https://github.com/onnx/onnx) \ No newline at end of file diff --git a/NOTICE.txt b/NOTICE.txt deleted file mode 100644 index ae3fdc115..000000000 --- a/NOTICE.txt +++ /dev/null @@ -1,20 +0,0 @@ -Eclipse Deeplearning4j -Copyright 2021 Eclipse Deeplearning4j Contributors - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes software developed at -* Skymind Inc (Apache 2.0). Copyright (C) 2015-2018 Skymind Inc . - -This product includes software developed at -* Konduit KK (Apache 2.0). Copyright (C) 2020. - - -This product includes software from the Tensorflow Project (Apache 2.0). -* Copyright (C) 2015-2018 Tensorflow Authors. - -# https://github.com/onnx/onnx - -This product includes software from the Onnx Project project (Apache 2.0). -* Copyright (C) 2020 Onnx Contributors (https://github.com/onnx/onnx) \ No newline at end of file diff --git a/README.md b/README.md index 23ed01183..e3eb6ba84 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,13 @@

- [![Documentation](https://img.shields.io/badge/user-documentation-blue.svg)](https://deeplearning4j.konduit.ai/) -[![Get help at the community forum](https://img.shields.io/badge/Get%20Help-Community%20Forum-blue)](https://community.konduit.ai/) -[![javadoc](https://javadoc.io/badge2/org.deeplearning4j/deeplearning4j-nn/DL4J%20API%20Doc.svg)](https://javadoc.io/doc/org.deeplearning4j/deeplearning4j-nn) -[![javadoc](https://javadoc.io/badge2/org.nd4j/nd4j-api/ND4J%20API%20Doc.svg)](https://javadoc.io/doc/org.nd4j/nd4j-api) + [![Documentation](https://img.shields.io/badge/user-documentation-blue.svg)](https://deeplearning4j.org) +[![Get help at the community forum](https://img.shields.io/badge/Get%20Help-Community%20Forum-blue)](https://www.reddit.com/r/deeplearning4j/) +[![javadoc](https://javadoc.io/badge2/org.deeplearning4j/deeplearning4j-nn/DL4J%20API%20Doc.svg)](https://deeplearning4j.org/api/latest/) [![License](https://img.shields.io/github/license/eclipse/deeplearning4j)](LICENSE) -![GitHub commit activity](https://img.shields.io/github/commit-activity/m/konduitai/deeplearning4j) -The **[Eclipse Deeplearning4J](https://deeplearning4j.konduit.ai/)** (DL4J) ecosystem is a set of projects intended to support all the needs of a JVM based deep learning application. This means starting with the raw data, loading and preprocessing it from wherever and whatever format it is in to building and tuning a wide variety of simple and complex deep learning networks. +The **[Eclipse Deeplearning4J](https://deeplearning4j.org)** (DL4J) ecosystem is a set of projects intended to support all the needs of a JVM based deep learning application. This means starting with the raw data, loading and preprocessing it from wherever and whatever format it is in to building and tuning a wide variety of simple and complex deep learning networks. Because Deeplearning4J runs on the JVM you can use it with a wide variety of JVM based languages other than Java, like Scala, Kotlin, Clojure and many more. @@ -22,7 +20,7 @@ The DL4J stack comprises of: - **Arbiter**: Library for hyperparameter search - **LibND4J** : C++ library that underpins everything. For more information on how the JVM acceses native arrays and operations refer to [JavaCPP](https://github.com/bytedeco/javacpp) -All projects in the DL4J ecosystem support Windows, Linux and macOS. Hardware support includes CUDA GPUs (10.0, 10.1, 10.2 except OSX), x86 CPU (x86_64, avx2, avx512), ARM CPU (arm, arm64, armhf) and PowerPC (ppc64le). +All projects in the DL4J ecosystem support Windows, Linux and macOS. Hardware support includes CUDA GPUs (11.2, 10.0, 10.1, 10.2 except OSX), x86 CPU (x86_64, avx2, avx512), ARM CPU (arm, arm64, armhf) and PowerPC (ppc64le). ## Using Eclipse Deeplearning4J in your project @@ -112,9 +110,3 @@ An example of GPU "CC" or compute capability is 61 for Titan X Pascal. ## License [Apache License 2.0](LICENSE) - - -## Commercial Support -Deeplearning4J is actively developed by the team at [Konduit K.K.](http://www.konduit.ai). - -[If you need any commercial support feel free to reach out to us.](https://konduit.ai/konduit-open-source-support/) diff --git a/arbiter/.travis.yml b/arbiter/.travis.yml new file mode 100644 index 000000000..30638a6a9 --- /dev/null +++ b/arbiter/.travis.yml @@ -0,0 +1,24 @@ +branches: + only: + - master +notifications: + email: false +dist: trusty +sudo: false +cache: + directories: + - $HOME/.m2 +language: java +jdk: + - openjdk8 +matrix: + include: + - os: linux + env: OS=linux-x86_64 SCALA=2.10 + install: true + script: bash ./ci/build-linux-x86_64.sh + - os: linux + env: OS=linux-x86_64 SCALA=2.11 + install: true + script: bash ./ci/build-linux-x86_64.sh + diff --git a/arbiter/README.md b/arbiter/README.md new file mode 100644 index 000000000..67124f30a --- /dev/null +++ b/arbiter/README.md @@ -0,0 +1,45 @@ +# Arbiter + +A tool dedicated to tuning (hyperparameter optimization) of machine learning models. Part of the DL4J Suite of Machine Learning / Deep Learning tools for the enterprise. + + +## Modules +Arbiter contains the following modules: + +- arbiter-core: Defines the API and core functionality, and also contains functionality for the Arbiter UI +- arbiter-deeplearning4j: For hyperparameter optimization of DL4J models (MultiLayerNetwork and ComputationGraph networks) + + +## Hyperparameter Optimization Functionality + +The open-source version of Arbiter currently defines two methods of hyperparameter optimization: + +- Grid search +- Random search + +For optimization of complex models such as neural networks (those with more than a few hyperparameters), random search is superior to grid search, though Bayesian hyperparameter optimization schemes +For a comparison of random and grid search methods, see [Random Search for Hyper-parameter Optimization (Bergstra and Bengio, 2012)](http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf). + +### Core Concepts and Classes in Arbiter for Hyperparameter Optimization + +In order to conduct hyperparameter optimization in Arbiter, it is necessary for the user to understand and define the following: + +- **Parameter Space**: A ```ParameterSpace

``` specifies the type and allowable values of hyperparameters for a model configuration of type ```P```. For example, ```P``` could be a MultiLayerConfiguration for DL4J +- **Candidate Generator**: A ```CandidateGenerator``` is used to generate candidate models configurations of some type ```C```. The following implementations are defined in arbiter-core: + - ```RandomSearchCandidateGenerator``` + - ```GridSearchCandidateGenerator``` +- **Score Function**: A ```ScoreFunction``` is used to score a model of type ```M``` given data of type ```D```. For example, in DL4J a score function might be used to calculate the classification accuracy from a DataSetIterator + - A key concept here is that they score is a single numerical (double precision) value that we either want to minimize or maximize - this is the goal of hyperparameter optimization +- **Termination Conditions**: One or more ```TerminationCondition``` instances must be provided to the ```OptimizationConfiguration```. ```TerminationCondition``` instances are used to control when hyperparameter optimization should be stopped. Some built-in termination conditions: + - ```MaxCandidatesCondition```: Terminate if more than the specified number of candidate hyperparameter configurations have been executed + - ```MaxTimeCondition```: Terminate after a specified amount of time has elapsed since starting the optimization +- **Result Saver**: The ```ResultSaver``` interface is used to specify how the results of each hyperparameter optimization run should be saved. For example, whether saving should be done to local disk, to a database, to HDFS, or simply stored in memory. + - Note that ```ResultSaver.saveModel``` method returns a ```ResultReference``` object, which provides a mechanism for re-loading both the model and score from wherever it may be saved. +- **Optimization Configuration**: An ```OptimizationConfiguration``` ties together the above configuration options in a fluent (builder) pattern. +- **Candidate Executor**: The ```CandidateExecutor``` interface provides a layer of abstraction between the configuration and execution of each instance of learning. Currently, the only option is the ```LocalCandidateExecutor```, which is used to execute learning on a single machine (in the current JVM). In principle, other execution methods (for example, on Spark or cloud computing machines) could be implemented. +- **Optimization Runner**: The ```OptimizationRunner``` uses an ```OptimizationConfiguration``` and a ```CandidateExecutor``` to actually run the optimization, and save the results. + + +### Optimization of DeepLearning4J Models + +(This section: forthcoming) diff --git a/arbiter/arbiter-core/pom.xml b/arbiter/arbiter-core/pom.xml new file mode 100644 index 000000000..ab5ded1b8 --- /dev/null +++ b/arbiter/arbiter-core/pom.xml @@ -0,0 +1,97 @@ + + + + + arbiter + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + arbiter-core + jar + + arbiter-core + + + + net.brutex.ai + nd4j-api + ${project.version} + + + com.google.code.findbugs + * + + + + + com.google.guava + guava + ${guava.jre.version} + + + org.apache.commons + commons-lang3 + ${commons.lang.version} + + + + org.apache.commons + commons-math3 + ${commons.math.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + joda-time + joda-time + ${jodatime.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + + net.brutex.ai + deeplearning4j-common-tests + ${project.version} + test + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + ${jackson.version} + + + net.brutex.ai + nd4j-native + ${project.version} + test + windows-x86_64 + + + diff --git a/arbiter/arbiter-core/src/assembly/bin.xml b/arbiter/arbiter-core/src/assembly/bin.xml new file mode 100644 index 000000000..c99d6b144 --- /dev/null +++ b/arbiter/arbiter-core/src/assembly/bin.xml @@ -0,0 +1,91 @@ + + + + bin + + + tar.gz + + + + + + + lib + + *:jar:* + + + *:sources + + + + + + + + + readme.txt + + + + + src/main/resources/bin/ + bin + + arbiter + + unix + 0755 + + + + examples + examples + + + + + + + + target + ./ + + *.jar + + + + + + \ No newline at end of file diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/AbstractParameterSpace.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/AbstractParameterSpace.java new file mode 100644 index 000000000..4ff9dd964 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/AbstractParameterSpace.java @@ -0,0 +1,74 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by Alex on 23/07/2017. + */ +public abstract class AbstractParameterSpace implements ParameterSpace { + + @Override + public Map getNestedSpaces() { + Map m = new LinkedHashMap<>(); + + //Need to manually build and walk the class heirarchy... + Class currClass = this.getClass(); + List> classHeirarchy = new ArrayList<>(); + while (currClass != Object.class) { + classHeirarchy.add(currClass); + currClass = currClass.getSuperclass(); + } + + for (int i = classHeirarchy.size() - 1; i >= 0; i--) { + //Use reflection here to avoid a mass of boilerplate code... + Field[] allFields = classHeirarchy.get(i).getDeclaredFields(); + + for (Field f : allFields) { + + String name = f.getName(); + Class fieldClass = f.getType(); + boolean isParamSpacefield = ParameterSpace.class.isAssignableFrom(fieldClass); + + if (!isParamSpacefield) { + continue; + } + + f.setAccessible(true); + + ParameterSpace p; + try { + p = (ParameterSpace) f.get(this); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + + if (p != null) { + m.put(name, p); + } + } + } + + return m; + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/Candidate.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/Candidate.java new file mode 100644 index 000000000..4f00d92e7 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/Candidate.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.deeplearning4j.arbiter.optimize.generator.util.SerializedSupplier; +import org.nd4j.common.function.Supplier; + +import java.io.Serializable; +import java.util.Map; + +/** + * Candidate: a proposed hyperparameter configuration. + * Also includes a map for data parameters, to configure things like data preprocessing, etc. + */ +@Data +@AllArgsConstructor +public class Candidate implements Serializable { + + private Supplier supplier; + private int index; + private double[] flatParameters; + private Map dataParameters; + private Exception exception; + + public Candidate(C value, int index, double[] flatParameters, Map dataParameters, Exception e) { + this(new SerializedSupplier(value), index, flatParameters, dataParameters, e); + } + + public Candidate(C value, int index, double[] flatParameters) { + this(new SerializedSupplier(value), index, flatParameters); + } + + public Candidate(Supplier value, int index, double[] flatParameters) { + this(value, index, flatParameters, null, null); + } + + public C getValue(){ + return supplier.get(); + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/CandidateGenerator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/CandidateGenerator.java new file mode 100644 index 000000000..3b070fd37 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/CandidateGenerator.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api; + +import org.deeplearning4j.arbiter.optimize.generator.GridSearchCandidateGenerator; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * A CandidateGenerator proposes candidates (i.e., hyperparameter configurations) for evaluation. + * This abstraction allows for different ways of generating the next configuration to test; for example, + * random search, grid search, Bayesian optimization methods, etc. + * + * @author Alex Black + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface CandidateGenerator { + + /** + * Is this candidate generator able to generate more candidates? This will always return true in some + * cases, but some search strategies have a limit (grid search, for example) + */ + boolean hasMoreCandidates(); + + /** + * Generate a candidate hyperparameter configuration + */ + Candidate getCandidate(); + + /** + * Report results for the candidate generator. + * + * @param result The results to report + */ + void reportResults(OptimizationResult result); + + /** + * @return Get the parameter space for this candidate generator + */ + ParameterSpace getParameterSpace(); + + /** + * @param rngSeed Set the random number generator seed for the candidate generator + */ + void setRngSeed(long rngSeed); + + /** + * @return The type (class) of the generated candidates + */ + Class getCandidateType(); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/OptimizationResult.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/OptimizationResult.java new file mode 100644 index 000000000..8868b73ba --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/OptimizationResult.java @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api; + +import lombok.Data; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; + +/** + * An optimization result represents the results of an optimization run, including the canditate configuration, the + * trained model, the score for that model, and index of the model + * + * @author Alex Black + */ +@Data +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +@JsonIgnoreProperties({"resultReference"}) +public class OptimizationResult implements Serializable { + @JsonProperty + private Candidate candidate; + @JsonProperty + private Double score; + @JsonProperty + private int index; + @JsonProperty + private Object modelSpecificResults; + @JsonProperty + private CandidateInfo candidateInfo; + private ResultReference resultReference; + + + public OptimizationResult(Candidate candidate, Double score, int index, Object modelSpecificResults, + CandidateInfo candidateInfo, ResultReference resultReference) { + this.candidate = candidate; + this.score = score; + this.index = index; + this.modelSpecificResults = modelSpecificResults; + this.candidateInfo = candidateInfo; + this.resultReference = resultReference; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/ParameterSpace.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/ParameterSpace.java new file mode 100644 index 000000000..7a2dff8e7 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/ParameterSpace.java @@ -0,0 +1,81 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.util.List; +import java.util.Map; + +/** + * ParameterSpace: defines the acceptable ranges of values a given parameter may take. + * Note that parameter spaces can be simple (like {@code ParameterSpace}) or complicated, including + * multiple nested ParameterSpaces + * + * @author Alex Black + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface ParameterSpace

{ + + /** + * Generate a candidate given a set of values. These values are then mapped to a specific candidate, using some + * mapping function (such as the prior probability distribution) + * + * @param parameterValues A set of values, each in the range [0,1], of length {@link #numParameters()} + */ + P getValue(double[] parameterValues); + + /** + * Get the total number of parameters (hyperparameters) to be optimized. This includes optional parameters from + * different parameter subpaces. (Thus, not every parameter may be used in every candidate) + * + * @return Number of hyperparameters to be optimized + */ + int numParameters(); + + /** + * Collect a list of parameters, recursively. Note that leaf parameters are parameters that do not have any + * nested parameter spaces + */ + List collectLeaves(); + + /** + * Get a list of nested parameter spaces by name. Note that the returned parameter spaces may in turn have further + * nested parameter spaces. The map should be empty for leaf parameter spaces + * + * @return A map of nested parameter spaces + */ + Map getNestedSpaces(); + + /** + * Is this ParameterSpace a leaf? (i.e., does it contain other ParameterSpaces internally?) + */ + @JsonIgnore + boolean isLeaf(); + + /** + * For leaf ParameterSpaces: set the indices of the leaf ParameterSpace. + * Expects input of length {@link #numParameters()}. Throws exception if {@link #isLeaf()} is false. + * + * @param indices Indices to set. Length should equal {@link #numParameters()} + */ + void setIndices(int... indices); + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/TaskCreator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/TaskCreator.java new file mode 100644 index 000000000..c6e58905d --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/TaskCreator.java @@ -0,0 +1,62 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api; + +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; + +import java.util.List; +import java.util.Properties; +import java.util.concurrent.Callable; + +/** + * The TaskCreator is used to take a candidate configuration, data provider and score function, and create something + * that can be executed as a Callable + * + * @author Alex Black + */ +public interface TaskCreator { + + /** + * Generate a callable that can be executed to conduct the training of this model (given the model configuration) + * + * @param candidate Candidate (model) configuration to be trained + * @param dataProvider DataProvider, for the data + * @param scoreFunction Score function to be used to evaluate the model + * @param statusListeners Status listeners, that can be used for callbacks (to UI, for example) + * @return A callable that returns an OptimizationResult, once optimization is complete + */ + @Deprecated + Callable create(Candidate candidate, DataProvider dataProvider, ScoreFunction scoreFunction, + List statusListeners, IOptimizationRunner runner); + + /** + * Generate a callable that can be executed to conduct the training of this model (given the model configuration) + * + * @param candidate Candidate (model) configuration to be trained + * @param dataSource Data source + * @param dataSourceProperties Properties (may be null) for the data source + * @param scoreFunction Score function to be used to evaluate the model + * @param statusListeners Status listeners, that can be used for callbacks (to UI, for example) + * @return A callable that returns an OptimizationResult, once optimization is complete + */ + Callable create(Candidate candidate, Class dataSource, Properties dataSourceProperties, + ScoreFunction scoreFunction, List statusListeners, IOptimizationRunner runner); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/TaskCreatorProvider.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/TaskCreatorProvider.java new file mode 100644 index 000000000..ea0a4f283 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/TaskCreatorProvider.java @@ -0,0 +1,43 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api; + +import java.util.HashMap; +import java.util.Map; + +public class TaskCreatorProvider { + + private static Map, Class> map = new HashMap<>(); + + public synchronized static TaskCreator defaultTaskCreatorFor(Class paramSpaceClass){ + Class c = map.get(paramSpaceClass); + try { + if(c == null){ + return null; + } + return c.newInstance(); + } catch (Exception e){ + throw new RuntimeException("Could not create new instance of task creator class: " + c + " - missing no-arg constructor?", e); + } + } + + public synchronized static void registerDefaultTaskCreatorClass(Class spaceClass, + Class creatorClass){ + map.put(spaceClass, creatorClass); + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/adapter/ParameterSpaceAdapter.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/adapter/ParameterSpaceAdapter.java new file mode 100644 index 000000000..56bd51d69 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/adapter/ParameterSpaceAdapter.java @@ -0,0 +1,82 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.adapter; + +import lombok.AllArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * An abstract class used for adapting one type into another. Subclasses of this need to merely implement 2 simple methods + * + * @param Type to convert from + * @param Type to convert to + * @author Alex Black + */ +@AllArgsConstructor +public abstract class ParameterSpaceAdapter implements ParameterSpace { + + + protected abstract T convertValue(F from); + + protected abstract ParameterSpace underlying(); + + protected abstract String underlyingName(); + + + @Override + public T getValue(double[] parameterValues) { + return convertValue(underlying().getValue(parameterValues)); + } + + @Override + public int numParameters() { + return underlying().numParameters(); + } + + @Override + public List collectLeaves() { + ParameterSpace p = underlying(); + if(p.isLeaf()){ + return Collections.singletonList(p); + } + return underlying().collectLeaves(); + } + + @Override + public Map getNestedSpaces() { + return Collections.singletonMap(underlyingName(), (ParameterSpace)underlying()); + } + + @Override + public boolean isLeaf() { + return false; //Underlying may be a leaf, however + } + + @Override + public void setIndices(int... indices) { + underlying().setIndices(indices); + } + + @Override + public String toString() { + return underlying().toString(); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataProvider.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataProvider.java new file mode 100644 index 000000000..23918373f --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataProvider.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.data; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; +import java.util.Map; + +/** + * DataProvider interface abstracts out the providing of data + * @deprecated Use {@link DataSource} + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +@Deprecated +public interface DataProvider extends Serializable { + + /** + * Get training data given some parameters for the data. + * Data parameters map is used to specify things like batch + * size data preprocessing + * + * @param dataParameters Parameters for data. May be null or empty for default data + * @return training data + */ + Object trainData(Map dataParameters); + + /** + * Get training data given some parameters for the data. Data parameters map is used to specify things like batch + * size data preprocessing + * + * @param dataParameters Parameters for data. May be null or empty for default data + * @return training data + */ + Object testData(Map dataParameters); + + Class getDataType(); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataSetIteratorFactoryProvider.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataSetIteratorFactoryProvider.java new file mode 100644 index 000000000..3766338a9 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataSetIteratorFactoryProvider.java @@ -0,0 +1,89 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.data; + +import lombok.Data; +import org.nd4j.linalg.dataset.api.iterator.DataSetIteratorFactory; + +import java.util.Map; + +/** + * This is a {@link DataProvider} for + * an {@link DataSetIteratorFactory} which + * based on a key of {@link DataSetIteratorFactoryProvider#FACTORY_KEY} + * will create {@link org.nd4j.linalg.dataset.api.iterator.DataSetIterator} + * for use with arbiter. + * + * This {@link DataProvider} is mainly meant for use for command line driven + * applications. + * + * @author Adam Gibson + */ +@Data +public class DataSetIteratorFactoryProvider implements DataProvider { + + public final static String FACTORY_KEY = "org.deeplearning4j.arbiter.data.data.factory"; + + /** + * Get training data given some parameters for the data. + * Data parameters map is used to specify things like batch + * size data preprocessing + * + * @param dataParameters Parameters for data. May be null or empty for default data + * @return training data + */ + @Override + public DataSetIteratorFactory trainData(Map dataParameters) { + return create(dataParameters); + } + + /** + * Get training data given some parameters for the data. Data parameters map + * is used to specify things like batch + * size data preprocessing + * + * @param dataParameters Parameters for data. May be null or empty for default data + * @return training data + */ + @Override + public DataSetIteratorFactory testData(Map dataParameters) { + return create(dataParameters); + } + + @Override + public Class getDataType() { + return DataSetIteratorFactory.class; + } + + private DataSetIteratorFactory create(Map dataParameters) { + if (dataParameters == null) + throw new IllegalArgumentException( + "Data parameters is null. Please specify a class name to create a dataset iterator."); + if (!dataParameters.containsKey(FACTORY_KEY)) + throw new IllegalArgumentException( + "No data set iterator factory class found. Please specify a class name with key " + + FACTORY_KEY); + String value = dataParameters.get(FACTORY_KEY).toString(); + try { + Class clazz = + (Class) Class.forName(value); + return clazz.newInstance(); + } catch (Exception e) { + throw new RuntimeException("Could not create DataSetIteratorFactory instance - missing no-arg constructor?", e); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataSource.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataSource.java new file mode 100644 index 000000000..0afe7bb70 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/data/DataSource.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.data; + +import java.io.Serializable; +import java.util.Properties; + +/** + * DataSource: defines where the data should come from for training and testing. + * Note that implementations must have a no-argument contsructor + * + * @author Alex Black + */ +public interface DataSource extends Serializable { + + /** + * Configure the current data source with the specified properties + * Note: These properties are fixed for the training instance, and are optionally provided by the user + * at the configuration stage. + * The properties could be anything - and are usually specific to each DataSource implementation. + * For example, values such as batch size could be set using these properties + * @param properties Properties to apply to the data source instance + */ + void configure(Properties properties); + + /** + * Get test data to be used for the optimization. Usually a DataSetIterator or MultiDataSetIterator + */ + Object trainData(); + + /** + * Get test data to be used for the optimization. Usually a DataSetIterator or MultiDataSetIterator + */ + Object testData(); + + /** + * The type of data returned by {@link #trainData()} and {@link #testData()}. + * Usually DataSetIterator or MultiDataSetIterator + * @return Class of the objects returned by trainData and testData + */ + Class getDataType(); + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/evaluation/ModelEvaluator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/evaluation/ModelEvaluator.java new file mode 100644 index 000000000..e5dd31d6e --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/evaluation/ModelEvaluator.java @@ -0,0 +1,40 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.evaluation; + +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; + +import java.io.Serializable; +import java.util.List; + +/** + * ModelEvaluator: Used to conduct additional evaluation. + * For example, this may be classification performance on a test set or similar + */ +public interface ModelEvaluator extends Serializable { + Object evaluateModel(Object model, DataProvider dataProvider); + + /** + * @return The model types supported by this class + */ + List> getSupportedModelTypes(); + + /** + * @return The datatypes supported by this class + */ + List> getSupportedDataTypes(); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/InMemoryResultSaver.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/InMemoryResultSaver.java new file mode 100644 index 000000000..43b914cb3 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/InMemoryResultSaver.java @@ -0,0 +1,63 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.saving; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * A simple class to store optimization results in-memory. + * Not recommended for large (or a large number of) models. + */ +@NoArgsConstructor +public class InMemoryResultSaver implements ResultSaver { + @Override + public ResultReference saveModel(OptimizationResult result, Object modelResult) throws IOException { + return new InMemoryResult(result, modelResult); + } + + @Override + public List> getSupportedCandidateTypes() { + return Collections.>singletonList(Object.class); + } + + @Override + public List> getSupportedModelTypes() { + return Collections.>singletonList(Object.class); + } + + @AllArgsConstructor + private static class InMemoryResult implements ResultReference { + private OptimizationResult result; + private Object modelResult; + + @Override + public OptimizationResult getResult() throws IOException { + return result; + } + + @Override + public Object getResultModel() throws IOException { + return modelResult; + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/ResultReference.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/ResultReference.java new file mode 100644 index 000000000..02e4ec453 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/ResultReference.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.saving; + +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.IOException; + +/** + * Idea: We can't store all results in memory in general (might have thousands of candidates with millions of + * parameters each) + * So instead: return a reference to the saved result. Idea is that the result may be saved to disk or a database, + * and we can easily load it back into memory (if/when required) using the getResult() method + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface ResultReference { + + OptimizationResult getResult() throws IOException; + + Object getResultModel() throws IOException; + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/ResultSaver.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/ResultSaver.java new file mode 100644 index 000000000..3506d536b --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/saving/ResultSaver.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.saving; + +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.IOException; +import java.util.List; + +/** + * The ResultSaver interface provides a means of saving models in such a way that they can be loaded back into memory later, + * regardless of where/how they are saved. + * + * @author Alex Black + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface ResultSaver { + + /** + * Save the model (including configuration and any additional evaluation/results) + * + * @param result Optimization result for the model to save + * @param modelResult Model result to save + * @return ResultReference, such that the result can be loaded back into memory + * @throws IOException If IO error occurs during model saving + */ + ResultReference saveModel(OptimizationResult result, Object modelResult) throws IOException; + + /** + * @return The candidate types supported by this class + */ + List> getSupportedCandidateTypes(); + + /** + * @return The model types supported by this class + */ + List> getSupportedModelTypes(); + + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/score/ScoreFunction.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/score/ScoreFunction.java new file mode 100644 index 000000000..c6ad6ed29 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/score/ScoreFunction.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.score; + +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * ScoreFunction defines the objective of hyperparameter optimization. + * Specifically, it is used to calculate a score for a given model, relative to the data set provided + * in the configuration. + * + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface ScoreFunction extends Serializable { + + /** + * Calculate and return the score, for the given model and data provider + * + * @param model Model to score + * @param dataProvider Data provider - data to use + * @param dataParameters Parameters for data + * @return Calculated score + */ + double score(Object model, DataProvider dataProvider, Map dataParameters); + + /** + * Calculate and return the score, for the given model and data provider + * + * @param model Model to score + * @param dataSource Data source + * @param dataSourceProperties data source properties + * @return Calculated score + */ + double score(Object model, Class dataSource, Properties dataSourceProperties); + + /** + * Should this score function be minimized or maximized? + * + * @return true if score should be minimized, false if score should be maximized + */ + boolean minimize(); + + /** + * @return The model types supported by this class + */ + List> getSupportedModelTypes(); + + /** + * @return The data types supported by this class + */ + List> getSupportedDataTypes(); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/MaxCandidatesCondition.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/MaxCandidatesCondition.java new file mode 100644 index 000000000..61b76dc90 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/MaxCandidatesCondition.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.termination; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Terminate hyperparameter search when the number of candidates exceeds a specified value. + * Note that this is counted as number of completed candidates, plus number of failed candidates. + */ +@AllArgsConstructor +@NoArgsConstructor +@Data +public class MaxCandidatesCondition implements TerminationCondition { + @JsonProperty + private int maxCandidates; + + @Override + public void initialize(IOptimizationRunner optimizationRunner) { + //No op + } + + @Override + public boolean terminate(IOptimizationRunner optimizationRunner) { + return optimizationRunner.numCandidatesCompleted() + optimizationRunner.numCandidatesFailed() >= maxCandidates; + } + + @Override + public String toString() { + return "MaxCandidatesCondition(" + maxCandidates + ")"; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/MaxTimeCondition.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/MaxTimeCondition.java new file mode 100644 index 000000000..c346c0ea5 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/MaxTimeCondition.java @@ -0,0 +1,81 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.termination; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.concurrent.TimeUnit; + +/** + * Terminate hyperparameter optimization after + * a fixed amount of time has passed + * @author Alex Black + */ +@NoArgsConstructor +@Data +public class MaxTimeCondition implements TerminationCondition { + private static final DateTimeFormatter formatter = DateTimeFormat.forPattern("dd-MMM HH:mm ZZ"); + + private long duration; + private TimeUnit timeUnit; + private long startTime; + private long endTime; + + + private MaxTimeCondition(@JsonProperty("duration") long duration, @JsonProperty("timeUnit") TimeUnit timeUnit, + @JsonProperty("startTime") long startTime, @JsonProperty("endTime") long endTime) { + this.duration = duration; + this.timeUnit = timeUnit; + this.startTime = startTime; + this.endTime = endTime; + } + + /** + * @param duration Duration of time + * @param timeUnit Unit that the duration is specified in + */ + public MaxTimeCondition(long duration, TimeUnit timeUnit) { + this.duration = duration; + this.timeUnit = timeUnit; + } + + @Override + public void initialize(IOptimizationRunner optimizationRunner) { + startTime = System.currentTimeMillis(); + this.endTime = startTime + timeUnit.toMillis(duration); + } + + @Override + public boolean terminate(IOptimizationRunner optimizationRunner) { + return System.currentTimeMillis() >= endTime; + } + + @Override + public String toString() { + if (startTime > 0) { + return "MaxTimeCondition(" + duration + "," + timeUnit + ",start=\"" + formatter.print(startTime) + + "\",end=\"" + formatter.print(endTime) + "\")"; + } else { + return "MaxTimeCondition(" + duration + "," + timeUnit + "\")"; + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/TerminationCondition.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/TerminationCondition.java new file mode 100644 index 000000000..ec5e1982f --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/api/termination/TerminationCondition.java @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.api.termination; + + +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * Global termination condition for conducting hyperparameter optimization. + * Termination conditions are used to determine if/when the optimization should stop. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +@JsonInclude(JsonInclude.Include.NON_NULL) +public interface TerminationCondition { + + /** + * Initialize the termination condition (such as starting timers, etc). + */ + void initialize(IOptimizationRunner optimizationRunner); + + /** + * Determine whether optimization should be terminated + * + * @param optimizationRunner Optimization runner + * @return true if learning should be terminated, false otherwise + */ + boolean terminate(IOptimizationRunner optimizationRunner); + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/config/OptimizationConfiguration.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/config/OptimizationConfiguration.java new file mode 100644 index 000000000..59b3e9a6a --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/config/OptimizationConfiguration.java @@ -0,0 +1,226 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.config; + +import lombok.*; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultSaver; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.api.termination.TerminationCondition; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +/** + * OptimizationConfiguration ties together all of the various + * components (such as data, score functions, result saving etc) + * required to execute hyperparameter optimization. + * + * @author Alex Black + */ +@Data +@NoArgsConstructor +@EqualsAndHashCode(exclude = {"dataProvider", "terminationConditions", "candidateGenerator", "resultSaver"}) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public class OptimizationConfiguration { + @JsonSerialize + private DataProvider dataProvider; + @JsonSerialize + private Class dataSource; + @JsonSerialize + private Properties dataSourceProperties; + @JsonSerialize + private CandidateGenerator candidateGenerator; + @JsonSerialize + private ResultSaver resultSaver; + @JsonSerialize + private ScoreFunction scoreFunction; + @JsonSerialize + private List terminationConditions; + @JsonSerialize + private Long rngSeed; + + @Getter + @Setter + private long executionStartTime; + + + private OptimizationConfiguration(Builder builder) { + this.dataProvider = builder.dataProvider; + this.dataSource = builder.dataSource; + this.dataSourceProperties = builder.dataSourceProperties; + this.candidateGenerator = builder.candidateGenerator; + this.resultSaver = builder.resultSaver; + this.scoreFunction = builder.scoreFunction; + this.terminationConditions = builder.terminationConditions; + this.rngSeed = builder.rngSeed; + + if (rngSeed != null) + candidateGenerator.setRngSeed(rngSeed); + + //Validate the configuration: data types, score types, etc + //TODO + + //Validate that the dataSource has a no-arg constructor + if (dataSource != null) { + try { + dataSource.getConstructor(); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Data source class " + dataSource.getName() + " does not have a public no-argument constructor"); + } + } + } + + public static class Builder { + + private DataProvider dataProvider; + private Class dataSource; + private Properties dataSourceProperties; + private CandidateGenerator candidateGenerator; + private ResultSaver resultSaver; + private ScoreFunction scoreFunction; + private List terminationConditions; + private Long rngSeed; + + /** + * @deprecated Use {@link #dataSource(Class, Properties)} + */ + @Deprecated + public Builder dataProvider(DataProvider dataProvider) { + this.dataProvider = dataProvider; + return this; + } + + /** + * DataSource: defines where the data should come from for training and testing. + * Note that implementations must have a no-argument contsructor + * + * @param dataSource Class for the data source + * @param dataSourceProperties May be null. Properties for configuring the data source + */ + public Builder dataSource(Class dataSource, Properties dataSourceProperties) { + this.dataSource = dataSource; + this.dataSourceProperties = dataSourceProperties; + return this; + } + + public Builder candidateGenerator(CandidateGenerator candidateGenerator) { + this.candidateGenerator = candidateGenerator; + return this; + } + + public Builder modelSaver(ResultSaver resultSaver) { + this.resultSaver = resultSaver; + return this; + } + + public Builder scoreFunction(ScoreFunction scoreFunction) { + this.scoreFunction = scoreFunction; + return this; + } + + /** + * Termination conditions to use + * + * @param conditions + * @return + */ + public Builder terminationConditions(TerminationCondition... conditions) { + terminationConditions = Arrays.asList(conditions); + return this; + } + + public Builder terminationConditions(List terminationConditions) { + this.terminationConditions = terminationConditions; + return this; + } + + public Builder rngSeed(long rngSeed) { + this.rngSeed = rngSeed; + return this; + } + + public OptimizationConfiguration build() { + return new OptimizationConfiguration(this); + } + } + + + /** + * Create an optimization configuration from the json + * + * @param json the json to create the config from + * For type definitions + * @see OptimizationConfiguration + */ + public static OptimizationConfiguration fromYaml(String json) { + try { + return JsonMapper.getYamlMapper().readValue(json, OptimizationConfiguration.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Create an optimization configuration from the json + * + * @param json the json to create the config from + * @see OptimizationConfiguration + */ + public static OptimizationConfiguration fromJson(String json) { + try { + return JsonMapper.getMapper().readValue(json, OptimizationConfiguration.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Return a json configuration of this optimization configuration + * + * @return + */ + public String toJson() { + try { + return JsonMapper.getMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Return a yaml configuration of this optimization configuration + * + * @return + */ + public String toYaml() { + try { + return JsonMapper.getYamlMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/DegenerateIntegerDistribution.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/DegenerateIntegerDistribution.java new file mode 100644 index 000000000..c613d08b6 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/DegenerateIntegerDistribution.java @@ -0,0 +1,96 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.distribution; + +import org.apache.commons.math3.distribution.IntegerDistribution; +import org.apache.commons.math3.exception.NumberIsTooLargeException; +import org.apache.commons.math3.exception.OutOfRangeException; + +/** + * Degenerate distribution: i.e., integer "distribution" that is just a fixed value + */ +public class DegenerateIntegerDistribution implements IntegerDistribution { + private int value; + + public DegenerateIntegerDistribution(int value) { + this.value = value; + } + + + @Override + public double probability(int x) { + return (x == value ? 1.0 : 0.0); + } + + @Override + public double cumulativeProbability(int x) { + return (x >= value ? 1.0 : 0.0); + } + + @Override + public double cumulativeProbability(int x0, int x1) throws NumberIsTooLargeException { + return (value >= x0 && value <= x1 ? 1.0 : 0.0); + } + + @Override + public int inverseCumulativeProbability(double p) throws OutOfRangeException { + throw new UnsupportedOperationException(); + } + + @Override + public double getNumericalMean() { + return value; + } + + @Override + public double getNumericalVariance() { + return 0; + } + + @Override + public int getSupportLowerBound() { + return value; + } + + @Override + public int getSupportUpperBound() { + return value; + } + + @Override + public boolean isSupportConnected() { + return true; + } + + @Override + public void reseedRandomGenerator(long seed) { + //no op + } + + @Override + public int sample() { + return value; + } + + @Override + public int[] sample(int sampleSize) { + int[] out = new int[sampleSize]; + for (int i = 0; i < out.length; i++) + out[i] = value; + return out; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/DistributionUtils.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/DistributionUtils.java new file mode 100644 index 000000000..24dafc726 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/DistributionUtils.java @@ -0,0 +1,149 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.distribution; + +import org.apache.commons.math3.distribution.*; + +/** + * Distribution utils for Apache Commons math distributions - which don't provide equals, hashcode, toString methods, + * don't implement serializable etc. + * Which makes unit testing etc quite difficult. + * + * @author Alex Black + */ +public class DistributionUtils { + + private DistributionUtils() {} + + + public static boolean distributionsEqual(RealDistribution a, RealDistribution b) { + if (a.getClass() != b.getClass()) + return false; + Class c = a.getClass(); + if (c == BetaDistribution.class) { + BetaDistribution ba = (BetaDistribution) a; + BetaDistribution bb = (BetaDistribution) b; + + return ba.getAlpha() == bb.getAlpha() && ba.getBeta() == bb.getBeta(); + } else if (c == CauchyDistribution.class) { + CauchyDistribution ca = (CauchyDistribution) a; + CauchyDistribution cb = (CauchyDistribution) b; + return ca.getMedian() == cb.getMedian() && ca.getScale() == cb.getScale(); + } else if (c == ChiSquaredDistribution.class) { + ChiSquaredDistribution ca = (ChiSquaredDistribution) a; + ChiSquaredDistribution cb = (ChiSquaredDistribution) b; + return ca.getDegreesOfFreedom() == cb.getDegreesOfFreedom(); + } else if (c == ExponentialDistribution.class) { + ExponentialDistribution ea = (ExponentialDistribution) a; + ExponentialDistribution eb = (ExponentialDistribution) b; + return ea.getMean() == eb.getMean(); + } else if (c == FDistribution.class) { + FDistribution fa = (FDistribution) a; + FDistribution fb = (FDistribution) b; + return fa.getNumeratorDegreesOfFreedom() == fb.getNumeratorDegreesOfFreedom() + && fa.getDenominatorDegreesOfFreedom() == fb.getDenominatorDegreesOfFreedom(); + } else if (c == GammaDistribution.class) { + GammaDistribution ga = (GammaDistribution) a; + GammaDistribution gb = (GammaDistribution) b; + return ga.getShape() == gb.getShape() && ga.getScale() == gb.getScale(); + } else if (c == LevyDistribution.class) { + LevyDistribution la = (LevyDistribution) a; + LevyDistribution lb = (LevyDistribution) b; + return la.getLocation() == lb.getLocation() && la.getScale() == lb.getScale(); + } else if (c == LogNormalDistribution.class) { + LogNormalDistribution la = (LogNormalDistribution) a; + LogNormalDistribution lb = (LogNormalDistribution) b; + return la.getScale() == lb.getScale() && la.getShape() == lb.getShape(); + } else if (c == NormalDistribution.class) { + NormalDistribution na = (NormalDistribution) a; + NormalDistribution nb = (NormalDistribution) b; + return na.getMean() == nb.getMean() && na.getStandardDeviation() == nb.getStandardDeviation(); + } else if (c == ParetoDistribution.class) { + ParetoDistribution pa = (ParetoDistribution) a; + ParetoDistribution pb = (ParetoDistribution) b; + return pa.getScale() == pb.getScale() && pa.getShape() == pb.getShape(); + } else if (c == TDistribution.class) { + TDistribution ta = (TDistribution) a; + TDistribution tb = (TDistribution) b; + return ta.getDegreesOfFreedom() == tb.getDegreesOfFreedom(); + } else if (c == TriangularDistribution.class) { + TriangularDistribution ta = (TriangularDistribution) a; + TriangularDistribution tb = (TriangularDistribution) b; + return ta.getSupportLowerBound() == tb.getSupportLowerBound() + && ta.getSupportUpperBound() == tb.getSupportUpperBound() && ta.getMode() == tb.getMode(); + } else if (c == UniformRealDistribution.class) { + UniformRealDistribution ua = (UniformRealDistribution) a; + UniformRealDistribution ub = (UniformRealDistribution) b; + return ua.getSupportLowerBound() == ub.getSupportLowerBound() + && ua.getSupportUpperBound() == ub.getSupportUpperBound(); + } else if (c == WeibullDistribution.class) { + WeibullDistribution wa = (WeibullDistribution) a; + WeibullDistribution wb = (WeibullDistribution) b; + return wa.getShape() == wb.getShape() && wa.getScale() == wb.getScale(); + } else if (c == LogUniformDistribution.class ){ + LogUniformDistribution lu_a = (LogUniformDistribution)a; + LogUniformDistribution lu_b = (LogUniformDistribution)b; + return lu_a.getMin() == lu_b.getMin() && lu_a.getMax() == lu_b.getMax(); + } else { + throw new UnsupportedOperationException("Unknown or not supported RealDistribution: " + c); + } + } + + public static boolean distributionEquals(IntegerDistribution a, IntegerDistribution b) { + if (a.getClass() != b.getClass()) + return false; + Class c = a.getClass(); + + if (c == BinomialDistribution.class) { + BinomialDistribution ba = (BinomialDistribution) a; + BinomialDistribution bb = (BinomialDistribution) b; + return ba.getNumberOfTrials() == bb.getNumberOfTrials() + && ba.getProbabilityOfSuccess() == bb.getProbabilityOfSuccess(); + } else if (c == GeometricDistribution.class) { + GeometricDistribution ga = (GeometricDistribution) a; + GeometricDistribution gb = (GeometricDistribution) b; + return ga.getProbabilityOfSuccess() == gb.getProbabilityOfSuccess(); + } else if (c == HypergeometricDistribution.class) { + HypergeometricDistribution ha = (HypergeometricDistribution) a; + HypergeometricDistribution hb = (HypergeometricDistribution) b; + return ha.getPopulationSize() == hb.getPopulationSize() + && ha.getNumberOfSuccesses() == hb.getNumberOfSuccesses() + && ha.getSampleSize() == hb.getSampleSize(); + } else if (c == PascalDistribution.class) { + PascalDistribution pa = (PascalDistribution) a; + PascalDistribution pb = (PascalDistribution) b; + return pa.getNumberOfSuccesses() == pb.getNumberOfSuccesses() + && pa.getProbabilityOfSuccess() == pb.getProbabilityOfSuccess(); + } else if (c == PoissonDistribution.class) { + PoissonDistribution pa = (PoissonDistribution) a; + PoissonDistribution pb = (PoissonDistribution) b; + return pa.getMean() == pb.getMean(); + } else if (c == UniformIntegerDistribution.class) { + UniformIntegerDistribution ua = (UniformIntegerDistribution) a; + UniformIntegerDistribution ub = (UniformIntegerDistribution) b; + return ua.getSupportUpperBound() == ub.getSupportUpperBound() + && ua.getSupportUpperBound() == ub.getSupportUpperBound(); + } else if (c == ZipfDistribution.class) { + ZipfDistribution za = (ZipfDistribution) a; + ZipfDistribution zb = (ZipfDistribution) b; + return za.getNumberOfElements() == zb.getNumberOfElements() && za.getExponent() == zb.getNumberOfElements(); + } else { + throw new UnsupportedOperationException("Unknown or not supported IntegerDistribution: " + c); + } + + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/LogUniformDistribution.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/LogUniformDistribution.java new file mode 100644 index 000000000..da790c422 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/distribution/LogUniformDistribution.java @@ -0,0 +1,155 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.distribution; + +import com.google.common.base.Preconditions; +import lombok.Getter; +import org.apache.commons.math3.distribution.RealDistribution; +import org.apache.commons.math3.exception.NumberIsTooLargeException; +import org.apache.commons.math3.exception.OutOfRangeException; + +import java.util.Random; + +/** + * Log uniform distribution, with support in range [min, max] for min > 0 + * + * Reference: https://www.vosesoftware.com/riskwiki/LogUniformdistribution.php + * + * @author Alex Black + */ +public class LogUniformDistribution implements RealDistribution { + + @Getter private final double min; + @Getter private final double max; + + private final double logMin; + private final double logMax; + + private transient Random rng = new Random(); + + /** + * + * @param min Minimum value + * @param max Maximum value + */ + public LogUniformDistribution(double min, double max) { + Preconditions.checkArgument(min > 0, "Minimum must be > 0. Got: " + min); + Preconditions.checkArgument(max > min, "Maximum must be > min. Got: (min, max)=(" + + min + "," + max + ")"); + this.min = min; + this.max = max; + + this.logMin = Math.log(min); + this.logMax = Math.log(max); + } + + @Override + public double probability(double x) { + if(x < min || x > max){ + return 0; + } + + return 1.0 / (x * (logMax - logMin)); + } + + @Override + public double density(double x) { + return probability(x); + } + + @Override + public double cumulativeProbability(double x) { + if(x <= min){ + return 0.0; + } else if(x >= max){ + return 1.0; + } + + return (Math.log(x)-logMin)/(logMax-logMin); + } + + @Override + public double cumulativeProbability(double x0, double x1) throws NumberIsTooLargeException { + return cumulativeProbability(x1) - cumulativeProbability(x0); + } + + @Override + public double inverseCumulativeProbability(double p) throws OutOfRangeException { + Preconditions.checkArgument(p >= 0 && p <= 1, "Invalid input: " + p); + return Math.exp(p * (logMax-logMin) + logMin); + } + + @Override + public double getNumericalMean() { + return (max-min)/(logMax-logMin); + } + + @Override + public double getNumericalVariance() { + double d1 = (logMax-logMin)*(max*max - min*min) - 2*(max-min)*(max-min); + return d1 / (2*Math.pow(logMax-logMin, 2.0)); + } + + @Override + public double getSupportLowerBound() { + return min; + } + + @Override + public double getSupportUpperBound() { + return max; + } + + @Override + public boolean isSupportLowerBoundInclusive() { + return true; + } + + @Override + public boolean isSupportUpperBoundInclusive() { + return true; + } + + @Override + public boolean isSupportConnected() { + return true; + } + + @Override + public void reseedRandomGenerator(long seed) { + rng.setSeed(seed); + } + + @Override + public double sample() { + return inverseCumulativeProbability(rng.nextDouble()); + } + + @Override + public double[] sample(int sampleSize) { + double[] d = new double[sampleSize]; + for( int i=0; i Type of candidates to generate + */ +@Data +@EqualsAndHashCode(exclude = {"rng", "candidateCounter"}) +public abstract class BaseCandidateGenerator implements CandidateGenerator { + protected ParameterSpace parameterSpace; + protected AtomicInteger candidateCounter = new AtomicInteger(0); + protected SynchronizedRandomGenerator rng = new SynchronizedRandomGenerator(new JDKRandomGenerator()); + protected Map dataParameters; + protected boolean initDone = false; + + public BaseCandidateGenerator(ParameterSpace parameterSpace, Map dataParameters, + boolean initDone) { + this.parameterSpace = parameterSpace; + this.dataParameters = dataParameters; + this.initDone = initDone; + } + + protected void initialize() { + if(!initDone) { + //First: collect leaf parameter spaces objects and remove duplicates + List noDuplicatesList = LeafUtils.getUniqueObjects(parameterSpace.collectLeaves()); + + //Second: assign each a number + int i = 0; + for (ParameterSpace ps : noDuplicatesList) { + int np = ps.numParameters(); + if (np == 1) { + ps.setIndices(i++); + } else { + int[] values = new int[np]; + for (int j = 0; j < np; j++) + values[j] = i++; + ps.setIndices(values); + } + } + initDone = true; + } + } + + @Override + public ParameterSpace getParameterSpace() { + return parameterSpace; + } + + @Override + public void reportResults(OptimizationResult result) { + //No op + } + + @Override + public void setRngSeed(long rngSeed) { + rng.setSeed(rngSeed); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/GeneticSearchCandidateGenerator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/GeneticSearchCandidateGenerator.java new file mode 100644 index 000000000..564c194ba --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/GeneticSearchCandidateGenerator.java @@ -0,0 +1,187 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.ChromosomeFactory; +import org.deeplearning4j.arbiter.optimize.generator.genetic.exceptions.GeneticGenerationException; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.EmptyPopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.generator.genetic.selection.GeneticSelectionOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.selection.SelectionOperator; + +import java.util.Map; + +/** + * Uses a genetic algorithm to generate candidates. + * + * @author Alexandre Boulanger + */ +@Slf4j +public class GeneticSearchCandidateGenerator extends BaseCandidateGenerator { + + @Getter + protected final PopulationModel populationModel; + + protected final ChromosomeFactory chromosomeFactory; + protected final SelectionOperator selectionOperator; + + protected boolean hasMoreCandidates = true; + + public static class Builder { + protected final ParameterSpace parameterSpace; + + protected Map dataParameters; + protected boolean initDone; + protected boolean minimizeScore; + protected PopulationModel populationModel; + protected ChromosomeFactory chromosomeFactory; + protected SelectionOperator selectionOperator; + + /** + * @param parameterSpace ParameterSpace from which to generate candidates + * @param scoreFunction The score function that will be used in the OptimizationConfiguration + */ + public Builder(ParameterSpace parameterSpace, ScoreFunction scoreFunction) { + this.parameterSpace = parameterSpace; + this.minimizeScore = scoreFunction.minimize(); + } + + /** + * @param populationModel The PopulationModel instance to use. + */ + public Builder populationModel(PopulationModel populationModel) { + this.populationModel = populationModel; + return this; + } + + /** + * @param selectionOperator The SelectionOperator to use. Default is GeneticSelectionOperator + */ + public Builder selectionOperator(SelectionOperator selectionOperator) { + this.selectionOperator = selectionOperator; + return this; + } + + public Builder dataParameters(Map dataParameters) { + + this.dataParameters = dataParameters; + return this; + } + + public GeneticSearchCandidateGenerator.Builder initDone(boolean initDone) { + this.initDone = initDone; + return this; + } + + /** + * @param chromosomeFactory The ChromosomeFactory to use + */ + public Builder chromosomeFactory(ChromosomeFactory chromosomeFactory) { + this.chromosomeFactory = chromosomeFactory; + return this; + } + + public GeneticSearchCandidateGenerator build() { + if (populationModel == null) { + PopulationInitializer defaultPopulationInitializer = new EmptyPopulationInitializer(); + populationModel = new PopulationModel.Builder().populationInitializer(defaultPopulationInitializer) + .build(); + } + + if (chromosomeFactory == null) { + chromosomeFactory = new ChromosomeFactory(); + } + + if (selectionOperator == null) { + selectionOperator = new GeneticSelectionOperator.Builder().build(); + } + + return new GeneticSearchCandidateGenerator(this); + } + } + + private GeneticSearchCandidateGenerator(Builder builder) { + super(builder.parameterSpace, builder.dataParameters, builder.initDone); + + initialize(); + + chromosomeFactory = builder.chromosomeFactory; + populationModel = builder.populationModel; + selectionOperator = builder.selectionOperator; + + chromosomeFactory.initializeInstance(builder.parameterSpace.numParameters()); + populationModel.initializeInstance(builder.minimizeScore); + selectionOperator.initializeInstance(populationModel, chromosomeFactory); + + } + + @Override + public boolean hasMoreCandidates() { + return hasMoreCandidates; + } + + @Override + public Candidate getCandidate() { + + double[] values = null; + Object value = null; + Exception e = null; + + try { + values = selectionOperator.buildNextGenes(); + value = parameterSpace.getValue(values); + } catch (GeneticGenerationException e2) { + log.warn("Error generating candidate", e2); + e = e2; + hasMoreCandidates = false; + } catch (Exception e2) { + log.warn("Error getting configuration for candidate", e2); + e = e2; + } + + return new Candidate(value, candidateCounter.getAndIncrement(), values, dataParameters, e); + } + + @Override + public Class getCandidateType() { + return null; + } + + @Override + public String toString() { + return "GeneticSearchCandidateGenerator"; + } + + @Override + public void reportResults(OptimizationResult result) { + if (result.getScore() == null) { + return; + } + + Chromosome newChromosome = chromosomeFactory.createChromosome(result.getCandidate().getFlatParameters(), + result.getScore()); + populationModel.add(newChromosome); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/GridSearchCandidateGenerator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/GridSearchCandidateGenerator.java new file mode 100644 index 000000000..4d056087f --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/GridSearchCandidateGenerator.java @@ -0,0 +1,232 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.math3.random.RandomAdaptor; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.util.LeafUtils; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + + +/** + * GridSearchCandidateGenerator: generates candidates in an exhaustive grid search manner.
+ * Note that:
+ * - For discrete parameters: the grid size (# values to check per hyperparameter) is equal to the number of values for + * that hyperparameter
+ * - For integer parameters: the grid size is equal to {@code min(discretizationCount,max-min+1)}. Some integer ranges can + * be large, and we don't necessarily want to exhaustively search them. {@code discretizationCount} is a constructor argument
+ * - For continuous parameters: the grid size is equal to {@code discretizationCount}.
+ * In all cases, the minimum, maximum and gridSize-2 values between the min/max will be generated.
+ * Also note that: if a probability distribution is provided for continuous hyperparameters, this will be taken into account + * when generating candidates. This allows the grid for a hyperparameter to be non-linear: i.e., for example, linear in log space + * + * @author Alex Black + */ +@Slf4j +@EqualsAndHashCode(exclude = {"order"}, callSuper = true) +@JsonIgnoreProperties({"numValuesPerParam", "totalNumCandidates", "order", "candidateCounter", "rng", "candidate"}) +public class GridSearchCandidateGenerator extends BaseCandidateGenerator { + + /** + * In what order should candidates be generated?
+ * Sequential: generate candidates in order. The first hyperparameter will be changed most rapidly, and the last + * will be changed least rapidly.
+ * RandomOrder: generate candidates in a random order
+ * In both cases, the same candidates will be generated; only the order of generation is different + */ + public enum Mode { + Sequential, RandomOrder + } + + private final int discretizationCount; + private final Mode mode; + + private int[] numValuesPerParam; + @Getter + private int totalNumCandidates; + private Queue order; + + /** + * @param parameterSpace ParameterSpace from which to generate candidates + * @param discretizationCount For continuous parameters: into how many values should we discretize them into? + * For example, suppose continuous parameter is in range [0,1] with 3 bins: + * do [0.0, 0.5, 1.0]. Note that if all values + * @param mode {@link GridSearchCandidateGenerator.Mode} specifies the order + * in which candidates should be generated. + */ + public GridSearchCandidateGenerator(@JsonProperty("parameterSpace") ParameterSpace parameterSpace, + @JsonProperty("discretizationCount") int discretizationCount, @JsonProperty("mode") Mode mode, + @JsonProperty("dataParameters") Map dataParameters, + @JsonProperty("initDone") boolean initDone) { + super(parameterSpace, dataParameters, initDone); + this.discretizationCount = discretizationCount; + this.mode = mode; + initialize(); + } + + /** + * @param parameterSpace ParameterSpace from which to generate candidates + * @param discretizationCount For continuous parameters: into how many values should we discretize them into? + * For example, suppose continuous parameter is in range [0,1] with 3 bins: + * do [0.0, 0.5, 1.0]. Note that if all values + * @param mode {@link GridSearchCandidateGenerator.Mode} specifies the order + * in which candidates should be generated. + */ + public GridSearchCandidateGenerator(ParameterSpace parameterSpace, int discretizationCount, Mode mode, + Map dataParameters){ + this(parameterSpace, discretizationCount, mode, dataParameters, false); + } + + @Override + protected void initialize() { + super.initialize(); + + List leaves = LeafUtils.getUniqueObjects(parameterSpace.collectLeaves()); + int nParams = leaves.size(); + + //Work out for each parameter: is it continuous or discrete? + // for grid search: discrete values are grid-searchable as-is + // continuous values: discretize using 'discretizationCount' bins + // integer values: use min(max-min+1, discretizationCount) values. i.e., discretize if necessary + numValuesPerParam = new int[nParams]; + long searchSize = 1; + for (int i = 0; i < nParams; i++) { + ParameterSpace ps = leaves.get(i); + if (ps instanceof DiscreteParameterSpace) { + DiscreteParameterSpace dps = (DiscreteParameterSpace) ps; + numValuesPerParam[i] = dps.numValues(); + } else if (ps instanceof IntegerParameterSpace) { + IntegerParameterSpace ips = (IntegerParameterSpace) ps; + int min = ips.getMin(); + int max = ips.getMax(); + //Discretize, as some integer ranges are much too large to search (i.e., num. neural network units, between 100 and 1000) + numValuesPerParam[i] = Math.min(max - min + 1, discretizationCount); + } else if (ps instanceof FixedValue){ + numValuesPerParam[i] = 1; + } else { + numValuesPerParam[i] = discretizationCount; + } + searchSize *= numValuesPerParam[i]; + } + + if (searchSize >= Integer.MAX_VALUE) + throw new IllegalStateException("Invalid search: cannot process search with " + searchSize + + " candidates > Integer.MAX_VALUE"); //TODO find a more reasonable upper bound? + + order = new ConcurrentLinkedQueue<>(); + + totalNumCandidates = (int) searchSize; + switch (mode) { + case Sequential: + for (int i = 0; i < totalNumCandidates; i++) { + order.add(i); + } + break; + case RandomOrder: + List tempList = new ArrayList<>(totalNumCandidates); + for (int i = 0; i < totalNumCandidates; i++) { + tempList.add(i); + } + + Collections.shuffle(tempList, new RandomAdaptor(rng)); + order.addAll(tempList); + break; + default: + throw new RuntimeException(); + } + + } + + @Override + public boolean hasMoreCandidates() { + return !order.isEmpty(); + } + + @Override + public Candidate getCandidate() { + int next = order.remove(); + + //Next: max integer (candidate number) to values + double[] values = indexToValues(numValuesPerParam, next, totalNumCandidates); + + Object value = null; + Exception e = null; + try { + value = parameterSpace.getValue(values); + } catch (Exception e2) { + log.warn("Error getting configuration for candidate", e2); + e = e2; + } + + return new Candidate(value, candidateCounter.getAndIncrement(), values, dataParameters, e); + } + + @Override + public Class getCandidateType() { + return null; + } + + public static double[] indexToValues(int[] numValuesPerParam, int candidateIdx, int product) { + //How? first map to index of num possible values. Then: to double values in range 0 to 1 + // 0-> [0,0,0], 1-> [1,0,0], 2-> [2,0,0], 3-> [0,1,0] etc + //Based on: Nd4j Shape.ind2sub + + int countNon1 = 0; + for( int i : numValuesPerParam) + if(i > 1) + countNon1++; + + int denom = product; + int num = candidateIdx; + int[] index = new int[numValuesPerParam.length]; + + for (int i = index.length - 1; i >= 0; i--) { + denom /= numValuesPerParam[i]; + index[i] = num / denom; + num %= denom; + } + + //Now: convert indexes to values in range [0,1] + //min value -> 0 + //max value -> 1 + double[] out = new double[countNon1]; + int outIdx = 0; + for (int i = 0; i < numValuesPerParam.length; i++) { + if (numValuesPerParam[i] > 1){ + out[outIdx++] = index[i] / ((double) (numValuesPerParam[i] - 1)); + } + } + + return out; + } + + @Override + public String toString() { + return "GridSearchCandidateGenerator(mode=" + mode + ")"; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/RandomSearchGenerator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/RandomSearchGenerator.java new file mode 100644 index 000000000..04b5c8da8 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/RandomSearchGenerator.java @@ -0,0 +1,93 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator; + +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +/** + * RandomSearchGenerator: generates candidates at random.
+ * Note: if a probability distribution is provided for continuous hyperparameters, + * this will be taken into account + * when generating candidates. This allows the search to be weighted more towards + * certain values according to a probability + * density. For example: generate samples for learning rate according to log uniform distribution + * + * @author Alex Black + */ +@Slf4j +@EqualsAndHashCode(callSuper = true) +@JsonIgnoreProperties({"numValuesPerParam", "totalNumCandidates", "order", "candidateCounter", "rng", "candidate"}) +public class RandomSearchGenerator extends BaseCandidateGenerator { + + @JsonCreator + public RandomSearchGenerator(@JsonProperty("parameterSpace") ParameterSpace parameterSpace, + @JsonProperty("dataParameters") Map dataParameters, + @JsonProperty("initDone") boolean initDone) { + super(parameterSpace, dataParameters, initDone); + initialize(); + } + + public RandomSearchGenerator(ParameterSpace parameterSpace, Map dataParameters){ + this(parameterSpace, dataParameters, false); + } + + public RandomSearchGenerator(ParameterSpace parameterSpace){ + this(parameterSpace, null, false); + } + + + @Override + public boolean hasMoreCandidates() { + return true; + } + + @Override + public Candidate getCandidate() { + double[] randomValues = new double[parameterSpace.numParameters()]; + for (int i = 0; i < randomValues.length; i++) + randomValues[i] = rng.nextDouble(); + + Object value = null; + Exception e = null; + try { + value = parameterSpace.getValue(randomValues); + } catch (Exception e2) { + log.warn("Error getting configuration for candidate", e2); + e = e2; + } + + return new Candidate(value, candidateCounter.getAndIncrement(), randomValues, dataParameters, e); + } + + @Override + public Class getCandidateType() { + return null; + } + + @Override + public String toString() { + return "RandomSearchGenerator"; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/Chromosome.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/Chromosome.java new file mode 100644 index 000000000..5d8d00f0f --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/Chromosome.java @@ -0,0 +1,42 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic; + +import lombok.Data; + +/** + * Candidates are stored as Chromosome in the population model + * + * @author Alexandre Boulanger + */ +@Data +public class Chromosome { + /** + * The fitness score of the genes. + */ + protected final double fitness; + + /** + * The genes. + */ + protected final double[] genes; + + public Chromosome(double[] genes, double fitness) { + this.genes = genes; + this.fitness = fitness; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/ChromosomeFactory.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/ChromosomeFactory.java new file mode 100644 index 000000000..ede86406a --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/ChromosomeFactory.java @@ -0,0 +1,51 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic; + +/** + * A factory that builds new chromosomes. Used by the GeneticSearchCandidateGenerator. + * + * @author Alexandre Boulanger + */ +public class ChromosomeFactory { + private int chromosomeLength; + + /** + * Called by the GeneticSearchCandidateGenerator. + */ + public void initializeInstance(int chromosomeLength) { + this.chromosomeLength = chromosomeLength; + } + + /** + * Create a new instance of a Chromosome + * + * @param genes The genes + * @param fitness The fitness score + * @return A new instance of Chromosome + */ + public Chromosome createChromosome(double[] genes, double fitness) { + return new Chromosome(genes, fitness); + } + + /** + * @return The number of genes in a chromosome + */ + public int getChromosomeLength() { + return chromosomeLength; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/ArithmeticCrossover.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/ArithmeticCrossover.java new file mode 100644 index 000000000..978e7166b --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/ArithmeticCrossover.java @@ -0,0 +1,120 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover; + +import org.apache.commons.math3.random.JDKRandomGenerator; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.random.SynchronizedRandomGenerator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.RandomTwoParentSelection; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.TwoParentSelection; +import org.nd4j.common.base.Preconditions; + +/** + * A crossover operator that linearly combines the genes of two parents.
+ * When a crossover is generated (with a of probability crossover rate), each genes is a linear combination of the corresponding genes of the parents. + *

+ * t*parentA + (1-t)*parentB, where t is [0, 1] and different for each gene. + * + * @author Alexandre Boulanger + */ +public class ArithmeticCrossover extends TwoParentsCrossoverOperator { + private static final double DEFAULT_CROSSOVER_RATE = 0.85; + + private final double crossoverRate; + private final RandomGenerator rng; + + public static class Builder { + private double crossoverRate = DEFAULT_CROSSOVER_RATE; + private RandomGenerator rng; + private TwoParentSelection parentSelection; + + /** + * The probability that the operator generates a crossover (default 0.85). + * + * @param rate A value between 0.0 and 1.0 + */ + public Builder crossoverRate(double rate) { + Preconditions.checkState(rate >= 0.0 && rate <= 1.0, "Rate must be between 0.0 and 1.0, got %s", rate); + + this.crossoverRate = rate; + return this; + } + + /** + * Use a supplied RandomGenerator + * + * @param rng An instance of RandomGenerator + */ + public Builder randomGenerator(RandomGenerator rng) { + this.rng = rng; + return this; + } + + /** + * The parent selection behavior. Default is random parent selection. + * + * @param parentSelection An instance of TwoParentSelection + */ + public Builder parentSelection(TwoParentSelection parentSelection) { + this.parentSelection = parentSelection; + return this; + } + + public ArithmeticCrossover build() { + if (rng == null) { + rng = new SynchronizedRandomGenerator(new JDKRandomGenerator()); + } + + if (parentSelection == null) { + parentSelection = new RandomTwoParentSelection(); + } + + return new ArithmeticCrossover(this); + } + } + + private ArithmeticCrossover(ArithmeticCrossover.Builder builder) { + super(builder.parentSelection); + + this.crossoverRate = builder.crossoverRate; + this.rng = builder.rng; + } + + /** + * Has a probability crossoverRate of performing the crossover where each gene is a linear combination of:
+ * t*parentA + (1-t)*parentB, where t is [0, 1] and different for each gene.
+ * Otherwise, returns the genes of a random parent. + * + * @return The crossover result. See {@link CrossoverResult}. + */ + @Override + public CrossoverResult crossover() { + double[][] parents = parentSelection.selectParents(); + + double[] offspringValues = new double[parents[0].length]; + + if (rng.nextDouble() < crossoverRate) { + for (int i = 0; i < offspringValues.length; ++i) { + double t = rng.nextDouble(); + offspringValues[i] = t * parents[0][i] + (1.0 - t) * parents[1][i]; + } + return new CrossoverResult(true, offspringValues); + } + + return new CrossoverResult(false, parents[0]); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/CrossoverOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/CrossoverOperator.java new file mode 100644 index 000000000..cfae61e09 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/CrossoverOperator.java @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; + +/** + * Abstract class for all crossover operators + * + * @author Alexandre Boulanger + */ +public abstract class CrossoverOperator { + protected PopulationModel populationModel; + + /** + * Will be called by the selection operator once the population model is instantiated. + */ + public void initializeInstance(PopulationModel populationModel) { + this.populationModel = populationModel; + } + + /** + * Performs the crossover + * + * @return The crossover result. See {@link CrossoverResult}. + */ + public abstract CrossoverResult crossover(); + + + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/CrossoverResult.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/CrossoverResult.java new file mode 100644 index 000000000..68b7bdecb --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/CrossoverResult.java @@ -0,0 +1,43 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover; + +import lombok.Data; + +/** + * Returned by a crossover operator + * + * @author Alexandre Boulanger + */ +@Data +public class CrossoverResult { + /** + * If false, there was no crossover and the operator simply returned the genes of a random parent. + * If true, the genes are the result of a crossover. + */ + private final boolean isModified; + + /** + * The genes returned by the operator. + */ + private final double[] genes; + + public CrossoverResult(boolean isModified, double[] genes) { + this.isModified = isModified; + this.genes = genes; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/KPointCrossover.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/KPointCrossover.java new file mode 100644 index 000000000..8a7bb3a2a --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/KPointCrossover.java @@ -0,0 +1,178 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover; + +import org.apache.commons.math3.random.JDKRandomGenerator; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.random.SynchronizedRandomGenerator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.RandomTwoParentSelection; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.TwoParentSelection; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.utils.CrossoverPointsGenerator; +import org.nd4j.common.base.Preconditions; + +import java.util.Deque; + +/** +* The K-Point crossover will select at random multiple crossover points.
+* Each gene comes from one of the two parents. Each time a crossover point is reached, the parent is switched. +*/ +public class KPointCrossover extends TwoParentsCrossoverOperator { + private static final double DEFAULT_CROSSOVER_RATE = 0.85; + private static final int DEFAULT_MIN_CROSSOVER = 1; + private static final int DEFAULT_MAX_CROSSOVER = 4; + + private final double crossoverRate; + private final int minCrossovers; + private final int maxCrossovers; + + private final RandomGenerator rng; + + public static class Builder { + private double crossoverRate = DEFAULT_CROSSOVER_RATE; + private int minCrossovers = DEFAULT_MIN_CROSSOVER; + private int maxCrossovers = DEFAULT_MAX_CROSSOVER; + private RandomGenerator rng; + private TwoParentSelection parentSelection; + + /** + * The probability that the operator generates a crossover (default 0.85). + * + * @param rate A value between 0.0 and 1.0 + */ + public Builder crossoverRate(double rate) { + Preconditions.checkState(rate >= 0.0 && rate <= 1.0, "Rate must be between 0.0 and 1.0, got %s", rate); + + this.crossoverRate = rate; + return this; + } + + /** + * The number of crossovers points (default is min 1, max 4) + * + * @param min The minimum number + * @param max The maximum number + */ + public Builder numCrossovers(int min, int max) { + Preconditions.checkState(max >= 0 && min >= 0, "Min and max must be positive"); + Preconditions.checkState(max >= min, "Max must be greater or equal to min"); + + this.minCrossovers = min; + this.maxCrossovers = max; + return this; + } + + /** + * Use a fixed number of crossover points + * + * @param num The number of crossovers + */ + public Builder numCrossovers(int num) { + Preconditions.checkState(num >= 0, "Num must be positive"); + + this.minCrossovers = num; + this.maxCrossovers = num; + return this; + } + + /** + * Use a supplied RandomGenerator + * + * @param rng An instance of RandomGenerator + */ + public Builder randomGenerator(RandomGenerator rng) { + this.rng = rng; + return this; + } + + /** + * The parent selection behavior. Default is random parent selection. + * + * @param parentSelection An instance of TwoParentSelection + */ + public Builder parentSelection(TwoParentSelection parentSelection) { + this.parentSelection = parentSelection; + return this; + } + + public KPointCrossover build() { + if (rng == null) { + rng = new SynchronizedRandomGenerator(new JDKRandomGenerator()); + } + + if (parentSelection == null) { + parentSelection = new RandomTwoParentSelection(); + } + + return new KPointCrossover(this); + } + } + + private CrossoverPointsGenerator crossoverPointsGenerator; + + private KPointCrossover(KPointCrossover.Builder builder) { + super(builder.parentSelection); + + this.crossoverRate = builder.crossoverRate; + this.maxCrossovers = builder.maxCrossovers; + this.minCrossovers = builder.minCrossovers; + this.rng = builder.rng; + } + + private CrossoverPointsGenerator getCrossoverPointsGenerator(int chromosomeLength) { + if (crossoverPointsGenerator == null) { + crossoverPointsGenerator = + new CrossoverPointsGenerator(chromosomeLength, minCrossovers, maxCrossovers, rng); + } + + return crossoverPointsGenerator; + } + + /** + * Has a probability crossoverRate of performing the crossover where the operator will select at random multiple crossover points.
+ * Each gene comes from one of the two parents. Each time a crossover point is reached, the parent is switched.
+ * Otherwise, returns the genes of a random parent. + * + * @return The crossover result. See {@link CrossoverResult}. + */ + @Override + public CrossoverResult crossover() { + double[][] parents = parentSelection.selectParents(); + + boolean isModified = false; + double[] resultGenes = parents[0]; + + if (rng.nextDouble() < crossoverRate) { + // Select crossover points + Deque crossoverPoints = getCrossoverPointsGenerator(parents[0].length).getCrossoverPoints(); + + // Crossover + resultGenes = new double[parents[0].length]; + int currentParent = 0; + int nextCrossover = crossoverPoints.pop(); + for (int i = 0; i < resultGenes.length; ++i) { + if (i == nextCrossover) { + currentParent = currentParent == 0 ? 1 : 0; + nextCrossover = crossoverPoints.pop(); + } + resultGenes[i] = parents[currentParent][i]; + } + isModified = true; + } + + return new CrossoverResult(isModified, resultGenes); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/SinglePointCrossover.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/SinglePointCrossover.java new file mode 100644 index 000000000..cbeca1232 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/SinglePointCrossover.java @@ -0,0 +1,123 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover; + +import org.apache.commons.math3.random.JDKRandomGenerator; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.random.SynchronizedRandomGenerator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.RandomTwoParentSelection; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.TwoParentSelection; +import org.nd4j.common.base.Preconditions; + +/** + * The single point crossover will select a random point where every genes before that point comes from one parent + * and after which every genes comes from the other parent. + * + * @author Alexandre Boulanger + */ +public class SinglePointCrossover extends TwoParentsCrossoverOperator { + private static final double DEFAULT_CROSSOVER_RATE = 0.85; + + private final RandomGenerator rng; + private final double crossoverRate; + + public static class Builder { + private double crossoverRate = DEFAULT_CROSSOVER_RATE; + private RandomGenerator rng; + private TwoParentSelection parentSelection; + + /** + * The probability that the operator generates a crossover (default 0.85). + * + * @param rate A value between 0.0 and 1.0 + */ + public Builder crossoverRate(double rate) { + Preconditions.checkState(rate >= 0.0 && rate <= 1.0, "Rate must be between 0.0 and 1.0, got %s", rate); + + this.crossoverRate = rate; + return this; + } + + /** + * Use a supplied RandomGenerator + * + * @param rng An instance of RandomGenerator + */ + public Builder randomGenerator(RandomGenerator rng) { + this.rng = rng; + return this; + } + + /** + * The parent selection behavior. Default is random parent selection. + * + * @param parentSelection An instance of TwoParentSelection + */ + public Builder parentSelection(TwoParentSelection parentSelection) { + this.parentSelection = parentSelection; + return this; + } + + public SinglePointCrossover build() { + if (rng == null) { + rng = new SynchronizedRandomGenerator(new JDKRandomGenerator()); + } + + if (parentSelection == null) { + parentSelection = new RandomTwoParentSelection(); + } + + return new SinglePointCrossover(this); + } + } + + private SinglePointCrossover(SinglePointCrossover.Builder builder) { + super(builder.parentSelection); + + this.crossoverRate = builder.crossoverRate; + this.rng = builder.rng; + } + + /** + * Has a probability crossoverRate of performing the crossover where the operator will select a random crossover point.
+ * Each gene before this point comes from one of the two parents and each gene at or after this point comes from the other parent. + * Otherwise, returns the genes of a random parent. + * + * @return The crossover result. See {@link CrossoverResult}. + */ + public CrossoverResult crossover() { + double[][] parents = parentSelection.selectParents(); + + boolean isModified = false; + double[] resultGenes = parents[0]; + + if (rng.nextDouble() < crossoverRate) { + int chromosomeLength = parents[0].length; + + // Crossover + resultGenes = new double[chromosomeLength]; + + int crossoverPoint = rng.nextInt(chromosomeLength); + for (int i = 0; i < resultGenes.length; ++i) { + resultGenes[i] = ((i < crossoverPoint) ? parents[0] : parents[1])[i]; + } + isModified = true; + } + + return new CrossoverResult(isModified, resultGenes); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/TwoParentsCrossoverOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/TwoParentsCrossoverOperator.java new file mode 100644 index 000000000..69f1fb105 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/TwoParentsCrossoverOperator.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.TwoParentSelection; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; + +/** + * Abstract class for all crossover operators that applies to two parents. + * + * @author Alexandre Boulanger + */ +public abstract class TwoParentsCrossoverOperator extends CrossoverOperator { + + protected final TwoParentSelection parentSelection; + + /** + * @param parentSelection A parent selection that selects two parents. + */ + protected TwoParentsCrossoverOperator(TwoParentSelection parentSelection) { + this.parentSelection = parentSelection; + } + + /** + * Will be called by the selection operator once the population model is instantiated. + */ + @Override + public void initializeInstance(PopulationModel populationModel) { + super.initializeInstance(populationModel); + parentSelection.initializeInstance(populationModel.getPopulation()); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/UniformCrossover.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/UniformCrossover.java new file mode 100644 index 000000000..8912a1298 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/UniformCrossover.java @@ -0,0 +1,136 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover; + +import org.apache.commons.math3.random.JDKRandomGenerator; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.random.SynchronizedRandomGenerator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.RandomTwoParentSelection; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.TwoParentSelection; +import org.nd4j.common.base.Preconditions; + +/** + * The uniform crossover will, for each gene, randomly select the parent that donates the gene. + * + * @author Alexandre Boulanger + */ +public class UniformCrossover extends TwoParentsCrossoverOperator { + private static final double DEFAULT_CROSSOVER_RATE = 0.85; + private static final double DEFAULT_PARENT_BIAS_FACTOR = 0.5; + + private final double crossoverRate; + private final double parentBiasFactor; + private final RandomGenerator rng; + + public static class Builder { + private double crossoverRate = DEFAULT_CROSSOVER_RATE; + private double parentBiasFactor = DEFAULT_PARENT_BIAS_FACTOR; + private RandomGenerator rng; + private TwoParentSelection parentSelection; + + /** + * The probability that the operator generates a crossover (default 0.85). + * + * @param rate A value between 0.0 and 1.0 + */ + public Builder crossoverRate(double rate) { + Preconditions.checkState(rate >= 0.0 && rate <= 1.0, "Rate must be between 0.0 and 1.0, got %s", rate); + + this.crossoverRate = rate; + return this; + } + + /** + * A factor that will introduce a bias in the parent selection.
+ * + * @param factor In the range [0, 1]. 0 will only select the first parent while 1 only select the second one. The default is 0.5; no bias. + */ + public Builder parentBiasFactor(double factor) { + Preconditions.checkState(factor >= 0.0 && factor <= 1.0, "Factor must be between 0.0 and 1.0, got %s", + factor); + + this.parentBiasFactor = factor; + return this; + } + + /** + * Use a supplied RandomGenerator + * + * @param rng An instance of RandomGenerator + */ + public Builder randomGenerator(RandomGenerator rng) { + this.rng = rng; + return this; + } + + /** + * The parent selection behavior. Default is random parent selection. + * + * @param parentSelection An instance of TwoParentSelection + */ + public Builder parentSelection(TwoParentSelection parentSelection) { + this.parentSelection = parentSelection; + return this; + } + + public UniformCrossover build() { + if (rng == null) { + rng = new SynchronizedRandomGenerator(new JDKRandomGenerator()); + } + if (parentSelection == null) { + parentSelection = new RandomTwoParentSelection(); + } + return new UniformCrossover(this); + } + } + + private UniformCrossover(UniformCrossover.Builder builder) { + super(builder.parentSelection); + + this.crossoverRate = builder.crossoverRate; + this.parentBiasFactor = builder.parentBiasFactor; + this.rng = builder.rng; + } + + /** + * Has a probability crossoverRate of performing the crossover where the operator will select randomly which parent donates the gene.
+ * One of the parent may be favored if the bias is different than 0.5 + * Otherwise, returns the genes of a random parent. + * + * @return The crossover result. See {@link CrossoverResult}. + */ + @Override + public CrossoverResult crossover() { + // select the parents + double[][] parents = parentSelection.selectParents(); + + double[] resultGenes = parents[0]; + boolean isModified = false; + + if (rng.nextDouble() < crossoverRate) { + // Crossover + resultGenes = new double[parents[0].length]; + + for (int i = 0; i < resultGenes.length; ++i) { + resultGenes[i] = ((rng.nextDouble() < parentBiasFactor) ? parents[0] : parents[1])[i]; + } + isModified = true; + } + + return new CrossoverResult(isModified, resultGenes); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/ParentSelection.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/ParentSelection.java new file mode 100644 index 000000000..4fa9ed17c --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/ParentSelection.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; + +import java.util.List; + +/** + * Abstract class for all parent selection behaviors + * + * @author Alexandre Boulanger + */ +public abstract class ParentSelection { + protected List population; + + /** + * Will be called by the crossover operator once the population model is instantiated. + */ + public void initializeInstance(List population) { + this.population = population; + } + + /** + * Performs the parent selection + * + * @return An array of parents genes. The outer array are the parents, and the inner array are the genes. + */ + public abstract double[][] selectParents(); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/RandomTwoParentSelection.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/RandomTwoParentSelection.java new file mode 100644 index 000000000..81baeb07c --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/RandomTwoParentSelection.java @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection; + +import org.apache.commons.math3.random.JDKRandomGenerator; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.random.SynchronizedRandomGenerator; + +/** + * A parent selection behavior that returns two random parents. + * + * @author Alexandre Boulanger + */ +public class RandomTwoParentSelection extends TwoParentSelection { + + private final RandomGenerator rng; + + public RandomTwoParentSelection() { + this(new SynchronizedRandomGenerator(new JDKRandomGenerator())); + } + + /** + * Use a supplied RandomGenerator + * + * @param rng An instance of RandomGenerator + */ + public RandomTwoParentSelection(RandomGenerator rng) { + this.rng = rng; + } + + /** + * Selects two random parents + * + * @return An array of parents genes. The outer array are the parents, and the inner array are the genes. + */ + @Override + public double[][] selectParents() { + double[][] parents = new double[2][]; + + int parent1Idx = rng.nextInt(population.size()); + int parent2Idx; + do { + parent2Idx = rng.nextInt(population.size()); + } while (parent1Idx == parent2Idx); + + parents[0] = population.get(parent1Idx).getGenes(); + parents[1] = population.get(parent2Idx).getGenes(); + + return parents; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/TwoParentSelection.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/TwoParentSelection.java new file mode 100644 index 000000000..b4b4f4843 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/parentselection/TwoParentSelection.java @@ -0,0 +1,25 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection; + +/** + * Abstract class for all parent selection behaviors that selects two parents. + * + * @author Alexandre Boulanger + */ +public abstract class TwoParentSelection extends ParentSelection { +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/utils/CrossoverPointsGenerator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/utils/CrossoverPointsGenerator.java new file mode 100644 index 000000000..7e6e799e7 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/crossover/utils/CrossoverPointsGenerator.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.utils; + +import org.apache.commons.math3.random.RandomGenerator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.KPointCrossover; + +import java.util.*; + +/** + * A helper class used by {@link KPointCrossover} to generate the crossover points + * + * @author Alexandre Boulanger + */ +public class CrossoverPointsGenerator { + private final int minCrossovers; + private final int maxCrossovers; + private final RandomGenerator rng; + private List parameterIndexes; + + /** + * Constructor + * + * @param chromosomeLength The number of genes + * @param minCrossovers The minimum number of crossover points to generate + * @param maxCrossovers The maximum number of crossover points to generate + * @param rng A RandomGenerator instance + */ + public CrossoverPointsGenerator(int chromosomeLength, int minCrossovers, int maxCrossovers, RandomGenerator rng) { + this.minCrossovers = minCrossovers; + this.maxCrossovers = maxCrossovers; + this.rng = rng; + parameterIndexes = new ArrayList(); + for (int i = 0; i < chromosomeLength; ++i) { + parameterIndexes.add(i); + } + } + + /** + * Generate a list of crossover points. + * + * @return An ordered list of crossover point indexes and with Integer.MAX_VALUE as the last element + */ + public Deque getCrossoverPoints() { + Collections.shuffle(parameterIndexes); + List crossoverPointLists = + parameterIndexes.subList(0, rng.nextInt(maxCrossovers - minCrossovers) + minCrossovers); + Collections.sort(crossoverPointLists); + Deque crossoverPoints = new ArrayDeque(crossoverPointLists); + crossoverPoints.add(Integer.MAX_VALUE); + + return crossoverPoints; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/CullOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/CullOperator.java new file mode 100644 index 000000000..95452a7eb --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/CullOperator.java @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.culling; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; + +/** + * The cull operator will remove from the population the least desirables chromosomes. + * + * @author Alexandre Boulanger + */ +public interface CullOperator { + /** + * Will be called by the population model once created. + */ + void initializeInstance(PopulationModel populationModel); + + /** + * Cull the population to the culled size. + */ + void cullPopulation(); + + /** + * @return The target population size after culling. + */ + int getCulledSize(); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/LeastFitCullOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/LeastFitCullOperator.java new file mode 100644 index 000000000..6ec5c64df --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/LeastFitCullOperator.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.culling; + +/** + * An elitist cull operator that discards the chromosomes with the worst fitness while keeping the best ones. + * + * @author Alexandre Boulanger + */ +public class LeastFitCullOperator extends RatioCullOperator { + + /** + * The default cull ratio is 1/3. + */ + public LeastFitCullOperator() { + super(); + } + + /** + * @param cullRatio The ratio of the maximum population size to be culled.
+ * For example, a ratio of 1/3 on a population with a maximum size of 30 will cull back a given population to 20. + */ + public LeastFitCullOperator(double cullRatio) { + super(cullRatio); + } + + /** + * Will discard the chromosomes with the worst fitness until the population size fall back at the culled size. + */ + @Override + public void cullPopulation() { + while (population.size() > culledSize) { + population.remove(population.size() - 1); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/RatioCullOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/RatioCullOperator.java new file mode 100644 index 000000000..9c838acc8 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/culling/RatioCullOperator.java @@ -0,0 +1,70 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.culling; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.nd4j.common.base.Preconditions; + +import java.util.List; + +/** + * An abstract base for cull operators that culls back the population to a ratio of its maximum size. + * + * @author Alexandre Boulanger + */ +public abstract class RatioCullOperator implements CullOperator { + private static final double DEFAULT_CULL_RATIO = 1.0 / 3.0; + protected int culledSize; + protected List population; + protected final double cullRatio; + + /** + * @param cullRatio The ratio of the maximum population size to be culled.
+ * For example, a ratio of 1/3 on a population with a maximum size of 30 will cull back a given population to 20. + */ + public RatioCullOperator(double cullRatio) { + Preconditions.checkState(cullRatio >= 0.0 && cullRatio <= 1.0, "Cull ratio must be between 0.0 and 1.0, got %s", + cullRatio); + + this.cullRatio = cullRatio; + } + + /** + * The default cull ratio is 1/3 + */ + public RatioCullOperator() { + this(DEFAULT_CULL_RATIO); + } + + /** + * Will be called by the population model once created. + */ + public void initializeInstance(PopulationModel populationModel) { + this.population = populationModel.getPopulation(); + culledSize = (int) (populationModel.getPopulationSize() * (1.0 - cullRatio) + 0.5); + } + + /** + * @return The target population size after culling. + */ + @Override + public int getCulledSize() { + return culledSize; + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/exceptions/GeneticGenerationException.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/exceptions/GeneticGenerationException.java new file mode 100644 index 000000000..b0a9a42b3 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/exceptions/GeneticGenerationException.java @@ -0,0 +1,23 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.exceptions; + +public class GeneticGenerationException extends RuntimeException { + public GeneticGenerationException(String message) { + super(message); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/mutation/MutationOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/mutation/MutationOperator.java new file mode 100644 index 000000000..56f459a73 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/mutation/MutationOperator.java @@ -0,0 +1,33 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.mutation; + +/** + * The mutation operator will apply a mutation to the given genes. + * + * @author Alexandre Boulanger + */ +public interface MutationOperator { + + /** + * Performs a mutation. + * + * @param genes The genes to be mutated + * @return True if the genes were mutated, otherwise false. + */ + boolean mutate(double[] genes); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/mutation/RandomMutationOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/mutation/RandomMutationOperator.java new file mode 100644 index 000000000..ba10676b6 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/mutation/RandomMutationOperator.java @@ -0,0 +1,93 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.mutation; + +import org.apache.commons.math3.random.JDKRandomGenerator; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.random.SynchronizedRandomGenerator; +import org.nd4j.common.base.Preconditions; + +/** + * A mutation operator where each gene has a chance of being mutated with a mutation rate probability. + * + * @author Alexandre Boulanger + */ +public class RandomMutationOperator implements MutationOperator { + private static final double DEFAULT_MUTATION_RATE = 0.005; + + private final double mutationRate; + private final RandomGenerator rng; + + public static class Builder { + private double mutationRate = DEFAULT_MUTATION_RATE; + private RandomGenerator rng; + + /** + * Each gene will have this probability of being mutated. + * + * @param rate The mutation rate. (default 0.005) + */ + public Builder mutationRate(double rate) { + Preconditions.checkState(rate >= 0.0 && rate <= 1.0, "Rate must be between 0.0 and 1.0, got %s", rate); + + this.mutationRate = rate; + return this; + } + + /** + * Use a supplied RandomGenerator + * + * @param rng An instance of RandomGenerator + */ + public Builder randomGenerator(RandomGenerator rng) { + this.rng = rng; + return this; + } + + public RandomMutationOperator build() { + if (rng == null) { + rng = new SynchronizedRandomGenerator(new JDKRandomGenerator()); + } + return new RandomMutationOperator(this); + } + } + + private RandomMutationOperator(RandomMutationOperator.Builder builder) { + this.mutationRate = builder.mutationRate; + this.rng = builder.rng; + } + + /** + * Performs the mutation. Each gene has a mutation rate probability of being mutated. + * + * @param genes The genes to be mutated + * @return True if the genes were mutated, otherwise false. + */ + @Override + public boolean mutate(double[] genes) { + boolean hasMutation = false; + + for (int i = 0; i < genes.length; ++i) { + if (rng.nextDouble() < mutationRate) { + genes[i] = rng.nextDouble(); + hasMutation = true; + } + } + + return hasMutation; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/EmptyPopulationInitializer.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/EmptyPopulationInitializer.java new file mode 100644 index 000000000..20c147385 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/EmptyPopulationInitializer.java @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.population; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; + +import java.util.ArrayList; +import java.util.List; + +/** + * A population initializer that build an empty population. + * + * @author Alexandre Boulanger + */ +public class EmptyPopulationInitializer implements PopulationInitializer { + + /** + * Initialize an empty population + * + * @param size The maximum size of the population. + * @return The initialized population. + */ + @Override + public List getInitializedPopulation(int size) { + return new ArrayList<>(size); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationInitializer.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationInitializer.java new file mode 100644 index 000000000..40dd4f438 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationInitializer.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.population; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; + +import java.util.List; + +/** + * An initializer that construct the population used by the population model. + * + * @author Alexandre Boulanger + */ +public interface PopulationInitializer { + /** + * Called by the population model to construct the population + * + * @param size The maximum size of the population + * @return An initialized population + */ + List getInitializedPopulation(int size); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationListener.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationListener.java new file mode 100644 index 000000000..aca266b57 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationListener.java @@ -0,0 +1,35 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.population; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; + +import java.util.List; + +/** + * A listener that is called when the population changes. + * + * @author Alexandre Boulanger + */ +public interface PopulationListener { + /** + * Called after the population has changed. + * + * @param population The population after it has changed. + */ + void onChanged(List population); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationModel.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationModel.java new file mode 100644 index 000000000..9c5a4c7e1 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/population/PopulationModel.java @@ -0,0 +1,182 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.population; + +import lombok.Getter; +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.culling.CullOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.culling.LeastFitCullOperator; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * The population model handles all aspects of the population (initialization, additions and culling) + * + * @author Alexandre Boulanger + */ +public class PopulationModel { + private static final int DEFAULT_POPULATION_SIZE = 30; + + private final CullOperator cullOperator; + private final List populationListeners = new ArrayList<>(); + private Comparator chromosomeComparator; + + /** + * The maximum population size + */ + @Getter + private final int populationSize; + + /** + * The population + */ + @Getter + public final List population; + + /** + * A comparator used when higher fitness value is better + */ + public static class MaximizeScoreComparator implements Comparator { + @Override + public int compare(Chromosome lhs, Chromosome rhs) { + return -Double.compare(lhs.getFitness(), rhs.getFitness()); + } + } + + /** + * A comparator used when lower fitness value is better + */ + public static class MinimizeScoreComparator implements Comparator { + @Override + public int compare(Chromosome lhs, Chromosome rhs) { + return Double.compare(lhs.getFitness(), rhs.getFitness()); + } + } + + public static class Builder { + private int populationSize = DEFAULT_POPULATION_SIZE; + private PopulationInitializer populationInitializer; + private CullOperator cullOperator; + + /** + * Use an alternate population initialization behavior. Default is empty population. + * + * @param populationInitializer An instance of PopulationInitializer + */ + public Builder populationInitializer(PopulationInitializer populationInitializer) { + this.populationInitializer = populationInitializer; + return this; + } + + /** + * The maximum population size.
+ * If using a ratio based culling, using a population with culled size of around 1.5 to 2 times the number of genes generally gives good results. + * (e.g. For a chromosome having 10 genes, the culled size should be between 15 and 20. And with a cull ratio of 1/3 we should set the population size to 23 to 30. (15 / (1 - 1/3)), rounded up) + * + * @param size The maximum size of the population + */ + public Builder populationSize(int size) { + populationSize = size; + return this; + } + + /** + * Use an alternate cull operator behavior. Default is least fit culling. + * + * @param cullOperator An instance of a CullOperator + */ + public Builder cullOperator(CullOperator cullOperator) { + this.cullOperator = cullOperator; + return this; + } + + public PopulationModel build() { + if (cullOperator == null) { + cullOperator = new LeastFitCullOperator(); + } + + if (populationInitializer == null) { + populationInitializer = new EmptyPopulationInitializer(); + } + + return new PopulationModel(this); + } + + } + + public PopulationModel(PopulationModel.Builder builder) { + populationSize = builder.populationSize; + population = new ArrayList<>(builder.populationSize); + PopulationInitializer populationInitializer = builder.populationInitializer; + + List initializedPopulation = populationInitializer.getInitializedPopulation(populationSize); + population.clear(); + population.addAll(initializedPopulation); + + cullOperator = builder.cullOperator; + cullOperator.initializeInstance(this); + } + + /** + * Called by the GeneticSearchCandidateGenerator + */ + public void initializeInstance(boolean minimizeScore) { + chromosomeComparator = minimizeScore ? new MinimizeScoreComparator() : new MaximizeScoreComparator(); + } + + /** + * Add a PopulationListener to the list of change listeners + * @param listener A PopulationListener instance + */ + public void addListener(PopulationListener listener) { + populationListeners.add(listener); + } + + /** + * Add a Chromosome to the population and call the PopulationListeners. Culling may be triggered. + * + * @param element The chromosome to be added + */ + public void add(Chromosome element) { + if (population.size() == populationSize) { + cullOperator.cullPopulation(); + } + + population.add(element); + + Collections.sort(population, chromosomeComparator); + + triggerPopulationChangedListeners(population); + } + + /** + * @return Return false when the population is below the culled size, otherwise true.
+ * Used by the selection operator to know if the population is still too small and should generate random genes. + */ + public boolean isReadyToBreed() { + return population.size() >= cullOperator.getCulledSize(); + } + + private void triggerPopulationChangedListeners(List population) { + for (PopulationListener listener : populationListeners) { + listener.onChanged(population); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/selection/GeneticSelectionOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/selection/GeneticSelectionOperator.java new file mode 100644 index 000000000..40b6a49c8 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/selection/GeneticSelectionOperator.java @@ -0,0 +1,197 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.selection; + +import org.apache.commons.math3.random.JDKRandomGenerator; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.random.SynchronizedRandomGenerator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.ChromosomeFactory; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverResult; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.SinglePointCrossover; +import org.deeplearning4j.arbiter.optimize.generator.genetic.exceptions.GeneticGenerationException; +import org.deeplearning4j.arbiter.optimize.generator.genetic.mutation.MutationOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.mutation.RandomMutationOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; + +import java.util.Arrays; + +/** + * A selection operator that will generate random genes initially. Once the population has reached the culled size, + * will start to generate offsprings of parents selected in the population. + * + * @author Alexandre Boulanger + */ +public class GeneticSelectionOperator extends SelectionOperator { + + private final static int PREVIOUS_GENES_TO_KEEP = 100; + private final static int MAX_NUM_GENERATION_ATTEMPTS = 1024; + + private final CrossoverOperator crossoverOperator; + private final MutationOperator mutationOperator; + private final RandomGenerator rng; + private double[][] previousGenes = new double[PREVIOUS_GENES_TO_KEEP][]; + private int previousGenesIdx = 0; + + public static class Builder { + private ChromosomeFactory chromosomeFactory; + private PopulationModel populationModel; + private CrossoverOperator crossoverOperator; + private MutationOperator mutationOperator; + private RandomGenerator rng; + + /** + * Use an alternate crossover behavior. Default is SinglePointCrossover. + * + * @param crossoverOperator An instance of CrossoverOperator + */ + public Builder crossoverOperator(CrossoverOperator crossoverOperator) { + this.crossoverOperator = crossoverOperator; + return this; + } + + /** + * Use an alternate mutation behavior. Default is RandomMutationOperator. + * + * @param mutationOperator An instance of MutationOperator + */ + public Builder mutationOperator(MutationOperator mutationOperator) { + this.mutationOperator = mutationOperator; + return this; + } + + /** + * Use a supplied RandomGenerator + * + * @param rng An instance of RandomGenerator + */ + public Builder randomGenerator(RandomGenerator rng) { + this.rng = rng; + return this; + } + + public GeneticSelectionOperator build() { + if (crossoverOperator == null) { + crossoverOperator = new SinglePointCrossover.Builder().build(); + } + + if (mutationOperator == null) { + mutationOperator = new RandomMutationOperator.Builder().build(); + } + + if (rng == null) { + rng = new SynchronizedRandomGenerator(new JDKRandomGenerator()); + } + + return new GeneticSelectionOperator(crossoverOperator, mutationOperator, rng); + } + } + + private GeneticSelectionOperator(CrossoverOperator crossoverOperator, MutationOperator mutationOperator, + RandomGenerator rng) { + this.crossoverOperator = crossoverOperator; + this.mutationOperator = mutationOperator; + this.rng = rng; + } + + /** + * Called by GeneticSearchCandidateGenerator + */ + @Override + public void initializeInstance(PopulationModel populationModel, ChromosomeFactory chromosomeFactory) { + super.initializeInstance(populationModel, chromosomeFactory); + crossoverOperator.initializeInstance(populationModel); + } + + /** + * Build a new set of genes. Has two distinct modes of operation + *

+ * @return Returns the generated set of genes + * @throws GeneticGenerationException If buildNextGenes() can't generate a set that has not already been tried, + * or if the crossover and the mutation operators can't generate a set, + * this exception is thrown. + */ + @Override + public double[] buildNextGenes() { + double[] result; + + boolean hasAlreadyBeenTried; + int attemptsRemaining = MAX_NUM_GENERATION_ATTEMPTS; + do { + if (populationModel.isReadyToBreed()) { + result = buildOffspring(); + } else { + result = buildRandomGenes(); + } + + hasAlreadyBeenTried = hasAlreadyBeenTried(result); + if (hasAlreadyBeenTried && --attemptsRemaining == 0) { + throw new GeneticGenerationException("Failed to generate a set of genes not already tried."); + } + } while (hasAlreadyBeenTried); + + previousGenes[previousGenesIdx] = result; + previousGenesIdx = ++previousGenesIdx % previousGenes.length; + + return result; + } + + private boolean hasAlreadyBeenTried(double[] genes) { + for (int i = 0; i < previousGenes.length; ++i) { + double[] current = previousGenes[i]; + if (current != null && Arrays.equals(current, genes)) { + return true; + } + } + + return false; + } + + private double[] buildOffspring() { + double[] offspringValues; + + boolean isModified; + int attemptsRemaining = MAX_NUM_GENERATION_ATTEMPTS; + do { + CrossoverResult crossoverResult = crossoverOperator.crossover(); + offspringValues = crossoverResult.getGenes(); + isModified = crossoverResult.isModified(); + isModified |= mutationOperator.mutate(offspringValues); + + if (!isModified && --attemptsRemaining == 0) { + throw new GeneticGenerationException( + String.format("Crossover and mutation operators failed to generate a new set of genes after %s attempts.", + MAX_NUM_GENERATION_ATTEMPTS)); + } + } while (!isModified); + + return offspringValues; + } + + private double[] buildRandomGenes() { + double[] randomValues = new double[chromosomeFactory.getChromosomeLength()]; + for (int i = 0; i < randomValues.length; ++i) { + randomValues[i] = rng.nextDouble(); + } + + return randomValues; + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/selection/SelectionOperator.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/selection/SelectionOperator.java new file mode 100644 index 000000000..7be470ea6 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/genetic/selection/SelectionOperator.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.genetic.selection; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.ChromosomeFactory; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; + +/** + * An abstract class for all selection operators. Used by the GeneticSearchCandidateGenerator to generate new candidates. + * + * @author Alexandre Boulanger + */ +public abstract class SelectionOperator { + protected PopulationModel populationModel; + protected ChromosomeFactory chromosomeFactory; + + /** + * Called by GeneticSearchCandidateGenerator + */ + public void initializeInstance(PopulationModel populationModel, ChromosomeFactory chromosomeFactory) { + + this.populationModel = populationModel; + this.chromosomeFactory = chromosomeFactory; + } + + /** + * Generate a new set of genes. + */ + public abstract double[] buildNextGenes(); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/util/SerializedSupplier.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/util/SerializedSupplier.java new file mode 100644 index 000000000..81109816d --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/generator/util/SerializedSupplier.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.generator.util; + +import org.nd4j.common.function.Supplier; + +import java.io.*; + +public class SerializedSupplier implements Serializable, Supplier { + + private byte[] asBytes; + + public SerializedSupplier(T obj){ + try(ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos)){ + oos.writeObject(obj); + oos.flush(); + oos.close(); + asBytes = baos.toByteArray(); + } catch (Exception e){ + throw new RuntimeException("Error serializing object - must be serializable",e); + } + } + + @Override + public T get() { + try(ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(asBytes))){ + return (T)ois.readObject(); + } catch (Exception e){ + throw new RuntimeException("Error deserializing object",e); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/BooleanSpace.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/BooleanSpace.java new file mode 100644 index 000000000..fd20afb47 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/BooleanSpace.java @@ -0,0 +1,76 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter; + +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * BooleanParameterSpace is a {@code ParameterSpace}; Defines {True, False} as a parameter space + * If argument to setValue is less than or equal to 0.5 it will return True else False + * + * @author susaneraly + */ +@EqualsAndHashCode +public class BooleanSpace implements ParameterSpace { + private int index = -1; + + @Override + public Boolean getValue(double[] input) { + if (index == -1) { + throw new IllegalStateException("Cannot get value: ParameterSpace index has not been set"); + } + if (input[index] <= 0.5) return Boolean.TRUE; + else return Boolean.FALSE; + } + + @Override + public int numParameters() { + return 1; + } + + @Override + public List collectLeaves() { + return Collections.singletonList((ParameterSpace) this); + } + + @Override + public Map getNestedSpaces() { + return Collections.emptyMap(); + } + + @Override + public boolean isLeaf() { + return true; + } + + @Override + public void setIndices(int... indices) { + if (indices == null || indices.length != 1) + throw new IllegalArgumentException("Invalid index"); + this.index = indices[0]; + } + + @Override + public String toString() { + return "BooleanSpace()"; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/FixedValue.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/FixedValue.java new file mode 100644 index 000000000..b22f77a52 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/FixedValue.java @@ -0,0 +1,90 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.serde.jackson.FixedValueDeserializer; +import org.deeplearning4j.arbiter.optimize.serde.jackson.FixedValueSerializer; +import org.deeplearning4j.arbiter.util.ObjectUtils; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * FixedValue is a ParameterSpace that defines only a single fixed value + * + * @param Type of (fixed) value + */ +@EqualsAndHashCode +@JsonSerialize(using = FixedValueSerializer.class) +@JsonDeserialize(using = FixedValueDeserializer.class) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public class FixedValue implements ParameterSpace { + @Getter + private Object value; + private int index; + + @JsonCreator + public FixedValue(@JsonProperty("value") T value) { + this.value = value; + } + + @Override + public String toString() { + return "FixedValue(" + ObjectUtils.valueToString(value) + ")"; + } + + @Override + public T getValue(double[] input) { + return (T) value; + } + + @Override + public int numParameters() { + return 0; + } + + @Override + public List collectLeaves() { + return Collections.emptyList(); + } + + @Override + public Map getNestedSpaces() { + return Collections.emptyMap(); + } + + @Override + public boolean isLeaf() { + return true; + } + + @Override + public void setIndices(int... indices) { + if (indices != null && indices.length != 0) + throw new IllegalArgumentException( + "Invalid call: FixedValue ParameterSpace " + "should not be given an index (0 params)"); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/continuous/ContinuousParameterSpace.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/continuous/ContinuousParameterSpace.java new file mode 100644 index 000000000..c8f139ebb --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/continuous/ContinuousParameterSpace.java @@ -0,0 +1,137 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter.continuous; + +import org.apache.commons.math3.distribution.RealDistribution; +import org.apache.commons.math3.distribution.UniformRealDistribution; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.distribution.DistributionUtils; +import org.deeplearning4j.arbiter.optimize.serde.jackson.RealDistributionDeserializer; +import org.deeplearning4j.arbiter.optimize.serde.jackson.RealDistributionSerializer; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * ContinuousParametSpace is a {@code ParameterSpace} that (optionally) takes an Apache Commons + * {@link RealDistribution} when used for random sampling (such as in a RandomSearchCandidateGenerator) + * + * @author Alex Black + */ +public class ContinuousParameterSpace implements ParameterSpace { + + //Need to use custom serializers/deserializers for commons RealDistribution instances + @JsonSerialize(using = RealDistributionSerializer.class) + @JsonDeserialize(using = RealDistributionDeserializer.class) + private RealDistribution distribution; + private int index = -1; + + /** + * ContinuousParameterSpace with uniform distribution between the minimum and maximum values + * + * @param min Minimum value that can be generated + * @param max Maximum value that can be generated + */ + public ContinuousParameterSpace(double min, double max) { + this(new UniformRealDistribution(min, max)); + } + + /** + * ConditiousParameterSpcae wiht a specified probability distribution. The provided distribution defines the min/max + * values, and (for random search, etc) will be used when generating random values + * + * @param distribution Distribution to sample from + */ + public ContinuousParameterSpace(@JsonProperty("distribution") RealDistribution distribution) { + this.distribution = distribution; + } + + + @Override + public Double getValue(double[] input) { + if (index == -1) { + throw new IllegalStateException("Cannot get value: ParameterSpace index has not been set"); + } + return distribution.inverseCumulativeProbability(input[index]); + } + + @Override + public int numParameters() { + return 1; + } + + @Override + public List collectLeaves() { + return Collections.singletonList((ParameterSpace) this); + } + + @Override + public Map getNestedSpaces() { + return Collections.emptyMap(); + } + + @Override + public boolean isLeaf() { + return true; + } + + @Override + public void setIndices(int... indices) { + if (indices == null || indices.length != 1) { + throw new IllegalArgumentException("Invalid index"); + } + this.index = indices[0]; + } + + + @Override + public String toString() { + if (distribution instanceof UniformRealDistribution) { + return "ContinuousParameterSpace(min=" + distribution.getSupportLowerBound() + ",max=" + + distribution.getSupportUpperBound() + ")"; + } else { + return "ContinuousParameterSpace(" + distribution + ")"; + } + } + + public boolean equals(Object o) { + if (o == this) + return true; + if (!(o instanceof ContinuousParameterSpace)) + return false; + final ContinuousParameterSpace other = (ContinuousParameterSpace) o; + if (distribution == null ? other.distribution != null + : !DistributionUtils.distributionsEqual(distribution, other.distribution)) + return false; + if (this.index != other.index) + return false; + return true; + } + + public int hashCode() { + final int PRIME = 59; + int result = 1; + result = result * PRIME + (distribution == null ? 43 : distribution.getClass().hashCode()); + result = result * PRIME + this.index; + return result; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/discrete/DiscreteParameterSpace.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/discrete/DiscreteParameterSpace.java new file mode 100644 index 000000000..3c70aaa03 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/discrete/DiscreteParameterSpace.java @@ -0,0 +1,113 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter.discrete; + +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.util.ObjectUtils; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.util.*; + +/** + * A DiscreteParameterSpace is used for a set of un-ordered values + * + * @param

Parameter type + * @author Alex Black + */ +@EqualsAndHashCode +public class DiscreteParameterSpace

implements ParameterSpace

{ + + @JsonSerialize + private List

values; + private int index = -1; + + public DiscreteParameterSpace(@JsonProperty("values") P... values) { + if (values != null) + this.values = Arrays.asList(values); + } + + public DiscreteParameterSpace(Collection

values) { + this.values = new ArrayList<>(values); + } + + public int numValues() { + return values.size(); + } + + @Override + public P getValue(double[] input) { + if (index == -1) { + throw new IllegalStateException("Cannot get value: ParameterSpace index has not been set"); + } + if (values == null) + throw new IllegalStateException("Values are null."); + //Map a value in range [0,1] to one of the list of values + //First value: [0,width], second: (width,2*width], third: (3*width,4*width] etc + int size = values.size(); + if (size == 1) + return values.get(0); + double width = 1.0 / size; + int val = (int) (input[index] / width); + return values.get(Math.min(val, size - 1)); + } + + @Override + public int numParameters() { + return 1; + } + + @Override + public List collectLeaves() { + return Collections.singletonList((ParameterSpace) this); + } + + @Override + public Map getNestedSpaces() { + return Collections.emptyMap(); + } + + @Override + public boolean isLeaf() { + return true; + } + + @Override + public void setIndices(int... indices) { + if (indices == null || indices.length != 1) { + throw new IllegalArgumentException("Invalid index"); + } + this.index = indices[0]; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("DiscreteParameterSpace("); + int n = values.size(); + for (int i = 0; i < n; i++) { + P value = values.get(i); + sb.append(ObjectUtils.valueToString(value)); + sb.append((i == n - 1 ? ")" : ",")); + } + return sb.toString(); + } + + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/integer/IntegerParameterSpace.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/integer/IntegerParameterSpace.java new file mode 100644 index 000000000..d76381244 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/integer/IntegerParameterSpace.java @@ -0,0 +1,151 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter.integer; + +import lombok.NoArgsConstructor; +import org.apache.commons.math3.distribution.IntegerDistribution; +import org.apache.commons.math3.distribution.UniformIntegerDistribution; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.distribution.DistributionUtils; +import org.deeplearning4j.arbiter.optimize.serde.jackson.IntegerDistributionDeserializer; +import org.deeplearning4j.arbiter.optimize.serde.jackson.IntegerDistributionSerializer; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * IntegerParameterSpace is a {@code ParameterSpace}; i.e., defines an ordered space of integers between + * some minimum and maximum value + * + * @author Alex Black + */ +@JsonIgnoreProperties({"min", "max"}) +@NoArgsConstructor +public class IntegerParameterSpace implements ParameterSpace { + + @JsonSerialize(using = IntegerDistributionSerializer.class) + @JsonDeserialize(using = IntegerDistributionDeserializer.class) + private IntegerDistribution distribution; + private int index = -1; + + /** + * Create an IntegerParameterSpace with a uniform distribution between the specified min/max (inclusive) + * + * @param min Min value, inclusive + * @param max Max value, inclusive + */ + public IntegerParameterSpace(int min, int max) { + this(new UniformIntegerDistribution(min, max)); + } + + /** + * Crate an IntegerParametSpace from the given IntegerDistribution + * + * @param distribution Distribution to use + */ + @JsonCreator + public IntegerParameterSpace(@JsonProperty("distribution") IntegerDistribution distribution) { + this.distribution = distribution; + } + + public int getMin() { + return distribution.getSupportLowerBound(); + } + + public int getMax() { + return distribution.getSupportUpperBound(); + } + + @Override + public Integer getValue(double[] input) { + if (index == -1) { + throw new IllegalStateException("Cannot get value: ParameterSpace index has not been set"); + } + return distribution.inverseCumulativeProbability(input[index]); + } + + @Override + public int numParameters() { + return 1; + } + + @Override + public List collectLeaves() { + return Collections.singletonList((ParameterSpace) this); + } + + @Override + public Map getNestedSpaces() { + return Collections.emptyMap(); + } + + @Override + public boolean isLeaf() { + return true; + } + + @Override + public void setIndices(int... indices) { + if (indices == null || indices.length != 1) + throw new IllegalArgumentException("Invalid index"); + this.index = indices[0]; + } + + @Override + public String toString() { + if (distribution instanceof UniformIntegerDistribution) { + return "IntegerParameterSpace(min=" + distribution.getSupportLowerBound() + ",max=" + + distribution.getSupportUpperBound() + ")"; + } else { + return "IntegerParameterSpace(" + distribution + ")"; + } + } + + public boolean equals(Object o) { + if (o == this) + return true; + if (!(o instanceof IntegerParameterSpace)) + return false; + final IntegerParameterSpace other = (IntegerParameterSpace) o; + if (!other.canEqual(this)) + return false; + if (distribution == null ? other.distribution != null + : !DistributionUtils.distributionEquals(distribution, other.distribution)) + return false; + if (this.index != other.index) + return false; + return true; + } + + public int hashCode() { + final int PRIME = 59; + int result = 1; + result = result * PRIME + (distribution == null ? 43 : distribution.getClass().hashCode()); + result = result * PRIME + this.index; + return result; + } + + protected boolean canEqual(Object other) { + return other instanceof IntegerParameterSpace; + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/MathOp.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/MathOp.java new file mode 100644 index 000000000..2d567536f --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/MathOp.java @@ -0,0 +1,69 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter.math; + +import org.deeplearning4j.arbiter.optimize.api.AbstractParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; + +import java.util.List; + +/** + * A simple parameter space that implements scalar mathematical operations on another parameter space. This allows you + * to do things like Y = X * 2, where X is a parameter space. For example, a layer size hyperparameter could be set + * using this to 2x the size of the previous layer + * + * @param Type of the parameter space + * @author Alex Black + */ +public class MathOp extends AbstractParameterSpace { + + private ParameterSpace parameterSpace; + private Op op; + private T scalar; + + public MathOp(ParameterSpace parameterSpace, Op op, T scalar){ + this.parameterSpace = parameterSpace; + this.op = op; + this.scalar = scalar; + } + + @Override + public T getValue(double[] parameterValues) { + T u = parameterSpace.getValue(parameterValues); + return op.doOp(u, scalar); + } + + @Override + public int numParameters() { + return parameterSpace.numParameters(); + } + + @Override + public List collectLeaves() { + return parameterSpace.collectLeaves(); + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + parameterSpace.setIndices(indices); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/Op.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/Op.java new file mode 100644 index 000000000..2102804ce --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/Op.java @@ -0,0 +1,76 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter.math; + +public enum Op { + ADD, SUB, MUL, DIV; + + + //Package private + T doOp(T first, T second){ + if(first instanceof Integer || first instanceof Long){ + long result; + switch (this){ + case ADD: + result = Long.valueOf(first.longValue() + second.longValue()); + break; + case SUB: + result = Long.valueOf(first.longValue() - second.longValue()); + break; + case MUL: + result = Long.valueOf(first.longValue() * second.longValue()); + break; + case DIV: + result = Long.valueOf(first.longValue() / second.longValue()); + break; + default: + throw new UnsupportedOperationException("Unknown op: " + this); + } + if(first instanceof Long){ + return (T)Long.valueOf(result); + } else { + return (T)Integer.valueOf((int)result); + } + } else if(first instanceof Double || first instanceof Float){ + double result; + switch (this){ + case ADD: + result = Double.valueOf(first.doubleValue() + second.doubleValue()); + break; + case SUB: + result = Double.valueOf(first.doubleValue() - second.doubleValue()); + break; + case MUL: + result = Double.valueOf(first.doubleValue() * second.doubleValue()); + break; + case DIV: + result = Double.valueOf(first.doubleValue() / second.doubleValue()); + break; + default: + throw new UnsupportedOperationException("Unknown op: " + this); + } + if(first instanceof Double){ + return (T)Double.valueOf(result); + } else { + return (T)Float.valueOf((float)result); + } + } else { + throw new UnsupportedOperationException("Not supported type: only Integer, Long, Double, Float supported" + + " here. Got type: " + first.getClass()); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/PairMathOp.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/PairMathOp.java new file mode 100644 index 000000000..db0a9c98b --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/parameter/math/PairMathOp.java @@ -0,0 +1,79 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter.math; + +import org.deeplearning4j.arbiter.optimize.api.AbstractParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * A simple parameter space that implements pairwise mathematical operations on another parameter space. This allows you + * to do things like Z = X + Y, where X and Y are parameter spaces. + * + * @param Type of the parameter space + * @author Alex Black + */ +public class PairMathOp extends AbstractParameterSpace { + + private ParameterSpace first; + private ParameterSpace second; + private Op op; + + public PairMathOp(ParameterSpace first, ParameterSpace second, Op op){ + this.first = first; + this.second = second; + this.op = op; + } + + @Override + public T getValue(double[] parameterValues) { + T f = first.getValue(parameterValues); + T s = second.getValue(parameterValues); + return op.doOp(f, s); + } + + @Override + public int numParameters() { + return first.numParameters() + second.numParameters(); + } + + @Override + public List collectLeaves() { + List l = new ArrayList<>(); + l.addAll(first.collectLeaves()); + l.addAll(second.collectLeaves()); + return l; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + int n1 = first.numParameters(); + int n2 = second.numParameters(); + int[] s1 = Arrays.copyOfRange(indices, 0, n1); + int[] s2 = Arrays.copyOfRange(indices, n1, n1+n2); + first.setIndices(s1); + second.setIndices(s2); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/BaseOptimizationRunner.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/BaseOptimizationRunner.java new file mode 100644 index 000000000..fa503ef6d --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/BaseOptimizationRunner.java @@ -0,0 +1,383 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.api.termination.TerminationCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * BaseOptimization runner: responsible for scheduling tasks, saving results using the result saver, etc. + * + * @author Alex Black + */ +@Slf4j +public abstract class BaseOptimizationRunner implements IOptimizationRunner { + private static final int POLLING_FREQUENCY = 1; + private static final TimeUnit POLLING_FREQUENCY_UNIT = TimeUnit.SECONDS; + + protected OptimizationConfiguration config; + protected Queue> queuedFutures = new ConcurrentLinkedQueue<>(); + protected BlockingQueue> completedFutures = new LinkedBlockingQueue<>(); + protected AtomicInteger totalCandidateCount = new AtomicInteger(); + protected AtomicInteger numCandidatesCompleted = new AtomicInteger(); + protected AtomicInteger numCandidatesFailed = new AtomicInteger(); + protected Double bestScore = null; + protected Long bestScoreTime = null; + protected AtomicInteger bestScoreCandidateIndex = new AtomicInteger(-1); + protected List allResults = new ArrayList<>(); + + protected Map currentStatus = new ConcurrentHashMap<>(); //TODO: better design possible? + + protected ExecutorService futureListenerExecutor; + + protected List statusListeners = new ArrayList<>(); + + + protected BaseOptimizationRunner(OptimizationConfiguration config) { + this.config = config; + + if (config.getTerminationConditions() == null || config.getTerminationConditions().size() == 0) { + throw new IllegalArgumentException("Cannot create BaseOptimizationRunner without TerminationConditions (" + + "termination conditions are null or empty)"); + } + + } + + protected void init() { + futureListenerExecutor = Executors.newFixedThreadPool(maxConcurrentTasks(), new ThreadFactory() { + private AtomicLong counter = new AtomicLong(0); + + @Override + public Thread newThread(Runnable r) { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + t.setName("ArbiterOptimizationRunner-" + counter.getAndIncrement()); + return t; + } + }); + } + + /** + * + */ + @Override + public void execute() { + log.info("{}: execution started", this.getClass().getSimpleName()); + config.setExecutionStartTime(System.currentTimeMillis()); + for (StatusListener listener : statusListeners) { + listener.onInitialization(this); + } + + //Initialize termination conditions (start timers, etc) + for (TerminationCondition c : config.getTerminationConditions()) { + c.initialize(this); + } + + //Queue initial tasks: + List> tempList = new ArrayList<>(100); + while (true) { + //Otherwise: add tasks if required + Future future = null; + try { + future = completedFutures.poll(POLLING_FREQUENCY, POLLING_FREQUENCY_UNIT); + } catch (InterruptedException e) { + //No op? + } + if (future != null) { + tempList.add(future); + } + completedFutures.drainTo(tempList); + + //Process results (if any) + for (Future f : tempList) { + queuedFutures.remove(f); + processReturnedTask(f); + } + + if (tempList.size() > 0) { + for (StatusListener sl : statusListeners) { + sl.onRunnerStatusChange(this); + } + } + tempList.clear(); + + //Check termination conditions: + if (terminate()) { + shutdown(true); + break; + } + + //Add additional tasks + while (config.getCandidateGenerator().hasMoreCandidates() && queuedFutures.size() < maxConcurrentTasks()) { + Candidate candidate = config.getCandidateGenerator().getCandidate(); + CandidateInfo status; + if (candidate.getException() != null) { + //Failed on generation... + status = processFailedCandidates(candidate); + } else { + long created = System.currentTimeMillis(); + ListenableFuture f; + if(config.getDataSource() != null){ + f = execute(candidate, config.getDataSource(), config.getDataSourceProperties(), config.getScoreFunction()); + } else { + f = execute(candidate, config.getDataProvider(), config.getScoreFunction()); + } + f.addListener(new OnCompletionListener(f), futureListenerExecutor); + queuedFutures.add(f); + totalCandidateCount.getAndIncrement(); + + status = new CandidateInfo(candidate.getIndex(), CandidateStatus.Created, null, + created, null, null, candidate.getFlatParameters(), null); + currentStatus.put(candidate.getIndex(), status); + } + + for (StatusListener listener : statusListeners) { + listener.onCandidateStatusChange(status, this, null); + } + } + } + + //Process any final (completed) tasks: + completedFutures.drainTo(tempList); + for (Future f : tempList) { + queuedFutures.remove(f); + processReturnedTask(f); + } + tempList.clear(); + + log.info("Optimization runner: execution complete"); + for (StatusListener listener : statusListeners) { + listener.onShutdown(this); + } + } + + + private CandidateInfo processFailedCandidates(Candidate candidate) { + //In case the candidate fails during the creation of the candidate + + long time = System.currentTimeMillis(); + String stackTrace = ExceptionUtils.getStackTrace(candidate.getException()); + CandidateInfo newStatus = new CandidateInfo(candidate.getIndex(), CandidateStatus.Failed, null, time, time, + time, candidate.getFlatParameters(), stackTrace); + currentStatus.put(candidate.getIndex(), newStatus); + + return newStatus; + } + + /** + * Process returned task (either completed or failed + */ + private void processReturnedTask(Future future) { + long currentTime = System.currentTimeMillis(); + OptimizationResult result; + try { + result = future.get(100, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Unexpected InterruptedException thrown for task", e); + } catch (ExecutionException e) { + //Note that most of the time, an OptimizationResult is returned even for an exception + //This is just to handle any that are missed there (or, by implementations that don't properly do this) + log.warn("Task failed", e); + + numCandidatesFailed.getAndIncrement(); + return; + } catch (TimeoutException e) { + throw new RuntimeException(e); //TODO + } + + //Update internal status: + CandidateInfo status = currentStatus.get(result.getIndex()); + CandidateInfo newStatus = new CandidateInfo(result.getIndex(), result.getCandidateInfo().getCandidateStatus(), + result.getScore(), status.getCreatedTime(), result.getCandidateInfo().getStartTime(), + currentTime, status.getFlatParams(), result.getCandidateInfo().getExceptionStackTrace()); + currentStatus.put(result.getIndex(), newStatus); + + //Listeners (on complete, etc) should be executed in underlying task + + + if (result.getCandidateInfo().getCandidateStatus() == CandidateStatus.Failed) { + log.info("Task {} failed during execution: {}", result.getIndex(), result.getCandidateInfo().getExceptionStackTrace()); + numCandidatesFailed.getAndIncrement(); + } else { + + //Report completion to candidate generator + config.getCandidateGenerator().reportResults(result); + + Double score = result.getScore(); + log.info("Completed task {}, score = {}", result.getIndex(), result.getScore()); + + boolean minimize = config.getScoreFunction().minimize(); + if (score != null && (bestScore == null + || ((minimize && score < bestScore) || (!minimize && score > bestScore)))) { + if (bestScore == null) { + log.info("New best score: {} (first completed model)", score); + } else { + int idx = result.getIndex(); + int lastBestIdx = bestScoreCandidateIndex.get(); + log.info("New best score: {}, model {} (prev={}, model {})", score, idx, bestScore, lastBestIdx); + } + bestScore = score; + bestScoreTime = System.currentTimeMillis(); + bestScoreCandidateIndex.set(result.getIndex()); + } + numCandidatesCompleted.getAndIncrement(); + + //Model saving is done in the optimization tasks, to avoid CUDA threading issues + ResultReference resultReference = result.getResultReference(); + + if (resultReference != null) + allResults.add(resultReference); + } + } + + @Override + public int numCandidatesTotal() { + return totalCandidateCount.get(); + } + + @Override + public int numCandidatesCompleted() { + return numCandidatesCompleted.get(); + } + + @Override + public int numCandidatesFailed() { + return numCandidatesFailed.get(); + } + + @Override + public int numCandidatesQueued() { + return queuedFutures.size(); + } + + @Override + public Double bestScore() { + return bestScore; + } + + @Override + public Long bestScoreTime() { + return bestScoreTime; + } + + @Override + public int bestScoreCandidateIndex() { + return bestScoreCandidateIndex.get(); + } + + @Override + public List getResults() { + return new ArrayList<>(allResults); + } + + @Override + public OptimizationConfiguration getConfiguration() { + return config; + } + + + @Override + public void addListeners(StatusListener... listeners) { + for (StatusListener l : listeners) { + if (!statusListeners.contains(l)) { + statusListeners.add(l); + } + } + } + + @Override + public void removeListeners(StatusListener... listeners) { + for (StatusListener l : listeners) { + if (statusListeners.contains(l)) { + statusListeners.remove(l); + } + } + } + + @Override + public void removeAllListeners() { + statusListeners.clear(); + } + + @Override + public List getCandidateStatus() { + List list = new ArrayList<>(); + list.addAll(currentStatus.values()); + return list; + } + + private boolean terminate() { + for (TerminationCondition c : config.getTerminationConditions()) { + if (c.terminate(this)) { + log.info("BaseOptimizationRunner global termination condition hit: {}", c); + return true; + } + } + return false; + } + + @AllArgsConstructor + @Data + private class FutureDetails { + private final Future future; + private final long startTime; + private final int index; + } + + @AllArgsConstructor + private class OnCompletionListener implements Runnable { + private Future future; + + @Override + public void run() { + completedFutures.add(future); + } + } + + + protected abstract int maxConcurrentTasks(); + + @Deprecated + protected abstract ListenableFuture execute(Candidate candidate, DataProvider dataProvider, + ScoreFunction scoreFunction); + @Deprecated + protected abstract List> execute(List candidates, + DataProvider dataProvider, ScoreFunction scoreFunction); + + protected abstract ListenableFuture execute(Candidate candidate, Class dataSource, + Properties dataSourceProperties, ScoreFunction scoreFunction); + + protected abstract List> execute(List candidates, Class dataSource, + Properties dataSourceProperties, ScoreFunction scoreFunction); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/CandidateInfo.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/CandidateInfo.java new file mode 100644 index 000000000..e8c7ccf25 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/CandidateInfo.java @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * Simple helper class to store status of a candidate that is/has been/will be executed + */ +@AllArgsConstructor +@Data +public class CandidateInfo { + + public CandidateInfo() { + //No arg constructor for Jackson + } + + private int index; + private CandidateStatus candidateStatus; + private Double score; + private long createdTime; + private Long startTime; + private Long endTime; + private double[] flatParams; //Same as parameters in Candidate class + private String exceptionStackTrace; +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/CandidateStatus.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/CandidateStatus.java new file mode 100644 index 000000000..a19f89a52 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/CandidateStatus.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner; + +/** + * Status for candidates + */ +public enum CandidateStatus { + Created, Running, Complete, Failed, Cancelled +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/IOptimizationRunner.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/IOptimizationRunner.java new file mode 100644 index 000000000..50e6dc4b0 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/IOptimizationRunner.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner; + +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.util.List; + +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface IOptimizationRunner { + + void execute(); + + /** Total number of candidates: created (scheduled), completed and failed */ + int numCandidatesTotal(); + + int numCandidatesCompleted(); + + int numCandidatesFailed(); + + /** Number of candidates running or queued */ + int numCandidatesQueued(); + + /** Best score found so far */ + Double bestScore(); + + /** Time that the best score was found at, or 0 if no jobs have completed successfully */ + Long bestScoreTime(); + + /** Index of the best scoring candidate, or -1 if no candidate has scored yet*/ + int bestScoreCandidateIndex(); + + List getResults(); + + OptimizationConfiguration getConfiguration(); + + void addListeners(StatusListener... listeners); + + void removeListeners(StatusListener... listeners); + + void removeAllListeners(); + + List getCandidateStatus(); + + /** + * @param awaitCompletion If true: await completion of currently scheduled tasks. If false: shutdown immediately, + * cancelling any currently executing tasks + */ + void shutdown(boolean awaitCompletion); +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/LocalOptimizationRunner.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/LocalOptimizationRunner.java new file mode 100644 index 000000000..a3992b09a --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/LocalOptimizationRunner.java @@ -0,0 +1,150 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.Setter; +import org.deeplearning4j.arbiter.optimize.api.*; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; + +/** + * LocalOptimizationRunner: execute hyperparameter optimization + * locally (on current machine, in current JVM). + * + * @author Alex Black + */ +public class LocalOptimizationRunner extends BaseOptimizationRunner { + + public static final int DEFAULT_MAX_CONCURRENT_TASKS = 1; + + private final int maxConcurrentTasks; + + private TaskCreator taskCreator; + private ListeningExecutorService executor; + @Setter + private long shutdownMaxWaitMS = 2L * 24 * 60 * 60 * 1000; + + public LocalOptimizationRunner(OptimizationConfiguration config){ + this(config, null); + } + + public LocalOptimizationRunner(OptimizationConfiguration config, TaskCreator taskCreator) { + this(DEFAULT_MAX_CONCURRENT_TASKS, config, taskCreator); + } + + public LocalOptimizationRunner(int maxConcurrentTasks, OptimizationConfiguration config){ + this(maxConcurrentTasks, config, null); + } + + public LocalOptimizationRunner(int maxConcurrentTasks, OptimizationConfiguration config, TaskCreator taskCreator) { + super(config); + if (maxConcurrentTasks <= 0) + throw new IllegalArgumentException("maxConcurrentTasks must be > 0 (got: " + maxConcurrentTasks + ")"); + this.maxConcurrentTasks = maxConcurrentTasks; + + if(taskCreator == null){ + Class psClass = config.getCandidateGenerator().getParameterSpace().getClass(); + taskCreator = TaskCreatorProvider.defaultTaskCreatorFor(psClass); + if(taskCreator == null){ + throw new IllegalStateException("No TaskCreator was provided and a default TaskCreator cannot be " + + "inferred for ParameterSpace class " + psClass.getName() + ". Please provide a TaskCreator " + + "via the LocalOptimizationRunner constructor"); + } + } + + this.taskCreator = taskCreator; + + ExecutorService exec = Executors.newFixedThreadPool(maxConcurrentTasks, new ThreadFactory() { + private AtomicLong counter = new AtomicLong(0); + + @Override + public Thread newThread(Runnable r) { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + t.setName("LocalCandidateExecutor-" + counter.getAndIncrement()); + return t; + } + }); + executor = MoreExecutors.listeningDecorator(exec); + + init(); + } + + @Override + protected int maxConcurrentTasks() { + return maxConcurrentTasks; + } + + @Override + protected ListenableFuture execute(Candidate candidate, DataProvider dataProvider, + ScoreFunction scoreFunction) { + return execute(Collections.singletonList(candidate), dataProvider, scoreFunction).get(0); + } + + @Override + protected List> execute(List candidates, DataProvider dataProvider, + ScoreFunction scoreFunction) { + List> list = new ArrayList<>(candidates.size()); + for (Candidate candidate : candidates) { + Callable task = + taskCreator.create(candidate, dataProvider, scoreFunction, statusListeners, this); + list.add(executor.submit(task)); + } + return list; + } + + @Override + protected ListenableFuture execute(Candidate candidate, Class dataSource, Properties dataSourceProperties, ScoreFunction scoreFunction) { + return execute(Collections.singletonList(candidate), dataSource, dataSourceProperties, scoreFunction).get(0); + } + + @Override + protected List> execute(List candidates, Class dataSource, Properties dataSourceProperties, ScoreFunction scoreFunction) { + List> list = new ArrayList<>(candidates.size()); + for (Candidate candidate : candidates) { + Callable task = taskCreator.create(candidate, dataSource, dataSourceProperties, scoreFunction, statusListeners, this); + list.add(executor.submit(task)); + } + return list; + } + + @Override + public void shutdown(boolean awaitTermination) { + if(awaitTermination){ + try { + executor.shutdown(); + executor.awaitTermination(shutdownMaxWaitMS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e){ + throw new RuntimeException(e); + } + } else { + executor.shutdownNow(); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/BaseStatusListener.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/BaseStatusListener.java new file mode 100644 index 000000000..aca25d95d --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/BaseStatusListener.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner.listener; + +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; + +/** + * BaseStatusListener: implements all methods of {@link StatusListener} as no-op. + * Users can extend this and override only the methods actually required + * + * @author Alex Black + */ +public abstract class BaseStatusListener implements StatusListener{ + @Override + public void onInitialization(IOptimizationRunner runner) { + //No op + } + + @Override + public void onShutdown(IOptimizationRunner runner) { + //No op + } + + @Override + public void onRunnerStatusChange(IOptimizationRunner runner) { + //No op + } + + @Override + public void onCandidateStatusChange(CandidateInfo candidateInfo, IOptimizationRunner runner, OptimizationResult result) { + //No op + } + + @Override + public void onCandidateIteration(CandidateInfo candidateInfo, Object candidate, int iteration) { + //No op + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/StatusChangeType.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/StatusChangeType.java new file mode 100644 index 000000000..d8e2f429b --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/StatusChangeType.java @@ -0,0 +1,26 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner.listener; + +/** + * Created by Alex on 20/07/2017. + */ +public enum StatusChangeType { + + CandidateCompleted, CandidateFailed, CandidateNewScheduled, CandidateNewBestScore + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/StatusListener.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/StatusListener.java new file mode 100644 index 000000000..fa5ba25a2 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/StatusListener.java @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner.listener; + +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; + +/** + * The status Listener interface is used to inspect/track the status of execution, both for individual candidates, + * and for the optimisation runner overall. + * + * @author Alex Black + */ +public interface StatusListener { + + /** Called when optimization runner starts execution */ + void onInitialization(IOptimizationRunner runner); + + /** Called when optimization runner terminates */ + void onShutdown(IOptimizationRunner runner); + + /** Called when any of the summary stats change, for the optimization runner: + * number scheduled, number completed, number failed, best score, etc. */ + void onRunnerStatusChange(IOptimizationRunner runner); + + /** + * Called when the status of the candidate is change. For example created, completed, failed. + * + * @param candidateInfo Candidate information + * @param runner Optimisation runner calling this method + * @param result Optimisation result. Maybe null. + */ + void onCandidateStatusChange(CandidateInfo candidateInfo, IOptimizationRunner runner, OptimizationResult result); + + /** + * This method may be called by tasks as they are executing. The intent of this method is to report partial results, + * such as different stages of learning, or scores/evaluations so far + * + * @param candidateInfo Candidate information + * @param candidate Current candidate value/configuration + * @param iteration Current iteration number + */ + void onCandidateIteration(CandidateInfo candidateInfo, Object candidate, int iteration); + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/impl/LoggingStatusListener.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/impl/LoggingStatusListener.java new file mode 100644 index 000000000..add0d4ff7 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/runner/listener/impl/LoggingStatusListener.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.runner.listener.impl; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; + +/** + * Created by Alex on 20/07/2017. + */ +@Slf4j +public class LoggingStatusListener implements StatusListener { + + + @Override + public void onInitialization(IOptimizationRunner runner) { + log.info("Optimization runner: initialized"); + } + + @Override + public void onShutdown(IOptimizationRunner runner) { + log.info("Optimization runner: shut down"); + } + + @Override + public void onRunnerStatusChange(IOptimizationRunner runner) { + log.info("Optimization runner: status change"); + } + + @Override + public void onCandidateStatusChange(CandidateInfo candidateInfo, IOptimizationRunner runner, + OptimizationResult result) { + log.info("Candidate status change: {}", candidateInfo); + } + + @Override + public void onCandidateIteration(CandidateInfo candidateInfo, Object candidate, int iteration) { + log.info("Candidate iteration #{} - {}", iteration, candidate); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/FixedValueDeserializer.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/FixedValueDeserializer.java new file mode 100644 index 000000000..7ca349878 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/FixedValueDeserializer.java @@ -0,0 +1,52 @@ +package org.deeplearning4j.arbiter.optimize.serde.jackson; + +import org.apache.commons.codec.binary.Base64; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; + +/** + * A custom deserializer to be used in conjunction with {@link FixedValueSerializer} + * @author Alex Black + */ +public class FixedValueDeserializer extends JsonDeserializer { + @Override + public FixedValue deserialize(JsonParser p, DeserializationContext deserializationContext) throws IOException, JsonProcessingException { + JsonNode node = p.getCodec().readTree(p); + String className = node.get("@valueclass").asText(); + Class c; + try { + c = Class.forName(className); + } catch (Exception e) { + throw new RuntimeException(e); + } + + if(node.has("value")){ + //Number, String, Enum + JsonNode valueNode = node.get("value"); + Object o = new ObjectMapper().treeToValue(valueNode, c); + return new FixedValue<>(o); + } else { + //Everything else + JsonNode valueNode = node.get("data"); + String data = valueNode.asText(); + + byte[] b = new Base64().decode(data); + ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(b)); + try { + Object o = ois.readObject(); + return new FixedValue<>(o); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/FixedValueSerializer.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/FixedValueSerializer.java new file mode 100644 index 000000000..80ff7d61d --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/FixedValueSerializer.java @@ -0,0 +1,52 @@ +package org.deeplearning4j.arbiter.optimize.serde.jackson; + +import org.apache.commons.net.util.Base64; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.type.WritableTypeId; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; + +import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; + + +/** + * A custom serializer to handle arbitrary object types + * Uses standard JSON where safe (number, string, enumerations) or Java object serialization (bytes -> base64) + * The latter is not an ideal approach, but Jackson doesn't support serialization/deserialization of arbitrary + * objects very well + * + * @author Alex Black + */ +public class FixedValueSerializer extends JsonSerializer { + @Override + public void serialize(FixedValue fixedValue, JsonGenerator j, SerializerProvider serializerProvider) throws IOException { + Object o = fixedValue.getValue(); + + j.writeStringField("@valueclass", o.getClass().getName()); + if(o instanceof Number || o instanceof String || o instanceof Enum){ + j.writeObjectField("value", o); + } else { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(o); + baos.close(); + byte[] b = baos.toByteArray(); + String base64 = new Base64().encodeToString(b); + j.writeStringField("data", base64); + } + } + + @Override + public void serializeWithType(FixedValue value, JsonGenerator gen, SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + WritableTypeId typeId = typeSer.typeId(value, START_OBJECT); + typeSer.writeTypePrefix(gen, typeId); + serialize(value, gen, serializers); + typeSer.writeTypeSuffix(gen, typeId); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/IntegerDistributionDeserializer.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/IntegerDistributionDeserializer.java new file mode 100644 index 000000000..6700e9753 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/IntegerDistributionDeserializer.java @@ -0,0 +1,59 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.serde.jackson; + +import org.apache.commons.math3.distribution.*; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; + +/** + * Custom Jackson deserializer for integer distributions + * + * @author Alex Black + */ +public class IntegerDistributionDeserializer extends JsonDeserializer { + + @Override + public IntegerDistribution deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + JsonNode node = p.getCodec().readTree(p); + String simpleName = node.get("distribution").asText(); + + switch (simpleName) { + case "BinomialDistribution": + return new BinomialDistribution(node.get("trials").asInt(), node.get("p").asDouble()); + case "GeometricDistribution": + return new GeometricDistribution(node.get("p").asDouble()); + case "HypergeometricDistribution": + return new HypergeometricDistribution(node.get("populationSize").asInt(), + node.get("numberOfSuccesses").asInt(), node.get("sampleSize").asInt()); + case "PascalDistribution": + return new PascalDistribution(node.get("r").asInt(), node.get("p").asDouble()); + case "PoissonDistribution": + return new PoissonDistribution(node.get("p").asDouble()); + case "UniformIntegerDistribution": + return new UniformIntegerDistribution(node.get("lower").asInt(), node.get("upper").asInt()); + case "ZipfDistribution": + return new ZipfDistribution(node.get("numElements").asInt(), node.get("exponent").asDouble()); + default: + throw new RuntimeException("Unknown or not supported distribution: " + simpleName); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/IntegerDistributionSerializer.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/IntegerDistributionSerializer.java new file mode 100644 index 000000000..4157df2f7 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/IntegerDistributionSerializer.java @@ -0,0 +1,74 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.serde.jackson; + +import org.apache.commons.math3.distribution.*; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +/** + * Custom Jackson serializer for integer distributions + * + * @author Alex Black + */ +public class IntegerDistributionSerializer extends JsonSerializer { + @Override + public void serialize(IntegerDistribution d, JsonGenerator j, SerializerProvider serializerProvider) + throws IOException { + Class c = d.getClass(); + String s = c.getSimpleName(); + + j.writeStartObject(); + j.writeStringField("distribution", s); + + if (c == BinomialDistribution.class) { + BinomialDistribution bd = (BinomialDistribution) d; + j.writeNumberField("trials", bd.getNumberOfTrials()); + j.writeNumberField("p", bd.getProbabilityOfSuccess()); + } else if (c == GeometricDistribution.class) { + GeometricDistribution gd = (GeometricDistribution) d; + j.writeNumberField("p", gd.getProbabilityOfSuccess()); + } else if (c == HypergeometricDistribution.class) { + HypergeometricDistribution hd = (HypergeometricDistribution) d; + j.writeNumberField("populationSize", hd.getPopulationSize()); + j.writeNumberField("numberOfSuccesses", hd.getNumberOfSuccesses()); + j.writeNumberField("sampleSize", hd.getSampleSize()); + } else if (c == PascalDistribution.class) { + PascalDistribution pd = (PascalDistribution) d; + j.writeNumberField("r", pd.getNumberOfSuccesses()); + j.writeNumberField("p", pd.getProbabilityOfSuccess()); + } else if (c == PoissonDistribution.class) { + PoissonDistribution pd = (PoissonDistribution) d; + j.writeNumberField("p", pd.getMean()); + } else if (c == UniformIntegerDistribution.class) { + UniformIntegerDistribution ud = (UniformIntegerDistribution) d; + j.writeNumberField("lower", ud.getSupportLowerBound()); + j.writeNumberField("upper", ud.getSupportUpperBound()); + } else if (c == ZipfDistribution.class) { + ZipfDistribution zd = (ZipfDistribution) d; + j.writeNumberField("numElements", zd.getNumberOfElements()); + j.writeNumberField("exponent", zd.getExponent()); + } else { + throw new UnsupportedOperationException("Unknown or not supported IntegerDistribution: " + c); + } + + j.writeEndObject(); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/JsonMapper.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/JsonMapper.java new file mode 100644 index 000000000..7ed1bfe45 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/JsonMapper.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.serde.jackson; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.joda.JodaModule; + +/** + * Created by Alex on 16/11/2016. + */ +public class JsonMapper { + + private static ObjectMapper mapper; + private static ObjectMapper yamlMapper; + + static { + mapper = new ObjectMapper(); + mapper.registerModule(new JodaModule()); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); + mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY); + mapper.setVisibility(PropertyAccessor.SETTER, JsonAutoDetect.Visibility.ANY); + yamlMapper = new ObjectMapper(new YAMLFactory()); + yamlMapper.registerModule(new JodaModule()); + yamlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + yamlMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + yamlMapper.enable(SerializationFeature.INDENT_OUTPUT); + yamlMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); + yamlMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + yamlMapper.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY); + } + + private JsonMapper() { + } + + + /** + * Return the yaml mapper + * + * @return + */ + public static ObjectMapper getYamlMapper() { + return yamlMapper; + } + + /** + * Return a json mapper + * + * @return + */ + public static ObjectMapper getMapper() { + return mapper; + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/RealDistributionDeserializer.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/RealDistributionDeserializer.java new file mode 100644 index 000000000..a30626560 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/RealDistributionDeserializer.java @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.serde.jackson; + +import org.apache.commons.math3.distribution.*; +import org.deeplearning4j.arbiter.optimize.distribution.LogUniformDistribution; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; + +/** + * Created by Alex on 14/02/2017. + */ +public class RealDistributionDeserializer extends JsonDeserializer { + + @Override + public RealDistribution deserialize(JsonParser p, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = p.getCodec().readTree(p); + String simpleName = node.get("distribution").asText(); + + switch (simpleName) { + case "BetaDistribution": + return new BetaDistribution(node.get("alpha").asDouble(), node.get("beta").asDouble()); + case "CauchyDistribution": + return new CauchyDistribution(node.get("median").asDouble(), node.get("scale").asDouble()); + case "ChiSquaredDistribution": + return new ChiSquaredDistribution(node.get("dof").asDouble()); + case "ExponentialDistribution": + return new ExponentialDistribution(node.get("mean").asDouble()); + case "FDistribution": + return new FDistribution(node.get("numeratorDof").asDouble(), node.get("denominatorDof").asDouble()); + case "GammaDistribution": + return new GammaDistribution(node.get("shape").asDouble(), node.get("scale").asDouble()); + case "LevyDistribution": + return new LevyDistribution(node.get("mu").asDouble(), node.get("c").asDouble()); + case "LogNormalDistribution": + return new LogNormalDistribution(node.get("scale").asDouble(), node.get("shape").asDouble()); + case "NormalDistribution": + return new NormalDistribution(node.get("mean").asDouble(), node.get("stdev").asDouble()); + case "ParetoDistribution": + return new ParetoDistribution(node.get("scale").asDouble(), node.get("shape").asDouble()); + case "TDistribution": + return new TDistribution(node.get("dof").asDouble()); + case "TriangularDistribution": + return new TriangularDistribution(node.get("a").asDouble(), node.get("b").asDouble(), + node.get("c").asDouble()); + case "UniformRealDistribution": + return new UniformRealDistribution(node.get("lower").asDouble(), node.get("upper").asDouble()); + case "WeibullDistribution": + return new WeibullDistribution(node.get("alpha").asDouble(), node.get("beta").asDouble()); + case "LogUniformDistribution": + return new LogUniformDistribution(node.get("min").asDouble(), node.get("max").asDouble()); + default: + throw new RuntimeException("Unknown or not supported distribution: " + simpleName); + } + + + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/RealDistributionSerializer.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/RealDistributionSerializer.java new file mode 100644 index 000000000..b108aad0a --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/RealDistributionSerializer.java @@ -0,0 +1,107 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.serde.jackson; + +import org.apache.commons.math3.distribution.*; +import org.deeplearning4j.arbiter.optimize.distribution.LogUniformDistribution; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +/** + * Custom JSON serializer for Apache commons RealDistribution instances. + * The custom serializer is set up to use the built-in c + */ +public class RealDistributionSerializer extends JsonSerializer { + + @Override + public void serialize(RealDistribution d, JsonGenerator j, SerializerProvider serializerProvider) + throws IOException { + Class c = d.getClass(); + String s = c.getSimpleName(); + + j.writeStartObject(); + j.writeStringField("distribution", s); + + + if (c == BetaDistribution.class) { + BetaDistribution bd = (BetaDistribution) d; + j.writeNumberField("alpha", bd.getAlpha()); + j.writeNumberField("beta", bd.getBeta()); + } else if (c == CauchyDistribution.class) { + CauchyDistribution cd = (CauchyDistribution) d; + j.writeNumberField("median", cd.getMedian()); + j.writeNumberField("scale", cd.getScale()); + } else if (c == ChiSquaredDistribution.class) { + ChiSquaredDistribution cd = (ChiSquaredDistribution) d; + j.writeNumberField("dof", cd.getDegreesOfFreedom()); + } else if (c == ExponentialDistribution.class) { + ExponentialDistribution ed = (ExponentialDistribution) d; + j.writeNumberField("mean", ed.getMean()); + } else if (c == FDistribution.class) { + FDistribution fd = (FDistribution) d; + j.writeNumberField("numeratorDof", fd.getNumeratorDegreesOfFreedom()); + j.writeNumberField("denominatorDof", fd.getDenominatorDegreesOfFreedom()); + } else if (c == GammaDistribution.class) { + GammaDistribution gd = (GammaDistribution) d; + j.writeNumberField("shape", gd.getShape()); + j.writeNumberField("scale", gd.getScale()); + } else if (c == LevyDistribution.class) { + LevyDistribution ld = (LevyDistribution) d; + j.writeNumberField("mu", ld.getLocation()); + j.writeNumberField("c", ld.getScale()); + } else if (c == LogNormalDistribution.class) { + LogNormalDistribution ln = (LogNormalDistribution) d; + j.writeNumberField("scale", ln.getScale()); + j.writeNumberField("shape", ln.getShape()); + } else if (c == NormalDistribution.class) { + NormalDistribution nd = (NormalDistribution) d; + j.writeNumberField("mean", nd.getMean()); + j.writeNumberField("stdev", nd.getStandardDeviation()); + } else if (c == ParetoDistribution.class) { + ParetoDistribution pd = (ParetoDistribution) d; + j.writeNumberField("scale", pd.getScale()); + j.writeNumberField("shape", pd.getShape()); + } else if (c == TDistribution.class) { + TDistribution td = (TDistribution) d; + j.writeNumberField("dof", td.getDegreesOfFreedom()); + } else if (c == TriangularDistribution.class) { + TriangularDistribution td = (TriangularDistribution) d; + j.writeNumberField("a", td.getSupportLowerBound()); + j.writeNumberField("b", td.getMode()); + j.writeNumberField("c", td.getSupportUpperBound()); + } else if (c == UniformRealDistribution.class) { + UniformRealDistribution u = (UniformRealDistribution) d; + j.writeNumberField("lower", u.getSupportLowerBound()); + j.writeNumberField("upper", u.getSupportUpperBound()); + } else if (c == WeibullDistribution.class) { + WeibullDistribution wb = (WeibullDistribution) d; + j.writeNumberField("alpha", wb.getShape()); + j.writeNumberField("beta", wb.getScale()); + } else if (c == LogUniformDistribution.class){ + LogUniformDistribution lud = (LogUniformDistribution) d; + j.writeNumberField("min", lud.getMin()); + j.writeNumberField("max", lud.getMax()); + } else { + throw new UnsupportedOperationException("Unknown or not supported RealDistribution: " + d.getClass()); + } + + j.writeEndObject(); + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/YamlMapper.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/YamlMapper.java new file mode 100644 index 000000000..b1aae22b2 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/optimize/serde/jackson/YamlMapper.java @@ -0,0 +1,52 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.serde.jackson; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.joda.JodaModule; + +/** + * Created by Alex on 16/11/2016. + */ +public class YamlMapper { + + private static final ObjectMapper mapper; + + static { + mapper = new ObjectMapper(new YAMLFactory()); + mapper.registerModule(new JodaModule()); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); + mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY); + } + + + private YamlMapper() {} + + public static ObjectMapper getMapper() { + return mapper; + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/ClassPathResource.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/ClassPathResource.java new file mode 100644 index 000000000..d22db15c8 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/ClassPathResource.java @@ -0,0 +1,233 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Simple utility class used to get access to files at the classpath, or packed into jar. + * Based on Spring ClassPathResource implementation + jar internals access implemented. + * + * @author raver119@gmail.com + */ +public class ClassPathResource { + + private String resourceName; + + private static Logger log = LoggerFactory.getLogger(ClassPathResource.class); + + /** + * Builds new ClassPathResource object + * + * @param resourceName String name of resource, to be retrieved + */ + public ClassPathResource(String resourceName) { + if (resourceName == null) + throw new IllegalStateException("Resource name can't be null"); + this.resourceName = resourceName; + } + + /** + * Returns URL of the requested resource + * + * @return URL of the resource, if it's available in current Jar + */ + private URL getUrl() { + ClassLoader loader = null; + try { + loader = Thread.currentThread().getContextClassLoader(); + } catch (Exception e) { + // do nothing + } + + if (loader == null) { + loader = ClassPathResource.class.getClassLoader(); + } + + URL url = loader.getResource(this.resourceName); + if (url == null) { + // try to check for mis-used starting slash + // TODO: see TODO below + if (this.resourceName.startsWith("/")) { + url = loader.getResource(this.resourceName.replaceFirst("[\\\\/]", "")); + if (url != null) + return url; + } else { + // try to add slash, to make clear it's not an issue + // TODO: change this mechanic to actual path purifier + url = loader.getResource("/" + this.resourceName); + if (url != null) + return url; + } + throw new IllegalStateException("Resource '" + this.resourceName + "' cannot be found."); + } + return url; + } + + /** + * Returns requested ClassPathResource as File object + *

+ * Please note: if this method called from compiled jar, temporary file will be created to provide File access + * + * @return File requested at constructor call + * @throws FileNotFoundException + */ + public File getFile() throws FileNotFoundException { + URL url = this.getUrl(); + + if (isJarURL(url)) { + /* + This is actually request for file, that's packed into jar. Probably the current one, but that doesn't matters. + */ + try { + url = extractActualUrl(url); + File file = File.createTempFile("canova_temp", "file"); + file.deleteOnExit(); + + ZipFile zipFile = new ZipFile(url.getFile()); + ZipEntry entry = zipFile.getEntry(this.resourceName); + if (entry == null) { + if (this.resourceName.startsWith("/")) { + entry = zipFile.getEntry(this.resourceName.replaceFirst("/", "")); + if (entry == null) { + throw new FileNotFoundException("Resource " + this.resourceName + " not found"); + } + } else + throw new FileNotFoundException("Resource " + this.resourceName + " not found"); + } + + long size = entry.getSize(); + + InputStream stream = zipFile.getInputStream(entry); + FileOutputStream outputStream = new FileOutputStream(file); + byte[] array = new byte[1024]; + int rd = 0; + long bytesRead = 0; + do { + rd = stream.read(array); + outputStream.write(array, 0, rd); + bytesRead += rd; + } while (bytesRead < size); + + outputStream.flush(); + outputStream.close(); + + stream.close(); + zipFile.close(); + + return file; + } catch (Exception e) { + throw new RuntimeException(e); + } + + } else { + /* + It's something in the actual underlying filesystem, so we can just go for it + */ + + try { + URI uri = new URI(url.toString().replaceAll(" ", "%20")); + return new File(uri.getSchemeSpecificPart()); + } catch (URISyntaxException e) { + return new File(url.getFile()); + } + } + } + + /** + * Checks, if proposed URL is packed into archive. + * + * @param url URL to be checked + * @return True, if URL is archive entry, False otherwise + */ + private boolean isJarURL(URL url) { + String protocol = url.getProtocol(); + return "jar".equals(protocol) || "zip".equals(protocol) || "wsjar".equals(protocol) + || "code-source".equals(protocol) && url.getPath().contains("!/"); + } + + /** + * Extracts parent Jar URL from original ClassPath entry URL. + * + * @param jarUrl Original URL of the resource + * @return URL of the Jar file, containing requested resource + * @throws MalformedURLException + */ + private URL extractActualUrl(URL jarUrl) throws MalformedURLException { + String urlFile = jarUrl.getFile(); + int separatorIndex = urlFile.indexOf("!/"); + if (separatorIndex != -1) { + String jarFile = urlFile.substring(0, separatorIndex); + + try { + return new URL(jarFile); + } catch (MalformedURLException var5) { + if (!jarFile.startsWith("/")) { + jarFile = "/" + jarFile; + } + + return new URL("file:" + jarFile); + } + } else { + return jarUrl; + } + } + + /** + * Returns requested ClassPathResource as InputStream object + * + * @return File requested at constructor call + * @throws FileNotFoundException + */ + public InputStream getInputStream() throws FileNotFoundException { + URL url = this.getUrl(); + if (isJarURL(url)) { + try { + url = extractActualUrl(url); + ZipFile zipFile = new ZipFile(url.getFile()); + ZipEntry entry = zipFile.getEntry(this.resourceName); + + if (entry == null) { + if (this.resourceName.startsWith("/")) { + entry = zipFile.getEntry(this.resourceName.replaceFirst("/", "")); + if (entry == null) { + throw new FileNotFoundException("Resource " + this.resourceName + " not found"); + } + } else + throw new FileNotFoundException("Resource " + this.resourceName + " not found"); + } + + InputStream stream = zipFile.getInputStream(entry); + return stream; + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + File srcFile = this.getFile(); + return new FileInputStream(srcFile); + } + } +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/CollectionUtils.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/CollectionUtils.java new file mode 100644 index 000000000..eb9275d82 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/CollectionUtils.java @@ -0,0 +1,49 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.util; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; + +public class CollectionUtils { + + /** + * Count the number of unique values in a collection + */ + public static int countUnique(Collection collection) { + HashSet set = new HashSet<>(collection); + return set.size(); + } + + /** + * Returns a list containing only unique values in a collection + */ + public static List getUnique(Collection collection) { + HashSet set = new HashSet<>(); + List out = new ArrayList<>(); + for (T t : collection) { + if (!set.contains(t)) { + out.add(t); + set.add(t); + } + } + return out; + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/LeafUtils.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/LeafUtils.java new file mode 100644 index 000000000..2a86dc48f --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/LeafUtils.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.util; + +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by Alex on 29/06/2017. + */ +public class LeafUtils { + + private LeafUtils() {} + + /** + * Returns a list of unique objects, not using the .equals() method, but rather using == + * + * @param allLeaves Leaf values to process + * @return A list of unique parameter space values + */ + public static List getUniqueObjects(List allLeaves) { + List unique = new ArrayList<>(); + for (ParameterSpace p : allLeaves) { + //This isn't especially efficient, but small number of parameters in general means it's fine + boolean found = false; + for (ParameterSpace q : unique) { + if (p == q) { + found = true; + } + } + if (!found) { + unique.add(p); + } + } + + return unique; + } + + /** + * Count the number of unique parameters in the specified leaf nodes + * + * @param allLeaves Leaf values to count the parameters fore + * @return Number of parameters for all unique objects + */ + public static int countUniqueParameters(List allLeaves) { + List unique = getUniqueObjects(allLeaves); + int count = 0; + for (ParameterSpace ps : unique) { + if (!ps.isLeaf()) { + throw new IllegalStateException("Method should only be used with leaf nodes"); + } + count += ps.numParameters(); + } + return count; + } + +} diff --git a/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/ObjectUtils.java b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/ObjectUtils.java new file mode 100644 index 000000000..9c3213430 --- /dev/null +++ b/arbiter/arbiter-core/src/main/java/org/deeplearning4j/arbiter/util/ObjectUtils.java @@ -0,0 +1,61 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.util; + +import java.util.Arrays; + +/** + * @author Alex Black + */ +public class ObjectUtils { + + private ObjectUtils() {} + + /** + * Get the string representation of the object. Arrays, including primitive arrays, are printed using + * Arrays.toString(...) methods. + * + * @param v Value to convert to a string + * @return String representation + */ + public static String valueToString(Object v) { + if (v.getClass().isArray()) { + if (v.getClass().getComponentType().isPrimitive()) { + Class c = v.getClass().getComponentType(); + if (c == int.class) { + return Arrays.toString((int[]) v); + } else if (c == double.class) { + return Arrays.toString((double[]) v); + } else if (c == float.class) { + return Arrays.toString((float[]) v); + } else if (c == long.class) { + return Arrays.toString((long[]) v); + } else if (c == byte.class) { + return Arrays.toString((byte[]) v); + } else if (c == short.class) { + return Arrays.toString((short[]) v); + } else { + return v.toString(); + } + } else { + return Arrays.toString((Object[]) v); + } + } else { + return v.toString(); + } + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/AssertTestsExtendBaseClass.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..cfb5e2556 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/AssertTestsExtendBaseClass.java @@ -0,0 +1,49 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.deeplearning4j.arbiter.optimize; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.deeplearning4j.arbiter.optimize"; + } + + @Override + protected Class getBaseClass() { + return BaseDL4JTest.class; + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/BraninFunction.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/BraninFunction.java new file mode 100644 index 000000000..4d507ee7d --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/BraninFunction.java @@ -0,0 +1,156 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.deeplearning4j.arbiter.optimize.api.*; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.CandidateStatus; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.Callable; + +public class BraninFunction { + public static class BraninSpace extends AbstractParameterSpace { + private int[] indices; + private ParameterSpace first = new ContinuousParameterSpace(-5, 10); + private ParameterSpace second = new ContinuousParameterSpace(0, 15); + + @Override + public BraninConfig getValue(double[] parameterValues) { + double f = first.getValue(parameterValues); + double s = second.getValue(parameterValues); + return new BraninConfig(f, s); //-5 to +10 and 0 to 15 + } + + @Override + public int numParameters() { + return 2; + } + + @Override + public List collectLeaves() { + List list = new ArrayList<>(); + list.addAll(first.collectLeaves()); + list.addAll(second.collectLeaves()); + return list; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + throw new UnsupportedOperationException(); + } + } + + @AllArgsConstructor + @Data + public static class BraninConfig implements Serializable { + private double x1; + private double x2; + } + + public static class BraninScoreFunction implements ScoreFunction { + private static final double a = 1.0; + private static final double b = 5.1 / (4.0 * Math.PI * Math.PI); + private static final double c = 5.0 / Math.PI; + private static final double r = 6.0; + private static final double s = 10.0; + private static final double t = 1.0 / (8.0 * Math.PI); + + @Override + public double score(Object m, DataProvider data, Map dataParameters) { + BraninConfig model = (BraninConfig) m; + double x1 = model.getX1(); + double x2 = model.getX2(); + + return a * Math.pow(x2 - b * x1 * x1 + c * x1 - r, 2.0) + s * (1 - t) * Math.cos(x1) + s; + } + + @Override + public double score(Object model, Class dataSource, Properties dataSourceProperties) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean minimize() { + return true; + } + + @Override + public List> getSupportedModelTypes() { + return Collections.>singletonList(BraninConfig.class); + } + + @Override + public List> getSupportedDataTypes() { + return Collections.>singletonList(Object.class); + } + } + + public static class BraninTaskCreator implements TaskCreator { + @Override + public Callable create(final Candidate c, DataProvider dataProvider, + final ScoreFunction scoreFunction, final List statusListeners, + IOptimizationRunner runner) { + + return new Callable() { + @Override + public OptimizationResult call() throws Exception { + + BraninConfig candidate = (BraninConfig) c.getValue(); + + double score = scoreFunction.score(candidate, null, (Map) null); +// System.out.println(candidate.getX1() + "\t" + candidate.getX2() + "\t" + score); + + Thread.sleep(20); + + if (statusListeners != null) { + for (StatusListener sl : statusListeners) { + sl.onCandidateIteration(null, null, 0); + } + } + + CandidateInfo ci = new CandidateInfo(-1, CandidateStatus.Complete, score, + System.currentTimeMillis(), null, null, null, null); + + return new OptimizationResult(c, score, c.getIndex(), null, ci, null); + } + }; + } + + @Override + public Callable create(Candidate candidate, Class dataSource, + Properties dataSourceProperties, ScoreFunction scoreFunction, + List statusListeners, IOptimizationRunner runner) { + throw new UnsupportedOperationException(); + } + } + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestGeneticSearch.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestGeneticSearch.java new file mode 100644 index 000000000..9410fa602 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestGeneticSearch.java @@ -0,0 +1,118 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.TerminationCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.generator.GeneticSearchCandidateGenerator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.exceptions.GeneticGenerationException; +import org.deeplearning4j.arbiter.optimize.generator.genetic.selection.SelectionOperator; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.CandidateStatus; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.impl.LoggingStatusListener; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestGeneticSearch extends BaseDL4JTest { + public class TestSelectionOperator extends SelectionOperator { + + @Override + public double[] buildNextGenes() { + throw new GeneticGenerationException("Forced exception to test exception handling."); + } + } + + public class TestTerminationCondition implements TerminationCondition { + + public boolean hasAFailedCandidate = false; + public int evalCount = 0; + + @Override + public void initialize(IOptimizationRunner optimizationRunner) {} + + @Override + public boolean terminate(IOptimizationRunner optimizationRunner) { + if (++evalCount == 50) { + // Generator did not handle GeneticGenerationException + return true; + } + + for (CandidateInfo candidateInfo : optimizationRunner.getCandidateStatus()) { + if (candidateInfo.getCandidateStatus() == CandidateStatus.Failed) { + hasAFailedCandidate = true; + return true; + } + } + + return false; + } + } + + @Test + public void GeneticSearchCandidateGenerator_getCandidate_ShouldGenerateCandidates() throws Exception { + + ScoreFunction scoreFunction = new BraninFunction.BraninScoreFunction(); + + //Define configuration: + CandidateGenerator candidateGenerator = + new GeneticSearchCandidateGenerator.Builder(new BraninFunction.BraninSpace(), scoreFunction) + .build(); + + TestTerminationCondition testTerminationCondition = new TestTerminationCondition(); + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).scoreFunction(scoreFunction) + .terminationConditions(new MaxCandidatesCondition(50), testTerminationCondition).build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new BraninFunction.BraninTaskCreator()); + + runner.addListeners(new LoggingStatusListener()); + runner.execute(); + + Assertions.assertFalse(testTerminationCondition.hasAFailedCandidate); + } + + @Test + public void GeneticSearchCandidateGenerator_getCandidate_GeneticExceptionShouldMarkCandidateAsFailed() { + + ScoreFunction scoreFunction = new BraninFunction.BraninScoreFunction(); + + //Define configuration: + CandidateGenerator candidateGenerator = + new GeneticSearchCandidateGenerator.Builder(new BraninFunction.BraninSpace(), scoreFunction) + .selectionOperator(new TestSelectionOperator()).build(); + + TestTerminationCondition testTerminationCondition = new TestTerminationCondition(); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).scoreFunction(scoreFunction) + .terminationConditions(testTerminationCondition).build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new BraninFunction.BraninTaskCreator()); + + runner.addListeners(new LoggingStatusListener()); + runner.execute(); + + Assertions.assertTrue(testTerminationCondition.hasAFailedCandidate); + } + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestGridSearch.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestGridSearch.java new file mode 100644 index 000000000..45a9aadf5 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestGridSearch.java @@ -0,0 +1,104 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.generator.GridSearchCandidateGenerator; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestGridSearch extends BaseDL4JTest { + + @Test + public void testIndexing() { + int[] nValues = {2, 3}; + int prod = 2 * 3; + double[][] expVals = new double[][] {{0.0, 0.0}, {1.0, 0.0}, {0.0, 0.5}, {1.0, 0.5}, {0.0, 1.0}, {1.0, 1.0}}; + for (int i = 0; i < prod; i++) { + double[] out = GridSearchCandidateGenerator.indexToValues(nValues, i, prod); + double[] exp = expVals[i]; + assertArrayEquals(exp, out, 1e-4); + } + } + + @Test + public void testGeneration() throws Exception { + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, new HashMap<>()); + + //Define configuration: + CandidateGenerator candidateGenerator = new GridSearchCandidateGenerator(new BraninFunction.BraninSpace(), 4, + GridSearchCandidateGenerator.Mode.Sequential, commands); + + //Check sequential: + double[] expValuesFirst = {-5, 0, 5, 10}; //Range: -5 to +10, with 4 values + double[] expValuesSecond = {0, 5, 10, 15}; //Range: 0 to +15, with 4 values + for (int i = 0; i < 4 * 4; i++) { + BraninFunction.BraninConfig conf = (BraninFunction.BraninConfig) candidateGenerator.getCandidate().getValue(); + double expF = expValuesFirst[i % 4]; //Changes most rapidly + double expS = expValuesSecond[i / 4]; + + double actF = conf.getX1(); + double actS = conf.getX2(); + + assertEquals(expF, actF, 1e-4); + assertEquals(expS, actS, 1e-4); + } + + //Check random order. specifically: check that all values are generated, in some order + double[][] orderedOutput = new double[16][2]; + for (int i = 0; i < expValuesFirst.length; i++) { + for (int j = 0; j < expValuesSecond.length; j++) { + orderedOutput[4 * j + i][0] = expValuesFirst[i]; + orderedOutput[4 * j + i][1] = expValuesSecond[j]; + } + } + + + candidateGenerator = new GridSearchCandidateGenerator(new BraninFunction.BraninSpace(), 4, + GridSearchCandidateGenerator.Mode.RandomOrder, commands); + boolean[] seen = new boolean[16]; + int seenCount = 0; + for (int i = 0; i < 4 * 4; i++) { + assertTrue(candidateGenerator.hasMoreCandidates()); + BraninFunction.BraninConfig config = (BraninFunction.BraninConfig) candidateGenerator.getCandidate().getValue(); + double x1 = config.getX1(); + double x2 = config.getX2(); + //Work out which of the values this is... + boolean matched = false; + for (int j = 0; j < 16; j++) { + if (Math.abs(orderedOutput[j][0] - x1) < 1e-5 && Math.abs(orderedOutput[j][1] - x2) < 1e-5) { + matched = true; + if (seen[j]) + fail("Same candidate generated multiple times"); + seen[j] = true; + seenCount++; + break; + } + } + assertTrue(matched, "Candidate " + x1 + ", " + x2 + " not found; invalid?"); + } + assertFalse(candidateGenerator.hasMoreCandidates()); + assertEquals(16, seenCount); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestJson.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestJson.java new file mode 100644 index 000000000..225894d6f --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestJson.java @@ -0,0 +1,122 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize; + +import org.apache.commons.math3.distribution.LogNormalDistribution; +import org.apache.commons.math3.distribution.NormalDistribution; +import org.apache.commons.math3.distribution.UniformIntegerDistribution; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.generator.GridSearchCandidateGenerator; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.parameter.BooleanSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.junit.jupiter.api.Test; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.joda.JodaModule; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Created by Alex on 02/02/2017. + */ +public class TestJson extends BaseDL4JTest { + + protected static ObjectMapper getObjectMapper(JsonFactory factory) { + ObjectMapper om = new ObjectMapper(factory); + om.registerModule(new JodaModule()); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + om.enable(SerializationFeature.INDENT_OUTPUT); + om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); + om.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + om.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY); + return om; + } + + private static ObjectMapper jsonMapper = getObjectMapper(new JsonFactory()); + private static ObjectMapper yamlMapper = getObjectMapper(new YAMLFactory()); + + + @Test + public void testParameterSpaceJson() throws Exception { + + List> l = new ArrayList<>(); + l.add(new FixedValue<>(1.0)); + l.add(new FixedValue<>(1)); + l.add(new FixedValue<>("string")); + l.add(new ContinuousParameterSpace(-1, 1)); + l.add(new ContinuousParameterSpace(new LogNormalDistribution(1, 1))); + l.add(new ContinuousParameterSpace(new NormalDistribution(2, 0.01))); + l.add(new DiscreteParameterSpace<>(1, 5, 7)); + l.add(new DiscreteParameterSpace<>("first", "second", "third")); + l.add(new IntegerParameterSpace(0, 10)); + l.add(new IntegerParameterSpace(new UniformIntegerDistribution(0, 50))); + l.add(new BooleanSpace()); + + for (ParameterSpace ps : l) { + String strJson = jsonMapper.writeValueAsString(ps); + String strYaml = yamlMapper.writeValueAsString(ps); + + ParameterSpace fromJson = jsonMapper.readValue(strJson, ParameterSpace.class); + ParameterSpace fromYaml = yamlMapper.readValue(strYaml, ParameterSpace.class); + + assertEquals(ps, fromJson); + assertEquals(ps, fromYaml); + } + } + + @Test + public void testCandidateGeneratorJson() throws Exception { + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, new HashMap<>()); + + List l = new ArrayList<>(); + l.add(new GridSearchCandidateGenerator(new DiscreteParameterSpace<>(0, 1, 2, 3, 4, 5), 10, + GridSearchCandidateGenerator.Mode.Sequential, commands)); + l.add(new GridSearchCandidateGenerator(new DiscreteParameterSpace<>(0, 1, 2, 3, 4, 5), 10, + GridSearchCandidateGenerator.Mode.RandomOrder, commands)); + l.add(new RandomSearchGenerator(new DiscreteParameterSpace<>(0, 1, 2, 3, 4, 5), commands)); + + for (CandidateGenerator cg : l) { + String strJson = jsonMapper.writeValueAsString(cg); + String strYaml = yamlMapper.writeValueAsString(cg); + + CandidateGenerator fromJson = jsonMapper.readValue(strJson, CandidateGenerator.class); + CandidateGenerator fromYaml = yamlMapper.readValue(strYaml, CandidateGenerator.class); + + assertEquals(cg, fromJson); + assertEquals(cg, fromYaml); + } + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestRandomSearch.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestRandomSearch.java new file mode 100644 index 000000000..db7702b76 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/TestRandomSearch.java @@ -0,0 +1,61 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.impl.LoggingStatusListener; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * + * Test random search on the Branin Function: + * http://www.sfu.ca/~ssurjano/branin.html + */ +public class TestRandomSearch extends BaseDL4JTest { + + @Test + public void test() throws Exception { + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, new HashMap<>()); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(new BraninFunction.BraninSpace(), commands); + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).scoreFunction(new BraninFunction.BraninScoreFunction()) + .terminationConditions(new MaxCandidatesCondition(50)).build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new BraninFunction.BraninTaskCreator()); + + runner.addListeners(new LoggingStatusListener()); + runner.execute(); + + +// System.out.println("----- Complete -----"); + } + + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/distribution/TestLogUniform.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/distribution/TestLogUniform.java new file mode 100644 index 000000000..e2a6044ce --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/distribution/TestLogUniform.java @@ -0,0 +1,70 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.distribution; + +import org.apache.commons.math3.distribution.RealDistribution; +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestLogUniform extends BaseDL4JTest { + + @Test + public void testSimple(){ + + double min = 0.5; + double max = 3; + + double logMin = Math.log(min); + double logMax = Math.log(max); + + RealDistribution rd = new LogUniformDistribution(min, max); + + for(double d = 0.1; d<= 3.5; d+= 0.1){ + double density = rd.density(d); + double cumulative = rd.cumulativeProbability(d); + double dExp; + double cumExp; + if(d < min){ + dExp = 0; + cumExp = 0; + } else if( d > max){ + dExp = 0; + cumExp = 1; + } else { + dExp = 1.0 / (d * (logMax-logMin)); + cumExp = (Math.log(d) - logMin) / (logMax - logMin); + } + + assertTrue(dExp >= 0); + assertTrue(cumExp >= 0); + assertTrue(cumExp <= 1.0); + assertEquals(dExp, density, 1e-5); + assertEquals(cumExp, cumulative, 1e-5); + } + + rd.reseedRandomGenerator(12345); + for( int i=0; i<100; i++ ){ + double d = rd.sample(); + assertTrue(d >= min); + assertTrue(d <= max); + } + } + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestCrossoverOperator.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestCrossoverOperator.java new file mode 100644 index 000000000..9297c3df7 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestCrossoverOperator.java @@ -0,0 +1,40 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverResult; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; + +public class TestCrossoverOperator extends CrossoverOperator { + + private final CrossoverResult[] results; + private int resultIdx = 0; + + public PopulationModel getPopulationModel() { + return populationModel; + } + + public TestCrossoverOperator(CrossoverResult[] results) { + this.results = results; + } + + @Override + public CrossoverResult crossover() { + return results[resultIdx++]; + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestMutationOperator.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestMutationOperator.java new file mode 100644 index 000000000..4718714d1 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestMutationOperator.java @@ -0,0 +1,34 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.mutation.MutationOperator; + +public class TestMutationOperator implements MutationOperator { + + private final boolean[] results; + private int resultIdx = 0; + + public TestMutationOperator(boolean[] results) { + this.results = results; + } + + @Override + public boolean mutate(double[] genes) { + return results[resultIdx++]; + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestParentSelection.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestParentSelection.java new file mode 100644 index 000000000..7f9c33b14 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestParentSelection.java @@ -0,0 +1,52 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.TwoParentSelection; + +import java.util.List; + +public class TestParentSelection extends TwoParentSelection { + + public boolean hasBeenInitialized; + + private final double[][] parents; + + public TestParentSelection(double[][] parents) { + this.parents = parents; + } + + public TestParentSelection() { + this(null); + } + + @Override + public void initializeInstance(List population) { + super.initializeInstance(population); + hasBeenInitialized = true; + } + + @Override + public double[][] selectParents() { + return parents; + } + + public List getPopulation() { + return population; + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestPopulationInitializer.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestPopulationInitializer.java new file mode 100644 index 000000000..926555f79 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestPopulationInitializer.java @@ -0,0 +1,30 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic; + +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; + +import java.util.ArrayList; +import java.util.List; + +public class TestPopulationInitializer implements PopulationInitializer { + @Override + public List getInitializedPopulation(int size) { + return new ArrayList<>(); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestRandomGenerator.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestRandomGenerator.java new file mode 100644 index 000000000..abeba96e8 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/TestRandomGenerator.java @@ -0,0 +1,88 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.commons.math3.random.RandomGenerator; + +public class TestRandomGenerator implements RandomGenerator { + private final int[] intRandomNumbers; + private int currentIntIdx = 0; + private final double[] doubleRandomNumbers; + private int currentDoubleIdx = 0; + + + public TestRandomGenerator(int[] intRandomNumbers, double[] doubleRandomNumbers) { + this.intRandomNumbers = intRandomNumbers; + this.doubleRandomNumbers = doubleRandomNumbers; + } + + @Override + public void setSeed(int i) { + + } + + @Override + public void setSeed(int[] ints) { + + } + + @Override + public void setSeed(long l) { + + } + + @Override + public void nextBytes(byte[] bytes) { + + } + + @Override + public int nextInt() { + return intRandomNumbers[currentIntIdx++]; + } + + @Override + public int nextInt(int i) { + return intRandomNumbers[currentIntIdx++]; + } + + @Override + public long nextLong() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public boolean nextBoolean() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public float nextFloat() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public double nextDouble() { + return doubleRandomNumbers[currentDoubleIdx++]; + } + + @Override + public double nextGaussian() { + throw new NotImplementedException("Not implemented"); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/ArithmeticCrossoverTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/ArithmeticCrossoverTests.java new file mode 100644 index 000000000..f234465b0 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/ArithmeticCrossoverTests.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.apache.commons.math3.random.RandomGenerator; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.ArithmeticCrossover; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverResult; +import org.deeplearning4j.arbiter.optimize.genetic.TestParentSelection; +import org.deeplearning4j.arbiter.optimize.genetic.TestRandomGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ArithmeticCrossoverTests extends BaseDL4JTest { + + @Test + public void ArithmeticCrossover_Crossover_OutsideCrossoverRate_ShouldReturnParent0() { + double[][] parents = new double[2][]; + parents[0] = new double[] {1.0}; + parents[1] = new double[] {2.0}; + + TestParentSelection parentSelection = new TestParentSelection(parents); + + RandomGenerator rng = new TestRandomGenerator(null, new double[] {1.0}); + + ArithmeticCrossover sut = + new ArithmeticCrossover.Builder().parentSelection(parentSelection).randomGenerator(rng).build(); + CrossoverResult result = sut.crossover(); + + Assertions.assertFalse(result.isModified()); + Assertions.assertEquals(1, result.getGenes().length); + Assertions.assertEquals(1.0, result.getGenes()[0], 0.001); + } + + @Test + public void ArithmeticCrossover_Crossover_WithinCrossoverRate_ShouldReturnLinearCombination() { + double[][] parents = new double[2][]; + parents[0] = new double[] {1.0}; + parents[1] = new double[] {2.0}; + + TestParentSelection parentSelection = new TestParentSelection(parents); + + RandomGenerator rng = new TestRandomGenerator(null, new double[] {0.1, 0.1}); + + ArithmeticCrossover sut = + new ArithmeticCrossover.Builder().parentSelection(parentSelection).randomGenerator(rng).build(); + CrossoverResult result = sut.crossover(); + + Assertions.assertTrue(result.isModified()); + Assertions.assertEquals(1, result.getGenes().length); + Assertions.assertEquals(1.9, result.getGenes()[0], 0.001); + } + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/CrossoverOperatorTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/CrossoverOperatorTests.java new file mode 100644 index 000000000..2cea0b608 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/CrossoverOperatorTests.java @@ -0,0 +1,43 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.genetic.TestCrossoverOperator; +import org.deeplearning4j.arbiter.optimize.genetic.TestPopulationInitializer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class CrossoverOperatorTests extends BaseDL4JTest { + + @Test + public void CrossoverOperator_initializeInstance_ShouldInitPopulationModel() throws IllegalAccessException { + TestCrossoverOperator sut = new TestCrossoverOperator(null); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel populationModel = + new PopulationModel.Builder().populationInitializer(populationInitializer).build(); + sut.initializeInstance(populationModel); + + Assertions.assertSame(populationModel, sut.getPopulationModel()); + + + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/CrossoverPointsGeneratorTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/CrossoverPointsGeneratorTests.java new file mode 100644 index 000000000..120fa8a28 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/CrossoverPointsGeneratorTests.java @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.apache.commons.math3.random.RandomGenerator; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.utils.CrossoverPointsGenerator; +import org.deeplearning4j.arbiter.optimize.genetic.TestRandomGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Deque; + +public class CrossoverPointsGeneratorTests extends BaseDL4JTest { + + @Test + public void CrossoverPointsGenerator_FixedNumberCrossovers() { + RandomGenerator rng = new TestRandomGenerator(new int[] {0}, null); + CrossoverPointsGenerator sut = new CrossoverPointsGenerator(10, 2, 2, rng); + + Deque result = sut.getCrossoverPoints(); + + Assertions.assertEquals(3, result.size()); + int a = result.pop(); + int b = result.pop(); + int c = result.pop(); + Assertions.assertTrue(a < b); + Assertions.assertTrue(b < c); + Assertions.assertEquals(Integer.MAX_VALUE, c); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/KPointCrossoverTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/KPointCrossoverTests.java new file mode 100644 index 000000000..64d56e5ac --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/KPointCrossoverTests.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.apache.commons.math3.random.RandomGenerator; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverResult; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.KPointCrossover; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.TwoParentSelection; +import org.deeplearning4j.arbiter.optimize.genetic.TestParentSelection; +import org.deeplearning4j.arbiter.optimize.genetic.TestRandomGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class KPointCrossoverTests extends BaseDL4JTest { + + @Test + public void KPointCrossover_BelowCrossoverRate_ShouldReturnParent0() { + RandomGenerator rng = new TestRandomGenerator(null, new double[] {1.0}); + + double[][] parents = new double[2][]; + parents[0] = new double[] {0.0}; + parents[1] = new double[] {1.0}; + TwoParentSelection parentSelection = new TestParentSelection(parents); + KPointCrossover sut = new KPointCrossover.Builder().randomGenerator(rng).crossoverRate(0.0) + .parentSelection(parentSelection).build(); + + CrossoverResult result = sut.crossover(); + + Assertions.assertFalse(result.isModified()); + Assertions.assertSame(parents[0], result.getGenes()); + } + + @Test + public void KPointCrossover_FixedNumberOfCrossovers() { + RandomGenerator rng = new TestRandomGenerator(new int[] {0, 1}, new double[] {0.0}); + + double[][] parents = new double[3][]; + parents[0] = new double[] {0.0, 0.0, 0.0, 0.0, 0.0}; + parents[1] = new double[] {1.0, 1.0, 1.0, 1.0, 1.0}; + parents[2] = new double[] {2.0, 2.0, 2.0, 2.0, 2.0}; + TwoParentSelection parentSelection = new TestParentSelection(parents); + KPointCrossover sut = new KPointCrossover.Builder().randomGenerator(rng).crossoverRate(1.0) + .parentSelection(parentSelection).numCrossovers(2).build(); + + CrossoverResult result = sut.crossover(); + + Assertions.assertTrue(result.isModified()); + for (double x : result.getGenes()) { + Assertions.assertTrue(x == 0.0 || x == 1.0); + } + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/ParentSelectionTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/ParentSelectionTests.java new file mode 100644 index 000000000..ca65e6ef0 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/ParentSelectionTests.java @@ -0,0 +1,39 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.genetic.TestParentSelection; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +public class ParentSelectionTests extends BaseDL4JTest { + + @Test + public void ParentSelection_InitializeInstance_ShouldInitPopulation() { + TestParentSelection sut = new TestParentSelection(); + + List population = new ArrayList<>(); + sut.initializeInstance(population); + + Assertions.assertSame(population, sut.getPopulation()); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/RandomTwoParentSelectionTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/RandomTwoParentSelectionTests.java new file mode 100644 index 000000000..214ee0181 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/RandomTwoParentSelectionTests.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.apache.commons.math3.random.RandomGenerator; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.RandomTwoParentSelection; +import org.deeplearning4j.arbiter.optimize.genetic.TestRandomGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +public class RandomTwoParentSelectionTests extends BaseDL4JTest { + @Test + public void RandomTwoParentSelection_ShouldReturnTwoDifferentParents() { + RandomGenerator rng = new TestRandomGenerator(new int[] {1, 1, 1, 0}, null); + RandomTwoParentSelection sut = new RandomTwoParentSelection(rng); + + List population = new ArrayList<>(); + population.add(new Chromosome(new double[] {1, 1, 1}, 1.0)); + population.add(new Chromosome(new double[] {2, 2, 2}, 2.0)); + population.add(new Chromosome(new double[] {3, 3, 3}, 3.0)); + sut.initializeInstance(population); + + double[][] result = sut.selectParents(); + + Assertions.assertSame(population.get(1).getGenes(), result[0]); + Assertions.assertSame(population.get(0).getGenes(), result[1]); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/SinglePointCrossoverTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/SinglePointCrossoverTests.java new file mode 100644 index 000000000..52fba0c59 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/SinglePointCrossoverTests.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.apache.commons.math3.random.RandomGenerator; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverResult; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.SinglePointCrossover; +import org.deeplearning4j.arbiter.optimize.genetic.TestParentSelection; +import org.deeplearning4j.arbiter.optimize.genetic.TestRandomGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SinglePointCrossoverTests extends BaseDL4JTest { + @Test + public void SinglePointCrossover_BelowCrossoverRate_ShouldReturnParent0() { + RandomGenerator rng = new TestRandomGenerator(null, new double[] {1.0}); + + double[][] parents = new double[2][]; + parents[0] = new double[] {1.0, 1.0, 1.0}; + parents[1] = new double[] {2.0, 2.0, 2.0}; + TestParentSelection parentSelection = new TestParentSelection(parents); + + SinglePointCrossover sut = new SinglePointCrossover.Builder().parentSelection(parentSelection) + .randomGenerator(rng).crossoverRate(0.0).build(); + + CrossoverResult result = sut.crossover(); + + Assertions.assertFalse(result.isModified()); + Assertions.assertSame(parents[0], result.getGenes()); + } + + @Test + public void SinglePointCrossover_ShouldReturnSingleSplit() { + RandomGenerator rng = new TestRandomGenerator(new int[] {2}, new double[] {0.1}); + + double[][] parents = new double[2][]; + parents[0] = new double[] {1.0, 1.0, 1.0}; + parents[1] = new double[] {2.0, 2.0, 2.0}; + TestParentSelection parentSelection = new TestParentSelection(parents); + + SinglePointCrossover sut = new SinglePointCrossover.Builder().parentSelection(parentSelection) + .randomGenerator(rng).crossoverRate(0.5).build(); + + CrossoverResult result = sut.crossover(); + + Assertions.assertTrue(result.isModified()); + Assertions.assertEquals(1.0, result.getGenes()[0], 0.0); + Assertions.assertEquals(1.0, result.getGenes()[1], 0.0); + Assertions.assertEquals(2.0, result.getGenes()[2], 0.0); + + } + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/TwoParentsCrossoverOperatorTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/TwoParentsCrossoverOperatorTests.java new file mode 100644 index 000000000..972d528ed --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/TwoParentsCrossoverOperatorTests.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.apache.commons.lang3.NotImplementedException; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverResult; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.TwoParentsCrossoverOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.parentselection.TwoParentSelection; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.genetic.TestParentSelection; +import org.deeplearning4j.arbiter.optimize.genetic.TestPopulationInitializer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TwoParentsCrossoverOperatorTests extends BaseDL4JTest { + + class TestTwoParentsCrossoverOperator extends TwoParentsCrossoverOperator { + + public TestTwoParentsCrossoverOperator(TwoParentSelection parentSelection) { + super(parentSelection); + } + + public TwoParentSelection getParentSelection() { + return parentSelection; + } + + @Override + public CrossoverResult crossover() { + throw new NotImplementedException("Not implemented"); + } + } + + @Test + public void TwoParentsCrossoverOperator_ctor_ShouldInitParentSelection() { + TestParentSelection parentSelection = new TestParentSelection(); + TestTwoParentsCrossoverOperator sut = new TestTwoParentsCrossoverOperator(parentSelection); + + Assertions.assertSame(parentSelection, sut.getParentSelection()); + } + + @Test + public void TwoParentsCrossoverOperator_initializeInstanceShouldInitializeParentSelection() { + TestParentSelection parentSelection = new TestParentSelection(); + TestTwoParentsCrossoverOperator sut = new TestTwoParentsCrossoverOperator(parentSelection); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + PopulationModel populationModel = + new PopulationModel.Builder().populationInitializer(populationInitializer).build(); + + sut.initializeInstance(populationModel); + + Assertions.assertTrue(parentSelection.hasBeenInitialized); + } + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/UniformCrossoverTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/UniformCrossoverTests.java new file mode 100644 index 000000000..5efff80b2 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/crossover/UniformCrossoverTests.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.crossover; + +import org.apache.commons.math3.random.RandomGenerator; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverResult; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.UniformCrossover; +import org.deeplearning4j.arbiter.optimize.genetic.TestParentSelection; +import org.deeplearning4j.arbiter.optimize.genetic.TestRandomGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class UniformCrossoverTests extends BaseDL4JTest { + + @Test + public void UniformCrossover_BelowCrossoverRate_ShouldReturnParent0() { + RandomGenerator rng = new TestRandomGenerator(null, new double[] {1.0}); + + double[][] parents = new double[2][]; + parents[0] = new double[] {1.0, 1.0, 1.0}; + parents[1] = new double[] {2.0, 2.0, 2.0}; + TestParentSelection parentSelection = new TestParentSelection(parents); + + UniformCrossover sut = new UniformCrossover.Builder().parentSelection(parentSelection).randomGenerator(rng) + .crossoverRate(0.0).build(); + + CrossoverResult result = sut.crossover(); + + Assertions.assertFalse(result.isModified()); + Assertions.assertSame(parents[0], result.getGenes()); + } + + @Test + public void UniformCrossover_ShouldReturnMixedParents() { + RandomGenerator rng = new TestRandomGenerator(null, new double[] {0.1, 0.1, 0.3, 0.2}); + + double[][] parents = new double[2][]; + parents[0] = new double[] {1.0, 1.0, 1.0}; + parents[1] = new double[] {2.0, 2.0, 2.0}; + TestParentSelection parentSelection = new TestParentSelection(parents); + + UniformCrossover sut = new UniformCrossover.Builder().parentSelection(parentSelection).randomGenerator(rng) + .crossoverRate(0.5).parentBiasFactor(0.3).build(); + + CrossoverResult result = sut.crossover(); + + Assertions.assertTrue(result.isModified()); + Assertions.assertEquals(1.0, result.getGenes()[0], 0.0); + Assertions.assertEquals(2.0, result.getGenes()[1], 0.0); + Assertions.assertEquals(1.0, result.getGenes()[2], 0.0); + } + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/culling/LeastFitCullOperatorTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/culling/LeastFitCullOperatorTests.java new file mode 100644 index 000000000..c5cde76d6 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/culling/LeastFitCullOperatorTests.java @@ -0,0 +1,62 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.culling; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.culling.LeastFitCullOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.genetic.TestPopulationInitializer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +public class LeastFitCullOperatorTests extends BaseDL4JTest { + + @Test + public void LeastFitCullingOperation_ShouldCullLastElements() { + LeastFitCullOperator sut = new LeastFitCullOperator(0.50); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel populationModel = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(10).build(); + sut.initializeInstance(populationModel); + + List originalChromosomes = new ArrayList<>(); + for (int i = 0; i < 10; ++i) { + originalChromosomes.add(new Chromosome(null, (double) i)); + } + + List chromosomes = populationModel.getPopulation(); + for (int i = 0; i < 10; ++i) { + chromosomes.add(originalChromosomes.get(i)); + } + + sut.cullPopulation(); + + Assertions.assertEquals(5, chromosomes.size()); + for (int i = 0; i < 5; ++i) { + Assertions.assertSame(originalChromosomes.get(i), chromosomes.get(i)); + } + } + + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/culling/RatioCullOperatorTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/culling/RatioCullOperatorTests.java new file mode 100644 index 000000000..ae09537f6 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/culling/RatioCullOperatorTests.java @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.culling; + +import org.apache.commons.lang3.NotImplementedException; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.culling.RatioCullOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.genetic.TestPopulationInitializer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class RatioCullOperatorTests extends BaseDL4JTest { + + class TestRatioCullOperator extends RatioCullOperator { + + public TestRatioCullOperator() { + super(); + } + + public TestRatioCullOperator(double ratio) { + super(ratio); + } + + public List getPopulation() { + return population; + } + + @Override + public void cullPopulation() { + throw new NotImplementedException("Not implemented"); + } + + public double getCullRatio() { + return cullRatio; + } + } + + @Test + public void RatioCullingOperation_ctorWithCullRatio_ShouldHaveParamRatio() { + TestRatioCullOperator sut = new TestRatioCullOperator(0.123); + + Assertions.assertEquals(0.123, sut.getCullRatio(), 0.0); + } + + @Test + public void RatioCullingOperation_initialize_shouldSetCulledSizeAndPopulation() throws IllegalAccessException { + TestRatioCullOperator sut = new TestRatioCullOperator(0.50); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel populationModel = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(10).build(); + sut.initializeInstance(populationModel); + + Assertions.assertSame(populationModel.getPopulation(), sut.getPopulation()); + Assertions.assertEquals(5, sut.getCulledSize()); + } + +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/mutation/RandomMutationOperatorTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/mutation/RandomMutationOperatorTests.java new file mode 100644 index 000000000..8b45ec9ad --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/mutation/RandomMutationOperatorTests.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.mutation; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.mutation.RandomMutationOperator; +import org.deeplearning4j.arbiter.optimize.genetic.TestRandomGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.Arrays; + +public class RandomMutationOperatorTests extends BaseDL4JTest { + @Test + public void RandomMutationOperator_DefaultBuild_ShouldNotBeNull() { + RandomMutationOperator sut = new RandomMutationOperator.Builder().build(); + Assertions.assertNotNull(sut); + } + + @Test + public void RandomMutationOperator_BuildWithMutationRate_ShouldUseSuppliedRate() throws Exception { + RandomMutationOperator sut = new RandomMutationOperator.Builder().mutationRate(0.123).build(); + + Field f = sut.getClass().getDeclaredField("mutationRate"); + f.setAccessible(true); + Double mutationRate = (Double) f.get(sut); + + Assertions.assertEquals(0.123, mutationRate, 0.0); + } + + @Test + public void RandomMutationOperator_BelowMutationRate_ShouldNotMutate() { + double[] randomNumbers = new double[] {0.1, 1.0, 1.0}; + + RandomMutationOperator sut = new RandomMutationOperator.Builder().mutationRate(0.1) + .randomGenerator(new TestRandomGenerator(null, randomNumbers)).build(); + + double[] genes = new double[] {-1.0, -1.0, -1.0}; + boolean hasMutated = sut.mutate(genes); + + Assertions.assertFalse(hasMutated); + Assertions.assertTrue(Arrays.equals(new double[] {-1.0, -1.0, -1.0}, genes)); + } + + @Test + public void RandomMutationOperator_AboveMutationRate_ShouldMutate() { + double[] randomNumbers = new double[] {0.099, 0.123, 1.0, 1.0}; + + RandomMutationOperator sut = new RandomMutationOperator.Builder().mutationRate(0.1) + .randomGenerator(new TestRandomGenerator(null, randomNumbers)).build(); + + double[] genes = new double[] {-1.0, -1.0, -1.0}; + boolean hasMutated = sut.mutate(genes); + + Assertions.assertTrue(hasMutated); + Assertions.assertTrue(Arrays.equals(new double[] {0.123, -1.0, -1.0}, genes)); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/population/PopulationModelTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/population/PopulationModelTests.java new file mode 100644 index 000000000..914a4be40 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/population/PopulationModelTests.java @@ -0,0 +1,195 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.population; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.Chromosome; +import org.deeplearning4j.arbiter.optimize.generator.genetic.culling.CullOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationListener; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.genetic.TestPopulationInitializer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class PopulationModelTests extends BaseDL4JTest { + + private class TestCullOperator implements CullOperator { + + private final int culledSize; + public boolean hasCulled = false; + + public TestCullOperator(int culledSize) { + this.culledSize = culledSize; + } + + @Override + public void initializeInstance(PopulationModel populationModel) { + + } + + @Override + public void cullPopulation() { + hasCulled = true; + } + + @Override + public int getCulledSize() { + return culledSize; + } + } + + private class TestPopulationListener implements PopulationListener { + + public List population; + + @Override + public void onChanged(List population) { + this.population = population; + } + } + + @Test + public void PopulationModel_IsReadyToBreed_NotReadyToBreed_ShouldReturnFalse() { + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel sut = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(5).cullOperator(new TestCullOperator(2)).build(); + + boolean result = sut.isReadyToBreed(); + + Assertions.assertFalse(result); + } + + @Test + public void PopulationModel_IsReadyToBreed_ReadyToBreed_ShouldReturnTrue() { + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel sut = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(5).cullOperator(new TestCullOperator(1)).build(); + + sut.getPopulation().add(null); + + boolean result = sut.isReadyToBreed(); + + Assertions.assertTrue(result); + } + + @Test + public void PopulationModel_Add_MaximizeScore_ShouldOrderDescendingPopulation() { + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel sut = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(5).cullOperator(new TestCullOperator(2)).build(); + + sut.initializeInstance(false); + + Chromosome[] chromosomes = new Chromosome[3]; + chromosomes[0] = new Chromosome(new double[0], 1.0); + chromosomes[1] = new Chromosome(new double[0], 100.0); + chromosomes[2] = new Chromosome(new double[0], 10.0); + sut.add(chromosomes[0]); + sut.add(chromosomes[1]); + sut.add(chromosomes[2]); + + Assertions.assertSame(chromosomes[1], sut.getPopulation().get(0)); + Assertions.assertSame(chromosomes[2], sut.getPopulation().get(1)); + Assertions.assertSame(chromosomes[0], sut.getPopulation().get(2)); + } + + @Test + public void PopulationModel_Add_MinimizeScore_ShouldOrderAscendingPopulation() { + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel sut = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(5).cullOperator(new TestCullOperator(2)).build(); + + sut.initializeInstance(true); + + Chromosome[] chromosomes = new Chromosome[3]; + chromosomes[0] = new Chromosome(new double[0], 100.0); + chromosomes[1] = new Chromosome(new double[0], 1.0); + chromosomes[2] = new Chromosome(new double[0], 10.0); + sut.add(chromosomes[0]); + sut.add(chromosomes[1]); + sut.add(chromosomes[2]); + + Assertions.assertSame(chromosomes[1], sut.getPopulation().get(0)); + Assertions.assertSame(chromosomes[2], sut.getPopulation().get(1)); + Assertions.assertSame(chromosomes[0], sut.getPopulation().get(2)); + } + + @Test + public void PopulationModel_Add_ShouldTriggerPopulationListeners() { + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel sut = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(5).cullOperator(new TestCullOperator(2)).build(); + + sut.initializeInstance(true); + + TestPopulationListener populationListener = new TestPopulationListener(); + sut.addListener(populationListener); + + sut.add(new Chromosome(new double[0], 100.0)); + + Assertions.assertSame(sut.getPopulation(), populationListener.population); + } + + @Test + public void PopulationModel_Add_BelowPopulationSize_ShouldNotCull() { + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + TestCullOperator cullOperator = new TestCullOperator(3); + + PopulationModel sut = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(5).cullOperator(cullOperator).build(); + + sut.initializeInstance(true); + + sut.add(new Chromosome(new double[0], 1.0)); + sut.add(new Chromosome(new double[0], 2.0)); + sut.add(new Chromosome(new double[0], 3.0)); + sut.add(new Chromosome(new double[0], 4.0)); + sut.add(new Chromosome(new double[0], 5.0)); + + Assertions.assertFalse(cullOperator.hasCulled); + } + + @Test + public void PopulationModel_Add_AbovePopulationSize_ShouldCull() { + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + TestCullOperator cullOperator = new TestCullOperator(3); + + PopulationModel sut = new PopulationModel.Builder().populationInitializer(populationInitializer) + .populationSize(5).cullOperator(cullOperator).build(); + + sut.initializeInstance(true); + + sut.add(new Chromosome(new double[0], 1.0)); + sut.add(new Chromosome(new double[0], 2.0)); + sut.add(new Chromosome(new double[0], 3.0)); + sut.add(new Chromosome(new double[0], 4.0)); + sut.add(new Chromosome(new double[0], 5.0)); + sut.add(new Chromosome(new double[0], 6.0)); + + Assertions.assertTrue(cullOperator.hasCulled); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/selection/GeneticSelectionOperatorTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/selection/GeneticSelectionOperatorTests.java new file mode 100644 index 000000000..4a0b2a498 --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/selection/GeneticSelectionOperatorTests.java @@ -0,0 +1,255 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.selection; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.commons.math3.random.RandomGenerator; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.ChromosomeFactory; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.crossover.CrossoverResult; +import org.deeplearning4j.arbiter.optimize.generator.genetic.culling.CullOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.exceptions.GeneticGenerationException; +import org.deeplearning4j.arbiter.optimize.generator.genetic.mutation.MutationOperator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.generator.genetic.selection.GeneticSelectionOperator; +import org.deeplearning4j.arbiter.optimize.genetic.TestCrossoverOperator; +import org.deeplearning4j.arbiter.optimize.genetic.TestMutationOperator; +import org.deeplearning4j.arbiter.optimize.genetic.TestPopulationInitializer; +import org.deeplearning4j.arbiter.optimize.genetic.TestRandomGenerator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +public class GeneticSelectionOperatorTests extends BaseDL4JTest { + + private class TestCullOperator implements CullOperator { + + private final int culledSize; + + public TestCullOperator(int culledSize) { + + this.culledSize = culledSize; + } + + @Override + public void initializeInstance(PopulationModel populationModel) { + + } + + @Override + public void cullPopulation() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public int getCulledSize() { + return culledSize; + } + } + + private class GeneticSelectionOperatorTestsMutationOperator implements MutationOperator { + + private boolean mutateResult; + + public GeneticSelectionOperatorTestsMutationOperator(boolean mutateResult) { + + this.mutateResult = mutateResult; + } + + @Override + public boolean mutate(double[] genes) { + return mutateResult; + } + } + + private class GeneticSelectionOperatorTestsCrossoverOperator extends CrossoverOperator { + + private CrossoverResult result; + + public GeneticSelectionOperatorTestsCrossoverOperator(CrossoverResult result) { + + this.result = result; + } + + @Override + public CrossoverResult crossover() { + return result; + } + } + + @Test + public void GeneticSelectionOperator_PopulationNotReadyToBreed_ShouldReturnRandomGenes() { + RandomGenerator rng = new TestRandomGenerator(null, new double[] {123.0}); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + TestCullOperator cullOperator = new TestCullOperator(1000); + PopulationModel populationModel = new PopulationModel.Builder().populationInitializer(populationInitializer) + .cullOperator(cullOperator).build(); + ChromosomeFactory chromosomeFactory = new ChromosomeFactory(); + chromosomeFactory.initializeInstance(1); + GeneticSelectionOperator sut = new GeneticSelectionOperator.Builder().randomGenerator(rng).build(); + sut.initializeInstance(populationModel, chromosomeFactory); + + double[] newGenes = sut.buildNextGenes(); + + Assertions.assertEquals(1, newGenes.length); + Assertions.assertEquals(123.0, newGenes[0], 0.0); + } + + @Test + public void GeneticSelectionOperator_NoModificationOnFirstTry() { + RandomGenerator rng = new TestRandomGenerator(null, new double[] {123.0}); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + TestCullOperator cullOperator = new TestCullOperator(-1); + + PopulationModel populationModel = new PopulationModel.Builder().populationInitializer(populationInitializer) + .cullOperator(cullOperator).build(); + + ChromosomeFactory chromosomeFactory = new ChromosomeFactory(); + chromosomeFactory.initializeInstance(1); + + CrossoverResult[] crossoverResults = new CrossoverResult[2]; + crossoverResults[0] = new CrossoverResult(false, new double[0]); + crossoverResults[1] = new CrossoverResult(true, new double[0]); + TestCrossoverOperator crossoverOperator = new TestCrossoverOperator(crossoverResults); + + boolean[] mutationResults = new boolean[] {false, false}; + TestMutationOperator mutationOperator = new TestMutationOperator(mutationResults); + + GeneticSelectionOperator sut = new GeneticSelectionOperator.Builder().randomGenerator(rng) + .crossoverOperator(crossoverOperator).mutationOperator(mutationOperator).build(); + sut.initializeInstance(populationModel, chromosomeFactory); + + double[] newGenes = sut.buildNextGenes(); + + Assertions.assertSame(crossoverResults[1].getGenes(), newGenes); + } + + @Test + public void GeneticSelectionOperator_MutationNoModificationOnFirstTry() { + RandomGenerator rng = new TestRandomGenerator(null, new double[] {123.0}); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + TestCullOperator cullOperator = new TestCullOperator(-1); + + PopulationModel populationModel = new PopulationModel.Builder().populationInitializer(populationInitializer) + .cullOperator(cullOperator).build(); + + ChromosomeFactory chromosomeFactory = new ChromosomeFactory(); + chromosomeFactory.initializeInstance(1); + + CrossoverResult[] crossoverResults = new CrossoverResult[3]; + crossoverResults[0] = new CrossoverResult(false, new double[0]); + crossoverResults[1] = new CrossoverResult(false, new double[0]); + crossoverResults[2] = new CrossoverResult(true, new double[0]); + TestCrossoverOperator crossoverOperator = new TestCrossoverOperator(crossoverResults); + + boolean[] mutationResults = new boolean[] {false, false, true}; + TestMutationOperator mutationOperator = new TestMutationOperator(mutationResults); + + GeneticSelectionOperator sut = new GeneticSelectionOperator.Builder().randomGenerator(rng) + .crossoverOperator(crossoverOperator).mutationOperator(mutationOperator).build(); + sut.initializeInstance(populationModel, chromosomeFactory); + + double[] newGenes = sut.buildNextGenes(); + + Assertions.assertSame(crossoverResults[2].getGenes(), newGenes); + } + + @Test + public void GeneticSelectionOperator_ShouldNotBuildDuplicates() { + RandomGenerator rng = new TestRandomGenerator(null, new double[] {123.0}); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + TestCullOperator cullOperator = new TestCullOperator(-1); + + PopulationModel populationModel = new PopulationModel.Builder().populationInitializer(populationInitializer) + .cullOperator(cullOperator).build(); + + ChromosomeFactory chromosomeFactory = new ChromosomeFactory(); + chromosomeFactory.initializeInstance(1); + + CrossoverResult[] crossoverResults = new CrossoverResult[3]; + crossoverResults[0] = new CrossoverResult(true, new double[] {1.0}); + crossoverResults[1] = new CrossoverResult(true, new double[] {1.0}); + crossoverResults[2] = new CrossoverResult(true, new double[] {2.0}); + TestCrossoverOperator crossoverOperator = new TestCrossoverOperator(crossoverResults); + + boolean[] mutationResults = new boolean[] {false, false, false}; + TestMutationOperator mutationOperator = new TestMutationOperator(mutationResults); + + GeneticSelectionOperator sut = new GeneticSelectionOperator.Builder().randomGenerator(rng) + .crossoverOperator(crossoverOperator).mutationOperator(mutationOperator).build(); + sut.initializeInstance(populationModel, chromosomeFactory); + + double[] newGenes = sut.buildNextGenes(); + assertArrayEquals(crossoverResults[0].getGenes(), newGenes, 1e-6); + + newGenes = sut.buildNextGenes(); + assertArrayEquals(crossoverResults[2].getGenes(), newGenes, 1e-6); + } + + @Test() + public void GeneticSelectionOperator_CrossoverAndMutationCantGenerateNew_ShouldThrow() { + Assertions.assertThrows(GeneticGenerationException.class, () -> { + TestCullOperator cullOperator = new TestCullOperator(-1); + + + PopulationModel populationModel = new PopulationModel.Builder().cullOperator(cullOperator).build(); + + MutationOperator mutationOperator = new GeneticSelectionOperatorTestsMutationOperator(false); + CrossoverOperator crossoverOperator = + new GeneticSelectionOperatorTestsCrossoverOperator(new CrossoverResult(false, null)); + + GeneticSelectionOperator sut = new GeneticSelectionOperator.Builder().crossoverOperator(crossoverOperator) + .mutationOperator(mutationOperator).build(); + sut.initializeInstance(populationModel, null); + + sut.buildNextGenes(); + }); + } + + @Test + public void GeneticSelectionOperator_CrossoverAndMutationAlwaysGenerateSame_ShouldThrow() { + Assertions.assertThrows(GeneticGenerationException.class, () -> { + TestCullOperator cullOperator = new TestCullOperator(-1); + + PopulationModel populationModel = new PopulationModel.Builder().cullOperator(cullOperator).build(); + + MutationOperator mutationOperator = new GeneticSelectionOperatorTestsMutationOperator(false); + CrossoverOperator crossoverOperator = new GeneticSelectionOperatorTestsCrossoverOperator( + new CrossoverResult(true, new double[]{1.0})); + + GeneticSelectionOperator sut = new GeneticSelectionOperator.Builder().crossoverOperator(crossoverOperator) + .mutationOperator(mutationOperator).build(); + sut.initializeInstance(populationModel, null); + + // This call is used to add the genes to the previousGenes collection + sut.buildNextGenes(); + + sut.buildNextGenes(); + }); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/selection/SelectionOperatorTests.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/selection/SelectionOperatorTests.java new file mode 100644 index 000000000..47bb3e37c --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/genetic/selection/SelectionOperatorTests.java @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.genetic.selection; + +import org.apache.commons.lang3.NotImplementedException; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.generator.genetic.ChromosomeFactory; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationInitializer; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.generator.genetic.selection.SelectionOperator; +import org.deeplearning4j.arbiter.optimize.genetic.TestPopulationInitializer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SelectionOperatorTests extends BaseDL4JTest { + private class TestSelectionOperator extends SelectionOperator { + + public PopulationModel getPopulationModel() { + return populationModel; + } + + public ChromosomeFactory getChromosomeFactory() { + return chromosomeFactory; + } + + @Override + public double[] buildNextGenes() { + throw new NotImplementedException("Not implemented"); + } + } + + @Test + public void SelectionOperator_InitializeInstance_ShouldInitializeFields() { + TestSelectionOperator sut = new TestSelectionOperator(); + + PopulationInitializer populationInitializer = new TestPopulationInitializer(); + + PopulationModel populationModel = + new PopulationModel.Builder().populationInitializer(populationInitializer).build(); + ChromosomeFactory chromosomeFactory = new ChromosomeFactory(); + sut.initializeInstance(populationModel, chromosomeFactory); + + Assertions.assertSame(populationModel, sut.getPopulationModel()); + Assertions.assertSame(chromosomeFactory, sut.getChromosomeFactory()); + } +} diff --git a/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/parameter/TestParameterSpaces.java b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/parameter/TestParameterSpaces.java new file mode 100644 index 000000000..5f477018c --- /dev/null +++ b/arbiter/arbiter-core/src/test/java/org/deeplearning4j/arbiter/optimize/parameter/TestParameterSpaces.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize.parameter; + +import org.apache.commons.math3.distribution.NormalDistribution; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestParameterSpaces extends BaseDL4JTest { + + + @Test + public void testContinuousParameterSpace() { + + ContinuousParameterSpace cps = new ContinuousParameterSpace(0, 1); + cps.setIndices(0); + + for (int i = 0; i < 10; i++) { + double d = i / 10.0; + assertEquals(d, cps.getValue(new double[]{d}), 0.0); + } + + cps = new ContinuousParameterSpace(10, 20); + cps.setIndices(0); + + for (int i = 0; i < 10; i++) { + double d = i / 10.0; + double exp = d * 10 + 10; + assertEquals(exp, cps.getValue(new double[]{d}), 0.0); + } + + + cps = new ContinuousParameterSpace(new NormalDistribution(0, 1)); + NormalDistribution nd = new NormalDistribution(0, 1); + cps.setIndices(0); + for (int i = 0; i < 11; i++) { + double d = i / 10.0; + assertEquals(nd.inverseCumulativeProbability(d), cps.getValue(new double[]{d}), 1e-4); + } + } + + @Test + public void testDiscreteParameterSpace() { + ParameterSpace dps = new DiscreteParameterSpace<>(0, 1, 2, 3, 4); + dps.setIndices(0); + + for (int i = 0; i < 5; i++) { + double d = i / 5.0 + 0.1; //Center + double dEdgeLower = i / 5.0 + 1e-8; //Edge case: just above split threshold + double dEdgeUpper = (i + 1) / 5.0 - 1e-8; //Edge case: just below split threshold + assertEquals(i, (int) dps.getValue(new double[]{d})); + assertEquals(i, (int) dps.getValue(new double[]{dEdgeLower})); + assertEquals(i, (int) dps.getValue(new double[]{dEdgeUpper})); + } + } + + @Test + public void testIntegerParameterSpace() { + ParameterSpace ips = new IntegerParameterSpace(0, 4); + ips.setIndices(0); + + for (int i = 0; i < 5; i++) { + double d = i / 5.0 + 0.1; //Center + double dEdgeLower = i / 5.0 + 1e-8; //Edge case: just above split threshold + double dEdgeUpper = (i + 1) / 5.0 - 1e-8; //Edge case: just below split threshold + assertEquals(i, (int) ips.getValue(new double[]{d})); + assertEquals(i, (int) ips.getValue(new double[]{dEdgeLower})); + assertEquals(i, (int) ips.getValue(new double[]{dEdgeUpper})); + } + } + + @Test + public void testBooleanSpace() { + ParameterSpace bSpace = new BooleanSpace(); + bSpace.setIndices(1); //randomly setting to non zero + + assertEquals(true, (boolean) bSpace.getValue(new double[]{0.0, 0.0})); + assertEquals(true, (boolean) bSpace.getValue(new double[]{0.1, 0.5})); + assertEquals(false, (boolean) bSpace.getValue(new double[]{0.2, 0.7})); + assertEquals(false, (boolean) bSpace.getValue(new double[]{0.3, 1.0})); + } + +} diff --git a/arbiter/arbiter-core/src/test/resources/logback.xml b/arbiter/arbiter-core/src/test/resources/logback.xml new file mode 100644 index 000000000..410bdaae9 --- /dev/null +++ b/arbiter/arbiter-core/src/test/resources/logback.xml @@ -0,0 +1,51 @@ + + + + + + logs/application.log + + %date - [%level] - from %logger in %thread + %n%message%n%xException%n + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/arbiter/arbiter-deeplearning4j/pom.xml b/arbiter/arbiter-deeplearning4j/pom.xml new file mode 100644 index 000000000..2f7e202a3 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/pom.xml @@ -0,0 +1,78 @@ + + + + + arbiter + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + arbiter-deeplearning4j + + + + + net.brutex.ai + arbiter-core + ${project.version} + + + + net.brutex.ai + deeplearning4j-core + ${project.version} + + + + ch.qos.logback + logback-classic + ${logback.version} + test + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.databind.version} + + + + com.google.code.gson + gson + ${gson.version} + + + + net.brutex.ai + deeplearning4j-common-tests + ${project.version} + test + + + + diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/BaseNetworkSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/BaseNetworkSpace.java new file mode 100644 index 000000000..69621330d --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/BaseNetworkSpace.java @@ -0,0 +1,615 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.adapter.ActivationParameterSpaceAdapter; +import org.deeplearning4j.arbiter.conf.dropout.DropoutSpace; +import org.deeplearning4j.arbiter.layers.LayerSpace; +import org.deeplearning4j.arbiter.optimize.api.AbstractParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import org.deeplearning4j.arbiter.optimize.serde.jackson.YamlMapper; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.api.layers.LayerConstraint; +import org.deeplearning4j.nn.conf.*; +import org.deeplearning4j.nn.conf.distribution.Distribution; +import org.deeplearning4j.nn.conf.dropout.Dropout; +import org.deeplearning4j.nn.conf.dropout.IDropout; +import org.deeplearning4j.nn.conf.stepfunctions.StepFunction; +import org.deeplearning4j.nn.conf.weightnoise.IWeightNoise; +import org.deeplearning4j.nn.weights.WeightInit; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.learning.config.IUpdater; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; + +import java.util.*; + +/** + * This is an abstract ParameterSpace for both MultiLayerNetworks (MultiLayerSpace) and ComputationGraph (ComputationGraphSpace) + *

+ * Functionality here should match {@link org.deeplearning4j.nn.conf.NeuralNetConfiguration.Builder} + * + * @param Type of network (MultiLayerNetwork or ComputationGraph) + * @author Alex Black + */ +@EqualsAndHashCode(callSuper = false) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +@Data +public abstract class BaseNetworkSpace extends AbstractParameterSpace { + + protected Long seed; + protected ParameterSpace optimizationAlgo; + protected ParameterSpace activationFunction; + protected ParameterSpace biasInit; + protected ParameterSpace weightInit; + protected ParameterSpace dist; + protected ParameterSpace maxNumLineSearchIterations; + protected ParameterSpace miniBatch; + protected ParameterSpace minimize; + protected ParameterSpace stepFunction; + protected ParameterSpace l1; + protected ParameterSpace l2; + protected ParameterSpace l1Bias; + protected ParameterSpace l2Bias; + protected ParameterSpace updater; + protected ParameterSpace biasUpdater; + protected ParameterSpace weightNoise; + private ParameterSpace dropout; + protected ParameterSpace gradientNormalization; + protected ParameterSpace gradientNormalizationThreshold; + protected ParameterSpace convolutionMode; + + protected List layerSpaces = new ArrayList<>(); + + //NeuralNetConfiguration.ListBuilder/MultiLayerConfiguration.Builder options: + protected ParameterSpace backpropType; + protected ParameterSpace tbpttFwdLength; + protected ParameterSpace tbpttBwdLength; + + protected ParameterSpace> allParamConstraints; + protected ParameterSpace> weightConstraints; + protected ParameterSpace> biasConstraints; + + protected int numEpochs = 1; + + + static { + JsonMapper.getMapper().registerSubtypes(ComputationGraphSpace.class, MultiLayerSpace.class); + YamlMapper.getMapper().registerSubtypes(ComputationGraphSpace.class, MultiLayerSpace.class); + } + + @SuppressWarnings("unchecked") + protected BaseNetworkSpace(Builder builder) { + this.seed = builder.seed; + this.optimizationAlgo = builder.optimizationAlgo; + this.activationFunction = builder.activationFunction; + this.biasInit = builder.biasInit; + this.weightInit = builder.weightInit; + this.dist = builder.dist; + this.maxNumLineSearchIterations = builder.maxNumLineSearchIterations; + this.miniBatch = builder.miniBatch; + this.minimize = builder.minimize; + this.stepFunction = builder.stepFunction; + this.l1 = builder.l1; + this.l2 = builder.l2; + this.l1Bias = builder.l1Bias; + this.l2Bias = builder.l2Bias; + this.updater = builder.updater; + this.biasUpdater = builder.biasUpdater; + this.weightNoise = builder.weightNoise; + this.dropout = builder.dropout; + this.gradientNormalization = builder.gradientNormalization; + this.gradientNormalizationThreshold = builder.gradientNormalizationThreshold; + this.convolutionMode = builder.convolutionMode; + this.allParamConstraints = builder.allParamConstraints; + this.weightConstraints = builder.weightConstraints; + this.biasConstraints = builder.biasConstraints; + + this.backpropType = builder.backpropType; + this.tbpttFwdLength = builder.tbpttFwdLength; + this.tbpttBwdLength = builder.tbpttBwdLength; + + this.numEpochs = builder.numEpochs; + } + + protected BaseNetworkSpace() { + //Default constructor for Jackson json/yaml serialization + } + + + protected NeuralNetConfiguration.Builder randomGlobalConf(double[] values) { + //Create MultiLayerConfiguration... + NeuralNetConfiguration.Builder builder = new NeuralNetConfiguration.Builder(); + if (seed != null) + builder.seed(seed); + if (optimizationAlgo != null) + builder.optimizationAlgo(optimizationAlgo.getValue(values)); + if (activationFunction != null) + builder.activation(activationFunction.getValue(values)); + if (biasInit != null) + builder.biasInit(biasInit.getValue(values)); + if (weightInit != null) + builder.weightInit(weightInit.getValue(values)); + if (dist != null) + builder.dist(dist.getValue(values)); + if (maxNumLineSearchIterations != null) + builder.maxNumLineSearchIterations(maxNumLineSearchIterations.getValue(values)); + if (miniBatch != null) + builder.miniBatch(miniBatch.getValue(values)); + if (minimize != null) + builder.minimize(minimize.getValue(values)); + if (stepFunction != null) + builder.stepFunction(stepFunction.getValue(values)); + if (l1 != null) + builder.l1(l1.getValue(values)); + if (l2 != null) + builder.l2(l2.getValue(values)); + if (l1Bias != null) + builder.l1Bias(l1Bias.getValue(values)); + if (l2Bias != null) + builder.l2Bias(l2Bias.getValue(values)); + if (updater != null) + builder.updater(updater.getValue(values)); + if (biasUpdater != null) + builder.biasUpdater(biasUpdater.getValue(values)); + if (weightNoise != null) + builder.weightNoise(weightNoise.getValue(values)); + if (dropout != null) + builder.dropOut(dropout.getValue(values)); + if (gradientNormalization != null) + builder.gradientNormalization(gradientNormalization.getValue(values)); + if (gradientNormalizationThreshold != null) + builder.gradientNormalizationThreshold(gradientNormalizationThreshold.getValue(values)); + if (convolutionMode != null) + builder.convolutionMode(convolutionMode.getValue(values)); + if (allParamConstraints != null){ + List c = allParamConstraints.getValue(values); + if(c != null){ + builder.constrainAllParameters(c.toArray(new LayerConstraint[c.size()])); + } + } + if (weightConstraints != null){ + List c = weightConstraints.getValue(values); + if(c != null){ + builder.constrainWeights(c.toArray(new LayerConstraint[c.size()])); + } + } + if (biasConstraints != null){ + List c = biasConstraints.getValue(values); + if(c != null){ + builder.constrainBias(c.toArray(new LayerConstraint[c.size()])); + } + } + + return builder; + } + + @Override + public List collectLeaves() { + Map global = getNestedSpaces(); + //Note: Results on previous line does NOT include the LayerSpaces, therefore we need to add these manually... + //This is because the type is a list, not a ParameterSpace + LinkedList stack = new LinkedList<>(); + stack.add(this); + + for (LayerConf layerConf : layerSpaces) { + LayerSpace ls = layerConf.getLayerSpace(); + stack.addAll(ls.collectLeaves()); + } + + List out = new ArrayList<>(); + while (!stack.isEmpty()) { + ParameterSpace next = stack.removeLast(); + if (next.isLeaf()) { + out.add(next); + } else { + Map m = next.getNestedSpaces(); + ParameterSpace[] arr = m.values().toArray(new ParameterSpace[m.size()]); + for (int i = arr.length - 1; i >= 0; i--) { + stack.add(arr[i]); + } + } + } + return out; + } + + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + throw new UnsupportedOperationException("Cannot set indices for non leaf"); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + for (Map.Entry e : getNestedSpaces().entrySet()) { + sb.append(e.getKey()).append(": ").append(e.getValue()).append("\n"); + } + + int i = 0; + for (LayerConf conf : layerSpaces) { + + sb.append("Layer config ").append(i++).append(": (Number layers:").append(conf.numLayers) + .append(", duplicate: ").append(conf.duplicateConfig).append("), ") + .append(conf.layerSpace.toString()).append("\n"); + } + + + return sb.toString(); + } + + @AllArgsConstructor + @Data + @NoArgsConstructor + public static class LayerConf { + protected LayerSpace layerSpace; + protected String layerName; + protected String[] inputs; + protected ParameterSpace numLayers; + protected boolean duplicateConfig; + protected InputPreProcessor preProcessor; + } + + @SuppressWarnings("unchecked") + protected abstract static class Builder> { + private Long seed; + private ParameterSpace optimizationAlgo; + private ParameterSpace activationFunction; + private ParameterSpace biasInit; + private ParameterSpace weightInit; + private ParameterSpace dist; + private ParameterSpace maxNumLineSearchIterations; + private ParameterSpace miniBatch; + private ParameterSpace minimize; + private ParameterSpace stepFunction; + private ParameterSpace l1; + private ParameterSpace l2; + private ParameterSpace l1Bias; + private ParameterSpace l2Bias; + private ParameterSpace updater; + private ParameterSpace biasUpdater; + private ParameterSpace weightNoise; + private ParameterSpace dropout; + private ParameterSpace gradientNormalization; + private ParameterSpace gradientNormalizationThreshold; + private ParameterSpace convolutionMode; + + private ParameterSpace> allParamConstraints; + private ParameterSpace> weightConstraints; + private ParameterSpace> biasConstraints; + + //NeuralNetConfiguration.ListBuilder/MultiLayerConfiguration.Builder options: + private ParameterSpace backpropType; + private ParameterSpace tbpttFwdLength; + private ParameterSpace tbpttBwdLength; + + //Early stopping configuration / (fixed) number of epochs: + private EarlyStoppingConfiguration earlyStoppingConfiguration; + private int numEpochs = 1; + + protected boolean validateOutputLayerConfig = true; + + public T seed(long seed) { + this.seed = seed; + return (T) this; + } + + public T optimizationAlgo(OptimizationAlgorithm optimizationAlgorithm) { + return optimizationAlgo(new FixedValue<>(optimizationAlgorithm)); + } + + public T optimizationAlgo(ParameterSpace parameterSpace) { + this.optimizationAlgo = parameterSpace; + return (T) this; + } + + + public T activation(Activation activationFunction) { + return activation(new FixedValue<>(activationFunction)); + } + + public T activation(ParameterSpace activationFunction) { + return activationFn(new ActivationParameterSpaceAdapter(activationFunction)); + } + + public T activationFn(ParameterSpace activationFunction) { + this.activationFunction = activationFunction; + return (T) this; + } + + public T biasInit(double biasInit){ + return biasInit(new FixedValue<>(biasInit)); + } + + public T biasInit(ParameterSpace biasInit){ + this.biasInit = biasInit; + return (T) this; + } + + public T weightInit(WeightInit weightInit) { + return weightInit(new FixedValue<>(weightInit)); + } + + public T weightInit(ParameterSpace weightInit) { + this.weightInit = weightInit; + return (T) this; + } + + public T dist(Distribution dist) { + return dist(new FixedValue<>(dist)); + } + + public T dist(ParameterSpace dist) { + this.dist = dist; + return (T) this; + } + + public T maxNumLineSearchIterations(int maxNumLineSearchIterations) { + return maxNumLineSearchIterations(new FixedValue<>(maxNumLineSearchIterations)); + } + + public T maxNumLineSearchIterations(ParameterSpace maxNumLineSearchIterations) { + this.maxNumLineSearchIterations = maxNumLineSearchIterations; + return (T) this; + } + + public T miniBatch(boolean minibatch) { + return miniBatch(new FixedValue<>(minibatch)); + } + + public T miniBatch(ParameterSpace miniBatch) { + this.miniBatch = miniBatch; + return (T) this; + } + + public T minimize(boolean minimize) { + return minimize(new FixedValue<>(minimize)); + } + + public T minimize(ParameterSpace minimize) { + this.minimize = minimize; + return (T) this; + } + + public T stepFunction(StepFunction stepFunction) { + return stepFunction(new FixedValue<>(stepFunction)); + } + + public T stepFunction(ParameterSpace stepFunction) { + this.stepFunction = stepFunction; + return (T) this; + } + + public T l1(double l1) { + return l1(new FixedValue<>(l1)); + } + + public T l1(ParameterSpace l1) { + this.l1 = l1; + return (T) this; + } + + public T l2(double l2) { + return l2(new FixedValue<>(l2)); + } + + public T l2(ParameterSpace l2) { + this.l2 = l2; + return (T) this; + } + public T l1Bias(double l1Bias) { + return l1Bias(new FixedValue<>(l1Bias)); + } + + public T l1Bias(ParameterSpace l1Bias) { + this.l1Bias = l1Bias; + return (T) this; + } + + public T l2Bias(double l2Bias) { + return l2Bias(new FixedValue<>(l2Bias)); + } + + public T l2Bias(ParameterSpace l2Bias) { + this.l2Bias = l2Bias; + return (T) this; + } + + public T updater(IUpdater updater){ + return updater(new FixedValue<>(updater)); + } + + public T updater(ParameterSpace updater) { + this.updater = updater; + return (T) this; + } + + public T biasUpdater(IUpdater biasUpdater){ + return biasUpdater(new FixedValue<>(biasUpdater)); + } + + public T biasUpdater(ParameterSpace biasUpdater){ + this.biasUpdater = biasUpdater; + return (T)this; + } + + public T weightNoise(IWeightNoise weightNoise){ + return weightNoise(new FixedValue<>(weightNoise)); + } + + public T weightNoise(ParameterSpace weightNoise){ + this.weightNoise = weightNoise; + return (T) this; + } + + public T dropOut(double dropout){ + return idropOut(new Dropout(dropout)); + } + + public T dropOut(ParameterSpace dropOut){ + return idropOut(new DropoutSpace(dropOut)); + } + + public T idropOut(IDropout idropOut){ + return idropOut(new FixedValue<>(idropOut)); + } + + public T idropOut(ParameterSpace idropOut){ + this.dropout = idropOut; + return (T) this; + } + + public T gradientNormalization(GradientNormalization gradientNormalization) { + return gradientNormalization(new FixedValue<>(gradientNormalization)); + } + + public T gradientNormalization(ParameterSpace gradientNormalization) { + this.gradientNormalization = gradientNormalization; + return (T) this; + } + + public T gradientNormalizationThreshold(double threshold) { + return gradientNormalizationThreshold(new FixedValue<>(threshold)); + } + + public T gradientNormalizationThreshold(ParameterSpace gradientNormalizationThreshold) { + this.gradientNormalizationThreshold = gradientNormalizationThreshold; + return (T) this; + } + + public T convolutionMode(ConvolutionMode convolutionMode) { + return convolutionMode(new FixedValue(convolutionMode)); + } + + public T convolutionMode(ParameterSpace convolutionMode) { + this.convolutionMode = convolutionMode; + return (T) this; + } + + public T backpropType(BackpropType backpropType) { + return backpropType(new FixedValue<>(backpropType)); + } + + public T backpropType(ParameterSpace backpropType) { + this.backpropType = backpropType; + return (T) this; + } + + public T tbpttFwdLength(int tbpttFwdLength) { + return tbpttFwdLength(new FixedValue<>(tbpttFwdLength)); + } + + public T tbpttFwdLength(ParameterSpace tbpttFwdLength) { + this.tbpttFwdLength = tbpttFwdLength; + return (T) this; + } + + public T tbpttBwdLength(int tbpttBwdLength) { + return tbpttBwdLength(new FixedValue<>(tbpttBwdLength)); + } + + public T tbpttBwdLength(ParameterSpace tbpttBwdLength) { + this.tbpttBwdLength = tbpttBwdLength; + return (T) this; + } + + public T constrainWeights(LayerConstraint... constraints){ + return constrainWeights(new FixedValue>(Arrays.asList(constraints))); + } + + public T constrainWeights(ParameterSpace> constraints){ + this.weightConstraints = constraints; + return (T) this; + } + + public T constrainBias(LayerConstraint... constraints){ + return constrainBias(new FixedValue>(Arrays.asList(constraints))); + } + + public T constrainBias(ParameterSpace> constraints){ + this.biasConstraints = constraints; + return (T) this; + } + + public T constrainAllParams(LayerConstraint... constraints){ + return constrainAllParams(new FixedValue>(Arrays.asList(constraints))); + } + + public T constrainAllParams(ParameterSpace> constraints){ + this.allParamConstraints = constraints; + return (T) this; + } + + public T validateOutputLayerConfig(boolean validate){ + this.validateOutputLayerConfig = validate; + return (T) this; + } + + /** + * Fixed number of training epochs. Default: 1 + * Note if both EarlyStoppingConfiguration and number of epochs is present, early stopping will be used in preference. + */ + public T numEpochs(int numEpochs) { + this.numEpochs = numEpochs; + return (T) this; + } + + + public abstract E build(); + } + + /** + * Return a json configuration of this configuration space. + * + * @return + */ + public String toJson() { + try { + return JsonMapper.getMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Return a yaml configuration of this configuration space. + * + * @return + */ + public String toYaml() { + try { + return YamlMapper.getMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/ComputationGraphSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/ComputationGraphSpace.java new file mode 100644 index 000000000..369300829 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/ComputationGraphSpace.java @@ -0,0 +1,316 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter; + +import lombok.*; +import org.deeplearning4j.arbiter.layers.LayerSpace; +import org.deeplearning4j.arbiter.layers.fixed.FixedLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.TaskCreatorProvider; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import org.deeplearning4j.arbiter.optimize.serde.jackson.YamlMapper; +import org.deeplearning4j.arbiter.task.ComputationGraphTaskCreator; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.WorkspaceMode; +import org.deeplearning4j.nn.conf.graph.GraphVertex; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.Layer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeName; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * ComputationGraphSpace: Defines the space of valid hyperparameters for a ComputationGraph. + * Note that this for fixed graph structures only + * + * @author Alex Black + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON ser/de +@Data +@EqualsAndHashCode(callSuper = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "@class") +@JsonTypeName("ComputationGraphSpace") +public class ComputationGraphSpace extends BaseNetworkSpace { + static { + TaskCreatorProvider.registerDefaultTaskCreatorClass(ComputationGraphSpace.class, ComputationGraphTaskCreator.class); + } + + @JsonProperty + protected List layerSpaces = new ArrayList<>(); + @JsonProperty + protected List vertices = new ArrayList<>(); + @JsonProperty + protected String[] networkInputs; + @JsonProperty + protected String[] networkOutputs; + @JsonProperty + protected ParameterSpace inputTypes; + @JsonProperty + protected int numParameters; + @JsonProperty + protected WorkspaceMode trainingWorkspaceMode; + @JsonProperty + protected WorkspaceMode inferenceWorkspaceMode; + @JsonProperty + protected boolean validateOutputLayerConfig = true; + + //Early stopping configuration / (fixed) number of epochs: + protected EarlyStoppingConfiguration earlyStoppingConfiguration; + + protected ComputationGraphSpace(Builder builder) { + super(builder); + + this.earlyStoppingConfiguration = builder.earlyStoppingConfiguration; + this.layerSpaces = builder.layerList; + this.vertices = builder.vertexList; + + this.networkInputs = builder.networkInputs; + this.networkOutputs = builder.networkOutputs; + this.inputTypes = builder.inputTypes; + this.trainingWorkspaceMode = builder.trainingWorkspaceMode; + this.inferenceWorkspaceMode = builder.inferenceWorkspaceMode; + this.validateOutputLayerConfig = builder.validateOutputLayerConfig; + + //Determine total number of parameters: + List list = LeafUtils.getUniqueObjects(collectLeaves()); + for (ParameterSpace ps : list) + numParameters += ps.numParameters(); + } + + + @Override + public GraphConfiguration getValue(double[] values) { + //Create ComputationGraphConfiguration... + NeuralNetConfiguration.Builder builder = randomGlobalConf(values); + + ComputationGraphConfiguration.GraphBuilder graphBuilder = builder.graphBuilder(); + graphBuilder.addInputs(this.networkInputs); + graphBuilder.setOutputs(this.networkOutputs); + if (inputTypes != null) + graphBuilder.setInputTypes(inputTypes.getValue(values)); + + //Build/add our layers and vertices: + for (LayerConf c : layerSpaces) { + org.deeplearning4j.nn.conf.layers.Layer l = c.layerSpace.getValue(values); + graphBuilder.addLayer(c.getLayerName(), l, c.getPreProcessor(), c.getInputs()); + } + for (VertexConf gv : vertices) { + graphBuilder.addVertex(gv.getVertexName(), gv.getGraphVertex(), gv.getInputs()); + } + + if (backpropType != null) + graphBuilder.backpropType(backpropType.getValue(values)); + if (tbpttFwdLength != null) + graphBuilder.tBPTTForwardLength(tbpttFwdLength.getValue(values)); + if (tbpttBwdLength != null) + graphBuilder.tBPTTBackwardLength(tbpttBwdLength.getValue(values)); + graphBuilder.validateOutputLayerConfig(validateOutputLayerConfig); + + ComputationGraphConfiguration configuration = graphBuilder.build(); + + if (trainingWorkspaceMode != null) + configuration.setTrainingWorkspaceMode(trainingWorkspaceMode); + if (inferenceWorkspaceMode != null) + configuration.setInferenceWorkspaceMode(inferenceWorkspaceMode); + + return new GraphConfiguration(configuration, earlyStoppingConfiguration, numEpochs); + } + + @Override + public int numParameters() { + return numParameters; + } + + @Override + public List collectLeaves() { + List list = super.collectLeaves(); + for (LayerConf lc : layerSpaces) { + list.addAll(lc.layerSpace.collectLeaves()); + } + if (inputTypes != null) + list.add(inputTypes); + return list; + } + + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + + for (LayerConf conf : layerSpaces) { + sb.append("Layer config: \"").append(conf.layerName).append("\", ").append(conf.layerSpace) + .append(", inputs: ").append(conf.inputs == null ? "[]" : Arrays.toString(conf.inputs)) + .append("\n"); + } + + for (VertexConf conf : vertices) { + sb.append("GraphVertex: \"").append(conf.vertexName).append("\", ").append(conf.graphVertex) + .append(", inputs: ").append(conf.inputs == null ? "[]" : Arrays.toString(conf.inputs)) + .append("\n"); + } + + if (earlyStoppingConfiguration != null) { + sb.append("Early stopping configuration:").append(earlyStoppingConfiguration.toString()).append("\n"); + } else { + sb.append("Training # epochs:").append(numEpochs).append("\n"); + } + + if (inputTypes != null) { + sb.append("Input types: ").append(inputTypes).append("\n"); + } + + return sb.toString(); + } + + @AllArgsConstructor + @Data + @NoArgsConstructor //For Jackson JSON + protected static class VertexConf { + protected GraphVertex graphVertex; + protected String vertexName; + protected String[] inputs; + } + + public static class Builder extends BaseNetworkSpace.Builder { + + protected List layerList = new ArrayList<>(); + protected List vertexList = new ArrayList<>(); + protected EarlyStoppingConfiguration earlyStoppingConfiguration; + protected String[] networkInputs; + protected String[] networkOutputs; + protected ParameterSpace inputTypes; + protected WorkspaceMode trainingWorkspaceMode; + protected WorkspaceMode inferenceWorkspaceMode; + + //Need: input types + //Early stopping configuration + //Graph nodes + + /** + * Early stopping configuration (optional). Note if both EarlyStoppingConfiguration and number of epochs is + * present, early stopping will be used in preference. + */ + public Builder earlyStoppingConfiguration( + EarlyStoppingConfiguration earlyStoppingConfiguration) { + this.earlyStoppingConfiguration = earlyStoppingConfiguration; + return this; + } + + public Builder layer(String layerName, LayerSpace layerSpace, String... layerInputs){ + return addLayer(layerName, layerSpace, layerInputs); + } + + public Builder layer(String layerName, LayerSpace layerSpace, InputPreProcessor preProcessor, + String... layerInputs) { + return addLayer(layerName, layerSpace, preProcessor, layerInputs); + } + + public Builder layer(String layerName, Layer layer, String... layerInputs){ + return layer(layerName, new FixedLayerSpace<>(layer), layerInputs); + } + + public Builder addLayer(String layerName, LayerSpace layerSpace, String... layerInputs) { + layerList.add(new LayerConf(layerSpace, layerName, layerInputs, new FixedValue<>(1), false, null)); + return this; + } + + public Builder addLayer(String layerName, LayerSpace layerSpace, InputPreProcessor preProcessor, + String... layerInputs){ + layerList.add(new LayerConf(layerSpace, layerName, layerInputs, new FixedValue<>(1), false, preProcessor)); + return this; + } + + public Builder addVertex(String vertexName, GraphVertex vertex, String... vertexInputs) { + vertexList.add(new VertexConf(vertex, vertexName, vertexInputs)); + return this; + } + + public Builder addInputs(String... networkInputs) { + this.networkInputs = networkInputs; + return this; + } + + public Builder setOutputs(String... networkOutputs) { + this.networkOutputs = networkOutputs; + return this; + } + + public Builder setInputTypes(InputType... inputTypes) { + return setInputTypes(new FixedValue(inputTypes)); + } + + public Builder setInputTypes(ParameterSpace inputTypes) { + this.inputTypes = inputTypes; + return this; + } + + public Builder trainingWorkspaceMode(WorkspaceMode workspaceMode){ + this.trainingWorkspaceMode = workspaceMode; + return this; + } + + public Builder inferenceWorkspaceMode(WorkspaceMode workspaceMode){ + this.inferenceWorkspaceMode = workspaceMode; + return this; + } + + @SuppressWarnings("unchecked") + public ComputationGraphSpace build() { + return new ComputationGraphSpace(this); + } + } + + + /** + * Instantiate a computation graph space from + * a raw json string + * @param json + * @return + */ + public static ComputationGraphSpace fromJson(String json) { + try { + return JsonMapper.getMapper().readValue(json, ComputationGraphSpace.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Instantiate a computation graph space + * from a raw yaml string + * @param yaml + * @return + */ + public static ComputationGraphSpace fromYaml(String yaml) { + try { + return YamlMapper.getMapper().readValue(yaml, ComputationGraphSpace.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/DL4JConfiguration.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/DL4JConfiguration.java new file mode 100644 index 000000000..15eb7e3ba --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/DL4JConfiguration.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import org.deeplearning4j.arbiter.optimize.serde.jackson.YamlMapper; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.io.Serializable; + +/** + * DL4JConfiguration: simple configuration method that contains the following:
+ * - MultiLayerConfiguration
+ * - Early stopping settings, OR number of epochs
+ * Note: if early stopping configuration is absent, a fixed number of epochs (default: 1) will be used. + * If both early stopping and number of epochs is present: early stopping will be used. + */ +@AllArgsConstructor +@Data +public class DL4JConfiguration implements Serializable { + @JsonSerialize + private MultiLayerConfiguration multiLayerConfiguration; + @JsonSerialize + private EarlyStoppingConfiguration earlyStoppingConfiguration; + @JsonSerialize + private Integer numEpochs; + + + /** + * Yaml mapping + * @return + */ + public String toYaml() { + try { + return YamlMapper.getMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Json mapping + * @return + */ + public String toJson() { + try { + return JsonMapper.getMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/GraphConfiguration.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/GraphConfiguration.java new file mode 100644 index 000000000..4cb6cf685 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/GraphConfiguration.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter; + + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import org.deeplearning4j.arbiter.optimize.serde.jackson.YamlMapper; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import com.fasterxml.jackson.core.JsonProcessingException; + +import java.io.Serializable; + +/** + * Analogous to {@link DL4JConfiguration}, GraphConfiguration includes a configuration for ComputationGraphs, as well + * as early stopping (or, optionally numEpochs) fields. + */ +@AllArgsConstructor +@Data +public class GraphConfiguration implements Serializable { + private ComputationGraphConfiguration configuration; + private EarlyStoppingConfiguration earlyStoppingConfiguration; + private Integer numEpochs; + + + + /** + * Yaml mapping + * @return + */ + public String toYaml() { + try { + return YamlMapper.getMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Json mapping + * @return + */ + public String toJson() { + try { + return JsonMapper.getMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/MultiLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/MultiLayerSpace.java new file mode 100644 index 000000000..beeb0420b --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/MultiLayerSpace.java @@ -0,0 +1,320 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.layers.LayerSpace; +import org.deeplearning4j.arbiter.layers.fixed.FixedLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.TaskCreatorProvider; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import org.deeplearning4j.arbiter.optimize.serde.jackson.YamlMapper; +import org.deeplearning4j.arbiter.task.MultiLayerNetworkTaskCreator; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.WorkspaceMode; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.Layer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class MultiLayerSpace extends BaseNetworkSpace { + + static { + TaskCreatorProvider.registerDefaultTaskCreatorClass(MultiLayerSpace.class, MultiLayerNetworkTaskCreator.class); + } + + @JsonProperty + protected ParameterSpace inputType; + @JsonProperty + protected ParameterSpace> inputPreProcessors; + + //Early stopping configuration / (fixed) number of epochs: + @JsonProperty + protected EarlyStoppingConfiguration earlyStoppingConfiguration; + @JsonProperty + protected int numParameters; + @JsonProperty + protected WorkspaceMode trainingWorkspaceMode; + @JsonProperty + protected WorkspaceMode inferenceWorkspaceMode; + @JsonProperty + protected boolean validateOutputLayerConfig = true; + + + protected MultiLayerSpace(Builder builder) { + super(builder); + this.inputType = builder.inputType; + this.inputPreProcessors = builder.inputPreProcessors; + + this.earlyStoppingConfiguration = builder.earlyStoppingConfiguration; + + this.layerSpaces = builder.layerSpaces; + + //Determine total number of parameters: + //Collect the leaves, and make sure they are unique. + //Note that the *object instances* must be unique - and consequently we don't want to use .equals(), as + // this would incorrectly filter out equal range parameter spaces + List allLeaves = collectLeaves(); + List list = LeafUtils.getUniqueObjects(allLeaves); + + for (ParameterSpace ps : list) { + int n = ps.numParameters(); + numParameters += ps.numParameters(); + } + + this.trainingWorkspaceMode = builder.trainingWorkspaceMode; + this.inferenceWorkspaceMode = builder.inferenceWorkspaceMode; + this.validateOutputLayerConfig = builder.validateOutputLayerConfig; + } + + protected MultiLayerSpace() { + //Default constructor for Jackson json/yaml serialization + } + + @Override + public DL4JConfiguration getValue(double[] values) { + //First: create layer configs + List layers = new ArrayList<>(); + for (LayerConf c : layerSpaces) { + int n = c.numLayers.getValue(values); + if (c.duplicateConfig) { + //Generate N identical configs + org.deeplearning4j.nn.conf.layers.Layer l = c.layerSpace.getValue(values); + for (int i = 0; i < n; i++) { + layers.add(l.clone()); + } + } else { + throw new UnsupportedOperationException("Not yet implemented"); + } + } + + //Create MultiLayerConfiguration... + NeuralNetConfiguration.Builder builder = randomGlobalConf(values); + + NeuralNetConfiguration.ListBuilder listBuilder = builder.list(); + for (int i = 0; i < layers.size(); i++) { + listBuilder.layer(i, layers.get(i)); + } + + if (backpropType != null) + listBuilder.backpropType(backpropType.getValue(values)); + if (tbpttFwdLength != null) + listBuilder.tBPTTForwardLength(tbpttFwdLength.getValue(values)); + if (tbpttBwdLength != null) + listBuilder.tBPTTBackwardLength(tbpttBwdLength.getValue(values)); + if (inputType != null) + listBuilder.setInputType(inputType.getValue(values)); + if (inputPreProcessors != null) + listBuilder.setInputPreProcessors(inputPreProcessors.getValue(values)); + listBuilder.validateOutputLayerConfig(validateOutputLayerConfig); + + MultiLayerConfiguration configuration = listBuilder.build(); + + if (trainingWorkspaceMode != null) + configuration.setTrainingWorkspaceMode(trainingWorkspaceMode); + if (inferenceWorkspaceMode != null) + configuration.setInferenceWorkspaceMode(inferenceWorkspaceMode); + + + return new DL4JConfiguration(configuration, earlyStoppingConfiguration, numEpochs); + } + + @Override + public int numParameters() { + return numParameters; + } + + @Override + public List collectLeaves() { + List list = super.collectLeaves(); + for (LayerConf lc : layerSpaces) { + list.addAll(lc.numLayers.collectLeaves()); + list.addAll(lc.layerSpace.collectLeaves()); + } + if (inputType != null) + list.addAll(inputType.collectLeaves()); + if (inputPreProcessors != null) + list.addAll(inputPreProcessors.collectLeaves()); + return list; + } + + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + + int i = 0; + for (LayerConf conf : layerSpaces) { + + sb.append("Layer config ").append(i++).append(": (Number layers:").append(conf.numLayers) + .append(", duplicate: ").append(conf.duplicateConfig).append("), ") + .append(conf.layerSpace.toString()).append("\n"); + } + + if (inputType != null) + sb.append("inputType: ").append(inputType).append("\n"); + if (inputPreProcessors != null) + sb.append("inputPreProcessors: ").append(inputPreProcessors).append("\n"); + + if (earlyStoppingConfiguration != null) { + sb.append("Early stopping configuration:").append(earlyStoppingConfiguration.toString()).append("\n"); + } else { + sb.append("Training # epochs:").append(numEpochs).append("\n"); + } + + return sb.toString(); + } + + public LayerSpace getLayerSpace(int layerNumber) { + return layerSpaces.get(layerNumber).getLayerSpace(); + } + + public static class Builder extends BaseNetworkSpace.Builder { + protected List layerSpaces = new ArrayList<>(); + protected ParameterSpace inputType; + protected ParameterSpace> inputPreProcessors; + protected WorkspaceMode trainingWorkspaceMode; + protected WorkspaceMode inferenceWorkspaceMode; + + //Early stopping configuration + protected EarlyStoppingConfiguration earlyStoppingConfiguration; + + + + public Builder setInputType(InputType inputType) { + return setInputType(new FixedValue<>(inputType)); + } + + public Builder setInputType(ParameterSpace inputType) { + this.inputType = inputType; + return this; + } + + public Builder layer(Layer layer){ + return layer(new FixedLayerSpace<>(layer)); + } + + public Builder layer(LayerSpace layerSpace) { + return layer(layerSpace, new FixedValue<>(1)); + } + + public Builder layer(LayerSpace layerSpace, ParameterSpace numLayersDistribution) { + return addLayer(layerSpace, numLayersDistribution); + } + + + public Builder addLayer(LayerSpace layerSpace) { + return addLayer(layerSpace, new FixedValue<>(1)); + } + + /** + * duplicateConfig not supported. Will always be true + * @param layerSpace + * @param numLayersDistribution + * @param duplicateConfig + * @return + */ + @Deprecated + public Builder addLayer(LayerSpace layerSpace, ParameterSpace numLayersDistribution, boolean duplicateConfig) { + if (!duplicateConfig) throw new IllegalArgumentException("Duplicate Config false not supported"); + String layerName = "layer_" + layerSpaces.size(); + duplicateConfig = true; //hard coded to always duplicate layers + layerSpaces.add(new LayerConf(layerSpace, layerName, null, numLayersDistribution, duplicateConfig, null)); + return this; + } + + /** + * @param layerSpace + * @param numLayersDistribution Distribution for number of layers to generate + */ + public Builder addLayer(LayerSpace layerSpace, ParameterSpace numLayersDistribution) { + String layerName = "layer_" + layerSpaces.size(); + boolean duplicateConfig = true; //hard coded to always duplicate layers + layerSpaces.add(new LayerConf(layerSpace, layerName, null, numLayersDistribution, duplicateConfig, null)); + return this; + } + + /** + * Early stopping configuration (optional). Note if both EarlyStoppingConfiguration and number of epochs is + * present, early stopping will be used in preference. + */ + public Builder earlyStoppingConfiguration( + EarlyStoppingConfiguration earlyStoppingConfiguration) { + this.earlyStoppingConfiguration = earlyStoppingConfiguration; + return this; + } + + /** + * @param inputPreProcessors Input preprocessors to set for the model + */ + public Builder setInputPreProcessors(Map inputPreProcessors) { + return setInputPreProcessors(new FixedValue<>(inputPreProcessors)); + } + + /** + * @param inputPreProcessors Input preprocessors to set for the model + */ + public Builder setInputPreProcessors(ParameterSpace> inputPreProcessors) { + this.inputPreProcessors = inputPreProcessors; + return this; + } + + public Builder trainingWorkspaceMode(WorkspaceMode workspaceMode){ + this.trainingWorkspaceMode = workspaceMode; + return this; + } + + public Builder inferenceWorkspaceMode(WorkspaceMode workspaceMode){ + this.inferenceWorkspaceMode = workspaceMode; + return this; + } + + @SuppressWarnings("unchecked") + public MultiLayerSpace build() { + return new MultiLayerSpace(this); + } + } + + public static MultiLayerSpace fromJson(String json) { + try { + return JsonMapper.getMapper().readValue(json, MultiLayerSpace.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static MultiLayerSpace fromYaml(String yaml) { + try { + return YamlMapper.getMapper().readValue(yaml, MultiLayerSpace.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/adapter/ActivationParameterSpaceAdapter.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/adapter/ActivationParameterSpaceAdapter.java new file mode 100644 index 000000000..5e666a00d --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/adapter/ActivationParameterSpaceAdapter.java @@ -0,0 +1,58 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.adapter; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.adapter.ParameterSpaceAdapter; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * A simple class to adapt a {@link Activation} parameter space to a {@link IActivation} parameter space + * + * @author Alex Black + */ +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = false) +public class ActivationParameterSpaceAdapter extends ParameterSpaceAdapter { + + private ParameterSpace activation; + + public ActivationParameterSpaceAdapter(@JsonProperty("activation") ParameterSpace activation) { + this.activation = activation; + } + + @Override + public IActivation convertValue(Activation from) { + return from.getActivationFunction(); + } + + @Override + protected ParameterSpace underlying() { + return activation; + } + + @Override + protected String underlyingName() { + return "activation"; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/adapter/LossFunctionParameterSpaceAdapter.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/adapter/LossFunctionParameterSpaceAdapter.java new file mode 100644 index 000000000..2c4c2899c --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/adapter/LossFunctionParameterSpaceAdapter.java @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.adapter; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.adapter.ParameterSpaceAdapter; +import org.nd4j.linalg.lossfunctions.ILossFunction; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * A simple class to adapt a {@link LossFunctions.LossFunction} parameter space to a {@link ILossFunction} parameter space + * + * @author Alex Black + */ +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = false) +public class LossFunctionParameterSpaceAdapter + extends ParameterSpaceAdapter { + + private ParameterSpace lossFunction; + + public LossFunctionParameterSpaceAdapter( + @JsonProperty("lossFunction") ParameterSpace lossFunction) { + this.lossFunction = lossFunction; + } + + @Override + protected ILossFunction convertValue(LossFunctions.LossFunction from) { + return from.getILossFunction(); + } + + @Override + protected ParameterSpace underlying() { + return lossFunction; + } + + @Override + protected String underlyingName() { + return "lossFunction"; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/dropout/DropoutSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/dropout/DropoutSpace.java new file mode 100644 index 000000000..76443109e --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/dropout/DropoutSpace.java @@ -0,0 +1,63 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.dropout; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.AbstractParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.nn.conf.dropout.Dropout; +import org.deeplearning4j.nn.conf.dropout.IDropout; + +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +public class DropoutSpace extends AbstractParameterSpace { + + private ParameterSpace dropout; + + @Override + public Dropout getValue(double[] parameterValues) { + double p = dropout.getValue(parameterValues); + if(p == 0){ + //Special case: 0 dropout = "disabled" in DL4J. But Dropout class doesn't support this + return null; + } + return new Dropout(p); + } + + @Override + public int numParameters() { + return dropout.numParameters(); + } + + @Override + public List collectLeaves() { + return dropout.collectLeaves(); + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + dropout.setIndices(indices); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdaGradSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdaGradSpace.java new file mode 100644 index 000000000..ca94a386a --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdaGradSpace.java @@ -0,0 +1,66 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.nd4j.linalg.learning.config.AdaGrad; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.schedule.ISchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = false) +public class AdaGradSpace extends BaseUpdaterSpace { + + private ParameterSpace learningRate; + private ParameterSpace lrSchedule; + + public AdaGradSpace(ParameterSpace learningRate) { + this(learningRate, null); + } + + public static AdaGradSpace withLR(ParameterSpace lr){ + return new AdaGradSpace(lr, null); + } + + public static AdaGradSpace withLRSchedule(ParameterSpace lrSchedule){ + return new AdaGradSpace(null, lrSchedule); + } + + protected AdaGradSpace(@JsonProperty("learningRate") ParameterSpace learningRate, + @JsonProperty("lrSchedule") ParameterSpace lrSchedule){ + this.learningRate = learningRate; + this.lrSchedule = lrSchedule; + } + + @Override + public IUpdater getValue(double[] parameterValues) { + if(lrSchedule != null){ + return new AdaGrad(lrSchedule.getValue(parameterValues)); + } else { + return new AdaGrad(learningRate.getValue(parameterValues)); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdaMaxSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdaMaxSpace.java new file mode 100644 index 000000000..137e62f8d --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdaMaxSpace.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.nd4j.linalg.learning.config.AdaMax; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.schedule.ISchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Data +@EqualsAndHashCode(callSuper = false) +public class AdaMaxSpace extends BaseUpdaterSpace { + + private ParameterSpace learningRate; + private ParameterSpace learningRateSchedule; + private ParameterSpace beta1; + private ParameterSpace beta2; + private ParameterSpace epsilon; + + public AdaMaxSpace(ParameterSpace learningRate) { + this(learningRate, null, null, null); + } + + public AdaMaxSpace(ParameterSpace learningRate, ParameterSpace beta1, + ParameterSpace beta2, ParameterSpace epsilon) { + this(learningRate, null, beta1, beta2, epsilon); + } + + public AdaMaxSpace(@JsonProperty("learningRate") ParameterSpace learningRate, + @JsonProperty("learningRateSchedule") ParameterSpace learningRateSchedule, + @JsonProperty("beta1") ParameterSpace beta1, + @JsonProperty("beta2") ParameterSpace beta2, + @JsonProperty("epsilon") ParameterSpace epsilon){ + this.learningRate = learningRate; + this.learningRateSchedule = learningRateSchedule; + this.beta1 = beta1; + this.beta2 = beta2; + this.epsilon = epsilon; + } + + public static AdaMaxSpace withLR(ParameterSpace lr){ + return new AdaMaxSpace(lr, null, null, null, null); + } + + public static AdaMaxSpace withLRSchedule(ParameterSpace lrSchedule){ + return new AdaMaxSpace(null, lrSchedule, null, null, null); + } + + @Override + public IUpdater getValue(double[] parameterValues) { + double lr = learningRate == null ? AdaMax.DEFAULT_ADAMAX_LEARNING_RATE : learningRate.getValue(parameterValues); + ISchedule lrS = learningRateSchedule == null ? null : learningRateSchedule.getValue(parameterValues); + double b1 = beta1 == null ? AdaMax.DEFAULT_ADAMAX_LEARNING_RATE : beta1.getValue(parameterValues); + double b2 = beta2 == null ? AdaMax.DEFAULT_ADAMAX_LEARNING_RATE : beta2.getValue(parameterValues); + double eps = epsilon == null ? AdaMax.DEFAULT_ADAMAX_LEARNING_RATE : epsilon.getValue(parameterValues); + if(lrS == null){ + return new AdaMax(lr, b1, b2, eps); + } else { + AdaMax a = new AdaMax(lrS); + a.setBeta1(b1); + a.setBeta2(b2); + a.setEpsilon(eps); + return a; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdamSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdamSpace.java new file mode 100644 index 000000000..638b60782 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/AdamSpace.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.schedule.ISchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Data +@EqualsAndHashCode(callSuper = false) +public class AdamSpace extends BaseUpdaterSpace { + + private ParameterSpace learningRate; + private ParameterSpace learningRateSchedule; + private ParameterSpace beta1; + private ParameterSpace beta2; + private ParameterSpace epsilon; + + public AdamSpace(ParameterSpace learningRate) { + this(learningRate, null, null, null); + } + + public AdamSpace(ParameterSpace learningRate, ParameterSpace beta1, + ParameterSpace beta2, ParameterSpace epsilon) { + this(learningRate, null, beta1, beta2, epsilon); + } + + public static AdamSpace withLR(ParameterSpace lr){ + return new AdamSpace(lr, null, null, null, null); + } + + public static AdamSpace withLRSchedule(ParameterSpace lrSchedule){ + return new AdamSpace(null, lrSchedule, null, null, null); + } + + protected AdamSpace(@JsonProperty("learningRate") ParameterSpace learningRate, + @JsonProperty("learningRateSchedule") ParameterSpace learningRateSchedule, + @JsonProperty("beta1") ParameterSpace beta1, + @JsonProperty("beta2") ParameterSpace beta2, + @JsonProperty("epsilon") ParameterSpace epsilon){ + this.learningRate = learningRate; + this.learningRateSchedule = learningRateSchedule; + this.beta1 = beta1; + this.beta2 = beta2; + this.epsilon = epsilon; + } + + @Override + public IUpdater getValue(double[] parameterValues) { + double lr = learningRate == null ? Adam.DEFAULT_ADAM_LEARNING_RATE : learningRate.getValue(parameterValues); + ISchedule lrS = learningRateSchedule == null ? null : learningRateSchedule.getValue(parameterValues); + double b1 = beta1 == null ? Adam.DEFAULT_ADAM_LEARNING_RATE : beta1.getValue(parameterValues); + double b2 = beta2 == null ? Adam.DEFAULT_ADAM_LEARNING_RATE : beta2.getValue(parameterValues); + double eps = epsilon == null ? Adam.DEFAULT_ADAM_LEARNING_RATE : epsilon.getValue(parameterValues); + if(lrS == null){ + return new Adam(lr, b1, b2, eps); + } else { + Adam a = new Adam(lrS); + a.setBeta1(b1); + a.setBeta2(b2); + a.setEpsilon(eps); + return a; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/BaseUpdaterSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/BaseUpdaterSpace.java new file mode 100644 index 000000000..ec1eca996 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/BaseUpdaterSpace.java @@ -0,0 +1,70 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater; + +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import org.deeplearning4j.arbiter.optimize.api.AbstractParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.nd4j.linalg.learning.config.IUpdater; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +@Data +public abstract class BaseUpdaterSpace extends AbstractParameterSpace { + + @Override + public int numParameters() { + int count = 0; + for(ParameterSpace p : collectLeaves()){ + count += p.numParameters(); + } + return count; + } + + @Override + public List collectLeaves() { + Map nested = getNestedSpaces(); + List out = new ArrayList<>(); + for(ParameterSpace p : nested.values()){ + out.addAll(p.collectLeaves()); + } + return out; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices){ + int soFar = 0; + for(ParameterSpace p : collectLeaves()){ + int numParams = p.numParameters(); + if(numParams <= 0){ + continue; + } + int[] subset = Arrays.copyOfRange(indices, soFar, soFar + numParams); + p.setIndices(subset); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/NadamSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/NadamSpace.java new file mode 100644 index 000000000..16bc09127 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/NadamSpace.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.nd4j.linalg.learning.config.Nadam; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.schedule.ISchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Data +@EqualsAndHashCode(callSuper = false) +public class NadamSpace extends BaseUpdaterSpace { + + private ParameterSpace learningRate; + private ParameterSpace learningRateSchedule; + private ParameterSpace beta1; + private ParameterSpace beta2; + private ParameterSpace epsilon; + + public NadamSpace(ParameterSpace learningRate) { + this(learningRate, null, null, null); + } + + public NadamSpace(ParameterSpace learningRate, ParameterSpace beta1, + ParameterSpace beta2, ParameterSpace epsilon) { + this(learningRate, null, beta1, beta2, epsilon); + } + + public NadamSpace(@JsonProperty("learningRate") ParameterSpace learningRate, + @JsonProperty("learningRateSchedule") ParameterSpace learningRateSchedule, + @JsonProperty("beta1") ParameterSpace beta1, + @JsonProperty("beta2") ParameterSpace beta2, + @JsonProperty("epsilon") ParameterSpace epsilon){ + this.learningRate = learningRate; + this.learningRateSchedule = learningRateSchedule; + this.beta1 = beta1; + this.beta2 = beta2; + this.epsilon = epsilon; + } + + public static NadamSpace withLR(ParameterSpace lr){ + return new NadamSpace(lr, null, null, null, null); + } + + public static NadamSpace withLRSchedule(ParameterSpace lrSchedule){ + return new NadamSpace(null, lrSchedule, null, null, null); + } + + @Override + public IUpdater getValue(double[] parameterValues) { + double lr = learningRate == null ? Nadam.DEFAULT_NADAM_LEARNING_RATE : learningRate.getValue(parameterValues); + ISchedule lrS = learningRateSchedule == null ? null : learningRateSchedule.getValue(parameterValues); + double b1 = beta1 == null ? Nadam.DEFAULT_NADAM_LEARNING_RATE : beta1.getValue(parameterValues); + double b2 = beta2 == null ? Nadam.DEFAULT_NADAM_LEARNING_RATE : beta2.getValue(parameterValues); + double eps = epsilon == null ? Nadam.DEFAULT_NADAM_LEARNING_RATE : epsilon.getValue(parameterValues); + if(lrS == null){ + return new Nadam(lr, b1, b2, eps); + } else { + Nadam a = new Nadam(lrS); + a.setBeta1(b1); + a.setBeta2(b2); + a.setEpsilon(eps); + return a; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/NesterovsSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/NesterovsSpace.java new file mode 100644 index 000000000..6bb059493 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/NesterovsSpace.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.schedule.ISchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Data +@EqualsAndHashCode(callSuper = false) +public class NesterovsSpace extends BaseUpdaterSpace { + + protected ParameterSpace learningRate; + protected ParameterSpace learningRateSchedule; + protected ParameterSpace momentum; + protected ParameterSpace momentumSchedule; + + public NesterovsSpace(ParameterSpace learningRate) { + this(learningRate, null); + } + + public NesterovsSpace(ParameterSpace learningRate, ParameterSpace momentum) { + this(learningRate, null, momentum, null); + } + + public NesterovsSpace(@JsonProperty("learningRate") ParameterSpace learningRate, + @JsonProperty("learningRateSchedule") ParameterSpace learningRateSchedule, + @JsonProperty("momentum") ParameterSpace momentum, + @JsonProperty("momentumSchedule") ParameterSpace momentumSchedule) { + this.learningRate = learningRate; + this.learningRateSchedule = learningRateSchedule; + this.momentum = momentum; + this.momentumSchedule = momentumSchedule; + } + + public static NesterovsSpace withLR(ParameterSpace lr){ + return new NesterovsSpace(lr, null, null, null); + } + + public static NesterovsSpace withLR(ParameterSpace lr, double momentum){ + return new NesterovsSpace(lr, null, new FixedValue<>(momentum), null); + } + + public static NesterovsSpace withLR(ParameterSpace lr, ParameterSpace momentum){ + return new NesterovsSpace(lr, null, momentum, null); + } + + public static NesterovsSpace withLRSchedule(ParameterSpace lrSchedule){ + return new NesterovsSpace(null, lrSchedule, null, null); + } + + public static NesterovsSpace withLRSchedule(ParameterSpace lrSchedule, double momentum){ + return new NesterovsSpace(null, lrSchedule, new FixedValue<>(momentum), null); + } + + public static NesterovsSpace withLRSchedule(ParameterSpace lrSchedule, ParameterSpace momentum){ + return new NesterovsSpace(null, lrSchedule, momentum, null); + } + + + @Override + public IUpdater getValue(double[] parameterValues) { + double lr = learningRate == null ? Nesterovs.DEFAULT_NESTEROV_LEARNING_RATE : learningRate.getValue(parameterValues); + ISchedule lrS = learningRateSchedule == null ? null : learningRateSchedule.getValue(parameterValues); + double m = momentum == null ? Nesterovs.DEFAULT_NESTEROV_MOMENTUM : momentum.getValue(parameterValues); + ISchedule mS = momentumSchedule == null ? null : momentumSchedule.getValue(parameterValues); + if(lrS == null){ + if(momentumSchedule == null){ + return new Nesterovs(lr, m); + } else { + return new Nesterovs(lr, mS); + } + } else { + if(momentumSchedule == null){ + return new Nesterovs(lrS, m); + } else { + return new Nesterovs(lrS, mS); + } + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/RmsPropSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/RmsPropSpace.java new file mode 100644 index 000000000..8590947d6 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/RmsPropSpace.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.learning.config.RmsProp; +import org.nd4j.linalg.schedule.ISchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Data +@EqualsAndHashCode(callSuper = false) +public class RmsPropSpace extends BaseUpdaterSpace { + + protected ParameterSpace learningRate; + protected ParameterSpace learningRateSchedule; + + public RmsPropSpace(ParameterSpace learningRate) { + this(learningRate, null); + } + + public RmsPropSpace(@JsonProperty("learningRate") ParameterSpace learningRate, + @JsonProperty("learningRateSchedule") ParameterSpace learningRateSchedule){ + this.learningRate = learningRate; + this.learningRateSchedule = learningRateSchedule; + } + + @Override + public IUpdater getValue(double[] parameterValues) { + double lr = learningRate == null ? RmsProp.DEFAULT_RMSPROP_LEARNING_RATE : learningRate.getValue(parameterValues); + ISchedule lrS = learningRateSchedule == null ? null : learningRateSchedule.getValue(parameterValues); + if(lrS == null){ + return new RmsProp(lr); + } else { + return new RmsProp(lrS); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/SgdSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/SgdSpace.java new file mode 100644 index 000000000..0c136e114 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/SgdSpace.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.schedule.ISchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Data +@EqualsAndHashCode(callSuper = false) +public class SgdSpace extends BaseUpdaterSpace { + + protected ParameterSpace learningRate; + protected ParameterSpace learningRateSchedule; + + public SgdSpace(ParameterSpace learningRate) { + this(learningRate, null); + } + + public SgdSpace(@JsonProperty("learningRate") ParameterSpace learningRate, + @JsonProperty("learningRateSchedule") ParameterSpace learningRateSchedule){ + this.learningRate = learningRate; + this.learningRateSchedule = learningRateSchedule; + } + + @Override + public IUpdater getValue(double[] parameterValues) { + double lr = learningRate == null ? Sgd.DEFAULT_SGD_LR : learningRate.getValue(parameterValues); + ISchedule lrS = learningRateSchedule == null ? null : learningRateSchedule.getValue(parameterValues); + if(lrS == null){ + return new Sgd(lr); + } else { + return new Sgd(lrS); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/ExponentialScheduleSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/ExponentialScheduleSpace.java new file mode 100644 index 000000000..3977faa25 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/ExponentialScheduleSpace.java @@ -0,0 +1,92 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater.schedule; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.nd4j.linalg.schedule.ExponentialSchedule; +import org.nd4j.linalg.schedule.ISchedule; +import org.nd4j.linalg.schedule.ScheduleType; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.*; + +@NoArgsConstructor //JSON +@Data +public class ExponentialScheduleSpace implements ParameterSpace { + + private ScheduleType scheduleType; + private ParameterSpace initialValue; + private ParameterSpace gamma; + + public ExponentialScheduleSpace(@NonNull ScheduleType scheduleType, + @NonNull ParameterSpace initialValue, double gamma){ + this(scheduleType, initialValue, new FixedValue<>(gamma)); + } + + public ExponentialScheduleSpace(@NonNull @JsonProperty("scheduleType") ScheduleType scheduleType, + @NonNull @JsonProperty("initialValue") ParameterSpace initialValue, + @NonNull @JsonProperty("gamma") ParameterSpace gamma){ + this.scheduleType = scheduleType; + this.initialValue = initialValue; + this.gamma = gamma; + } + + @Override + public ISchedule getValue(double[] parameterValues) { + return new ExponentialSchedule(scheduleType, initialValue.getValue(parameterValues), gamma.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return initialValue.numParameters() + gamma.numParameters(); + } + + @Override + public List collectLeaves() { + return Arrays.asList(initialValue, gamma); + } + + @Override + public Map getNestedSpaces() { + Map out = new LinkedHashMap<>(); + out.put("initialValue", initialValue); + out.put("gamma", gamma); + return out; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + if(initialValue.numParameters() > 0){ + int[] sub = Arrays.copyOfRange(indices, 0, initialValue.numParameters()); + initialValue.setIndices(sub); + } + if(gamma.numParameters() > 0){ + int inp = initialValue.numParameters(); + int[] sub = Arrays.copyOfRange(indices, inp, inp + gamma.numParameters()); + gamma.setIndices(sub); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/InverseScheduleSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/InverseScheduleSpace.java new file mode 100644 index 000000000..a22c640a9 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/InverseScheduleSpace.java @@ -0,0 +1,106 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater.schedule; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.nd4j.linalg.schedule.ExponentialSchedule; +import org.nd4j.linalg.schedule.ISchedule; +import org.nd4j.linalg.schedule.InverseSchedule; +import org.nd4j.linalg.schedule.ScheduleType; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@NoArgsConstructor //JSON +@Data +public class InverseScheduleSpace implements ParameterSpace { + + private ScheduleType scheduleType; + private ParameterSpace initialValue; + private ParameterSpace gamma; + private ParameterSpace power; + + public InverseScheduleSpace(@NonNull ScheduleType scheduleType, @NonNull ParameterSpace initialValue, + double gamma, double power){ + this(scheduleType, initialValue, new FixedValue<>(gamma), new FixedValue<>(power)); + } + + public InverseScheduleSpace(@NonNull @JsonProperty("scheduleType") ScheduleType scheduleType, + @NonNull @JsonProperty("initialValue") ParameterSpace initialValue, + @NonNull @JsonProperty("gamma") ParameterSpace gamma, + @NonNull @JsonProperty("power") ParameterSpace power){ + this.scheduleType = scheduleType; + this.initialValue = initialValue; + this.gamma = gamma; + this.power = power; + } + + @Override + public ISchedule getValue(double[] parameterValues) { + return new InverseSchedule(scheduleType, initialValue.getValue(parameterValues), + gamma.getValue(parameterValues), power.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return initialValue.numParameters() + gamma.numParameters() + power.numParameters(); + } + + @Override + public List collectLeaves() { + return Arrays.asList(initialValue, gamma, power); + } + + @Override + public Map getNestedSpaces() { + Map out = new LinkedHashMap<>(); + out.put("initialValue", initialValue); + out.put("gamma", gamma); + out.put("power", power); + return out; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + if(initialValue.numParameters() > 0){ + int[] sub = Arrays.copyOfRange(indices, 0, initialValue.numParameters()); + initialValue.setIndices(sub); + } + if(gamma.numParameters() > 0){ + int inp = initialValue.numParameters(); + int[] sub = Arrays.copyOfRange(indices, inp, inp + gamma.numParameters()); + gamma.setIndices(sub); + } + if(power.numParameters() > 0){ + int np = initialValue.numParameters() + gamma.numParameters(); + int[] sub = Arrays.copyOfRange(indices, np, np + power.numParameters()); + power.setIndices(sub); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/PolyScheduleSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/PolyScheduleSpace.java new file mode 100644 index 000000000..9beff30b5 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/PolyScheduleSpace.java @@ -0,0 +1,106 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater.schedule; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.nd4j.linalg.schedule.ISchedule; +import org.nd4j.linalg.schedule.InverseSchedule; +import org.nd4j.linalg.schedule.PolySchedule; +import org.nd4j.linalg.schedule.ScheduleType; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@NoArgsConstructor //JSON +@Data +public class PolyScheduleSpace implements ParameterSpace { + + private ScheduleType scheduleType; + private ParameterSpace initialValue; + private ParameterSpace power; + private ParameterSpace maxIter; + + public PolyScheduleSpace(@NonNull ScheduleType scheduleType, @NonNull ParameterSpace initialValue, + double power, int maxIter){ + this(scheduleType, initialValue, new FixedValue<>(power), new FixedValue<>(maxIter)); + } + + public PolyScheduleSpace(@NonNull @JsonProperty("scheduleType") ScheduleType scheduleType, + @NonNull @JsonProperty("initialValue") ParameterSpace initialValue, + @NonNull @JsonProperty("power") ParameterSpace power, + @NonNull @JsonProperty("maxIter") ParameterSpace maxIter){ + this.scheduleType = scheduleType; + this.initialValue = initialValue; + this.power = power; + this.maxIter = maxIter; + } + + @Override + public ISchedule getValue(double[] parameterValues) { + return new PolySchedule(scheduleType, initialValue.getValue(parameterValues), + power.getValue(parameterValues), maxIter.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return initialValue.numParameters() + power.numParameters() + maxIter.numParameters(); + } + + @Override + public List collectLeaves() { + return Arrays.asList(initialValue, power, maxIter); + } + + @Override + public Map getNestedSpaces() { + Map out = new LinkedHashMap<>(); + out.put("initialValue", initialValue); + out.put("power", power); + out.put("maxIter", maxIter); + return out; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + if(initialValue.numParameters() > 0){ + int[] sub = Arrays.copyOfRange(indices, 0, initialValue.numParameters()); + initialValue.setIndices(sub); + } + if(power.numParameters() > 0){ + int np = initialValue.numParameters(); + int[] sub = Arrays.copyOfRange(indices, np, np + power.numParameters()); + power.setIndices(sub); + } + if(maxIter.numParameters() > 0){ + int np = initialValue.numParameters() + power.numParameters(); + int[] sub = Arrays.copyOfRange(indices, np, np + maxIter.numParameters()); + maxIter.setIndices(sub); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/SigmoidScheduleSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/SigmoidScheduleSpace.java new file mode 100644 index 000000000..c8c5e4c3c --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/SigmoidScheduleSpace.java @@ -0,0 +1,106 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater.schedule; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.nd4j.linalg.schedule.ISchedule; +import org.nd4j.linalg.schedule.PolySchedule; +import org.nd4j.linalg.schedule.ScheduleType; +import org.nd4j.linalg.schedule.SigmoidSchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@NoArgsConstructor //JSON +@Data +public class SigmoidScheduleSpace implements ParameterSpace { + + private ScheduleType scheduleType; + private ParameterSpace initialValue; + private ParameterSpace gamma; + private ParameterSpace stepSize; + + public SigmoidScheduleSpace(@NonNull ScheduleType scheduleType, @NonNull ParameterSpace initialValue, + double gamma, int stepSize){ + this(scheduleType, initialValue, new FixedValue<>(gamma), new FixedValue<>(stepSize)); + } + + public SigmoidScheduleSpace(@NonNull @JsonProperty("scheduleType") ScheduleType scheduleType, + @NonNull @JsonProperty("initialValue") ParameterSpace initialValue, + @NonNull @JsonProperty("gamma") ParameterSpace gamma, + @NonNull @JsonProperty("stepSize") ParameterSpace stepSize){ + this.scheduleType = scheduleType; + this.initialValue = initialValue; + this.gamma = gamma; + this.stepSize = stepSize; + } + + @Override + public ISchedule getValue(double[] parameterValues) { + return new SigmoidSchedule(scheduleType, initialValue.getValue(parameterValues), + gamma.getValue(parameterValues), stepSize.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return initialValue.numParameters() + gamma.numParameters() + stepSize.numParameters(); + } + + @Override + public List collectLeaves() { + return Arrays.asList(initialValue, gamma, stepSize); + } + + @Override + public Map getNestedSpaces() { + Map out = new LinkedHashMap<>(); + out.put("initialValue", initialValue); + out.put("gamma", gamma); + out.put("stepSize", stepSize); + return out; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + if(initialValue.numParameters() > 0){ + int[] sub = Arrays.copyOfRange(indices, 0, initialValue.numParameters()); + initialValue.setIndices(sub); + } + if(gamma.numParameters() > 0){ + int np = initialValue.numParameters(); + int[] sub = Arrays.copyOfRange(indices, np, np + gamma.numParameters()); + gamma.setIndices(sub); + } + if(stepSize.numParameters() > 0){ + int np = initialValue.numParameters() + gamma.numParameters(); + int[] sub = Arrays.copyOfRange(indices, np, np + stepSize.numParameters()); + stepSize.setIndices(sub); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/StepScheduleSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/StepScheduleSpace.java new file mode 100644 index 000000000..d37638e8d --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/conf/updater/schedule/StepScheduleSpace.java @@ -0,0 +1,106 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.conf.updater.schedule; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.nd4j.linalg.schedule.ISchedule; +import org.nd4j.linalg.schedule.InverseSchedule; +import org.nd4j.linalg.schedule.ScheduleType; +import org.nd4j.linalg.schedule.StepSchedule; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@NoArgsConstructor //JSON +@Data +public class StepScheduleSpace implements ParameterSpace { + + private ScheduleType scheduleType; + private ParameterSpace initialValue; + private ParameterSpace decayRate; + private ParameterSpace step; + + public StepScheduleSpace(@NonNull ScheduleType scheduleType, @NonNull ParameterSpace initialValue, + double decayRate, double step){ + this(scheduleType, initialValue, new FixedValue<>(decayRate), new FixedValue<>(step)); + } + + public StepScheduleSpace(@NonNull @JsonProperty("scheduleType") ScheduleType scheduleType, + @NonNull @JsonProperty("initialValue") ParameterSpace initialValue, + @NonNull @JsonProperty("decayRate") ParameterSpace decayRate, + @NonNull @JsonProperty("step") ParameterSpace step){ + this.scheduleType = scheduleType; + this.initialValue = initialValue; + this.decayRate = decayRate; + this.step = step; + } + + @Override + public ISchedule getValue(double[] parameterValues) { + return new StepSchedule(scheduleType, initialValue.getValue(parameterValues), + decayRate.getValue(parameterValues), step.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return initialValue.numParameters() + decayRate.numParameters() + step.numParameters(); + } + + @Override + public List collectLeaves() { + return Arrays.asList(initialValue, decayRate, step); + } + + @Override + public Map getNestedSpaces() { + Map out = new LinkedHashMap<>(); + out.put("initialValue", initialValue); + out.put("decayRate", decayRate); + out.put("step", step); + return out; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + if(initialValue.numParameters() > 0){ + int[] sub = Arrays.copyOfRange(indices, 0, initialValue.numParameters()); + initialValue.setIndices(sub); + } + if(decayRate.numParameters() > 0){ + int inp = initialValue.numParameters(); + int[] sub = Arrays.copyOfRange(indices, inp, inp + decayRate.numParameters()); + decayRate.setIndices(sub); + } + if(step.numParameters() > 0){ + int np = initialValue.numParameters() + decayRate.numParameters(); + int[] sub = Arrays.copyOfRange(indices, np, np + step.numParameters()); + step.setIndices(sub); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/data/DataSetIteratorFactoryProvider.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/data/DataSetIteratorFactoryProvider.java new file mode 100644 index 000000000..e5443c0d3 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/data/DataSetIteratorFactoryProvider.java @@ -0,0 +1,85 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.data; + +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.nd4j.linalg.dataset.api.iterator.DataSetIteratorFactory; + +import java.util.Map; + +/** + * This is a {@link DataProvider} for + * an {@link DataSetIteratorFactory} which + * based on a key of {@link DataSetIteratorFactoryProvider#FACTORY_KEY} + * will create {@link org.nd4j.linalg.dataset.api.iterator.DataSetIterator} + * for use with arbiter. + * + * This {@link DataProvider} is mainly meant for use for command line driven + * applications. + * + * @author Adam Gibson + */ +public class DataSetIteratorFactoryProvider implements DataProvider { + + public final static String FACTORY_KEY = "org.deeplearning4j.arbiter.data.data.factory"; + + /** + * Get training data given some parameters for the data. + * Data parameters map is used to specify things like batch + * size data preprocessing + * + * @param dataParameters Parameters for data. May be null or empty for default data + * @return training data + */ + @Override + public DataSetIteratorFactory trainData(Map dataParameters) { + return create(dataParameters); + } + + /** + * Get training data given some parameters for the data. Data parameters map + * is used to specify things like batch + * size data preprocessing + * + * @param dataParameters Parameters for data. May be null or empty for default data + * @return training data + */ + @Override + public DataSetIteratorFactory testData(Map dataParameters) { + return create(dataParameters); + } + + @Override + public Class getDataType() { + return DataSetIteratorFactory.class; + } + + private DataSetIteratorFactory create(Map dataParameters) { + if (!dataParameters.containsKey(FACTORY_KEY)) + throw new IllegalArgumentException( + "No data set iterator factory class found. Please specify a class name with key " + + FACTORY_KEY); + String value = dataParameters.get(FACTORY_KEY).toString(); + try { + Class clazz = + (Class) Class.forName(value); + return clazz.newInstance(); + } catch (Exception e) { + throw new RuntimeException("Could not create DataSetIteratorFactory instance - missing no-arg constructor?", e); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/data/MnistDataProvider.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/data/MnistDataProvider.java new file mode 100644 index 000000000..c42837896 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/data/MnistDataProvider.java @@ -0,0 +1,80 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.data; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultipleEpochsIterator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.IOException; +import java.util.Map; +import java.util.Random; + +/** + * + * MnistDataProvider - a DataProvider for the MNIST data set, with configurable number of epochs, batch size + * and RNG seed + * + * @author Alex Black + */ +@Data +@NoArgsConstructor +public class MnistDataProvider implements DataProvider{ + + private int numEpochs; + private int batchSize; + private int rngSeed; + + public MnistDataProvider(int numEpochs, int batchSize){ + this(numEpochs, batchSize, new Random().nextInt()); + } + + public MnistDataProvider(@JsonProperty("numEpochs") int numEpochs, @JsonProperty("batchSize") int batchSize, + @JsonProperty("rngSeed") int rngSeed) { + this.numEpochs = numEpochs; + this.batchSize = batchSize; + this.rngSeed = rngSeed; + } + + + @Override + public Object trainData(Map dataParameters) { + try { + return new MultipleEpochsIterator(numEpochs, new MnistDataSetIterator(batchSize, true, rngSeed)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Object testData(Map dataParameters) { + try { + return new MnistDataSetIterator(batchSize, false, 12345); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/AlphaDropoutSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/AlphaDropoutSpace.java new file mode 100644 index 000000000..f4e3801f5 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/AlphaDropoutSpace.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.dropout; + +import lombok.AllArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.conf.dropout.AlphaDropout; +import org.deeplearning4j.nn.conf.dropout.IDropout; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@AllArgsConstructor +public class AlphaDropoutSpace implements ParameterSpace { + + private ParameterSpace dropout; + + public AlphaDropoutSpace(double activationRetainProbability){ + this(new FixedValue<>(activationRetainProbability)); + } + + @Override + public IDropout getValue(double[] parameterValues) { + return new AlphaDropout(dropout.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return dropout.numParameters(); + } + + @Override + public List collectLeaves() { + return Collections.singletonList(dropout); + } + + @Override + public Map getNestedSpaces() { + return Collections.singletonMap("dropout", dropout); + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + dropout.setIndices(indices); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/DropoutSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/DropoutSpace.java new file mode 100644 index 000000000..52dea3155 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/DropoutSpace.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.dropout; + +import lombok.AllArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.conf.dropout.Dropout; +import org.deeplearning4j.nn.conf.dropout.IDropout; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@AllArgsConstructor +public class DropoutSpace implements ParameterSpace { + + private ParameterSpace dropout; + + public DropoutSpace(double activationRetainProbability){ + this(new FixedValue<>(activationRetainProbability)); + } + + @Override + public IDropout getValue(double[] parameterValues) { + return new Dropout(dropout.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return dropout.numParameters(); + } + + @Override + public List collectLeaves() { + return Collections.singletonList(dropout); + } + + @Override + public Map getNestedSpaces() { + return Collections.singletonMap("dropout", dropout); + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + dropout.setIndices(indices); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/GaussianDropoutSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/GaussianDropoutSpace.java new file mode 100644 index 000000000..0cb345f40 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/GaussianDropoutSpace.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.dropout; + +import lombok.AllArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.conf.dropout.Dropout; +import org.deeplearning4j.nn.conf.dropout.GaussianDropout; +import org.deeplearning4j.nn.conf.dropout.IDropout; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@AllArgsConstructor +public class GaussianDropoutSpace implements ParameterSpace { + + private ParameterSpace rate; + + public GaussianDropoutSpace(double rate){ + this(new FixedValue<>(rate)); + } + + @Override + public IDropout getValue(double[] parameterValues) { + return new GaussianDropout(rate.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return rate.numParameters(); + } + + @Override + public List collectLeaves() { + return Collections.singletonList(rate); + } + + @Override + public Map getNestedSpaces() { + return Collections.singletonMap("rate", rate); + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + rate.setIndices(indices); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/GaussianNoiseSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/GaussianNoiseSpace.java new file mode 100644 index 000000000..706d389ee --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/dropout/GaussianNoiseSpace.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.dropout; + +import lombok.AllArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.conf.dropout.GaussianNoise; +import org.deeplearning4j.nn.conf.dropout.IDropout; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@AllArgsConstructor +public class GaussianNoiseSpace implements ParameterSpace { + + private ParameterSpace stddev; + + public GaussianNoiseSpace(double stddev){ + this(new FixedValue<>(stddev)); + } + + @Override + public IDropout getValue(double[] parameterValues) { + return new GaussianNoise(stddev.getValue(parameterValues)); + } + + @Override + public int numParameters() { + return stddev.numParameters(); + } + + @Override + public List collectLeaves() { + return Collections.singletonList(stddev); + } + + @Override + public Map getNestedSpaces() { + return Collections.singletonMap("stddev", stddev); + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + stddev.setIndices(indices); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/evaluator/multilayer/ClassificationEvaluator.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/evaluator/multilayer/ClassificationEvaluator.java new file mode 100644 index 000000000..3d13ea2f1 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/evaluator/multilayer/ClassificationEvaluator.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.evaluator.multilayer; + +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.evaluation.ModelEvaluator; +import org.deeplearning4j.arbiter.scoring.util.ScoreUtil; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * A model evaluator for doing additional + * evaluation (classification evaluation) + * for a {@link MultiLayerNetwork} given a {@link DataSetIterator} + * + * @author Alex Black + */ +@NoArgsConstructor +@AllArgsConstructor +public class ClassificationEvaluator implements ModelEvaluator { + private Map params = null; + + + @Override + public Evaluation evaluateModel(Object model, DataProvider dataProvider) { + + if (model instanceof MultiLayerNetwork) { + DataSetIterator iterator = ScoreUtil.getIterator(dataProvider.testData(params)); + return ScoreUtil.getEvaluation((MultiLayerNetwork) model, iterator); + } else { + DataSetIterator iterator = ScoreUtil.getIterator(dataProvider.testData(params)); + return ScoreUtil.getEvaluation((ComputationGraph) model, iterator); + } + } + + @Override + public List> getSupportedModelTypes() { + return Arrays.>asList(MultiLayerNetwork.class, ComputationGraph.class); + } + + @Override + public List> getSupportedDataTypes() { + return Arrays.>asList(DataSetIterator.class, MultiDataSetIterator.class); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/evaluator/multilayer/RegressionDataEvaluator.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/evaluator/multilayer/RegressionDataEvaluator.java new file mode 100644 index 000000000..7973b11e0 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/evaluator/multilayer/RegressionDataEvaluator.java @@ -0,0 +1,62 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.evaluator.multilayer; + +import lombok.AllArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.evaluation.ModelEvaluator; +import org.deeplearning4j.arbiter.scoring.RegressionValue; +import org.deeplearning4j.arbiter.scoring.util.ScoreUtil; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Created by agibsonccc on 3/12/17. + */ +@AllArgsConstructor +public class RegressionDataEvaluator implements ModelEvaluator { + private RegressionValue regressionValue; + private Map params = null; + + @Override + public Double evaluateModel(Object model, DataProvider dataProvider) { + + if (model instanceof MultiLayerNetwork) { + DataSetIterator iterator = ScoreUtil.getIterator(dataProvider.testData(params)); + return ScoreUtil.score((MultiLayerNetwork) model, iterator, regressionValue); + } else { + DataSetIterator iterator = ScoreUtil.getIterator(dataProvider.testData(params)); + return ScoreUtil.score((ComputationGraph) model, iterator, regressionValue); + } + } + + @Override + public List> getSupportedModelTypes() { + return Arrays.>asList(MultiLayerNetwork.class, ComputationGraph.class); + } + + @Override + public List> getSupportedDataTypes() { + return Arrays.>asList(DataSetIterator.class, MultiDataSetIterator.class); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/AbstractLSTMLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/AbstractLSTMLayerSpace.java new file mode 100644 index 000000000..7cad81a82 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/AbstractLSTMLayerSpace.java @@ -0,0 +1,108 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.adapter.ActivationParameterSpaceAdapter; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.AbstractLSTM; +import org.deeplearning4j.nn.conf.layers.GravesLSTM; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; + +/** + * Layer space for LSTM layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public abstract class AbstractLSTMLayerSpace extends FeedForwardLayerSpace { + + protected ParameterSpace forgetGateBiasInit; + protected ParameterSpace gateActivationFn; + + protected AbstractLSTMLayerSpace(Builder builder) { + super(builder); + this.forgetGateBiasInit = builder.forgetGateBiasInit; + this.gateActivationFn = builder.gateActivationFn; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + protected void setLayerOptionsBuilder(AbstractLSTM.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (forgetGateBiasInit != null) + builder.forgetGateBiasInit(forgetGateBiasInit.getValue(values)); + if(gateActivationFn != null) + builder.gateActivationFunction(gateActivationFn.getValue(values)); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder(); //"AbstractLSTMLayerSpace("); + if (forgetGateBiasInit != null) + sb.append("forgetGateBiasInit: ").append(forgetGateBiasInit).append(delim); + if (gateActivationFn != null) + sb.append("gateActivationFn: ").append(gateActivationFn).append(delim); + sb.append(super.toString(delim)); + return sb.toString(); + } + + public static abstract class Builder extends FeedForwardLayerSpace.Builder { + + private ParameterSpace forgetGateBiasInit; + private ParameterSpace gateActivationFn; + + public T forgetGateBiasInit(double forgetGateBiasInit) { + return forgetGateBiasInit(new FixedValue<>(forgetGateBiasInit)); + } + + public T forgetGateBiasInit(ParameterSpace forgetGateBiasInit) { + this.forgetGateBiasInit = forgetGateBiasInit; + return (T)this; + } + + public T gateActivationFn(Activation activation){ + return gateActivationFn(activation.getActivationFunction()); + } + + public T gateActivation(ParameterSpace gateActivationFn){ + return gateActivationFn(new ActivationParameterSpaceAdapter(gateActivationFn)); + } + + public T gateActivationFn(IActivation gateActivationFn){ + return gateActivationFn(new FixedValue<>(gateActivationFn)); + } + + public T gateActivationFn(ParameterSpace gateActivationFn){ + this.gateActivationFn = gateActivationFn; + return (T)this; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/ActivationLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/ActivationLayerSpace.java new file mode 100644 index 000000000..1d45d23c8 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/ActivationLayerSpace.java @@ -0,0 +1,94 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.adapter.ActivationParameterSpaceAdapter; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.ActivationLayer; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; + +/** + * Layer space for {@link ActivationLayer} + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class ActivationLayerSpace extends LayerSpace { + + private ParameterSpace activationFunction; + + protected ActivationLayerSpace(Builder builder) { + super(builder); + this.activationFunction = builder.activationFunction; + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + + @Override + public ActivationLayer getValue(double[] parameterValues) { + ActivationLayer.Builder b = new ActivationLayer.Builder(); + super.setLayerOptionsBuilder(b, parameterValues); + b.activation(activationFunction.getValue(parameterValues)); + return b.build(); + } + + public static class Builder extends LayerSpace.Builder { + + private ParameterSpace activationFunction; + + public Builder activation(Activation activation) { + return activation(new FixedValue<>(activation)); + } + + public Builder activation(IActivation iActivation) { + return activationFn(new FixedValue<>(iActivation)); + } + + public Builder activation(ParameterSpace activationFunction) { + return activationFn(new ActivationParameterSpaceAdapter(activationFunction)); + } + + public Builder activationFn(ParameterSpace activationFunction) { + this.activationFunction = activationFunction; + return this; + } + + @SuppressWarnings("unchecked") + public ActivationLayerSpace build() { + return new ActivationLayerSpace(this); + } + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + return "ActivationLayerSpace(" + super.toString(delim) + ")"; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/AutoEncoderLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/AutoEncoderLayerSpace.java new file mode 100644 index 000000000..a429a2c96 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/AutoEncoderLayerSpace.java @@ -0,0 +1,107 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.AutoEncoder; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Layer space for autoencoder layers + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class AutoEncoderLayerSpace extends BasePretrainNetworkLayerSpace { + @JsonProperty + private ParameterSpace corruptionLevel; + @JsonProperty + private ParameterSpace sparsity; + + private AutoEncoderLayerSpace(Builder builder) { + super(builder); + this.corruptionLevel = builder.corruptionLevel; + this.sparsity = builder.sparsity; + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public AutoEncoder getValue(double[] values) { + AutoEncoder.Builder b = new AutoEncoder.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(AutoEncoder.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (corruptionLevel != null) + builder.corruptionLevel(corruptionLevel.getValue(values)); + if (sparsity != null) + builder.sparsity(sparsity.getValue(values)); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder("AutoEncoderLayerSpace("); + if (corruptionLevel != null) + sb.append("corruptionLevel: ").append(corruptionLevel).append(delim); + if (sparsity != null) + sb.append("sparsity: ").append(sparsity).append(delim); + sb.append(super.toString(delim)).append(")"); + return sb.toString(); + } + + public static class Builder extends BasePretrainNetworkLayerSpace.Builder { + + private ParameterSpace corruptionLevel; + private ParameterSpace sparsity; + + public Builder corruptionLevel(double corruptionLevel) { + return corruptionLevel(new FixedValue<>(corruptionLevel)); + } + + public Builder corruptionLevel(ParameterSpace corruptionLevel) { + this.corruptionLevel = corruptionLevel; + return this; + } + + public Builder sparsity(double sparsity) { + return sparsity(new FixedValue<>(sparsity)); + } + + public Builder sparsity(ParameterSpace sparsity) { + this.sparsity = sparsity; + return this; + } + + public AutoEncoderLayerSpace build() { + return new AutoEncoderLayerSpace(this); + } + + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseConvolutionLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseConvolutionLayerSpace.java new file mode 100644 index 000000000..11bf1f274 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseConvolutionLayerSpace.java @@ -0,0 +1,162 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; + +/** + * Layer space for convolutional layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public abstract class BaseConvolutionLayerSpace extends FeedForwardLayerSpace { + protected ParameterSpace dilation; + protected ParameterSpace kernelSize; + protected ParameterSpace stride; + protected ParameterSpace padding; + protected ParameterSpace convolutionMode; + protected ParameterSpace hasBias; + + protected BaseConvolutionLayerSpace(Builder builder) { + super(builder); + this.dilation = builder.dilation; + this.kernelSize = builder.kernelSize; + this.stride = builder.stride; + this.padding = builder.padding; + this.convolutionMode = builder.convolutionMode; + this.hasBias = builder.hasBias; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + protected void setLayerOptionsBuilder(ConvolutionLayer.BaseConvBuilder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (dilation != null) + builder.dilation(dilation.getValue(values)); + if (kernelSize != null) + builder.kernelSize(kernelSize.getValue(values)); + if (stride != null) + builder.stride(stride.getValue(values)); + if (padding != null) + builder.padding(padding.getValue(values)); + if (convolutionMode != null) + builder.convolutionMode(convolutionMode.getValue(values)); + if (hasBias != null) + builder.hasBias(hasBias.getValue(values)); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder(); + if (dilation != null) + sb.append("dilation: ").append(dilation).append(delim); + if (kernelSize != null) + sb.append("kernelSize: ").append(kernelSize).append(delim); + if (stride != null) + sb.append("stride: ").append(stride).append(delim); + if (padding != null) + sb.append("padding: ").append(padding).append(delim); + if (convolutionMode != null) + sb.append("convolutionMode: ").append(convolutionMode).append(delim); + if (hasBias != null) + sb.append("hasBias: ").append(hasBias).append(delim); + sb.append(super.toString(delim)); + return sb.toString(); + } + + + public static abstract class Builder extends FeedForwardLayerSpace.Builder { + protected ParameterSpace dilation; + protected ParameterSpace kernelSize; + protected ParameterSpace stride; + protected ParameterSpace padding; + protected ParameterSpace convolutionMode; + protected ParameterSpace hasBias; + + public T dilation(int... dilation) { + return dilation(new FixedValue<>(dilation)); + } + + public T dilation(ParameterSpace dilation) { + this.dilation = dilation; + return (T) this; + } + public T kernelSize(int... kernelSize) { + return kernelSize(new FixedValue<>(kernelSize)); + } + + public T kernelSize(ParameterSpace kernelSize) { + this.kernelSize = kernelSize; + return (T)this; + } + + public T stride(int... stride) { + return stride(new FixedValue<>(stride)); + } + + public T stride(ParameterSpace stride) { + this.stride = stride; + return (T)this; + } + + public T padding(int... padding) { + return padding(new FixedValue<>(padding)); + } + + public T padding(ParameterSpace padding) { + this.padding = padding; + return (T)this; + } + + public T convolutionMode(ConvolutionMode convolutionMode) { + return convolutionMode(new FixedValue<>(convolutionMode)); + } + + public T convolutionMode(ParameterSpace convolutionMode) { + this.convolutionMode = convolutionMode; + return (T)this; + } + + public T hasBias(boolean hasBias){ + return hasBias(new FixedValue<>(hasBias)); + } + + public T hasBias(ParameterSpace hasBias){ + this.hasBias = hasBias; + return (T)this; + } + + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseLayerSpace.java new file mode 100644 index 000000000..255e76be5 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseLayerSpace.java @@ -0,0 +1,292 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import com.google.common.base.Preconditions; +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.adapter.ActivationParameterSpaceAdapter; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.Updater; +import org.deeplearning4j.nn.conf.distribution.Distribution; +import org.deeplearning4j.nn.conf.layers.BaseLayer; +import org.deeplearning4j.nn.conf.weightnoise.IWeightNoise; +import org.deeplearning4j.nn.weights.WeightInit; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.learning.config.IUpdater; +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.Map; + +/** + * BaseLayerSpace contains the common Layer hyperparameters; should match {@link BaseLayer} in terms of features + * + * @author Alex Black + */ +@JsonInclude(JsonInclude.Include.NON_NULL) + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public abstract class BaseLayerSpace extends LayerSpace { + protected ParameterSpace activationFunction; + protected ParameterSpace weightInit; + protected ParameterSpace biasInit; + protected ParameterSpace dist; + protected ParameterSpace l1; + protected ParameterSpace l2; + protected ParameterSpace l1Bias; + protected ParameterSpace l2Bias; + protected ParameterSpace updater; + protected ParameterSpace biasUpdater; + protected ParameterSpace weightNoise; + protected ParameterSpace gradientNormalization; + protected ParameterSpace gradientNormalizationThreshold; + protected int numParameters; + + @SuppressWarnings("unchecked") + protected BaseLayerSpace(Builder builder) { + super(builder); + this.activationFunction = builder.activationFunction; + this.weightInit = builder.weightInit; + this.biasInit = builder.biasInit; + this.dist = builder.dist; + this.l1 = builder.l1; + this.l2 = builder.l2; + this.l1Bias = builder.l1Bias; + this.l2Bias = builder.l2Bias; + this.updater = builder.updater; + this.biasUpdater = builder.biasUpdater; + this.weightNoise = builder.weightNoise; + this.gradientNormalization = builder.gradientNormalization; + this.gradientNormalizationThreshold = builder.gradientNormalizationThreshold; + } + + @Override + public int numParameters() { + return numParameters; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + throw new UnsupportedOperationException("Cannot set indices for non-leaf parameter space"); + } + + + protected void setLayerOptionsBuilder(BaseLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (activationFunction != null) + builder.activation(activationFunction.getValue(values)); + if (biasInit != null) + builder.biasInit(biasInit.getValue(values)); + if (weightInit != null) + builder.weightInit(weightInit.getValue(values)); + if (dist != null) + builder.dist(dist.getValue(values)); + if (l1 != null) + builder.l1(l1.getValue(values)); + if (l2 != null) + builder.l2(l2.getValue(values)); + if (l1Bias != null) + builder.l1Bias(l1Bias.getValue(values)); + if (l2Bias != null) + builder.l2Bias(l2Bias.getValue(values)); + if (updater != null) + builder.updater(updater.getValue(values)); + if (biasUpdater != null) + builder.biasUpdater(biasUpdater.getValue(values)); + if (weightNoise != null) + builder.weightNoise(weightNoise.getValue(values)); + if (gradientNormalization != null) + builder.gradientNormalization(gradientNormalization.getValue(values)); + if (gradientNormalizationThreshold != null) + builder.gradientNormalizationThreshold(gradientNormalizationThreshold.getValue(values)); + } + + + @Override + public String toString() { + return toString(", "); + } + + protected String toString(String delim) { + StringBuilder sb = new StringBuilder(); + + for (Map.Entry e : getNestedSpaces().entrySet()) { + sb.append(e.getKey()).append(": ").append(e.getValue()).append("\n"); + } + return sb.toString(); + } + + @SuppressWarnings("unchecked") + public abstract static class Builder extends LayerSpace.Builder { + protected ParameterSpace activationFunction; + protected ParameterSpace weightInit; + protected ParameterSpace biasInit; + protected ParameterSpace dist; + protected ParameterSpace l1; + protected ParameterSpace l2; + protected ParameterSpace l1Bias; + protected ParameterSpace l2Bias; + protected ParameterSpace updater; + protected ParameterSpace biasUpdater; + protected ParameterSpace weightNoise; + protected ParameterSpace gradientNormalization; + protected ParameterSpace gradientNormalizationThreshold; + + public T activation(Activation... activations){ + Preconditions.checkArgument(activations.length > 0, "Activations length must be 1 or more"); + if(activations.length == 1){ + return activation(activations[0]); + } + return activation(new DiscreteParameterSpace<>(activations)); + } + + public T activation(Activation activation) { + return activation(new FixedValue<>(activation)); + } + + public T activation(IActivation iActivation) { + return activationFn(new FixedValue<>(iActivation)); + } + + public T activation(ParameterSpace activationFunction) { + return activationFn(new ActivationParameterSpaceAdapter(activationFunction)); + } + + public T activationFn(ParameterSpace activationFunction) { + this.activationFunction = activationFunction; + return (T) this; + } + + public T weightInit(WeightInit weightInit) { + return (T) weightInit(new FixedValue(weightInit)); + } + + public T weightInit(ParameterSpace weightInit) { + this.weightInit = weightInit; + return (T) this; + } + + public T weightInit(Distribution distribution){ + weightInit(WeightInit.DISTRIBUTION); + return dist(distribution); + } + + public T biasInit(double biasInit){ + return biasInit(new FixedValue<>(biasInit)); + } + + public T biasInit(ParameterSpace biasInit){ + this.biasInit = biasInit; + return (T) this; + } + + public T dist(Distribution dist) { + return dist(new FixedValue<>(dist)); + } + + public T dist(ParameterSpace dist) { + this.dist = dist; + return (T) this; + } + + public T l1(double l1) { + return l1(new FixedValue(l1)); + } + + public T l1(ParameterSpace l1) { + this.l1 = l1; + return (T) this; + } + + public T l2(double l2) { + return l2(new FixedValue(l2)); + } + + public T l2(ParameterSpace l2) { + this.l2 = l2; + return (T) this; + } + + public T l1Bias(double l1Bias) { + return l1Bias(new FixedValue(l1Bias)); + } + + public T l1Bias(ParameterSpace l1Bias) { + this.l1Bias = l1Bias; + return (T) this; + } + + public T l2Bias(double l2Bias) { + return l2Bias(new FixedValue<>(l2Bias)); + } + + public T l2Bias(ParameterSpace l2Bias) { + this.l2Bias = l2Bias; + return (T) this; + } + + public T updater(IUpdater updater) { + return updater(new FixedValue<>(updater)); + } + + public T updater(ParameterSpace updater) { + this.updater = updater; + return (T) this; + } + + public T biasUpdater(IUpdater biasUpdater) { + return biasUpdater(new FixedValue<>(biasUpdater)); + } + + public T biasUpdater(ParameterSpace biasUpdater) { + this.biasUpdater = biasUpdater; + return (T) this; + } + + public T gradientNormalization(GradientNormalization gradientNormalization) { + return gradientNormalization(new FixedValue(gradientNormalization)); + } + + public T gradientNormalization(ParameterSpace gradientNormalization) { + this.gradientNormalization = gradientNormalization; + return (T) this; + } + + public T gradientNormalizationThreshold(double threshold) { + return gradientNormalizationThreshold(new FixedValue<>(threshold)); + } + + public T gradientNormalizationThreshold(ParameterSpace gradientNormalizationThreshold) { + this.gradientNormalizationThreshold = gradientNormalizationThreshold; + return (T) this; + } + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseOutputLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseOutputLayerSpace.java new file mode 100644 index 000000000..857f729ad --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BaseOutputLayerSpace.java @@ -0,0 +1,87 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.adapter.LossFunctionParameterSpaceAdapter; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.conf.layers.BaseOutputLayer; +import org.nd4j.linalg.lossfunctions.ILossFunction; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; + +/** + * @param Type of the (concrete) output layer + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PUBLIC) //For Jackson JSON/YAML deserialization +public abstract class BaseOutputLayerSpace extends FeedForwardLayerSpace { + + protected ParameterSpace lossFunction; + protected ParameterSpace hasBias; + + protected BaseOutputLayerSpace(Builder builder) { + super(builder); + this.lossFunction = builder.lossFunction; + this.hasBias = builder.hasBias; + } + + protected void setLayerOptionsBuilder(BaseOutputLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (lossFunction != null) + builder.lossFunction(lossFunction.getValue(values)); + if (hasBias != null) + builder.hasBias(hasBias.getValue(values)); + } + + @SuppressWarnings("unchecked") + public static abstract class Builder extends FeedForwardLayerSpace.Builder { + + protected ParameterSpace lossFunction; + protected ParameterSpace hasBias; + + public T lossFunction(LossFunction lossFunction) { + return lossFunction(new FixedValue<>(lossFunction)); + } + + public T lossFunction(ParameterSpace lossFunction) { + return iLossFunction(new LossFunctionParameterSpaceAdapter(lossFunction)); + } + + public T iLossFunction(ILossFunction lossFunction) { + return iLossFunction(new FixedValue<>(lossFunction)); + } + + public T iLossFunction(ParameterSpace lossFunction) { + this.lossFunction = lossFunction; + return (T) this; + } + + public T hasBias(boolean hasBias){ + return hasBias(new FixedValue<>(hasBias)); + } + + public T hasBias(ParameterSpace hasBias){ + this.hasBias = hasBias; + return (T)this; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BasePretrainNetworkLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BasePretrainNetworkLayerSpace.java new file mode 100644 index 000000000..9f554911b --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BasePretrainNetworkLayerSpace.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.conf.layers.BasePretrainNetwork; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; +import com.fasterxml.jackson.annotation.JsonProperty; + + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public abstract class BasePretrainNetworkLayerSpace extends FeedForwardLayerSpace { + @JsonProperty + protected ParameterSpace lossFunction; + + protected BasePretrainNetworkLayerSpace(Builder builder) { + super(builder); + this.lossFunction = builder.lossFunction; + } + + + public static abstract class Builder extends FeedForwardLayerSpace.Builder { + protected ParameterSpace lossFunction; + + public T lossFunction(LossFunction lossFunction) { + return lossFunction(new FixedValue(lossFunction)); + } + + public T lossFunction(ParameterSpace lossFunction) { + this.lossFunction = lossFunction; + return (T) this; + } + + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BatchNormalizationSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BatchNormalizationSpace.java new file mode 100644 index 000000000..9b55555ed --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/BatchNormalizationSpace.java @@ -0,0 +1,214 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.api.layers.LayerConstraint; +import org.deeplearning4j.nn.conf.layers.BatchNormalization; + +import java.util.Arrays; +import java.util.List; + +/** + * LayerSpace for batch normalization layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class BatchNormalizationSpace extends FeedForwardLayerSpace { + + protected ParameterSpace decay; + protected ParameterSpace eps; + protected ParameterSpace isMinibatch; + protected ParameterSpace lockGammaBeta; + protected ParameterSpace gamma; + protected ParameterSpace beta; + protected ParameterSpace> constrainBeta; + protected ParameterSpace> constrainGamma; + + private BatchNormalizationSpace(Builder builder) { + super(builder); + this.decay = builder.decay; + this.eps = builder.eps; + this.isMinibatch = builder.isMinibatch; + this.lockGammaBeta = builder.lockGammaBeta; + this.gamma = builder.gamma; + this.beta = builder.beta; + this.constrainBeta = builder.betaConstraints; + this.constrainGamma = builder.gammaConstraints; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public BatchNormalization getValue(double[] parameterValues) { + BatchNormalization.Builder b = new BatchNormalization.Builder(); + setLayerOptionsBuilder(b, parameterValues); + return b.build(); + } + + protected void setLayerOptionsBuilder(BatchNormalization.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (decay != null) + builder.decay(decay.getValue(values)); + if (eps != null) + builder.eps(eps.getValue(values)); + if (isMinibatch != null) + builder.minibatch(isMinibatch.getValue(values)); + if (lockGammaBeta != null) + builder.lockGammaBeta(lockGammaBeta.getValue(values)); + if (gamma != null) + builder.gamma(gamma.getValue(values)); + if (beta != null) + builder.beta(beta.getValue(values)); + if (constrainBeta != null){ + List c = constrainBeta.getValue(values); + if(c != null){ + builder.constrainBeta(c.toArray(new LayerConstraint[c.size()])); + } + } + if (constrainGamma != null){ + List c = constrainGamma.getValue(values); + if(c != null){ + builder.constrainGamma(c.toArray(new LayerConstraint[c.size()])); + } + } + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder(); + sb.append("BatchNormalizationSpace(").append(super.toString(delim)); + if (decay != null) + sb.append("decay: ").append(decay).append(delim); + if (eps != null) + sb.append("eps: ").append(eps).append(delim); + if (isMinibatch != null) + sb.append("isMinibatch: ").append(isMinibatch).append(delim); + if (lockGammaBeta != null) + sb.append("lockGammaBeta: ").append(lockGammaBeta).append(delim); + if (gamma != null) + sb.append("gamma: ").append(gamma).append(delim); + if (beta != null) + sb.append("beta: ").append(beta).append(delim); + sb.append(")"); + return sb.toString(); + } + + public static class Builder extends FeedForwardLayerSpace.Builder { + + protected ParameterSpace decay; + protected ParameterSpace eps; + protected ParameterSpace isMinibatch; + protected ParameterSpace lockGammaBeta; + protected ParameterSpace gamma; + protected ParameterSpace beta; + protected ParameterSpace> betaConstraints; + protected ParameterSpace> gammaConstraints; + + public Builder minibatch(boolean minibatch) { + return minibatch(new FixedValue<>(minibatch)); + } + + public Builder minibatch(ParameterSpace minibatch) { + this.isMinibatch = minibatch; + return this; + } + + public Builder gamma(double gamma) { + return gamma(new FixedValue<>(gamma)); + } + + public Builder gamma(ParameterSpace gamma) { + this.gamma = gamma; + return this; + } + + public Builder beta(double beta) { + return beta(new FixedValue<>(beta)); + } + + public Builder beta(ParameterSpace beta) { + this.beta = beta; + return this; + } + + public Builder eps(double eps) { + return eps(new FixedValue<>(eps)); + } + + public Builder eps(ParameterSpace eps) { + this.eps = eps; + return this; + } + + public Builder decay(double decay) { + return decay(new FixedValue(decay)); + } + + public Builder decay(ParameterSpace decay) { + this.decay = decay; + return this; + } + + public Builder lockGammaBeta(boolean lockGammaBeta) { + return lockGammaBeta(new FixedValue<>(lockGammaBeta)); + } + + public Builder lockGammaBeta(ParameterSpace lockGammaBeta) { + this.lockGammaBeta = lockGammaBeta; + return this; + } + + public Builder constrainBeta(LayerConstraint... constraints) { + return constrainBeta(new FixedValue<>(Arrays.asList(constraints))); + } + + public Builder constrainBeta(ParameterSpace> constraints) { + this.betaConstraints = constraints; + return this; + } + + public Builder constrainGamma(LayerConstraint... constraints) { + return constrainGamma(new FixedValue<>(Arrays.asList(constraints))); + } + + public Builder constrainGamma(ParameterSpace> constraints) { + this.gammaConstraints = constraints; + return this; + } + + + @Override + public BatchNormalizationSpace build() { + return new BatchNormalizationSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/Bidirectional.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/Bidirectional.java new file mode 100644 index 000000000..64cdfd369 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/Bidirectional.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.nn.conf.layers.Layer; + +import java.util.List; + +/** + * Bidirectional layer wrapper. Can be used wrap an existing layer space, in the same way that + * {@link org.deeplearning4j.nn.conf.layers.recurrent.Bidirectional} wraps a DL4J layer + * + * @author Alex Black + */ +@NoArgsConstructor //JSON +@Data +public class Bidirectional extends LayerSpace { + + protected LayerSpace layerSpace; + + public Bidirectional(LayerSpace layerSpace){ + this.layerSpace = layerSpace; + } + + @Override + public Layer getValue(double[] parameterValues) { + Layer underlying = layerSpace.getValue(parameterValues); + return new org.deeplearning4j.nn.conf.layers.recurrent.Bidirectional(underlying); + } + + @Override + public int numParameters() { + return layerSpace.numParameters(); + } + + @Override + public List collectLeaves() { + return layerSpace.collectLeaves(); + } + + @Override + public boolean isLeaf() { + return layerSpace.isLeaf(); + } + + @Override + public void setIndices(int... indices) { + layerSpace.setIndices(indices); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/CenterLossOutputLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/CenterLossOutputLayerSpace.java new file mode 100644 index 000000000..ecba732c3 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/CenterLossOutputLayerSpace.java @@ -0,0 +1,87 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public class CenterLossOutputLayerSpace extends BaseOutputLayerSpace { + + ParameterSpace alpha; + ParameterSpace lambda; + + protected CenterLossOutputLayerSpace(Builder builder){ + super(builder); + this.alpha = builder.alpha; + this.lambda = builder.lambda; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public CenterLossOutputLayer getValue(double[] parameterValues) { + CenterLossOutputLayer.Builder b = new CenterLossOutputLayer.Builder(); + setLayerOptionsBuilder(b, parameterValues); + return b.build(); + } + + protected void setLayerBuilderOptions(CenterLossOutputLayer.Builder builder, double[] values){ + super.setLayerOptionsBuilder(builder, values); + if(alpha != null) + builder.alpha(alpha.getValue(values)); + if(lambda != null) + builder.lambda(lambda.getValue(values)); + } + + public static class Builder extends BaseOutputLayerSpace.Builder { + + ParameterSpace alpha; + ParameterSpace lambda; + + public Builder alpha(double alpha){ + return alpha(new FixedValue<>(alpha)); + } + + public Builder alpha(ParameterSpace alpha){ + this.alpha = alpha; + return this; + } + + public Builder lambda(double lambda){ + return lambda(new FixedValue<>(lambda)); + } + + public Builder lambda(ParameterSpace lambda){ + this.lambda = lambda; + return this; + } + + @Override + public CenterLossOutputLayerSpace build() { + return new CenterLossOutputLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/ConvolutionLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/ConvolutionLayerSpace.java new file mode 100644 index 000000000..110e5b6e7 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/ConvolutionLayerSpace.java @@ -0,0 +1,172 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; + +/** + * Layer space for convolutional layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class ConvolutionLayerSpace extends FeedForwardLayerSpace { + protected ParameterSpace dilation; + protected ParameterSpace kernelSize; + protected ParameterSpace stride; + protected ParameterSpace padding; + protected ParameterSpace convolutionMode; + protected ParameterSpace hasBias; + + private ConvolutionLayerSpace(Builder builder) { + super(builder); + this.dilation = builder.dilation; + this.kernelSize = builder.kernelSize; + this.stride = builder.stride; + this.padding = builder.padding; + this.convolutionMode = builder.convolutionMode; + this.hasBias = builder.hasBias; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public ConvolutionLayer getValue(double[] values) { + ConvolutionLayer.Builder b = new ConvolutionLayer.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(ConvolutionLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (dilation != null) + builder.dilation(dilation.getValue(values)); + if (kernelSize != null) + builder.kernelSize(kernelSize.getValue(values)); + if (stride != null) + builder.stride(stride.getValue(values)); + if (padding != null) + builder.padding(padding.getValue(values)); + if (convolutionMode != null) + builder.convolutionMode(convolutionMode.getValue(values)); + if (hasBias != null) + builder.hasBias(hasBias.getValue(values)); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder("ConvolutionLayerSpace("); + if (dilation != null) + sb.append("dilation: ").append(dilation).append(delim); + if (kernelSize != null) + sb.append("kernelSize: ").append(kernelSize).append(delim); + if (stride != null) + sb.append("stride: ").append(stride).append(delim); + if (padding != null) + sb.append("padding: ").append(padding).append(delim); + if (convolutionMode != null) + sb.append("convolutionMode: ").append(convolutionMode).append(delim); + if (hasBias != null) + sb.append("hasBias: ").append(hasBias).append(delim); + sb.append(super.toString(delim)).append(")"); + return sb.toString(); + } + + + public static class Builder extends FeedForwardLayerSpace.Builder { + protected ParameterSpace dilation; + protected ParameterSpace kernelSize; + protected ParameterSpace stride; + protected ParameterSpace padding; + protected ParameterSpace convolutionMode; + protected ParameterSpace hasBias; + + public Builder dilation(int... dilation) { + return dilation(new FixedValue<>(dilation)); + } + + public Builder dilation(ParameterSpace dilation) { + this.dilation = dilation; + return this; + } + public Builder kernelSize(int... kernelSize) { + return kernelSize(new FixedValue<>(kernelSize)); + } + + public Builder kernelSize(ParameterSpace kernelSize) { + this.kernelSize = kernelSize; + return this; + } + + public Builder stride(int... stride) { + return stride(new FixedValue<>(stride)); + } + + public Builder stride(ParameterSpace stride) { + this.stride = stride; + return this; + } + + public Builder padding(int... padding) { + return padding(new FixedValue<>(padding)); + } + + public Builder padding(ParameterSpace padding) { + this.padding = padding; + return this; + } + + public Builder convolutionMode(ConvolutionMode convolutionMode) { + return convolutionMode(new FixedValue<>(convolutionMode)); + } + + public Builder convolutionMode(ParameterSpace convolutionMode) { + this.convolutionMode = convolutionMode; + return this; + } + + public Builder hasBias(boolean hasBias){ + return hasBias(new FixedValue<>(hasBias)); + } + + public Builder hasBias(ParameterSpace hasBias){ + this.hasBias = hasBias; + return this; + } + + public ConvolutionLayerSpace build() { + return new ConvolutionLayerSpace(this); + } + + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/Deconvolution2DLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/Deconvolution2DLayerSpace.java new file mode 100644 index 000000000..72231f246 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/Deconvolution2DLayerSpace.java @@ -0,0 +1,52 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.nn.conf.layers.Deconvolution2D; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public class Deconvolution2DLayerSpace extends BaseConvolutionLayerSpace { + + protected Deconvolution2DLayerSpace(Builder builder) { + super(builder); + } + + @Override + public Deconvolution2D getValue(double[] parameterValues) { + Deconvolution2D.Builder b = new Deconvolution2D.Builder(); + setLayerOptionsBuilder(b, parameterValues); + return b.build(); + } + + protected void setLayerOptionsBuilder(Deconvolution2D.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + } + + public static class Builder extends BaseConvolutionLayerSpace.Builder { + @Override + public Deconvolution2DLayerSpace build() { + return new Deconvolution2DLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/DenseLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/DenseLayerSpace.java new file mode 100644 index 000000000..4a7ac3f28 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/DenseLayerSpace.java @@ -0,0 +1,90 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.DenseLayer; + +/** + * layer hyperparameter configuration space for dense layers (i.e., multi-layer perceptron layers) + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor //For Jackson JSON/YAML deserialization +public class DenseLayerSpace extends FeedForwardLayerSpace { + + protected ParameterSpace hasBias; + + private DenseLayerSpace(Builder builder) { + super(builder); + + this.hasBias = builder.hasBias; + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public DenseLayer getValue(double[] values) { + //Using the builder here, to get default options + DenseLayer.Builder b = new DenseLayer.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(DenseLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if(hasBias != null) + builder.hasBias(hasBias.getValue(values)); + } + + public static class Builder extends FeedForwardLayerSpace.Builder { + + protected ParameterSpace hasBias; + + public Builder hasBias(boolean hasBias){ + return hasBias(new FixedValue<>(hasBias)); + } + + public Builder hasBias(ParameterSpace hasBias){ + this.hasBias = hasBias; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public DenseLayerSpace build() { + return new DenseLayerSpace(this); + } + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + return "DenseLayerSpace(" + super.toString(delim) + ")"; + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/DropoutLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/DropoutLayerSpace.java new file mode 100644 index 000000000..1e6ca7157 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/DropoutLayerSpace.java @@ -0,0 +1,89 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.*; +import org.deeplearning4j.arbiter.dropout.DropoutSpace; +import org.deeplearning4j.arbiter.optimize.api.AbstractParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.conf.dropout.IDropout; +import org.deeplearning4j.nn.conf.layers.DropoutLayer; + +import java.util.Collections; +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public class DropoutLayerSpace extends LayerSpace { + + public DropoutLayerSpace(@NonNull ParameterSpace dropout){ + this.dropOut = dropout; + } + + protected DropoutLayerSpace(Builder builder){ + super(builder); + } + + @Override + public DropoutLayer getValue(double[] parameterValues) { + return new DropoutLayer.Builder().dropOut(dropOut.getValue(parameterValues)).build(); + } + + @Override + public int numParameters() { + return dropOut.numParameters(); + } + + @Override + public List collectLeaves() { + return dropOut.collectLeaves(); + } + + + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + dropOut.setIndices(indices); + } + + public static class Builder extends LayerSpace.Builder { + + public Builder dropOut(double d){ + return iDropOut(new DropoutSpace(new FixedValue<>(d))); + } + + public Builder dropOut(ParameterSpace dropOut){ + return iDropOut(new DropoutSpace(dropOut)); + } + + public Builder iDropOut(ParameterSpace dropout){ + this.dropOut = dropout; + return this; + } + + public DropoutLayerSpace build(){ + return new DropoutLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/EmbeddingLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/EmbeddingLayerSpace.java new file mode 100644 index 000000000..7aa5c5444 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/EmbeddingLayerSpace.java @@ -0,0 +1,88 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.EmbeddingLayer; + +/** + * Layer hyperparameter configuration space for {@link org.deeplearning4j.nn.conf.layers.EmbeddingLayer} + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class EmbeddingLayerSpace extends FeedForwardLayerSpace { + private ParameterSpace hasBias; + + private EmbeddingLayerSpace(Builder builder) { + super(builder); + this.hasBias = builder.hasBias; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public EmbeddingLayer getValue(double[] values) { + //Using the builder here, to get default options + EmbeddingLayer.Builder b = new EmbeddingLayer.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(EmbeddingLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if(hasBias != null) + builder.hasBias(hasBias.getValue(values)); + } + + public static class Builder extends FeedForwardLayerSpace.Builder { + protected ParameterSpace hasBias; + + public Builder hasBias(boolean hasBias){ + return hasBias(new FixedValue<>(hasBias)); + } + + public Builder hasBias(ParameterSpace hasBias){ + this.hasBias = hasBias; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public EmbeddingLayerSpace build() { + return new EmbeddingLayerSpace(this); + } + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + return "EmbeddingLayerSpace(" + super.toString(delim) + ")"; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/FeedForwardLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/FeedForwardLayerSpace.java new file mode 100644 index 000000000..3ba0f3a06 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/FeedForwardLayerSpace.java @@ -0,0 +1,154 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.api.layers.LayerConstraint; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; + +import java.util.Arrays; +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor //For Jackson JSON/YAML deserialization +public abstract class FeedForwardLayerSpace extends BaseLayerSpace { + protected ParameterSpace nIn; + protected ParameterSpace nOut; + protected ParameterSpace> constrainWeights; + protected ParameterSpace> constrainBias; + protected ParameterSpace> constrainAll; + + + protected FeedForwardLayerSpace(Builder builder) { + super(builder); + nIn = builder.nIn; + nOut = builder.nOut; + constrainWeights = builder.constrainWeights; + constrainBias = builder.constrainBias; + constrainAll = builder.constrainAll; + } + + protected void setLayerOptionsBuilder(FeedForwardLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (nIn != null) + builder.nIn(nIn.getValue(values)); + if (nOut != null) + builder.nOut(nOut.getValue(values)); + if (constrainWeights != null){ + List c = constrainWeights.getValue(values); + if(c != null){ + builder.constrainWeights(c.toArray(new LayerConstraint[c.size()])); + } + } + if (constrainBias != null){ + List c = constrainBias.getValue(values); + if(c != null){ + builder.constrainBias(c.toArray(new LayerConstraint[c.size()])); + } + } + if (constrainAll != null){ + List c = constrainAll.getValue(values); + if(c != null){ + builder.constrainAllParameters(c.toArray(new LayerConstraint[c.size()])); + } + } + + } + + + public abstract static class Builder extends BaseLayerSpace.Builder { + + protected ParameterSpace nIn; + protected ParameterSpace nOut; + protected ParameterSpace> constrainWeights; + protected ParameterSpace> constrainBias; + protected ParameterSpace> constrainAll; + + public T nIn(int nIn) { + return nIn(new FixedValue<>(nIn)); + } + + public T nIn(ParameterSpace nIn) { + this.nIn = nIn; + return (T) this; + } + + public T nOut(int nOut) { + return nOut(new FixedValue<>(nOut)); + } + + public T nOut(ParameterSpace nOut) { + this.nOut = nOut; + return (T) this; + } + + public T constrainWeights(LayerConstraint... constraints){ + return constrainWeights(new FixedValue>(Arrays.asList(constraints))); + } + + public T constrainWeights(ParameterSpace> constraints){ + this.constrainWeights = constraints; + return (T) this; + } + + public T constrainBias(LayerConstraint... constraints){ + return constrainBias(new FixedValue>(Arrays.asList(constraints))); + } + + public T constrainBias(ParameterSpace> constraints){ + this.constrainBias = constraints; + return (T) this; + } + + public T constrainAllParams(LayerConstraint... constraints){ + return constrainAllParams(new FixedValue>(Arrays.asList(constraints))); + } + + public T constrainAllParams(ParameterSpace> constraints){ + this.constrainAll = constraints; + return (T) this; + } + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + protected String toString(String delim) { + StringBuilder sb = new StringBuilder(); + if (nIn != null) + sb.append("nIn: ").append(nIn).append(delim); + if (nOut != null) + sb.append("nOut: ").append(nOut).append(delim); + if (constrainWeights != null) + sb.append("constrainWeights: ").append(constrainWeights).append(delim); + if (constrainBias != null) + sb.append("constrainBias: ").append(constrainBias).append(delim); + if (constrainAll != null) + sb.append("constrainAllParams: ").append(constrainAll).append(delim); + sb.append(super.toString(delim)); + return sb.toString(); + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GlobalPoolingLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GlobalPoolingLayerSpace.java new file mode 100644 index 000000000..17bd22103 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GlobalPoolingLayerSpace.java @@ -0,0 +1,135 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer; +import org.deeplearning4j.nn.conf.layers.PoolingType; + +/** + * Layer space for a {@link GlobalPoolingLayer} + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class GlobalPoolingLayerSpace extends LayerSpace { + + protected ParameterSpace poolingDimensions; + protected ParameterSpace collapseDimensions; + protected ParameterSpace poolingType; + protected ParameterSpace pNorm; + + private int numParameters; + + private GlobalPoolingLayerSpace(Builder builder) { + super(builder); + this.poolingDimensions = builder.poolingDimensions; + this.collapseDimensions = builder.collapseDimensions; + this.poolingType = builder.poolingType; + this.pNorm = builder.pNorm; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public GlobalPoolingLayer getValue(double[] parameterValues) { + GlobalPoolingLayer.Builder builder = new GlobalPoolingLayer.Builder(); + super.setLayerOptionsBuilder(builder, parameterValues); + if (poolingDimensions != null) + builder.poolingDimensions(poolingDimensions.getValue(parameterValues)); + if (collapseDimensions != null) + builder.collapseDimensions(collapseDimensions.getValue(parameterValues)); + if (poolingType != null) + builder.poolingType(poolingType.getValue(parameterValues)); + if (pNorm != null) + builder.pnorm(pNorm.getValue(parameterValues)); + return builder.build(); + } + + @Override + public int numParameters() { + return numParameters; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + throw new UnsupportedOperationException("Cannot set indices for non-leaf parameter space"); + } + + + + public static class Builder extends LayerSpace.Builder { + + protected ParameterSpace poolingDimensions; + protected ParameterSpace collapseDimensions; + protected ParameterSpace poolingType; + protected ParameterSpace pNorm; + + public Builder poolingDimensions(int... poolingDimensions) { + return poolingDimensions(new FixedValue<>(poolingDimensions)); + } + + public Builder poolingDimensions(ParameterSpace poolingDimensions) { + this.poolingDimensions = poolingDimensions; + return this; + } + + public Builder collapseDimensions(boolean collapseDimensions) { + return collapseDimensions(new FixedValue<>(collapseDimensions)); + } + + public Builder collapseDimensions(ParameterSpace collapseDimensions) { + this.collapseDimensions = collapseDimensions; + return this; + } + + public Builder poolingType(PoolingType poolingType) { + return poolingType(new FixedValue<>(poolingType)); + } + + public Builder poolingType(ParameterSpace poolingType) { + this.poolingType = poolingType; + return this; + } + + public Builder pNorm(int pNorm) { + return pNorm(new FixedValue<>(pNorm)); + } + + public Builder pNorm(ParameterSpace pNorm) { + this.pNorm = pNorm; + return this; + } + + public GlobalPoolingLayerSpace build() { + return new GlobalPoolingLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GravesBidirectionalLSTMLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GravesBidirectionalLSTMLayerSpace.java new file mode 100644 index 000000000..e42deacbe --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GravesBidirectionalLSTMLayerSpace.java @@ -0,0 +1,97 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM; + +import java.util.List; + +/** + * Layer space for Bidirectional LSTM layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class GravesBidirectionalLSTMLayerSpace extends FeedForwardLayerSpace { + + private ParameterSpace forgetGateBiasInit; + + private GravesBidirectionalLSTMLayerSpace(Builder builder) { + super(builder); + this.forgetGateBiasInit = builder.forgetGateBiasInit; + + List l = collectLeaves(); + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + + @Override + public GravesBidirectionalLSTM getValue(double[] values) { + GravesBidirectionalLSTM.Builder b = new GravesBidirectionalLSTM.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(GravesBidirectionalLSTM.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (forgetGateBiasInit != null) + builder.forgetGateBiasInit(forgetGateBiasInit.getValue(values)); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder("GravesBidirectionalLSTMLayerSpace("); + if (forgetGateBiasInit != null) + sb.append("forgetGateBiasInit: ").append(forgetGateBiasInit).append(delim); + sb.append(super.toString(delim)).append(")"); + return sb.toString(); + } + + public static class Builder extends FeedForwardLayerSpace.Builder { + + private ParameterSpace forgetGateBiasInit; + + public Builder forgetGateBiasInit(double forgetGateBiasInit) { + return forgetGateBiasInit(new FixedValue<>(forgetGateBiasInit)); + } + + public Builder forgetGateBiasInit(ParameterSpace forgetGateBiasInit) { + this.forgetGateBiasInit = forgetGateBiasInit; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public GravesBidirectionalLSTMLayerSpace build() { + return new GravesBidirectionalLSTMLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GravesLSTMLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GravesLSTMLayerSpace.java new file mode 100644 index 000000000..9707836fa --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/GravesLSTMLayerSpace.java @@ -0,0 +1,76 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.GravesLSTM; + +/** + * Layer space for LSTM layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class GravesLSTMLayerSpace extends AbstractLSTMLayerSpace { + + private GravesLSTMLayerSpace(Builder builder) { + super(builder); + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + + @Override + public GravesLSTM getValue(double[] values) { + GravesLSTM.Builder b = new GravesLSTM.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(GravesLSTM.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder("GravesLSTMLayerSpace("); + sb.append(super.toString(delim)).append(")"); + return sb.toString(); + } + + public static class Builder extends AbstractLSTMLayerSpace.Builder { + + @Override + @SuppressWarnings("unchecked") + public GravesLSTMLayerSpace build() { + return new GravesLSTMLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LSTMLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LSTMLayerSpace.java new file mode 100644 index 000000000..10e297134 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LSTMLayerSpace.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.GravesLSTM; +import org.deeplearning4j.nn.conf.layers.LSTM; + +/** + * Layer space for LSTM layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class LSTMLayerSpace extends AbstractLSTMLayerSpace { + + private LSTMLayerSpace(Builder builder) { + super(builder); + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + + @Override + public LSTM getValue(double[] values) { + LSTM.Builder b = new LSTM.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(LSTM.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder("LSTMLayerSpace("); + sb.append(super.toString(delim)).append(")"); + return sb.toString(); + } + + public static class Builder extends AbstractLSTMLayerSpace.Builder { + + @Override + @SuppressWarnings("unchecked") + public LSTMLayerSpace build() { + return new LSTMLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LayerSpace.java new file mode 100644 index 000000000..eb77196d2 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LayerSpace.java @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.dropout.DropoutSpace; +import org.deeplearning4j.arbiter.optimize.api.AbstractParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.conf.dropout.IDropout; +import org.deeplearning4j.nn.conf.layers.Layer; +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * LayerSpace contains common Layer hyperparameters; should match {@link Layer} in terms of features + * + * @author Alex Black + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@Data +@EqualsAndHashCode(callSuper = false) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public abstract class LayerSpace extends AbstractParameterSpace { + protected ParameterSpace dropOut; + protected int numParameters; + + protected LayerSpace(Builder builder) { + this.dropOut = builder.dropOut; + } + + @Override + public List collectLeaves() { + //To avoid manually coding EVERY parameter, in every layer: + // Do a depth-first search of nested spaces + LinkedList stack = new LinkedList<>(); + stack.add(this); + + List out = new ArrayList<>(); + while (!stack.isEmpty()) { + ParameterSpace next = stack.removeLast(); + if (next.isLeaf()) { + out.add(next); + } else { + Map m = next.getNestedSpaces(); + ParameterSpace[] arr = m.values().toArray(new ParameterSpace[m.size()]); + for (int i = arr.length - 1; i >= 0; i--) { + stack.add(arr[i]); + } + } + } + + return out; + } + + @Override + public int numParameters() { + return numParameters; + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void setIndices(int... indices) { + throw new UnsupportedOperationException("Cannot set indices for non-leaf parameter space"); + } + + + protected void setLayerOptionsBuilder(Layer.Builder builder, double[] values) { + if (dropOut != null) + builder.dropOut(dropOut.getValue(values)); + } + + + @Override + public String toString() { + return toString(", "); + } + + protected String toString(String delim) { + StringBuilder sb = new StringBuilder(); + if (dropOut != null) + sb.append("dropOut: ").append(dropOut).append(delim); + String s = sb.toString(); + + if (s.endsWith(delim)) { + //Remove final delimiter + int last = s.lastIndexOf(delim); + return s.substring(0, last); + } else + return s; + } + + @SuppressWarnings("unchecked") + public abstract static class Builder { + protected ParameterSpace dropOut; + + public T dropOut(double dropOut) { + return dropOut(new FixedValue<>(dropOut)); + } + + public T dropOut(ParameterSpace dropOut) { + return iDropOut(new DropoutSpace(dropOut)); + } + + public T iDropOut(ParameterSpace dropOut){ + this.dropOut = dropOut; + return (T) this; + } + + public abstract E build(); + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LocalResponseNormalizationLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LocalResponseNormalizationLayerSpace.java new file mode 100644 index 000000000..eeeb5837f --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LocalResponseNormalizationLayerSpace.java @@ -0,0 +1,119 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.LocalResponseNormalization; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class LocalResponseNormalizationLayerSpace extends LayerSpace { + + private ParameterSpace n; + private ParameterSpace k; + private ParameterSpace alpha; + private ParameterSpace beta; + + + private LocalResponseNormalizationLayerSpace(Builder builder) { + super(builder); + this.n = builder.n; + this.k = builder.k; + this.alpha = builder.alpha; + this.beta = builder.beta; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public LocalResponseNormalization getValue(double[] values) { + LocalResponseNormalization.Builder b = new LocalResponseNormalization.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(LocalResponseNormalization.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (n != null) + builder.n(n.getValue(values)); + if (k != null) + builder.k(k.getValue(values)); + if (alpha != null) + builder.alpha(alpha.getValue(values)); + if (beta != null) + builder.beta(beta.getValue(values)); + } + + + public static class Builder extends LayerSpace.Builder { + + private ParameterSpace n; + private ParameterSpace k; + private ParameterSpace alpha; + private ParameterSpace beta; + + + public Builder n(double n) { + return n(new FixedValue<>(n)); + } + + public Builder n(ParameterSpace n) { + this.n = n; + return this; + } + + public Builder k(double k) { + return k(new FixedValue<>(k)); + } + + public Builder k(ParameterSpace k) { + this.k = k; + return this; + } + + public Builder alpha(double alpha) { + return alpha(new FixedValue<>(alpha)); + } + + public Builder alpha(ParameterSpace alpha) { + this.alpha = alpha; + return this; + } + + public Builder beta(double beta) { + return beta(new FixedValue<>(beta)); + } + + public Builder beta(ParameterSpace beta) { + this.beta = beta; + return this; + } + + public LocalResponseNormalizationLayerSpace build() { + return new LocalResponseNormalizationLayerSpace(this); + } + + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LossLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LossLayerSpace.java new file mode 100644 index 000000000..fc0b8c4d1 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/LossLayerSpace.java @@ -0,0 +1,105 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.adapter.ActivationParameterSpaceAdapter; +import org.deeplearning4j.arbiter.adapter.LossFunctionParameterSpaceAdapter; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.LossLayer; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.lossfunctions.ILossFunction; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public class LossLayerSpace extends LayerSpace { + + private ParameterSpace activationFunction; + protected ParameterSpace lossFunction; + + public LossLayerSpace(Builder builder){ + super(builder); + this.activationFunction = builder.activationFunction; + this.lossFunction = builder.lossFunction; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public LossLayer getValue(double[] parameterValues) { + LossLayer.Builder b = new LossLayer.Builder(); + if(activationFunction != null) + b.activation(activationFunction.getValue(parameterValues)); + if(lossFunction != null) + b.lossFunction(lossFunction.getValue(parameterValues)); + return b.build(); + } + + + public static class Builder extends LayerSpace.Builder{ + + private ParameterSpace activationFunction; + protected ParameterSpace lossFunction; + + public Builder lossFunction(LossFunctions.LossFunction lossFunction) { + return lossFunction(new FixedValue<>(lossFunction)); + } + + public Builder lossFunction(ParameterSpace lossFunction) { + return iLossFunction(new LossFunctionParameterSpaceAdapter(lossFunction)); + } + + public Builder iLossFunction(ILossFunction lossFunction) { + return iLossFunction(new FixedValue<>(lossFunction)); + } + + public Builder iLossFunction(ParameterSpace lossFunction) { + this.lossFunction = lossFunction; + return this; + } + + public Builder activation(Activation activation) { + return activation(new FixedValue<>(activation)); + } + + public Builder activation(IActivation iActivation) { + return activationFn(new FixedValue<>(iActivation)); + } + + public Builder activation(ParameterSpace activationFunction) { + return activationFn(new ActivationParameterSpaceAdapter(activationFunction)); + } + + public Builder activationFn(ParameterSpace activationFunction) { + this.activationFunction = activationFunction; + return this; + } + + @Override + public LossLayerSpace build() { + return new LossLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/OCNNLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/OCNNLayerSpace.java new file mode 100644 index 000000000..d4fc9553b --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/OCNNLayerSpace.java @@ -0,0 +1,153 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer; + + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class OCNNLayerSpace extends BaseOutputLayerSpace { + + + protected ParameterSpace nuSpace; + protected ParameterSpace initialRValue; + protected ParameterSpace hiddenLayerSize; + protected ParameterSpace windowSize; + protected ParameterSpace configureR; + + private OCNNLayerSpace(Builder builder) { + super(builder); + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + this.nuSpace = builder.nuSpace; + this.initialRValue = builder.initialRValue; + this.hiddenLayerSize = builder.hiddenLayerSize; + this.configureR = builder.configureR; + } + + + @Override + public OCNNOutputLayer getValue(double[] parameterValues) { + OCNNOutputLayer.Builder o = new OCNNOutputLayer.Builder(); + setLayerOptionsBuilder(o, parameterValues); + return o.build(); + } + + protected void setLayerOptionsBuilder(OCNNOutputLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + builder.nu(nuSpace.getValue(values)); + builder.hiddenLayerSize(hiddenLayerSize.getValue(values)); + builder.initialRValue(initialRValue.getValue(values)); + builder.configureR(configureR.getValue(values)); + builder.windowSize(windowSize.getValue(values)); + } + + + public static class Builder extends BaseOutputLayerSpace.Builder { + protected ParameterSpace nuSpace; + protected ParameterSpace initialRValue; + protected ParameterSpace hiddenLayerSize; + protected ParameterSpace windowSize; + protected ParameterSpace configureR; + + public Builder nu(ParameterSpace nuSpace) { + this.nuSpace = nuSpace; + return this; + } + + /** + * Use hiddenLayerSize instead + * @param numHiddenSpace + * @return + */ + @Deprecated + public Builder numHidden(ParameterSpace numHiddenSpace) { + return hiddenLayerSize(numHiddenSpace); + } + + /** + * Use hiddenLayerSize instead + * @param numHidden + * @return + */ + @Deprecated + public Builder numHidden(int numHidden) { + return hiddenLayerSize(numHidden); + } + + public Builder hiddenLayerSize(ParameterSpace hiddenLayerSize) { + this.hiddenLayerSize = hiddenLayerSize; + return this; + } + + public Builder hiddenLayerSize(int hiddenLayerSize) { + this.hiddenLayerSize = new FixedValue<>(hiddenLayerSize); + return this; + } + + public Builder nu(double nu) { + this.nuSpace = new FixedValue<>(nu); + return this; + } + + public Builder initialRValue(double initialRValue) { + this.initialRValue = new FixedValue<>(initialRValue); + return this; + } + + public Builder initialRValue(ParameterSpace initialRValue) { + this.initialRValue = initialRValue; + return this; + } + + public Builder windowSize(int windowSize) { + this.windowSize = new FixedValue<>(windowSize); + return this; + } + + public Builder windowSize(ParameterSpace windowSize) { + this.windowSize = windowSize; + return this; + } + + public Builder configureR(boolean configureR) { + this.configureR = new FixedValue<>(configureR); + return this; + } + + public Builder configureR(ParameterSpace configureR) { + this.configureR = configureR; + return this; + } + + + @Override + @SuppressWarnings("unchecked") + public OCNNLayerSpace build() { + return new OCNNLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/OutputLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/OutputLayerSpace.java new file mode 100644 index 000000000..5e6479fce --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/OutputLayerSpace.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.OutputLayer; + +/** + * Layer hyperparameter configuration space for output layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class OutputLayerSpace extends BaseOutputLayerSpace { + + private OutputLayerSpace(Builder builder) { + super(builder); + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public OutputLayer getValue(double[] values) { + OutputLayer.Builder o = new OutputLayer.Builder(); + setLayerOptionsBuilder(o, values); + return o.build(); + } + + protected void setLayerOptionsBuilder(OutputLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + } + + public static class Builder extends BaseOutputLayerSpace.Builder { + + @Override + @SuppressWarnings("unchecked") + public OutputLayerSpace build() { + return new OutputLayerSpace(this); + } + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + return "OutputLayerSpace(" + super.toString(delim) + ")"; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/RnnOutputLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/RnnOutputLayerSpace.java new file mode 100644 index 000000000..4fba80d81 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/RnnOutputLayerSpace.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; + +/** + * Layer hyperparametor configuration space for RnnOutputLayer + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class RnnOutputLayerSpace extends BaseOutputLayerSpace { + + private RnnOutputLayerSpace(Builder builder) { + super(builder); + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public RnnOutputLayer getValue(double[] values) { + RnnOutputLayer.Builder b = new RnnOutputLayer.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(RnnOutputLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + return "RnnOutputLayerSpace(" + super.toString(delim) + ")"; + } + + public static class Builder extends BaseOutputLayerSpace.Builder { + + @Override + @SuppressWarnings("unchecked") + public RnnOutputLayerSpace build() { + return new RnnOutputLayerSpace(this); + } + } + + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/SeparableConvolution2DLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/SeparableConvolution2DLayerSpace.java new file mode 100644 index 000000000..64a0d26a6 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/SeparableConvolution2DLayerSpace.java @@ -0,0 +1,101 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.nn.api.layers.LayerConstraint; +import org.deeplearning4j.nn.conf.layers.SeparableConvolution2D; + +import java.util.Arrays; +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For Jackson JSON/YAML deserialization +public class SeparableConvolution2DLayerSpace extends BaseConvolutionLayerSpace { + + private ParameterSpace depthMultiplier; + protected ParameterSpace> pointWiseConstraints; + + protected SeparableConvolution2DLayerSpace(Builder builder){ + super(builder); + this.depthMultiplier = builder.depthMultiplier; + this.pointWiseConstraints = builder.pointWiseConstraints; + } + + @Override + public SeparableConvolution2D getValue(double[] parameterValues) { + SeparableConvolution2D.Builder b = new SeparableConvolution2D.Builder(); + setLayerOptionsBuilder(b, parameterValues); + return b.build(); + } + + protected void setLayerOptionsBuilder(SeparableConvolution2D.Builder builder, double[] values){ + super.setLayerOptionsBuilder(builder, values); + if (kernelSize != null) + builder.kernelSize(kernelSize.getValue(values)); + if (stride != null) + builder.stride(stride.getValue(values)); + if (padding != null) + builder.padding(padding.getValue(values)); + if (convolutionMode != null) + builder.convolutionMode(convolutionMode.getValue(values)); + if (hasBias != null) + builder.hasBias(hasBias.getValue(values)); + if (depthMultiplier != null) + builder.depthMultiplier(depthMultiplier.getValue(values)); + if (pointWiseConstraints != null){ + List c = pointWiseConstraints.getValue(values); + if(c != null){ + builder.constrainPointWise(c.toArray(new LayerConstraint[c.size()])); + } + } + } + + + public static class Builder extends BaseConvolutionLayerSpace.Builder{ + private ParameterSpace depthMultiplier; + protected ParameterSpace> pointWiseConstraints; + + public Builder constrainPointWise(LayerConstraint... constraints){ + return constrainPointWise(new FixedValue>(Arrays.asList(constraints))); + } + + public Builder constrainPointWise(ParameterSpace> constraints){ + this.pointWiseConstraints = constraints; + return this; + } + + public Builder depthMultiplier(int depthMultiplier){ + return depthMultiplier(new FixedValue<>(depthMultiplier)); + } + + public Builder depthMultiplier(ParameterSpace depthMultiplier){ + this.depthMultiplier = depthMultiplier; + return this; + } + + public SeparableConvolution2DLayerSpace build(){ + return new SeparableConvolution2DLayerSpace(this); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/SubsamplingLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/SubsamplingLayerSpace.java new file mode 100644 index 000000000..5f1e32dab --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/SubsamplingLayerSpace.java @@ -0,0 +1,208 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; + +/** + * Layer hyperparameter configuration space for subsampling layers + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class SubsamplingLayerSpace extends LayerSpace { + + protected ParameterSpace convolutionMode; + protected ParameterSpace poolingType; + protected ParameterSpace dilation; + protected ParameterSpace kernelSize; + protected ParameterSpace stride; + protected ParameterSpace padding; + protected ParameterSpace pnorm; + protected ParameterSpace eps; + + private SubsamplingLayerSpace(Builder builder) { + super(builder); + this.convolutionMode = builder.convolutionMode; + this.poolingType = builder.poolingType; + this.kernelSize = builder.kernelSize; + this.dilation = builder.dilation; + this.stride = builder.stride; + this.padding = builder.padding; + this.pnorm = builder.pnorm; + this.eps = builder.eps; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public SubsamplingLayer getValue(double[] values) { + SubsamplingLayer.Builder b = new SubsamplingLayer.Builder(); + setLayerOptionsBuilder(b, values); + return b.build(); + } + + protected void setLayerOptionsBuilder(SubsamplingLayer.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (convolutionMode != null) + builder.convolutionMode(convolutionMode.getValue(values)); + if (poolingType != null) + builder.poolingType(poolingType.getValue(values)); + if (dilation !=null) + builder.dilation(dilation.getValue(values)); + if (kernelSize != null) + builder.kernelSize(kernelSize.getValue(values)); + if (stride != null) + builder.stride(stride.getValue(values)); + if (padding != null) + builder.padding(padding.getValue(values)); + if(pnorm != null) + builder.pnorm(pnorm.getValue(values)); + if(eps != null) + builder.eps(eps.getValue(values)); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder("SubsamplingLayerSpace("); + if (convolutionMode != null) + sb.append("convolutionMode: ").append(convolutionMode).append(delim); + if (poolingType != null) + sb.append("poolingType: ").append(poolingType).append(delim); + if (dilation != null) + sb.append("dilation: ").append(dilation).append(delim); + if (kernelSize != null) + sb.append("kernelSize: ").append(kernelSize).append(delim); + if (stride != null) + sb.append("stride: ").append(stride).append(delim); + if (padding != null) + sb.append("padding: ").append(padding).append(delim); + if (pnorm != null) + sb.append("pnorm: ").append(pnorm).append(delim); + if (eps != null) + sb.append("eps: ").append(eps).append(delim); + sb.append(super.toString(delim)).append(")"); + return sb.toString(); + } + + + public static class Builder extends FeedForwardLayerSpace.Builder { + + protected ParameterSpace convolutionMode; + protected ParameterSpace poolingType; + protected ParameterSpace dilation; + protected ParameterSpace kernelSize; + protected ParameterSpace stride; + protected ParameterSpace padding; + protected ParameterSpace pnorm; + protected ParameterSpace eps; + + public Builder convolutionMode(ConvolutionMode convolutionMode){ + return convolutionMode(new FixedValue<>(convolutionMode)); + } + + public Builder convolutionMode(ParameterSpace convolutionMode){ + this.convolutionMode = convolutionMode; + return this; + } + + public Builder poolingType(SubsamplingLayer.PoolingType poolingType) { + return poolingType(new FixedValue<>(poolingType)); + } + + public Builder poolingType(ParameterSpace poolingType) { + this.poolingType = poolingType; + return this; + } + + public Builder dilation(int... dilation) { + return dilation(new FixedValue<>(dilation)); + } + + public Builder dilation(ParameterSpace dilation) { + this.dilation = dilation; + return this; + } + + public Builder kernelSize(int... kernelSize) { + return kernelSize(new FixedValue<>(kernelSize)); + } + + public Builder kernelSize(ParameterSpace kernelSize) { + this.kernelSize = kernelSize; + return this; + } + + public Builder stride(int... stride) { + return stride(new FixedValue(stride)); + } + + public Builder stride(ParameterSpace stride) { + this.stride = stride; + return this; + } + + public Builder padding(int... padding) { + return padding(new FixedValue(padding)); + } + + public Builder padding(ParameterSpace padding) { + this.padding = padding; + return this; + } + + public Builder pnorm(int pnorm){ + return pnorm(new FixedValue<>(pnorm)); + } + + public Builder pnorm(ParameterSpace pnorm){ + this.pnorm = pnorm; + return this; + } + + public Builder eps(double eps){ + return eps(new FixedValue<>(eps)); + } + + public Builder eps(ParameterSpace eps){ + this.eps = eps; + return this; + } + + @SuppressWarnings("unchecked") + public SubsamplingLayerSpace build() { + return new SubsamplingLayerSpace(this); + } + + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/VariationalAutoencoderLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/VariationalAutoencoderLayerSpace.java new file mode 100644 index 000000000..2138ea8ec --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/VariationalAutoencoderLayerSpace.java @@ -0,0 +1,182 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.layers.variational.LossFunctionWrapper; +import org.deeplearning4j.nn.conf.layers.variational.ReconstructionDistribution; +import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.lossfunctions.ILossFunction; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +/** + * Layer space for {@link VariationalAutoencoder} + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PRIVATE) //For Jackson JSON/YAML deserialization +public class VariationalAutoencoderLayerSpace extends BasePretrainNetworkLayerSpace { + + private ParameterSpace encoderLayerSizes; + private ParameterSpace decoderLayerSizes; + private ParameterSpace outputDistribution; + private ParameterSpace pzxActivationFn; + private ParameterSpace numSamples; + + protected VariationalAutoencoderLayerSpace(Builder builder) { + super(builder); + + this.encoderLayerSizes = builder.encoderLayerSizes; + this.decoderLayerSizes = builder.decoderLayerSizes; + this.outputDistribution = builder.outputDistribution; + this.pzxActivationFn = builder.pzxActivationFn; + this.numSamples = builder.numSamples; + + this.numParameters = LeafUtils.countUniqueParameters(collectLeaves()); + } + + @Override + public VariationalAutoencoder getValue(double[] parameterValues) { + VariationalAutoencoder.Builder b = new VariationalAutoencoder.Builder(); + setLayerOptionsBuilder(b, parameterValues); + return b.build(); + } + + protected void setLayerOptionsBuilder(VariationalAutoencoder.Builder builder, double[] values) { + super.setLayerOptionsBuilder(builder, values); + if (encoderLayerSizes != null) + builder.encoderLayerSizes(encoderLayerSizes.getValue(values)); + if (decoderLayerSizes != null) + builder.decoderLayerSizes(decoderLayerSizes.getValue(values)); + if (outputDistribution != null) + builder.reconstructionDistribution(outputDistribution.getValue(values)); + if (pzxActivationFn != null) + builder.pzxActivationFn(pzxActivationFn.getValue(values)); + if (numSamples != null) + builder.numSamples(numSamples.getValue(values)); + } + + @Override + public String toString() { + return toString(", "); + } + + @Override + public String toString(String delim) { + StringBuilder sb = new StringBuilder("VariationalAutoencoderLayerSpace("); + if (encoderLayerSizes != null) + sb.append("encoderLayerSizes: ").append(encoderLayerSizes).append(delim); + if (decoderLayerSizes != null) + sb.append("decoderLayerSizes: ").append(decoderLayerSizes).append(delim); + if (outputDistribution != null) + sb.append("reconstructionDistribution: ").append(outputDistribution).append(delim); + if (pzxActivationFn != null) + sb.append("pzxActivationFn: ").append(pzxActivationFn).append(delim); + if (numSamples != null) + sb.append("numSamples: ").append(numSamples).append(delim); + sb.append(super.toString(delim)).append(")"); + return sb.toString(); + } + + public static class Builder extends BasePretrainNetworkLayerSpace.Builder { + + private ParameterSpace encoderLayerSizes; + private ParameterSpace decoderLayerSizes; + private ParameterSpace outputDistribution; + private ParameterSpace pzxActivationFn; + private ParameterSpace numSamples; + + + public Builder encoderLayerSizes(int... encoderLayerSizes) { + return encoderLayerSizes(new FixedValue<>(encoderLayerSizes)); + } + + public Builder encoderLayerSizes(ParameterSpace encoderLayerSizes) { + this.encoderLayerSizes = encoderLayerSizes; + return this; + } + + public Builder decoderLayerSizes(int... decoderLayerSizes) { + return decoderLayerSizes(new FixedValue<>(decoderLayerSizes)); + } + + public Builder decoderLayerSizes(ParameterSpace decoderLayerSizes) { + this.decoderLayerSizes = decoderLayerSizes; + return this; + } + + public Builder reconstructionDistribution(ReconstructionDistribution distribution) { + return reconstructionDistribution(new FixedValue<>(distribution)); + } + + public Builder reconstructionDistribution(ParameterSpace distribution) { + this.outputDistribution = distribution; + return this; + } + + public Builder lossFunction(IActivation outputActivationFn, LossFunctions.LossFunction lossFunction) { + return lossFunction(outputActivationFn, lossFunction.getILossFunction()); + } + + public Builder lossFunction(Activation outputActivationFn, LossFunctions.LossFunction lossFunction) { + return lossFunction(outputActivationFn.getActivationFunction(), lossFunction.getILossFunction()); + } + + public Builder lossFunction(IActivation outputActivationFn, ILossFunction lossFunction) { + return reconstructionDistribution(new LossFunctionWrapper(outputActivationFn, lossFunction)); + } + + public Builder pzxActivationFn(IActivation activationFunction) { + return pzxActivationFn(new FixedValue<>(activationFunction)); + } + + public Builder pzxActivationFn(ParameterSpace activationFunction) { + this.pzxActivationFn = activationFunction; + return this; + } + + public Builder pzxActivationFunction(Activation activation) { + return pzxActivationFn(activation.getActivationFunction()); + } + + public Builder numSamples(int numSamples) { + return numSamples(new FixedValue<>(numSamples)); + } + + public Builder numSamples(ParameterSpace numSamples) { + this.numSamples = numSamples; + return this; + } + + + @Override + public E build() { + return (E) new VariationalAutoencoderLayerSpace(this); + } + + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/fixed/FixedLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/fixed/FixedLayerSpace.java new file mode 100644 index 000000000..fb1afc299 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/layers/fixed/FixedLayerSpace.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.layers.fixed; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.layers.LayerSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.nn.conf.layers.Layer; + +import java.util.Collections; +import java.util.List; + +/** + * A layer space that wraps a DL4J layer, without any optimizable hyperparameters + * + * @param Type of layer + * + * @author Alex Black + */ +@AllArgsConstructor +@NoArgsConstructor +@Data +@EqualsAndHashCode(callSuper = false) +public class FixedLayerSpace extends LayerSpace { + + protected T layer; + + @Override + public T getValue(double[] parameterValues) { + return (T)layer.clone(); + } + + @Override + public int numParameters() { + return 0; + } + + @Override + public boolean isLeaf() { + return true; + } + + @Override + public void setIndices(int[] idxs){ + if(idxs != null && idxs.length > 0){ + throw new IllegalStateException("Cannot set indices: no parameters"); + } + } + + @Override + public List collectLeaves() { + return Collections.singletonList(this); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/listener/DL4JArbiterStatusReportingListener.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/listener/DL4JArbiterStatusReportingListener.java new file mode 100644 index 000000000..0c89984c9 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/listener/DL4JArbiterStatusReportingListener.java @@ -0,0 +1,49 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.listener; + +import lombok.AllArgsConstructor; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.optimize.api.BaseTrainingListener; +import org.deeplearning4j.optimize.api.IterationListener; + +import java.util.List; + +/** + * A simple DL4J Iteration listener that calls Arbiter's status listeners + * + * @author Alex Black + */ +@AllArgsConstructor +public class DL4JArbiterStatusReportingListener extends BaseTrainingListener { + + private List statusListeners; + private CandidateInfo candidateInfo; + + @Override + public void iterationDone(Model model, int iteration, int epoch) { + if (statusListeners == null) { + return; + } + + for (StatusListener sl : statusListeners) { + sl.onCandidateIteration(candidateInfo, model, iteration); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/saver/local/FileModelSaver.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/saver/local/FileModelSaver.java new file mode 100644 index 000000000..167f1f9d1 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/saver/local/FileModelSaver.java @@ -0,0 +1,147 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.saver.local; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.deeplearning4j.arbiter.DL4JConfiguration; +import org.deeplearning4j.arbiter.GraphConfiguration; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultSaver; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.util.ModelSerializer; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Basic MultiLayerNetwork saver. Saves config, parameters and score to: baseDir/0/, baseDir/1/, etc + * where index is given by OptimizationResult.getIndex() + * + * @author Alex Black + */ +@Slf4j +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode +public class FileModelSaver implements ResultSaver { + @JsonProperty + private String path; + private File fPath; + + @JsonCreator + public FileModelSaver(@NonNull String path) { + this(new File(path)); + } + + public FileModelSaver(@NonNull File file){ + this.path = file.getPath(); + this.fPath = file; + + if(!fPath.exists()){ + fPath.mkdirs(); + } else if (!fPath.isDirectory()) { + throw new IllegalArgumentException("Invalid path: exists and is not directory. " + path); + } + + log.info("FileModelSaver saving networks to local directory: {}", path); + } + + @Override + public ResultReference saveModel(OptimizationResult result, Object modelResult) throws IOException { + String dir = new File(path, result.getIndex() + "/").getAbsolutePath(); + + File f = new File(dir); + f.mkdir(); + + File modelFile = new File(FilenameUtils.concat(dir, "model.bin")); + File scoreFile = new File(FilenameUtils.concat(dir, "score.txt")); + File additionalResultsFile = new File(FilenameUtils.concat(dir, "additionalResults.bin")); + File esConfigFile = new File(FilenameUtils.concat(dir, "earlyStoppingConfig.bin")); + File numEpochsFile = new File(FilenameUtils.concat(dir, "numEpochs.txt")); + + FileUtils.writeStringToFile(scoreFile, String.valueOf(result.getScore())); + + Model m = (Model) modelResult; + ModelSerializer.writeModel(m, modelFile, true); + + + Object additionalResults = result.getModelSpecificResults(); + if (additionalResults != null && additionalResults instanceof Serializable) { + try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(additionalResultsFile))) { + oos.writeObject(additionalResults); + } + } + + //Write early stopping configuration (if present) to file: + int nEpochs; + EarlyStoppingConfiguration esc; + if (result.getCandidate().getValue() instanceof DL4JConfiguration) { + DL4JConfiguration c = ((DL4JConfiguration) result.getCandidate().getValue()); + esc = c.getEarlyStoppingConfiguration(); + nEpochs = c.getNumEpochs(); + } else { + GraphConfiguration c = ((GraphConfiguration) result.getCandidate().getValue()); + esc = c.getEarlyStoppingConfiguration(); + nEpochs = c.getNumEpochs(); + } + + + if (esc != null) { + try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(esConfigFile))) { + oos.writeObject(esc); + } + } else { + FileUtils.writeStringToFile(numEpochsFile, String.valueOf(nEpochs)); + } + + log.debug("Deeplearning4j model result (id={}, score={}) saved to directory: {}", result.getIndex(), + result.getScore(), dir); + + boolean isGraph = m instanceof ComputationGraph; + return new LocalFileNetResultReference(result.getIndex(), dir, isGraph, modelFile, scoreFile, + additionalResultsFile, esConfigFile, numEpochsFile, result.getCandidate()); + } + + @Override + public List> getSupportedCandidateTypes() { + return Collections.>singletonList(Object.class); + } + + @Override + public List> getSupportedModelTypes() { + return Arrays.>asList(MultiLayerNetwork.class, ComputationGraph.class); + } + + @Override + public String toString() { + return "FileModelSaver(path=" + path + ")"; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/saver/local/LocalFileNetResultReference.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/saver/local/LocalFileNetResultReference.java new file mode 100644 index 000000000..db46e011e --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/saver/local/LocalFileNetResultReference.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.saver.local; + +import lombok.AllArgsConstructor; +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.arbiter.DL4JConfiguration; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.util.ModelSerializer; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; + +/** + * Result reference for MultiLayerNetworks and ComputationGraphs saved to local file system + */ +@AllArgsConstructor +public class LocalFileNetResultReference implements ResultReference { + + private int index; + private String dir; + private boolean isGraph; + private File modelFile; + private File scoreFile; + private File additionalResultsFile; + private File esConfigFile; + private File numEpochsFile; + private Candidate candidate; + + @Override + public OptimizationResult getResult() throws IOException { + + + String scoreStr = FileUtils.readFileToString(scoreFile); + //TODO: properly parsing. Probably want to store additional info other than just score... + double d = Double.parseDouble(scoreStr); + + EarlyStoppingConfiguration earlyStoppingConfiguration = null; + if (esConfigFile != null && esConfigFile.exists()) { + try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(esConfigFile))) { + earlyStoppingConfiguration = (EarlyStoppingConfiguration) ois.readObject(); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Error loading early stopping configuration", e); + } + } + int nEpochs = 1; + if (numEpochsFile != null && numEpochsFile.exists()) { + String numEpochs = FileUtils.readFileToString(numEpochsFile); + nEpochs = Integer.parseInt(numEpochs); + } + + + + Object additionalResults; + if (additionalResultsFile.exists()) { + try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(additionalResultsFile))) { + additionalResults = ois.readObject(); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Error loading additional results", e); + } + } else { + additionalResults = null; + } + + return new OptimizationResult(candidate, d, index, additionalResults, null, this); + } + + @Override + public Object getResultModel() throws IOException { + Model m; + if (isGraph) { + m = ModelSerializer.restoreComputationGraph(modelFile, false); + } else { + m = ModelSerializer.restoreMultiLayerNetwork(modelFile, false); + } + return m; + } + + @Override + public String toString() { + return "LocalFileNetResultReference(" + dir + ")"; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/RegressionValue.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/RegressionValue.java new file mode 100644 index 000000000..304750dc8 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/RegressionValue.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring; + +/** + * Enumeration used to select the type of regression statistics to optimize on, with the various regression score functions + * - MSE: mean squared error
+ * - MAE: mean absolute error
+ * - RMSE: root mean squared error
+ * - RSE: relative squared error
+ * - CorrCoeff: correlation coefficient
+ * + * @deprecated Use {@link org.deeplearning4j.eval.RegressionEvaluation.Metric} + */ +@Deprecated +public enum RegressionValue { + MSE, MAE, RMSE, RSE, CorrCoeff +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/ScoreFunctions.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/ScoreFunctions.java new file mode 100644 index 000000000..f9e57a597 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/ScoreFunctions.java @@ -0,0 +1,66 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring; + + +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.scoring.impl.TestSetAccuracyScoreFunction; +import org.deeplearning4j.arbiter.scoring.impl.TestSetF1ScoreFunction; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.arbiter.scoring.impl.TestSetRegressionScoreFunction; + +/** + * ScoreFunctions provides static methods for getting score functions for DL4J MultiLayerNetwork and ComputationGraph + * + * @author Alex Black + */ +public class ScoreFunctions { + + private ScoreFunctions() {} + + /** + * Calculate the loss (score/loss function value) on a test set, for a MultiLayerNetwork + * + * @param average Average (divide by number of examples) + */ + public static ScoreFunction testSetLoss(boolean average) { + return new TestSetLossScoreFunction(average); + } + + /** + * Calculate the accuracy on a test set, for a MultiLayerNetwork + */ + public static ScoreFunction testSetAccuracy() { + return new TestSetAccuracyScoreFunction(); + } + + + /** + * Calculate the f1 score on a test set + */ + public static ScoreFunction testSetF1() { + return new TestSetF1ScoreFunction(); + } + + /** + * Calculate a regression value (MSE, MAE etc) on a test set + */ + public static ScoreFunction testSetRegression(RegressionValue regressionValue) { + return new TestSetRegressionScoreFunction(regressionValue); + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/BaseNetScoreFunction.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/BaseNetScoreFunction.java new file mode 100644 index 000000000..1d38ada7c --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/BaseNetScoreFunction.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.impl; + +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIteratorFactory; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Created by Alex on 23/07/2017. + */ +@EqualsAndHashCode +public abstract class BaseNetScoreFunction implements ScoreFunction { + + + @Override + public double score(Object model, DataProvider dataProvider, Map dataParameters) { + Object testData = dataProvider.testData(dataParameters); + return score(model, testData); + } + + @Override + public double score(Object model, Class dataSource, Properties dataSourceProperties) { + DataSource ds; + try{ + ds = dataSource.newInstance(); + if (dataSourceProperties != null) { + ds.configure(dataSourceProperties); + } + } catch (Exception e){ + throw new RuntimeException("Error creating DataSource instance - missing no-arg constructor?", e); + } + return score(model, ds.testData()); + } + + protected double score(Object model, Object testData){ + if (model instanceof MultiLayerNetwork) { + if (testData instanceof DataSetIterator) { + return score((MultiLayerNetwork) model, (DataSetIterator) testData); + } else if(testData instanceof MultiDataSetIterator){ + return score((MultiLayerNetwork) model, (MultiDataSetIterator) testData); + } else if(testData instanceof DataSetIteratorFactory){ + return score((MultiLayerNetwork)model, ((DataSetIteratorFactory)testData).create()); + } else { + throw new RuntimeException("Unknown type of data: " + testData.getClass()); + } + } else { + if (testData instanceof DataSetIterator) { + return score((ComputationGraph) model, (DataSetIterator) testData); + } else if(testData instanceof DataSetIteratorFactory){ + return score((ComputationGraph) model, ((DataSetIteratorFactory)testData).create()); + } else if(testData instanceof MultiDataSetIterator) { + return score((ComputationGraph) model, (MultiDataSetIterator) testData); + } else { + throw new RuntimeException("Unknown type of data: " + testData.getClass()); + } + } + } + + @Override + public List> getSupportedModelTypes() { + return Arrays.>asList(MultiLayerNetwork.class, ComputationGraph.class); + } + + @Override + public List> getSupportedDataTypes() { + return Arrays.>asList(DataSetIterator.class, MultiDataSetIterator.class); + } + + public abstract double score(MultiLayerNetwork net, DataSetIterator iterator); + + public abstract double score(MultiLayerNetwork net, MultiDataSetIterator iterator); + + public abstract double score(ComputationGraph graph, DataSetIterator iterator); + + public abstract double score(ComputationGraph graph, MultiDataSetIterator iterator); +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/EvaluationScoreFunction.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/EvaluationScoreFunction.java new file mode 100644 index 000000000..7e71425d5 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/EvaluationScoreFunction.java @@ -0,0 +1,86 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.impl; + +import lombok.*; +import org.deeplearning4j.datasets.iterator.MultiDataSetWrapperIterator; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +/** + * Score function that calculates an evaluation {@link Evaluation.Metric} on the test set for a + * {@link MultiLayerNetwork} or {@link ComputationGraph} + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //JSON +public class EvaluationScoreFunction extends BaseNetScoreFunction { + + protected Evaluation.Metric metric; + + /** + * @param metric Evaluation metric to calculate + */ + public EvaluationScoreFunction(@NonNull org.deeplearning4j.eval.Evaluation.Metric metric) { + this(metric.toNd4j()); + } + + /** + * @param metric Evaluation metric to calculate + */ + public EvaluationScoreFunction(@NonNull Evaluation.Metric metric) { + this.metric = metric; + } + + @Override + public String toString() { + return "EvaluationScoreFunction(metric=" + metric + ")"; + } + + @Override + public double score(MultiLayerNetwork net, DataSetIterator iterator) { + Evaluation e = net.evaluate(iterator); + return e.scoreForMetric(metric); + } + + @Override + public double score(MultiLayerNetwork net, MultiDataSetIterator iterator) { + return score(net, new MultiDataSetWrapperIterator(iterator)); + } + + @Override + public double score(ComputationGraph graph, DataSetIterator iterator) { + Evaluation e = graph.evaluate(iterator); + return e.scoreForMetric(metric); + } + + @Override + public double score(ComputationGraph graph, MultiDataSetIterator iterator) { + Evaluation e = graph.evaluate(iterator); + return e.scoreForMetric(metric); + } + + @Override + public boolean minimize() { + return false; //Want to maximize all evaluation metrics: Accuracy, F1, precision, recall, g-measure, mcc + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/ROCScoreFunction.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/ROCScoreFunction.java new file mode 100644 index 000000000..9203963e3 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/ROCScoreFunction.java @@ -0,0 +1,122 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.impl; + +import lombok.*; +import org.deeplearning4j.datasets.iterator.MultiDataSetWrapperIterator; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.evaluation.classification.ROC; +import org.nd4j.evaluation.classification.ROCBinary; +import org.nd4j.evaluation.classification.ROCMultiClass; +import org.nd4j.linalg.dataset.adapter.MultiDataSetIteratorAdapter; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +/** + * Score function that calculates AUC (area under ROC curve) or AUPRC (area under precision/recall curve) on a test set + * for a {@link MultiLayerNetwork} or {@link ComputationGraph} + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //JSON +public class ROCScoreFunction extends BaseNetScoreFunction { + + /** + * Type of ROC evaluation to perform:
+ * ROC: use {@link ROC} to perform evaluation (single output binary classification)
+ * BINARY: use {@link ROCBinary} to perform evaluation (multi-output/multi-task binary classification)
+ * MULTICLASS: use {@link ROCMultiClass} to perform evaluation (1 vs. all multi-class classification) + * + */ + public enum ROCType {ROC, BINARY, MULTICLASS} + + /** + * Metric to calculate.
+ * AUC: Area under ROC curve
+ * AUPRC: Area under precision/recall curve + */ + public enum Metric {AUC, AUPRC}; + + protected ROCType type; + protected Metric metric; + + /** + * @param type ROC type to use for evaluation + * @param metric Evaluation metric to calculate + */ + public ROCScoreFunction(@NonNull ROCType type, @NonNull Metric metric) { + this.type = type; + this.metric = metric; + } + + @Override + public String toString() { + return "ROCScoreFunction(type=" + type + ",metric=" + metric + ")"; + } + + @Override + public double score(MultiLayerNetwork net, DataSetIterator iterator) { + switch (type){ + case ROC: + ROC r = net.evaluateROC(iterator); + return metric == Metric.AUC ? r.calculateAUC() : r.calculateAUCPR(); + case BINARY: + ROCBinary r2 = net.doEvaluation(iterator, new ROCBinary())[0]; + return metric == Metric.AUC ? r2.calculateAverageAuc() : r2.calculateAverageAUCPR(); + case MULTICLASS: + ROCMultiClass r3 = net.evaluateROCMultiClass(iterator); + return metric == Metric.AUC ? r3.calculateAverageAUC() : r3.calculateAverageAUCPR(); + default: + throw new RuntimeException("Unknown type: " + type); + } + } + + @Override + public double score(MultiLayerNetwork net, MultiDataSetIterator iterator) { + return score(net, new MultiDataSetWrapperIterator(iterator)); + } + + @Override + public double score(ComputationGraph graph, DataSetIterator iterator) { + return score(graph, new MultiDataSetIteratorAdapter(iterator)); + } + + @Override + public double score(ComputationGraph net, MultiDataSetIterator iterator) { + switch (type){ + case ROC: + ROC r = net.evaluateROC(iterator); + return metric == Metric.AUC ? r.calculateAUC() : r.calculateAUCPR(); + case BINARY: + ROCBinary r2 = net.doEvaluation(iterator, new ROCBinary())[0]; + return metric == Metric.AUC ? r2.calculateAverageAuc() : r2.calculateAverageAUCPR(); + case MULTICLASS: + ROCMultiClass r3 = net.evaluateROCMultiClass(iterator, 0); + return metric == Metric.AUC ? r3.calculateAverageAUC() : r3.calculateAverageAUCPR(); + default: + throw new RuntimeException("Unknown type: " + type); + } + } + + @Override + public boolean minimize() { + return false; //Want to maximize all evaluation metrics: Accuracy, F1, precision, recall, g-measure, mcc + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/RegressionScoreFunction.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/RegressionScoreFunction.java new file mode 100644 index 000000000..51fcd9898 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/RegressionScoreFunction.java @@ -0,0 +1,92 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.impl; + +import lombok.*; +import org.deeplearning4j.datasets.iterator.MultiDataSetWrapperIterator; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.evaluation.regression.RegressionEvaluation; +import org.nd4j.evaluation.regression.RegressionEvaluation.Metric; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +/** + * Score function for regression (including multi-label regression) for a MultiLayerNetwork or ComputationGraph + * on a test set. Supports all regression metrics: {@link Metric} + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For JSON +public class RegressionScoreFunction extends BaseNetScoreFunction { + + protected Metric metric; + + public RegressionScoreFunction(@NonNull org.deeplearning4j.eval.RegressionEvaluation.Metric metric) { + this(metric.toNd4j()); + } + + public RegressionScoreFunction(@NonNull Metric metric) { + this.metric = metric; + } + + @Override + public boolean minimize() { + switch (metric) { + case MSE: + case MAE: + case RMSE: + case RSE: + return true; + case PC: + case R2: + return false; + default: + throw new IllegalStateException("Unknown metric: " + metric); + } + } + + @Override + public String toString() { + return "RegressionScoreFunction(metric=" + metric + ")"; + } + + @Override + public double score(MultiLayerNetwork net, DataSetIterator iterator) { + RegressionEvaluation e = net.evaluateRegression(iterator); + return e.scoreForMetric(metric); + } + + @Override + public double score(MultiLayerNetwork net, MultiDataSetIterator iterator) { + return score(net, new MultiDataSetWrapperIterator(iterator)); + } + + @Override + public double score(ComputationGraph graph, DataSetIterator iterator) { + RegressionEvaluation e = graph.evaluateRegression(iterator); + return e.scoreForMetric(metric); + } + + @Override + public double score(ComputationGraph graph, MultiDataSetIterator iterator) { + RegressionEvaluation e = graph.evaluateRegression(iterator); + return e.scoreForMetric(metric); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetAccuracyScoreFunction.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetAccuracyScoreFunction.java new file mode 100644 index 000000000..34b051663 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetAccuracyScoreFunction.java @@ -0,0 +1,72 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.impl; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +/** + * Score function that calculates the accuracy on a + * test set for a {@link MultiLayerNetwork} or {@link ComputationGraph} + * + * @author Alex Black + * @deprecated Use {@link EvaluationScoreFunction} + */ +@Data +@EqualsAndHashCode(callSuper = true) +@Deprecated +public class TestSetAccuracyScoreFunction extends BaseNetScoreFunction { + + + @Override + public String toString() { + return "TestSetAccuracyScoreFunction()"; + } + + @Override + public double score(MultiLayerNetwork net, DataSetIterator iterator) { + Evaluation e = net.evaluate(iterator); + return e.accuracy(); + } + + @Override + public double score(MultiLayerNetwork net, MultiDataSetIterator iterator) { + throw new UnsupportedOperationException("Cannot evaluate MultiLayerNetwork on MultiDataSetIterator"); + } + + @Override + public double score(ComputationGraph graph, DataSetIterator iterator) { + Evaluation e = graph.evaluate(iterator); + return e.accuracy(); + } + + @Override + public double score(ComputationGraph graph, MultiDataSetIterator iterator) { + Evaluation e = graph.evaluate(iterator); + return e.accuracy(); + } + + @Override + public boolean minimize() { + return false; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetF1ScoreFunction.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetF1ScoreFunction.java new file mode 100644 index 000000000..24516a1d7 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetF1ScoreFunction.java @@ -0,0 +1,72 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.impl; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +/** + * Score function that calculates the F1 score + * on a test set for a {@link MultiLayerNetwork} or {@link ComputationGraph} + * + * @author Alex Black + * @deprecated Use {@link EvaluationScoreFunction} + */ +@Data +@EqualsAndHashCode(callSuper = true) +@Deprecated +public class TestSetF1ScoreFunction extends BaseNetScoreFunction { + + @Override + public boolean minimize() { + return false; //false -> maximize + } + + + @Override + public String toString() { + return "TestSetF1ScoreFunction"; + } + + @Override + public double score(MultiLayerNetwork net, DataSetIterator iterator) { + Evaluation e = net.evaluate(iterator); + return e.f1(); + } + + @Override + public double score(MultiLayerNetwork net, MultiDataSetIterator iterator) { + throw new UnsupportedOperationException("Cannot evaluate MultiLayerNetwork on MultiDataSetIterator"); + } + + @Override + public double score(ComputationGraph graph, DataSetIterator iterator) { + Evaluation e = graph.evaluate(iterator); + return e.f1(); + } + + @Override + public double score(ComputationGraph graph, MultiDataSetIterator iterator) { + Evaluation e = graph.evaluate(iterator); + return e.f1(); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetLossScoreFunction.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetLossScoreFunction.java new file mode 100644 index 000000000..f44df800e --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetLossScoreFunction.java @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.impl; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.arbiter.scoring.util.ScoreUtil; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Score function that calculates the test set loss + * on a test set for a {@link MultiLayerNetwork} or {@link ComputationGraph} + * + * @author Alex Black + */ +@Data +@EqualsAndHashCode(callSuper = false) +public class TestSetLossScoreFunction extends BaseNetScoreFunction { + @JsonProperty + private final boolean average; + + public TestSetLossScoreFunction() { + this(true); + } + + public TestSetLossScoreFunction(boolean average) { + this.average = average; + } + + + @Override + public boolean minimize() { + return true; + } + + @Override + public String toString() { + return "TestSetLossScoreFunction()"; + } + + @Override + public double score(MultiLayerNetwork net, DataSetIterator iterator) { + return ScoreUtil.score(net, iterator, average); + } + + @Override + public double score(MultiLayerNetwork net, MultiDataSetIterator iterator) { + throw new UnsupportedOperationException("Cannot evaluate MultiLayerNetwork on MultiDataSetIterator"); + } + + @Override + public double score(ComputationGraph graph, DataSetIterator iterator) { + return ScoreUtil.score(graph, iterator, average); + } + + @Override + public double score(ComputationGraph graph, MultiDataSetIterator iterator) { + return ScoreUtil.score(graph, iterator, average); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetRegressionScoreFunction.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetRegressionScoreFunction.java new file mode 100644 index 000000000..0a27cea4e --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/impl/TestSetRegressionScoreFunction.java @@ -0,0 +1,85 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.impl; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.deeplearning4j.arbiter.scoring.RegressionValue; +import org.deeplearning4j.arbiter.scoring.util.ScoreUtil; +import org.deeplearning4j.eval.RegressionEvaluation; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +/** + * Score function for regression (including multi-label regression) for a MultiLayerNetwork or ComputationGraph + * on a test set + * + * @author Alex Black + * @deprecated Use {@link RegressionScoreFunction} + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) //For JSON +@Deprecated +public class TestSetRegressionScoreFunction extends BaseNetScoreFunction { + private RegressionValue regressionValue; + + /** + * @param regressionValue The type of evaluation to do: MSE, MAE, RMSE, etc + */ + public TestSetRegressionScoreFunction(RegressionValue regressionValue) { + this.regressionValue = regressionValue; + } + + + @Override + public boolean minimize() { + return regressionValue != RegressionValue.CorrCoeff; //Maximize correlation coefficient, minimize the remaining ones + } + + @Override + public String toString() { + return "TestSetRegressionScoreFunction(type=" + regressionValue + ")"; + } + + @Override + public double score(MultiLayerNetwork net, DataSetIterator iterator) { + RegressionEvaluation e = net.evaluateRegression(iterator); + return ScoreUtil.getScoreFromRegressionEval(e, regressionValue); + } + + @Override + public double score(MultiLayerNetwork net, MultiDataSetIterator iterator) { + throw new UnsupportedOperationException("Cannot evaluate MultiLayerNetwork on MultiDataSetIterator"); + } + + @Override + public double score(ComputationGraph graph, DataSetIterator iterator) { + RegressionEvaluation e = graph.evaluateRegression(iterator); + return ScoreUtil.getScoreFromRegressionEval(e, regressionValue); + } + + @Override + public double score(ComputationGraph graph, MultiDataSetIterator iterator) { + RegressionEvaluation e = graph.evaluateRegression(iterator); + return ScoreUtil.getScoreFromRegressionEval(e, regressionValue); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/util/ScoreUtil.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/util/ScoreUtil.java new file mode 100644 index 000000000..303defe35 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/scoring/util/ScoreUtil.java @@ -0,0 +1,328 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.scoring.util; + +import org.deeplearning4j.arbiter.scoring.RegressionValue; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.eval.RegressionEvaluation; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.adapter.MultiDataSetIteratorAdapter; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIteratorFactory; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIteratorFactory; + + + +/** + * Various utilities for functions used in arbiter. + * + * @author Adam Gibson + */ +public class ScoreUtil { + + + + /** + * Get a {@link DataSetIterator} + * from the given object whether it's a {@link DataSetIterator} + * or {@link DataSetIteratorFactory}, any other type will throw + * an {@link IllegalArgumentException} + * @param o the object to get the iterator from + * @return the datasetiterator from the given objects + */ + public static MultiDataSetIterator getMultiIterator(Object o) { + if (o instanceof MultiDataSetIterator) { + return (MultiDataSetIterator) o; + } else if (o instanceof MultiDataSetIteratorFactory) { + MultiDataSetIteratorFactory factory = (MultiDataSetIteratorFactory) o; + return factory.create(); + } else if( o instanceof DataSetIterator ){ + return new MultiDataSetIteratorAdapter((DataSetIterator)o); + } else if( o instanceof DataSetIteratorFactory ){ + return new MultiDataSetIteratorAdapter(((DataSetIteratorFactory)o).create()); + } + + throw new IllegalArgumentException("Type must either be DataSetIterator or DataSetIteratorFactory"); + } + + + /** + * Get a {@link DataSetIterator} + * from the given object whether it's a {@link DataSetIterator} + * or {@link DataSetIteratorFactory}, any other type will throw + * an {@link IllegalArgumentException} + * @param o the object to get the iterator from + * @return the datasetiterator from the given objects + */ + public static DataSetIterator getIterator(Object o) { + if (o instanceof DataSetIterator) + return (DataSetIterator) o; + else if (o instanceof DataSetIteratorFactory) { + DataSetIteratorFactory factory = (DataSetIteratorFactory) o; + return factory.create(); + } + + throw new IllegalArgumentException("Type must either be DataSetIterator or DataSetIteratorFactory"); + } + + /** + * + * @param model + * @param testData + * @return + */ + public static Evaluation getEvaluation(MultiLayerNetwork model, DataSetIterator testData) { + return model.evaluate(testData); + } + + /** + * Get the evaluation + * for the given model and test dataset + * @param model the model to get the evaluation from + * @param testData the test data to do the evaluation on + * @return the evaluation object with accumulated statistics + * for the current test data + */ + public static Evaluation getEvaluation(ComputationGraph model, MultiDataSetIterator testData) { + if (model.getNumOutputArrays() != 1) + throw new IllegalStateException("GraphSetSetAccuracyScoreFunction cannot be " + + "applied to ComputationGraphs with more than one output. NumOutputs = " + + model.getNumOutputArrays()); + + return model.evaluate(testData); + } + + + /** + * Get the evaluation + * for the given model and test dataset + * @param model the model to get the evaluation from + * @param testData the test data to do the evaluation on + * @return the evaluation object with accumulated statistics + * for the current test data + */ + public static Evaluation getEvaluation(ComputationGraph model, DataSetIterator testData) { + if (model.getNumOutputArrays() != 1) + throw new IllegalStateException("GraphSetSetAccuracyScoreFunctionDataSet cannot be " + + "applied to ComputationGraphs with more than one output. NumOutputs = " + + model.getNumOutputArrays()); + + return model.evaluate(testData); + } + + + + /** + * Score based on the loss function + * @param model the model to score with + * @param testData the test data to score + * @param average whether to average the score + * for the whole batch or not + * @return the score for the given test set + */ + public static double score(ComputationGraph model, MultiDataSetIterator testData, boolean average) { + //TODO: do this properly taking into account division by N, L1/L2 etc + double sumScore = 0.0; + int totalExamples = 0; + while (testData.hasNext()) { + MultiDataSet ds = testData.next(); + long numExamples = ds.getFeatures(0).size(0); + sumScore += numExamples * model.score(ds); + totalExamples += numExamples; + } + + if (!average) + return sumScore; + return sumScore / totalExamples; + } + + /** + * Score based on the loss function + * @param model the model to score with + * @param testData the test data to score + * @param average whether to average the score + * for the whole batch or not + * @return the score for the given test set + */ + public static double score(ComputationGraph model, DataSetIterator testData, boolean average) { + //TODO: do this properly taking into account division by N, L1/L2 etc + double sumScore = 0.0; + int totalExamples = 0; + while (testData.hasNext()) { + DataSet ds = testData.next(); + int numExamples = ds.numExamples(); + + sumScore += numExamples * model.score(ds); + totalExamples += numExamples; + } + + if (!average) + return sumScore; + return sumScore / totalExamples; + } + + + /** + * + * @param model + * @param testSet + * @param regressionValue + * @return + */ + public static double score(ComputationGraph model, MultiDataSetIterator testSet, RegressionValue regressionValue) { + int nOutputs = model.getNumOutputArrays(); + + RegressionEvaluation[] evaluations = new RegressionEvaluation[nOutputs]; + for (int i = 0; i < evaluations.length; i++) + evaluations[i] = new RegressionEvaluation(); + + while (testSet.hasNext()) { + MultiDataSet next = testSet.next(); + INDArray[] labels = next.getLabels(); + + if (next.hasMaskArrays()) { + INDArray[] fMasks = next.getFeaturesMaskArrays(); + INDArray[] lMasks = next.getLabelsMaskArrays(); + + model.setLayerMaskArrays(fMasks, lMasks); + + INDArray[] outputs = model.output(false, next.getFeatures()); + for (int i = 0; i < evaluations.length; i++) { + if (lMasks != null && lMasks[i] != null) { + evaluations[i].evalTimeSeries(labels[i], outputs[i], lMasks[i]); + } else { + evaluations[i].evalTimeSeries(labels[i], outputs[i]); + } + } + + model.clearLayerMaskArrays(); + } else { + INDArray[] outputs = model.output(false, next.getFeatures()); + for (int i = 0; i < evaluations.length; i++) { + if (labels[i].rank() == 3) { + evaluations[i].evalTimeSeries(labels[i], outputs[i]); + } else { + evaluations[i].eval(labels[i], outputs[i]); + } + } + } + } + + double sum = 0.0; + int totalColumns = 0; + for (int i = 0; i < evaluations.length; i++) { + int nColumns = evaluations[i].numColumns(); + totalColumns += nColumns; + sum += getScoreFromRegressionEval(evaluations[i], regressionValue); + } + if (regressionValue == RegressionValue.CorrCoeff) + sum /= totalColumns; + + return sum; + } + + + /** + * Run a {@link RegressionEvaluation} + * over a {@link DataSetIterator} + * @param model the model to use + * @param testSet the test set iterator + * @param regressionValue the regression type to use + * @return + */ + public static double score(ComputationGraph model, DataSetIterator testSet, RegressionValue regressionValue) { + RegressionEvaluation evaluation = model.evaluateRegression(testSet); + return getScoreFromRegressionEval(evaluation, regressionValue); + } + + + /** + * Score the given test data + * with the given multi layer network + * @param model model to use + * @param testData the test data to test with + * @param average whether to average the score or not + * @return the score for the given test data given the model + */ + public static double score(MultiLayerNetwork model, DataSetIterator testData, boolean average) { + //TODO: do this properly taking into account division by N, L1/L2 etc + double sumScore = 0.0; + int totalExamples = 0; + while (testData.hasNext()) { + DataSet ds = testData.next(); + int numExamples = ds.numExamples(); + + sumScore += numExamples * model.score(ds); + totalExamples += numExamples; + } + + if (!average) + return sumScore; + return sumScore / totalExamples; + } + + + /** + * Score the given multi layer network + * @param model the model to score + * @param testSet the test set + * @param regressionValue the regression function to use + * @return the score from the given test set + */ + public static double score(MultiLayerNetwork model, DataSetIterator testSet, RegressionValue regressionValue) { + RegressionEvaluation eval = model.evaluateRegression(testSet); + return getScoreFromRegressionEval(eval, regressionValue); + } + + + @Deprecated + public static double getScoreFromRegressionEval(RegressionEvaluation eval, RegressionValue regressionValue) { + double sum = 0.0; + int nColumns = eval.numColumns(); + switch (regressionValue) { + case MSE: + for (int i = 0; i < nColumns; i++) + sum += eval.meanSquaredError(i); + break; + case MAE: + for (int i = 0; i < nColumns; i++) + sum += eval.meanAbsoluteError(i); + break; + case RMSE: + for (int i = 0; i < nColumns; i++) + sum += eval.rootMeanSquaredError(i); + break; + case RSE: + for (int i = 0; i < nColumns; i++) + sum += eval.relativeSquaredError(i); + break; + case CorrCoeff: + for (int i = 0; i < nColumns; i++) + sum += eval.correlationR2(i); + sum /= nColumns; + break; + } + + return sum; + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/ComputationGraphTaskCreator.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/ComputationGraphTaskCreator.java new file mode 100644 index 000000000..53a9fe0aa --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/ComputationGraphTaskCreator.java @@ -0,0 +1,267 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.task; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.deeplearning4j.arbiter.GraphConfiguration; +import org.deeplearning4j.arbiter.listener.DL4JArbiterStatusReportingListener; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.api.TaskCreator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.evaluation.ModelEvaluator; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultSaver; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.CandidateStatus; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; +import org.deeplearning4j.arbiter.scoring.util.ScoreUtil; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.EarlyStoppingResult; +import org.deeplearning4j.earlystopping.trainer.EarlyStoppingGraphTrainer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.Callable; + +/** + * Task creator for ComputationGraph + * + * @author Alex Black + */ +@AllArgsConstructor +@NoArgsConstructor +@Slf4j +public class ComputationGraphTaskCreator implements TaskCreator { + + private ModelEvaluator modelEvaluator; + @Getter + @Setter + private TaskListener taskListener; + + public ComputationGraphTaskCreator(ModelEvaluator modelEvaluator){ + this(modelEvaluator, null); + } + + @Override + public Callable create(Candidate candidate, DataProvider dataProvider, + ScoreFunction scoreFunction, List statusListener, + IOptimizationRunner runner) { + + return new GraphLearningTask(candidate, dataProvider, scoreFunction, modelEvaluator, statusListener, + taskListener, runner); + } + + @Override + public Callable create(Candidate candidate, Class dataSource, Properties dataSourceProperties, + ScoreFunction scoreFunction, List statusListeners, IOptimizationRunner runner) { + return new GraphLearningTask(candidate, dataSource, dataSourceProperties, scoreFunction, modelEvaluator, statusListeners, + taskListener, runner); + } + + @AllArgsConstructor + private static class GraphLearningTask implements Callable { + + private Candidate candidate; + private DataProvider dataProvider; + private Class dataSource; + private Properties dataSourceProperties; + private ScoreFunction scoreFunction; + private ModelEvaluator modelEvaluator; + private List listeners; + private TaskListener taskListener; + private IOptimizationRunner runner; + + private long startTime; + + public GraphLearningTask(Candidate candidate, DataProvider dataProvider, ScoreFunction scoreFunction, + ModelEvaluator modelEvaluator, List listeners, + TaskListener taskListener, IOptimizationRunner runner) { + this.candidate = candidate; + this.dataProvider = dataProvider; + this.scoreFunction = scoreFunction; + this.modelEvaluator = modelEvaluator; + this.listeners = listeners; + this.taskListener = taskListener; + this.runner = runner; + } + + public GraphLearningTask(Candidate candidate, Class dataSource, Properties dataSourceProperties, + ScoreFunction scoreFunction, ModelEvaluator modelEvaluator, List listeners, + TaskListener taskListener, IOptimizationRunner runner) { + this.candidate = candidate; + this.dataSource = dataSource; + this.dataSourceProperties = dataSourceProperties; + this.scoreFunction = scoreFunction; + this.modelEvaluator = modelEvaluator; + this.listeners = listeners; + this.taskListener = taskListener; + this.runner = runner; + } + + + @Override + public OptimizationResult call() throws Exception { + + try { + OptimizationResult result = callHelper(); + if(listeners != null && !listeners.isEmpty()){ + CandidateInfo ci = new CandidateInfo(candidate.getIndex(), CandidateStatus.Complete, result.getScore(), + startTime, startTime, System.currentTimeMillis(), candidate.getFlatParameters(), null); + for(StatusListener sl : listeners){ + try{ + sl.onCandidateStatusChange(ci, runner, result); + } catch (Exception e){ + log.error("Error in status listener for candidate {}", candidate.getIndex(), e); + } + } + } + return result; + } catch (Throwable e) { + String stackTrace = ExceptionUtils.getStackTrace(e); + log.warn("Execution failed for task {}", candidate.getIndex(), e); + + CandidateInfo ci = new CandidateInfo(candidate.getIndex(), CandidateStatus.Failed, null, startTime, + null, null, candidate.getFlatParameters(), stackTrace); + return new OptimizationResult(candidate, null, candidate.getIndex(), null, ci, null); + } finally { + //Destroy workspaces to free memory + Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); + System.gc(); + try { + //Sleep for a few seconds - workspace destruction and memory deallocation happens quickly but doesn't + // happen instantly; if we didn't have this, we may run into a situation where the next thread/task + // tries to allocate before WS memory is fully deallocated, resulting in an OOM in memory constrained + // environments + Thread.sleep(2000L); + } catch (Exception e){ } + } + } + + private OptimizationResult callHelper() throws Exception { + startTime = System.currentTimeMillis(); + CandidateInfo ci = new CandidateInfo(candidate.getIndex(), CandidateStatus.Running, null, startTime, startTime, + null, candidate.getFlatParameters(), null); + + //Create network + ComputationGraph net = new ComputationGraph(((GraphConfiguration) candidate.getValue()).getConfiguration()); + net.init(); + + if(taskListener != null){ + net = taskListener.preProcess(net, candidate); + } + + if (listeners != null) { + net.addListeners(new DL4JArbiterStatusReportingListener(listeners, ci)); + } + + //For DataSetIterator: wraps in a MultiDataSetIterator, hence method can be used for both + MultiDataSetIterator iterator; + if(dataSource != null){ + try { + DataSource dsInstance = dataSource.newInstance(); + if (dataSourceProperties != null) + dsInstance.configure(dataSourceProperties); + iterator = ScoreUtil.getMultiIterator(dsInstance.trainData()); + } catch (Exception e){ + throw new RuntimeException("Error instantiating instance of DataSource for class " + dataSource.getName() + + " - no zero-arg constructor?",e); + } + } else { + iterator = ScoreUtil.getMultiIterator(dataProvider.trainData(candidate.getDataParameters())); + } + + + EarlyStoppingConfiguration esConfig = + ((GraphConfiguration) candidate.getValue()).getEarlyStoppingConfiguration(); + EarlyStoppingResult esResult = null; + if (esConfig != null) { + EarlyStoppingGraphTrainer trainer = new EarlyStoppingGraphTrainer(esConfig, net, iterator, null); + esResult = trainer.fit(); + net = esResult.getBestModel(); //Can return null if failed OR if + + switch (esResult.getTerminationReason()) { + case Error: + ci.setCandidateStatus(CandidateStatus.Failed); + ci.setExceptionStackTrace(esResult.getTerminationDetails()); + break; + case IterationTerminationCondition: + case EpochTerminationCondition: + ci.setCandidateStatus(CandidateStatus.Complete); + break; + } + + } else { + //Fixed number of epochs + int nEpochs = ((GraphConfiguration) candidate.getValue()).getNumEpochs(); + for (int i = 0; i < nEpochs; i++) { + net.fit(iterator); + } + ci.setCandidateStatus(CandidateStatus.Complete); + } + Nd4j.getExecutioner().commit(); + + Object additionalEvaluation = null; + if (esConfig != null && esResult.getTerminationReason() != EarlyStoppingResult.TerminationReason.Error) { + additionalEvaluation = + (modelEvaluator != null ? modelEvaluator.evaluateModel(net, dataProvider) : null); + } + + Double score = null; + if (net != null) { + if(dataSource != null){ + score = scoreFunction.score(net, dataSource, dataSourceProperties); + } else { + score = scoreFunction.score(net, dataProvider, candidate.getDataParameters()); + } + ci.setScore(score); + } + + if(taskListener != null){ + taskListener.postProcess(net, candidate); + } + + OptimizationResult result = new OptimizationResult(candidate, score, candidate.getIndex(), additionalEvaluation, ci, null); + + //Save the model: + ResultSaver saver = runner.getConfiguration().getResultSaver(); + ResultReference resultReference = null; + if (saver != null) { + try { + resultReference = saver.saveModel(result, net); + } catch (IOException e) { + //TODO: Do we want ta warn or fail on IOException? + log.warn("Error saving model (id={}): IOException thrown. ", result.getIndex(), e); + } + } + result.setResultReference(resultReference); + return result; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/MultiLayerNetworkTaskCreator.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/MultiLayerNetworkTaskCreator.java new file mode 100644 index 000000000..5c2fb0703 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/MultiLayerNetworkTaskCreator.java @@ -0,0 +1,265 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.task; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.deeplearning4j.arbiter.DL4JConfiguration; +import org.deeplearning4j.arbiter.listener.DL4JArbiterStatusReportingListener; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.api.TaskCreator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.evaluation.ModelEvaluator; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultSaver; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.CandidateStatus; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; +import org.deeplearning4j.arbiter.scoring.util.ScoreUtil; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.EarlyStoppingResult; +import org.deeplearning4j.earlystopping.trainer.EarlyStoppingTrainer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.Callable; + +/** + * Task creator for MultiLayerNetworks + * + * @author Alex Black + */ +@AllArgsConstructor +@NoArgsConstructor +@Slf4j +public class MultiLayerNetworkTaskCreator implements TaskCreator { + + private ModelEvaluator modelEvaluator; + @Getter + @Setter + private TaskListener taskListener; + + public MultiLayerNetworkTaskCreator(ModelEvaluator modelEvaluator){ + this(modelEvaluator, null); + } + + @Override + public Callable create(Candidate candidate, DataProvider dataProvider, + ScoreFunction scoreFunction, List statusListeners, + IOptimizationRunner runner) { + + return new DL4JLearningTask(candidate, dataProvider, scoreFunction, modelEvaluator, statusListeners, taskListener, runner); + } + + @Override + public Callable create(Candidate candidate, Class dataSource, Properties dataSourceProperties, + ScoreFunction scoreFunction, List statusListeners, IOptimizationRunner runner) { + return new DL4JLearningTask(candidate, dataSource, dataSourceProperties, scoreFunction, modelEvaluator, statusListeners, taskListener, runner); + } + + + private static class DL4JLearningTask implements Callable { + + private Candidate candidate; + private DataProvider dataProvider; + private Class dataSource; + private Properties dataSourceProperties; + private ScoreFunction scoreFunction; + private ModelEvaluator modelEvaluator; + private List listeners; + private TaskListener taskListener; + private IOptimizationRunner runner; + + private long startTime; + + public DL4JLearningTask(Candidate candidate, DataProvider dataProvider, ScoreFunction scoreFunction, + ModelEvaluator modelEvaluator, List listeners, TaskListener taskListener, + IOptimizationRunner runner) { + this.candidate = candidate; + this.dataProvider = dataProvider; + this.scoreFunction = scoreFunction; + this.modelEvaluator = modelEvaluator; + this.listeners = listeners; + this.taskListener = taskListener; + this.runner = runner; + } + + public DL4JLearningTask(Candidate candidate, Class dataSource, Properties dataSourceProperties, + ScoreFunction scoreFunction, ModelEvaluator modelEvaluator, List listeners, TaskListener taskListener, + IOptimizationRunner runner) { + this.candidate = candidate; + this.dataSource = dataSource; + this.dataSourceProperties = dataSourceProperties; + this.scoreFunction = scoreFunction; + this.modelEvaluator = modelEvaluator; + this.listeners = listeners; + this.taskListener = taskListener; + this.runner = runner; + } + + + @Override + public OptimizationResult call() { + + try { + OptimizationResult result = callHelper(); + if(listeners != null && !listeners.isEmpty()){ + CandidateInfo ci = new CandidateInfo(candidate.getIndex(), CandidateStatus.Complete, result.getScore(), + startTime, startTime, System.currentTimeMillis(), candidate.getFlatParameters(), null); + for(StatusListener sl : listeners){ + try{ + sl.onCandidateStatusChange(ci, runner, result); + } catch (Exception e){ + log.error("Error in status listener for candidate {}", candidate.getIndex(), e); + } + } + } + return result; + } catch (Throwable e) { + String stackTrace = ExceptionUtils.getStackTrace(e); + log.warn( "Execution failed for task {}", candidate.getIndex(), e ); + + CandidateInfo ci = new CandidateInfo(candidate.getIndex(), CandidateStatus.Failed, null, startTime, + null, null, candidate.getFlatParameters(), stackTrace); + return new OptimizationResult(candidate, null, candidate.getIndex(), null, ci, null); + } finally { + //Destroy workspaces to free memory + Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); + System.gc(); + try { + //Sleep for a few seconds - workspace destruction and memory deallocation happens quickly but doesn't + // happen instantly; if we didn't have this, we may run into a situation where the next thread/task + // tries to allocate before WS memory is fully deallocated, resulting in an OOM in memory constrained + // environments + Thread.sleep(2000L); + } catch (Exception e){ } + } + } + + private OptimizationResult callHelper() { + startTime = System.currentTimeMillis(); + CandidateInfo ci = new CandidateInfo(candidate.getIndex(), CandidateStatus.Running, null, + startTime, startTime, null, candidate.getFlatParameters(), null); + + //Create network + MultiLayerNetwork net = new MultiLayerNetwork( + ((DL4JConfiguration) candidate.getValue()).getMultiLayerConfiguration()); + net.init(); + + if(taskListener != null){ + net = taskListener.preProcess(net, candidate); + } + + if (listeners != null) { + net.addListeners(new DL4JArbiterStatusReportingListener(listeners, ci)); + } + + //Early stopping or fixed number of epochs: + DataSetIterator dataSetIterator; + if(dataSource != null){ + DataSource dsInstance; + try{ + dsInstance = dataSource.newInstance(); + } catch (Exception e){ + throw new RuntimeException("Error instantiating instance of DataSource for class " + dataSource.getName() + + " - no zero-arg constructor?",e); + } + if(dataSourceProperties != null) + dsInstance.configure(dataSourceProperties); + dataSetIterator = ScoreUtil.getIterator(dsInstance.trainData()); + } else { + dataSetIterator = ScoreUtil.getIterator(dataProvider.trainData(candidate.getDataParameters())); + } + + + EarlyStoppingConfiguration esConfig = + ((DL4JConfiguration) candidate.getValue()).getEarlyStoppingConfiguration(); + EarlyStoppingResult esResult = null; + if (esConfig != null) { + EarlyStoppingTrainer trainer = new EarlyStoppingTrainer(esConfig, net, dataSetIterator, null); + esResult = trainer.fit(); + net = esResult.getBestModel(); //Can return null if failed OR if + + switch (esResult.getTerminationReason()) { + case Error: + ci.setCandidateStatus(CandidateStatus.Failed); + ci.setExceptionStackTrace(esResult.getTerminationDetails()); + break; + case IterationTerminationCondition: + case EpochTerminationCondition: + ci.setCandidateStatus(CandidateStatus.Complete); + break; + } + + } else { + //Fixed number of epochs + int nEpochs = ((DL4JConfiguration) candidate.getValue()).getNumEpochs(); + for (int i = 0; i < nEpochs; i++) { + net.fit(dataSetIterator); + } + ci.setCandidateStatus(CandidateStatus.Complete); + } + + Object additionalEvaluation = null; + if (esConfig != null && esResult.getTerminationReason() != EarlyStoppingResult.TerminationReason.Error) { + additionalEvaluation = + (modelEvaluator != null ? modelEvaluator.evaluateModel(net, dataProvider) : null); + } + + Double score = null; + if (net != null) { + if(dataSource != null){ + score = scoreFunction.score(net, dataSource, dataSourceProperties); + } else { + score = scoreFunction.score(net, dataProvider, candidate.getDataParameters()); + } + ci.setScore(score); + } + + if(taskListener != null){ + taskListener.postProcess(net, candidate); + } + + OptimizationResult result = new OptimizationResult(candidate, score, candidate.getIndex(), additionalEvaluation, ci, null); + //Save the model: + ResultSaver saver = runner.getConfiguration().getResultSaver(); + ResultReference resultReference = null; + if (saver != null) { + try { + resultReference = saver.saveModel(result, net); + } catch (IOException e) { + //TODO: Do we want ta warn or fail on IOException? + log.warn("Error saving model (id={}): IOException thrown. ", result.getIndex(), e); + } + } + result.setResultReference(resultReference); + return result; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/TaskListener.java b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/TaskListener.java new file mode 100644 index 000000000..ecf262548 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/main/java/org/deeplearning4j/arbiter/task/TaskListener.java @@ -0,0 +1,49 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.task; + +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.nn.api.Model; + +import java.io.Serializable; + +/** + * TaskListener: can be used to preprocess and post process a model (MultiLayerNetwork or ComputationGraph) before/after + * training, in a {@link MultiLayerNetworkTaskCreator} or {@link ComputationGraphTaskCreator} + * + * @author Alex Black + */ +public interface TaskListener extends Serializable { + + /** + * Preprocess the model, before any training has taken place. + *
+ * Can be used to (for example) set listeners on a model before training starts + * @param model Model to preprocess + * @param candidate Candidate information, for the current model + * @return The updated model (usually the same one as the input, perhaps with modifications) + */ + T preProcess(T model, Candidate candidate); + + /** + * Post process the model, after any training has taken place + * @param model Model to postprocess + * @param candidate Candidate information, for the current model + */ + void postProcess(Model model, Candidate candidate); + +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/AssertTestsExtendBaseClass.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..06e00219f --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.deeplearning4j.arbiter; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.nd4j.common.tests.AbstractAssertTestsClass; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.deeplearning4j.arbiter"; + } + + @Override + protected Class getBaseClass() { + return BaseDL4JTest.class; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/TestUtils.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/TestUtils.java new file mode 100644 index 000000000..ea5e0eddd --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/TestUtils.java @@ -0,0 +1,243 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter; + +import org.apache.commons.compress.utils.IOUtils; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.layers.BaseLayer; +import org.deeplearning4j.nn.conf.layers.samediff.AbstractSameDiffLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.util.ModelSerializer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.regularization.L1Regularization; +import org.nd4j.linalg.learning.regularization.L2Regularization; +import org.nd4j.linalg.learning.regularization.Regularization; +import org.nd4j.linalg.learning.regularization.WeightDecay; + +import java.io.*; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class TestUtils { + + public static MultiLayerNetwork testModelSerialization(MultiLayerNetwork net){ + + MultiLayerNetwork restored; + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ModelSerializer.writeModel(net, baos, true); + byte[] bytes = baos.toByteArray(); + + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + restored = ModelSerializer.restoreMultiLayerNetwork(bais, true); + + assertEquals(net.getLayerWiseConfigurations(), restored.getLayerWiseConfigurations()); + assertEquals(net.params(), restored.params()); + } catch (IOException e){ + //Should never happen + throw new RuntimeException(e); + } + + //Also check the MultiLayerConfiguration is serializable (required by Spark etc) + MultiLayerConfiguration conf = net.getLayerWiseConfigurations(); + serializeDeserializeJava(conf); + + return restored; + } + + public static ComputationGraph testModelSerialization(ComputationGraph net){ + + ComputationGraph restored; + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ModelSerializer.writeModel(net, baos, true); + byte[] bytes = baos.toByteArray(); + + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + restored = ModelSerializer.restoreComputationGraph(bais, true); + + assertEquals(net.getConfiguration(), restored.getConfiguration()); + assertEquals(net.params(), restored.params()); + } catch (IOException e){ + //Should never happen + throw new RuntimeException(e); + } + + //Also check the ComputationGraphConfiguration is serializable (required by Spark etc) + ComputationGraphConfiguration conf = net.getConfiguration(); + serializeDeserializeJava(conf); + + return restored; + } + + private static T serializeDeserializeJava(T object){ + byte[] bytes; + try(ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos)){ + oos.writeObject(object); + oos.close(); + bytes = baos.toByteArray(); + } catch (IOException e){ + //Should never happen + throw new RuntimeException(e); + } + + T out; + try(ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))){ + out = (T)ois.readObject(); + } catch (IOException | ClassNotFoundException e){ + throw new RuntimeException(e); + } + + assertEquals(object, out); + return out; + } + + public static INDArray randomOneHot(long examples, long nOut){ + return randomOneHot(examples, nOut, new Random(12345)); + } + + public static INDArray randomOneHot(long examples, long nOut, long rngSeed){ + return randomOneHot(examples, nOut, new Random(rngSeed)); + } + + public static INDArray randomOneHot(long examples, long nOut, Random rng){ + INDArray arr = Nd4j.create(examples, nOut); + for( int i=0; i l){ + for(Regularization r : l){ + if(r instanceof L1Regularization){ + return (L1Regularization) r; + } + } + return null; + } + + public static L2Regularization getL2Reg(BaseLayer baseLayer){ + return getL2Reg(baseLayer.getRegularization()); + } + + public static L2Regularization getL2Reg(List l){ + for(Regularization r : l){ + if(r instanceof L2Regularization){ + return (L2Regularization) r; + } + } + return null; + } + + public static WeightDecay getWeightDecayReg(BaseLayer bl){ + return getWeightDecayReg(bl.getRegularization()); + } + + public static WeightDecay getWeightDecayReg(List l){ + for(Regularization r : l){ + if(r instanceof WeightDecay){ + return (WeightDecay) r; + } + } + return null; + } + + public static double getL1(BaseLayer layer) { + List l = layer.getRegularization(); + return getL1(l); + } + + public static double getL1(List l){ + L1Regularization l1Reg = null; + for(Regularization reg : l){ + if(reg instanceof L1Regularization) + l1Reg = (L1Regularization) reg; + } + assertNotNull(l1Reg); + return l1Reg.getL1().valueAt(0,0); + } + + public static double getL2(BaseLayer layer) { + List l = layer.getRegularization(); + return getL2(l); + } + + public static double getL2(List l){ + L2Regularization l2Reg = null; + for(Regularization reg : l){ + if(reg instanceof L2Regularization) + l2Reg = (L2Regularization) reg; + } + assertNotNull(l2Reg); + return l2Reg.getL2().valueAt(0,0); + } + + public static double getL1(AbstractSameDiffLayer layer){ + return getL1(layer.getRegularization()); + } + + public static double getL2(AbstractSameDiffLayer layer){ + return getL2(layer.getRegularization()); + } + + public static double getWeightDecay(BaseLayer layer) { + return getWeightDecayReg(layer.getRegularization()).getCoeff().valueAt(0,0); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestComputationGraphSpace.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestComputationGraphSpace.java new file mode 100644 index 000000000..b34280911 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestComputationGraphSpace.java @@ -0,0 +1,168 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.computationgraph; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.ComputationGraphSpace; +import org.deeplearning4j.arbiter.TestUtils; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.graph.LayerVertex; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.BaseLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; + +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestComputationGraphSpace extends BaseDL4JTest { + + @Test + public void testBasic() { + + ComputationGraphConfiguration expected = new NeuralNetConfiguration.Builder() + .updater(new Sgd(0.005)) + .seed(12345) + .graphBuilder().addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in") + .addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).build(), "0").addLayer("2", + new OutputLayer.Builder().lossFunction(LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .nIn(10).nOut(5) + .build(), + "1") + .setOutputs("2").build(); + + ComputationGraphSpace cgs = new ComputationGraphSpace.Builder() + .updater(new Sgd(0.005)) + .seed(12345).addInputs("in") + .addLayer("0", new DenseLayerSpace.Builder().nIn(10).nOut(10).build(), "in") + .addLayer("1", new DenseLayerSpace.Builder().nIn(10).nOut(10).build(), "0") + .addLayer("2", new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(5) + .build(), "1") + .setOutputs("2").setInputTypes(InputType.feedForward(10)) + .build(); + + int nParams = cgs.numParameters(); + assertEquals(0, nParams); + + ComputationGraphConfiguration conf = cgs.getValue(new double[0]).getConfiguration(); + + assertEquals(expected, conf); + } + + @Test + public void testBasic2() { + + ComputationGraphSpace mls = new ComputationGraphSpace.Builder() + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.2, 0.5)) + .addInputs("in").addLayer("0", + new DenseLayerSpace.Builder().nIn(10).nOut(10) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), + "in") + .addLayer("1", new OutputLayerSpace.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX) + .build(), "0") + .setOutputs("1").setInputTypes(InputType.feedForward(10)).build(); + + int nParams = mls.numParameters(); + assertEquals(3, nParams); + + //Assign numbers to each leaf ParameterSpace object (normally done by candidate generator) + List noDuplicatesList = LeafUtils.getUniqueObjects(mls.collectLeaves()); + + //Second: assign each a number + int c = 0; + for (ParameterSpace ps : noDuplicatesList) { + int np = ps.numParameters(); + if (np == 1) { + ps.setIndices(c++); + } else { + int[] values = new int[np]; + for (int j = 0; j < np; j++) + values[c++] = j; + ps.setIndices(values); + } + } + + int reluCount = 0; + int tanhCount = 0; + + Random r = new Random(12345); + + for (int i = 0; i < 50; i++) { + + double[] rvs = new double[nParams]; + for (int j = 0; j < rvs.length; j++) + rvs[j] = r.nextDouble(); + + + ComputationGraphConfiguration conf = mls.getValue(rvs).getConfiguration(); + + int nLayers = conf.getVertexInputs().size(); + assertEquals(2, nLayers); + + for (int j = 0; j < nLayers; j++) { + NeuralNetConfiguration layerConf = + ((LayerVertex) conf.getVertices().get(String.valueOf(j))).getLayerConf(); + + double lr = ((Sgd)((BaseLayer) layerConf.getLayer()).getIUpdater()).getLearningRate(); + assertTrue(lr >= 0.0001 && lr <= 0.1); + double l2 = TestUtils.getL2(((BaseLayer) layerConf.getLayer())); + assertTrue(l2 >= 0.2 && l2 <= 0.5); + + if (j == nLayers - 1) { //Output layer + assertEquals(Activation.SOFTMAX.getActivationFunction(), + ((BaseLayer) layerConf.getLayer()).getActivationFn()); + } else { + IActivation actFn = ((BaseLayer) layerConf.getLayer()).getActivationFn(); + assertTrue(Activation.RELU.getActivationFunction().equals(actFn) || + Activation.TANH.getActivationFunction().equals(actFn)); + if (Activation.RELU.getActivationFunction().equals(actFn)) + reluCount++; + else + tanhCount++; + } + } + } + +// System.out.println("ReLU vs. Tanh: " + reluCount + "\t" + tanhCount); + assertTrue(reluCount > 0); + assertTrue(tanhCount > 0); + + } + + +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecution.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecution.java new file mode 100644 index 000000000..e9b8a7f73 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecution.java @@ -0,0 +1,373 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.computationgraph; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.ComputationGraphSpace; +import org.deeplearning4j.arbiter.conf.updater.AdamSpace; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.evaluator.multilayer.ClassificationEvaluator; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.multilayernetwork.TestDL4JLocalExecution; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxTimeCondition; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.ScoreFunctions; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.arbiter.task.ComputationGraphTaskCreator; +import org.deeplearning4j.arbiter.util.TestDataFactoryProviderMnist; +import org.deeplearning4j.datasets.iterator.MultipleEpochsIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; +import org.deeplearning4j.earlystopping.scorecalc.DataSetLossCalculatorCG; +import org.deeplearning4j.earlystopping.scorecalc.ScoreCalculator; +import org.deeplearning4j.earlystopping.termination.MaxEpochsTerminationCondition; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.dataset.adapter.MultiDataSetIteratorAdapter; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.function.Supplier; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class TestGraphLocalExecution extends BaseDL4JTest { + + @TempDir + public File testDir; + + @BeforeAll + public static void before(){ + Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); + } + + @Override + public long getTimeoutMilliseconds() { + return 120_000L; + } + + @Test + public void testLocalExecutionDataSources() throws Exception { + + for( int dataApproach = 0; dataApproach<3; dataApproach++ ) { + log.info("////////////////// Starting Test: {} ///////////////////", dataApproach); + + //Define: network config (hyperparameter space) + ComputationGraphSpace mls = new ComputationGraphSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)) + .addInputs("in") + .addLayer("0", + new DenseLayerSpace.Builder().nIn(784).nOut(new IntegerParameterSpace(10, 20)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), "in") //1-2 identical layers (except nIn) + .addLayer("1", new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "0") + .setOutputs("1") + .setInputTypes(InputType.feedForward(784)) + .numEpochs(3).build(); + + DataProvider dp = null; + Class ds = null; + Properties dsP = null; + CandidateGenerator candidateGenerator; + + if(dataApproach == 0){ + ds = TestDL4JLocalExecution.MnistDataSource.class; + dsP = new Properties(); + dsP.setProperty("minibatch", "2"); + candidateGenerator = new RandomSearchGenerator(mls); + } else if(dataApproach == 1) { + //DataProvider approach + dp = new TestDL4JLocalExecution.MnistDataProvider(); + candidateGenerator = new RandomSearchGenerator(mls); + } else { + //Factory approach + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + candidateGenerator = new RandomSearchGenerator(mls, commands); + dp = new DataSetIteratorFactoryProvider(); + } + + File f = testDir; + File modelSave = new File(f, "modelSaveDir"); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dp) + .dataSource(ds, dsP) + .modelSaver(new FileModelSaver(modelSave)) + .scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(20, TimeUnit.SECONDS), + new MaxCandidatesCondition(3)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration,new ComputationGraphTaskCreator(new ClassificationEvaluator())); + + runner.execute(); + + List results = runner.getResults(); + assertTrue(results.size() > 0); + +// System.out.println("----- COMPLETE - " + results.size() + " results -----"); + } + } + + + @Test + public void testLocalExecution() throws Exception { + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + //Define: network config (hyperparameter space) + ComputationGraphSpace mls = new ComputationGraphSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)).addInputs("in") + .setInputTypes(InputType.feedForward(4)) + .addLayer("layer0", + new DenseLayerSpace.Builder().nIn(784).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), + "in") + .addLayer("out", new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "layer0") + .setOutputs("out").numEpochs(3).build(); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); + DataProvider dataProvider = new DataSetIteratorFactoryProvider(); + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterDL4JTest\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + f.deleteOnExit(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)).scoreFunction(ScoreFunctions.testSetLoss(true)) + .terminationConditions(new MaxTimeCondition(30, TimeUnit.SECONDS), + new MaxCandidatesCondition(3)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, + new ComputationGraphTaskCreator(new ClassificationEvaluator())); + + runner.execute(); + + assertEquals(0, runner.numCandidatesFailed()); + assertTrue(runner.numCandidatesCompleted() > 0); + } + + @Test + public void testLocalExecutionMDS() throws Exception { + //Define: network config (hyperparameter space) + ComputationGraphSpace mls = new ComputationGraphSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)).addInputs("in") + .setInputTypes(InputType.feedForward(784)) + .addLayer("layer0", + new DenseLayerSpace.Builder().nIn(784).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), + "in") + .addLayer("out", new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "layer0") + .setOutputs("out").numEpochs(3).build(); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, null); + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterDL4JTest\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + f.deleteOnExit(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(new TestMdsDataProvider(1, 32)) + .modelSaver(new FileModelSaver(modelSavePath)).scoreFunction(ScoreFunctions.testSetLoss(true)) + .terminationConditions(new MaxTimeCondition(30, TimeUnit.SECONDS), + new MaxCandidatesCondition(3)) + .scoreFunction(ScoreFunctions.testSetAccuracy()) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new ComputationGraphTaskCreator()); + + runner.execute(); + + assertEquals(0, runner.numCandidatesFailed()); + assertTrue(runner.numCandidatesCompleted() > 0); + } + + public static class TestMdsDataProvider implements DataProvider { + private int numEpochs; + private int batchSize; + + public TestMdsDataProvider(@JsonProperty("numEpochs") int numEpochs, @JsonProperty("batchSize") int batchSize) { + this.numEpochs = numEpochs; + this.batchSize = batchSize; + } + + private TestMdsDataProvider() { + } + + + @Override + public Object trainData(Map dataParameters) { + try { + DataSetIterator underlying = new MnistDataSetIterator(batchSize, Math.min(60000, 3 * batchSize), false, true, true, 12345); + return new MultiDataSetIteratorAdapter(new MultipleEpochsIterator(numEpochs, underlying)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Object testData(Map dataParameters) { + try { + DataSetIterator underlying = new MnistDataSetIterator(batchSize, Math.min(10000, 2 * batchSize), false, false, false, 12345); + return new MultiDataSetIteratorAdapter(underlying); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return MultiDataSetIterator.class; + } + } + + @Test + public void testLocalExecutionEarlyStopping() throws Exception { + EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(2)) + .scoreCalculator(new ScoreProvider()) + .modelSaver(new InMemoryModelSaver()).build(); + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + //Define: network config (hyperparameter space) + ComputationGraphSpace cgs = new ComputationGraphSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new AdamSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)).addInputs("in") + .setInputTypes(InputType.feedForward(784)) + .addLayer("first", + new DenseLayerSpace.Builder().nIn(784).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), + "in") //1-2 identical layers (except nIn) + .addLayer("out", new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "first") + .setOutputs("out").earlyStoppingConfiguration(esConf).build(); + + //Define configuration: + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs, commands); + DataProvider dataProvider = new DataSetIteratorFactoryProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterDL4JTest2CG\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + f.deleteOnExit(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dataProvider) + .scoreFunction(ScoreFunctions.testSetF1()) + .modelSaver(new FileModelSaver(modelSavePath)) + .terminationConditions(new MaxTimeCondition(15, TimeUnit.SECONDS), + new MaxCandidatesCondition(3)) + .build(); + + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new ComputationGraphTaskCreator()); + runner.execute(); + + assertEquals(0, runner.numCandidatesFailed()); + assertTrue(runner.numCandidatesCompleted() > 0); + } + + private static class ScoreProvider implements Supplier, Serializable { + @Override + public ScoreCalculator get() { + try { + return new DataSetLossCalculatorCG(new MnistDataSetIterator(4, 8), true); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecutionGenetic.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecutionGenetic.java new file mode 100644 index 000000000..05815a020 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/computationgraph/TestGraphLocalExecutionGenetic.java @@ -0,0 +1,212 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.computationgraph; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.ComputationGraphSpace; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.evaluator.multilayer.ClassificationEvaluator; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.multilayernetwork.TestDL4JLocalExecution; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxTimeCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.generator.GeneticSearchCandidateGenerator; +import org.deeplearning4j.arbiter.optimize.generator.genetic.population.PopulationModel; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.arbiter.task.ComputationGraphTaskCreator; +import org.deeplearning4j.arbiter.util.TestDataFactoryProviderMnist; +import org.deeplearning4j.datasets.iterator.MultipleEpochsIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.earlystopping.scorecalc.DataSetLossCalculatorCG; +import org.deeplearning4j.earlystopping.scorecalc.ScoreCalculator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.dataset.adapter.MultiDataSetIteratorAdapter; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.common.function.Supplier; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class TestGraphLocalExecutionGenetic extends BaseDL4JTest { + + @TempDir + public File testDir; + + @Override + public long getTimeoutMilliseconds() { + return 120_000L; + } + + @Test + public void testLocalExecutionDataSources() throws Exception { + for (int dataApproach = 0; dataApproach < 3; dataApproach++) { + log.info("////////////////// Starting Test: {} ///////////////////", dataApproach); + + //Define: network config (hyperparameter space) + ComputationGraphSpace mls = new ComputationGraphSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)) + .addInputs("in") + .addLayer("0", + new DenseLayerSpace.Builder().nIn(784).nOut(new IntegerParameterSpace(5, 32)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH, Activation.LEAKYRELU)) + .build(), "in") //1-2 identical layers (except nIn) + .addLayer("1", new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "0") + .setOutputs("1") + .setInputTypes(InputType.feedForward(784)) + .numEpochs(3).build(); + + DataProvider dp = null; + Class ds = null; + Properties dsP = null; + CandidateGenerator candidateGenerator; + + TestSetLossScoreFunction scoreFunction = new TestSetLossScoreFunction(); + + if (dataApproach == 0) { + ds = TestDL4JLocalExecution.MnistDataSource.class; + dsP = new Properties(); + dsP.setProperty("minibatch", "2"); + + candidateGenerator = new GeneticSearchCandidateGenerator.Builder(mls, scoreFunction) + .populationModel(new PopulationModel.Builder().populationSize(5).build()) + .build(); + } else if (dataApproach == 1) { + //DataProvider approach + dp = new TestDL4JLocalExecution.MnistDataProvider(); + + candidateGenerator = new GeneticSearchCandidateGenerator.Builder(mls, scoreFunction) + .populationModel(new PopulationModel.Builder().populationSize(5).build()) + .build(); + } else { + //Factory approach + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + candidateGenerator = new GeneticSearchCandidateGenerator.Builder(mls, scoreFunction) + .dataParameters(commands) + .populationModel(new PopulationModel.Builder().populationSize(5).build()) + .build(); + dp = new DataSetIteratorFactoryProvider(); + } + + File f = testDir; + File modelSave = new File(f, "modelSaveDir"); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dp) + .dataSource(ds, dsP) + .modelSaver(new FileModelSaver(modelSave)) + .scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(20, TimeUnit.SECONDS), + new MaxCandidatesCondition(3)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new ComputationGraphTaskCreator(new ClassificationEvaluator())); + + runner.execute(); + + List results = runner.getResults(); + assertTrue(results.size() > 0); + +// System.out.println("----- COMPLETE - " + results.size() + " results -----"); + } + } + + public static class TestMdsDataProvider implements DataProvider { + private int numEpochs; + private int batchSize; + + public TestMdsDataProvider(@JsonProperty("numEpochs") int numEpochs, @JsonProperty("batchSize") int batchSize) { + this.numEpochs = numEpochs; + this.batchSize = batchSize; + } + + private TestMdsDataProvider() { + } + + + @Override + public Object trainData(Map dataParameters) { + try { + DataSetIterator underlying = new MnistDataSetIterator(batchSize, Math.min(60000, 10 * batchSize), false, true, true, 12345); + return new MultiDataSetIteratorAdapter(new MultipleEpochsIterator(numEpochs, underlying)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Object testData(Map dataParameters) { + try { + DataSetIterator underlying = new MnistDataSetIterator(batchSize, Math.min(10000, 5 * batchSize), false, false, false, 12345); + return new MultiDataSetIteratorAdapter(underlying); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return MultiDataSetIterator.class; + } + } + + private static class ScoreProvider implements Supplier, Serializable { + @Override + public ScoreCalculator get() { + try { + return new DataSetLossCalculatorCG(new MnistDataSetIterator(128, 1280), true); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/json/TestJson.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/json/TestJson.java new file mode 100644 index 000000000..f4d539089 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/json/TestJson.java @@ -0,0 +1,268 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.json; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.ComputationGraphSpace; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.conf.updater.AdaMaxSpace; +import org.deeplearning4j.arbiter.conf.updater.AdamSpace; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.multilayernetwork.MnistDataSetIteratorFactory; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxTimeCondition; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import org.deeplearning4j.arbiter.scoring.RegressionValue; +import org.deeplearning4j.arbiter.scoring.ScoreFunctions; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.arbiter.util.TestDataFactoryProviderMnist; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; +import org.deeplearning4j.earlystopping.scorecalc.ClassificationScoreCalculator; +import org.deeplearning4j.earlystopping.scorecalc.DataSetLossCalculatorCG; +import org.deeplearning4j.earlystopping.scorecalc.base.BaseIEvaluationScoreCalculator; +import org.deeplearning4j.earlystopping.termination.MaxEpochsTerminationCondition; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.eval.IEvaluation; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Created by Alex on 14/02/2017. + */ +public class TestJson extends BaseDL4JTest { + + @Test + public void testMultiLayerSpaceJson() { + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .addLayer(new DenseLayerSpace.Builder().nIn(1).nOut(new IntegerParameterSpace(5, 30)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.SOFTPLUS, + Activation.LEAKYRELU)) + .build(), new IntegerParameterSpace(1, 2), true) //1-2 identical layers + .addLayer(new DenseLayerSpace.Builder().nIn(4).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers + .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .iLossFunction(LossFunctions.LossFunction.MCXENT.getILossFunction()).build()) + .setInputType(InputType.convolutional(28, 28, 1)).build(); + + String asJson = mls.toJson(); + // System.out.println(asJson); + + MultiLayerSpace fromJson = MultiLayerSpace.fromJson(asJson); + + assertEquals(mls, fromJson); + } + + + + @Test + public void testOptimizationFromJson() { + EarlyStoppingConfiguration esConf = + new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(100)) + .scoreCalculator(new DataSetLossCalculatorCG(new IrisDataSetIterator(150, 150), + true)) + .modelSaver(new InMemoryModelSaver()).build(); + + //Define: network config (hyperparameter space) + ComputationGraphSpace cgs = new ComputationGraphSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new AdaMaxSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)).addInputs("in") + .setInputTypes(InputType.feedForward(4)) + .addLayer("first", + new DenseLayerSpace.Builder().nIn(4).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), + "in") //1-2 identical layers (except nIn) + .addLayer("out", new OutputLayerSpace.Builder().nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "first") + .setOutputs("out").earlyStoppingConfiguration(esConf).build(); + + //Define configuration: + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs, commands); + DataProvider dataProvider = new DataSetIteratorFactoryProvider(); + + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder().candidateGenerator(candidateGenerator) + .dataProvider(dataProvider).scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(2, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + String json = configuration.toJson(); + OptimizationConfiguration loadConf = OptimizationConfiguration.fromJson(json); + assertEquals(configuration, loadConf); + } + + @Test + public void testOptimizationFromJsonDataSource() { + for(boolean withProperties : new boolean[]{false, true}) { + //Define: network config (hyperparameter space) + ComputationGraphSpace cgs = new ComputationGraphSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new AdaMaxSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)).addInputs("in") + .setInputTypes(InputType.feedForward(4)) + .addLayer("first", + new DenseLayerSpace.Builder().nIn(4).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), + "in") //1-2 identical layers (except nIn) + .addLayer("out", new OutputLayerSpace.Builder().nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "first") + .setOutputs("out").build(); + + //Define configuration: + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs, commands); + + Properties p = new Properties(); + p.setProperty("minibatch", "16"); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder().candidateGenerator(candidateGenerator) + .dataSource(MnistDataSource.class, (withProperties ? p : null)) + .scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(2, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + String json = configuration.toJson(); + OptimizationConfiguration loadConf = OptimizationConfiguration.fromJson(json); + assertEquals(configuration, loadConf); + assertNotNull(loadConf.getDataSource()); + if(withProperties){ + assertNotNull(loadConf.getDataSourceProperties()); + } + } + } + + @Test + public void testComputationGraphSpaceJson() { + ParameterSpace p = new IntegerParameterSpace(10, 100); + ComputationGraphSpace cgs = + new ComputationGraphSpace.Builder() + .updater(new AdamSpace(new DiscreteParameterSpace<>(0.1, 0.5, 1.0))) + .seed(12345).addInputs("in") + .addLayer("0", new DenseLayerSpace.Builder() + .nIn(new IntegerParameterSpace(1, 100)).nOut(p).build(), "in") + .addLayer("1", new DenseLayerSpace.Builder().nIn(p).nOut(10).build(), "0") + .addLayer("2", new OutputLayerSpace.Builder().iLossFunction( + LossFunctions.LossFunction.MCXENT.getILossFunction()).nIn(10) + .nOut(5).build(), "1") + .setOutputs("2").build(); + + String asJson = cgs.toJson(); + ComputationGraphSpace fromJson = ComputationGraphSpace.fromJson(asJson); + + assertEquals(cgs, fromJson); + } + + @Test + public void testScoreFunctionJson() throws Exception { + + ScoreFunction[] scoreFunctions = new ScoreFunction[]{ + ScoreFunctions.testSetAccuracy(), ScoreFunctions.testSetF1(), + ScoreFunctions.testSetLoss(true), ScoreFunctions.testSetRegression(RegressionValue.MAE), + ScoreFunctions.testSetRegression(RegressionValue.RMSE)}; + + for(ScoreFunction sc : scoreFunctions){ + String json = JsonMapper.getMapper().writeValueAsString(sc); + ScoreFunction fromJson = JsonMapper.getMapper().readValue(json, ScoreFunction.class); + + assertEquals(sc, fromJson); + } + } + + + public static class MnistDataSource implements DataSource { + private int minibatch; + + public MnistDataSource(){ + + } + + @Override + public void configure(Properties properties) { + this.minibatch = Integer.parseInt(properties.getProperty("minibatch", "16")); + } + + @Override + public Object trainData() { + try { + return new MnistDataSetIterator(minibatch, true, 12345); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + @Override + public Object testData() { + try { + return new MnistDataSetIterator(minibatch, true, 12345); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/MNISTOptimizationTest.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/MNISTOptimizationTest.java new file mode 100644 index 000000000..ea754990a --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/MNISTOptimizationTest.java @@ -0,0 +1,166 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.multilayernetwork; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.layers.ConvolutionLayerSpace; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxTimeCondition; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.arbiter.task.MultiLayerNetworkTaskCreator; +import org.deeplearning4j.arbiter.util.TestDataFactoryProviderMnist; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; +import org.deeplearning4j.earlystopping.scorecalc.DataSetLossCalculator; +import org.deeplearning4j.earlystopping.termination.MaxEpochsTerminationCondition; +import org.deeplearning4j.earlystopping.termination.MaxScoreIterationTerminationCondition; +import org.deeplearning4j.earlystopping.termination.MaxTimeIterationTerminationCondition; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +// import org.deeplearning4j.arbiter.optimize.ui.ArbiterUIServer; +// import org.deeplearning4j.arbiter.optimize.ui.listener.UIOptimizationRunnerStatusListener; + +/** Not strictly a unit test. Rather: part example, part debugging on MNIST */ +public class MNISTOptimizationTest extends BaseDL4JTest { + + public static void main(String[] args) throws Exception { + EarlyStoppingConfiguration esConf = + new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(3)) + .iterationTerminationConditions( + new MaxTimeIterationTerminationCondition(5, TimeUnit.MINUTES), + new MaxScoreIterationTerminationCondition(4.6) //Random score: -log_e(0.1) ~= 2.3 + ).scoreCalculator(new DataSetLossCalculator(new MnistDataSetIterator(64, 2000, false, false, true, 123), true)).modelSaver(new InMemoryModelSaver()).build(); + + //Define: network config (hyperparameter space) + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .addLayer( + new ConvolutionLayerSpace.Builder().nIn(1) + .nOut(new IntegerParameterSpace(5, 30)) + .kernelSize(new DiscreteParameterSpace<>(new int[] {3, 3}, + new int[] {4, 4}, new int[] {5, 5})) + .stride(new DiscreteParameterSpace<>(new int[] {1, 1}, + new int[] {2, 2})) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.SOFTPLUS, Activation.LEAKYRELU)) + .build(), + new IntegerParameterSpace(1, 2)) //1-2 identical layers + .addLayer(new DenseLayerSpace.Builder().nIn(4).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), new IntegerParameterSpace(0, 1)) //0 to 1 layers + .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .earlyStoppingConfiguration(esConf).build(); + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); + DataProvider dataProvider = new MnistDataSetProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterMNISTSmall\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)).scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(120, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + + // ArbiterUIServer server = ArbiterUIServer.getInstance(); + // runner.addListeners(new UIOptimizationRunnerStatusListener(server)); + + runner.execute(); + + + System.out.println("----- COMPLETE -----"); + } + + + private static class MnistDataSetProvider implements DataProvider { + + @Override + public DataSetIterator trainData(Map dataParameters) { + try { + if (dataParameters == null || dataParameters.isEmpty()) { + return new MnistDataSetIterator(64, 10000, false, true, true, 123); + } + if (dataParameters.containsKey("batchsize")) { + int b = (Integer) dataParameters.get("batchsize"); + return new MnistDataSetIterator(b, 10000, false, true, true, 123); + } + return new MnistDataSetIterator(64, 10000, false, true, true, 123); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public DataSetIterator testData(Map dataParameters) { + return trainData(dataParameters); + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + + @Override + public String toString() { + return "MnistDataSetProvider()"; + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/MnistDataSetIteratorFactory.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/MnistDataSetIteratorFactory.java new file mode 100644 index 000000000..55c2643a9 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/MnistDataSetIteratorFactory.java @@ -0,0 +1,42 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.multilayernetwork; + +import lombok.Data; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIteratorFactory; + +import java.io.IOException; + +/** + * Created by agibsonccc on 3/13/17. + */ +@Data +public class MnistDataSetIteratorFactory implements DataSetIteratorFactory { + /** + * @return + */ + @Override + public DataSetIterator create() { + try { + return new MnistDataSetIterator(1000, 1000); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestDL4JLocalExecution.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestDL4JLocalExecution.java new file mode 100644 index 000000000..aee1d022c --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestDL4JLocalExecution.java @@ -0,0 +1,381 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.multilayernetwork; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.evaluator.multilayer.ClassificationEvaluator; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OCNNLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxTimeCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.generator.GridSearchCandidateGenerator; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.arbiter.task.MultiLayerNetworkTaskCreator; +import org.deeplearning4j.arbiter.util.TestDataFactoryProviderMnist; +import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; +import org.deeplearning4j.earlystopping.scorecalc.DataSetLossCalculator; +import org.deeplearning4j.earlystopping.termination.MaxEpochsTerminationCondition; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class TestDL4JLocalExecution extends BaseDL4JTest { + + @TempDir + public File testDir; + + @BeforeAll + public static void before(){ + Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); + } + + @Test + public void testLocalExecution() throws Exception { + + for( int dataApproach = 0; dataApproach<3; dataApproach++ ) { + log.info("////////////////// Starting Test: {} ///////////////////", dataApproach); + + //Define: network config (hyperparameter space) + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)) + .addLayer( + new DenseLayerSpace.Builder().nIn(784).nOut(new IntegerParameterSpace(10, 20)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build()) //1-2 identical layers (except nIn) + .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .numEpochs(3).build(); + + DataProvider dp = null; + Class ds = null; + Properties dsP = null; + CandidateGenerator candidateGenerator; + + if(dataApproach == 0){ + ds = MnistDataSource.class; + dsP = new Properties(); + dsP.setProperty("minibatch", "2"); + candidateGenerator = new RandomSearchGenerator(mls); + } else if(dataApproach == 1) { + //DataProvider approach + dp = new MnistDataProvider(); + candidateGenerator = new RandomSearchGenerator(mls); + } else { + //Factory approach + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + candidateGenerator = new RandomSearchGenerator(mls, commands); + dp = new DataSetIteratorFactoryProvider(); + } + + File f = testDir; + File modelSave = new File(f, "modelSaveDir"); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dp) + .dataSource(ds, dsP) + .modelSaver(new FileModelSaver(modelSave)) + .scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(5, TimeUnit.SECONDS), + new MaxCandidatesCondition(5)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, + new MultiLayerNetworkTaskCreator(new ClassificationEvaluator())); + + runner.execute(); + + List results = runner.getResults(); + assertTrue(results.size() > 0); + + System.out.println("----- COMPLETE - " + results.size() + " results -----"); + } + } + + public static class MnistDataSource implements DataSource { + private int minibatch; + + public MnistDataSource(){ + + } + + @Override + public void configure(Properties properties) { + this.minibatch = Integer.parseInt(properties.getProperty("minibatch", "16")); + } + + @Override + public Object trainData() { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(minibatch, true, 12345), 3); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + @Override + public Object testData() { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(minibatch, true, 12345), 3); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + } + + public static class MnistDataProvider implements DataProvider { + private int minibatch = 8; + + @Override + public Object trainData(Map dataParameters) { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(minibatch, true, 12345), 3); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + @Override + public Object testData(Map dataParameters) { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(minibatch, true, 12345), 3); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + } + + @Test + //@org.junit.Ignore + public void testLocalExecutionGridSearch() throws Exception { + + //Define: network config (hyperparameter space) + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)) + .addLayer( + new DenseLayerSpace.Builder().nIn(4).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), + new IntegerParameterSpace(1, 2)) //1-2 identical layers (except nIn) + .addLayer(new OutputLayerSpace.Builder().nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .numEpochs(3).build(); + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + CandidateGenerator candidateGenerator = new GridSearchCandidateGenerator(mls, 5, + GridSearchCandidateGenerator.Mode.Sequential, commands); + DataProvider dataProvider = new DataSetIteratorFactoryProvider(); + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterDL4JTest/").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + f.deleteOnExit(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)).scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(2, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, + new MultiLayerNetworkTaskCreator(new ClassificationEvaluator())); + + runner.execute(); + + System.out.println("----- COMPLETE -----"); + } + + @Test + //@Ignore + public void testLocalExecutionEarlyStopping() throws Exception { + EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(100)) + .scoreCalculator(new DataSetLossCalculator(new IrisDataSetIterator(150, 150), true)) + .modelSaver(new InMemoryModelSaver()).build(); + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + + //Define: network config (hyperparameter space) + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)) + .addLayer(new DenseLayerSpace.Builder().nIn(4).nOut(new IntegerParameterSpace(2, 10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), + new IntegerParameterSpace(1, 2)) //1-2 identical layers (except nIn) + .addLayer(new OutputLayerSpace.Builder().nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .earlyStoppingConfiguration(esConf).build(); + + //Define configuration: + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); + DataProvider dataProvider = new DataSetIteratorFactoryProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterDL4JTest2\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + f.deleteOnExit(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)).scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(2, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, + new MultiLayerNetworkTaskCreator(new ClassificationEvaluator())); + + runner.execute(); + System.out.println("----- COMPLETE -----"); + } + + + @Test + public void testOcnn() { + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + + //Define: network config (hyperparameter space) + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)) + .addLayer( + new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(250, 500)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), + new IntegerParameterSpace(1, 2)) //1-2 identical layers (except nIn) + .addLayer(new OCNNLayerSpace.Builder().nu(new ContinuousParameterSpace(0.0001, 0.1)) + .numHidden(new DiscreteParameterSpace(784 / 2,784 / 4)) + .activation(Activation.HARDSIGMOID) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.convolutionalFlat(28,28,1)) + .build(); + + //Define configuration: + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); + DataProvider dataProvider = new DataSetIteratorFactoryProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterDL4JTest3\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + f.deleteOnExit(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)).scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(2, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + + //candidate generation: uncomment execute if you want to run + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, + new MultiLayerNetworkTaskCreator(new ClassificationEvaluator())); + + Candidate candidate = candidateGenerator.getCandidate(); + + // runner.execute(); + System.out.println("----- COMPLETE -----"); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestErrors.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestErrors.java new file mode 100644 index 000000000..a3d4e3657 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestErrors.java @@ -0,0 +1,158 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.multilayernetwork; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.ComputationGraphSpace; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.arbiter.task.MultiLayerNetworkTaskCreator; +import org.deeplearning4j.arbiter.util.TestDataProviderMnist; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; + +@Timeout(20) +public class TestErrors extends BaseDL4JTest { + + @TempDir + public File temp; + + @Test + public void testAllInvalidConfig() throws Exception { + //Invalid config - basically check that this actually terminates + + File f = temp; + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .addLayer(new DenseLayerSpace.Builder().nIn(4).nOut(new FixedValue<>(0)) //INVALID: nOut of 0 + .activation(Activation.TANH) + .build()) + .addLayer(new OutputLayerSpace.Builder().nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(new TestDataProviderMnist(32, 3)) + .modelSaver(new FileModelSaver(f)).scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions( + new MaxCandidatesCondition(5)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration); + runner.execute(); + } + + + @Test + public void testAllInvalidDataConfigMismatch() throws Exception { + //Valid config - but mismatched with provided data + + File f = temp; + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .addLayer(new DenseLayerSpace.Builder().nIn(4).nOut(10) //INVALID: nOut of 0 + .activation(Activation.TANH) + .build()) + .addLayer(new OutputLayerSpace.Builder().nIn(10).nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(new TestDataProviderMnist(32, 3)) + .modelSaver(new FileModelSaver(f)).scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions( + new MaxCandidatesCondition(5)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration); + runner.execute(); + } + + + @Test + public void testAllInvalidConfigCG() throws Exception { + //Invalid config - basically check that this actually terminates + + File f = temp; + ComputationGraphSpace mls = new ComputationGraphSpace.Builder() + .addInputs("in") + .layer("0", new DenseLayerSpace.Builder().nIn(4).nOut(new FixedValue<>(0)) //INVALID: nOut of 0 + .activation(Activation.TANH) + .build(), "in") + .layer("1", new OutputLayerSpace.Builder().nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "0") + .setOutputs("1") + .build(); + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(new TestDataProviderMnist(32, 3)) + .modelSaver(new FileModelSaver(f)).scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxCandidatesCondition(5)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration); + runner.execute(); + } + + + @Test + public void testAllInvalidDataConfigMismatchCG() throws Exception { + //Valid config - but mismatched with provided data + + File f = temp; + ComputationGraphSpace mls = new ComputationGraphSpace.Builder() + .addInputs("in") + .layer("0", new DenseLayerSpace.Builder().nIn(4).nOut(10) + .activation(Activation.TANH).build(), "in") + .addLayer("1", new OutputLayerSpace.Builder().nIn(10).nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "0") + .setOutputs("1") + .build(); + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls); + + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(new TestDataProviderMnist(32, 3)) + .modelSaver(new FileModelSaver(f)).scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions( + new MaxCandidatesCondition(5)) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + runner.execute(); + } + +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestLayerSpace.java new file mode 100644 index 000000000..3f25c66db --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestLayerSpace.java @@ -0,0 +1,314 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.multilayernetwork; + +import org.apache.commons.lang3.ArrayUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.TestUtils; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.layers.*; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.BooleanSpace; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.nn.api.layers.LayerConstraint; +import org.deeplearning4j.nn.conf.constraint.MaxNormConstraint; +import org.deeplearning4j.nn.conf.constraint.MinMaxNormConstraint; +import org.deeplearning4j.nn.conf.constraint.NonNegativeConstraint; +import org.deeplearning4j.nn.conf.constraint.UnitNormConstraint; +import org.deeplearning4j.nn.conf.layers.*; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.learning.config.Sgd; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestLayerSpace extends BaseDL4JTest { + + @Test + public void testBasic1() { + + DenseLayer expected = new DenseLayer.Builder().nOut(13).activation(Activation.RELU).build(); + + DenseLayerSpace space = new DenseLayerSpace.Builder().nOut(13).activation(Activation.RELU).build(); + + int nParam = space.numParameters(); + assertEquals(0, nParam); + DenseLayer actual = space.getValue(new double[nParam]); + + assertEquals(expected, actual); + } + + @Test + public void testBasic2() { + + Activation[] actFns = new Activation[]{Activation.SOFTSIGN, Activation.RELU, Activation.LEAKYRELU}; + Random r = new Random(12345); + + for (int i = 0; i < 20; i++) { + + new DenseLayer.Builder().build(); + + DenseLayerSpace ls = + new DenseLayerSpace.Builder().nOut(20) + .updater(new SgdSpace(new ContinuousParameterSpace(0.3, 0.4))) + .l2(new ContinuousParameterSpace(0.01, 0.1)) + .activation(new DiscreteParameterSpace<>(actFns)).build(); + + //Set the parameter numbers... + List list = ls.collectLeaves(); + int k = 0; + for (int j = 0; j < list.size(); j++) { + if (list.get(j).numParameters() > 0) { + list.get(j).setIndices(k++); + } + } + + int nParam = ls.numParameters(); + assertEquals(3, nParam); + + double[] d = new double[nParam]; + for (int j = 0; j < d.length; j++) { + d[j] = r.nextDouble(); + } + + DenseLayer l = ls.getValue(d); + + assertEquals(20, l.getNOut()); + double lr = ((Sgd) l.getIUpdater()).getLearningRate(); + double l2 = TestUtils.getL2(l); + IActivation activation = l.getActivationFn(); + +// System.out.println(lr + "\t" + l2 + "\t" + activation); + + assertTrue(lr >= 0.3 && lr <= 0.4); + assertTrue(l2 >= 0.01 && l2 <= 0.1); + assertTrue(containsActivationFunction(actFns, activation)); + } + } + + @Test + public void testBatchNorm() { + BatchNormalizationSpace sp = new BatchNormalizationSpace.Builder().gamma(1.5) + .beta(new ContinuousParameterSpace(2, 3)).lockGammaBeta(true).build(); + + //Set the parameter numbers... + List list = sp.collectLeaves(); + int k = 0; + for (int j = 0; j < list.size(); j++) { + if (list.get(j).numParameters() > 0) { + list.get(j).setIndices(k++); + } + } + + BatchNormalization bn = sp.getValue(new double[]{0.6}); + assertTrue(bn.isLockGammaBeta()); + assertEquals(1.5, bn.getGamma(), 0.0); + assertEquals(0.6 * (3 - 2) + 2, bn.getBeta(), 1e-4); + } + + @Test + public void testBatchNormConstrain() { + + ArrayList> constrainListOptions = new ArrayList>(); + constrainListOptions.add(Collections.singletonList((LayerConstraint) new MaxNormConstraint(0.5, 1))); + constrainListOptions.add(Collections.singletonList((LayerConstraint) new MinMaxNormConstraint(0.3, 0.4, 1.0, 1))); + constrainListOptions.add(Collections.singletonList((LayerConstraint) new NonNegativeConstraint())); + constrainListOptions.add(Collections.singletonList((LayerConstraint) new UnitNormConstraint(1))); + + DiscreteParameterSpace> constrainParamSpace = new DiscreteParameterSpace<>(constrainListOptions); + BatchNormalizationSpace sp = new BatchNormalizationSpace.Builder().gamma(1.5) + .beta(0.6).lockGammaBeta(true).constrainBeta(constrainParamSpace).constrainGamma(new NonNegativeConstraint()).build(); + + BatchNormalization bnExpected = new BatchNormalization.Builder().gamma(1.5) + .beta(0.6).lockGammaBeta(true).constrainBeta(new NonNegativeConstraint()).constrainGamma(new NonNegativeConstraint()).build(); + //Set the parameter numbers... + List list = sp.collectLeaves(); + int k = 0; + for( + int j = 0; j 0) { + list.get(j).setIndices(k++); + } + } + + assertEquals(1,sp.getNumParameters()); + BatchNormalization bn = sp.getValue(new double[]{0.6}); + assertEquals(bnExpected,bn); //0.6 should pick the 3rd value in discrete param space + + //assertEquals(bn.getConstraints().size(),2); This throws an NPE but I believe this is an issue with actual impl of BatchNormalization not arbiter +} + + @Test + public void testActivationLayer() { + Activation[] actFns = new Activation[]{Activation.SOFTSIGN, Activation.RELU, Activation.LEAKYRELU}; + + ActivationLayerSpace als = + new ActivationLayerSpace.Builder().activation(new DiscreteParameterSpace<>(actFns)).build(); + //Set the parameter numbers... + List list = als.collectLeaves(); + for (int j = 0; j < list.size(); j++) { + list.get(j).setIndices(j); + } + + int nParam = als.numParameters(); + assertEquals(1, nParam); + + Random r = new Random(12345); + + for (int i = 0; i < 20; i++) { + + double[] d = new double[nParam]; + for (int j = 0; j < d.length; j++) { + d[j] = r.nextDouble(); + } + + ActivationLayer al = als.getValue(d); + IActivation activation = al.getActivationFn(); + +// System.out.println(activation); + + assertTrue(containsActivationFunction(actFns, activation)); + } + } + + @Test + public void testEmbeddingLayer() { + + Activation[] actFns = new Activation[]{Activation.SOFTSIGN, Activation.RELU, Activation.LEAKYRELU}; + + EmbeddingLayerSpace els = new EmbeddingLayerSpace.Builder().activation(new DiscreteParameterSpace<>(actFns)) + .nIn(10).nOut(new IntegerParameterSpace(10, 20)).build(); + //Set the parameter numbers... + List list = els.collectLeaves(); + int k = 0; + for (int j = 0; j < list.size(); j++) { + if (list.get(j).numParameters() > 0) { + list.get(j).setIndices(k++); + } + } + + int nParam = els.numParameters(); + assertEquals(2, nParam); + + Random r = new Random(12345); + + for (int i = 0; i < 20; i++) { + + double[] d = new double[nParam]; + for (int j = 0; j < d.length; j++) { + d[j] = r.nextDouble(); + } + + EmbeddingLayer el = els.getValue(d); + IActivation activation = el.getActivationFn(); + long nOut = el.getNOut(); + +// System.out.println(activation + "\t" + nOut); + + assertTrue(containsActivationFunction(actFns, activation)); + assertTrue(nOut >= 10 && nOut <= 20); + } + } + + @Test + public void testSimpleConv() { + ConvolutionLayer conv2d = new Convolution2D.Builder().dilation(1,2).kernelSize(2,2).nIn(2).nOut(3).build(); + ConvolutionLayerSpace conv2dSpace = new ConvolutionLayerSpace.Builder().dilation(1,2).kernelSize(2,2).nIn(2).nOut(3).build(); + assertEquals(0,conv2dSpace.getNumParameters()); + assertEquals(conv2d, conv2dSpace.getValue(new double[0])); + + Deconvolution2DLayerSpace deconvd2dls = new Deconvolution2DLayerSpace.Builder().dilation(2,1).nIn(2).nOut(2).hasBias(new BooleanSpace()).build(); + assertEquals(1, deconvd2dls.getNumParameters()); + //Set the parameter numbers... + List list = deconvd2dls.collectLeaves(); + int k = 0; + for( + int j = 0; j 0) { + list.get(j).setIndices(k++); + } + } + Deconvolution2D actual = deconvd2dls.getValue(new double[]{0.9}); + assertTrue(!actual.hasBias()); + assertEquals(ArrayUtils.toString(new int[] {2,1} ),ArrayUtils.toString(actual.getDilation())); + } + + @Test + public void testGravesBidirectionalLayer() { + + Activation[] actFns = new Activation[]{Activation.SOFTSIGN, Activation.RELU, Activation.LEAKYRELU}; + + GravesBidirectionalLSTMLayerSpace ls = + new GravesBidirectionalLSTMLayerSpace.Builder().activation(new DiscreteParameterSpace<>(actFns)) + .forgetGateBiasInit(new ContinuousParameterSpace(0.5, 0.8)).nIn(10) + .nOut(new IntegerParameterSpace(10, 20)).build(); + //Set the parameter numbers... + List list = ls.collectLeaves(); + int k = 0; + for (int j = 0; j < list.size(); j++) { + if (list.get(j).numParameters() > 0) { + list.get(j).setIndices(k++); + } + } + + int nParam = ls.numParameters(); + assertEquals(3, nParam); //Excluding fixed value for nIn + + Random r = new Random(12345); + + for (int i = 0; i < 20; i++) { + + double[] d = new double[nParam]; + for (int j = 0; j < d.length; j++) { + d[j] = r.nextDouble(); + } + + GravesBidirectionalLSTM el = ls.getValue(d); + IActivation activation = el.getActivationFn(); + long nOut = el.getNOut(); + double forgetGate = el.getForgetGateBiasInit(); + +// System.out.println(activation + "\t" + nOut + "\t" + forgetGate); + + assertTrue(containsActivationFunction(actFns, activation)); + assertTrue(nOut >= 10 && nOut <= 20); + assertTrue(forgetGate >= 0.5 && forgetGate <= 0.8); + } + } + + private static boolean containsActivationFunction(Activation[] activationFunctions, + IActivation activationFunction) { + for (Activation af : activationFunctions) { + if (activationFunction.equals(af.getActivationFunction())) + return true; + } + return false; + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestMultiLayerSpace.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestMultiLayerSpace.java new file mode 100644 index 000000000..784df4628 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestMultiLayerSpace.java @@ -0,0 +1,819 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.multilayernetwork; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.DL4JConfiguration; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.TestUtils; +import org.deeplearning4j.arbiter.conf.updater.AdamSpace; +import org.deeplearning4j.arbiter.conf.updater.NesterovsSpace; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.layers.*; +import org.deeplearning4j.arbiter.optimize.api.Candidate; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultSaver; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.TerminationCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.generator.GridSearchCandidateGenerator; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.parameter.FixedValue; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.math.MathOp; +import org.deeplearning4j.arbiter.optimize.parameter.math.Op; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.impl.TestSetAccuracyScoreFunction; +import org.deeplearning4j.arbiter.task.MultiLayerNetworkTaskCreator; +import org.deeplearning4j.arbiter.util.LeafUtils; +import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.constraint.NonNegativeConstraint; +import org.deeplearning4j.nn.conf.constraint.UnitNormConstraint; +import org.deeplearning4j.nn.conf.dropout.Dropout; +import org.deeplearning4j.nn.conf.dropout.IDropout; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.variational.BernoulliReconstructionDistribution; +import org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution; +import org.deeplearning4j.nn.conf.layers.variational.ReconstructionDistribution; +import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.nn.layers.recurrent.BidirectionalLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.ILossFunction; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; +import org.nd4j.linalg.lossfunctions.impl.LossMCXENT; +import org.nd4j.linalg.lossfunctions.impl.LossMSE; +import org.nd4j.common.primitives.Pair; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestMultiLayerSpace extends BaseDL4JTest { + + @TempDir + public File testDir; + + @BeforeAll + public static void before(){ + Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); + } + + @Test + public void testBasic() { + + MultiLayerConfiguration expected = + new NeuralNetConfiguration.Builder() + .updater(new Sgd(0.005)).seed(12345).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2, + new OutputLayer.Builder().lossFunction(LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(10).nOut(5).build()) + + .build(); + + MultiLayerSpace mls = + new MultiLayerSpace.Builder() + .updater(new Sgd(0.005)).seed(12345) + .addLayer(new DenseLayerSpace.Builder().nIn(10).nOut(10).build(), + new FixedValue<>(2)) //2 identical layers + .addLayer(new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .nIn(10).nOut(5).build()).build(); + + int nParams = mls.numParameters(); + assertEquals(0, nParams); + + MultiLayerConfiguration conf = mls.getValue(new double[0]).getMultiLayerConfiguration(); + + assertEquals(expected, conf); + } + + @Test + public void testBasic0() { + MultiLayerConfiguration expected = + new NeuralNetConfiguration.Builder() + .l1Bias(0.4) + .l2Bias(0.5) + .constrainBias(new NonNegativeConstraint()) + .updater(new Sgd(0.005)).seed(12345).list() + .layer(0, new DenseLayer.Builder().l1Bias(0.6).nIn(10).nOut(10).build()) + .layer(1, new DenseLayer.Builder().l2Bias(0.7).constrainBias(new UnitNormConstraint()).nIn(10).nOut(10).build()).layer(2, + new OutputLayer.Builder().lossFunction(LossFunction.MCXENT).activation(Activation.SOFTMAX) + .nIn(10).nOut(5).build()) + .build(); + + MultiLayerSpace mls = + new MultiLayerSpace.Builder() + .l1Bias(0.4) + .l2Bias(0.5) + .constrainBias(new NonNegativeConstraint()) + .updater(new Sgd(0.005)).seed(12345) + .addLayer(new DenseLayerSpace.Builder().l1Bias(new ContinuousParameterSpace(0,1)).nIn(10).nOut(10).build()) + .addLayer(new DenseLayerSpace.Builder().l2Bias(0.7).constrainBias(new UnitNormConstraint()).nIn(10).nOut(10).build()) + .addLayer(new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT).activation(Activation.SOFTMAX) + .nIn(10).nOut(5).build()) + .build(); + + int nParams = mls.numParameters(); + assertEquals(1, nParams); + + //Assign numbers to each leaf ParameterSpace object (normally done by candidate generator - manual here for testing) + List noDuplicatesList = LeafUtils.getUniqueObjects(mls.collectLeaves()); + + //Second: assign each a number + int c = 0; + for (ParameterSpace ps : noDuplicatesList) { + int np = ps.numParameters(); + if (np == 1) { + ps.setIndices(c++); + } else { + int[] values = new int[np]; + for (int j = 0; j < np; j++) + values[c++] = j; + ps.setIndices(values); + } + } + MultiLayerConfiguration conf = mls.getValue(new double[] {0.6}).getMultiLayerConfiguration(); + + assertEquals(expected, conf); + } + + @Test + public void testILossFunctionGetsSet() { + ILossFunction lossFunction = new LossMCXENT(Nd4j.create(new float[] {1f, 2f}, new long[]{1,2})); + + MultiLayerConfiguration expected = + new NeuralNetConfiguration.Builder().updater(new Sgd(0.005)).seed(12345).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2, + new OutputLayer.Builder().lossFunction(lossFunction) + .activation(Activation.SOFTMAX).nIn(10).nOut(5).build()) + .build(); + + MultiLayerSpace mls = new MultiLayerSpace.Builder().updater(new Sgd(0.005)).seed(12345) + .addLayer(new DenseLayerSpace.Builder().nIn(10).nOut(10).build(), new FixedValue<>(2)) //2 identical layers + .addLayer(new OutputLayerSpace.Builder().iLossFunction(lossFunction).activation(Activation.SOFTMAX).nIn(10).nOut(5).build()) + .build(); + + int nParams = mls.numParameters(); + assertEquals(0, nParams); + + MultiLayerConfiguration conf = mls.getValue(new double[0]).getMultiLayerConfiguration(); + + assertEquals(expected, conf); + } + + @Test + public void testBasic2() { + + MultiLayerSpace mls = + new MultiLayerSpace.Builder().updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.2, 0.5)) + .convolutionMode(ConvolutionMode.Same) + .addLayer(new ConvolutionLayerSpace.Builder().nIn(3).nOut(3).kernelSize(2, 2) + .stride(1, 1).build()) + .addLayer(new DenseLayerSpace.Builder().nIn(10).nOut(10) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.TANH)) + .build(), new IntegerParameterSpace(1, 3)) //1-3 identical layers + .addLayer(new OutputLayerSpace.Builder().nIn(10).nOut(10) + .activation(Activation.SOFTMAX).build()) + .build(); + + int nParams = mls.numParameters(); + assertEquals(4, nParams); + + //Assign numbers to each leaf ParameterSpace object (normally done by candidate generator - manual here for testing) + List noDuplicatesList = LeafUtils.getUniqueObjects(mls.collectLeaves()); + + //Second: assign each a number + int c = 0; + for (ParameterSpace ps : noDuplicatesList) { + int np = ps.numParameters(); + if (np == 1) { + ps.setIndices(c++); + } else { + int[] values = new int[np]; + for (int j = 0; j < np; j++) + values[c++] = j; + ps.setIndices(values); + } + } + + + int[] nLayerCounts = new int[3]; + int reluCount = 0; + int tanhCount = 0; + + Random r = new Random(12345); + + for (int i = 0; i < 50; i++) { + + double[] rvs = new double[nParams]; + for (int j = 0; j < rvs.length; j++) + rvs[j] = r.nextDouble(); + + + MultiLayerConfiguration conf = mls.getValue(rvs).getMultiLayerConfiguration(); + + int nLayers = conf.getConfs().size(); + assertTrue(nLayers >= 3 && nLayers <= 5); //1 conv + 1-3 dense layers + 1 output layer: 2 to 4 + + int nLayersExOutputLayer = nLayers - 1; + nLayerCounts[nLayersExOutputLayer - 2]++; + + for (int j = 0; j < nLayers; j++) { + NeuralNetConfiguration layerConf = conf.getConf(j); + + double lr = ((Sgd)((BaseLayer) layerConf.getLayer()).getIUpdater()).getLearningRate(); + assertTrue(lr >= 0.0001 && lr <= 0.1); + double l2 = TestUtils.getL2((BaseLayer) layerConf.getLayer()); + assertTrue(l2 >= 0.2 && l2 <= 0.5); + + if (j == nLayers - 1) { //Output layer + assertEquals(Activation.SOFTMAX.getActivationFunction(), ((BaseLayer) layerConf.getLayer()).getActivationFn()); + } else if (j == 0) { + //Conv layer + ConvolutionLayer cl = (ConvolutionLayer) layerConf.getLayer(); + assertEquals(3, cl.getNIn()); + assertEquals(3, cl.getNOut()); + assertEquals(ConvolutionMode.Same, cl.getConvolutionMode()); + } else { + IActivation actFn = ((BaseLayer) layerConf.getLayer()).getActivationFn(); + assertTrue(Activation.RELU.getActivationFunction().equals(actFn) || + Activation.TANH.getActivationFunction().equals(actFn)); + if (Activation.RELU.getActivationFunction().equals(actFn)) + reluCount++; + else + tanhCount++; + } + } + } + + for (int i = 0; i < 3; i++) { + assertTrue(nLayerCounts[i] >= 5); //Expect approx equal (50/3 each), but some variation randomly + } + +// System.out.println("Number of layers: " + Arrays.toString(nLayerCounts)); +// System.out.println("ReLU vs. Tanh: " + reluCount + "\t" + tanhCount); + + } + + @Test + public void testGlobalPoolingBasic() { + + MultiLayerConfiguration expected = new NeuralNetConfiguration.Builder().updater(new Sgd(0.005)).seed(12345).list() + .layer(0, new GravesLSTM.Builder().nIn(10).nOut(10).build()) + .layer(1, new GlobalPoolingLayer.Builder().poolingType(PoolingType.SUM).pnorm(7).build()) + .layer(2, new OutputLayer.Builder().lossFunction(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(5).build()) + .build(); + + MultiLayerSpace mls = + new MultiLayerSpace.Builder().updater(new Sgd(0.005)).seed(12345) + .addLayer(new GravesLSTMLayerSpace.Builder().nIn(10).nOut(10).build()) + .addLayer(new GlobalPoolingLayerSpace.Builder().poolingType(PoolingType.SUM) + .pNorm(7).build()) + .addLayer(new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .nIn(10).nOut(5).build()) + .build(); + + int nParams = mls.numParameters(); + assertEquals(0, nParams); + + MultiLayerConfiguration conf = mls.getValue(new double[0]).getMultiLayerConfiguration(); + + assertEquals(expected, conf); + } + + + @Test + public void testVariationalAutoencoderLayerSpaceBasic() { + MultiLayerSpace mls = + new MultiLayerSpace.Builder() + .updater(new Sgd(0.005)).seed( + 12345) + .addLayer(new VariationalAutoencoderLayerSpace.Builder() + .nIn(new IntegerParameterSpace(50, 75)).nOut(200) + .encoderLayerSizes(234, 567).decoderLayerSizes(123, 456) + .reconstructionDistribution( + new DiscreteParameterSpace( + new GaussianReconstructionDistribution(), + new BernoulliReconstructionDistribution())) + .build()) + .build(); + + int numParams = mls.numParameters(); + + //Assign numbers to each leaf ParameterSpace object (normally done by candidate generator - manual here for testing) + List noDuplicatesList = LeafUtils.getUniqueObjects(mls.collectLeaves()); + + //Second: assign each a number + int c = 0; + for (ParameterSpace ps : noDuplicatesList) { + int np = ps.numParameters(); + if (np == 1) { + ps.setIndices(c++); + } else { + int[] values = new int[np]; + for (int j = 0; j < np; j++) + values[c++] = j; + ps.setIndices(values); + } + } + + double[] zeros = new double[numParams]; + + DL4JConfiguration configuration = mls.getValue(zeros); + + MultiLayerConfiguration conf = configuration.getMultiLayerConfiguration(); + assertEquals(1, conf.getConfs().size()); + + NeuralNetConfiguration nnc = conf.getConf(0); + VariationalAutoencoder vae = (VariationalAutoencoder) nnc.getLayer(); + + assertEquals(50, vae.getNIn()); + assertEquals(200, vae.getNOut()); + + assertArrayEquals(new int[] {234, 567}, vae.getEncoderLayerSizes()); + assertArrayEquals(new int[] {123, 456}, vae.getDecoderLayerSizes()); + + assertTrue(vae.getOutputDistribution() instanceof GaussianReconstructionDistribution); + + + + double[] ones = new double[numParams]; + for (int i = 0; i < ones.length; i++) + ones[i] = 1.0; + + configuration = mls.getValue(ones); + + conf = configuration.getMultiLayerConfiguration(); + assertEquals(1, conf.getConfs().size()); + + nnc = conf.getConf(0); + vae = (VariationalAutoencoder) nnc.getLayer(); + + assertEquals(75, vae.getNIn()); + assertEquals(200, vae.getNOut()); + + assertArrayEquals(new int[] {234, 567}, vae.getEncoderLayerSizes()); + assertArrayEquals(new int[] {123, 456}, vae.getDecoderLayerSizes()); + + assertTrue(vae.getOutputDistribution() instanceof BernoulliReconstructionDistribution); + } + + @Test + public void testInputTypeBasic() throws Exception { + + ParameterSpace layerSizeHyperparam = new IntegerParameterSpace(20, 60); + + MultiLayerSpace hyperparameterSpace = new MultiLayerSpace.Builder().l2(0.0001) + .weightInit(WeightInit.XAVIER).updater(new Nesterovs()) + .addLayer(new ConvolutionLayerSpace.Builder().kernelSize(5, 5).nIn(1).stride(1, 1) + .nOut(layerSizeHyperparam).activation(Activation.IDENTITY).build()) + .addLayer(new SubsamplingLayerSpace.Builder().poolingType(SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2).build()) + .addLayer(new ConvolutionLayerSpace.Builder().kernelSize(5, 5) + //Note that nIn need not be specified in later layers + .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()) + .addLayer(new SubsamplingLayerSpace.Builder().poolingType(SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2).build()) + .addLayer(new DenseLayerSpace.Builder().activation(Activation.RELU).nOut(500).build()) + .addLayer(new OutputLayerSpace.Builder() + .lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(10) + .activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + + DataProvider dataProvider = new TestDataSetProvider(); + + File f = testDir; + if (f.exists()) + f.delete(); + f.mkdir(); + ResultSaver modelSaver = new FileModelSaver(f.getAbsolutePath()); + + ScoreFunction scoreFunction = new TestSetAccuracyScoreFunction(); + + int maxCandidates = 4; + TerminationCondition[] terminationConditions; + terminationConditions = new TerminationCondition[] {new MaxCandidatesCondition(maxCandidates)}; + + //Given these configuration options, let's put them all together: + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(new RandomSearchGenerator(hyperparameterSpace, null)) + .dataProvider(dataProvider).modelSaver(modelSaver).scoreFunction(scoreFunction) + .terminationConditions(terminationConditions).build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + runner.execute(); + + assertEquals(maxCandidates, runner.getResults().size()); + } + + + @Test + public void testSameRanges() { + + ParameterSpace l1Hyperparam = new ContinuousParameterSpace(0.001, 0.1); + ParameterSpace l2Hyperparam = new ContinuousParameterSpace(0.001, 0.1); + + MultiLayerSpace hyperparameterSpace = + new MultiLayerSpace.Builder().addLayer(new DenseLayerSpace.Builder().nIn(10).nOut(10).build()) + .l1(l1Hyperparam).l2(l2Hyperparam).build(); + + CandidateGenerator c = new RandomSearchGenerator(hyperparameterSpace, null); + + Candidate candidate = c.getCandidate(); + } + + @Test + public void testWeightedLossFunction() { + + MultiLayerConfiguration expected = + new NeuralNetConfiguration.Builder().updater(new Sgd(0.005)).seed(12345).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2, + new OutputLayer.Builder() + .lossFunction(new LossMSE(Nd4j.create( + new double[] {1, 2, 3, 4, 5}, new long[]{1,5}))) + .nIn(10).nOut(5).build()) + .build(); + + MultiLayerSpace mls = + new MultiLayerSpace.Builder().updater(new Sgd(0.005)).seed(12345) + .addLayer(new DenseLayerSpace.Builder().nIn(10).nOut(10).build(), + new FixedValue<>(2)) //2 identical layers + .addLayer(new OutputLayerSpace.Builder() + .iLossFunction(new LossMSE(Nd4j.create(new double[] {1, 2, 3, 4, 5}, new long[]{1,5}))) + .nIn(10).nOut(5).build()) + .build(); + + int nParams = mls.numParameters(); + assertEquals(0, nParams); + + MultiLayerConfiguration conf = mls.getValue(new double[0]).getMultiLayerConfiguration(); + + assertEquals(expected, conf); + + String json = mls.toJson(); + MultiLayerSpace fromJson = MultiLayerSpace.fromJson(json); + + assertEquals(mls, fromJson); + } + + + @Test + public void testBidirectional() throws Exception { + + MultiLayerSpace mls = + new MultiLayerSpace.Builder().updater(new Sgd(0.005)) + .seed(12345) + .layer(new Bidirectional(new LSTMLayerSpace.Builder() + .nIn(10).nOut(10).build())) + .build(); + + DL4JConfiguration conf = mls.getValue(new double[0]); + MultiLayerConfiguration c2 = conf.getMultiLayerConfiguration(); + + MultiLayerNetwork net = new MultiLayerNetwork(c2); + net.init(); + + assertEquals(1, net.getnLayers()); + assertTrue(net.getLayer(0) instanceof BidirectionalLayer); + BidirectionalLayer bl = (BidirectionalLayer)net.getLayer(0); + + Field f = BidirectionalLayer.class.getDeclaredField("fwd"); + Field b = BidirectionalLayer.class.getDeclaredField("bwd"); + f.setAccessible(true); + b.setAccessible(true); + org.deeplearning4j.nn.layers.recurrent.LSTM lstmFwd = (org.deeplearning4j.nn.layers.recurrent.LSTM) f.get(bl); + org.deeplearning4j.nn.layers.recurrent.LSTM lstmBwd = (org.deeplearning4j.nn.layers.recurrent.LSTM) b.get(bl); + + assertEquals(10, ((LSTM)lstmFwd.conf().getLayer()).getNIn()); + assertEquals(10, ((LSTM)lstmFwd.conf().getLayer()).getNOut()); + assertEquals(10, ((LSTM)lstmBwd.conf().getLayer()).getNIn()); + assertEquals(10, ((LSTM)lstmBwd.conf().getLayer()).getNOut()); + } + + + @Test + public void testMathOps() { + + ParameterSpace firstLayerSize = new IntegerParameterSpace(10,30); + ParameterSpace secondLayerSize = new MathOp<>(firstLayerSize, Op.MUL, 3); + ParameterSpace firstLayerLR = new ContinuousParameterSpace(0.01, 0.1); + ParameterSpace secondLayerLR = new MathOp<>(firstLayerLR, Op.ADD, 0.2); + + MultiLayerSpace mls = + new MultiLayerSpace.Builder().updater(new Sgd(0.005)) + .seed(12345) + .layer(new DenseLayerSpace.Builder().nOut(firstLayerSize) + .updater(new AdamSpace(firstLayerLR)) + .build()) + .layer(new OutputLayerSpace.Builder().nOut(secondLayerSize) + .updater(new AdamSpace(secondLayerLR)) + .activation(Activation.SOFTMAX) + .build()) + .setInputType(InputType.feedForward(10)) + .build(); + + int nParams = mls.numParameters(); + assertEquals(2, nParams); + + new RandomSearchGenerator(mls, null); //Initializes the indices + + Random r = new Random(12345); + for( int i=0; i<10; i++ ){ + double[] d = new double[nParams]; + for( int j=0; j dropout = new DiscreteParameterSpace<>(0.0, 0.5); + + MultiLayerSpace mls = + new MultiLayerSpace.Builder().updater(new Sgd(0.005)) + .dropOut(dropout) + .seed(12345) + .layer(new DenseLayerSpace.Builder().nOut(10) + .build()) + .layer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .build()) + .setInputType(InputType.feedForward(10)) + .build(); + + int nParams = mls.numParameters(); + assertEquals(1, nParams); + + new RandomSearchGenerator(mls, null); //Initializes the indices + + Random r = new Random(12345); + int countNull = 0; + int count05 = 0; + for( int i=0; i<10; i++ ){ + double[] d = new double[nParams]; + for( int j=0; j 0); + assertTrue(count05 > 0); + } + + + private static class TestDataSetProvider implements DataProvider { + + @Override + public Object trainData(Map dataParameters) { + return new ExistingDataSetIterator( + Collections.singletonList(new DataSet(Nd4j.create(1, 1, 28, 28), Nd4j.create(1,10)))); + } + + @Override + public Object testData(Map dataParameters) { + return new ExistingDataSetIterator( + Collections.singletonList(new DataSet(Nd4j.create(1, 1, 28, 28), Nd4j.create(1,10)))); + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + } + + + @Test + public void testDropout(){ + + MultiLayerSpace mls = new MultiLayerSpace.Builder().updater(new Sgd(0.005)).seed(12345) + .addLayer(new ConvolutionLayerSpace.Builder().nOut(2) + .dropOut(new ContinuousParameterSpace(0.4,0.6)) + .build()) + .addLayer(new GlobalPoolingLayerSpace.Builder().dropOut(new ContinuousParameterSpace(0.4,0.6)).build()) + .addLayer(new OutputLayerSpace.Builder().activation(Activation.SOFTMAX).nIn(10).nOut(5).build()) + .setInputType(InputType.convolutional(28, 28, 1)) + .build(); + + int nParams = mls.numParameters(); + List l = LeafUtils.getUniqueObjects(mls.collectLeaves()); + int x=0; + for( ParameterSpace p : l){ + int n = p.numParameters(); + int[] arr = new int[n]; + for(int i=0; i l = LeafUtils.getUniqueObjects(mls.collectLeaves()); + int x=0; + for( ParameterSpace p : l){ + int n = p.numParameters(); + int[] arr = new int[n]; + for(int i=0; i learningRateHyperparam = new DiscreteParameterSpace<>(0.003, 0.005, 0.01, 0.05); + ParameterSpace layerSizeHyperparam1 = new DiscreteParameterSpace<>(32, 64, 96, 128); + ParameterSpace layerSizeHyperparam2 = new DiscreteParameterSpace<>(32, 64, 96, 128); + ParameterSpace dropoutHyperparam = new DiscreteParameterSpace<>(0.8, 0.9); + + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .updater(new AdamSpace(learningRateHyperparam)) + .weightInit(WeightInit.XAVIER) + .l2(0.0001) + .addLayer(new DenseLayerSpace.Builder() + .nIn(10) + .nOut(layerSizeHyperparam1) + .build()) + .addLayer(new BatchNormalizationSpace.Builder() + .nOut(layerSizeHyperparam1) + .activation(Activation.RELU) + .build()) + .addLayer(new DropoutLayerSpace.Builder() + .dropOut(dropoutHyperparam) + .build()) + .addLayer(new DenseLayerSpace.Builder() + .nOut(layerSizeHyperparam2) + .build()) + .addLayer(new BatchNormalizationSpace.Builder() + .nOut(layerSizeHyperparam2) + .activation(Activation.RELU) + .build()) + .addLayer(new DropoutLayerSpace.Builder() + .dropOut(dropoutHyperparam) + .build()) + .addLayer(new OutputLayerSpace.Builder() + .nOut(10) + .activation(Activation.SOFTMAX) + .lossFunction(LossFunction.MCXENT) + .build()) + .build(); + + assertEquals(4, mls.getNumParameters()); + + for( int discreteCount : new int[]{1, 5}) { + GridSearchCandidateGenerator generator = new GridSearchCandidateGenerator(mls, discreteCount, GridSearchCandidateGenerator.Mode.Sequential, null); + + int expCandidates = 4 * 4 * 4 * 2; + assertEquals(expCandidates, generator.getTotalNumCandidates()); + + int count = 0; + while (generator.hasMoreCandidates()) { + generator.getCandidate(); + count++; + } + + + assertEquals(expCandidates, count); + } + } + + + @Test + public void testGridCandidateGenerator(){ + ParameterSpace layerSizeParam = new DiscreteParameterSpace<>(32, 48, 64); + ParameterSpace learningRateParam = new DiscreteParameterSpace<>(0.005, 0.007, 0.01); + + MultiLayerSpace hyperParamaterSpace = new MultiLayerSpace.Builder() + .seed(12345) + .biasInit(1) + .l2(1e-4) + .updater(new NesterovsSpace(learningRateParam)) + .addLayer(new DenseLayerSpace.Builder().nIn(10).nOut(layerSizeParam) + .weightInit(WeightInit.XAVIER) + .activation(Activation.RELU) + .build()) + .addLayer(new DenseLayerSpace.Builder().nIn(layerSizeParam).nOut(layerSizeParam) + .weightInit(WeightInit.XAVIER) + .activation(Activation.RELU) + .build()) + .addLayer(new OutputLayerSpace.Builder() + .lossFunction(LossFunctions.LossFunction.MSE) + .weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX) + .nIn(layerSizeParam).nOut(10).build()) + .build(); + + CandidateGenerator candidateGenerator = new GridSearchCandidateGenerator(hyperParamaterSpace, 30, GridSearchCandidateGenerator.Mode.Sequential, null); +// CandidateGenerator candidateGenerator = new RandomSearchGenerator(hyperParamaterSpace); + + Set> expCandidates = new HashSet<>(); + for(Double d : new double[]{0.005, 0.007, 0.01}){ + for(int i : new int[]{32, 48, 64}){ + expCandidates.add(new Pair<>(d, i)); + } + } + + Set> actCandidates = new HashSet<>(); + while(candidateGenerator.hasMoreCandidates()) { + Candidate conf = candidateGenerator.getCandidate(); + MultiLayerConfiguration mlc = conf.getValue().getMultiLayerConfiguration(); + FeedForwardLayer ffl = ((FeedForwardLayer) mlc.getConf(0).getLayer()); +// System.out.println(ffl.getIUpdater() + ", " + ffl.getNOut()); + actCandidates.add(new Pair<>(ffl.getIUpdater().getLearningRate(0,0), (int)ffl.getNOut())); + } + + assertEquals(expCandidates, actCandidates); + } +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestScoreFunctions.java b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestScoreFunctions.java new file mode 100644 index 000000000..e9c34c947 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/java/org/deeplearning4j/arbiter/multilayernetwork/TestScoreFunctions.java @@ -0,0 +1,220 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.multilayernetwork; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.conf.updater.AdamSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.saving.InMemoryResultSaver; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultReference; +import org.deeplearning4j.arbiter.optimize.api.saving.ResultSaver; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.scoring.impl.ROCScoreFunction; +import org.deeplearning4j.arbiter.task.MultiLayerNetworkTaskCreator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.eval.ROC; +import org.deeplearning4j.eval.ROCBinary; +import org.deeplearning4j.eval.ROCMultiClass; +import org.deeplearning4j.nn.conf.WorkspaceMode; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.DataSetPreProcessor; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +public class TestScoreFunctions extends BaseDL4JTest { + + + @Override + public long getTimeoutMilliseconds() { + return 60000L; + } + + @Test + public void testROCScoreFunctions() throws Exception { + + + for (boolean auc : new boolean[]{true, false}) { + for (ROCScoreFunction.ROCType rocType : ROCScoreFunction.ROCType.values()) { + String msg = (auc ? "AUC" : "AUPRC") + " - " + rocType; + log.info("Starting: " + msg); + + ParameterSpace lr = new ContinuousParameterSpace(1e-5, 1e-3); + + int nOut = (rocType == ROCScoreFunction.ROCType.ROC ? 2 : 10); + LossFunctions.LossFunction lf = (rocType == ROCScoreFunction.ROCType.BINARY ? + LossFunctions.LossFunction.XENT : LossFunctions.LossFunction.MCXENT); + Activation a = (rocType == ROCScoreFunction.ROCType.BINARY ? Activation.SIGMOID : Activation.SOFTMAX); + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .trainingWorkspaceMode(WorkspaceMode.NONE) + .inferenceWorkspaceMode(WorkspaceMode.NONE) + .updater(new AdamSpace(lr)) + .weightInit(WeightInit.XAVIER) + .layer(new OutputLayerSpace.Builder().nIn(784).nOut(nOut) + .activation(a) + .lossFunction(lf).build()) + .build(); + + CandidateGenerator cg = new RandomSearchGenerator(mls); + ResultSaver rs = new InMemoryResultSaver(); + ScoreFunction sf = new ROCScoreFunction(rocType, (auc ? ROCScoreFunction.Metric.AUC : ROCScoreFunction.Metric.AUPRC)); + + + OptimizationConfiguration oc = new OptimizationConfiguration.Builder() + .candidateGenerator(cg) + .dataProvider(new DP(rocType)) + .modelSaver(rs) + .scoreFunction(sf) + .terminationConditions(new MaxCandidatesCondition(3)) + .rngSeed(12345) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(oc, new MultiLayerNetworkTaskCreator()); + runner.execute(); + + List list = runner.getResults(); + + for (ResultReference rr : list) { + DataSetIterator testIter = new MnistDataSetIterator(4, 16, false, false, false, 12345); + testIter.setPreProcessor(new PreProc(rocType)); + + OptimizationResult or = rr.getResult(); + MultiLayerNetwork net = (MultiLayerNetwork) or.getResultReference().getResultModel(); + + double expScore; + switch (rocType){ + case ROC: + if(auc){ + expScore = net.doEvaluation(testIter, new ROC())[0].calculateAUC(); + } else { + expScore = net.doEvaluation(testIter, new ROC())[0].calculateAUCPR(); + } + break; + case BINARY: + if(auc){ + expScore = net.doEvaluation(testIter, new ROCBinary())[0].calculateAverageAuc(); + } else { + expScore = net.doEvaluation(testIter, new ROCBinary())[0].calculateAverageAUCPR(); + } + break; + case MULTICLASS: + if(auc){ + expScore = net.doEvaluation(testIter, new ROCMultiClass())[0].calculateAverageAUC(); + } else { + expScore = net.doEvaluation(testIter, new ROCMultiClass())[0].calculateAverageAUCPR(); + } + break; + default: + throw new RuntimeException(); + } + + + DataSetIterator iter = new MnistDataSetIterator(4, 16, false, false, false, 12345); + iter.setPreProcessor(new PreProc(rocType)); + + assertEquals(expScore, or.getScore(), 1e-4, msg); + } + } + } + } + + @AllArgsConstructor + public static class DP implements DataProvider { + + protected ROCScoreFunction.ROCType rocType; + + @Override + public Object trainData(Map dataParameters) { + try { + DataSetIterator iter = new MnistDataSetIterator(4, 16, false, false, false, 12345); + iter.setPreProcessor(new PreProc(rocType)); + return iter; + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + @Override + public Object testData(Map dataParameters) { + try { + DataSetIterator iter = new MnistDataSetIterator(4, 16, false, false, false, 12345); + iter.setPreProcessor(new PreProc(rocType)); + return iter; + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + } + + @AllArgsConstructor + public static class PreProc implements DataSetPreProcessor { + protected ROCScoreFunction.ROCType rocType; + + @Override + public void preProcess(DataSet toPreProcess) { + switch (rocType){ + case ROC: + //Convert to binary + long mb = toPreProcess.getLabels().size(0); + INDArray argMax = Nd4j.argMax(toPreProcess.getLabels(), 1); + INDArray newLabel = Nd4j.create(mb, 2); + for( int i=0; i dataParameters) { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(batchSize, true, 12345), terminationIter); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + @Override + public Object testData(Map dataParameters) { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(batchSize, false, 12345), terminationIter); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + + +} diff --git a/arbiter/arbiter-deeplearning4j/src/test/resources/logback.xml b/arbiter/arbiter-deeplearning4j/src/test/resources/logback.xml new file mode 100644 index 000000000..410bdaae9 --- /dev/null +++ b/arbiter/arbiter-deeplearning4j/src/test/resources/logback.xml @@ -0,0 +1,51 @@ + + + + + + logs/application.log + + %date - [%level] - from %logger in %thread + %n%message%n%xException%n + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/arbiter/arbiter-server/pom.xml b/arbiter/arbiter-server/pom.xml new file mode 100644 index 000000000..c38549354 --- /dev/null +++ b/arbiter/arbiter-server/pom.xml @@ -0,0 +1,63 @@ + + + + + arbiter + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + arbiter-server + jar + + arbiter-server + + + UTF-8 + + + + + com.beust + jcommander + 1.27 + + + net.brutex.ai + arbiter-deeplearning4j + ${project.version} + + + + net.brutex.ai + deeplearning4j-common-tests + ${project.version} + test + + + + + + test-nd4j-native + + + test-nd4j-cuda-10.2 + + + diff --git a/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/ArbiterCliGenerator.java b/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/ArbiterCliGenerator.java new file mode 100644 index 000000000..af19a81f7 --- /dev/null +++ b/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/ArbiterCliGenerator.java @@ -0,0 +1,286 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.server; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.arbiter.ComputationGraphSpace; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxTimeCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.TerminationCondition; +import org.deeplearning4j.arbiter.optimize.generator.GridSearchCandidateGenerator; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.RegressionValue; +import org.deeplearning4j.arbiter.scoring.ScoreFunctions; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Generate an {@link OptimizationConfiguration} + * via the command line interface. + * You can then use this configuration json file from + * {@link ArbiterCliRunner} + * + * @author Adam Gibson + */ +public class ArbiterCliGenerator { + @Parameter(names = {"--searchSpacePath"}) + private String searchSpacePath = null; + @Parameter(names = {"--candidateType"},required = true) + private String candidateType = null; + @Parameter(names = {"--discretizationCount"}) + private int discretizationCount = 5; + @Parameter(names = {"--gridSearchOrder"}) + private String gridSearchOrder = null; + @Parameter(names = {"--neuralNetType"},required = true) + private String neuralNetType = null; + @Parameter(names = {"--dataSetIteratorClass"},required = true) + private String dataSetIteratorClass = null; + @Parameter(names = {"--modelOutputPath"},required = true) + private String modelOutputPath = null; + @Parameter(names = {"--score"},required = true) + private String score = null; + @Parameter(names = {"--problemType"},required = true) + private String problemType = CLASSIFICIATION; + @Parameter(names = {"--configSavePath"},required = true) + private String configSavePath = null; + + @Parameter(names = {"--duration"},description = "The number of minutes to run for. Default is -1 which means run till convergence.") + private long duration = -1; + @Parameter(names = {"--numCandidates"},description = "The number of candidates to generate. Default is 1.") + private int numCandidates = 1; + + public final static String REGRESSION_MULTI = "regression"; + public final static String REGRESSION = "regression"; + public final static String CLASSIFICIATION = "classification"; + + public final static String RANDOM_CANDIDATE = "random"; + public final static String GRID_SEARCH_CANDIDATE = "gridsearch"; + + public final static String SEQUENTIAL_ORDER = "sequence"; + public final static String RANDOM_ORDER = "random"; + + public final static String COMP_GRAPH = "compgraph"; + public final static String MULTI_LAYER = "multilayer"; + + public final static String ACCURACY = "accuracy"; + public final static String F1 = "f1"; + + public final static String ACCURACY_MULTI = "accuracy_multi"; + public final static String F1_MULTI = "f1_multi"; + + + public final static String REGRESSION_SCORE = "regression_score"; + public final static String REGRESSION_SCORE_MULTI = "regression_score_multi"; + + public void runMain(String...args) throws Exception { + JCommander jcmdr = new JCommander(this); + + try { + jcmdr.parse(args); + } catch(ParameterException e) { + System.err.println(e.getMessage()); + //User provides invalid input -> print the usage info + jcmdr.usage(); + try{ Thread.sleep(500); } catch(Exception e2){ } + System.exit(1); + } + + + DataProvider dataProvider = new DataSetIteratorFactoryProvider(); + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY,dataSetIteratorClass); + + + if(neuralNetType.equals(MULTI_LAYER)) { + MultiLayerSpace multiLayerSpace = loadMultiLayer(); + CandidateGenerator candidateGenerator = null; + if(candidateType.equals(GRID_SEARCH_CANDIDATE)) { + candidateGenerator = new RandomSearchGenerator(multiLayerSpace,commands); + + + + } + else if(candidateType.equals(RANDOM_CANDIDATE)) { + candidateGenerator = new RandomSearchGenerator(multiLayerSpace,commands); + + } + + if(problemType.equals(CLASSIFICIATION)) { + OptimizationConfiguration configuration + = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelOutputPath)) + .scoreFunction(scoreFunctionMultiLayerNetwork()) + .terminationConditions(getConditions()) + .build(); + FileUtils.writeStringToFile(new File(configSavePath),configuration.toJson()); + + } + else if(problemType.equals(REGRESSION)) { + OptimizationConfiguration configuration + = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelOutputPath)) + .scoreFunction(scoreFunctionMultiLayerNetwork()) + .terminationConditions(getConditions()) + .build(); + FileUtils.writeStringToFile(new File(configSavePath),configuration.toJson()); + + } + + + } + else if(neuralNetType.equals(COMP_GRAPH)) { + ComputationGraphSpace computationGraphSpace = loadCompGraph(); + CandidateGenerator candidateGenerator = null; + if(candidateType.equals(GRID_SEARCH_CANDIDATE)) { + candidateGenerator = new RandomSearchGenerator(computationGraphSpace,commands); + + } + else if(candidateType.equals(RANDOM_CANDIDATE)) { + candidateGenerator = new RandomSearchGenerator(computationGraphSpace,commands); + + } + + + if(problemType.equals(CLASSIFICIATION)) { + OptimizationConfiguration configuration + = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelOutputPath)) + .scoreFunction(scoreFunctionCompGraph()) + .terminationConditions(getConditions()) + .build(); + + FileUtils.writeStringToFile(new File(configSavePath),configuration.toJson()); + } + else { + OptimizationConfiguration configuration + = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelOutputPath)) + .scoreFunction(scoreFunctionCompGraph()) + .terminationConditions(getConditions()) + .build(); + FileUtils.writeStringToFile(new File(configSavePath),configuration.toJson()); + + + } + + + } + + + } + + public static void main(String...args) throws Exception { + new ArbiterCliGenerator().runMain(args); + } + + private List getConditions() { + List ret = new ArrayList<>(); + if(duration > 0) { + ret.add(new MaxTimeCondition(duration,TimeUnit.MINUTES)); + } + + if(numCandidates > 0) + ret.add(new MaxCandidatesCondition(numCandidates)); + if(ret.isEmpty()) { + ret.add(new MaxCandidatesCondition(1)); + } + return ret; + } + + + private GridSearchCandidateGenerator.Mode getMode() { + if(gridSearchOrder.equals(RANDOM_ORDER)) + return GridSearchCandidateGenerator.Mode.RandomOrder; + else if(gridSearchOrder.equals(SEQUENTIAL_ORDER)) { + return GridSearchCandidateGenerator.Mode.Sequential; + } + else throw new IllegalArgumentException("Illegal mode " + gridSearchOrder); + } + + private ScoreFunction scoreFunctionCompGraph() { + if(problemType.equals(CLASSIFICIATION)) { + switch(score) { + case ACCURACY: return ScoreFunctions.testSetAccuracy(); + case F1: return ScoreFunctions.testSetF1(); + case F1_MULTI : return ScoreFunctions.testSetF1(); + case ACCURACY_MULTI: return ScoreFunctions.testSetAccuracy(); + + default: throw new IllegalArgumentException("Score " + score + " not valid for type " + problemType); + } + } + else if(problemType.equals(REGRESSION)) { + switch(score) { + case REGRESSION_SCORE: return ScoreFunctions.testSetRegression(RegressionValue.valueOf(score)); + case REGRESSION_SCORE_MULTI: return ScoreFunctions.testSetRegression(RegressionValue.valueOf(score)); + default: throw new IllegalArgumentException("Score " + score + " not valid for type " + problemType); + } + } + throw new IllegalStateException("Illegal problem type " + problemType); + } + + private ScoreFunction scoreFunctionMultiLayerNetwork() { + if(problemType.equals(CLASSIFICIATION)) { + switch(score) { + case ACCURACY: return ScoreFunctions.testSetAccuracy(); + case F1: return ScoreFunctions.testSetF1(); + + default: throw new IllegalArgumentException("Score " + score + " not valid for type " + problemType); + } + } + else if(problemType.equals(REGRESSION)) { + switch(score) { + case REGRESSION_SCORE: return ScoreFunctions.testSetRegression(RegressionValue.valueOf(score)); + default: throw new IllegalArgumentException("Score " + score + " not valid for type " + problemType); + + } + } + throw new IllegalStateException("Illegal problem type " + problemType); + } + + private ComputationGraphSpace loadCompGraph() throws Exception { + ComputationGraphSpace multiLayerSpace = ComputationGraphSpace.fromJson(FileUtils.readFileToString(new File(searchSpacePath))); + return multiLayerSpace; + } + + private MultiLayerSpace loadMultiLayer() throws Exception { + MultiLayerSpace multiLayerSpace = MultiLayerSpace.fromJson(FileUtils.readFileToString(new File(searchSpacePath))); + return multiLayerSpace; + } +} diff --git a/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/ArbiterCliRunner.java b/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/ArbiterCliRunner.java new file mode 100644 index 000000000..c845828cf --- /dev/null +++ b/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/ArbiterCliRunner.java @@ -0,0 +1,152 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.server; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.arbiter.evaluator.multilayer.ClassificationEvaluator; +import org.deeplearning4j.arbiter.evaluator.multilayer.RegressionDataEvaluator; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.scoring.RegressionValue; +import org.deeplearning4j.arbiter.server.cli.NeuralNetTypeValidator; +import org.deeplearning4j.arbiter.server.cli.ProblemTypeValidator; +import org.deeplearning4j.arbiter.task.ComputationGraphTaskCreator; +import org.deeplearning4j.arbiter.task.MultiLayerNetworkTaskCreator; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +/** + * Options: + * --dataSetIteratorClass + --modelSavePath + Default: /tmp + * --neuralNetType + --optimizationConfigPath + --problemType + Default: classification + --regressionType + + + + @author Adam Gibson + */ +public class ArbiterCliRunner { + @Parameter(names = {"--modelSavePath"}) + private String modelSavePath = System.getProperty("java.io.tmpdir"); + @Parameter(names = {"--optimizationConfigPath"}) + private String optimizationConfigPath = null; + @Parameter(names = {"--problemType"},validateWith = ProblemTypeValidator.class) + private String problemType = CLASSIFICATION; + @Parameter(names = {"--regressionType"}) + private String regressionType = null; + @Parameter(names = {"--dataSetIteratorClass"},required = true) + private String dataSetIteratorClass = null; + @Parameter(names = {"--neuralNetType"},required = true,validateWith = NeuralNetTypeValidator.class) + private String neuralNetType = null; + + public final static String CLASSIFICATION = "classification"; + public final static String REGRESSION = "regression"; + + + public final static String COMP_GRAPH = "compgraph"; + public final static String MULTI_LAYER_NETWORK = "multilayernetwork"; + + public void runMain(String...args) throws Exception { + JCommander jcmdr = new JCommander(this); + + try { + jcmdr.parse(args); + } catch(ParameterException e) { + System.err.println(e.getMessage()); + //User provides invalid input -> print the usage info + jcmdr.usage(); + try{ Thread.sleep(500); } catch(Exception e2){ } + System.exit(1); + } + + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY,dataSetIteratorClass); + + File f = new File(modelSavePath); + + if(f.exists()) f.delete(); + f.mkdir(); + f.deleteOnExit(); + + if(problemType.equals(REGRESSION)) { + if(neuralNetType.equals(COMP_GRAPH)) { + OptimizationConfiguration configuration + = OptimizationConfiguration.fromJson( + FileUtils.readFileToString(new File(optimizationConfigPath))); + + IOptimizationRunner runner + = new LocalOptimizationRunner(configuration, new ComputationGraphTaskCreator( + new RegressionDataEvaluator(RegressionValue.valueOf(regressionType),commands))); + runner.execute(); + } + else if(neuralNetType.equals(MULTI_LAYER_NETWORK)) { + OptimizationConfiguration configuration = OptimizationConfiguration. + fromJson(FileUtils.readFileToString(new File(optimizationConfigPath))); + + IOptimizationRunner runner + = new LocalOptimizationRunner( + configuration, + new MultiLayerNetworkTaskCreator( + new RegressionDataEvaluator( + RegressionValue.valueOf(regressionType), + commands))); + runner.execute(); + } + } + + else if(problemType.equals(CLASSIFICATION)) { + if(neuralNetType.equals(COMP_GRAPH)) { + OptimizationConfiguration configuration + = OptimizationConfiguration.fromJson(FileUtils.readFileToString(new File(optimizationConfigPath))); + + IOptimizationRunner runner + = new LocalOptimizationRunner( + configuration,new ComputationGraphTaskCreator(new ClassificationEvaluator())); + + runner.execute(); + } + else if(neuralNetType.equals(MULTI_LAYER_NETWORK)) { + OptimizationConfiguration configuration = OptimizationConfiguration + .fromJson(FileUtils.readFileToString(new File(optimizationConfigPath))); + + IOptimizationRunner runner + = new LocalOptimizationRunner(configuration, + new MultiLayerNetworkTaskCreator( + new ClassificationEvaluator()) + ); + + runner.execute(); + } + } + } + public static void main(String...args) throws Exception { + new ArbiterCliRunner().runMain(args); + } + +} diff --git a/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/cli/NeuralNetTypeValidator.java b/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/cli/NeuralNetTypeValidator.java new file mode 100644 index 000000000..1a338bdc0 --- /dev/null +++ b/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/cli/NeuralNetTypeValidator.java @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.server.cli; + +import com.beust.jcommander.IParameterValidator; +import com.beust.jcommander.ParameterException; +import org.deeplearning4j.arbiter.server.ArbiterCliRunner; + +/** + * Created by agibsonccc on 3/13/17. + */ +public class NeuralNetTypeValidator implements IParameterValidator { + /** + * Validate the parameter. + * + * @param name The name of the parameter (e.g. "-host"). + * @param value The value of the parameter that we need to validate + * @throws ParameterException Thrown if the value of the parameter is invalid. + */ + @Override + public void validate(String name, String value) throws ParameterException { + if(!value.equals(ArbiterCliRunner.MULTI_LAYER_NETWORK) || value.equals(ArbiterCliRunner.COMP_GRAPH)) { + throw new ParameterException("Neural net type can only be " + ArbiterCliRunner.COMP_GRAPH + " or " + ArbiterCliRunner.MULTI_LAYER_NETWORK); + + } + } +} diff --git a/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/cli/ProblemTypeValidator.java b/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/cli/ProblemTypeValidator.java new file mode 100644 index 000000000..3df2f6449 --- /dev/null +++ b/arbiter/arbiter-server/src/main/java/org/deeplearning4j/arbiter/server/cli/ProblemTypeValidator.java @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.server.cli; + +import com.beust.jcommander.IParameterValidator; +import com.beust.jcommander.ParameterException; +import org.deeplearning4j.arbiter.server.ArbiterCliGenerator; + +/** + * Created by agibsonccc on 3/13/17. + */ +public class ProblemTypeValidator implements IParameterValidator { + /** + * Validate the parameter. + * + * @param name The name of the parameter (e.g. "-host"). + * @param value The value of the parameter that we need to validate + * @throws ParameterException Thrown if the value of the parameter is invalid. + */ + @Override + public void validate(String name, String value) throws ParameterException { + if(!value.equals(ArbiterCliGenerator.REGRESSION) || value.equals(ArbiterCliGenerator.CLASSIFICIATION)) { + throw new ParameterException("Problem type can only be " + ArbiterCliGenerator.REGRESSION + " or " + ArbiterCliGenerator.CLASSIFICIATION); + + } + } +} diff --git a/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/ArbiterCLIRunnerTest.java b/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/ArbiterCLIRunnerTest.java new file mode 100644 index 000000000..5efcd9657 --- /dev/null +++ b/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/ArbiterCLIRunnerTest.java @@ -0,0 +1,121 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.server; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSetIteratorFactoryProvider; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxTimeCondition; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Created by agibsonccc on 3/12/17. + */ +@Slf4j +public class ArbiterCLIRunnerTest extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return 90000; + } + + @Test + public void testCliRunner() throws Exception { + ArbiterCliRunner cliRunner = new ArbiterCliRunner(); + + //Define: network config (hyperparameter space) + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.1))) + .l2(new ContinuousParameterSpace(0.0001, 0.01)) + .addLayer(new DenseLayerSpace.Builder().nIn(784).nOut(new IntegerParameterSpace(2,10)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build()) + .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .numEpochs(3).build(); + assertEquals(mls,MultiLayerSpace.fromJson(mls.toJson())); + //Define configuration: + Map commands = new HashMap<>(); + commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY,TestDataFactoryProviderMnist.class.getCanonicalName()); + + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls,commands); + DataProvider dataProvider = new DataSetIteratorFactoryProvider(); + + +// String modelSavePath = FilenameUtils.concat(System.getProperty("java.io.tmpdir"),"ArbiterDL4JTest/"); + String modelSavePath = new File(System.getProperty("java.io.tmpdir"),"ArbiterDL4JTest/").getAbsolutePath(); + File dir = new File(modelSavePath); + if(!dir.exists()) + dir.mkdirs(); + String configPath = System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID().toString() + ".json"; + OptimizationConfiguration configuration + = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator) + .dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction()) + .terminationConditions(new MaxTimeCondition(30, TimeUnit.SECONDS), + new MaxCandidatesCondition(5)) + .build(); + assertEquals(configuration,OptimizationConfiguration.fromJson(configuration.toJson())); + + FileUtils.writeStringToFile(new File(configPath),configuration.toJson()); +// System.out.println(configuration.toJson()); + configuration.toJson(); + + log.info("Starting test"); + cliRunner.runMain( + "--dataSetIteratorClass", + TestDataFactoryProviderMnist.class.getCanonicalName(), + "--neuralNetType", + ArbiterCliRunner.MULTI_LAYER_NETWORK, + "--optimizationConfigPath", + configPath + ); + } + + + +} diff --git a/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/AssertTestsExtendBaseClass.java b/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..256a8af9b --- /dev/null +++ b/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.deeplearning4j.arbiter.server; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.nd4j.common.tests.AbstractAssertTestsClass; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.deeplearning4j.arbiter.server"; + } + + @Override + protected Class getBaseClass() { + return BaseDL4JTest.class; + } +} diff --git a/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/MnistDataSetIteratorFactory.java b/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/MnistDataSetIteratorFactory.java new file mode 100644 index 000000000..57bef758d --- /dev/null +++ b/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/MnistDataSetIteratorFactory.java @@ -0,0 +1,43 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.server; + +import lombok.Data; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIteratorFactory; + +import java.io.IOException; + +/** + * Created by agibsonccc on 3/13/17. + */ +@Data +public class MnistDataSetIteratorFactory extends BaseDL4JTest implements DataSetIteratorFactory { + /** + * @return + */ + @Override + public DataSetIterator create() { + try { + return new MnistDataSetIterator(1000,1000); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/TestDataFactoryProviderMnist.java b/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/TestDataFactoryProviderMnist.java new file mode 100644 index 000000000..c4a75ffb4 --- /dev/null +++ b/arbiter/arbiter-server/src/test/java/org/deeplearning4j/arbiter/server/TestDataFactoryProviderMnist.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.server; + +import lombok.AllArgsConstructor; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.DataSetIteratorFactory; + +@AllArgsConstructor +public class TestDataFactoryProviderMnist extends BaseDL4JTest implements DataSetIteratorFactory { + + private int batchSize; + private int terminationIter; + + public TestDataFactoryProviderMnist(){ + this(16, 10); + } + + @Override + public DataSetIterator create() { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(batchSize, true, 12345), terminationIter); + } catch (Exception e){ + throw new RuntimeException(e); + } + } +} diff --git a/arbiter/arbiter-ui/pom.xml b/arbiter/arbiter-ui/pom.xml new file mode 100644 index 000000000..86f4530a9 --- /dev/null +++ b/arbiter/arbiter-ui/pom.xml @@ -0,0 +1,73 @@ + + + + + + arbiter + net.brutex.ai + 1.0.0-SNAPSHOT + + + 4.0.0 + + arbiter-ui + arbiter-ui + + + + net.brutex.ai + arbiter-core + ${project.version} + + + + net.brutex.ai + deeplearning4j-ui + ${project.version} + + + + net.brutex.ai + deeplearning4j-common-tests + ${project.version} + test + + + + net.brutex.ai + arbiter-deeplearning4j + ${project.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.databind.version} + + + diff --git a/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/UpdateStatus.java b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/UpdateStatus.java new file mode 100644 index 000000000..a92b4f0e7 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/UpdateStatus.java @@ -0,0 +1,33 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.ui; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode +@Data +public class UpdateStatus { + + private long statusUpdateTime; + private long settingsUpdateTime; + private long resultsUpdateTime; +} diff --git a/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/BaseJavaPersistable.java b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/BaseJavaPersistable.java new file mode 100644 index 000000000..1fb699e0b --- /dev/null +++ b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/BaseJavaPersistable.java @@ -0,0 +1,159 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.ui.data; + +import lombok.AllArgsConstructor; +import org.apache.commons.compress.utils.IOUtils; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.arbiter.ui.module.ArbiterModule; + +import java.io.*; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * Common implementation + * + * @author Alex Black + */ +@AllArgsConstructor +public abstract class BaseJavaPersistable implements Persistable { + + private String sessionId; + private long timestamp; + + public BaseJavaPersistable(Builder builder){ + this.sessionId = builder.sessionId; + this.timestamp = builder.timestamp; + } + + protected BaseJavaPersistable(){ + //No-arg costructor for Pesistable encoding/decoding + } + + @Override + public String getTypeID() { + return ArbiterModule.ARBITER_UI_TYPE_ID; + } + + @Override + public long getTimeStamp() { + return timestamp; + } + + @Override + public String getSessionID() { + return sessionId; + } + + @Override + public int encodingLengthBytes() { + //TODO - presumably a more efficient way to do this + byte[] encoded = encode(); + return encoded.length; + } + + @Override + public byte[] encode() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ObjectOutputStream oos = new ObjectOutputStream(baos)) { + oos.writeObject(this); + } catch (IOException e) { + throw new RuntimeException(e); //Should never happen + } + return baos.toByteArray(); + } + + @Override + public void encode(ByteBuffer buffer) { + buffer.put(encode()); + } + + @Override + public void encode(OutputStream outputStream) throws IOException { + try (ObjectOutputStream oos = new ObjectOutputStream(outputStream)) { + oos.writeObject(this); + } + } + + @Override + public void decode(byte[] decode) { + BaseJavaPersistable r; + try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(decode))) { + r = (BaseJavaPersistable) ois.readObject(); + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException(e); //Should never happen + } + + //Need to manually build and walk the class heirarchy... + Class currClass = this.getClass(); + List> classHeirarchy = new ArrayList<>(); + while (currClass != Object.class) { + classHeirarchy.add(currClass); + currClass = currClass.getSuperclass(); + } + + for (int i = classHeirarchy.size() - 1; i >= 0; i--) { + //Use reflection here to avoid a mass of boilerplate code... + Field[] allFields = classHeirarchy.get(i).getDeclaredFields(); + + for (Field f : allFields) { + if (Modifier.isStatic(f.getModifiers())) { + //Skip static fields + continue; + } + f.setAccessible(true); + try { + f.set(this, f.get(r)); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); //Should never happen + } + } + } + } + + @Override + public void decode(ByteBuffer buffer) { + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + decode(bytes); + } + + @Override + public void decode(InputStream inputStream) throws IOException { + decode(IOUtils.toByteArray(inputStream)); + } + + public static abstract class Builder> { + protected String sessionId; + protected long timestamp; + + public T sessionId(String sessionId){ + this.sessionId = sessionId; + return (T) this; + } + + public T timestamp(long timestamp){ + this.timestamp = timestamp; + return (T) this; + } + + } +} diff --git a/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/GlobalConfigPersistable.java b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/GlobalConfigPersistable.java new file mode 100644 index 000000000..9a6c3faa9 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/GlobalConfigPersistable.java @@ -0,0 +1,119 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.ui.data; + +import lombok.Getter; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import org.deeplearning4j.arbiter.ui.module.ArbiterModule; +import org.deeplearning4j.core.storage.Persistable; + +import java.io.IOException; + +/** + * + * A {@link Persistable} implemention for global settings + * @author Alex Black + */ +@Getter +public class GlobalConfigPersistable extends BaseJavaPersistable { + public static final String GLOBAL_WORKER_ID = "global"; + + private String optimizationConfigJson; + private int[] candidateCounts; //queued, completed, failed, total + private String optimizationRunner; + + public GlobalConfigPersistable(String sessionId, long timestamp){ + super(sessionId, timestamp); + } + + public GlobalConfigPersistable(Builder builder){ + super(builder); + this.optimizationConfigJson = builder.optimizationConfigJson; + this.candidateCounts = builder.candidateCounts; + if(this.candidateCounts == null){ + this.candidateCounts = new int[4]; + } + this.optimizationRunner = builder.optimizationRunner; + } + + public GlobalConfigPersistable(){ + //No-arg costructor for Pesistable encoding/decoding + } + + @Override + public String getTypeID() { + return ArbiterModule.ARBITER_UI_TYPE_ID; + } + + @Override + public String getWorkerID() { + return GLOBAL_WORKER_ID; + } + + + public OptimizationConfiguration getOptimizationConfiguration(){ + try { + return JsonMapper.getMapper().readValue(optimizationConfigJson, OptimizationConfiguration.class); + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + public int getCandidatesQueued(){ + return candidateCounts[0]; + } + + public int getCandidatesCompleted(){ + return candidateCounts[1]; + } + + public int getCandidatesFailed(){ + return candidateCounts[2]; + } + + public int getCandidatesTotal(){ + return candidateCounts[3]; + } + + public static class Builder extends BaseJavaPersistable.Builder{ + + private String optimizationConfigJson; + private int[] candidateCounts; //queued, completed, failed, total + private String optimizationRunner; + + public Builder optimizationConfigJson(String optimizationConfigJson){ + this.optimizationConfigJson = optimizationConfigJson; + return this; + } + + public Builder candidateCounts(int queued, int completed, int failed, int total){ + this.candidateCounts = new int[]{queued, completed, failed, total}; + return this; + } + + public Builder optimizationRunner(String optimizationRunner){ + this.optimizationRunner = optimizationRunner; + return this; + } + + public GlobalConfigPersistable build(){ + return new GlobalConfigPersistable(this); + } + + } +} diff --git a/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/ModelInfoPersistable.java b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/ModelInfoPersistable.java new file mode 100644 index 000000000..4d1ee4e5f --- /dev/null +++ b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/data/ModelInfoPersistable.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.ui.data; + +import lombok.Data; +import org.deeplearning4j.arbiter.optimize.runner.CandidateStatus; +import org.deeplearning4j.core.storage.Persistable; + +/** + * A {@link Persistable} implemention for model results - i.e., results for + * each model + * + * @author Alex BLack + */ +@Data +public class ModelInfoPersistable extends BaseJavaPersistable { + + private String workerId; + private Integer modelIdx; + private Double score; + private CandidateStatus status; + private long lastUpdateTime; + private long numParameters; + private int numLayers; + //From candidate generator - this + model hyperparam space means we can work out specific hyperparam + // settings for this model + private double[] paramSpaceValues; + private int totalNumUpdates; + //Values for score vs. iteration chart + private int[] iter; + private float[] scoreVsIter; + private String modelConfigJson; + private String exceptionStackTrace; + + public ModelInfoPersistable(String sessionId, String workerId, long timeStamp){ + super(sessionId, timeStamp); + + this.workerId = workerId; + } + + private ModelInfoPersistable(Builder builder){ + super(builder); + this.workerId = builder.workerId; + this.modelIdx = builder.modelIdx; + this.score = builder.score; + this.status = builder.status; + this.iter = builder.iter; + this.scoreVsIter = builder.scoreVsIter; + this.lastUpdateTime = builder.lastUpdateTime; + this.numParameters = builder.numParameters; + this.numLayers = builder.numLayers; + this.paramSpaceValues = builder.paramSpaceValues; + this.modelConfigJson = builder.modelConfigJson; + this.totalNumUpdates = builder.totalNumUpdates; + this.exceptionStackTrace = builder.exceptionStackTrace; + } + + public ModelInfoPersistable(){ + //No-arg costructor for Pesistable encoding/decoding + } + + @Override + public String getWorkerID() { + return workerId; + } + + + public static class Builder extends BaseJavaPersistable.Builder { + + private String workerId; + private Integer modelIdx; + private Double score; + private CandidateStatus status; + private long lastUpdateTime;; + private long numParameters; + private int numLayers; + private int totalNumUpdates; + private double[] paramSpaceValues; + private int[] iter; + private float[] scoreVsIter; + private String modelConfigJson; + private String exceptionStackTrace; + + public Builder workerId(String workerId){ + this.workerId = workerId; + return this; + } + + public Builder modelIdx(Integer idx){ + this.modelIdx = idx; + return this; + } + + public Builder score(Double score){ + this.score = score; + return this; + } + + public Builder status(CandidateStatus status){ + this.status = status; + return this; + } + + public Builder scoreVsIter(int[] iter, float[] scoreVsIter){ + this.iter = iter; + this.scoreVsIter = scoreVsIter; + return this; + } + + public Builder lastUpdateTime(long lastUpdateTime){ + this.lastUpdateTime = lastUpdateTime; + return this; + } + + public Builder numParameters(long numParameters){ + this.numParameters = numParameters; + return this; + } + + public Builder numLayers(int numLayers){ + this.numLayers = numLayers; + return this; + } + + public Builder totalNumUpdates(int totalNumUpdates){ + this.totalNumUpdates = totalNumUpdates; + return this; + } + + public Builder paramSpaceValues(double[] paramSpaceValues){ + this.paramSpaceValues = paramSpaceValues; + return this; + } + + public Builder modelConfigJson(String modelConfigJson){ + this.modelConfigJson = modelConfigJson; + return this; + } + + public Builder exceptionStackTrace(String exceptionStackTrace){ + this.exceptionStackTrace = exceptionStackTrace; + return this; + } + + public ModelInfoPersistable build(){ + return new ModelInfoPersistable(this); + } + } +} diff --git a/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/listener/ArbiterStatusListener.java b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/listener/ArbiterStatusListener.java new file mode 100644 index 000000000..c14258be2 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/listener/ArbiterStatusListener.java @@ -0,0 +1,238 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.ui.listener; + +import it.unimi.dsi.fastutil.floats.FloatArrayList; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.arbiter.optimize.api.OptimizationResult; +import org.deeplearning4j.arbiter.optimize.runner.CandidateInfo; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; +import org.deeplearning4j.arbiter.optimize.serde.jackson.JsonMapper; +import org.deeplearning4j.arbiter.ui.data.GlobalConfigPersistable; +import org.deeplearning4j.arbiter.ui.data.ModelInfoPersistable; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.common.primitives.Pair; + +import java.io.IOException; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A {@link StatusListener} for reporting Arbiter/DL4J optimization results to a {@link StatsStorageRouter} + * + * @author Alex Black + */ +@Slf4j +public class ArbiterStatusListener implements StatusListener { + + public static final int MAX_SCORE_VS_ITER_PTS = 1024; //Above this: subsample... every 2nd, 4th, 8th etc + + private final String sessionId; + private final StatsStorageRouter statsStorage; + + private String ocJson; + private long startTime = 0; + + private Map candidateScoreVsIterSubsampleFreq = new ConcurrentHashMap<>(); + private Map> candidateScoreVsIter = new ConcurrentHashMap<>(); + + private Map lastModelInfoPersistable = new ConcurrentHashMap<>(); + + public ArbiterStatusListener(@NonNull StatsStorageRouter statsStorage) { + this(UUID.randomUUID().toString(), statsStorage); + } + + public ArbiterStatusListener(@NonNull String sessionId, @NonNull StatsStorageRouter statsStorage){ + this.sessionId = sessionId; + this.statsStorage = statsStorage; + } + + @Override + public void onInitialization(IOptimizationRunner r) { + Persistable p = getNewStatusPersistable(r); + statsStorage.putStaticInfo(p); + } + + @Override + public void onShutdown(IOptimizationRunner runner) { + //No op? + + } + + @Override + public void onRunnerStatusChange(IOptimizationRunner r) { + Persistable p = getNewStatusPersistable(r); + statsStorage.putStaticInfo(p); + } + + @Override + public void onCandidateStatusChange(CandidateInfo candidateInfo, IOptimizationRunner runner, OptimizationResult result) { + ModelInfoPersistable p = lastModelInfoPersistable.get(candidateInfo.getIndex()); + if(p == null){ + p = new ModelInfoPersistable.Builder() + .timestamp(candidateInfo.getCreatedTime()) + .sessionId(sessionId) + .workerId(String.valueOf(candidateInfo.getIndex())) + .modelIdx(candidateInfo.getIndex()) + .score(candidateInfo.getScore()) + .status(candidateInfo.getCandidateStatus()) + .exceptionStackTrace(candidateInfo.getExceptionStackTrace()) + .build(); + + lastModelInfoPersistable.put(candidateInfo.getIndex(), p); + } + + if(p.getScore() == null){ + p.setScore(candidateInfo.getScore()); + } + + if(result != null && p.getExceptionStackTrace() == null && result.getCandidateInfo().getExceptionStackTrace() != null){ + //Update exceptions that may have occurred since earlier model info instance + p.setExceptionStackTrace(result.getCandidateInfo().getExceptionStackTrace()); + } + + p.setStatus(candidateInfo.getCandidateStatus()); + + statsStorage.putUpdate(p); + } + + @Override + public void onCandidateIteration(CandidateInfo candidateInfo, Object candidate, int iteration) { + double score; + long numParams; + int numLayers; + String modelConfigJson; + int totalNumUpdates; + if(candidate instanceof MultiLayerNetwork){ + MultiLayerNetwork m = (MultiLayerNetwork)candidate; + score = m.score(); + numParams = m.numParams(); + numLayers = m.getnLayers(); + modelConfigJson = m.getLayerWiseConfigurations().toJson(); + totalNumUpdates = m.getLayerWiseConfigurations().getIterationCount(); + } else if(candidate instanceof ComputationGraph) { + ComputationGraph cg = (ComputationGraph)candidate; + score = cg.score(); + numParams = cg.numParams(); + numLayers = cg.getNumLayers(); + modelConfigJson = cg.getConfiguration().toJson(); + totalNumUpdates = cg.getConfiguration().getIterationCount(); + } else { + score = 0; + numParams = 0; + numLayers = 0; + totalNumUpdates = 0; + modelConfigJson = ""; + } + + int idx = candidateInfo.getIndex(); + + Pair pair = candidateScoreVsIter.computeIfAbsent(idx, k -> new Pair<>(new IntArrayList(), new FloatArrayList())); + + IntArrayList iter = pair.getFirst(); + FloatArrayList scores = pair.getSecond(); + + //Do we need subsampling to avoid having too many data points? + int subsamplingFreq = candidateScoreVsIterSubsampleFreq.computeIfAbsent(idx, k -> 1); + if(iteration / subsamplingFreq > MAX_SCORE_VS_ITER_PTS){ + //Double subsampling frequency and re-parse data + subsamplingFreq *= 2; + candidateScoreVsIterSubsampleFreq.put(idx, subsamplingFreq); + + IntArrayList newIter = new IntArrayList(); + FloatArrayList newScores = new FloatArrayList(); + for( int i=0; i(iter, scores)); + } + + if(iteration % subsamplingFreq == 0) { + iter.add(iteration); + scores.add((float) score); + } + + + int[] iters = iter.toIntArray(); + float[] fScores = new float[iters.length]; + for( int i=0; i T fromJson(String json, Class type){ + try{ + return getMapper().readValue(json, type); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static ObjectMapper getInstance(){ + return MAPPER; + } + +} diff --git a/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/misc/UIUtils.java b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/misc/UIUtils.java new file mode 100644 index 000000000..8ea969c82 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/misc/UIUtils.java @@ -0,0 +1,112 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.ui.misc; + +import org.joda.time.Period; +import org.joda.time.PeriodType; +import org.joda.time.format.PeriodFormatter; +import org.joda.time.format.PeriodFormatterBuilder; + +/** + * Created by Alex on 20/07/2017. + */ +public class UIUtils { + + /** + * Convert the "messy" min/max values on a dataset to something clean. For example, 0.895732 becomes 1.0 + * + * @param max Maximum data point value + * @param min Minimum data point value + * @param nTick Number of tick marks desired on chart (good setting: 5) + * @return double[] of length 2 - with new minimum and maximum + */ + public static double[] graphNiceRange(double max, double min, int nTick){ + if(max == min || !Double.isFinite(max)){ + if(max == 0.0 || !Double.isFinite(max)){ + return new double[]{0.0, 1.0}; + } + + return graphNiceRange(1.5 * max, 0.5 * max, nTick); + } + + double range = niceNum(max-min, false); + double d = niceNum(range / (nTick-1), true ); + double graphMin = Math.floor(min/d)*d; + double graphMax = Math.ceil(max/d)*d; + + + return new double[]{graphMin, graphMax}; + } + + public static double niceNum(double x, boolean round){ + double exp = Math.floor(Math.log10(x)); + double f = x / Math.pow(10, exp); + + double nf; + if(round){ + if(f < 1.5 ){ + nf = 1; + } else if( f < 3){ + nf = 2; + } else if( f < 7){ + nf = 5; + } else { + nf = 10; + } + } else { + if(f <= 1 ){ + nf = 1; + } else if( f <= 2){ + nf = 2; + } else if( f <= 5){ + nf = 5; + } else { + nf = 10; + } + } + return nf * Math.pow(10, exp); + } + + /** + * Format the duration in milliseconds to a human readable String, with "yr", "days", "hr" etc prefixes + * + * + * @param durationMs Duration in milliseconds + * @return Human readable string + */ + public static String formatDuration(long durationMs){ + Period period = Period.seconds((int)(durationMs/1000L)); + Period p2 = period.normalizedStandard(PeriodType.yearMonthDayTime()); + + PeriodFormatter formatter = new PeriodFormatterBuilder() + .appendYears() + .appendSuffix(" yr ") + .appendMonths() + .appendSuffix(" months ") + .appendDays() + .appendSuffix(" days ") + .appendHours() + .appendSuffix(" hr ") + .appendMinutes() + .appendSuffix(" min ") + .appendSeconds() + .appendSuffix(" sec") + .toFormatter(); + + return formatter.print(p2); + } +} diff --git a/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/module/ArbiterModule.java b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/module/ArbiterModule.java new file mode 100644 index 000000000..1ee0ce729 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/java/org/deeplearning4j/arbiter/ui/module/ArbiterModule.java @@ -0,0 +1,943 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.ui.module; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.ext.web.RoutingContext; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StatsStorage; +import org.deeplearning4j.core.storage.StatsStorageEvent; +import org.deeplearning4j.core.storage.StatsStorageListener; +import org.deeplearning4j.arbiter.BaseNetworkSpace; +import org.deeplearning4j.arbiter.layers.LayerSpace; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.termination.TerminationCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.runner.CandidateStatus; +import org.deeplearning4j.arbiter.ui.UpdateStatus; +import org.deeplearning4j.arbiter.ui.data.GlobalConfigPersistable; +import org.deeplearning4j.arbiter.ui.data.ModelInfoPersistable; +import org.deeplearning4j.arbiter.ui.misc.UIUtils; +import org.deeplearning4j.arbiter.util.ObjectUtils; +import org.deeplearning4j.nn.conf.serde.JsonMappers; +import org.deeplearning4j.ui.VertxUIServer; +import org.deeplearning4j.ui.api.Component; +import org.deeplearning4j.ui.api.*; +import org.deeplearning4j.ui.components.chart.ChartLine; +import org.deeplearning4j.ui.components.chart.ChartScatter; +import org.deeplearning4j.ui.components.chart.style.StyleChart; +import org.deeplearning4j.ui.components.component.ComponentDiv; +import org.deeplearning4j.ui.components.component.style.StyleDiv; +import org.deeplearning4j.ui.components.table.ComponentTable; +import org.deeplearning4j.ui.components.table.style.StyleTable; +import org.deeplearning4j.ui.components.text.ComponentText; +import org.deeplearning4j.ui.components.text.style.StyleText; +import org.deeplearning4j.ui.i18n.I18NResource; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.nd4j.common.function.Function; +import org.nd4j.common.primitives.Pair; + +import java.awt.*; +import java.text.DecimalFormat; +import java.util.List; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A Deeplearning4j {@link UIModule}, for integration with DL4J's user interface + * + * @author Alex Black + */ +@Slf4j +public class ArbiterModule implements UIModule { + + private static final DecimalFormat DECIMAL_FORMAT_2DP = new DecimalFormat("#.00"); + private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormat.forPattern("YYYY-MM-dd HH:mm ZZ"); + public static final String ARBITER_UI_TYPE_ID = "ArbiterUI"; + + private AtomicBoolean loggedArbiterAddress = new AtomicBoolean(false); + private Map knownSessionIDs = Collections.synchronizedMap(new LinkedHashMap<>()); + private String currentSessionID; + + private Map lastUpdateForSession = Collections.synchronizedMap(new HashMap<>()); + + //Styles for UI: + private static final StyleTable STYLE_TABLE = new StyleTable.Builder() + .width(100, LengthUnit.Percent) + .backgroundColor(Color.WHITE) + .borderWidth(1) + .columnWidths(LengthUnit.Percent, 30, 70) + .build(); + + private static final StyleTable STYLE_TABLE3_25_25_50 = new StyleTable.Builder() + .width(100, LengthUnit.Percent) + .backgroundColor(Color.WHITE) + .borderWidth(1) + .columnWidths(LengthUnit.Percent, 25, 25, 50) + .build(); + + private static final StyleDiv STYLE_DIV_WIDTH_100_PC = new StyleDiv.Builder() + .width(100, LengthUnit.Percent) + .build(); + + private static final ComponentDiv DIV_SPACER_20PX = new ComponentDiv(new StyleDiv.Builder() + .width(100,LengthUnit.Percent) + .height(20, LengthUnit.Px).build()); + + private static final ComponentDiv DIV_SPACER_60PX = new ComponentDiv(new StyleDiv.Builder() + .width(100,LengthUnit.Percent) + .height(60, LengthUnit.Px).build()); + + private static final StyleChart STYLE_CHART_560_320 = new StyleChart.Builder() + .width(560, LengthUnit.Px) + .height(320, LengthUnit.Px) + .build(); + + private static final StyleChart STYLE_CHART_800_400 = new StyleChart.Builder() + .width(800, LengthUnit.Px) + .height(400, LengthUnit.Px) + .build(); + + + private StyleText STYLE_TEXT_SZ12 = new StyleText.Builder() + .fontSize(12) + .build(); + + //Set whitespacePre(true) to avoid losing new lines, tabs, multiple spaces etc + private StyleText STYLE_TEXT_SZ10_WHITESPACE_PRE = new StyleText.Builder() + .fontSize(10) + .whitespacePre(true) + .build(); + + + @Override + public List getCallbackTypeIDs() { + return Collections.singletonList(ARBITER_UI_TYPE_ID); + } + + @Override + public List getRoutes() { + boolean multiSession = VertxUIServer.getMultiSession().get(); + List r = new ArrayList<>(); + r.add(new Route("/arbiter/multisession", HttpMethod.GET, + (path, rc) -> rc.response().end(multiSession ? "true" : "false"))); + if (multiSession) { + r.add(new Route("/arbiter", HttpMethod.GET, (path, rc) -> this.listSessions(rc))); + r.add(new Route("/arbiter/:sessionId", HttpMethod.GET, (path, rc) -> { + if (knownSessionIDs.containsKey(path.get(0))) { + rc.response() + .putHeader("content-type", "text/html; charset=utf-8") + .sendFile("templates/ArbiterUI.html"); + } else { + sessionNotFound(path.get(0), rc.request().path(), rc); + } + })); + + r.add(new Route("/arbiter/:sessionId/lastUpdate", HttpMethod.GET, (path, rc) -> { + if (knownSessionIDs.containsKey(path.get(0))) { + this.getLastUpdateTime(path.get(0), rc); + } else { + sessionNotFound(path.get(0), rc.request().path(), rc); + } + })); + r.add(new Route("/arbiter/:sessionId/candidateInfo/:id", HttpMethod.GET, (path, rc) -> { + if (knownSessionIDs.containsKey(path.get(0))) { + this.getCandidateInfo(path.get(0), path.get(1), rc); + } else { + sessionNotFound(path.get(0), rc.request().path(), rc); + } + })); + r.add(new Route("/arbiter/:sessionId/config", HttpMethod.GET, (path, rc) -> { + if (knownSessionIDs.containsKey(path.get(0))) { + this.getOptimizationConfig(path.get(0), rc); + } else { + sessionNotFound(path.get(0), rc.request().path(), rc); + } + })); + r.add(new Route("/arbiter/:sessionId/results", HttpMethod.GET, (path, rc) -> { + if (knownSessionIDs.containsKey(path.get(0))) { + this.getSummaryResults(path.get(0), rc); + } else { + sessionNotFound(path.get(0), rc.request().path(), rc); + } + })); + r.add(new Route("/arbiter/:sessionId/summary", HttpMethod.GET, (path, rc) -> { + if (knownSessionIDs.containsKey(path.get(0))) { + this.getSummaryStatus(path.get(0), rc); + } else { + sessionNotFound(path.get(0), rc.request().path(), rc); + } + })); + } else { + r.add(new Route("/arbiter", HttpMethod.GET, (path, rc) -> rc.response() + .putHeader("content-type", "text/html; charset=utf-8") + .sendFile("templates/ArbiterUI.html"))); + r.add(new Route("/arbiter/lastUpdate", HttpMethod.GET, (path, rc) -> this.getLastUpdateTime(null, rc))); + r.add(new Route("/arbiter/candidateInfo/:id", HttpMethod.GET, + (path, rc) -> this.getCandidateInfo(null, path.get(0), rc))); + r.add(new Route("/arbiter/config", HttpMethod.GET, (path, rc) -> this.getOptimizationConfig(null, rc))); + r.add(new Route("/arbiter/results", HttpMethod.GET, (path, rc) -> this.getSummaryResults(null, rc))); + r.add(new Route("/arbiter/summary", HttpMethod.GET, (path, rc) -> this.getSummaryStatus(null, rc))); + + r.add(new Route("/arbiter/sessions/current", HttpMethod.GET, (path, rc) -> this.currentSession(rc))); + r.add(new Route("/arbiter/sessions/set/:to", HttpMethod.GET, + (path, rc) -> this.setSession(path.get(0), rc))); + } + // common for single- and multi-session mode + r.add(new Route("/arbiter/sessions/all", HttpMethod.GET, (path, rc) -> this.sessionInfo(rc))); + + return r; + } + + + /** + * Load StatsStorage via provider, or return "not found" + * + * @param sessionId session ID to look fo with provider + * @param targetPath one of overview / model / system, or null + * @param rc routing context + */ + private void sessionNotFound(String sessionId, String targetPath, RoutingContext rc) { + Function loader = VertxUIServer.getInstance().getStatsStorageLoader(); + if (loader != null && loader.apply(sessionId)) { + if (targetPath != null) { + rc.reroute(targetPath); + } else { + rc.response().end(); + } + } else { + rc.response().setStatusCode(HttpResponseStatus.NOT_FOUND.code()) + .end("Unknown session ID: " + sessionId); + } + } + + + /** + * List optimization sessions. Returns a HTML list of arbiter sessions + */ + private synchronized void listSessions(RoutingContext rc) { + StringBuilder sb = new StringBuilder("\n" + + "\n" + + "\n" + + " \n" + + " Optimization sessions - DL4J Arbiter UI\n" + + " \n" + + "\n" + + " \n" + + "

DL4J Arbiter UI

\n" + + "

UI server is in multi-session mode." + + " To visualize an optimization session, please select one from the following list.

\n" + + "

List of attached optimization sessions

\n"); + if (!knownSessionIDs.isEmpty()) { + sb.append(" "); + } else { + sb.append("No optimization session attached."); + } + + sb.append(" \n" + + "\n"); + + rc.response() + .putHeader("content-type", "text/html; charset=utf-8") + .end(sb.toString()); + } + + @Override + public void reportStorageEvents(Collection events) { + boolean attachedArbiter = false; + for (StatsStorageEvent sse : events) { + if (ARBITER_UI_TYPE_ID.equals(sse.getTypeID())) { + if (sse.getEventType() == StatsStorageListener.EventType.PostStaticInfo) { + knownSessionIDs.put(sse.getSessionID(), sse.getStatsStorage()); + } + + Long lastUpdate = lastUpdateForSession.get(sse.getSessionID()); + if (lastUpdate == null) { + lastUpdateForSession.put(sse.getSessionID(), sse.getTimestamp()); + } else if (sse.getTimestamp() > lastUpdate) { + lastUpdateForSession.put(sse.getSessionID(), sse.getTimestamp()); //Should be thread safe - read only elsewhere + } + attachedArbiter = true; + } + } + + if(currentSessionID == null){ + getDefaultSession(); + } + + if(attachedArbiter && !loggedArbiterAddress.getAndSet(true)){ + String address = UIServer.getInstance().getAddress(); + address += "/arbiter"; + log.info("DL4J Arbiter Hyperparameter Optimization UI: {}", address); + } + } + + @Override + public synchronized void onAttach(StatsStorage statsStorage) { + for (String sessionID : statsStorage.listSessionIDs()) { + for (String typeID : statsStorage.listTypeIDsForSession(sessionID)) { + if (!ARBITER_UI_TYPE_ID.equals(typeID)) + continue; + knownSessionIDs.put(sessionID, statsStorage); + } + } + + if (currentSessionID == null) + getDefaultSession(); + } + + private void currentSession(RoutingContext rc) { + String sid = currentSessionID == null ? "" : currentSessionID; + rc.response() + .putHeader("content-type", "application/json") + .end(asJson(sid)); + } + + private void sessionInfo(RoutingContext rc) { + rc.response() + .putHeader("content-type", "application/json") + .end(asJson(knownSessionIDs.keySet())); + } + + private void setSession(String newSessionID, RoutingContext rc) { + log.debug("Arbiter UI: Set to session {}", newSessionID); + + if (knownSessionIDs.containsKey(newSessionID)) { + currentSessionID = newSessionID; + rc.response().end(); + } else { + rc.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()).end("Unknown session ID: " + newSessionID); + } + } + + private void getDefaultSession() { + if (currentSessionID != null) + return; + + long mostRecentTime = Long.MIN_VALUE; + String sessionID = null; + for (Map.Entry entry : knownSessionIDs.entrySet()) { + List staticInfos = entry.getValue().getAllStaticInfos(entry.getKey(), ARBITER_UI_TYPE_ID); + if (staticInfos == null || staticInfos.size() == 0) + continue; + Persistable p = staticInfos.get(0); + long thisTime = p.getTimeStamp(); + if (thisTime > mostRecentTime) { + mostRecentTime = thisTime; + sessionID = entry.getKey(); + } + } + + if (sessionID != null) { + currentSessionID = sessionID; + } + } + + @Override + public void onDetach(StatsStorage statsStorage) { + for (String s : knownSessionIDs.keySet()) { + if (knownSessionIDs.get(s) == statsStorage) { + knownSessionIDs.remove(s); + } + } + } + + @Override + public List getInternationalizationResources() { + return Collections.emptyList(); + } + + /** + * Return the last update time for the page + * @param sessionId session ID (optional, for multi-session mode) + * @param rc routing context + */ + private void getLastUpdateTime(String sessionId, RoutingContext rc){ + if (sessionId == null) { + sessionId = currentSessionID; + } + StatsStorage ss = knownSessionIDs.get(sessionId); + List latestUpdates = ss.getLatestUpdateAllWorkers(sessionId, ARBITER_UI_TYPE_ID); + long t = 0; + if (latestUpdates.isEmpty()) { + t = System.currentTimeMillis(); + } else { + for (Persistable update : latestUpdates) { + if (update.getTimeStamp() > t) { + t = update.getTimeStamp(); + } + } + } + UpdateStatus us = new UpdateStatus(t, t, t); + + rc.response().putHeader("content-type", "application/json").end(asJson(us)); + } + + private String asJson(Object o){ + try{ + return JsonMappers.getMapper().writeValueAsString(o); + } catch (JsonProcessingException e){ + throw new RuntimeException("Error converting object to JSON", e); + } + } + + /** + * Get the info for a specific candidate - last section in the UI + * @param sessionId session ID (optional, for multi-session mode) + * @param candidateId ID for the candidate + * @param rc routing context + */ + private void getCandidateInfo(String sessionId, String candidateId, RoutingContext rc){ + if (sessionId == null) { + sessionId = currentSessionID; + } + StatsStorage ss = knownSessionIDs.get(sessionId); + if(ss == null){ + log.debug("getModelLastUpdateTimes(): Session ID is unknown: {}", sessionId); + rc.response().end(); + return; + } + + GlobalConfigPersistable gcp = (GlobalConfigPersistable)ss + .getStaticInfo(sessionId, ARBITER_UI_TYPE_ID, GlobalConfigPersistable.GLOBAL_WORKER_ID); + OptimizationConfiguration oc = gcp.getOptimizationConfiguration(); + + Persistable p = ss.getLatestUpdate(sessionId, ARBITER_UI_TYPE_ID, candidateId); + if(p == null){ + String title = "No results found for model " + candidateId + "."; + ComponentText ct = new ComponentText.Builder(title,STYLE_TEXT_SZ12).build(); + rc.response() + .putHeader("content-type", "application/json") + .end(asJson(ct)); + return; + } + + ModelInfoPersistable mip = (ModelInfoPersistable)p; + + //First: static info + // Hyperparameter configuration/settings + // Number of parameters + // Maybe memory info in the future? + + //Second: dynamic info + //Runtime + // Performance stats (total minibatches, total time, + // Score vs. time + + List components = new ArrayList<>(); + + //First table: mix of static + dynamic in a table + long runtimeDurationMs = mip.getLastUpdateTime() - mip.getTimeStamp(); + double avgMinibatchesPerSec = mip.getTotalNumUpdates() / (runtimeDurationMs/1000.0); + String avgMinibatchesPerSecStr = DECIMAL_FORMAT_2DP.format(avgMinibatchesPerSec); + String runtimeStr = UIUtils.formatDuration(runtimeDurationMs); + + if(mip.getStatus() == CandidateStatus.Failed){ + runtimeStr = ""; + avgMinibatchesPerSecStr = ""; + } + + String[][] table = new String[][]{ + {"Model Index", String.valueOf(mip.getModelIdx())}, + {"Status", mip.getStatus().toString()}, + {"Model Score", mip.getScore() == null ? "" : String.valueOf(mip.getScore())}, + {"Created", TIME_FORMATTER.print(mip.getTimeStamp())}, + {"Runtime", runtimeStr}, + {"Total Number of Model Updates", String.valueOf(mip.getTotalNumUpdates())}, + {"Average # Updates / Sec", avgMinibatchesPerSecStr}, + {"Number of Parameters", String.valueOf(mip.getNumParameters())}, + {"Number of Layers", String.valueOf(mip.getNumLayers())} + }; + + ComponentTable cTable = new ComponentTable.Builder(STYLE_TABLE) + .content(table) + .header("Model Information", "") + .build(); + components.add(cTable); + + + //Second: parameter space values, in multiple tables + double[] paramSpaceValues = mip.getParamSpaceValues(); + if(paramSpaceValues != null){ + BaseNetworkSpace bns = (BaseNetworkSpace)oc.getCandidateGenerator().getParameterSpace(); + Map m = bns.getNestedSpaces(); + + String[][] hSpaceTable = new String[m.size()][3]; + int i=0; + for(Map.Entry e : m.entrySet()){ + hSpaceTable[i][0] = e.getKey(); + Object currCandidateValue = e.getValue().getValue(paramSpaceValues); + hSpaceTable[i][1] = ObjectUtils.valueToString(currCandidateValue); + hSpaceTable[i][2] = e.getValue().toString(); + i++; + } + + String[] hSpaceTableHeader = new String[]{"Hyperparameter", "Model Value", "Hyperparameter Space"}; + + ComponentTable ct2 = new ComponentTable.Builder(STYLE_TABLE3_25_25_50) + .content(hSpaceTable) + .header(hSpaceTableHeader) + .build(); + + + String title = "Global Network Configuration"; + components.add(DIV_SPACER_20PX); + components.add(new ComponentText.Builder(title, STYLE_TEXT_SZ12).build()); + components.add(ct2); + + List layerConfs = bns.getLayerSpaces(); + + for(BaseNetworkSpace.LayerConf l : layerConfs){ + LayerSpace ls = l.getLayerSpace(); + Map lpsm = ls.getNestedSpaces(); + + String[][] t = new String[lpsm.size()][3]; + i=0; + for(Map.Entry e : lpsm.entrySet()){ + t[i][0] = e.getKey(); + Object currCandidateValue = e.getValue().getValue(paramSpaceValues); + t[i][1] = ObjectUtils.valueToString(currCandidateValue); + t[i][2] = e.getValue().toString(); + i++; + } + + ComponentTable ct3 = new ComponentTable.Builder(STYLE_TABLE3_25_25_50) + .content(t) + .header(hSpaceTableHeader) + .build(); + + title = "Layer Space: " + ls.getClass().getSimpleName() + ", Name: " + l.getLayerName(); + + components.add(DIV_SPACER_20PX); + components.add(new ComponentText.Builder(title, STYLE_TEXT_SZ12).build()); + components.add(ct3); + } + } + + + //Third: Score vs. time chart + int[] iters = mip.getIter(); + float[] scores = mip.getScoreVsIter(); + + if(iters != null) { + double[] si = new double[iters.length]; + double[] scoresD = new double[iters.length]; + + double minScore = Double.MAX_VALUE; + double maxScore = -Double.MAX_VALUE; + for( int i=0; i components = new ArrayList<>(); + + GlobalConfigPersistable gcp = (GlobalConfigPersistable)p; + OptimizationConfiguration oc = gcp.getOptimizationConfiguration(); + + //Report optimization settings/configuration. + String[] tableHeader = {"Configuration", "Value"}; + String [] dataSourceOrProvider; + if (oc.getDataProvider() != null) { + dataSourceOrProvider = new String[] {"Data Provider", oc.getDataProvider().toString()}; + } + else { + dataSourceOrProvider = new String[] {"Data Source", oc.getDataSource().getCanonicalName()}; + } + String[][] table = new String[][]{ + {"Candidate Generator", oc.getCandidateGenerator().getClass().getSimpleName()}, + dataSourceOrProvider, + {"Score Function", oc.getScoreFunction().toString()}, + {"Result Saver", oc.getResultSaver().toString()}, + }; + + ComponentTable ct = new ComponentTable.Builder(STYLE_TABLE) + .content(table) + .header(tableHeader) + .build(); + components.add(ct); + + + String title = "Global Network Configuration"; + components.add(DIV_SPACER_20PX); + components.add(new ComponentText.Builder(title, STYLE_TEXT_SZ12).build()); + BaseNetworkSpace ps = (BaseNetworkSpace)oc.getCandidateGenerator().getParameterSpace(); + Map m = ps.getNestedSpaces(); + + String[][] hSpaceTable = new String[m.size()][2]; + int i=0; + for(Map.Entry e : m.entrySet()){ + hSpaceTable[i][0] = e.getKey(); + hSpaceTable[i][1] = e.getValue().toString(); + i++; + } + + components.add(DIV_SPACER_20PX); + String[] hSpaceTableHeader = new String[]{"Hyperparameter", "Hyperparameter Configuration"}; + + ComponentTable ct2 = new ComponentTable.Builder(STYLE_TABLE) + .content(hSpaceTable) + .header(hSpaceTableHeader) + .build(); + components.add(ct2); + + //Configuration for each layer: + List layerConfs = ps.getLayerSpaces(); + for(BaseNetworkSpace.LayerConf l : layerConfs){ + LayerSpace ls = l.getLayerSpace(); + Map lpsm = ls.getNestedSpaces(); + + String[][] t = new String[lpsm.size()][2]; + i=0; + for(Map.Entry e : lpsm.entrySet()){ + t[i][0] = e.getKey(); + t[i][1] = e.getValue().toString(); + i++; + } + + ComponentTable ct3 = new ComponentTable.Builder(STYLE_TABLE) + .content(t) + .header(hSpaceTableHeader) + .build(); + + title = "Layer Space: " + ls.getClass().getSimpleName() + ", Name: " + l.getLayerName(); + + components.add(DIV_SPACER_20PX); + components.add(new ComponentText.Builder(title, STYLE_TEXT_SZ12).build()); + components.add(ct3); + } + + ComponentDiv cd = new ComponentDiv(STYLE_DIV_WIDTH_100_PC, components); + + rc.response().putHeader("content-type", "application/json").end(asJson(cd)); + } + + /** + * Get candidates summary results list - third section on the page: Results table + * @param sessionId session ID (optional, for multi-session mode) + * @param rc routing context + */ + private void getSummaryResults(String sessionId, RoutingContext rc){ + if (sessionId == null) { + sessionId = currentSessionID; + } + StatsStorage ss = knownSessionIDs.get(sessionId); + if(ss == null){ + log.debug("getSummaryResults(): Session ID is unknown: {}", sessionId); + rc.response().end(); + return; + } + + List allModelInfoTemp = new ArrayList<>(ss.getLatestUpdateAllWorkers(sessionId, ARBITER_UI_TYPE_ID)); + List table = new ArrayList<>(); + for(Persistable per : allModelInfoTemp){ + ModelInfoPersistable mip = (ModelInfoPersistable)per; + String score = (mip.getScore() == null ? "" : mip.getScore().toString()); + table.add(new String[]{mip.getModelIdx().toString(), score, mip.getStatus().toString()}); + } + + rc.response().putHeader("content-type", "application/json").end(asJson(table)); + } + + /** + * Get summary status information: first section in the page + * @param sessionId session ID (optional, for multi-session mode) + * @param rc routing context + */ + private void getSummaryStatus(String sessionId, RoutingContext rc){ + if (sessionId == null) { + sessionId = currentSessionID; + } + StatsStorage ss = knownSessionIDs.get(sessionId); + if(ss == null){ + log.debug("getOptimizationConfig(): Session ID is unknown: {}", sessionId); + rc.response().end(); + return; + } + + Persistable p = ss.getStaticInfo(sessionId, ARBITER_UI_TYPE_ID, GlobalConfigPersistable.GLOBAL_WORKER_ID); + + if(p == null){ + log.info("No static info"); + rc.response().end(); + return; + } + + GlobalConfigPersistable gcp = (GlobalConfigPersistable)p; + OptimizationConfiguration oc = gcp.getOptimizationConfiguration(); + long execStartTime = oc.getExecutionStartTime(); + + + + //Charts: + //Best model score vs. time + //All candidate scores (scatter plot vs. time) + + //How to get this? query all model infos... + + List allModelInfoTemp = new ArrayList<>(ss.getLatestUpdateAllWorkers(sessionId, ARBITER_UI_TYPE_ID)); + List allModelInfo = new ArrayList<>(); + for(Persistable per : allModelInfoTemp){ + ModelInfoPersistable mip = (ModelInfoPersistable)per; + if(mip.getStatus() == CandidateStatus.Complete && mip.getScore() != null && Double.isFinite(mip.getScore())){ + allModelInfo.add(mip); + } + } + + allModelInfo.sort(Comparator.comparingLong(Persistable::getTimeStamp)); + + Pair, ModelInfoPersistable> chartsAndBest = getSummaryChartsAndBest(allModelInfo, oc.getScoreFunction().minimize(), execStartTime ); + + //First: table - number completed, queued, running, failed, total + //Best model index, score, and time + //Total runtime + //Termination conditions + List components = new ArrayList<>(); + + + + List tcs = oc.getTerminationConditions(); + + //TODO: I18N + + long bestTime; + Double bestScore = null; + String bestModelString = null; + if(chartsAndBest.getSecond() != null){ + bestTime = chartsAndBest.getSecond().getTimeStamp(); + bestScore = chartsAndBest.getSecond().getScore(); + String sinceBest = UIUtils.formatDuration(System.currentTimeMillis() - bestTime); + + bestModelString = "Model " + chartsAndBest.getSecond().getModelIdx() + ", Found at " + + TIME_FORMATTER.print(bestTime) + " (" + sinceBest + " ago)"; + } + + String execStartTimeStr = ""; + String execTotalRuntimeStr = ""; + if(execStartTime > 0){ + execStartTimeStr = TIME_FORMATTER.print(execStartTime); + // allModelInfo is sorted by Persistable::getTimeStamp + long lastCompleteTime = execStartTime; + if (!allModelInfo.isEmpty()) { + lastCompleteTime = allModelInfo.get(allModelInfo.size() - 1).getTimeStamp(); + } + execTotalRuntimeStr = UIUtils.formatDuration(lastCompleteTime - execStartTime); + } + + + String[][] table = new String[][]{ + {"Models Completed", String.valueOf(gcp.getCandidatesCompleted())}, + {"Models Queued/Running", String.valueOf(gcp.getCandidatesQueued())}, + {"Models Failed", String.valueOf(gcp.getCandidatesFailed())}, + {"Models Total", String.valueOf(gcp.getCandidatesTotal())}, + {"Best Score", (bestScore != null ? String.valueOf(bestScore) : "")}, + {"Best Scoring Model", bestModelString != null ? bestModelString : ""}, + {"Optimization Runner", gcp.getOptimizationRunner()}, + {"Execution Start Time", execStartTimeStr}, + {"Total Runtime", execTotalRuntimeStr} + }; + + + + ComponentTable ct = new ComponentTable.Builder(STYLE_TABLE) + .content(table) + .header("Status", "") + .build(); + + components.add(ct); + + String[][] tcTable = new String[tcs.size()][2]; + for( int i=0; i,ModelInfoPersistable> getSummaryChartsAndBest(List allModelInfo, + boolean minimize, long execStartTime){ + List bestX = new ArrayList<>(); + List bestY = new ArrayList<>(); + + double[] allX = new double[allModelInfo.size()]; + double[] allY = new double[allModelInfo.size()]; + + double bestScore = (minimize ? Double.MAX_VALUE : -Double.MAX_VALUE); + double worstScore = (minimize ? -Double.MAX_VALUE : Double.MAX_VALUE); + double lastTime = -1L; + ModelInfoPersistable bestModel = null; + for(int i=0; i bestScore) || (minimize && currScore < bestScore)){ + bestX.add(t); + bestY.add(bestScore); + bestX.add(t); //TODO non-real time rendering support... + bestY.add(currScore); + + bestScore = currScore; + bestModel = mip; + } + + if((!minimize && currScore < worstScore) || (minimize && currScore > worstScore)){ + worstScore = currScore; + } + + if(t > lastTime){ + lastTime = t; + } + } + + + double[] scatterGraphMinMax = UIUtils.graphNiceRange(Math.max(bestScore, worstScore), Math.min(bestScore, worstScore), 5); + double[] lineGraphMinMax = UIUtils.graphNiceRange( + bestY.stream().mapToDouble(s -> s).max().orElse(0),bestY.stream().mapToDouble(s -> s).min().orElse(0), 5 + ); + + if(bestX.size() > 0) { + bestX.add(lastTime); + bestY.add(bestY.get(bestY.size() - 1)); + } + + + double[] bestXd = new double[bestX.size()]; + double[] bestYd = new double[bestXd.length]; + for( int i=0; i components = new ArrayList<>(2); + + ChartLine cl = new ChartLine.Builder("Best Model Score vs. Time (Minutes)", STYLE_CHART_560_320) + .addSeries("Best Score vs. Time", bestXd, bestYd) + .setYMin(lineGraphMinMax[0]) + .setYMax(lineGraphMinMax[1]) + .build(); + components.add(cl); + + ChartScatter cs = new ChartScatter.Builder("All Candidate Scores vs. Time (Minutes)", STYLE_CHART_560_320) + .addSeries("Candidates", allX, allY) + .setYMin(scatterGraphMinMax[0]) + .setYMax(scatterGraphMinMax[1]) + .build(); + + components.add(cs); + + return new Pair<>(components, bestModel); + } +} diff --git a/arbiter/arbiter-ui/src/main/resources/META-INF/services/org.deeplearning4j.ui.api.UIModule b/arbiter/arbiter-ui/src/main/resources/META-INF/services/org.deeplearning4j.ui.api.UIModule new file mode 100644 index 000000000..083fd24c9 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/resources/META-INF/services/org.deeplearning4j.ui.api.UIModule @@ -0,0 +1,17 @@ +################################################################################ +# Copyright (c) 2015-2018 Skymind, Inc. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + +org.deeplearning4j.arbiter.ui.module.ArbiterModule \ No newline at end of file diff --git a/arbiter/arbiter-ui/src/main/resources/deeplearning4jUiAssets/dl4j-ui.js b/arbiter/arbiter-ui/src/main/resources/deeplearning4jUiAssets/dl4j-ui.js new file mode 100644 index 000000000..4c99517d0 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/resources/deeplearning4jUiAssets/dl4j-ui.js @@ -0,0 +1,1319 @@ +var __extends = (this && this.__extends) || function (d, b) { + for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); +}; +var Style = (function () { + function Style(jsonObj) { + var _this = this; + this.getWidth = function () { return _this.width; }; + this.getHeight = function () { return _this.height; }; + this.getWidthUnit = function () { return _this.widthUnit; }; + this.getHeightUnit = function () { return _this.heightUnit; }; + this.getMarginTop = function () { return _this.marginTop; }; + this.getMarginBottom = function () { return _this.marginBottom; }; + this.getMarginLeft = function () { return _this.marginLeft; }; + this.getMarginRight = function () { return _this.marginRight; }; + this.getBackgroundColor = function () { return _this.backgroundColor; }; + this.width = jsonObj['width']; + this.height = jsonObj['height']; + this.widthUnit = TSUtils.normalizeLengthUnit(jsonObj['widthUnit']); + this.heightUnit = TSUtils.normalizeLengthUnit(jsonObj['heightUnit']); + this.marginTop = jsonObj['marginTop']; + this.marginBottom = jsonObj['marginBottom']; + this.marginLeft = jsonObj['marginLeft']; + this.marginRight = jsonObj['marginRight']; + this.backgroundColor = jsonObj['backgroundColor']; + } + Style.getMargins = function (s) { + var mTop = (s ? s.getMarginTop() : 0); + var mBottom = (s ? s.getMarginBottom() : 0); + var mLeft = (s ? s.getMarginLeft() : 0); + var mRight = (s ? s.getMarginRight() : 0); + return { top: mTop, + right: mRight, + bottom: mBottom, + left: mLeft, + widthExMargins: s.getWidth() - mLeft - mRight, + heightExMargins: s.getHeight() - mTop - mBottom }; + }; + return Style; +}()); +var ComponentType; +(function (ComponentType) { + ComponentType[ComponentType["ComponentText"] = 0] = "ComponentText"; + ComponentType[ComponentType["ComponentTable"] = 1] = "ComponentTable"; + ComponentType[ComponentType["ComponentDiv"] = 2] = "ComponentDiv"; + ComponentType[ComponentType["ChartHistogram"] = 3] = "ChartHistogram"; + ComponentType[ComponentType["ChartHorizontalBar"] = 4] = "ChartHorizontalBar"; + ComponentType[ComponentType["ChartLine"] = 5] = "ChartLine"; + ComponentType[ComponentType["ChartScatter"] = 6] = "ChartScatter"; + ComponentType[ComponentType["ChartStackedArea"] = 7] = "ChartStackedArea"; + ComponentType[ComponentType["ChartTimeline"] = 8] = "ChartTimeline"; + ComponentType[ComponentType["DecoratorAccordion"] = 9] = "DecoratorAccordion"; +})(ComponentType || (ComponentType = {})); +var Component = (function () { + function Component(componentType) { + this.componentType = componentType; + } + Component.prototype.getComponentType = function () { + return this.componentType; + }; + Component.getComponent = function (jsonStr) { + var json = JSON.parse(jsonStr); + var key; + if (json["componentType"]) + key = json["componentType"]; + else + key = Object.keys(json)[0]; + switch (key) { + case ComponentType[ComponentType.ComponentText]: + return new ComponentText(jsonStr); + case ComponentType[ComponentType.ComponentTable]: + return new ComponentTable(jsonStr); + case ComponentType[ComponentType.ChartHistogram]: + return new ChartHistogram(jsonStr); + case ComponentType[ComponentType.ChartHorizontalBar]: + throw new Error("Horizontal bar chart: not yet implemented"); + case ComponentType[ComponentType.ChartLine]: + return new ChartLine(jsonStr); + case ComponentType[ComponentType.ChartScatter]: + return new ChartScatter(jsonStr); + case ComponentType[ComponentType.ChartStackedArea]: + return new ChartStackedArea(jsonStr); + case ComponentType[ComponentType.ChartTimeline]: + return new ChartTimeline(jsonStr); + case ComponentType[ComponentType.DecoratorAccordion]: + return new DecoratorAccordion(jsonStr); + case ComponentType[ComponentType.ComponentDiv]: + return new ComponentDiv(jsonStr); + default: + throw new Error("Unknown component type \"" + key + "\" or invalid JSON: \"" + jsonStr + "\""); + } + }; + return Component; +}()); +var ChartConstants = (function () { + function ChartConstants() { + } + ChartConstants.DEFAULT_CHART_STROKE_WIDTH = 1.0; + ChartConstants.DEFAULT_CHART_POINT_SIZE = 3.0; + ChartConstants.DEFAULT_AXIS_STROKE_WIDTH = 1.0; + ChartConstants.DEFAULT_TITLE_COLOR = "#000000"; + return ChartConstants; +}()); +var TSUtils = (function () { + function TSUtils() { + } + TSUtils.max = function (input) { + var max = -Number.MAX_VALUE; + for (var i = 0; i < input.length; i++) { + for (var j = 0; j < input[i].length; j++) { + max = Math.max(max, input[i][j]); + } + } + return max; + }; + TSUtils.min = function (input) { + var min = Number.MAX_VALUE; + for (var i = 0; i < input.length; i++) { + for (var j = 0; j < input[i].length; j++) { + min = Math.min(min, input[i][j]); + } + } + return min; + }; + TSUtils.normalizeLengthUnit = function (input) { + if (input == null) + return input; + switch (input.toLowerCase()) { + case "px": + return "px"; + case "percent": + case "%": + return "%"; + case "cm": + return "cm"; + case "mm": + return "mm"; + case "in": + return "in"; + default: + return input; + } + }; + return TSUtils; +}()); +var Chart = (function (_super) { + __extends(Chart, _super); + function Chart(componentType, jsonStr) { + _super.call(this, componentType); + var jsonOrig = JSON.parse(jsonStr); + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[componentType]]; + this.suppressAxisHorizontal = json['suppressAxisHorizontal']; + this.suppressAxisVertical = json['suppressAxisVertical']; + this.showLegend = json['showLegend']; + this.title = json['title']; + this.setXMin = json['setXMin']; + this.setXMax = json['setXMax']; + this.setYMin = json['setYMin']; + this.setYMax = json['setYMax']; + this.gridVerticalStrokeWidth = json['gridVerticalStrokeWidth']; + this.gridHorizontalStrokeWidth = json['gridHorizontalStrokeWidth']; + if (json['style']) + this.style = new StyleChart(json['style']); + } + Chart.prototype.getStyle = function () { + return this.style; + }; + Chart.appendTitle = function (svg, title, margin, titleStyle) { + var text = svg.append("text") + .text(title) + .attr("x", (margin.widthExMargins / 2)) + .attr("y", 0 - ((margin.top - 30) / 2)) + .attr("text-anchor", "middle"); + if (titleStyle) { + if (titleStyle.getFont()) + text.attr("font-family", titleStyle.getFont); + if (titleStyle.getFontSize() != null) + text.attr("font-size", titleStyle.getFontSize() + "pt"); + if (titleStyle.getUnderline() != null) + text.style("text-decoration", "underline"); + if (titleStyle.getColor()) + text.style("fill", titleStyle.getColor); + else + text.style("fill", ChartConstants.DEFAULT_TITLE_COLOR); + } + else { + text.style("text-decoration", "underline"); + text.style("fill", ChartConstants.DEFAULT_TITLE_COLOR); + } + }; + return Chart; +}(Component)); +var ChartHistogram = (function (_super) { + __extends(ChartHistogram, _super); + function ChartHistogram(jsonStr) { + _super.call(this, ComponentType.ChartHistogram, jsonStr); + this.render = function (appendToObject) { + var s = this.getStyle(); + var margin = Style.getMargins(s); + var xMin; + var xMax; + var yMin; + var yMax; + if (this.setXMin) + xMin = this.setXMin; + else + xMin = (this.lowerBounds ? d3.min(this.lowerBounds) : 0); + if (this.setXMax) + xMax = this.setXMax; + else + xMax = (this.upperBounds ? d3.max(this.upperBounds) : 1); + if (this.setYMin) + yMin = this.setYMin; + else + yMin = 0; + if (this.setYMax) + yMax = this.setYMax; + else + yMax = (this.yValues ? d3.max(this.yValues) : 1); + var xScale = d3.scale.linear() + .domain([xMin, xMax]) + .range([0, margin.widthExMargins]); + var xAxis = d3.svg.axis().scale(xScale) + .orient("bottom").ticks(5); + if (this.gridVerticalStrokeWidth && this.gridVerticalStrokeWidth > 0) { + xAxis.innerTickSize(-margin.heightExMargins); + } + var yScale = d3.scale.linear() + .domain([0, yMax]) + .range([margin.heightExMargins, 0]); + var yAxis = d3.svg.axis().scale(yScale) + .orient("left").ticks(5); + if (this.gridHorizontalStrokeWidth && this.gridHorizontalStrokeWidth > 0) { + yAxis.innerTickSize(-margin.widthExMargins); + } + if (this.suppressAxisHorizontal === true) + xAxis.tickValues([]); + if (this.suppressAxisVertical === true) + yAxis.tickValues([]); + var lowerBounds = this.lowerBounds; + var upperBounds = this.upperBounds; + var yValues = this.yValues; + var data = lowerBounds.map(function (d, i) { + return { 'width': upperBounds[i] - lowerBounds[i], 'height': yValues[i], 'offset': lowerBounds[i] }; + }); + var svg = d3.select("#" + appendToObject.attr("id")) + .append("svg") + .style("fill", "none") + .attr("width", s.getWidth()) + .attr("height", s.getHeight()) + .attr("padding", "20px") + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + svg.selectAll(".bin") + .data(data) + .enter().append("rect") + .attr("class", "bin") + .style("fill", "steelblue") + .attr("x", function (d) { return xScale(d.offset); }) + .attr("width", function (d) { return xScale(xMin + d.width) - 1; }) + .attr("y", function (d) { return yScale(d.height); }) + .attr("height", function (d) { return margin.heightExMargins - yScale(d.height); }); + var xAxisNode = svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + margin.heightExMargins + ")") + .style("stroke", "#000") + .style("stroke-width", (s != null && s.getAxisStrokeWidth() != null ? s.getAxisStrokeWidth() : ChartConstants.DEFAULT_AXIS_STROKE_WIDTH)) + .style("fill", "none") + .call(xAxis); + xAxisNode.selectAll('text').style("stroke-width", 0).style("fill", "#000000"); + if (this.gridVerticalStrokeWidth != null) + xAxisNode.selectAll('.axis line').style({ 'stroke-width': this.gridVerticalStrokeWidth }); + var yAxisNode = svg.append("g") + .attr("class", "y axis") + .style("stroke", "#000") + .style("stroke-width", (s != null && s.getAxisStrokeWidth() != null ? s.getAxisStrokeWidth() : ChartConstants.DEFAULT_AXIS_STROKE_WIDTH)) + .style("fill", "none") + .call(yAxis); + yAxisNode.selectAll('text').style("stroke-width", 0).style("fill", "#000000"); + if (this.gridHorizontalStrokeWidth != null) + yAxisNode.selectAll('.axis line').style({ 'stroke-width': this.gridHorizontalStrokeWidth }); + if (this.title) { + var titleStyle; + if (this.style) + titleStyle = this.style.getTitleStyle(); + Chart.appendTitle(svg, this.title, margin, titleStyle); + } + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.ChartHistogram]]; + this.lowerBounds = json['lowerBounds']; + this.upperBounds = json['upperBounds']; + this.yValues = json['yvalues']; + } + return ChartHistogram; +}(Chart)); +var ChartLine = (function (_super) { + __extends(ChartLine, _super); + function ChartLine(jsonStr) { + _super.call(this, ComponentType.ChartLine, jsonStr); + this.render = function (appendToObject) { + var nSeries = (!this.xData ? 0 : this.xData.length); + var s = this.getStyle(); + var margin = Style.getMargins(s); + var xScale = d3.scale.linear().range([0, margin.widthExMargins]); + var yScale = d3.scale.linear().range([margin.heightExMargins, 0]); + var xAxis = d3.svg.axis().scale(xScale) + .orient("bottom").ticks(5); + if (this.gridVerticalStrokeWidth != null && this.gridVerticalStrokeWidth > 0) { + xAxis.innerTickSize(-margin.heightExMargins); + } + var yAxis = d3.svg.axis().scale(yScale) + .orient("left").ticks(5); + if (this.gridHorizontalStrokeWidth != null && this.gridHorizontalStrokeWidth > 0) { + yAxis.innerTickSize(-margin.widthExMargins); + } + if (this.suppressAxisHorizontal === true) + xAxis.tickValues([]); + if (this.suppressAxisVertical === true) + yAxis.tickValues([]); + var valueline = d3.svg.line() + .x(function (d) { + return xScale(d.xPos); + }) + .y(function (d) { + return yScale(d.yPos); + }); + var svg = d3.select("#" + appendToObject.attr("id")) + .append("svg") + .style("stroke-width", (s && s.getStrokeWidth() ? s.getStrokeWidth() : ChartConstants.DEFAULT_CHART_STROKE_WIDTH)) + .style("fill", "none") + .attr("width", s.getWidth()) + .attr("height", s.getHeight()) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + var xMin; + var xMax; + var yMin; + var yMax; + if (this.setXMin != null) + xMin = this.setXMin; + else + xMin = (this.xData ? TSUtils.min(this.xData) : 0); + if (this.setXMax != null) + xMax = this.setXMax; + else + xMax = (this.xData ? TSUtils.max(this.xData) : 1); + if (this.setYMin != null) + yMin = this.setYMin; + else + yMin = (this.yData ? TSUtils.min(this.yData) : 0); + if (this.setYMax != null) + yMax = this.setYMax; + else + yMax = (this.yData ? TSUtils.max(this.yData) : 1); + xScale.domain([xMin, xMax]); + yScale.domain([yMin, yMax]); + var defaultColor = d3.scale.category10(); + for (var i = 0; i < nSeries; i++) { + var xVals = this.xData[i]; + var yVals = this.yData[i]; + var data = xVals.map(function (d, i) { + return { 'xPos': xVals[i], 'yPos': yVals[i] }; + }); + svg.append("path") + .attr("class", "line") + .style("stroke", (s && s.getSeriesColor(i) ? s.getSeriesColor(i) : defaultColor(String(i)))) + .attr("d", valueline(data)); + } + var xAxisNode = svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + margin.heightExMargins + ")") + .style("stroke", "#000") + .style("stroke-width", (s != null && s.getAxisStrokeWidth() != null ? s.getAxisStrokeWidth() : ChartConstants.DEFAULT_AXIS_STROKE_WIDTH)) + .style("fill", "none") + .call(xAxis); + xAxisNode.selectAll('text').style("stroke-width", 0).style("fill", "#000000"); + if (this.gridVerticalStrokeWidth != null) + xAxisNode.selectAll('.axis line').style({ 'stroke-width': this.gridVerticalStrokeWidth }); + var yAxisNode = svg.append("g") + .attr("class", "y axis") + .style("stroke", "#000") + .style("stroke-width", (s != null && s.getAxisStrokeWidth() != null ? s.getAxisStrokeWidth() : ChartConstants.DEFAULT_AXIS_STROKE_WIDTH)) + .style("fill", "none") + .call(yAxis); + yAxisNode.selectAll('text').style("stroke-width", 0).style("fill", "#000000"); + if (this.gridHorizontalStrokeWidth != null) + yAxisNode.selectAll('.axis line').style({ 'stroke-width': this.gridHorizontalStrokeWidth }); + if (this.seriesNames && this.showLegend === true) { + var legendSpace = margin.widthExMargins / i; + for (var i = 0; i < nSeries; i++) { + var values = this.xData[i]; + var yValues = this.yData[i]; + var lastX = values[values.length - 1]; + var lastY = yValues[yValues.length - 1]; + var toDisplay = this.seriesNames[i]; + svg.append("text") + .attr("x", (legendSpace / 2) + i * legendSpace) + .attr("y", margin.heightExMargins + (margin.bottom / 2) + 5) + .attr("class", "legend") + .style("fill", (s && s.getSeriesColor(i) ? s.getSeriesColor(i) : defaultColor(String(i)))) + .text(toDisplay); + } + } + if (this.title) { + var titleStyle; + if (this.style) + titleStyle = this.style.getTitleStyle(); + Chart.appendTitle(svg, this.title, margin, titleStyle); + } + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.ChartLine]]; + this.xData = json['x']; + this.yData = json['y']; + this.seriesNames = json['seriesNames']; + } + return ChartLine; +}(Chart)); +var ChartScatter = (function (_super) { + __extends(ChartScatter, _super); + function ChartScatter(jsonStr) { + _super.call(this, ComponentType.ChartScatter, jsonStr); + this.render = function (appendToObject) { + var nSeries = (!this.xData ? 0 : this.xData.length); + var s = this.getStyle(); + var margin = Style.getMargins(s); + var xScale = d3.scale.linear().range([0, margin.widthExMargins]); + var yScale = d3.scale.linear().range([margin.heightExMargins, 0]); + var xAxis = d3.svg.axis().scale(xScale) + .innerTickSize(-margin.heightExMargins) + .orient("bottom").ticks(5); + var yAxis = d3.svg.axis().scale(yScale) + .innerTickSize(-margin.widthExMargins) + .orient("left").ticks(5); + if (this.suppressAxisHorizontal === true) + xAxis.tickValues([]); + if (this.suppressAxisVertical === true) + yAxis.tickValues([]); + var svg = d3.select("#" + appendToObject.attr("id")) + .append("svg") + .style("stroke-width", (s && s.getStrokeWidth() ? s.getStrokeWidth() : 1)) + .style("fill", "none") + .attr("width", s.getWidth()) + .attr("height", s.getHeight()) + .attr("padding", "20px") + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + var xMin; + var xMax; + var yMin; + var yMax; + if (this.setXMin) + xMin = this.setXMin; + else + xMin = (this.xData ? TSUtils.min(this.xData) : 0); + if (this.setXMax) + xMax = this.setXMax; + else + xMax = (this.xData ? TSUtils.max(this.xData) : 1); + if (this.setYMin) + yMin = this.setYMin; + else + yMin = (this.yData ? TSUtils.min(this.yData) : 0); + if (this.setYMax) + yMax = this.setYMax; + else + yMax = (this.yData ? TSUtils.max(this.yData) : 1); + xScale.domain([xMin, xMax]); + yScale.domain([yMin, yMax]); + var defaultColor = d3.scale.category10(); + for (var i = 0; i < nSeries; i++) { + var xVals = this.xData[i]; + var yVals = this.yData[i]; + var data = xVals.map(function (d, i) { + return { 'xPos': xVals[i], 'yPos': yVals[i] }; + }); + svg.selectAll("circle") + .data(data) + .enter() + .append("circle") + .style("fill", (s && s.getSeriesColor(i) ? s.getSeriesColor(i) : defaultColor(String(i)))) + .attr("r", (s && s.getPointSize() ? s.getPointSize() : ChartConstants.DEFAULT_CHART_POINT_SIZE)) + .attr("cx", function (d) { + return xScale(d['xPos']); + }) + .attr("cy", function (d) { + return yScale(d['yPos']); + }); + } + var xAxisNode = svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + margin.heightExMargins + ")") + .style("stroke", "#000") + .style("stroke-width", (s != null && s.getAxisStrokeWidth() != null ? s.getAxisStrokeWidth() : ChartConstants.DEFAULT_AXIS_STROKE_WIDTH)) + .style("fill", "none") + .call(xAxis); + xAxisNode.selectAll('text').style("stroke-width", 0).style("fill", "#000000"); + if (this.gridVerticalStrokeWidth != null) + xAxisNode.selectAll('.axis line').style({ 'stroke-width': this.gridVerticalStrokeWidth }); + var yAxisNode = svg.append("g") + .attr("class", "y axis") + .style("stroke", "#000") + .style("stroke-width", (s != null && s.getAxisStrokeWidth() != null ? s.getAxisStrokeWidth() : ChartConstants.DEFAULT_AXIS_STROKE_WIDTH)) + .style("fill", "none") + .call(yAxis); + yAxisNode.selectAll('text').style("stroke-width", 0).style("fill", "#000000"); + if (this.gridHorizontalStrokeWidth != null) + yAxisNode.selectAll('.axis line').style({ 'stroke-width': this.gridHorizontalStrokeWidth }); + if (this.seriesNames && this.showLegend === true) { + var legendSpace = margin.widthExMargins / i; + for (var i = 0; i < nSeries; i++) { + var values = this.xData[i]; + var yValues = this.yData[i]; + var lastX = values[values.length - 1]; + var lastY = yValues[yValues.length - 1]; + var toDisplay; + if (!lastX || !lastY) + toDisplay = this.seriesNames[i] + " (no data)"; + else + toDisplay = this.seriesNames[i] + " (" + lastX.toPrecision(5) + "," + lastY.toPrecision(5) + ")"; + svg.append("text") + .attr("x", (legendSpace / 2) + i * legendSpace) + .attr("y", margin.heightExMargins + (margin.bottom / 2) + 5) + .attr("class", "legend") + .style("fill", (s && s.getSeriesColor(i) ? s.getSeriesColor(i) : defaultColor(String(i)))) + .text(toDisplay); + } + } + if (this.title) { + var titleStyle; + if (this.style) + titleStyle = this.style.getTitleStyle(); + Chart.appendTitle(svg, this.title, margin, titleStyle); + } + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.ChartScatter]]; + this.xData = json['x']; + this.yData = json['y']; + this.seriesNames = json['seriesNames']; + } + return ChartScatter; +}(Chart)); +var Legend = (function () { + function Legend() { + } + Legend.offsetX = 15; + Legend.offsetY = 15; + Legend.padding = 8; + Legend.separation = 12; + Legend.boxSize = 10; + Legend.fillColor = "#FFFFFF"; + Legend.legendOpacity = 0.75; + Legend.borderStrokeColor = "#000000"; + Legend.legendFn = (function (g) { + var svg = d3.select(g.property("nearestViewportElement")); + var legendBox = g.selectAll(".outerRect").data([true]); + var legendItems = g.selectAll(".legendElement").data([true]); + legendBox.enter().append("rect").attr("class", "outerRect"); + legendItems.enter().append("g").attr("class", "legendElement"); + var legendElements = []; + svg.selectAll("[data-legend]").each(function () { + var thisVar = d3.select(this); + legendElements.push({ + label: thisVar.attr("data-legend"), + color: thisVar.style("fill") + }); + }); + legendItems.selectAll("rect") + .data(legendElements, function (d) { return d.label; }) + .call(function (d) { d.enter().append("rect"); }) + .call(function (d) { d.exit().remove(); }) + .attr("x", 0) + .attr("y", function (d, i) { return i * Legend.separation - Legend.boxSize + "px"; }) + .attr("width", Legend.boxSize) + .attr("height", Legend.boxSize) + .style("fill", function (d) { return d.color; }); + legendItems.selectAll("text") + .data(legendElements, function (d) { return d.label; }) + .call(function (d) { d.enter().append("text"); }) + .call(function (d) { d.exit().remove(); }) + .attr("y", function (d, i) { return i * Legend.separation + "px"; }) + .attr("x", (Legend.padding + Legend.boxSize) + "px") + .text(function (d) { return d.label; }); + var legendBoundingBox = legendItems[0][0].getBBox(); + legendBox.attr("x", (legendBoundingBox.x - Legend.padding)) + .attr("y", (legendBoundingBox.y - Legend.padding)) + .attr("height", (legendBoundingBox.height + 2 * Legend.padding)) + .attr("width", (legendBoundingBox.width + 2 * Legend.padding)) + .style("fill", Legend.fillColor) + .style("stroke", Legend.borderStrokeColor) + .style("opacity", Legend.legendOpacity); + svg.selectAll(".legend").attr("transform", "translate(" + Legend.offsetX + "," + Legend.offsetY + ")"); + }); + return Legend; +}()); +var ChartStackedArea = (function (_super) { + __extends(ChartStackedArea, _super); + function ChartStackedArea(jsonStr) { + _super.call(this, ComponentType.ChartStackedArea, jsonStr); + this.render = function (appendToObject) { + var nSeries = (!this.xData ? 0 : this.xData.length); + var s = this.getStyle(); + var margin = Style.getMargins(s); + var xScale = d3.scale.linear().range([0, margin.widthExMargins]); + var yScale = d3.scale.linear().range([margin.heightExMargins, 0]); + var xAxis = d3.svg.axis().scale(xScale) + .orient("bottom").ticks(5); + if (this.gridVerticalStrokeWidth != null && this.gridVerticalStrokeWidth > 0) { + xAxis.innerTickSize(-margin.heightExMargins); + } + var yAxis = d3.svg.axis().scale(yScale) + .orient("left").ticks(5); + if (this.gridHorizontalStrokeWidth != null && this.gridHorizontalStrokeWidth > 0) { + yAxis.innerTickSize(-margin.widthExMargins); + } + if (this.suppressAxisHorizontal === true) + xAxis.tickValues([]); + if (this.suppressAxisVertical === true) + yAxis.tickValues([]); + var data = []; + for (var i = 0; i < this.xData.length; i++) { + var obj = {}; + for (var j = 0; j < this.labels.length; j++) { + obj[this.labels[j]] = this.yData[j][i]; + obj['xValue'] = this.xData[i]; + } + data.push(obj); + } + var area = d3.svg.area() + .x(function (d) { return xScale(d.xValue); }) + .y0(function (d) { return yScale(d.y0); }) + .y1(function (d) { return yScale(d.y0 + d.y); }); + var stack = d3.layout.stack() + .values(function (d) { return d.values; }); + var svg = d3.select("#" + appendToObject.attr("id")).append("svg") + .attr("width", margin.widthExMargins + margin.left + margin.right) + .attr("height", margin.heightExMargins + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + var color = d3.scale.category20(); + color.domain(d3.keys(data[0]).filter(function (key) { + return key !== "xValue"; + })); + var browsers = stack(color.domain().map(function (name) { + return { + name: name, + values: data.map(function (d) { + return { xValue: d.xValue, y: d[name] * 1 }; + }) + }; + })); + var maxX = d3.max(data, function (d) { + var vals = d3.keys(d).map(function (key) { + return key !== "xValue" ? d[key] : 0; + }); + return d3.sum(vals); + }); + xScale.domain(d3.extent(data, function (d) { + return d.xValue; + })); + yScale.domain([0, maxX]); + var browser = svg.selectAll(".browser") + .data(browsers) + .enter().append("g") + .attr("class", "browser"); + var tempLabels = this.labels; + var defaultColor = d3.scale.category20(); + browser.append("path") + .attr("class", "area") + .attr("data-legend", function (d) { return d.name; }) + .attr("d", function (d) { + return area(d.values); + }) + .style("fill", function (d) { + if (s && s.getSeriesColor(tempLabels.indexOf(d.name))) { + return s.getSeriesColor(tempLabels.indexOf(d.name)); + } + else { + return defaultColor(String(tempLabels.indexOf(d.name))); + } + }) + .style({ "stroke-width": "0px" }); + var xAxisNode = svg.append("g") + .attr("class", "x axis") + .style("stroke", "#000") + .style("stroke-width", (s != null && s.getAxisStrokeWidth() != null ? s.getAxisStrokeWidth() : ChartConstants.DEFAULT_AXIS_STROKE_WIDTH)) + .style("fill", "none") + .attr("transform", "translate(0," + margin.heightExMargins + ")") + .call(xAxis); + xAxisNode.selectAll('text').style("stroke-width", 0).style("fill", "#000000"); + var yAxisNode = svg.append("g") + .attr("class", "y axis") + .style("stroke", "#000") + .style("stroke-width", (s != null && s.getAxisStrokeWidth() != null ? s.getAxisStrokeWidth() : ChartConstants.DEFAULT_AXIS_STROKE_WIDTH)) + .style("fill", "none") + .call(yAxis); + yAxisNode.selectAll('text').style("stroke-width", 0).style("fill", "#000000"); + if (this.title) { + var titleStyle; + if (this.style) + titleStyle = this.style.getTitleStyle(); + Chart.appendTitle(svg, this.title, margin, titleStyle); + } + var legend = svg.append("g") + .attr("class", "legend") + .attr("transform", "translate(40,40)") + .style("font-size", "12px") + .call(Legend.legendFn); + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.ChartStackedArea]]; + this.xData = json['x']; + this.yData = json['y']; + this.labels = json['labels']; + } + return ChartStackedArea; +}(Chart)); +var ChartTimeline = (function (_super) { + __extends(ChartTimeline, _super); + function ChartTimeline(jsonStr) { + _super.call(this, ComponentType.ChartTimeline, jsonStr); + this.render = function (appendToObject) { + var instance = this; + var s = this.getStyle(); + var margin = Style.getMargins(s); + this.itemData = []; + var count = 0; + for (var i = 0; i < this.laneData.length; i++) { + for (var j = 0; j < this.laneData[i].length; j++) { + var obj = {}; + obj["start"] = this.laneData[i][j]["startTimeMs"]; + obj["end"] = this.laneData[i][j]["endTimeMs"]; + obj["id"] = count++; + obj["lane"] = i; + obj["color"] = this.laneData[i][j]["color"]; + obj["label"] = this.laneData[i][j]["entryLabel"]; + this.itemData.push(obj); + } + } + this.lanes = []; + for (var i = 0; i < this.laneNames.length; i++) { + var obj = {}; + obj["label"] = this.laneNames[i]; + obj["id"] = i; + this.lanes.push(obj); + } + var svg = d3.select("#" + appendToObject.attr("id")) + .append("svg") + .style("stroke-width", (s && s.getStrokeWidth() ? s.getStrokeWidth() : ChartConstants.DEFAULT_CHART_STROKE_WIDTH)) + .style("fill", "none") + .attr("width", s.getWidth()) + .attr("height", s.getHeight()) + .append("g"); + var heightExMargins = s.getHeight() - margin.top - margin.bottom; + var widthExMargins = s.getWidth() - margin.left - margin.right; + var miniHeight = this.laneNames.length * ChartTimeline.MINI_LANE_HEIGHT_PX; + var mainHeight = s.getHeight() - miniHeight - margin.top - margin.bottom - 25; + var minTime = d3.min(this.itemData, function (d) { return d.start; }); + var maxTime = d3.max(this.itemData, function (d) { return d.end; }); + this.x = d3.time.scale() + .domain([minTime, maxTime]) + .range([0, widthExMargins]); + this.x1 = d3.time.scale().range([0, widthExMargins]); + this.y1 = d3.scale.linear().domain([0, this.laneNames.length]).range([0, mainHeight]); + this.y2 = d3.scale.linear().domain([0, this.laneNames.length]).range([0, miniHeight]); + this.rect = svg.append('defs').append('clipPath') + .attr('id', 'clip') + .append('rect') + .attr('width', widthExMargins) + .attr('height', s.getHeight() - 100); + this.mainView = svg.append('g') + .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')') + .attr('width', widthExMargins) + .attr('height', mainHeight) + .attr('font-size', '12px') + .attr('font', 'sans-serif'); + this.miniView = svg.append('g') + .attr('transform', 'translate(' + margin.left + ',' + (mainHeight + margin.top + 25) + ')') + .attr('width', widthExMargins) + .attr('height', miniHeight) + .attr('font-size', '10px') + .attr('font', 'sans-serif'); + this.mainView.append('g').selectAll('.laneLines') + .data(this.lanes) + .enter().append('line') + .attr('x1', 0) + .attr('y1', function (d) { + return d3.round(instance.y1(d.id)) + 0.5; + }) + .attr('x2', widthExMargins) + .attr('y2', function (d) { + return d3.round(instance.y1(d.id)) + 0.5; + }) + .attr('stroke', 'lightgray') + .attr('stroke-width', 1); + this.mainView.append('g').selectAll('.laneText') + .data(this.lanes) + .enter().append('text') + .text(function (d) { + if (d.label) + return d.label; + return ""; + }) + .attr('x', -10) + .attr('y', function (d) { + return instance.y1(d.id + .5); + }) + .attr('text-anchor', 'end') + .attr("font", "8pt sans-serif") + .attr('fill', 'black'); + this.miniView.append('g').selectAll('.laneLines') + .data(this.lanes) + .enter().append('line') + .attr('x1', 0) + .attr('y1', function (d) { return d3.round(instance.y2(d.id)) + 0.5; }) + .attr('x2', widthExMargins) + .attr('y2', function (d) { return d3.round(instance.y2(d.id)) + 0.5; }) + .attr('stroke', 'gray') + .attr('stroke-width', 1.0); + this.miniView.append('g').selectAll('.laneText') + .data(this.lanes) + .enter().append('text') + .text(function (d) { + if (d.label) + return d.label; + return ""; + }) + .attr('x', -10) + .attr('y', function (d) { + return instance.y2(d.id + .5); + }) + .attr('dy', '0.5ex') + .attr('text-anchor', 'end') + .attr('fill', 'black'); + this.xTimeAxis = d3.svg.axis() + .scale(this.x1) + .orient('bottom') + .ticks(d3.time.days, 1) + .tickFormat(d3.time.format('%a %d')) + .tickSize(6, 0); + var temp = this.mainView.append('g') + .attr('transform', 'translate(0,' + mainHeight + ')') + .attr('class', 'timeAxis') + .attr('fill', 'black') + .style("stroke", "black").style("stroke-width", 1.0).style("fill", "black") + .attr("font", "10px sans-serif") + .call(this.xTimeAxis); + temp.selectAll('text').style("stroke-width", 0.0).attr('stroke-width', 0.0); + this.itemRects = this.mainView.append('g') + .attr('clip-path', 'url(#clip)'); + this.miniView.append('g').selectAll('miniItems') + .data(this.getMiniViewPaths(this.itemData)) + .enter().append('path') + .attr('class', function (d) { + return 'miniItem ' + d.class; + }) + .attr('d', function (d) { + return d.path; + }) + .attr('stroke', 'black') + .attr('stroke-width', 'black'); + this.miniView.append('rect') + .attr('pointer-events', 'painted') + .attr('width', widthExMargins) + .attr('height', miniHeight) + .attr('visibility', 'hidden') + .on('mouseup', this.moveBrush); + this.brush = d3.svg.brush() + .x(this.x) + .extent([minTime, maxTime]) + .on("brush", this.renderChart); + this.miniView.append('g') + .attr('class', 'x brush') + .call(this.brush) + .selectAll('rect') + .attr('y', 1) + .attr('height', miniHeight - 1) + .style('fill', 'gray') + .style('fill-opacity', '0.2') + .style('stroke', 'DarkSlateGray') + .style('stroke-width', 1); + this.miniView.selectAll('rect.background').remove(); + this.renderChart(); + if (this.title) { + var titleStyle; + if (this.style) + titleStyle = this.style.getTitleStyle(); + var text = svg.append("text") + .text(this.title) + .attr("x", (s.getWidth() / 2)) + .attr("y", ((margin.top - 30) / 2)) + .attr("text-anchor", "middle"); + if (titleStyle) { + if (titleStyle.getFont()) + text.attr("font-family", titleStyle.getFont); + if (titleStyle.getFontSize() != null) + text.attr("font-size", titleStyle.getFontSize() + "pt"); + if (titleStyle.getUnderline() != null) + text.style("text-decoration", "underline"); + if (titleStyle.getColor()) + text.style("fill", titleStyle.getColor); + else + text.style("fill", ChartConstants.DEFAULT_TITLE_COLOR); + } + else { + text.style("text-decoration", "underline"); + text.style("fill", ChartConstants.DEFAULT_TITLE_COLOR); + } + } + }; + this.renderChart = function () { + var instance = this; + var extent = this.brush.extent(); + var minExtent = extent[0]; + var maxExtent = extent[1]; + var visibleItems = this.itemData.filter(function (d) { + return d.start < maxExtent && d.end > minExtent; + }); + this.miniView.select('.brush').call(this.brush.extent([minExtent, maxExtent])); + this.x1.domain([minExtent, maxExtent]); + var range = maxExtent - minExtent; + if (range > 2 * ChartTimeline.MILLISEC_PER_WEEK) { + this.xTimeAxis.ticks(d3.time.mondays, 1).tickFormat(d3.time.format('%a %d')); + } + else if (range > 2 * ChartTimeline.MILLISEC_PER_DAY) { + this.xTimeAxis.ticks(d3.time.days, 1).tickFormat(d3.time.format('%a %d')); + } + else if (range > 2 * ChartTimeline.MILLISEC_PER_HOUR) { + this.xTimeAxis.ticks(d3.time.hours, 4).tickFormat(d3.time.format('%H %p')); + } + else if (range > 2 * ChartTimeline.MILLISEC_PER_MINUTE) { + this.xTimeAxis.ticks(d3.time.minutes, 1).tickFormat(d3.time.format('%H:%M')); + } + else if (range >= 30000) { + this.xTimeAxis.ticks(d3.time.seconds, 10).tickFormat(d3.time.format('%H:%M:%S')); + } + else { + this.xTimeAxis.ticks(d3.time.seconds, 1).tickFormat(d3.time.format('%H:%M:%S')); + } + this.mainView.select('.timeAxis').call(this.xTimeAxis); + var rects = this.itemRects.selectAll('rect') + .data(visibleItems, function (d) { return d.id; }) + .attr('x', function (d) { return instance.x1(d.start); }) + .attr('width', function (d) { return instance.x1(d.end) - instance.x1(d.start); }); + rects.enter().append('rect') + .attr('x', function (d) { return instance.x1(d.start); }) + .attr('y', function (d) { return instance.y1(d.lane) + ChartTimeline.ENTRY_LANE_HEIGHT_OFFSET_FRACTION * instance.y1(1) + 0.5; }) + .attr('width', function (d) { return instance.x1(d.end) - instance.x1(d.start); }) + .attr('height', function (d) { return ChartTimeline.ENTRY_LANE_HEIGHT_TOTAL_FRACTION * instance.y1(1); }) + .attr('stroke', 'black') + .attr('fill', function (d) { + if (d.color) + return d.color; + return ChartTimeline.DEFAULT_COLOR; + }) + .attr('stroke-width', 1); + rects.exit().remove(); + var labels = this.itemRects.selectAll('text') + .data(visibleItems, function (d) { + return d.id; + }) + .attr('x', function (d) { + return instance.x1(Math.max(d.start, minExtent)) + 2; + }) + .attr('fill', 'black'); + labels.enter().append('text') + .text(function (d) { + if (instance.x1(d.end) - instance.x1(d.start) <= 30) + return ""; + if (d.label) + return d.label; + return ""; + }) + .attr('x', function (d) { + return instance.x1(Math.max(d.start, minExtent)) + 2; + }) + .attr('y', function (d) { + return instance.y1(d.lane) + .4 * instance.y1(1) + 0.5; + }) + .attr('text-anchor', 'start') + .attr('class', 'itemLabel') + .attr('fill', 'black'); + labels.exit().remove(); + }; + this.moveBrush = function () { + var origin = d3.mouse(this.rect[0]); + var time = this.x.invert(origin[0]).getTime(); + var halfExtent = (this.brush.extent()[1].getTime() - this.brush.extent()[0].getTime()) / 2; + this.brush.extent([new Date(time - halfExtent), new Date(time + halfExtent)]); + this.renderChart(); + }; + this.getMiniViewPaths = function (items) { + var paths = {}, d, offset = .5 * this.y2(1) + 0.5, result = []; + for (var i = 0; i < items.length; i++) { + d = items[i]; + if (!paths[d.class]) + paths[d.class] = ''; + paths[d.class] += ['M', this.x(d.start), (this.y2(d.lane) + offset), 'H', this.x(d.end)].join(' '); + } + for (var className in paths) { + result.push({ class: className, path: paths[className] }); + } + return result; + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.ChartTimeline]]; + this.laneNames = json['laneNames']; + this.laneData = json['laneData']; + } + ChartTimeline.MINI_LANE_HEIGHT_PX = 12; + ChartTimeline.ENTRY_LANE_HEIGHT_OFFSET_FRACTION = 0.05; + ChartTimeline.ENTRY_LANE_HEIGHT_TOTAL_FRACTION = 0.90; + ChartTimeline.MILLISEC_PER_MINUTE = 60 * 1000; + ChartTimeline.MILLISEC_PER_HOUR = 60 * ChartTimeline.MILLISEC_PER_MINUTE; + ChartTimeline.MILLISEC_PER_DAY = 24 * ChartTimeline.MILLISEC_PER_HOUR; + ChartTimeline.MILLISEC_PER_WEEK = 7 * ChartTimeline.MILLISEC_PER_DAY; + ChartTimeline.DEFAULT_COLOR = "LightGrey"; + return ChartTimeline; +}(Chart)); +var StyleChart = (function (_super) { + __extends(StyleChart, _super); + function StyleChart(jsonObj) { + var _this = this; + _super.call(this, jsonObj['StyleChart']); + this.getStrokeWidth = function () { return _this.strokeWidth; }; + this.getPointSize = function () { return _this.pointSize; }; + this.getSeriesColors = function () { return _this.seriesColors; }; + this.getSeriesColor = function (idx) { + if (!this.seriesColors || idx < 0 || idx > this.seriesColors.length) + return null; + return _this.seriesColors[idx]; + }; + this.getAxisStrokeWidth = function () { return _this.axisStrokeWidth; }; + this.getTitleStyle = function () { return _this.titleStyle; }; + var style = jsonObj['StyleChart']; + if (style) { + this.strokeWidth = style['strokeWidth']; + this.pointSize = style['pointSize']; + this.seriesColors = style['seriesColors']; + if (style['titleStyle']) + this.titleStyle = new StyleText(style['titleStyle']); + } + } + return StyleChart; +}(Style)); +var ComponentDiv = (function (_super) { + __extends(ComponentDiv, _super); + function ComponentDiv(jsonStr) { + _super.call(this, ComponentType.ComponentDiv); + this.render = function (appendToObject) { + var newDiv = $('
'); + newDiv.uniqueId(); + if (this.style) { + if (this.style.getWidth()) { + var unit = this.style.getWidthUnit(); + newDiv.width(this.style.getWidth() + (unit ? unit : "")); + } + if (this.style.getHeight()) { + var unit = this.style.getHeightUnit(); + newDiv.height(this.style.getHeight() + (unit ? unit : "")); + } + if (this.style.getBackgroundColor()) + newDiv.css("background-color", this.style.getBackgroundColor()); + if (this.style.getFloatValue()) + newDiv.css("float", this.style.getFloatValue()); + } + appendToObject.append(newDiv); + if (this.components) { + for (var i = 0; i < this.components.length; i++) { + this.components[i].render(newDiv); + } + } + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.ComponentDiv]]; + var components = json['components']; + if (components) { + this.components = []; + for (var i = 0; i < components.length; i++) { + var asStr = JSON.stringify(components[i]); + this.components.push(Component.getComponent(asStr)); + } + } + if (json['style']) + this.style = new StyleDiv(json['style']); + } + return ComponentDiv; +}(Component)); +var StyleDiv = (function (_super) { + __extends(StyleDiv, _super); + function StyleDiv(jsonObj) { + var _this = this; + _super.call(this, jsonObj['StyleDiv']); + this.getFloatValue = function () { return _this.floatValue; }; + if (jsonObj && jsonObj['StyleDiv']) + this.floatValue = jsonObj['StyleDiv']['floatValue']; + } + return StyleDiv; +}(Style)); +var DecoratorAccordion = (function (_super) { + __extends(DecoratorAccordion, _super); + function DecoratorAccordion(jsonStr) { + _super.call(this, ComponentType.DecoratorAccordion); + this.render = function (appendToObject) { + var s = this.style; + var outerDiv = $('
'); + outerDiv.uniqueId(); + var titleDiv; + if (this.title) + titleDiv = $('
' + this.title + '
'); + else + titleDiv = $('
'); + titleDiv.uniqueId(); + outerDiv.append(titleDiv); + var innerDiv = $('
'); + innerDiv.uniqueId(); + outerDiv.append(innerDiv); + if (this.innerComponents) { + for (var i = 0; i < this.innerComponents.length; i++) { + this.innerComponents[i].render(innerDiv); + } + } + appendToObject.append(outerDiv); + if (this.defaultCollapsed) + outerDiv.accordion({ collapsible: true, heightStyle: "content", active: false }); + else + outerDiv.accordion({ collapsible: true, heightStyle: "content" }); + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.DecoratorAccordion]]; + this.title = json['title']; + this.defaultCollapsed = json['defaultCollapsed']; + var innerCs = json['innerComponents']; + if (innerCs) { + this.innerComponents = []; + for (var i = 0; i < innerCs.length; i++) { + var asStr = JSON.stringify(innerCs[i]); + this.innerComponents.push(Component.getComponent(asStr)); + } + } + if (json['style']) + this.style = new StyleAccordion(json['style']); + } + return DecoratorAccordion; +}(Component)); +var StyleAccordion = (function (_super) { + __extends(StyleAccordion, _super); + function StyleAccordion(jsonObj) { + _super.call(this, jsonObj['StyleAccordion']); + } + return StyleAccordion; +}(Style)); +var ComponentTable = (function (_super) { + __extends(ComponentTable, _super); + function ComponentTable(jsonStr) { + _super.call(this, ComponentType.ComponentTable); + this.render = function (appendToObject) { + var s = this.style; + var margin = Style.getMargins(s); + var tbl = document.createElement('table'); + tbl.style.width = '100%'; + if (s && s.getBorderWidthPx() != null) + tbl.setAttribute('border', String(s.getBorderWidthPx())); + if (s && s.getBackgroundColor()) + tbl.style.backgroundColor = s.getBackgroundColor(); + if (s && s.getWhitespaceMode()) + tbl.style.whiteSpace = s.getWhitespaceMode(); + if (s && s.getColumnWidths()) { + var colWidths = s.getColumnWidths(); + var unit = TSUtils.normalizeLengthUnit(s.getColumnWidthUnit()); + for (var i = 0; i < colWidths.length; i++) { + var col = document.createElement('col'); + col.setAttribute('width', colWidths[i] + unit); + tbl.appendChild(col); + } + } + var padTop = 1; + var padRight = 1; + var padBottom = 1; + var padLeft = 1; + if (this.header) { + var theader = document.createElement('thead'); + var headerRow = document.createElement('tr'); + if (s && s.getHeaderColor()) + headerRow.style.backgroundColor = s.getHeaderColor(); + for (var i = 0; i < this.header.length; i++) { + var headerd = document.createElement('th'); + headerd.style.padding = padTop + 'px ' + padRight + 'px ' + padBottom + 'px ' + padLeft + 'px'; + headerd.appendChild(document.createTextNode(this.header[i])); + headerRow.appendChild(headerd); + } + tbl.appendChild(headerRow); + } + if (this.content) { + var tbdy = document.createElement('tbody'); + for (var i = 0; i < this.content.length; i++) { + var tr = document.createElement('tr'); + for (var j = 0; j < this.content[i].length; j++) { + var td = document.createElement('td'); + td.style.padding = padTop + 'px ' + padRight + 'px ' + padBottom + 'px ' + padLeft + 'px'; + td.appendChild(document.createTextNode(this.content[i][j])); + tr.appendChild(td); + } + tbdy.appendChild(tr); + } + tbl.appendChild(tbdy); + } + appendToObject.append(tbl); + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.ComponentTable]]; + this.header = json['header']; + this.content = json['content']; + if (json['style']) + this.style = new StyleTable(json['style']); + } + return ComponentTable; +}(Component)); +var StyleTable = (function (_super) { + __extends(StyleTable, _super); + function StyleTable(jsonObj) { + var _this = this; + _super.call(this, jsonObj['StyleTable']); + this.getColumnWidths = function () { return _this.columnWidths; }; + this.getColumnWidthUnit = function () { return _this.columnWidthUnit; }; + this.getBorderWidthPx = function () { return _this.borderWidthPx; }; + this.getHeaderColor = function () { return _this.headerColor; }; + this.getWhitespaceMode = function () { return _this.whitespaceMode; }; + var style = jsonObj['StyleTable']; + if (style) { + this.columnWidths = jsonObj['StyleTable']['columnWidths']; + this.borderWidthPx = jsonObj['StyleTable']['borderWidthPx']; + this.headerColor = jsonObj['StyleTable']['headerColor']; + this.columnWidthUnit = jsonObj['StyleTable']['columnWidthUnit']; + this.whitespaceMode = jsonObj['StyleTable']['whitespaceMode']; + } + } + return StyleTable; +}(Style)); +var ComponentText = (function (_super) { + __extends(ComponentText, _super); + function ComponentText(jsonStr) { + var _this = this; + _super.call(this, ComponentType.ComponentText); + this.render = function (appendToObject) { + var textNode = document.createTextNode(_this.text); + if (_this.style) { + var newSpan = document.createElement('span'); + if (_this.style.getFont()) + newSpan.style.font = _this.style.getFont(); + if (_this.style.getFontSize() != null) + newSpan.style.fontSize = _this.style.getFontSize() + "pt"; + if (_this.style.getUnderline() != null) + newSpan.style.textDecoration = 'underline'; + if (_this.style.getColor()) + newSpan.style.color = _this.style.getColor(); + if (_this.style.getMarginTop()) + newSpan.style.marginTop = _this.style.getMarginTop() + "px"; + if (_this.style.getMarginBottom()) + newSpan.style.marginBottom = _this.style.getMarginBottom() + "px"; + if (_this.style.getMarginLeft()) + newSpan.style.marginLeft = _this.style.getMarginLeft() + "px"; + if (_this.style.getMarginRight()) + newSpan.style.marginRight = _this.style.getMarginRight() + "px"; + if (_this.style.getWhitespacePre()) + newSpan.style.whiteSpace = 'pre'; + newSpan.appendChild(textNode); + appendToObject.append(newSpan); + } + else { + var newSpan = document.createElement('span'); + newSpan.appendChild(textNode); + appendToObject.append(newSpan); + } + }; + var json = JSON.parse(jsonStr); + if (!json["componentType"]) + json = json[ComponentType[ComponentType.ComponentText]]; + this.text = json['text']; + if (json['style']) + this.style = new StyleText(json['style']); + } + return ComponentText; +}(Component)); +var StyleText = (function (_super) { + __extends(StyleText, _super); + function StyleText(jsonObj) { + var _this = this; + _super.call(this, jsonObj['StyleText']); + this.getFont = function () { return _this.font; }; + this.getFontSize = function () { return _this.fontSize; }; + this.getUnderline = function () { return _this.underline; }; + this.getColor = function () { return _this.color; }; + this.getWhitespacePre = function () { return _this.whitespacePre; }; + var style = jsonObj['StyleText']; + if (style) { + this.font = style['font']; + this.fontSize = style['fontSize']; + this.underline = style['underline']; + this.color = style['color']; + this.whitespacePre = style['whitespacePre']; + } + } + return StyleText; +}(Style)); +//# sourceMappingURL=dl4j-ui.js.map \ No newline at end of file diff --git a/arbiter/arbiter-ui/src/main/resources/deeplearning4jUiAssets/dl4j-ui.js.map b/arbiter/arbiter-ui/src/main/resources/deeplearning4jUiAssets/dl4j-ui.js.map new file mode 100644 index 000000000..3545aed31 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/resources/deeplearning4jUiAssets/dl4j-ui.js.map @@ -0,0 +1 @@ +{"version":3,"file":"dl4j-ui.js","sourceRoot":"","sources":["../../typescript/org/deeplearning4j/ui/api/Style.ts","../../typescript/org/deeplearning4j/ui/api/ComponentType.ts","../../typescript/org/deeplearning4j/ui/api/Component.ts","../../typescript/org/deeplearning4j/ui/api/Constants.ts","../../typescript/org/deeplearning4j/ui/api/Margin.ts","../../typescript/org/deeplearning4j/ui/api/Renderable.ts","../../typescript/org/deeplearning4j/ui/util/TSUtils.ts","../../typescript/org/deeplearning4j/ui/components/chart/Chart.ts","../../typescript/org/deeplearning4j/ui/components/chart/ChartHistogram.ts","../../typescript/org/deeplearning4j/ui/components/chart/ChartLine.ts","../../typescript/org/deeplearning4j/ui/components/chart/ChartScatter.ts","../../typescript/org/deeplearning4j/ui/components/chart/Legend.ts","../../typescript/org/deeplearning4j/ui/components/chart/ChartStackedArea.ts","../../typescript/org/deeplearning4j/ui/components/chart/ChartTimeline.ts","../../typescript/org/deeplearning4j/ui/components/chart/style/StyleChart.ts","../../typescript/org/deeplearning4j/ui/components/component/ComponentDiv.ts","../../typescript/org/deeplearning4j/ui/components/component/style/StyleDiv.ts","../../typescript/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.ts","../../typescript/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.ts","../../typescript/org/deeplearning4j/ui/components/table/ComponentTable.ts","../../typescript/org/deeplearning4j/ui/components/table/style/StyleTable.ts","../../typescript/org/deeplearning4j/ui/components/text/ComponentText.ts","../../typescript/org/deeplearning4j/ui/components/text/style/StyleText.ts"],"names":[],"mappings":";;;;;AAkBA;IAcI,eAAa,OAAY;QAd7B,iBAmDC;QAzBG,aAAQ,GAAG,cAAM,OAAA,KAAI,CAAC,KAAK,EAAV,CAAU,CAAC;QAC5B,cAAS,GAAG,cAAM,OAAA,KAAI,CAAC,MAAM,EAAX,CAAW,CAAC;QAC9B,iBAAY,GAAG,cAAM,OAAA,KAAI,CAAC,SAAS,EAAd,CAAc,CAAC;QACpC,kBAAa,GAAG,cAAM,OAAA,KAAI,CAAC,UAAU,EAAf,CAAe,CAAC;QACtC,iBAAY,GAAG,cAAM,OAAA,KAAI,CAAC,SAAS,EAAd,CAAc,CAAC;QACpC,oBAAe,GAAG,cAAM,OAAA,KAAI,CAAC,YAAY,EAAjB,CAAiB,CAAC;QAC1C,kBAAa,GAAG,cAAM,OAAA,KAAI,CAAC,UAAU,EAAf,CAAe,CAAC;QACtC,mBAAc,GAAG,cAAM,OAAA,KAAI,CAAC,WAAW,EAAhB,CAAgB,CAAC;QACxC,uBAAkB,GAAG,cAAM,OAAA,KAAI,CAAC,eAAe,EAApB,CAAoB,CAAC;QAnB5C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,mBAAmB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,mBAAmB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QACrE,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;QAC5C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QAC1C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACtD,CAAC;IAaM,gBAAU,GAAjB,UAAkB,CAAQ;QACtB,IAAI,IAAI,GAAW,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAW,CAAC,CAAC,GAAG,CAAC,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC;QACpD,IAAI,KAAK,GAAW,CAAC,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,CAAC;QAChD,IAAI,MAAM,GAAW,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,CAAC;QAGlD,MAAM,CAAC,EAAC,GAAG,EAAE,IAAI;YACb,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,OAAO;YACf,IAAI,EAAE,KAAK;YACX,cAAc,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,KAAK,GAAG,MAAM;YAC7C,eAAe,EAAE,CAAC,CAAC,SAAS,EAAE,GAAG,IAAI,GAAG,OAAO,EAAC,CAAC;IACzD,CAAC;IACL,YAAC;AAAD,CAAC,AAnDD,IAmDC;ACjDD,IAAK,aAWJ;AAXD,WAAK,aAAa;IACd,mEAAa,CAAA;IACb,qEAAc,CAAA;IACd,iEAAY,CAAA;IACZ,qEAAc,CAAA;IACd,6EAAkB,CAAA;IAClB,2DAAS,CAAA;IACT,iEAAY,CAAA;IACZ,yEAAgB,CAAA;IAChB,mEAAa,CAAA;IACb,6EAAkB,CAAA;AACtB,CAAC,EAXI,aAAa,KAAb,aAAa,QAWjB;ACTD;IAII,mBAAY,aAA4B;QACpC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAEM,oCAAgB,GAAvB;QACI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAKa,sBAAY,GAA1B,UAA2B,OAAe;QAEtC,IAAI,IAAI,GAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,GAAW,CAAC;QAChB,EAAE,CAAA,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;QACtD,IAAI;YAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAIhC,MAAM,CAAA,CAAC,GAAG,CAAC,CAAA,CAAC;YACR,KAAK,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC;gBAC3C,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC;YAEtC,KAAK,aAAa,CAAC,aAAa,CAAC,cAAc,CAAC;gBAC5C,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;YAEvC,KAAK,aAAa,CAAC,aAAa,CAAC,cAAc,CAAC;gBAC5C,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;YAEvC,KAAK,aAAa,CAAC,aAAa,CAAC,kBAAkB,CAAC;gBAChD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAEjE,KAAK,aAAa,CAAC,aAAa,CAAC,SAAS,CAAC;gBACvC,MAAM,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC;YAElC,KAAK,aAAa,CAAC,aAAa,CAAC,YAAY,CAAC;gBAC1C,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;YAErC,KAAK,aAAa,CAAC,aAAa,CAAC,gBAAgB,CAAC;gBAC9C,MAAM,CAAC,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAEzC,KAAK,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC;gBAC3C,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC;YAEtC,KAAK,aAAa,CAAC,aAAa,CAAC,kBAAkB,CAAC;gBAChD,MAAM,CAAC,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAE3C,KAAK,aAAa,CAAC,aAAa,CAAC,YAAY,CAAC;gBAC1C,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;YAErC;gBACI,MAAM,IAAI,KAAK,CAAC,2BAA2B,GAAG,GAAG,GAAG,wBAAwB,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;QACvG,CAAC;IACL,CAAC;IACL,gBAAC;AAAD,CAAC,AA3DD,IA2DC;AChED;IAAA;IAMA,CAAC;IAJU,yCAA0B,GAAG,GAAG,CAAC;IACjC,uCAAwB,GAAG,GAAG,CAAC;IAC/B,wCAAyB,GAAG,GAAG,CAAC;IAChC,kCAAmB,GAAG,SAAS,CAAC;IAC3C,qBAAC;AAAD,CAAC,AAND,IAMC;AGJD;IAAA;IA6CA,CAAC;IA1CU,WAAG,GAAV,UAAW,KAAiB;QACxB,IAAI,GAAG,GAAW,CAAC,MAAM,CAAC,SAAS,CAAC;QACpC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,GAAG,CAAA,CAAE,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;QACL,CAAC;QACD,MAAM,CAAC,GAAG,CAAC;IACf,CAAC;IAGM,WAAG,GAAV,UAAW,KAAiB;QACxB,IAAI,GAAG,GAAW,MAAM,CAAC,SAAS,CAAC;QACnC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,GAAG,CAAA,CAAE,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;QACL,CAAC;QACD,MAAM,CAAC,GAAG,CAAC;IACf,CAAC;IAGM,2BAAmB,GAA1B,UAA2B,KAAa;QACpC,EAAE,CAAA,CAAC,KAAK,IAAI,IAAI,CAAC;YAAC,MAAM,CAAC,KAAK,CAAC;QAE/B,MAAM,CAAA,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA,CAAC;YACxB,KAAK,IAAI;gBACL,MAAM,CAAC,IAAI,CAAC;YAChB,KAAK,SAAS,CAAC;YACf,KAAK,GAAG;gBACJ,MAAM,CAAC,GAAG,CAAC;YACf,KAAK,IAAI;gBACL,MAAM,CAAC,IAAI,CAAC;YAChB,KAAK,IAAI;gBACL,MAAM,CAAC,IAAI,CAAC;YAChB,KAAK,IAAI;gBACL,MAAM,CAAC,IAAI,CAAC;YAChB;gBACI,MAAM,CAAC,KAAK,CAAC;QACrB,CAAC;IAEL,CAAC;IACL,cAAC;AAAD,CAAC,AA7CD,IA6CC;ACxCD;IAA6B,yBAAS;IAiBlC,eAAY,aAA4B,EAAE,OAAe;QACrD,kBAAM,aAAa,CAAC,CAAC;QAErB,IAAI,QAAQ,GAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;QAErE,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC7D,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACzD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;QAErC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAE/B,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAC/D,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAEnE,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAC,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,wBAAQ,GAAR;QACI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAEgB,iBAAW,GAA5B,UAA6B,GAAQ,EAAE,KAAa,EAAE,MAAc,EAAE,UAAqB;QACvF,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;aACxB,IAAI,CAAC,KAAK,CAAC;aACX,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;aACtC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;aACtC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAEnC,EAAE,CAAA,CAAC,UAAU,CAAC,CAAA,CAAC;YACX,EAAE,CAAA,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACrE,EAAE,CAAA,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAC,UAAU,CAAC,WAAW,EAAE,GAAC,IAAI,CAAC,CAAC;YAC1F,EAAE,CAAA,CAAC,UAAU,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC;gBAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;YACjF,EAAE,CAAA,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;gBAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACjE,IAAI;gBAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;QAC/D,CAAC;QAAC,IAAI,CAAC,CAAC;YACJ,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,MAAM,EAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;QAC1D,CAAC;IACL,CAAC;IACL,YAAC;AAAD,CAAC,AA9DD,CAA6B,SAAS,GA8DrC;AChED;IAA6B,kCAAK;IAM9B,wBAAY,OAAe;QACvB,kBAAM,aAAa,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAYjD,WAAM,GAAG,UAAC,cAAsB;YAC5B,IAAI,CAAC,GAAe,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,MAAM,GAAW,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAGzC,IAAI,IAAY,CAAC;YACjB,IAAI,IAAY,CAAC;YACjB,IAAI,IAAY,CAAC;YACjB,IAAI,IAAY,CAAC;YACjB,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YACrC,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9D,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YACrC,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9D,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YACrC,IAAI;gBAAC,IAAI,GAAG,CAAC,CAAC;YACd,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YACrC,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAGtD,IAAI,MAAM,GAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE;iBAC9B,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;iBACpB,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;YAEvC,IAAI,KAAK,GAAQ,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;iBACvC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAE/B,EAAE,CAAA,CAAC,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAA,CAAC;gBACjE,KAAK,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;YACjD,CAAC;YAED,IAAI,MAAM,GAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE;iBAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;iBACjB,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,KAAK,GAAQ,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;iBACvC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7B,EAAE,CAAA,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAA,CAAC;gBACrE,KAAK,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAChD,CAAC;YAID,EAAE,CAAA,CAAC,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC;gBAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAE9D,EAAE,CAAA,CAAC,IAAI,CAAC,oBAAoB,KAAK,IAAI,CAAC;gBAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAG5D,IAAI,WAAW,GAAa,IAAI,CAAC,WAAW,CAAC;YAC7C,IAAI,WAAW,GAAa,IAAI,CAAC,WAAW,CAAC;YAC7C,IAAI,OAAO,GAAa,IAAI,CAAC,OAAO,CAAC;YAErC,IAAI,IAAI,GAAQ,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1C,MAAM,CAAC,EAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,EAAC,CAAC;YACtG,CAAC,CAAC,CAAC;YAGH,IAAI,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC/C,MAAM,CAAC,KAAK,CAAC;iBACb,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC;iBACrB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC3B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;iBAC7B,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC;iBACvB,MAAM,CAAC,GAAG,CAAC;iBACX,IAAI,CAAC,WAAW,EACb,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YAI7D,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC;iBAChB,IAAI,CAAC,IAAI,CAAC;iBACV,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACtB,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;iBACpB,KAAK,CAAC,MAAM,EAAC,WAAW,CAAC;iBACzB,IAAI,CAAC,GAAG,EAAE,UAAS,CAAM,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxD,IAAI,CAAC,OAAO,EAAE,UAAS,CAAM,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;iBACtE,IAAI,CAAC,GAAG,EAAE,UAAS,CAAM,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxD,IAAI,CAAC,QAAQ,EAAE,UAAS,CAAM,IAAI,MAAM,CAAC,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAG5F,IAAI,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;iBACvB,IAAI,CAAC,WAAW,EAAE,cAAc,GAAG,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC;iBAChE,KAAK,CAAC,QAAQ,EAAC,MAAM,CAAC;iBACtB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAE,GAAG,cAAc,CAAC,yBAAyB,CAAC,CAAC;iBACxI,KAAK,CAAC,MAAM,EAAC,MAAM,CAAC;iBACpB,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,SAAS,CAAC,CAAC;YAE5E,EAAE,CAAA,CAAC,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC;gBAAC,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,EAAC,cAAc,EAAE,IAAI,CAAC,uBAAuB,EAAC,CAAC,CAAC;YAGjI,IAAI,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;iBACvB,KAAK,CAAC,QAAQ,EAAC,MAAM,CAAC;iBACtB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAE,GAAG,cAAc,CAAC,yBAAyB,CAAC,CAAC;iBACxI,KAAK,CAAC,MAAM,EAAC,MAAM,CAAC;iBACpB,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,SAAS,CAAC,CAAC;YAE5E,EAAE,CAAA,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC;gBAAC,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,EAAC,cAAc,EAAE,IAAI,CAAC,yBAAyB,EAAC,CAAC,CAAC;YAGrI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACb,IAAI,UAAqB,CAAC;gBAC1B,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC;oBAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBACvD,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC,CAAA;QApHG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,CAAC;QAGpF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IA8GL,qBAAC;AAAD,CAAC,AA9HD,CAA6B,KAAK,GA8HjC;AC9HD;IAAwB,6BAAK;IAMzB,mBAAY,OAAe;QACvB,kBAAM,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAU5C,WAAM,GAAG,UAAC,cAAsB;YAE5B,IAAI,OAAO,GAAW,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5D,IAAI,CAAC,GAAe,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,MAAM,GAAW,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAGzC,IAAI,MAAM,GAAmC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;YACjG,IAAI,MAAM,GAAmC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC;YAGlG,IAAI,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;iBAClC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/B,EAAE,CAAA,CAAC,IAAI,CAAC,uBAAuB,IAAI,IAAI,IAAI,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAA,CAAC;gBACzE,KAAK,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;YACjD,CAAC;YAGD,IAAI,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;iBAClC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7B,EAAE,CAAA,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,IAAI,IAAI,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAA,CAAC;gBAC7E,KAAK,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAChD,CAAC;YAED,EAAE,CAAA,CAAC,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC;gBAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAE9D,EAAE,CAAA,CAAC,IAAI,CAAC,oBAAoB,KAAK,IAAI,CAAC;gBAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAG5D,IAAI,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE;iBACxB,CAAC,CAAC,UAAU,CAAM;gBACf,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC;iBACD,CAAC,CAAC,UAAU,CAAM;gBACf,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAIP,IAAI,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC/C,MAAM,CAAC,KAAK,CAAC;iBACb,KAAK,CAAC,cAAc,EAAE,CAAE,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,cAAc,EAAE,GAAG,cAAc,CAAC,0BAA0B,CAAC,CAAC;iBAClH,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC;iBACrB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC3B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;iBAC7B,MAAM,CAAC,GAAG,CAAC;iBACX,IAAI,CAAC,WAAW,EAAE,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YAG5E,IAAI,IAAY,CAAC;YACjB,IAAI,IAAY,CAAC;YACjB,IAAI,IAAY,CAAC;YACjB,IAAI,IAAY,CAAC;YACjB,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7C,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACvD,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7C,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACvD,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7C,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACvD,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YAC7C,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAEvD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAC5B,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAG5B,IAAI,YAAY,GAA2B,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YACjE,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,IAAI,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpC,IAAI,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAEpC,IAAI,IAAI,GAAU,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBACtC,MAAM,CAAC,EAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAC,CAAC;gBAChD,CAAC,CAAC,CAAC;gBAEH,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;qBACb,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;qBACrB,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;qBAC3F,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;YACpC,CAAC;YAGD,IAAI,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;iBACvB,IAAI,CAAC,WAAW,EAAE,cAAc,GAAG,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC;iBAChE,KAAK,CAAC,QAAQ,EAAC,MAAM,CAAC;iBACtB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAE,GAAG,cAAc,CAAC,yBAAyB,CAAC,CAAC;iBACxI,KAAK,CAAC,MAAM,EAAC,MAAM,CAAC;iBACpB,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,SAAS,CAAC,CAAC;YAE5E,EAAE,CAAA,CAAC,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC;gBAAC,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,EAAC,cAAc,EAAE,IAAI,CAAC,uBAAuB,EAAC,CAAC,CAAC;YAGjI,IAAI,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;iBACvB,KAAK,CAAC,QAAQ,EAAC,MAAM,CAAC;iBACtB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAE,GAAG,cAAc,CAAC,yBAAyB,CAAC,CAAC;iBACxI,KAAK,CAAC,MAAM,EAAC,MAAM,CAAC;iBACpB,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,SAAS,CAAC,CAAC;YAE5E,EAAE,CAAA,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC;gBAAC,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,EAAC,cAAc,EAAE,IAAI,CAAC,yBAAyB,EAAC,CAAC,CAAC;YAGrI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC;gBAC/C,IAAI,WAAW,GAAG,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;gBAC5C,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC3B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC5B,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACtC,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACxC,IAAI,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;oBACpC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;yBACb,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;yBAC9C,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;yBAC3D,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;yBACvB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;yBACzF,IAAI,CAAC,SAAS,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC;YAGD,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACb,IAAI,UAAqB,CAAC;gBAC1B,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC;oBAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBACvD,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC,CAAA;QAxIG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC;QAE/E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC;IAmIL,gBAAC;AAAD,CAAC,AAlJD,CAAwB,KAAK,GAkJ5B;AClJD;IAA2B,gCAAK;IAM5B,sBAAY,OAAc;QACtB,kBAAM,aAAa,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAW/C,WAAM,GAAG,UAAC,cAAqB;YAE3B,IAAI,OAAO,GAAU,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3D,IAAI,CAAC,GAAc,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,MAAM,GAAU,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAGxC,IAAI,MAAM,GAAkC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;YAChG,IAAI,MAAM,GAAkC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC;YAGjG,IAAI,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;iBAClC,aAAa,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC;iBACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;iBAClC,aAAa,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC;iBACrC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAE7B,EAAE,CAAC,CAAC,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC;gBAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAE/D,EAAE,CAAC,CAAC,IAAI,CAAC,oBAAoB,KAAK,IAAI,CAAC;gBAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAI7D,IAAI,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC/C,MAAM,CAAC,KAAK,CAAC;iBACb,KAAK,CAAC,cAAc,EAAE,CAAE,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,CAAC;iBAC1E,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC;iBACrB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC3B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;iBAC7B,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC;iBACvB,MAAM,CAAC,GAAG,CAAC;iBACX,IAAI,CAAC,WAAW,EACb,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YAG7D,IAAI,IAAW,CAAC;YAChB,IAAI,IAAW,CAAC;YAChB,IAAI,IAAW,CAAC;YAChB,IAAI,IAAW,CAAC;YAChB,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YACtC,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACvD,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YACtC,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACvD,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YACtC,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACvD,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;YACtC,IAAI;gBAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAEvD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAC5B,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAG5B,IAAI,YAAY,GAA0B,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YAChE,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAE1B,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC/B,MAAM,CAAC,EAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAC,CAAC;gBAChD,CAAC,CAAC,CAAC;gBAEH,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC;qBAClB,IAAI,CAAC,IAAI,CAAC;qBACV,KAAK,EAAE;qBACP,MAAM,CAAC,QAAQ,CAAC;qBAChB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;qBACzF,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,YAAY,EAAE,GAAG,cAAc,CAAC,wBAAwB,CAAC,CAAC;qBAC/F,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC;oBACnB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC7B,CAAC,CAAC;qBACD,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC;oBACnB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC7B,CAAC,CAAC,CAAC;YACX,CAAC;YAGD,IAAI,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;iBACvB,IAAI,CAAC,WAAW,EAAE,cAAc,GAAG,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC;iBAChE,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC;iBACvB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAE,GAAG,cAAc,CAAC,yBAAyB,CAAC,CAAC;iBACxI,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC;iBACrB,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,SAAS,CAAC,CAAC;YAE5E,EAAE,CAAC,CAAC,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC;gBAAC,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,EAAC,cAAc,EAAE,IAAI,CAAC,uBAAuB,EAAC,CAAC,CAAC;YAGlI,IAAI,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;iBACvB,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC;iBACvB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAE,GAAG,cAAc,CAAC,yBAAyB,CAAC,CAAC;iBACxI,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC;iBACrB,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,SAAS,CAAC,CAAC;YAE5E,EAAE,CAAC,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC;gBAAC,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,EAAC,cAAc,EAAE,IAAI,CAAC,yBAAyB,EAAC,CAAC,CAAC;YAGtI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC;gBAC/C,IAAI,WAAW,GAAG,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;gBAC5C,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC3B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC5B,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACtC,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACxC,IAAI,SAAS,CAAC;oBACd,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC;wBAAC,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC;oBACrE,IAAI;wBAAC,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;oBACtG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;yBACb,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;yBAC9C,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;yBAC3D,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;yBACvB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;yBACzF,IAAI,CAAC,SAAS,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC;YAGD,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACb,IAAI,UAAqB,CAAC;gBAC1B,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC;oBAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBACvD,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC,CAAA;QAtIG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC;IAiIL,mBAAC;AAAD,CAAC,AAhJD,CAA2B,KAAK,GAgJ/B;ACpJD;IAAA;IAiEA,CAAC;IA9DkB,cAAO,GAAW,EAAE,CAAC;IACrB,cAAO,GAAW,EAAE,CAAC;IACrB,cAAO,GAAW,CAAC,CAAC;IACpB,iBAAU,GAAW,EAAE,CAAC;IACxB,cAAO,GAAW,EAAE,CAAC;IACrB,gBAAS,GAAW,SAAS,CAAC;IAC9B,oBAAa,GAAW,IAAI,CAAC;IAC7B,wBAAiB,GAAW,SAAS,CAAC;IAG9C,eAAQ,GAAG,CAAC,UAAS,CAAM;QAE9B,IAAI,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC,CAAC;QAC1D,IAAI,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,IAAI,WAAW,GAAG,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAE7D,SAAS,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAC,WAAW,CAAC,CAAC;QAC3D,WAAW,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,EAAC,eAAe,CAAC,CAAC;QAE9D,IAAI,cAAc,GAAU,EAAE,CAAC;QAC/B,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC;YAChC,IAAI,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9B,cAAc,CAAC,IAAI,CAAC;gBAChB,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC;gBAClC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;aAC/B,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAIH,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC;aACxB,IAAI,CAAC,cAAc,EAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA,CAAA,CAAC,CAAC;aAClD,IAAI,CAAC,UAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA,CAAA,CAAC,CAAC;aAC7C,IAAI,CAAC,UAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAA,CAAA,CAAC,CAAC;aACtC,IAAI,CAAC,GAAG,EAAC,CAAC,CAAC;aACX,IAAI,CAAC,GAAG,EAAC,UAAS,CAAC,EAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAC,MAAM,CAAC,UAAU,GAAC,MAAM,CAAC,OAAO,GAAC,IAAI,CAAA,CAAA,CAAC,CAAC;aACzE,IAAI,CAAC,OAAO,EAAC,MAAM,CAAC,OAAO,CAAC;aAC5B,IAAI,CAAC,QAAQ,EAAC,MAAM,CAAC,OAAO,CAAC;aAE7B,KAAK,CAAC,MAAM,EAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA,CAAA,CAAC,CAAC,CAAC;QAGjD,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC;aACxB,IAAI,CAAC,cAAc,EAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA,CAAA,CAAC,CAAC;aAClD,IAAI,CAAC,UAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA,CAAA,CAAC,CAAC;aAC7C,IAAI,CAAC,UAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAA,CAAA,CAAC,CAAC;aACtC,IAAI,CAAC,GAAG,EAAC,UAAS,CAAC,EAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAA,CAAA,CAAC,CAAC;aAC5D,IAAI,CAAC,GAAG,EAAC,CAAC,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;aAClD,IAAI,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA,CAAA,CAAC,CAAC,CAAC;QAGzC,IAAI,iBAAiB,GAAQ,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QACzD,SAAS,CAAC,IAAI,CAAC,GAAG,EAAC,CAAC,iBAAiB,CAAC,CAAC,GAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aACnD,IAAI,CAAC,GAAG,EAAC,CAAC,iBAAiB,CAAC,CAAC,GAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aAC9C,IAAI,CAAC,QAAQ,EAAC,CAAC,iBAAiB,CAAC,MAAM,GAAC,CAAC,GAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aAC1D,IAAI,CAAC,OAAO,EAAC,CAAC,iBAAiB,CAAC,KAAK,GAAC,CAAC,GAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aACxD,KAAK,CAAC,MAAM,EAAC,MAAM,CAAC,SAAS,CAAC;aAC9B,KAAK,CAAC,QAAQ,EAAC,MAAM,CAAC,iBAAiB,CAAC;aACxC,KAAK,CAAC,SAAS,EAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAE3C,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,WAAW,EAAC,YAAY,GAAG,MAAM,CAAC,OAAO,GAAG,GAAG,GAAG,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC;IAC1G,CAAC,CAAC,CAAC;IACP,aAAC;AAAD,CAAC,AAjED,IAiEC;AC1DD;IAA+B,oCAAK;IAKhC,0BAAY,OAAe;QACvB,kBAAM,aAAa,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;QAYnD,WAAM,GAAG,UAAC,cAAsB;YAE5B,IAAI,OAAO,GAAW,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5D,IAAI,CAAC,GAAe,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,MAAM,GAAW,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAGzC,IAAI,MAAM,GAAmC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;YACjG,IAAI,MAAM,GAAmC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC;YAGlG,IAAI,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;iBAClC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/B,EAAE,CAAA,CAAC,IAAI,CAAC,uBAAuB,IAAI,IAAI,IAAI,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAA,CAAC;gBACzE,KAAK,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;YACjD,CAAC;YAGD,IAAI,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;iBAClC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7B,EAAE,CAAA,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,IAAI,IAAI,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAA,CAAC;gBAC7E,KAAK,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAChD,CAAC;YAED,EAAE,CAAA,CAAC,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC;gBAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAE9D,EAAE,CAAA,CAAC,IAAI,CAAC,oBAAoB,KAAK,IAAI,CAAC;gBAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAE5D,IAAI,IAAI,GAAU,EAAE,CAAC;YACrB,GAAG,CAAA,CAAC,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,EAAE,CAAC;gBACb,GAAG,CAAA,CAAE,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACtC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACvC,GAAG,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAClC,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YAED,IAAI,IAAI,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE;iBACnB,CAAC,CAAC,UAAS,CAAM,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;iBAChD,EAAE,CAAC,UAAS,CAAM,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC7C,EAAE,CAAC,UAAS,CAAM,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEzD,IAAI,KAAK,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE;iBACxB,MAAM,CAAC,UAAS,CAAM,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAEnD,IAAI,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC7D,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,cAAc,GAAG,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC;iBACjE,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;iBACnE,MAAM,CAAC,GAAG,CAAC;iBACX,IAAI,CAAC,WAAW,EAAE,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YAE5E,IAAI,KAAK,GAAQ,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YACvC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG;gBAC9C,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC;YAC5B,CAAC,CAAC,CAAC,CAAC;YAEJ,IAAI,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI;gBAClD,MAAM,CAAC;oBACH,IAAI,EAAE,IAAI;oBACV,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;wBACxB,MAAM,CAAC,EAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAC,CAAC;oBAC9C,CAAC,CAAC;iBACL,CAAC;YACN,CAAC,CAAC,CAAC,CAAC;YAGJ,IAAI,IAAI,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC;gBAC/B,IAAI,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,GAAG;oBACnC,MAAM,CAAC,GAAG,KAAK,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACxC,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YAGH,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC;gBACrC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACpB,CAAC,CAAC,CAAC,CAAC;YAEJ,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;YAEzB,IAAI,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC;iBAClC,IAAI,CAAC,QAAQ,CAAC;iBACd,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC;iBACnB,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAE9B,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;YAE7B,IAAI,YAAY,GAA2B,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YACjE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;iBACjB,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;iBACrB,IAAI,CAAC,aAAa,EAAC,UAAS,CAAM,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA,CAAA,CAAC,CAAC;iBACrD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAM;gBACvB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC,CAAC;iBACD,KAAK,CAAC,MAAM,EAAE,UAAS,CAAM;gBAC1B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC;oBAClD,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACxD,CAAC;gBAAC,IAAI,CAAA,CAAC;oBACH,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBAC3D,CAAC;YACL,CAAC,CAAC;iBACD,KAAK,CAAC,EAAC,cAAc,EAAE,KAAK,EAAC,CAAC,CAAC;YAGpC,IAAI,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;iBACvB,KAAK,CAAC,QAAQ,EAAC,MAAM,CAAC;iBACtB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAE,GAAG,cAAc,CAAC,yBAAyB,CAAC,CAAC;iBACxI,KAAK,CAAC,MAAM,EAAC,MAAM,CAAC;iBACpB,IAAI,CAAC,WAAW,EAAE,cAAc,GAAG,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC;iBAChE,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,SAAS,CAAC,CAAC;YAG5E,IAAI,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;iBACvB,KAAK,CAAC,QAAQ,EAAC,MAAM,CAAC;iBACtB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAE,GAAG,cAAc,CAAC,yBAAyB,CAAC,CAAC;iBACxI,KAAK,CAAC,MAAM,EAAC,MAAM,CAAC;iBACpB,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAC,SAAS,CAAC,CAAC;YAG5E,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACb,IAAI,UAAqB,CAAC;gBAC1B,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC;oBAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBACvD,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC3D,CAAC;YAGD,IAAI,MAAM,GAAQ,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC5B,IAAI,CAAC,OAAO,EAAC,QAAQ,CAAC;iBACtB,IAAI,CAAC,WAAW,EAAC,kBAAkB,CAAC;iBACpC,KAAK,CAAC,WAAW,EAAC,MAAM,CAAC;iBACzB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC,CAAA;QAlJG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAGtF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IA4IL,uBAAC;AAAD,CAAC,AA3JD,CAA+B,KAAK,GA2JnC;AC9JD;IAA4B,iCAAK;IAgC7B,uBAAY,OAAc;QACtB,kBAAM,aAAa,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QAUhD,WAAM,GAAG,UAAC,cAAqB;YAC3B,IAAI,QAAQ,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,GAAc,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,MAAM,GAAU,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAGxC,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;YACnB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/C,IAAI,GAAG,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;oBAClD,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;oBAC9C,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;oBACpB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAChB,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;oBAC5C,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;oBACjD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5B,CAAC;YACL,CAAC;YAED,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAChB,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,IAAI,GAAG,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACd,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;YAID,IAAI,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC/C,MAAM,CAAC,KAAK,CAAC;iBACb,KAAK,CAAC,cAAc,EAAE,CAAE,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,cAAc,EAAE,GAAG,cAAc,CAAC,0BAA0B,CAAC,CAAC;iBAClH,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC;iBACrB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC3B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;iBAC7B,MAAM,CAAC,GAAG,CAAC,CAAC;YAEjB,IAAI,eAAe,GAAG,CAAC,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;YACjE,IAAI,cAAc,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC;YAC/D,IAAI,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,aAAa,CAAC,mBAAmB,CAAC;YAC3E,IAAI,UAAU,GAAG,CAAC,CAAC,SAAS,EAAE,GAAG,UAAU,GAAG,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;YAE9E,IAAI,OAAO,GAAU,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAK,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACjF,IAAI,OAAO,GAAU,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAK,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE;iBACnB,MAAM,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;iBAC1B,KAAK,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;YAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;YAErD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;YACtF,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;YAGtF,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;iBAC5C,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;iBAClB,MAAM,CAAC,MAAM,CAAC;iBACd,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC;iBAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,GAAG,GAAG,CAAC,CAAC;YAEzC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,WAAW,EAAE,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC;iBACtE,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC;iBAC7B,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC;iBAC1B,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC;iBACzB,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YAEhC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1B,IAAI,CAAC,WAAW,EAAE,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC;iBAC1F,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC;iBAC7B,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC;iBAC1B,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC;iBACzB,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YAGhC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC;iBAC5C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;iBAChB,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACtB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;iBACb,IAAI,CAAC,IAAI,EAAE,UAAU,CAAK;gBACvB,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;YAC7C,CAAC,CAAC;iBACD,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC;iBAC1B,IAAI,CAAC,IAAI,EAAE,UAAU,CAAK;gBACvB,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;YAC7C,CAAC,CAAC;iBACD,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC;iBAC3B,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;YAG7B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC;iBAC3C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;iBAChB,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACtB,IAAI,CAAC,UAAU,CAAK;gBACjB,EAAE,CAAA,CAAC,CAAC,CAAC,KAAK,CAAC;oBAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3B,MAAM,CAAC,EAAE,CAAC;YACd,CAAC,CAAC;iBACD,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;iBACd,IAAI,CAAC,GAAG,EAAE,UAAU,CAAK;gBACtB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAClC,CAAC,CAAC;iBACD,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC;iBAC1B,IAAI,CAAC,MAAM,EAAC,gBAAgB,CAAC;iBAC7B,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAG3B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC;iBAC5C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;iBAChB,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACtB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;iBACb,IAAI,CAAC,IAAI,EAAE,UAAU,CAAK,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;iBAC1E,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC;iBAC1B,IAAI,CAAC,IAAI,EAAE,UAAU,CAAK,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;iBAC1E,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC;iBACtB,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YAG/B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC;iBAC3C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;iBAChB,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACtB,IAAI,CAAC,UAAU,CAAK;gBACjB,EAAE,CAAA,CAAC,CAAC,CAAC,KAAK,CAAC;oBAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3B,MAAM,CAAC,EAAE,CAAC;YACd,CAAC,CAAC;iBACD,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;iBACd,IAAI,CAAC,GAAG,EAAE,UAAU,CAAK;gBACtB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAClC,CAAC,CAAC;iBACD,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC;iBACnB,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC;iBAC1B,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAG3B,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE;iBACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;iBACd,MAAM,CAAC,QAAQ,CAAC;iBAChB,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;iBACtB,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;iBACnC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAGpB,IAAI,IAAI,GAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC;iBACnC,IAAI,CAAC,WAAW,EAAE,cAAc,GAAG,UAAU,GAAG,GAAG,CAAC;iBAEpD,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC;iBACzB,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;iBACrB,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC;iBAC1E,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC;iBAC/B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YAG5E,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC;iBACrC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAGrC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC;iBAC3C,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;iBAC1C,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACtB,IAAI,CAAC,OAAO,EAAE,UAAU,CAAK;gBAC1B,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC;YACjC,CAAC,CAAC;iBACD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAK;gBACtB,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YAClB,CAAC,CAAC;iBACD,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC;iBACvB,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;YAGnC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;iBACvB,IAAI,CAAC,gBAAgB,EAAE,SAAS,CAAC;iBACjC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC;iBAC7B,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC;iBAC1B,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC;iBAC5B,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YACnC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE;iBACtB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;iBACT,MAAM,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;iBAC1B,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC;iBACpB,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;iBACxB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;iBAChB,SAAS,CAAC,MAAM,CAAC;iBACjB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;iBACZ,IAAI,CAAC,QAAQ,EAAE,UAAU,GAAG,CAAC,CAAC;iBAC9B,KAAK,CAAC,MAAM,EAAC,MAAM,CAAC;iBACpB,KAAK,CAAC,cAAc,EAAC,KAAK,CAAC;iBAC3B,KAAK,CAAC,QAAQ,EAAC,eAAe,CAAC;iBAC/B,KAAK,CAAC,cAAc,EAAC,CAAC,CAAC,CAAC;YAG7B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,CAAC;YACpD,IAAI,CAAC,WAAW,EAAE,CAAC;YAGnB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACb,IAAI,UAAoB,CAAC;gBACzB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;oBAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBACxD,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;qBAChB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;qBAC7B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;qBAClC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBAEnC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;oBACb,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;wBAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;oBACvE,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC;wBAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,CAAC;oBAC9F,EAAE,CAAC,CAAC,UAAU,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC;wBAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;oBAClF,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;wBAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;oBACnE,IAAI;wBAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,cAAc,CAAC,mBAAmB,CAAC,CAAC;gBAChE,CAAC;gBAAC,IAAI,CAAC,CAAC;oBACJ,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;oBAC3C,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,cAAc,CAAC,mBAAmB,CAAC,CAAC;gBAC3D,CAAC;YACL,CAAC;QACL,CAAC,CAAC;QAGF,gBAAW,GAAG;YACV,IAAI,QAAQ,GAAO,IAAI,CAAC;YAExB,IAAI,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAC1C,IAAI,SAAS,GAAU,MAAM,CAAC,CAAC,CAAC,CAAC;YACjC,IAAI,SAAS,GAAU,MAAM,CAAC,CAAC,CAAC,CAAC;YAEjC,IAAI,YAAY,GAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC;gBACnD,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,IAAI,CAAC,CAAC,GAAG,GAAG,SAAS,CAAA;YACnD,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;YAE/E,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;YAGvC,IAAI,KAAK,GAAG,SAAS,GAAG,SAAS,CAAC;YAClC,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAC9C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YACjF,CAAC;YAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACpD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9E,CAAC;YAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/E,CAAC;YAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC,mBAAmB,CAAC,CAAC,CAAC;gBACvD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YACjF,CAAC;YAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC;gBACxB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YACrF,CAAC;YAAC,IAAI,CAAC,CAAC;gBACJ,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YACpF,CAAC;YAGD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAGvD,IAAI,KAAK,GAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC;iBACvC,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;iBACjD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxD,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAG3F,KAAK,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACvB,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,iCAAiC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;iBAChI,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;iBACjF,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,aAAa,CAAC,gCAAgC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC;iBACvB,IAAI,CAAC,MAAM,EAAE,UAAS,CAAC;gBACpB,EAAE,CAAA,CAAC,CAAC,CAAC,KAAK,CAAC;oBAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3B,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC;YACvC,CAAC,CAAC;iBACD,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;YAC7B,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;YAGtB,IAAI,MAAM,GAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC;iBAC5C,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC;gBAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAChB,CAAC,CAAC;iBACD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC;gBAClB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,CAAC;iBACD,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAE3B,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;iBACxB,IAAI,CAAC,UAAU,CAAC;gBACb,EAAE,CAAA,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;oBAAC,MAAM,CAAC,EAAE,CAAC;gBAC9D,EAAE,CAAA,CAAC,CAAC,CAAC,KAAK,CAAC;oBAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBAC3B,MAAM,CAAC,EAAE,CAAC;YACd,CAAC,CAAC;iBACD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC;gBAClB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,CAAC;iBACD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC;gBAClB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YAC3D,CAAC,CAAC;iBACD,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC;iBAC5B,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC;iBAC1B,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAE3B,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;QAC3B,CAAC,CAAC;QAEF,cAAS,GAAG;YACR,IAAI,MAAM,GAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,IAAI,GAAQ,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YACnD,IAAI,UAAU,GAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;YAEnG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC9E,IAAI,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC,CAAC;QAEF,qBAAgB,GAAG,UAAC,KAAS;YACzB,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,MAAM,GAAG,EAAE,CAAC;YAC/D,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACb,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;gBACzC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvG,CAAC;YAED,GAAG,CAAC,CAAC,IAAI,SAAS,IAAI,KAAK,CAAC,CAAC,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,EAAC,CAAC,CAAC;YAC5D,CAAC;YACD,MAAM,CAAC,MAAM,CAAC;QAClB,CAAC,CAAA;QA3UG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;QAEpF,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IApBc,iCAAmB,GAAG,EAAE,CAAC;IACzB,+CAAiC,GAAU,IAAI,CAAC;IAChD,8CAAgC,GAAU,IAAI,CAAC;IAE/C,iCAAmB,GAAU,EAAE,GAAG,IAAI,CAAC;IACvC,+BAAiB,GAAU,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC;IAClE,8BAAgB,GAAU,EAAE,GAAG,aAAa,CAAC,iBAAiB,CAAC;IAC/D,+BAAiB,GAAU,CAAC,GAAG,aAAa,CAAC,gBAAgB,CAAC;IAE9D,2BAAa,GAAG,WAAW,CAAC;IAkV/C,oBAAC;AAAD,CAAC,AA/WD,CAA4B,KAAK,GA+WhC;ACpXD;IAAyB,8BAAK;IAQ1B,oBAAa,OAAY;QAR7B,iBAgCC;QAvBO,kBAAM,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QAYjC,mBAAc,GAAG,cAAM,OAAA,KAAI,CAAC,WAAW,EAAhB,CAAgB,CAAC;QACxC,iBAAY,GAAG,cAAM,OAAA,KAAI,CAAC,SAAS,EAAd,CAAc,CAAC;QACpC,oBAAe,GAAG,cAAM,OAAA,KAAI,CAAC,YAAY,EAAjB,CAAiB,CAAC;QAE1C,mBAAc,GAAG,UAAC,GAAW;YACzB,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;gBAAC,MAAM,CAAC,IAAI,CAAC;YAChF,MAAM,CAAC,KAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC,CAAC;QAEF,uBAAkB,GAAG,cAAM,OAAA,KAAI,CAAC,eAAe,EAApB,CAAoB,CAAC;QAChD,kBAAa,GAAG,cAAM,OAAA,KAAI,CAAC,UAAU,EAAf,CAAe,CAAC;QApBlC,IAAI,KAAK,GAAQ,OAAO,CAAC,YAAY,CAAC,CAAC;QAEvC,EAAE,CAAA,CAAC,KAAK,CAAC,CAAA,CAAC;YACN,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;YACpC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;YAC1C,EAAE,CAAA,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBAAC,IAAI,CAAC,UAAU,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;QACjF,CAAC;IACL,CAAC;IAaL,iBAAC;AAAD,CAAC,AAhCD,CAAyB,KAAK,GAgC7B;AC/BD;IAA2B,gCAAS;IAKhC,sBAAY,OAAe;QACvB,kBAAM,aAAa,CAAC,YAAY,CAAC,CAAC;QAoBtC,WAAM,GAAG,UAAC,cAAsB;YAE5B,IAAI,MAAM,GAAW,CAAC,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAElB,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA,CAAC;gBAEX,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAA,CAAC;oBACtB,IAAI,IAAI,GAAW,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;oBAC7C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;gBACD,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA,CAAC;oBACvB,IAAI,IAAI,GAAW,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;oBAC9C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC/D,CAAC;gBACD,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;oBAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,EAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC;gBACnG,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;oBAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;YACnF,CAAC;YAGD,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAG9B,EAAE,CAAA,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA,CAAC;gBAChB,GAAG,CAAA,CAAE,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACtC,CAAC;YACL,CAAC;QACL,CAAC,CAAA;QA9CG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,IAAI,UAAU,GAAU,IAAI,CAAC,YAAY,CAAC,CAAC;QAE3C,EAAE,CAAA,CAAC,UAAU,CAAC,CAAA,CAAC;YACX,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;YACrB,GAAG,CAAA,CAAE,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,IAAI,KAAK,GAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,CAAC;QACL,CAAC;QAED,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAC,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAG/D,CAAC;IAgCL,mBAAC;AAAD,CAAC,AAxDD,CAA2B,SAAS,GAwDnC;ACxDD;IAAuB,4BAAK;IAIxB,kBAAa,OAAY;QAJ7B,iBAcC;QATO,kBAAM,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QAM/B,kBAAa,GAAG,cAAM,OAAA,KAAI,CAAC,UAAU,EAAf,CAAe,CAAC;QAJlC,EAAE,CAAA,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;YAAC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC;IAE3F,CAAC;IAKL,eAAC;AAAD,CAAC,AAdD,CAAuB,KAAK,GAc3B;ACRD;IAAiC,sCAAS;IAOtC,4BAAY,OAAe;QACvB,kBAAM,aAAa,CAAC,kBAAkB,CAAC,CAAC;QAqB5C,WAAM,GAAG,UAAC,cAAsB;YAE5B,IAAI,CAAC,GAAkB,IAAI,CAAC,KAAK,CAAC;YAElC,IAAI,QAAQ,GAAW,CAAC,CAAC,aAAa,CAAC,CAAC;YACxC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAEpB,IAAI,QAAgB,CAAC;YACrB,EAAE,CAAA,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAC,QAAQ,GAAG,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC;YAC7D,IAAI;gBAAC,QAAQ,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;YACjC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACpB,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAE1B,IAAI,QAAQ,GAAW,CAAC,CAAC,aAAa,CAAC,CAAC;YACxC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACpB,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAG1B,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;gBACvB,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAEnD,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC7C,CAAC;YACL,CAAC;YAED,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEhC,EAAE,CAAA,CAAC,IAAI,CAAC,gBAAgB,CAAC;gBAAC,QAAQ,CAAC,SAAS,CAAC,EAAC,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAC,CAAC,CAAC;YACzG,IAAI;gBAAC,QAAQ,CAAC,SAAS,CAAC,EAAC,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAC,CAAC,CAAC;QASzE,CAAC,CAAA;QAxDG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAExF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAEjD,IAAI,OAAO,GAAU,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAE7C,EAAE,CAAA,CAAC,OAAO,CAAC,CAAA,CAAC;YACR,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;YAC1B,GAAG,CAAA,CAAE,IAAI,CAAC,GAAC,CAAC,EAAE,CAAC,GAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,IAAI,KAAK,GAAW,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACL,CAAC;QAED,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAC,IAAI,CAAC,KAAK,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACrE,CAAC;IA0CL,yBAAC;AAAD,CAAC,AArED,CAAiC,SAAS,GAqEzC;AC3ED;IAA6B,kCAAK;IAE9B,wBAAa,OAAY;QACrB,kBAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAGrC,CAAC;IAEL,qBAAC;AAAD,CAAC,AARD,CAA6B,KAAK,GAQjC;ACLD;IAA6B,kCAAS;IAOlC,wBAAY,OAAe;QACvB,kBAAM,aAAa,CAAC,cAAc,CAAC,CAAC;QAUxC,WAAM,GAAG,UAAC,cAAsB;YAE5B,IAAI,CAAC,GAAe,IAAI,CAAC,KAAK,CAAC;YAC/B,IAAI,MAAM,GAAW,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAEzC,IAAI,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YAE1C,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;YACzB,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,IAAI,IAAK,CAAC;gBAAC,GAAG,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;YAChG,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,CAAC,kBAAkB,EAAE,CAAC;gBAAC,GAAG,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC,kBAAkB,EAAE,CAAC;YACnF,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,CAAC,iBAAiB,EAAE,CAAC;gBAAC,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,iBAAiB,EAAE,CAAC;YAE5E,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;gBAE3B,IAAI,SAAS,GAAa,CAAC,CAAC,eAAe,EAAE,CAAC;gBAC9C,IAAI,IAAI,GAAW,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;gBACvE,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACxC,IAAI,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBACxC,GAAG,CAAC,YAAY,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;oBAC/C,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC;YAGD,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,OAAO,GAAG,CAAC,CAAC;YAEhB,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBACd,IAAI,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;gBAC9C,IAAI,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBAE7C,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;oBAAC,SAAS,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC,cAAc,EAAE,CAAC;gBAEjF,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,IAAI,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;oBAC3C,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,GAAG,OAAO,GAAG,IAAI,CAAC;oBAC/F,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC7D,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBACnC,CAAC;gBACD,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;YAC/B,CAAC;YAGD,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAEf,IAAI,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;gBAC3C,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3C,IAAI,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;oBAEtC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC9C,IAAI,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;wBACtC,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,GAAG,OAAO,GAAG,IAAI,CAAC;wBAC1F,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC5D,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;oBACvB,CAAC;oBAED,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBACzB,CAAC;gBACD,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YAED,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC,CAAA;QAxEG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,CAAC;QAEpF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAC,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,CAAC;IAqEL,qBAAC;AAAD,CAAC,AArFD,CAA6B,SAAS,GAqFrC;ACzFD;IAAyB,8BAAK;IAQ1B,oBAAa,OAAY;QAR7B,iBA0BC;QAjBO,kBAAM,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QAYjC,oBAAe,GAAG,cAAM,OAAA,KAAI,CAAC,YAAY,EAAjB,CAAiB,CAAC;QAC1C,uBAAkB,GAAG,cAAM,OAAA,KAAI,CAAC,eAAe,EAApB,CAAoB,CAAC;QAChD,qBAAgB,GAAG,cAAM,OAAA,KAAI,CAAC,aAAa,EAAlB,CAAkB,CAAC;QAC5C,mBAAc,GAAG,cAAM,OAAA,KAAI,CAAC,WAAW,EAAhB,CAAgB,CAAC;QACxC,sBAAiB,GAAG,cAAM,OAAA,KAAI,CAAC,cAAc,EAAnB,CAAmB,CAAC;QAd1C,IAAI,KAAK,GAAQ,OAAO,CAAC,YAAY,CAAC,CAAC;QACvC,EAAE,CAAA,CAAC,KAAK,CAAC,CAAA,CAAC;YACN,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,cAAc,CAAC,CAAC;YAC1D,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,CAAC;YAC5D,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,aAAa,CAAC,CAAC;YACxD,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAChE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;IAOL,iBAAC;AAAD,CAAC,AA1BD,CAAyB,KAAK,GA0B7B;ACtBD;IAA4B,iCAAS;IAKjC,uBAAY,OAAe;QAL/B,iBAwCC;QAlCO,kBAAM,aAAa,CAAC,aAAa,CAAC,CAAC;QASvC,WAAM,GAAG,UAAC,cAAsB;YAE5B,IAAI,QAAQ,GAAS,QAAQ,CAAC,cAAc,CAAC,KAAI,CAAC,IAAI,CAAC,CAAC;YACxD,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,CAAA,CAAC;gBACX,IAAI,OAAO,GAAoB,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAC9D,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,IAAI,GAAG,KAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnE,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAI,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC;gBAC9F,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,cAAc,GAAC,WAAW,CAAC;gBAC/E,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,KAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACtE,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,SAAS,GAAG,KAAI,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC;gBACzF,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,YAAY,GAAG,KAAI,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC;gBAClG,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,KAAI,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC;gBAC5F,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,WAAW,GAAG,KAAI,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC;gBAC/F,EAAE,CAAA,CAAC,KAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC;gBAEnE,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC9B,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnC,CAAC;YAAC,IAAI,CAAC,CAAC;gBACJ,IAAI,OAAO,GAAoB,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAE9D,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC9B,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnC,CAAC;QACL,CAAC,CAAA;QA/BG,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,EAAE,CAAA,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAAC,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC;QAEnF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzB,EAAE,CAAA,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAAC,IAAI,CAAC,KAAK,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAChE,CAAC;IA2BL,oBAAC;AAAD,CAAC,AAxCD,CAA4B,SAAS,GAwCpC;AC5CD;IAAwB,6BAAK;IAQzB,mBAAa,OAAY;QAR7B,iBA0BC;QAjBO,kBAAM,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;QAYhC,YAAO,GAAG,cAAM,OAAA,KAAI,CAAC,IAAI,EAAT,CAAS,CAAC;QAC1B,gBAAW,GAAG,cAAM,OAAA,KAAI,CAAC,QAAQ,EAAb,CAAa,CAAC;QAClC,iBAAY,GAAG,cAAM,OAAA,KAAI,CAAC,SAAS,EAAd,CAAc,CAAC;QACpC,aAAQ,GAAG,cAAM,OAAA,KAAI,CAAC,KAAK,EAAV,CAAU,CAAC;QAC5B,qBAAgB,GAAG,cAAM,OAAA,KAAI,CAAC,aAAa,EAAlB,CAAkB,CAAC;QAdxC,IAAI,KAAK,GAAQ,OAAO,CAAC,WAAW,CAAC,CAAC;QACtC,EAAE,CAAA,CAAC,KAAK,CAAC,CAAA,CAAC;YACN,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5B,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC;QAChD,CAAC;IACL,CAAC;IAOL,gBAAC;AAAD,CAAC,AA1BD,CAAwB,KAAK,GA0B5B"} \ No newline at end of file diff --git a/arbiter/arbiter-ui/src/main/resources/templates/ArbiterUI.html b/arbiter/arbiter-ui/src/main/resources/templates/ArbiterUI.html new file mode 100644 index 000000000..a1b4e92a0 --- /dev/null +++ b/arbiter/arbiter-ui/src/main/resources/templates/ArbiterUI.html @@ -0,0 +1,638 @@ + + + + + + DL4J - Arbiter UI + + + + + + + + + + + + + + + + + + + + +
+
Deeplearning4J - Arbiter UI
 + +
+ + +
+
+
+

Summary

+
+
+
+
+ +
+
+

Optimization Settings

+
+
+
+ + +
+
Results
+
+ + + + + + +
+
+
+ +
+
+

Selected Result

+
+
+
+
+ + \ No newline at end of file diff --git a/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/AssertTestsExtendBaseClass.java b/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..fcf3066e2 --- /dev/null +++ b/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.deeplearning4j.arbiter.optimize; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.deeplearning4j.BaseDL4JTest; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.deeplearning4j.arbiter.optimize"; + } + + @Override + protected Class getBaseClass() { + return BaseDL4JTest.class; + } +} diff --git a/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/TestBasic.java b/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/TestBasic.java new file mode 100644 index 000000000..b7502c84b --- /dev/null +++ b/arbiter/arbiter-ui/src/test/java/org/deeplearning4j/arbiter/optimize/TestBasic.java @@ -0,0 +1,791 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.arbiter.optimize; + +import io.netty.handler.codec.http.HttpResponseStatus; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.core.storage.StatsStorage; +import org.deeplearning4j.arbiter.ComputationGraphSpace; +import org.deeplearning4j.arbiter.MultiLayerSpace; +import org.deeplearning4j.arbiter.conf.updater.SgdSpace; +import org.deeplearning4j.arbiter.layers.ConvolutionLayerSpace; +import org.deeplearning4j.arbiter.layers.DenseLayerSpace; +import org.deeplearning4j.arbiter.layers.OutputLayerSpace; +import org.deeplearning4j.arbiter.optimize.api.CandidateGenerator; +import org.deeplearning4j.arbiter.optimize.api.ParameterSpace; +import org.deeplearning4j.arbiter.optimize.api.data.DataProvider; +import org.deeplearning4j.arbiter.optimize.api.data.DataSource; +import org.deeplearning4j.arbiter.optimize.api.score.ScoreFunction; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxCandidatesCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.MaxTimeCondition; +import org.deeplearning4j.arbiter.optimize.api.termination.TerminationCondition; +import org.deeplearning4j.arbiter.optimize.config.OptimizationConfiguration; +import org.deeplearning4j.arbiter.optimize.generator.RandomSearchGenerator; +import org.deeplearning4j.arbiter.optimize.parameter.continuous.ContinuousParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.discrete.DiscreteParameterSpace; +import org.deeplearning4j.arbiter.optimize.parameter.integer.IntegerParameterSpace; +import org.deeplearning4j.arbiter.optimize.runner.IOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.LocalOptimizationRunner; +import org.deeplearning4j.arbiter.optimize.runner.listener.StatusListener; +import org.deeplearning4j.arbiter.saver.local.FileModelSaver; +import org.deeplearning4j.arbiter.scoring.impl.EvaluationScoreFunction; +import org.deeplearning4j.arbiter.scoring.impl.TestSetLossScoreFunction; +import org.deeplearning4j.arbiter.task.ComputationGraphTaskCreator; +import org.deeplearning4j.arbiter.task.MultiLayerNetworkTaskCreator; +import org.deeplearning4j.arbiter.ui.listener.ArbiterStatusListener; +import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.ui.api.UIServer; +import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; + +import org.junit.jupiter.api.Test; +import org.nd4j.common.function.Function; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Created by Alex on 19/07/2017. + */ +@Slf4j +public class TestBasic extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return 3600_000L; + } + + @Test + //@Ignore + public void testBasicUiOnly() throws Exception { + + UIServer.getInstance(); + + Thread.sleep(1000_000); + } + + @Test + //@Ignore + public void testBasicMnist() throws Exception { + Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); + + MultiLayerSpace mls = getMultiLayerSpaceMnist(); + Map commands = new HashMap<>(); +// commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); + DataProvider dataProvider = new MnistDataSetProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(120, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + + StatsStorage ss = new InMemoryStatsStorage(); + StatusListener sl = new ArbiterStatusListener(ss); + runner.addListeners(sl); + + UIServer.getInstance().attach(ss); + + runner.execute(); + Thread.sleep(1000_000); + } + + private static MultiLayerSpace getMultiLayerSpaceMnist() { + return new MultiLayerSpace.Builder() + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .addLayer( + new ConvolutionLayerSpace.Builder().nIn(1) + .nOut(new IntegerParameterSpace(5, 30)) + .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3}, + new int[]{4, 4}, new int[]{5, 5})) + .stride(new DiscreteParameterSpace<>(new int[]{1, 1}, + new int[]{2, 2})) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.SOFTPLUS, Activation.LEAKYRELU)) + .build()) + .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers + .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) + .build(); + } + + @Test + //@Ignore + public void testBasicMnistDataSource() throws InterruptedException { + ParameterSpace learningRateHyperparam = new ContinuousParameterSpace(0.0001, 0.1); + ParameterSpace layerSizeHyperparam = new IntegerParameterSpace(16, 256); + + MultiLayerSpace hyperparameterSpace = new MultiLayerSpace.Builder() + .weightInit(WeightInit.XAVIER) + .l2(0.0001) + .updater(new SgdSpace(learningRateHyperparam)) + .addLayer(new DenseLayerSpace.Builder() + .nIn(784) + .activation(Activation.LEAKYRELU) + .nOut(layerSizeHyperparam) + .build()) + .addLayer(new OutputLayerSpace.Builder() + .nOut(10) + .activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT) + .build()) + .build(); + CandidateGenerator candidateGenerator = new RandomSearchGenerator(hyperparameterSpace, null); + ScoreFunction scoreFunction = new EvaluationScoreFunction(Evaluation.Metric.ACCURACY); + TerminationCondition[] terminationConditions = { + new MaxTimeCondition(5, TimeUnit.MINUTES), + new MaxCandidatesCondition(2)}; + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + Class ds = MnistDataSource.class; + Properties dsp = new Properties(); + dsp.setProperty("minibatch", "8"); + OptimizationConfiguration configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(scoreFunction) + .terminationConditions(terminationConditions) + .build(); + + IOptimizationRunner runner = new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + + StatsStorage ss = new InMemoryStatsStorage(); + StatusListener sl = new ArbiterStatusListener(ss); + runner.addListeners(sl); + + UIServer.getInstance().attach(ss); + + runner.execute(); + Thread.sleep(90000); + } + + + @Test + //@Ignore + public void testBasicMnistCompGraph() throws Exception { + + ComputationGraphSpace cgs = new ComputationGraphSpace.Builder() + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .addInputs("in") + .addLayer("0", + new ConvolutionLayerSpace.Builder().nIn(1) + .nOut(new IntegerParameterSpace(5, 30)) + .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3}, + new int[]{4, 4}, new int[]{5, 5})) + .stride(new DiscreteParameterSpace<>(new int[]{1, 1}, + new int[]{2, 2})) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.SOFTPLUS, Activation.LEAKYRELU)) + .build(), "in") + .addLayer("1", new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), "0") + .addLayer("out", new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "1") + .setOutputs("out") + .setInputTypes(InputType.convolutionalFlat(28, 28, 1)) + .build(); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs); + DataProvider dataProvider = new MnistDataSetProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnistCG\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(120, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new ComputationGraphTaskCreator()); + + StatsStorage ss = new InMemoryStatsStorage(); + StatusListener sl = new ArbiterStatusListener(ss); + runner.addListeners(sl); + + UIServer.getInstance().attach(ss); + + runner.execute(); + Thread.sleep(100000); + } + + + @Test + //@Ignore + public void testCandidateGenerationExceptionsMnist() throws Exception { + + //Idea: Create a configuration that is not physically realizable, which should throw an exception + // during the candidate generation phase + //This exception should be visible in UI, but training should continue otherwise + + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .dropOut(new ContinuousParameterSpace(0.2, 0.7)) + .addLayer( + new ConvolutionLayerSpace.Builder().nIn(1) + .nOut(new IntegerParameterSpace(5, 5)) + .kernelSize(new DiscreteParameterSpace<>(new int[]{14, 14}, new int[]{30, 30})) + .stride(2, 2) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.SOFTPLUS, Activation.LEAKYRELU)) + .build()) + .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers + .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) + .build(); + Map commands = new HashMap<>(); +// commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); + DataProvider dataProvider = new MnistDataSetProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(120, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + + StatsStorage ss = new InMemoryStatsStorage(); + StatusListener sl = new ArbiterStatusListener(ss); + runner.addListeners(sl); + + UIServer.getInstance().attach(ss); + + runner.execute(); + Thread.sleep(1000_000); + } + + + @Test + //@Ignore + public void testCandidateExecutionExceptionsMnist() throws Exception { + //Idea: Create a configuration that will throw an exception in the *execution* stage + // How? let's set wrong nOut + //This exception should be visible in UI, but training should continue otherwise + + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .dropOut(new ContinuousParameterSpace(0.2, 0.7)) + .addLayer( + new ConvolutionLayerSpace.Builder().nIn(1) + .nOut(new IntegerParameterSpace(5, 5)) + .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3}, new int[]{4, 4})) + .stride(2, 2) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.SOFTPLUS, Activation.LEAKYRELU)) + .build()) + .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 64)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers + .addLayer(new OutputLayerSpace.Builder().nOut(99).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) + .build(); + Map commands = new HashMap<>(); +// commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); + DataProvider dataProvider = new MnistDataSetProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(120, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + + StatsStorage ss = new InMemoryStatsStorage(); + StatusListener sl = new ArbiterStatusListener(ss); + runner.addListeners(sl); + + UIServer.getInstance().attach(ss); + + runner.execute(); + Thread.sleep(1000_000); + } + + + @Test + //@Ignore + public void testExecutionExceptionMnistCompGraph() throws Exception { + + //Idea: Create a configuration that will throw an exception in the *execution* stage + // How? let's set wrong nOut + //This exception should be visible in UI, but training should continue otherwise + + ComputationGraphSpace cgs = new ComputationGraphSpace.Builder() + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .dropOut(new ContinuousParameterSpace(0.2, 0.7)) + .addInputs("in") + .addLayer("0", + new ConvolutionLayerSpace.Builder().nIn(1) + .nOut(new IntegerParameterSpace(5, 30)) + .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3}, + new int[]{4, 4}, new int[]{5, 5})) + .stride(new DiscreteParameterSpace<>(new int[]{1, 1}, + new int[]{2, 2})) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.SOFTPLUS, Activation.LEAKYRELU)) + .build(), "in") + .addLayer("1", new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 64)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), "0") + .addLayer("out", new OutputLayerSpace.Builder().nIn(99).nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "1") + .setOutputs("out") + .setInputTypes(InputType.convolutionalFlat(28, 28, 1)) + .build(); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(cgs); + DataProvider dataProvider = new MnistDataSetProvider(); + + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnistCG\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataProvider(dataProvider) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(120, TimeUnit.MINUTES), + new MaxCandidatesCondition(100)) + .build(); + + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new ComputationGraphTaskCreator()); + + StatsStorage ss = new InMemoryStatsStorage(); + StatusListener sl = new ArbiterStatusListener(ss); + runner.addListeners(sl); + + UIServer.getInstance().attach(ss); + + runner.execute(); + Thread.sleep(1000_000); + } + + + /** + * Visualize multiple optimization sessions run one after another on single-session mode UI + * @throws InterruptedException if current thread has been interrupted + */ + @Test + //@Ignore + public void testBasicMnistMultipleSessions() throws InterruptedException { + + MultiLayerSpace mls = new MultiLayerSpace.Builder() + .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2))) + .l2(new ContinuousParameterSpace(0.0001, 0.05)) + .dropOut(new ContinuousParameterSpace(0.2, 0.7)) + .addLayer( + new ConvolutionLayerSpace.Builder().nIn(1) + .nOut(new IntegerParameterSpace(5, 30)) + .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3}, + new int[]{4, 4}, new int[]{5, 5})) + .stride(new DiscreteParameterSpace<>(new int[]{1, 1}, + new int[]{2, 2})) + .activation(new DiscreteParameterSpace<>(Activation.RELU, + Activation.SOFTPLUS, Activation.LEAKYRELU)) + .build()) + .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128)) + .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH)) + .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers + .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) + .build(); + Map commands = new HashMap<>(); +// commands.put(DataSetIteratorFactoryProvider.FACTORY_KEY, TestDataFactoryProviderMnist.class.getCanonicalName()); + + //Define configuration: + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls, commands); + + Class ds = MnistDataSource.class; + Properties dsp = new Properties(); + dsp.setProperty("minibatch", "8"); + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\").getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(1, TimeUnit.MINUTES), + new MaxCandidatesCondition(3)) + .build(); + + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + + StatsStorage ss = new InMemoryStatsStorage(); + + + StatusListener sl = new ArbiterStatusListener(ss); + runner.addListeners(sl); + + UIServer.getInstance().attach(ss); + runner.execute(); + + + candidateGenerator = new RandomSearchGenerator(mls, commands); + configuration = new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(1, TimeUnit.MINUTES), + new MaxCandidatesCondition(3)) + .build(); + + runner = new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + sl = new ArbiterStatusListener(ss); + runner.addListeners(sl); + + UIServer.getInstance().attach(ss); + + runner.execute(); + + Thread.sleep(1000_000); + } + + /** + * Auto-attach multiple optimization sessions to multi-session mode UI + * @throws IOException if could not connect to the server + */ + @Test + public void testUiMultiSessionAutoAttach() throws IOException { + + //Define configuration: + MultiLayerSpace mls = getMultiLayerSpaceMnist(); + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls); + + Class ds = MnistDataSource.class; + Properties dsp = new Properties(); + dsp.setProperty("minibatch", "8"); + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestMultiSessionAutoAttach\\") + .getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(10, TimeUnit.SECONDS), + new MaxCandidatesCondition(1)) + .build(); + + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + + // add 3 different sessions to the same execution + HashMap statsStorageForSession = new HashMap<>(); + for (int i = 0; i < 3; i++) { + StatsStorage ss = new InMemoryStatsStorage(); + @NonNull String sessionId = "sid" + i; + statsStorageForSession.put(sessionId, ss); + StatusListener sl = new ArbiterStatusListener(sessionId, ss); + runner.addListeners(sl); + } + + Function statsStorageProvider = statsStorageForSession::get; + UIServer uIServer = UIServer.getInstance(true, statsStorageProvider); + String serverAddress = uIServer.getAddress(); + + runner.execute(); + + for (String sessionId : statsStorageForSession.keySet()) { + /* + * Visiting /arbiter/:sessionId to auto-attach StatsStorage + */ + String sessionUrl = sessionUrl(uIServer.getAddress(), sessionId); + HttpURLConnection conn = (HttpURLConnection) new URL(sessionUrl).openConnection(); + conn.connect(); + + log.info("Checking auto-attaching Arbiter session at {}", sessionUrl(serverAddress, sessionId)); + assertEquals(HttpResponseStatus.OK.code(), conn.getResponseCode()); + assertTrue(uIServer.isAttached(statsStorageForSession.get(sessionId))); + } + } + + /** + * Attach multiple optimization sessions to multi-session mode UI by manually visiting session URL + * @throws Exception if an error occurred + */ + @Test + //@Ignore + public void testUiMultiSessionManualAttach() throws Exception { + Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); + + //Define configuration: + MultiLayerSpace mls = getMultiLayerSpaceMnist(); + CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls); + + Class ds = MnistDataSource.class; + Properties dsp = new Properties(); + dsp.setProperty("minibatch", "8"); + + String modelSavePath = new File(System.getProperty("java.io.tmpdir"), "ArbiterUiTestBasicMnist\\") + .getAbsolutePath(); + + File f = new File(modelSavePath); + if (f.exists()) + f.delete(); + f.mkdir(); + if (!f.exists()) + throw new RuntimeException(); + + OptimizationConfiguration configuration = + new OptimizationConfiguration.Builder() + .candidateGenerator(candidateGenerator).dataSource(ds, dsp) + .modelSaver(new FileModelSaver(modelSavePath)) + .scoreFunction(new TestSetLossScoreFunction(true)) + .terminationConditions(new MaxTimeCondition(10, TimeUnit.MINUTES), + new MaxCandidatesCondition(10)) + .build(); + + + // parallel execution of multiple optimization sessions + HashMap statsStorageForSession = new HashMap<>(); + for (int i = 0; i < 3; i++) { + String sessionId = "sid" + i; + IOptimizationRunner runner = + new LocalOptimizationRunner(configuration, new MultiLayerNetworkTaskCreator()); + StatsStorage ss = new InMemoryStatsStorage(); + statsStorageForSession.put(sessionId, ss); + StatusListener sl = new ArbiterStatusListener(sessionId, ss); + runner.addListeners(sl); + // Asynchronous execution + new Thread(runner::execute).start(); + } + + Function statsStorageProvider = statsStorageForSession::get; + UIServer uIServer = UIServer.getInstance(true, statsStorageProvider); + String serverAddress = uIServer.getAddress(); + + for (String sessionId : statsStorageForSession.keySet()) { + log.info("Arbiter session can be attached at {}", sessionUrl(serverAddress, sessionId)); + } + + Thread.sleep(1000_000); + } + + + /** + * Get URL for arbiter session on given server address + * @param serverAddress server address, e.g.: http://localhost:9000 + * @param sessionId session ID (will be URL-encoded) + * @return URL + * @throws UnsupportedEncodingException if the character encoding is not supported + */ + private static String sessionUrl(String serverAddress, String sessionId) throws UnsupportedEncodingException { + return String.format("%s/arbiter/%s", serverAddress, URLEncoder.encode(sessionId, "UTF-8")); + } + + private static class MnistDataSetProvider implements DataProvider { + + @Override + public DataSetIterator trainData(Map dataParameters) { + try { + if (dataParameters == null || dataParameters.isEmpty()) { + return new MnistDataSetIterator(64, 10000, false, true, true, 123); + } + if (dataParameters.containsKey("batchsize")) { + int b = (Integer) dataParameters.get("batchsize"); + return new MnistDataSetIterator(b, 10000, false, true, true, 123); + } + return new MnistDataSetIterator(64, 10000, false, true, true, 123); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public DataSetIterator testData(Map dataParameters) { + return trainData(dataParameters); + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + + @Override + public String toString() { + return "MnistDataSetProvider()"; + } + } + + public static class MnistDataSource implements DataSource { + private int minibatch; + + public MnistDataSource() { + + } + + @Override + public void configure(Properties properties) { + this.minibatch = Integer.parseInt(properties.getProperty("minibatch", "16")); + } + + @Override + public Object trainData() { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(minibatch, true, 12345), 3); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public Object testData() { + try { + return new EarlyTerminationDataSetIterator(new MnistDataSetIterator(minibatch, true, 12345), 3); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public Class getDataType() { + return DataSetIterator.class; + } + } + +} diff --git a/arbiter/arbiter-ui/src/test/resources/logback.xml b/arbiter/arbiter-ui/src/test/resources/logback.xml new file mode 100644 index 000000000..410bdaae9 --- /dev/null +++ b/arbiter/arbiter-ui/src/test/resources/logback.xml @@ -0,0 +1,51 @@ + + + + + + logs/application.log + + %date - [%level] - from %logger in %thread + %n%message%n%xException%n + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/arbiter/buildmultiplescalaversions.sh b/arbiter/buildmultiplescalaversions.sh new file mode 100644 index 000000000..e04610a02 --- /dev/null +++ b/arbiter/buildmultiplescalaversions.sh @@ -0,0 +1,53 @@ +#! /bin/bash +################################################################################ +# Copyright (c) 2015-2018 Skymind, Inc. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + +BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function echoError() { + (>&2 echo "$1") +} + +function scalaError() { + echoError "Changing Scala major version to 2.10 in the build did not change the state of your working copy, is Scala 2.11 still the default ?" + exit 2 +} + +function whatchanged() { + cd "$BASEDIR" + for i in $(git status -s --porcelain -- $(find ./ -mindepth 2 -name pom.xml)|awk '{print $2}'); do + echo "$(dirname $i)" + cd "$BASEDIR" + done +} + +set -eu +./change-scala-versions.sh 2.11 # should be idempotent, this is the default +mvn "$@" +./change-scala-versions.sh 2.10 +if [ -z "$(whatchanged)" ]; then + scalaError; +else + if [[ "${@#-pl}" = "$@" ]]; then + mvn -Dmaven.clean.skip=true -pl $(whatchanged| tr '\n' ',') -amd "$@" + else + # the arguments already tweak the project list ! don't tweak them more + # as this can lead to conflicts (excluding a project that's not part of + # the reactor) + mvn "$@" + fi +fi +./change-scala-versions.sh 2.11 # back to the default diff --git a/arbiter/contrib/formatter.xml b/arbiter/contrib/formatter.xml new file mode 100644 index 000000000..d6cc96bf6 --- /dev/null +++ b/arbiter/contrib/formatter.xml @@ -0,0 +1,353 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/arbiter/pom.xml b/arbiter/pom.xml new file mode 100644 index 000000000..a3321c0ab --- /dev/null +++ b/arbiter/pom.xml @@ -0,0 +1,182 @@ + + + + + + + + net.brutex.ai + deeplearning4j + 1.0.0-SNAPSHOT + + + 4.0.0 + + net.brutex.ai + arbiter + pom + + Arbiter + Model Evaluation and Testing + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + arbiter-deeplearning4j + arbiter-core + arbiter-server + arbiter-ui + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + generate-javadoc + prepare-package + + javadoc + + + + + + + + + + + net.alchim31.maven + scala-maven-plugin + ${maven-scala-plugin.version} + + + -deprecation + -explaintypes + -nobootcp + + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + add-source + testCompile + + + + + + + + + + + test-nd4j-native + + + net.brutex.ai + nd4j-native + ${project.version} + test + + + net.brutex.ai + dl4j-test-resources + ${dl4j-test-resources.version} + test + + + + + + test-nd4j-cuda-10.2 + + + net.brutex.ai + nd4j-cuda-${cuda.version} + ${project.version} + test + + + net.brutex.ai + dl4j-test-resources + ${dl4j-test-resources.version} + test + + + + + only-eclipse + + + m2e.version + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + com.lewisd + lint-maven-plugin + [0.0.11,) + + check + + + + + + + + + + + + + + + + diff --git a/brutex-extended-tests/build.gradle b/brutex-extended-tests/build.gradle new file mode 100644 index 000000000..d21da53de --- /dev/null +++ b/brutex-extended-tests/build.gradle @@ -0,0 +1,66 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +apply plugin: 'java' +apply plugin: 'maven-publish' + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +ext { + buildTarget = rootProject.ext.buildTarget + scalaVersion = rootProject.ext.scalaVersion +} + +dependencies { + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.google.guava:guava" + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisUi.cavisUiStandalone + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisDnn.cavisDnnApi + implementation "org.slf4j:slf4j-api" + implementation "org.apache.hadoop:hadoop-client" + compileOnly "org.apache.spark:spark-core_${scalaVersion}" + compileOnly "org.apache.spark:spark-sql_${scalaVersion}" + compileOnly "org.scala-lang:scala-library" + testImplementation "org.apache.spark:spark-core_${scalaVersion}" + testImplementation "org.apache.spark:spark-sql_${scalaVersion}" + testCompileOnly "org.scala-lang:scala-library" + + implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkCore + implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkParameterserver + implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisUi.cavisUiCommon + implementation projects.cavisUi.cavisUiVertx + implementation projects.cavisUi.cavisUiModel + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators + implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage +} + +test { + dependsOn jar +} + diff --git a/brutex-extended-tests/src/main/java/net/brutex/ai/Dummy.java b/brutex-extended-tests/src/main/java/net/brutex/ai/Dummy.java new file mode 100644 index 000000000..3179dc3e5 --- /dev/null +++ b/brutex-extended-tests/src/main/java/net/brutex/ai/Dummy.java @@ -0,0 +1,25 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package net.brutex.ai; + +public class Dummy { +} diff --git a/brutex-extended-tests/src/main/java/net/brutex/ai/package-info.java b/brutex-extended-tests/src/main/java/net/brutex/ai/package-info.java new file mode 100644 index 000000000..e8635d3f5 --- /dev/null +++ b/brutex-extended-tests/src/main/java/net/brutex/ai/package-info.java @@ -0,0 +1,22 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package net.brutex.ai; \ No newline at end of file diff --git a/brutex-extended-tests/src/test/java/net/brutex/ai/nd4j/tests/LoadBackendTests.java b/brutex-extended-tests/src/test/java/net/brutex/ai/nd4j/tests/LoadBackendTests.java new file mode 100644 index 000000000..4ce2844d5 --- /dev/null +++ b/brutex-extended-tests/src/test/java/net/brutex/ai/nd4j/tests/LoadBackendTests.java @@ -0,0 +1,55 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package net.brutex.ai.nd4j.tests; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.lang.reflect.Field; +import java.util.logging.Logger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class LoadBackendTests { + + @Test + public void loadBackend() throws ClassNotFoundException, NoSuchFieldException, IllegalAccessException { + // check if Nd4j is there + //Logger.getLogger(LoadBackendTests.class.getName()).info("System java.library.path: " + System.getProperty("java.library.path")); + final Field sysPathsField = ClassLoader.class.getDeclaredField("sys_paths"); + sysPathsField.setAccessible(true); + sysPathsField.set(null, null); + //System.loadLibrary("jnind4jcpu"); + log.info("Backend: {}", Nd4j.getBackend().buildInfo()); + double d1 = 2.0; + double d2 = 5.0; + INDArray arr = Nd4j.scalar(d1); + INDArray arr2 = Nd4j.scalar( d2); + INDArray res = arr.add(arr2); + Number n = res.sumNumber(); + assertEquals(n.doubleValue(), 7.0, String.format("Addition of two scalar values %g and %g", d1, d2)); + } +} diff --git a/brutex-extended-tests/src/test/java/net/brutex/spark/BaseSparkSessionTest.java b/brutex-extended-tests/src/test/java/net/brutex/spark/BaseSparkSessionTest.java new file mode 100644 index 000000000..5f81489e0 --- /dev/null +++ b/brutex-extended-tests/src/test/java/net/brutex/spark/BaseSparkSessionTest.java @@ -0,0 +1,67 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package net.brutex.spark; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +import java.io.Serializable; + +@Slf4j +public abstract class BaseSparkSessionTest implements Serializable { + private static SparkSession spark; + + public static SparkSession getSession() { + SparkConf sparkConf = new SparkConf() + .setMaster("spark://10.5.5.200:7077") + .setAppName(BaseSparkSessionTest.class.getSimpleName()) + .set("spark.driver.bindAddress", "10.5.5.145") + .set("spark.network.timeout", "240000") + .set("spark.driver.host", "10.5.5.145") + .set("spark.deploy.mode", "client") + .set("spark.executor.memory", "4g") + .set("spark.cores.max", "4") + .set("spark.worker.cleanup.enabled", "true") + .set("spark.driver.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.executor.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.hadoop.fs.defaultFS", "hdfs://10.5.5.200:9000"); + + spark = SparkSession.builder() + .config(sparkConf) + .getOrCreate(); + + return spark; + } + + @BeforeAll + public static void beforeAll() { + + } + + @AfterAll + public static synchronized void afterAll() { + getSession().close(); + + } +} diff --git a/brutex-extended-tests/src/test/java/net/brutex/spark/BrianTest.java b/brutex-extended-tests/src/test/java/net/brutex/spark/BrianTest.java new file mode 100644 index 000000000..d3a7179f6 --- /dev/null +++ b/brutex-extended-tests/src/test/java/net/brutex/spark/BrianTest.java @@ -0,0 +1,342 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +package net.brutex.spark; + +import com.fasterxml.jackson.core.Version; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.filter.FilterInvalidValues; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.Writable; +import org.datavec.spark.transform.SparkTransformExecutor; +import org.datavec.spark.transform.misc.StringToWritablesFunction; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.spark.api.RDDTrainingApproach; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.datavec.DataVecDataSetFunction; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.deeplearning4j.ui.api.UIServer; +import org.junit.jupiter.api.*; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +/** + * Tests for new Spark Word2Vec implementation + * + * @author raver119@gmail.com + */ +@Slf4j +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Tag("integration") +public class BrianTest /*extends BaseDL4JTest*/ { + static { + String OS = System.getProperty("os.name").toLowerCase(); + + if (OS.contains("win")) { + System.setProperty("hadoop.home.dir", Paths.get("c:\\java\\winutils").toAbsolutePath().toString()); + } else { + System.setProperty("hadoop.home.dir", "/"); + } + } + + public long getTimeoutMilliseconds() { + return 400000L; + } + + private JavaSparkContext sc; + private JavaRDD rdd; + + /* + @BeforeAll + public void loadData() { + + + /* + sc.addFile("https://www.openml.org/data/get_csv/1595261/phpMawTba"); + org.apache.hadoop.fs.FileSystem hdfs = FileSystem.get( sc.hadoopConfiguration()); + try { + String file = SparkFiles.get("phpMawTba"); + Path target = new Path("/user/brian/" + "mydata.csv"); + //Apache Commons + FileUtils.copyFile(new File(file), hdfs.create(target)); + } catch (IOException e) { + e.printStackTrace(); + } + + + } +*/ + + @BeforeAll + public void setUp() throws Exception { + log.info("Running @BeforeEach scope"); + System.setProperty("hadoop.home.dir", Paths.get("c:\\java\\winutils").toAbsolutePath().toString()); + Version version = com.fasterxml.jackson.databind.cfg.PackageVersion.VERSION; + System.out.println("Jackson version found: " + version); + SparkConf sparkConf = new SparkConf() + .setMaster("spark://10.5.5.200:7077") + .setAppName("Brian3") + .set("spark.driver.bindAddress", "10.5.5.145") + .set("spark.network.timeout", "240000") + .set("spark.driver.host", "10.5.5.145") + .set("spark.driver.bindAddress", "10.5.5.145") + .set("spark.deploy.mode", "cluster") + .set("spark.executor.memory", "2g") + .set("spark.executor.cores", "2") + .set("spark.cores.max", "4") + .set("spark.worker.cleanup.enabled", "false") + .set("spark.driver.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.executor.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.driver.extraClassPath", "brian-spark-dl4j-tests-1.0.1-SNAPSHOT-bin.jar;brian-spark-dl4j-tests-1.0.1-SNAPSHOT-tests.jar") + .set("spark.executor.extraClassPath", "brian-spark-dl4j-tests-1.0.1-SNAPSHOT-bin.jar;brian-spark-dl4j-tests-1.0.1-SNAPSHOT-tests.jar") + .set("spark.hadoop.fs.defaultFS", "hdfs://10.5.5.200:9000"); + //.set("spark.driver.cores", "2") + //.set("spark.driver.memory", "8g") + //.set("spark.driver.host", "10.5.5.145") + //.setExecutorEnv("spark.executor.cores", "2") + //.setExecutorEnv("spark.executor.memory", "2g") + //.set("spark.submit.deployMode", "client") + ; + +/* + SparkSession spark = SparkSession + .builder() + .master("spark://10.5.5.200:7077") + .config("spark.driver.bindAddress", "10.5.5.145") + .config("spark.driver.host", "10.5.5.145") + //.config("spark.driver.memory", "5g") + .appName("BrianTest2") + .getOrCreate(); +*/ + sc = new JavaSparkContext(sparkConf); + + // sc.addJar("C:\\Users\\brian\\_projects\\deeplearning4j\\deeplearning4j\\deeplearning4j-scaleout\\spark\\dl4j-spark-nlp-java8\\target\\dl4j-spark-nlp-java8_2.12-1.0.0-SNAPSHOT-tests.jar"); + // sc.addJar("C:\\Users\\brian\\_projects\\deeplearning4j\\datavec\\datavec-api\\target\\datavec-api-1.0.0-SNAPSHOT.jar"); + // sc.addJar("C:\\Users\\brian\\_projects\\deeplearning4j\\nd4j\\nd4j-uberjar\\target\\nd4j-uberjar-1.0.0-SNAPSHOT.jar"); + // sc.addJar("C:\\Users\\brian\\_projects\\deeplearning4j\\nd4j\\nd4j-common\\target\\nd4j-common-1.0.0-SNAPSHOT.jar"); + // sc.addJar("C:\\Users\\brian\\_projects\\deeplearning4j\\datavec\\datavec-spark\\target\\datavec-spark_2.12-1.0.0-SNAPSHOT.jar"); + sc.addJar("C:\\Users\\brian\\_projects\\Brian-Spark-DL4J-Tests\\target\\brian-spark-dl4j-tests-1.0.1-SNAPSHOT-bin.jar"); + sc.addJar("C:\\Users\\brian\\_projects\\Brian-Spark-DL4J-Tests\\target\\brian-spark-dl4j-tests-1.0.1-SNAPSHOT-tests.jar"); + + + rdd = sc.textFile("hdfs://10.5.5.200:9000/user/zeppelin/cities_full.csv.gz"); + + + + + } + + @AfterAll + public void tearDown() throws Exception { + sc.close(); + sc.stop(); + UIServer.stopInstance(); + + } + + @Test + ////@Ignore("AB 2019/05/21 - Failing - Issue #7657") + public void testStringsTokenization1() throws Exception { + + //shrink for Test + //List list = Arrays.asList(new String[]{"asdsad", "asdasdasd", "asdasdasd", "3easdasd"}); + //JavaRDD rdd = sc.parallelize(list); + + // rdd = rdd.sample(true, 1.0, 1); + log.info("Datenmenge: " + rdd.count()); + log.info("Sample: " + rdd.top(3)); + + Assertions.assertEquals(146889, rdd.count()); + } + + @Test + public void testSchemaCreation() throws Exception { + + + rdd.cache(); + + JavaRDD cities = rdd.map( (Function) line -> { + return line.split(",")[1]; + }).cache(); + + JavaRDD stateCodeList = rdd.map( (Function) line -> { + return line.split(",")[2]; + }).cache(); + + JavaRDD countryCodeList = rdd.map( (Function) line -> { + return line.split(",")[3]; + }).cache(); + + + CSVRecordReader recordReader = new CSVRecordReader(0, ','); + JavaRDD> convertedRDD = rdd.map((Function>) s -> { + return new StringToWritablesFunction( recordReader).call(s); + }); + + //Source Schema + Schema inputSchema = new Schema.Builder() + .addColumnLong("city_id") + .addColumnsString("city_name", "state_code", "country_code") + .addColumnsString("country_full") + .addColumnsDouble("lat", "lon") + .build(); + + //Running Transformation + /* + TransformProcess tp = new TransformProcess.Builder(inputSchema) + .removeColumns("country_full", "lat", "lon") + .addConstantIntegerColumn("dummy_spalte", 1) + .stringToCategorical("state_code", stateCodeList.distinct().collect()) + .stringToCategorical("country_code", countryCodeList.distinct().collect()) + .stringToCategorical("city_name", cities.distinct().collect()) + .filter(new FilterInvalidValues()) + .categoricalToOneHot("city_name") + .categoricalToOneHot("state_code") + .categoricalToOneHot("country_code") + .build(); + */ + TransformProcess tp = new TransformProcess.Builder(inputSchema) + .removeAllColumnsExceptFor("country_code", "lat", "lon") + .stringToCategorical("country_code", Arrays.asList(new String[] {"GR", "FR", "DE", "CH"})) + .filter(new FilterInvalidValues()) + .categoricalToOneHot("country_code") + .build(); + + //log.info("Final Schema: " +tp.getFinalSchema().toString()); + //Execute Transformation Process + convertedRDD.repartition(8); + convertedRDD.cache(); + JavaRDD> processedData = SparkTransformExecutor.execute(convertedRDD, tp); + processedData.repartition(8); + processedData.cache(); + //log.info("Datenmenge nach processing: " + processedData.count()); + + + //Vectorisieren + int labelIndex = 0; //in welcher Spalte ist das Label + int numLabels = 4; //Anzahl der Klassen 0-236 = 237 Werte + + DataVecDataSetFunction datavecFunction = new DataVecDataSetFunction(labelIndex, numLabels, false); + JavaRDD rddDataSet = processedData.map(datavecFunction); + log.info("rddDataset: " + rddDataSet.toDebugString()); + Random rand = new Random(); + rddDataSet.sortBy( (Function) s -> {return rand.nextDouble(); }, true, 8); + + //og.info("Sample: " + rddDataSet.sample(false, 0.005, 0).collect()); + + /* Skip, this will save each record one by one to hdfs + */ + //Now save this hard work + /* + int miniBatchSize = 1; //Minibatch size of the saved DataSet objects + final String exportPath = "hdfs://10.5.5.200:9000/user/brian/data"; + JavaRDD paths = rddDataSet.mapPartitionsWithIndex( + new BatchAndExportDataSetsFunction(miniBatchSize, exportPath), + true) + ; + paths.collect(); + */ + + //Create Trainingmaster + + TrainingMaster trainingMaster = new ParameterAveragingTrainingMaster.Builder(4) + .rddTrainingApproach(RDDTrainingApproach.Direct) //when "export", tries to save everything first + .batchSizePerWorker(1000) + .collectTrainingStats(true) + .build(); + + //Define Network + + MultiLayerConfiguration multiLayerConfiguration = new NeuralNetConfiguration.Builder() + .seed(123) + .updater(new Nesterovs(0.1, 0.9)) + .list() + .layer(0, new DenseLayer.Builder().nIn(5).nOut(20).weightInit(WeightInit.XAVIER).activation(Activation.RELU).l2(0.001).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(20).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()) + //.layer(2, new DenseLayer.Builder().nIn(9).nOut(9).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(20).nOut(4).weightInit(WeightInit.XAVIER).activation(Activation.SIGMOID).build()) + .build(); + + //Define SparkNet + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, multiLayerConfiguration, trainingMaster); + + + JavaRDD[] split = rddDataSet.randomSplit(new double[] {0.9, 0.1}, 123); + //JavaRDD trainingData = split[0]; + JavaRDD trainingData = rddDataSet; + JavaRDD testData = split[1]; + + //Run Training on subset + for(int i =0; i<20; i++) { + sparkNet.fit(trainingData); + } + + //Evaluieren + MultiLayerNetwork finalNet = sparkNet.getNetwork(); + + //Speichern + Configuration conf = sc.hadoopConfiguration(); + conf.set("hadoop.tmp.dir", "/user/brian/tmp"); + FileSystem fs = FileSystem.get(conf); + Path p = new Path("hdfs://10.5.5.200:9000/user/brian/model"); + //fs.mkdirs(p); + //ModelSerializer.writeModel(finalNet, fs.create(p), true ); + + Evaluation eval = new Evaluation(4); // outputNum = 10: number of output classes + Iterator iter = testData.toLocalIterator(); + log.info("testData has " + testData.count() + " DataSets"); + while(iter.hasNext()){ + DataSet next = iter.next(); + //log.info("getFeatures " + next.getFeatures() ); + INDArray output = finalNet.output(next.getFeatures()); //get the networks prediction + //log.info("output "+ output.toStringFull()); + eval.eval(next.getLabels(), output); //check the prediction against the true class + //log.info("Predict " + finalNet.predict(next)); + } + log.info("Evaluation stats: " + eval.stats()); + } + +} diff --git a/brutex-extended-tests/src/test/java/net/brutex/spark/BrianTest2.java b/brutex-extended-tests/src/test/java/net/brutex/spark/BrianTest2.java new file mode 100644 index 000000000..436016352 --- /dev/null +++ b/brutex-extended-tests/src/test/java/net/brutex/spark/BrianTest2.java @@ -0,0 +1,348 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +package net.brutex.spark; + +import com.fasterxml.jackson.core.Version; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.sql.SparkSession; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.filter.FilterInvalidValues; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.Writable; +import org.datavec.spark.transform.SparkTransformExecutor; +import org.datavec.spark.transform.misc.StringToWritablesFunction; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.datavec.DataVecDataSetFunction; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingMaster; +import org.deeplearning4j.ui.api.UIServer; +import org.junit.jupiter.api.*; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; + +import java.io.File; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +/** + * Tests for new Spark Word2Vec implementation + * + * @author raver119@gmail.com + */ +@Slf4j +@Tag("integration") +public class BrianTest2 /*extends BaseDL4JTest*/ { + static { + String OS = System.getProperty("os.name").toLowerCase(); + + if (OS.contains("win")) { + System.setProperty("hadoop.home.dir", Paths.get("c:\\java\\winutils").toAbsolutePath().toString()); + } else { + System.setProperty("hadoop.home.dir", "/"); + } + } + + public long getTimeoutMilliseconds() { + return 400000L; + } + + private JavaSparkContext sc; + + + /* + @BeforeAll + public void loadData() { + + + /* + sc.addFile("https://www.openml.org/data/get_csv/1595261/phpMawTba"); + org.apache.hadoop.fs.FileSystem hdfs = FileSystem.get( sc.hadoopConfiguration()); + try { + String file = SparkFiles.get("phpMawTba"); + Path target = new Path("/user/brian/" + "mydata.csv"); + //Apache Commons + FileUtils.copyFile(new File(file), hdfs.create(target)); + } catch (IOException e) { + e.printStackTrace(); + } + + + } +*/ + + @BeforeEach + public void setUp() throws Exception { + log.info("Running @BeforeEach scope"); + System.setProperty("hadoop.home.dir", Paths.get("c:\\java\\winutils").toAbsolutePath().toString()); + Version version = com.fasterxml.jackson.databind.cfg.PackageVersion.VERSION; + System.out.println("Jackson version found: " + version); + System.out.println(System.getProperty("java.vm.name")+"\n"+System.getProperty("java.runtime.version")); + + SparkConf sparkConf = new SparkConf() + .setMaster("spark://10.5.5.200:7077") + .setAppName("Brian3") + .set("spark.driver.bindAddress", "10.5.5.145") + .set("spark.network.timeout", "240000") + .set("spark.driver.host", "10.5.5.145") + .set("spark.deploy.mode", "client") + .set("spark.executor.memory", "4g") + .set("spark.cores.max", "2") + .set("spark.worker.cleanup.enabled", "false") + .set("spark.driver.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.executor.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.hadoop.fs.defaultFS", "hdfs://10.5.5.200:9000"); + + SparkSession spark = SparkSession.builder() + .master("spark://10.5.5.200:7077") + .appName("BrianTest2") + .config(sparkConf) + .getOrCreate(); + + this.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + /* + Whatever is in classpath (driver), is added to the Spark Executors + */ + final String clpath = System.getProperty("java.class.path"); + log.info("java.class.path=\r\n{}\r\n", clpath); + final String separator = System.getProperty("path.separator"); + final String[] a = clpath.split(separator); + for(String s : a) { + File f = new File(s); + if(f.exists() && f.isFile() && s.endsWith(".jar")) { + log.info("Adding jar to SparkContext '{}'.", f.getName()); + this.sc.addJar(s); + } + } + } + + @AfterEach + public void tearDown() throws Exception { + if(sc!=null) this.sc.stop(); + UIServer.stopInstance(); + } + + @Test + public void testStringsTokenization1() throws Exception { + + final JavaRDD rdd = sc.textFile("hdfs://10.5.5.200:9000/user/zeppelin/cities_full.csv.gz"); + //shrink for Test + //List list = Arrays.asList(new String[]{"asdsad", "asdasdasd", "asdasdasd", "3easdasd"}); + //JavaRDD rdd = sc.parallelize(list); + + // rdd = rdd.sample(true, 1.0, 1); + log.info("Datenmenge: " + rdd.count()); + log.info("Sample: " + rdd.top(3)); + + Assertions.assertEquals(146889, rdd.count()); + } + + @Test + public void testSchemaCreation() throws Exception { + log.info(System.getProperty("java.vm.name")+"\n"+System.getProperty("java.runtime.version")); + final JavaRDD rdd = sc.textFile("hdfs://10.5.5.200:9000/user/zeppelin/cities_full.csv.gz"); + rdd.cache(); + + JavaRDD cities = rdd.map( (Function) line -> { + return line.split(",")[1]; + }).cache(); + + JavaRDD stateCodeList = rdd.map( (Function) line -> { + return line.split(",")[2]; + }).cache(); + + JavaRDD countryCodeList = rdd.map( (Function) line -> { + return line.split(",")[3]; + }).cache(); + + + CSVRecordReader recordReader = new CSVRecordReader(0, ','); + JavaRDD> convertedRDD = rdd.map((Function>) s -> { + return new StringToWritablesFunction( recordReader).call(s); + }); + + //Source Schema + Schema inputSchema = new Schema.Builder() + .addColumnLong("city_id") + .addColumnsString("city_name", "state_code", "country_code") + .addColumnsString("country_full") + .addColumnsDouble("lat", "lon") + .build(); + + //Running Transformation + /* + TransformProcess tp = new TransformProcess.Builder(inputSchema) + .removeColumns("country_full", "lat", "lon") + .addConstantIntegerColumn("dummy_spalte", 1) + .stringToCategorical("state_code", stateCodeList.distinct().collect()) + .stringToCategorical("country_code", countryCodeList.distinct().collect()) + .stringToCategorical("city_name", cities.distinct().collect()) + .filter(new FilterInvalidValues()) + .categoricalToOneHot("city_name") + .categoricalToOneHot("state_code") + .categoricalToOneHot("country_code") + .build(); + */ + TransformProcess tp = new TransformProcess.Builder(inputSchema) + .removeAllColumnsExceptFor("country_code", "lat", "lon") + .stringToCategorical("country_code", Arrays.asList(new String[] {"GR", "FR", "DE", "CH"})) + .filter(new FilterInvalidValues()) + .categoricalToOneHot("country_code") + .build(); + + //log.info("Final Schema: " +tp.getFinalSchema().toString()); + //Execute Transformation Process + //convertedRDD.repartition(1); + //convertedRDD.cache(); + JavaRDD> processedData = SparkTransformExecutor.execute(convertedRDD, tp); + //processedData.repartition(1); + //processedData.cache(); + //log.info("Datenmenge nach processing: " + processedData.count()); + + + //Vectorisieren + int labelIndex = 0; //in welcher Spalte ist das Label + int numLabels = 4; //Anzahl der Klassen 0-236 = 237 Werte + + DataVecDataSetFunction datavecFunction = new DataVecDataSetFunction(labelIndex, numLabels, false); + JavaRDD rddDataSet = processedData.map(datavecFunction); + log.info("rddDataset: " + rddDataSet.toDebugString()); + Random rand = new Random(); + rddDataSet.sortBy( (Function) s -> {return rand.nextDouble(); }, true, 8); + + //og.info("Sample: " + rddDataSet.sample(false, 0.005, 0).collect()); + + /* Skip, this will save each record one by one to hdfs + */ + //Now save this hard work + /* + int miniBatchSize = 1; //Minibatch size of the saved DataSet objects + final String exportPath = "hdfs://10.5.5.200:9000/user/brian/data"; + JavaRDD paths = rddDataSet.mapPartitionsWithIndex( + new BatchAndExportDataSetsFunction(miniBatchSize, exportPath), + true) + ; + paths.collect(); + */ + + + // Configure distributed training required for gradient sharing implementation + VoidConfiguration conf = VoidConfiguration.builder() + .unicastPort(40123) //Port that workers will use to communicate. Use any free port + //.networkMask("10.0.0.0/16") //Network mask for communication. Examples 10.0.0.0/24, or 192.168.0.0/16 etc + .controllerAddress("10.5.5.145") + .build(); + +//Create the TrainingMaster instance + TrainingMaster trainingMaster = new SharedTrainingMaster.Builder(conf, 1000) + .batchSizePerWorker(20000) //Batch size for training + .updatesThreshold(1e-3) //Update threshold for quantization/compression. See technical explanation page + .workersPerNode(1) // equal to number of GPUs. For CPUs: use 1; use > 1 for large core count + .exportDirectory("/user/brian/") + .build(); + + //Create Trainingmaster +/* + TrainingMaster trainingMaster = new ParameterAveragingTrainingMaster.Builder(4) + .rddTrainingApproach(RDDTrainingApproach.Direct) //when "export", tries to save everything first + .collectTrainingStats(false).build(); + */ + /* + TrainingMaster tm = new SharedTrainingMaster.Builder(voidConfiguration, minibatch) + .thresholdAlgorithm(new AdaptiveThresholdAlgorithm(this.gradientThreshold)) + .residualPostProcessor(new ResidualClippingPostProcessor(5, 5)) + .build(); +*/ + //Define Network + + MultiLayerConfiguration multiLayerConfiguration = new NeuralNetConfiguration.Builder() + .seed(123) + .updater(new Nesterovs(0.1, 0.9)) + .list() + .layer(0, new DenseLayer.Builder().nIn(5).nOut(20).weightInit(WeightInit.XAVIER).activation(Activation.RELU).l2(0.001).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(20).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()) + //.layer(2, new DenseLayer.Builder().nIn(9).nOut(9).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(20).nOut(4).weightInit(WeightInit.XAVIER).activation(Activation.SIGMOID).build()) + .build(); + + //Define SparkNet + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, multiLayerConfiguration, trainingMaster); + + + JavaRDD[] split = rddDataSet.randomSplit(new double[] {0.9, 0.1}, 123); + //JavaRDD trainingData = split[0]; + JavaRDD trainingData = rddDataSet; + JavaRDD testData = split[1]; + + //Run Training on subset + for(int i =0; i<4; i++) { + sparkNet.fit(trainingData); + } + + //Evaluieren + MultiLayerNetwork finalNet = sparkNet.getNetwork(); + + //Speichern + Configuration hconf = sc.hadoopConfiguration(); + hconf.set("hadoop.tmp.dir", "/user/brian/tmp"); + FileSystem fs = FileSystem.get(hconf); + Path p = new Path("hdfs://10.5.5.200:9000/user/brian/model"); + //fs.mkdirs(p); + //ModelSerializer.writeModel(finalNet, fs.create(p), true ); + + Evaluation eval = new Evaluation(4); // outputNum = 10: number of output classes + Iterator iter = testData.toLocalIterator(); + log.info("testData has " + testData.count() + " DataSets"); + while(iter.hasNext()){ + DataSet next = iter.next(); + //log.info("getFeatures " + next.getFeatures() ); + INDArray output = finalNet.output(next.getFeatures()); //get the networks prediction + //log.info("output "+ output.toStringFull()); + eval.eval(next.getLabels(), output); //check the prediction against the true class + //log.info("Predict " + finalNet.predict(next)); + } + log.info("Evaluation stats: " + eval.stats()); + + } + +} diff --git a/brutex-extended-tests/src/test/java/net/brutex/spark/LoadBackendTest.java b/brutex-extended-tests/src/test/java/net/brutex/spark/LoadBackendTest.java new file mode 100644 index 000000000..d09894ae1 --- /dev/null +++ b/brutex-extended-tests/src/test/java/net/brutex/spark/LoadBackendTest.java @@ -0,0 +1,35 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package net.brutex.spark; + +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.factory.Nd4j; + +public class LoadBackendTest extends BaseSparkSessionTest { + + @Test + public void loadBackend() { + SparkSession spark = getSession(); + Nd4j.create(1, 0); + } +} diff --git a/brutex-extended-tests/src/test/java/net/brutex/spark/TestServer.java b/brutex-extended-tests/src/test/java/net/brutex/spark/TestServer.java new file mode 100644 index 000000000..353195da4 --- /dev/null +++ b/brutex-extended-tests/src/test/java/net/brutex/spark/TestServer.java @@ -0,0 +1,243 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package net.brutex.spark; + +import lombok.extern.log4j.Log4j2; +//import net.brutex.ai.performance.storage.PostgresStatsStorage; +import lombok.extern.slf4j.Slf4j; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.collection.ListStringRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.ListStringSplit; +import org.deeplearning4j.core.storage.StatsStorage; +import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.LSTM; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.ui.api.UIServer; +import org.deeplearning4j.ui.model.stats.StatsListener; +import org.deeplearning4j.ui.model.storage.FileStatsStorage; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.SplitTestAndTrain; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.impl.LossMCXENT; + +import java.io.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +@Slf4j +@Tag("integration") +public class TestServer { + + @AfterAll + public static void tidyUp() throws Exception { + UIServer.stopInstance(); + } + + @Test + public void runServer() throws InterruptedException, IOException { + log.info("Using backend: " + Nd4j.getBackend()); + UIServer ui = UIServer.getInstance(); + log.info("Port:" + ui.getPort()); + + //Get our network and training data + //MultiLayerNetwork net = UIExampleUtils.getMnistNetwork(); + //DataSetIterator trainData = UIExampleUtils.getMnistData(); + int i = 2000; + int numClasses = 10; + int numBatchSize = 100; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(1234) + .weightInit(WeightInit.XAVIER) + .updater(new Nesterovs.Builder().learningRate(0.15).build()) + .activation(Activation.RELU) + .l2(0) + .list() + //.layer(0, new ConvolutionLayer.Builder().nIn(1).kernelSize(1, 5).stride(1,1).padding(0,2).nOut(1).name("1st Filter").updater(new Adam.Builder().learningRate(0.2).build()).build()) + //.layer(1, new ConvolutionLayer.Builder().nIn(1).kernelSize(1, 2).stride(1,2).padding(0,0).nOut(1).name("2nd Filter").updater(new Adam.Builder().learningRate(0.1).build()).build()) + // .layer(1, new DenseLayer.Builder().nIn(10).nOut(64).activation(Activation.RELU).build()) + .layer(0, new DenseLayer.Builder().nIn(10).nOut(100).activation(Activation.RELU).l2(0.003).build()) + .layer(1, new LSTM.Builder().nIn(100).nOut(100).activation(Activation.TANH).build()) + .layer(2, new LSTM.Builder().nIn(100).nOut(100).activation(Activation.TANH).build()) + .layer(3, new DenseLayer.Builder().nIn(100).nOut(16).activation(Activation.RELU).l2(0.001).build()) + + .layer(4, new OutputLayer.Builder().nIn(16).nOut(numClasses) + .activation(Activation.SOFTMAX) + .lossFunction(new LossMCXENT()) + .build() + ) + //.inputPreProcessor(0, new FeedForwardToCnnPreProcessor(1,10, 1)) + //.inputPreProcessor(2, new CnnToFeedForwardPreProcessor()) + .inputPreProcessor(1, new FeedForwardToRnnPreProcessor()) + .inputPreProcessor(3, new RnnToFeedForwardPreProcessor()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + + RecordReader trainrecords = new CSVRecordReader(0, ';'); + File dataFile = new File("c://temp/werte2-medium.csv"); + + trainrecords.initialize(new FileSplit(dataFile)); + DataSetIterator iter = new RecordReaderDataSetIterator.Builder(trainrecords, numBatchSize) + .classification(10, numClasses) + .build() + ; + + + List featuresTrain = new ArrayList(); + List labelsTrain = new ArrayList(); + List featuresTest = new ArrayList(); + List labelsTest = new ArrayList(); + List rawLabels = new ArrayList(); + List rawTrainLabels = new ArrayList(); + + INDArray indexes = null; + + + + while(iter.hasNext()) { + DataSet next = iter.next(); + SplitTestAndTrain split = next.splitTestAndTrain(0.9); + DataSet dsTest = split.getTest(); + DataSet dsTrain = split.getTrain(); + + + DataNormalization normalizer = new NormalizerStandardize(); + normalizer.fit(dsTrain); + normalizer.transform(dsTrain); + normalizer.transform(dsTest); + + featuresTrain.add(dsTrain.getFeatures()); + labelsTrain.add(dsTrain.getLabels()); + rawTrainLabels.add(dsTrain.getLabels()); + + + + featuresTest.add(dsTest.getFeatures()); + rawLabels.add(dsTest.getLabels()); + indexes = Nd4j.argMax(dsTest.getLabels(),1); + labelsTest.add(indexes); + } + + + //Configure where the network information (gradients, activations, score vs. time etc) is to be stored + //Then add the StatsListener to collect this information from the network, as it trains + File logFile = new File("c://temp/", "ui-stats-brian.dl4j"); + logFile.delete(); + //StatsStorage statsStorage = new FileStatsStorage(logFile); + //PostgresStatsStorage psqlStore = new PostgresStatsStorage(); + int listenerFrequency = 2; + //net.setListeners(new StatsListener(psqlStore, listenerFrequency), new StatsListener(statsStorage, listenerFrequency), new ScoreIterationListener(200)); + //net.setListeners(new StatsListener(statsStorage, listenerFrequency), new ScoreIterationListener(200)); + + + //Attach the StatsStorage instance to the UI: this allows the contents of the StatsStorage to be visualized + //ui.attach(statsStorage); + + Iterator labelsTrainIterator = labelsTrain.iterator(); + + + File deb = new File("c:\\temp\\debug.txt"); + OutputStream out = new BufferedOutputStream(new FileOutputStream(deb)); + + while(i>0) { + for(INDArray a : featuresTrain) { + net.fit(a, labelsTrainIterator.next()); + } + labelsTrainIterator = labelsTrain.iterator(); + i--; + + //Play Visualisation + /* + NDArrayStrings fm = new NDArrayStrings(" | "); + Nd4j.writeTxt(net.getLayer(1).getGradientsViewArray(),"c:/temp/dump"+i+".txt"); + out.write(fm.format(net.getLayer(1).getGradientsViewArray(), false).getBytes(StandardCharsets.UTF_8)); + + out.write(10); + out.write(13); + + out.write(net.getLayer(1).toString().getBytes(StandardCharsets.UTF_8)); + + out.write(10); + out.write(13); +*/ + } + out.close(); + + + //Thread.sleep(60000); + + List tt = new ArrayList<>(); + tt.addAll(Arrays.asList("1", "2", "3", "3", "5", "6", "7", "8", "9","5")); + List> ttt = new ArrayList(); + ttt.add(tt); + + RecordReader rr = new ListStringRecordReader(); + rr.initialize(new ListStringSplit(ttt)); + DataSetIterator dataIter = new RecordReaderDataSetIterator(rr, 1); + org.nd4j.linalg.dataset.DataSet set = dataIter.next(); + log.info( "Brian out:" + net.score(set)); + log.info( "Brian out:" + net.f1Score(set)); + + log.info("============================== Training Data ======================================="); + runEval(numClasses, featuresTrain, rawTrainLabels, net); + log.info("===================================================================================="); + log.info("============================== Test Data ======================================="); + runEval(numClasses, featuresTest, rawLabels, net); + log.info("===================================================================================="); + + } + + void runEval(int numClasses, List trainingData, List trainingLabels, MultiLayerNetwork network) { + + Evaluation eval = new Evaluation(numClasses); + + Iterator testIterator = trainingData.iterator(); + Iterator labelsIterator = trainingLabels.iterator(); + while(testIterator.hasNext()) { + INDArray output = network.output(testIterator.next()); + eval.eval(labelsIterator.next(), output); + } + log.info(eval.stats()); + } +} diff --git a/brutex-extended-tests/src/test/java/net/brutex/spark/TestServer2.java b/brutex-extended-tests/src/test/java/net/brutex/spark/TestServer2.java new file mode 100644 index 000000000..d6ac22e11 --- /dev/null +++ b/brutex-extended-tests/src/test/java/net/brutex/spark/TestServer2.java @@ -0,0 +1,282 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package net.brutex.spark; + +import lombok.extern.log4j.Log4j2; +//import net.brutex.ai.performance.storage.PostgresStatsStorage; +import lombok.extern.slf4j.Slf4j; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.collection.ListStringRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.ListStringSplit; +import org.datavec.image.recordreader.ImageRecordReader; +import org.deeplearning4j.core.storage.StatsStorage; +import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.LSTM; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.ui.api.UIServer; +import org.deeplearning4j.ui.model.stats.StatsListener; +import org.deeplearning4j.ui.model.storage.FileStatsStorage; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.SplitTestAndTrain; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization; +import org.nd4j.linalg.dataset.api.preprocessor.ImagePreProcessingScaler; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.impl.LossMCXENT; + +import java.io.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +@Slf4j +@Tag("integration") +public class TestServer2 { + + @AfterAll + public static void tidyUp() throws Exception { + UIServer.stopInstance(); + } + + @Test + public void runServer() throws InterruptedException, IOException { + +/* + for(int page=1; page<=10;page++) { + Connection xx = Jsoup.connect("https://www.ebay.de/b/Bier-Bierdeckel-fur-Sammler/8734/bn_16579776?rt=nc&_dmd=1&_pgn=" + page); + Elements xxx = xx.get().body().select(".s-item__image-img"); + File datafile = new File("c:\\temp\\img_dump.csv"); + + int ifile = 0; + for (Element e : xxx) { + log.info(e.toString()); + String imgurl = e.attr("src"); + if (!imgurl.endsWith(".jpg")) { + imgurl = e.attr("data-src"); + } + Connection.Response res = Jsoup.connect(imgurl).ignoreContentType(true).execute(); + FileOutputStream out = new FileOutputStream(new File("c:\\temp\\imgdump\\" + page+ ifile + ".jpg")); + out.write(res.bodyAsBytes()); + out.close(); + FileUtils.writeStringToFile(datafile, e.attr("alt").toLowerCase().replace(";", "") + ";" +page+ ifile + ".jpg" + "\r\n", Charset.defaultCharset(), true); + ifile++; + } + } +*/ + RecordReader rrr = new ImageRecordReader(32,32,3); + + rrr.initialize(new FileSplit(new File("c:\\temp\\imgdump\\"))); + + DataSetIterator diter = new RecordReaderDataSetIterator.Builder(rrr,12) + .classification(1, 3) + .preProcessor( new ImagePreProcessingScaler()) + .build(); + + + + log.info("Using backend: " + Nd4j.getBackend()); + UIServer ui = UIServer.getInstance(); + log.info("Port:" + ui.getPort()); + + //Get our network and training data + //MultiLayerNetwork net = UIExampleUtils.getMnistNetwork(); + //DataSetIterator trainData = UIExampleUtils.getMnistData(); + int i = 2000; + int numClasses = 10; + int numBatchSize = 100; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(1234) + .weightInit(WeightInit.XAVIER) + .updater(new Nesterovs.Builder().learningRate(0.15).build()) + .activation(Activation.RELU) + .l2(0) + .list() + //.layer(0, new ConvolutionLayer.Builder().nIn(1).kernelSize(1, 5).stride(1,1).padding(0,2).nOut(1).name("1st Filter").updater(new Adam.Builder().learningRate(0.2).build()).build()) + //.layer(1, new ConvolutionLayer.Builder().nIn(1).kernelSize(1, 2).stride(1,2).padding(0,0).nOut(1).name("2nd Filter").updater(new Adam.Builder().learningRate(0.1).build()).build()) + // .layer(1, new DenseLayer.Builder().nIn(10).nOut(64).activation(Activation.RELU).build()) + .layer(0, new DenseLayer.Builder().nIn(10).nOut(100).activation(Activation.RELU).l2(0.003).build()) + .layer(1, new LSTM.Builder().nIn(100).nOut(100).activation(Activation.TANH).build()) + .layer(2, new LSTM.Builder().nIn(100).nOut(100).activation(Activation.TANH).build()) + .layer(3, new DenseLayer.Builder().nIn(100).nOut(16).activation(Activation.RELU).l2(0.001).build()) + + .layer(4, new OutputLayer.Builder().nIn(16).nOut(numClasses) + .activation(Activation.SOFTMAX) + .lossFunction(new LossMCXENT()) + .build() + ) + //.inputPreProcessor(0, new FeedForwardToCnnPreProcessor(1,10, 1)) + //.inputPreProcessor(2, new CnnToFeedForwardPreProcessor()) + .inputPreProcessor(1, new FeedForwardToRnnPreProcessor()) + .inputPreProcessor(3, new RnnToFeedForwardPreProcessor()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + + RecordReader trainrecords = new CSVRecordReader(0, ';'); + File dataFile = new File("c://temp/werte2-medium.csv"); + + trainrecords.initialize(new FileSplit(dataFile)); + /* + DataSetIterator iter = new RecordReaderDataSetIterator.Builder(trainrecords, numBatchSize) + .classification(10, numClasses) + .build() + ; + + */ + + + List featuresTrain = new ArrayList(); + List labelsTrain = new ArrayList(); + List featuresTest = new ArrayList(); + List labelsTest = new ArrayList(); + List rawLabels = new ArrayList(); + List rawTrainLabels = new ArrayList(); + + INDArray indexes = null; + + + + while(diter.hasNext()) { + DataSet next = diter.next(); + SplitTestAndTrain split = next.splitTestAndTrain(0.9); + DataSet dsTest = split.getTest(); + DataSet dsTrain = split.getTrain(); + + + DataNormalization normalizer = new NormalizerStandardize(); + normalizer.fit(dsTrain); + normalizer.transform(dsTrain); + normalizer.transform(dsTest); + + featuresTrain.add(dsTrain.getFeatures()); + labelsTrain.add(dsTrain.getLabels()); + rawTrainLabels.add(dsTrain.getLabels()); + + + + featuresTest.add(dsTest.getFeatures()); + rawLabels.add(dsTest.getLabels()); + indexes = Nd4j.argMax(dsTest.getLabels(),1); + labelsTest.add(indexes); + } + + + //Configure where the network information (gradients, activations, score vs. time etc) is to be stored + //Then add the StatsListener to collect this information from the network, as it trains + File logFile = new File("c://temp/", "ui-stats-brian.dl4j"); + logFile.delete(); + StatsStorage statsStorage = new FileStatsStorage(logFile); + //PostgresStatsStorage psqlStore = new PostgresStatsStorage(); + int listenerFrequency = 2; + //net.setListeners(new StatsListener(psqlStore, listenerFrequency), new StatsListener(statsStorage, listenerFrequency), new ScoreIterationListener(200)); + net.setListeners(new StatsListener(statsStorage, listenerFrequency), new ScoreIterationListener(200)); + + + //Attach the StatsStorage instance to the UI: this allows the contents of the StatsStorage to be visualized + ui.attach(statsStorage); + + Iterator labelsTrainIterator = labelsTrain.iterator(); + + + File deb = new File("c:\\temp\\debug.txt"); + OutputStream out = new BufferedOutputStream(new FileOutputStream(deb)); + + while(i>0) { + for(INDArray a : featuresTrain) { + net.fit(a, labelsTrainIterator.next()); + } + labelsTrainIterator = labelsTrain.iterator(); + i--; + + //Play Visualisation + /* + NDArrayStrings fm = new NDArrayStrings(" | "); + Nd4j.writeTxt(net.getLayer(1).getGradientsViewArray(),"c:/temp/dump"+i+".txt"); + out.write(fm.format(net.getLayer(1).getGradientsViewArray(), false).getBytes(StandardCharsets.UTF_8)); + + out.write(10); + out.write(13); + + out.write(net.getLayer(1).toString().getBytes(StandardCharsets.UTF_8)); + + out.write(10); + out.write(13); +*/ + } + out.close(); + + + //Thread.sleep(60000); + + List tt = new ArrayList<>(); + tt.addAll(Arrays.asList("1", "2", "3", "3", "5", "6", "7", "8", "9","5")); + List> ttt = new ArrayList(); + ttt.add(tt); + + RecordReader rr = new ListStringRecordReader(); + rr.initialize(new ListStringSplit(ttt)); + DataSetIterator dataIter = new RecordReaderDataSetIterator(rr, 1); + org.nd4j.linalg.dataset.DataSet set = dataIter.next(); + log.info( "Brian out:" + net.score(set)); + log.info( "Brian out:" + net.f1Score(set)); + + log.info("============================== Training Data ======================================="); + runEval(numClasses, featuresTrain, rawTrainLabels, net); + log.info("===================================================================================="); + log.info("============================== Test Data ======================================="); + runEval(numClasses, featuresTest, rawLabels, net); + log.info("===================================================================================="); + + } + + void runEval(int numClasses, List trainingData, List trainingLabels, MultiLayerNetwork network) { + + Evaluation eval = new Evaluation(numClasses); + + Iterator testIterator = trainingData.iterator(); + Iterator labelsIterator = trainingLabels.iterator(); + while(testIterator.hasNext()) { + INDArray output = network.output(testIterator.next()); + eval.eval(labelsIterator.next(), output); + } + log.info(eval.stats()); + } +} diff --git a/build.gradle b/build.gradle new file mode 100644 index 000000000..f0c49da06 --- /dev/null +++ b/build.gradle @@ -0,0 +1,140 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +buildscript { + repositories { + mavenCentral() + } + dependencies { + classpath "com.vanniktech:gradle-dependency-graph-generator-plugin:0.6.0" + classpath 'com.google.gradle:osdetector-gradle-plugin:1.7.0' + } +} +apply plugin: "com.vanniktech.dependency.graph.generator" +apply plugin: 'com.google.osdetector' + +ext { + buildTarget = (properties.CAVIS_TARGET ?: osdetector.classifier).toLowerCase() //if not defined otherwise, we build target is the same as build host + logger.quiet("Building host platform is '{}' and build target(s) are '{}'", osdetector.classifier, buildTarget) + buildSupportMatrix = [[host: "windows-x86_64", + canBuild: ["windows-x86_64", + "windows-x86"] + ], + [host: "linux-x86_64", + canBuild: ["linux-x86_64", "linux-arm64"] + ]] + logger.quiet("Print {}", buildSupportMatrix) + + scalaVersion = "2.12" + logger.quiet("Scala main version is set to {}", scalaVersion) +} + +configurations.all { + resolutionStrategy { + // fail eagerly on version conflict (includes transitive dependencies) + // e.g. multiple different versions of the same dependency (group and name are equal) + failOnVersionConflict() + } +} + + +allprojects { Project proj -> + apply plugin: 'com.google.osdetector' + + version = "1.0.0-SNAPSHOT" + group = "net.brutex.cavis" + + + plugins.withType(JavaPlugin) { + dependencies { + + implementation platform(project(":cavis-common-platform")) + compileOnly platform(project(":cavis-common-platform")) + annotationProcessor platform(project(":cavis-common-platform")) + testCompileOnly platform(project(":cavis-common-platform")) + testAnnotationProcessor platform(project(":cavis-common-platform")) + testImplementation platform(project(":cavis-common-platform")) + + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + testCompileOnly 'org.projectlombok:lombok' + testAnnotationProcessor 'org.projectlombok:lombok' + testImplementation 'org.junit.jupiter:junit-jupiter-engine' + testImplementation 'org.junit.jupiter:junit-jupiter-api' + + } + test { + useJUnitPlatform { + if( project.hasProperty("includeTags") ) { + it.includeTags=project.getProperty("includeTags").split(",") + } + if( project.hasProperty("excludeTags") ) { + it.excludeTags=project.getProperty("excludeTags").split(",") + } + } + ignoreFailures = true + testLogging { + events "PASSED", "SKIPPED", "FAILED", "STANDARD_OUT", "STANDARD_ERROR" + } + } + } + + plugins.withType(MavenPublishPlugin) { + publishing { + publications { + mavenJava(MavenPublication) { + /* Need to verify the property exists, as some + modules may not declare it (i.e. the java-platform plugin) + */ + if (components.hasProperty("java") && !proj.name.equals("cavis-native-lib")) { + from components.java + } + } + } + repositories { + + maven { + name = 'LocalRemote' + def releasesRepoUrl = 'https://archiva.brutex.net/repository/internal/' + def snapshotsRepoUrl = 'https://archiva.brutex.net/repository/snapshots/' + url = proj.version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + allowInsecureProtocol = false + credentials { + username = mavenuser + password = mavenpass + } + } + /* + maven { + name = 'OSSRH' + def releasesRepoUrl = 'https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/' + def snapshotsRepoUrl = 'https://s01.oss.sonatype.org/content/repositories/snapshots/' + url = proj.version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + credentials { + username = ossrhUsername + password = ossrhPassword + } + } + + */ + } + } + } +} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 000000000..c2f90c670 --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,33 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +plugins { + id 'groovy' +} + + repositories { + mavenCentral() + } + + dependencies { + + //implementation "org.bytedeco:gradle-javacpp:1.5.5" + } diff --git a/cavis-common-platform/build.gradle b/cavis-common-platform/build.gradle new file mode 100644 index 000000000..7df8e9da7 --- /dev/null +++ b/cavis-common-platform/build.gradle @@ -0,0 +1,170 @@ +plugins { + id 'java-platform' + id 'maven-publish' +} + +ext { + scalaVersion = rootProject.ext.scalaVersion +} + + def javacpp = [version: "1.5.6", presetsVersion: "1.5.6"] + def hdf5 = [version: "1.12.1"] + def jackson = [version: "2.10.5.20201202"] + def cuda = [version: "11.4"] + def cudnn = [version: "8.2"] + def openblas = [version: "0.3.17"] + + def javacv = [version:"1.5.6"] + def opencv = [version: "4.5.3"] + def leptonica = [version: "1.81.1"] + def junit = [version: "5.7.1"] + + def flatbuffers = [version: "1.10.0"] + + def spark = [version: "3.1.2"] + def scala = [version:"2.12.10"] //[version:"2.13.5"] + + def netty = [version: "4.1.68.Final"] + + +javaPlatform { + allowDependencies() +} + +dependencies { + + api enforcedPlatform("io.netty:netty-bom:${netty.version}") + api enforcedPlatform("com.fasterxml.jackson:jackson-bom:${jackson.version}") + + + constraints { + + api enforcedPlatform("io.netty:netty-bom:${netty.version}") + api enforcedPlatform("com.fasterxml.jackson:jackson-bom:${jackson.version}") + + api 'com.google.guava:guava:30.1-jre' + api "com.google.protobuf:protobuf-java:3.15.6" + api "com.google.code.gson:gson:2.8.6" + api "com.google.protobuf:protobuf-java-util:3.15.6" + api "com.google.flatbuffers:flatbuffers-java:${flatbuffers.version}" + api "com.google.flatbuffers:flatbuffers-java:${flatbuffers.version}" + + /* + api "com.fasterxml.jackson.core:jackson-core:${jackson.version}" + api "com.fasterxml.jackson.core:jackson-databind:${jackson.version}" + api "com.fasterxml.jackson.core:jackson-annotations:${jackson.version}" + + api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${jackson.version}" + */ + // api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jackson.version}" + // api "com.fasterxml.jackson.datatype:jackson-datatype-joda:${jackson.version}" + // api "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}" + + + api "org.projectlombok:lombok:1.18.24" + + /*Logging*/ + api 'org.slf4j:slf4j-api:1.7.30' + + api "org.apache.logging.log4j:log4j-core:2.17.0" + api "ch.qos.logback:logback-classic:1.2.3" + api 'ch.qos.logback:logback-core:1.2.3' + + + api 'commons-io:commons-io:2.5' + api 'commons-codec:commons-codec:1.11' + api 'commons-net:commons-net:3.6' + api 'commons-collections:commons-collections:3.2.2' + + api 'org.apache.commons:commons-math3:3.6.1' + api 'org.apache.commons:commons-lang3:3.9' + api 'org.apache.commons:commons-compress:1.19' + api 'org.apache.commons:commons-collections4:4.1' + api "joda-time:joda-time:2.2" + api "org.reflections:reflections:0.9.10" + api 'org.springframework:spring-core:5.0.2.RELEASE' + + api "org.junit.jupiter:junit-jupiter-api:${junit.version}" + api "org.junit.jupiter:junit-jupiter-engine:${junit.version}" + api "org.junit.jupiter:junit-jupiter-params:${junit.version}" + + + api 'com.jakewharton.byteunits:byteunits:0.9.1' + api 'net.ericaro:neoitertools:1.0.0' + api 'com.github.oshi:oshi-core:3.4.2' + api 'com.github.oshi:oshi-json:3.4.2' + + + api 'com.github.jai-imageio:jai-imageio-core:1.3.0' + api 'com.twelvemonkeys.imageio:imageio-jpeg:3.1.1' + api 'com.twelvemonkeys.imageio:imageio-tiff:3.1.1' + api 'com.twelvemonkeys.imageio:imageio-psd:3.1.1' + api 'com.twelvemonkeys.imageio:imageio-bmp:3.1.1' + + api('com.google.android:android:4.1.1.4') + + api "org.bytedeco:javacpp:${javacpp.version}" + api "org.bytedeco:javacv:${javacv.version}" + api "org.bytedeco:opencv:${opencv.version}-${javacpp.presetsVersion}" + api "org.bytedeco:openblas:${openblas.version}-${javacpp.presetsVersion}" + api "org.bytedeco:leptonica-platform:${leptonica.version}-${javacpp.presetsVersion}" + api "org.bytedeco:hdf5-platform:${hdf5.version}-${javacpp.presetsVersion}" + api "org.bytedeco:hdf5:${hdf5.version}-${javacpp.presetsVersion}" + api "org.bytedeco:hdf5:${hdf5.version}-${javacpp.presetsVersion}:windows-x86_64" + api "org.bytedeco:hdf5:${hdf5.version}-${javacpp.presetsVersion}:linux-x86_64" + + + + api "org.bytedeco:cuda:${cuda.version}-${cudnn.version}-${javacpp.presetsVersion}" + api "org.bytedeco:cuda-platform-redist:${cuda.version}-${cudnn.version}-${javacpp.presetsVersion}" + api "org.bytedeco:mkl-dnn:0.21.5-${javacpp.presetsVersion}" + api "org.bytedeco:tensorflow:1.15.5-${javacpp.presetsVersion}" + api "org.bytedeco:cpython:3.9.6-${javacpp.presetsVersion}" + api "org.bytedeco:numpy:1.21.1-${javacpp.presetsVersion}" + + /* Apache Spark */ + api "org.apache.spark:spark-core_${scalaVersion}:${spark.version}" + api "org.apache.spark:spark-mllib_${scalaVersion}:${spark.version}" + api "org.apache.spark:spark-sql_${scalaVersion}:${spark.version}" + + api "org.apache.hadoop:hadoop-client:3.2.0" + + api("org.scala-lang:scala-library:${scala.version}") { + version { + strictly "${scala.version}" + because("Scala versions need to match, it is a mess otherwise.") + } + } + api("org.scala-lang:scala-reflect:${scala.version}") { + version { + strictly "${scala.version}" + because( "Scala versions need to match, it is a mess otherwise.") + } + } + api("org.scala-lang:scala-compiler:${scala.version}") { + version { + strictly "${scala.version}" + because( "Scala versions need to match, it is a mess otherwise.") + } + } + + api "org.agrona:agrona:1.12.0" + + } +} + +publishing { + publications { + myPlatform(MavenPublication) { + from components.javaPlatform + } + } +} + + +tasks.withType(GenerateModuleMetadata).configureEach { + // The value 'enforced-platform' is provided in the validation + // error message you got + suppressedValidationErrors.add('enforced-platform') +} + diff --git a/datavec/README.md b/cavis-datavec/README.md similarity index 100% rename from datavec/README.md rename to cavis-datavec/README.md diff --git a/cavis-datavec/build.gradle b/cavis-datavec/build.gradle new file mode 100644 index 000000000..5684a1d1d --- /dev/null +++ b/cavis-datavec/build.gradle @@ -0,0 +1,10 @@ +subprojects { + group = "net.brutex.cavis-datavec" + + apply plugin: "java-library" + apply plugin: "maven-publish" + apply plugin: "signing" + + + +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-api/build.gradle b/cavis-datavec/cavis-datavec-api/build.gradle new file mode 100644 index 000000000..ecad3eda0 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/build.gradle @@ -0,0 +1,120 @@ +plugins { + id 'java-library' + id 'maven-publish' + id 'signing' + id 'idea' +} + +ext { + buildTarget = rootProject.ext.buildTarget +} + +idea { + module { + downloadJavadoc = true // defaults to false + downloadSources = true + } +} + +apply from: "../../chooseBackend.gradle" + +chipList.each { thisChip -> + configurations.register("${thisChip}TestImplementation") { + it.extendsFrom configurations.testImplementation + it.extendsFrom configurations.implementation + } + configurations.register("${thisChip}TestRuntime") { + it.extendsFrom configurations.testRuntimeOnly + it.extendsFrom configurations.api + it.extendsFrom configurations.implementation + it.extendsFrom configurations.testImplementation + } + + tasks.register("${thisChip}Test", Test) { + it.testClassesDirs = sourceSets.test.output.classesDirs + it.useJUnitPlatform() + it.classpath = configurations.getByName("${thisChip}TestRuntime") + it.classpath += sourceSets.test.output.classesDirs + it.classpath += sourceSets.main.output.classesDirs + it.ignoreFailures = true + it.testLogging { + events "PASSED", "SKIPPED", "FAILED", "STANDARD_OUT", "STANDARD_ERROR" + } + //it.jvmArgs("-Dorg.bytedeco.javacpp.logger.debug=true") + + // it.debug = true + } + + tasks.test.dependsOn "${thisChip}Test" +} + +test { + enabled = false +} + +dependencies { + testImplementation 'org.junit.jupiter:junit-jupiter-params' + testRuntimeOnly project(":cavis-native:cavis-native-blas") + testRuntimeOnly project(":cavis-nd4j:cavis-nd4j-common-tests") + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" + + + if(withCuda()) { + cudaTestRuntime platform(project(":cavis-common-platform")) + cudaTestRuntime project(":cavis-native:cavis-native-jcublas") + cudaTestRuntime group: "org.bytedeco", name: "openblas" + cudaTestRuntime group: "org.bytedeco", name: "openblas", classifier: buildTarget + cudaTestRuntime "org.bytedeco:cuda" + cudaTestRuntime (project(":cavis-native:cavis-native-lib")) { + capabilities{ + it.requireCapabilities "net.brutex.cavis-native:cavis-native-lib-cuda-support:1.0.0-SNAPSHOT" + } + } + } + + if(withCpu()) { + cpuTestRuntime project(":cavis-native:cavis-native-cpu") + cpuTestRuntime group: "org.bytedeco", name: "openblas" + cpuTestRuntime group: "org.bytedeco", name: "openblas", classifier: buildTarget + cpuTestRuntime (project(":cavis-native:cavis-native-lib")) { + capabilities{ + it.requireCapabilities "net.brutex.cavis-native:cavis-native-lib-cpu-support:1.0.0-SNAPSHOT" + } + } + } + + + + implementation platform(project(':cavis-common-platform')) + + implementation project(':cavis-dnn:cavis-dnn-common') + implementation project(':cavis-dnn:cavis-dnn-api') + implementation project(":cavis-nd4j:cavis-nd4j-common-tests") + + implementation 'org.apache.commons:commons-lang3' + implementation 'commons-io:commons-io' + implementation "commons-codec:commons-codec" + implementation 'org.slf4j:slf4j-api' + implementation 'joda-time:joda-time' + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" + + implementation "com.google.guava:guava" + implementation "org.freemarker:freemarker:2.3.23" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.clearspring.analytics:stream:2.9.8" + implementation "net.sf.opencsv:opencsv:2.3" + implementation "com.tdunning:t-digest:3.2" + implementation "it.unimi.dsi:fastutil:8.1.1" + testImplementation 'com.tngtech.archunit:archunit-junit5-engine:0.17.0' + implementation "com.fasterxml.jackson.datatype:jackson-datatype-joda" + testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml" + testImplementation 'org.hamcrest:hamcrest-api:1.0' + testImplementation 'org.hamcrest:hamcrest-core:1.3' + + + + implementation 'org.bytedeco:javacpp' + +} + diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/conf/Configurable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/conf/Configurable.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/conf/Configurable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/conf/Configurable.java diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/conf/Configuration.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/conf/Configuration.java new file mode 100644 index 000000000..71b7f7c2a --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/conf/Configuration.java @@ -0,0 +1,1392 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.conf; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import org.apache.commons.lang3.StringUtils; +import org.datavec.api.util.ReflectionUtils; +import org.datavec.api.writable.Writable; +import org.datavec.api.writable.WritableType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.*; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.*; +import java.net.URL; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +public class Configuration implements Iterable>, Writable, Serializable { + private static final Logger LOG = LoggerFactory.getLogger(Configuration.class); + + private boolean quietmode = true; + + /** + * List of configuration resources. + */ + private ArrayList resources = new ArrayList<>(); + + /** + * List of configuration parameters marked final. + */ + private Set finalParameters = new HashSet<>(); + + private boolean loadDefaults = true; + + /** + * Configuration objects + */ + private static final WeakHashMap REGISTRY = new WeakHashMap<>(); + + /** + * List of default Resources. Resources are loaded in the order of the list + * entries + */ + private static final CopyOnWriteArrayList defaultResources = new CopyOnWriteArrayList<>(); + + private static final ConcurrentMap>> CACHE_CLASSES = new ConcurrentHashMap<>(); + + /** + * Flag to indicate if the storage of resource which updates a key needs + * to be stored for each key + */ + private boolean storeResource; + + /** + * Stores the mapping of key to the resource which modifies or loads + * the key most recently + */ + private HashMap updatingResource; + + static { + //print deprecation warning if hadoop-site.xml is found in classpath + ClassLoader cL = Thread.currentThread().getContextClassLoader(); + if (cL == null) { + cL = Configuration.class.getClassLoader(); + } + if (cL.getResource("hadoop-site.xml") != null) { + LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " + + "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, " + + "mapred-site.xml and hdfs-site.xml to override properties of " + + "core-default.xml, mapred-default.xml and hdfs-default.xml " + "respectively"); + } + addDefaultResource("core-default.xml"); + addDefaultResource("core-site.xml"); + } + + private Properties properties; + private Properties overlay; + private transient ClassLoader classLoader; + { + classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = Configuration.class.getClassLoader(); + } + } + + + + /** A new configuration. */ + public Configuration() { + this(true); + } + + /** A new configuration where the behavior of reading from the default + * resources can be turned off. + * + * If the parameter {@code loadDefaults} is false, the new instance + * will not load resources from the default files. + * @param loadDefaults specifies whether to load from the default files + */ + public Configuration(boolean loadDefaults) { + this.loadDefaults = loadDefaults; + synchronized (Configuration.class) { + REGISTRY.put(this, null); + } + this.storeResource = false; + } + + /** + * A new configuration with the same settings and additional facility for + * storage of resource to each key which loads or updates + * the key most recently + * @param other the configuration from which to clone settings + * @param storeResource flag to indicate if the storage of resource to + * each key is to be stored + */ + private Configuration(Configuration other, boolean storeResource) { + this(other); + this.loadDefaults = other.loadDefaults; + this.storeResource = storeResource; + if (storeResource) { + updatingResource = new HashMap<>(); + } + } + + /** + * A new configuration with the same settings cloned from another. + * + * @param other the configuration from which to clone settings. + */ + @SuppressWarnings("unchecked") + public Configuration(Configuration other) { + this.resources = (ArrayList) other.resources.clone(); + synchronized (other) { + if (other.properties != null) { + this.properties = (Properties) other.properties.clone(); + } + + if (other.overlay != null) { + this.overlay = (Properties) other.overlay.clone(); + } + } + + this.finalParameters = new HashSet<>(other.finalParameters); + synchronized (Configuration.class) { + REGISTRY.put(this, null); + } + } + + /** + * Add a default resource. Resources are loaded in the order of the resources + * added. + * @param name file name. File should be present in the classpath. + */ + public static void addDefaultResource(String name) { + // The lock hierarchy is that we must always lock + // instances before locking the class. Since reloadConfiguration + // is synchronized on the instance, we must not call conf.reloadConfiguration + // while holding a lock on Configuration.class. Otherwise we could deadlock + // if that conf is attempting to lock the Class + ArrayList toReload; + synchronized (Configuration.class) { + if (defaultResources.contains(name)) { + return; + } + defaultResources.add(name); + // Make a copy so we don't iterate while not holding the lock + toReload = new ArrayList<>(REGISTRY.size()); + toReload.addAll(REGISTRY.keySet()); + } + for (Configuration conf : toReload) { + if (conf.loadDefaults) { + conf.reloadConfiguration(); + } + } + } + + /** + * Add a configuration resource. + * + * The properties of this resource will override properties of previously + * added resources, unless they were marked final. + * + * @param name resource to be added, the classpath is examined for a file + * with that name. + */ + public void addResource(String name) { + addResourceObject(name); + } + + /** + * Add a configuration resource. + * + * The properties of this resource will override properties of previously + * added resources, unless they were marked final. + * + * @param url url of the resource to be added, the local filesystem is + * examined directly to find the resource, without referring to + * the classpath. + */ + public void addResource(URL url) { + addResourceObject(url); + } + + + /** + * Add a configuration resource. + * + * The properties of this resource will override properties of previously + * added resources, unless they were marked final. + * + * @param in InputStream to deserialize the object from. + */ + public void addResource(InputStream in) { + addResourceObject(in); + } + + + /** + * Reload configuration from previously added resources. + * + * This method will clear all the configuration read from the added + * resources, and final parameters. This will make the resources to + * be read again before accessing the values. Values that are added + * via set methods will overlay values read from the resources. + */ + public synchronized void reloadConfiguration() { + properties = null; // trigger reload + finalParameters.clear(); // clear site-limits + } + + private synchronized void addResourceObject(Object resource) { + resources.add(resource); // add to resources + reloadConfiguration(); + } + + private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); + + private String substituteVars(String expr) { + if (expr == null) { + return null; + } + Matcher match = varPat.matcher(""); + String eval = expr; + int MAX_SUBST = 20; + for (int s = 0; s < MAX_SUBST; s++) { + match.reset(eval); + if (!match.find()) { + return eval; + } + String var = match.group(); + var = var.substring(2, var.length() - 1); // remove ${ .. } + String val = null; + try { + val = System.getProperty(var); + } catch (SecurityException se) { + LOG.warn("Unexpected SecurityException in Configuration", se); + } + if (val == null) { + val = getRaw(var); + } + if (val == null) { + return eval; // return literal ${var}: var is unbound + } + // substitute + eval = eval.substring(0, match.start()) + val + eval.substring(match.end()); + } + throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr); + } + + /** + * Get the value of the name property, null if + * no such property exists. + * + * Values are processed for variable expansion + * before being returned. + * + * @param name the property name. + * @return the value of the name property, + * or null if no such property exists. + */ + public String get(String name) { + return substituteVars(getProps().getProperty(name)); + } + + /** + * Get the value of the name property, without doing + * variable expansion. + * + * @param name the property name. + * @return the value of the name property, + * or null if no such property exists. + */ + public String getRaw(String name) { + return getProps().getProperty(name); + } + + /** + * Get the char value of the name property, null if + * no such property exists. + * + * Values are processed for variable expansion + * before being returned. + * + * @param name the property name. + * @return the value of the name property, + * or null if no such property exists. + */ + public char getChar(String name) { + return getProps().getProperty(name).charAt(0); + } + + /** + * Get the char value of the name property, null if + * no such property exists. + * + * Values are processed for variable expansion + * before being returned. + * + * @param name the property name. + * @return the value of the name property, + * or null if no such property exists. + */ + public char getChar(String name, char defaultValue) { + return getProps().getProperty(name, String.valueOf(defaultValue)).charAt(0); + } + + /** + * Set the value of the name property. + * + * @param name property name. + * @param value property value. + */ + public void set(String name, String value) { + getOverlay().setProperty(name, value); + getProps().setProperty(name, value); + } + + /** + * Sets a property if it is currently unset. + * @param name the property name + * @param value the new value + */ + public void setIfUnset(String name, String value) { + if (get(name) == null) { + set(name, value); + } + } + + private synchronized Properties getOverlay() { + if (overlay == null) { + overlay = new Properties(); + } + return overlay; + } + + /** + * Get the value of the name property. If no such property + * exists, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value, or defaultValue if the property + * doesn't exist. + */ + public String get(String name, String defaultValue) { + return substituteVars(getProps().getProperty(name, defaultValue)); + } + + /** + * Get the value of the name property as an int. + * + * If no such property exists, or if the specified value is not a valid + * int, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value as an int, + * or defaultValue. + */ + public int getInt(String name, int defaultValue) { + String valueString = get(name); + if (valueString == null) + return defaultValue; + try { + String hexString = getHexDigits(valueString); + if (hexString != null) { + return Integer.parseInt(hexString, 16); + } + return Integer.parseInt(valueString); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * Set the value of the name property to an int. + * + * @param name property name. + * @param value int value of the property. + */ + public void setInt(String name, int value) { + set(name, Integer.toString(value)); + } + + + /** + * Get the value of the name property as a long. + * If no such property is specified, or if the specified value is not a valid + * long, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value as a long, + * or defaultValue. + */ + public long getLong(String name, long defaultValue) { + String valueString = get(name); + if (valueString == null) + return defaultValue; + try { + String hexString = getHexDigits(valueString); + if (hexString != null) { + return Long.parseLong(hexString, 16); + } + return Long.parseLong(valueString); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + private String getHexDigits(String value) { + boolean negative = false; + String str = value; + String hexString; + if (value.startsWith("-")) { + negative = true; + str = value.substring(1); + } + if (str.startsWith("0x") || str.startsWith("0X")) { + hexString = str.substring(2); + if (negative) { + hexString = "-" + hexString; + } + return hexString; + } + return null; + } + + /** + * Set the value of the name property to a long. + * + * @param name property name. + * @param value long value of the property. + */ + public void setLong(String name, long value) { + set(name, Long.toString(value)); + } + + /** + * Get the value of the name property as a float. + * If no such property is specified, or if the specified value is not a valid + * float, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value as a float, + * or defaultValue. + */ + public float getFloat(String name, float defaultValue) { + String valueString = get(name); + if (valueString == null) + return defaultValue; + try { + return Float.parseFloat(valueString); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * Set the value of the name property to a float. + * + * @param name property name. + * @param value property value. + */ + public void setFloat(String name, float value) { + set(name, Float.toString(value)); + } + + /** + * Get the value of the name property as a boolean. + * If no such property is specified, or if the specified value is not a valid + * boolean, then defaultValue is returned. + * + * @param name property name. + * @param defaultValue default value. + * @return property value as a boolean, + * or defaultValue. + */ + public boolean getBoolean(String name, boolean defaultValue) { + String valueString = get(name); + return "true".equals(valueString) || !"false".equals(valueString) && defaultValue; + } + + /** + * Set the value of the name property to a boolean. + * + * @param name property name. + * @param value boolean value of the property. + */ + public void setBoolean(String name, boolean value) { + set(name, Boolean.toString(value)); + } + + /** + * Set the given property, if it is currently unset. + * @param name property name + * @param value new value + */ + public void setBooleanIfUnset(String name, boolean value) { + setIfUnset(name, Boolean.toString(value)); + } + + /** + * Get the value of the name property as a Pattern. + * If no such property is specified, or if the specified value is not a valid + * Pattern, then DefaultValue is returned. + * + * @param name property name + * @param defaultValue default value + * @return property value as a compiled Pattern, or defaultValue + */ + public Pattern getPattern(String name, Pattern defaultValue) { + String valString = get(name); + if (null == valString || "".equals(valString)) { + return defaultValue; + } + try { + return Pattern.compile(valString); + } catch (PatternSyntaxException pse) { + LOG.warn("Regular expression '" + valString + "' for property '" + name + "' not valid. Using default", + pse); + return defaultValue; + } + } + + /** + * Set the given property to Pattern. + * If the pattern is passed as null, sets the empty pattern which results in + * further calls to getPattern(...) returning the default value. + * + * @param name property name + * @param pattern new value + */ + public void setPattern(String name, Pattern pattern) { + if (null == pattern) { + set(name, null); + } else { + set(name, pattern.pattern()); + } + } + + @Override + public void write(DataOutput out) throws IOException { + + } + + @Override + public void readFields(DataInput in) throws IOException { + + } + + /** + * A class that represents a set of positive integer ranges. It parses + * strings of the form: "2-3,5,7-" where ranges are separated by comma and + * the lower/upper bounds are separated by dash. Either the lower or upper + * bound may be omitted meaning all values up to or over. So the string + * above means 2, 3, 5, and 7, 8, 9, ... + */ + public static class IntegerRanges { + private static class Range { + int start; + int end; + } + + List ranges = new ArrayList(); + + public IntegerRanges() {} + + public IntegerRanges(String newValue) { + StringTokenizer itr = new StringTokenizer(newValue, ","); + while (itr.hasMoreTokens()) { + String rng = itr.nextToken().trim(); + String[] parts = rng.split("-", 3); + if (parts.length < 1 || parts.length > 2) { + throw new IllegalArgumentException("integer range badly formed: " + rng); + } + Range r = new Range(); + r.start = convertToInt(parts[0], 0); + if (parts.length == 2) { + r.end = convertToInt(parts[1], Integer.MAX_VALUE); + } else { + r.end = r.start; + } + if (r.start > r.end) { + throw new IllegalArgumentException("IntegerRange from " + r.start + " to " + r.end + " is invalid"); + } + ranges.add(r); + } + } + + /** + * Convert a string to an int treating empty strings as the default value. + * @param value the string value + * @param defaultValue the value for if the string is empty + * @return the desired integer + */ + private static int convertToInt(String value, int defaultValue) { + String trim = value.trim(); + if (trim.length() == 0) { + return defaultValue; + } + return Integer.parseInt(trim); + } + + /** + * Is the given value in the set of ranges + * @param value the value to check + * @return is the value in the ranges? + */ + public boolean isIncluded(int value) { + for (Range r : ranges) { + if (r.start <= value && value <= r.end) { + return true; + } + } + return false; + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + boolean first = true; + for (Range r : ranges) { + if (first) { + first = false; + } else { + result.append(','); + } + result.append(r.start); + result.append('-'); + result.append(r.end); + } + return result.toString(); + } + } + + /** + * Parse the given attribute as a set of integer ranges + * @param name the attribute name + * @param defaultValue the default value if it is not set + * @return a new set of ranges from the configured value + */ + public IntegerRanges getRange(String name, String defaultValue) { + return new IntegerRanges(get(name, defaultValue)); + } + + /** + * Get the comma delimited values of the name property as + * a collection of Strings. + * If no such property is specified then empty collection is returned. + *

+ * This is an optimized version of {@link #getStrings(String)} + * + * @param name property name. + * @return property value as a collection of Strings. + */ + public Collection getStringCollection(String name) { + String valueString = get(name); + if(valueString == null) + return null; + return Arrays.asList(StringUtils.split(valueString, ",")); + } + + /** + * Get the comma delimited values of the name property as + * an array of Strings. + * If no such property is specified then null is returned. + * + * @param name property name. + * @return property value as an array of Strings, + * or null. + */ + public String[] getStrings(String name) { + String valueString = get(name); + return StringUtils.split(valueString, ","); + } + + /** + * Get the comma delimited values of the name property as + * an array of Strings. + * If no such property is specified then default value is returned. + * + * @param name property name. + * @param defaultValue The default value + * @return property value as an array of Strings, + * or default value. + */ + public String[] getStrings(String name, String... defaultValue) { + String valueString = get(name); + if (valueString == null) { + return defaultValue; + } else { + return StringUtils.split(valueString, ","); + } + } + + /** + * Get the comma delimited values of the name property as + * a collection of Strings, trimmed of the leading and trailing whitespace. + * If no such property is specified then empty Collection is returned. + * + * @param name property name. + * @return property value as a collection of Strings, or empty Collection + */ + public Collection getTrimmedStringCollection(String name) { + String valueString = get(name); + if (null == valueString) { + return Collections.emptyList(); + } + return Arrays.asList(StringUtils.stripAll(StringUtils.split(valueString, ","))); + } + + /** + * Get the comma delimited values of the name property as + * an array of Strings, trimmed of the leading and trailing whitespace. + * If no such property is specified then an empty array is returned. + * + * @param name property name. + * @return property value as an array of trimmed Strings, + * or empty array. + */ + public String[] getTrimmedStrings(String name) { + String valueString = get(name); + return StringUtils.stripAll(StringUtils.split(valueString, ",")); + } + + /** + * Get the comma delimited values of the name property as + * an array of Strings, trimmed of the leading and trailing whitespace. + * If no such property is specified then default value is returned. + * + * @param name property name. + * @param defaultValue The default value + * @return property value as an array of trimmed Strings, + * or default value. + */ + public String[] getTrimmedStrings(String name, String... defaultValue) { + String valueString = get(name); + if (null == valueString) { + return defaultValue; + } else { + return StringUtils.stripAll(StringUtils.split(valueString, ",")); + } + } + + /** + * Set the array of string values for the name property as + * as comma delimited values. + * + * @param name property name. + * @param values The values + */ + public void setStrings(String name, String... values) { + set(name, StringUtils.join(values, ",")); + } + + /** + * Load a class by name. + * + * @param name the class name. + * @return the class object. + * @throws ClassNotFoundException if the class is not found. + */ + public Class getClassByName(String name) throws ClassNotFoundException { + Map> map = CACHE_CLASSES.get(classLoader); + if (map == null) { + Map> newMap = new ConcurrentHashMap<>(); + map = CACHE_CLASSES.putIfAbsent(classLoader, newMap); + if (map == null) { + map = newMap; + } + } + + Class clazz = map.get(name); + if (clazz == null) { + clazz = Class.forName(name, true, classLoader); + if (clazz != null) { + map.put(name, clazz); + } + } + + return clazz; + } + + /** + * Get the value of the name property + * as an array of Class. + * The value of the property specifies a list of comma separated class names. + * If no such property is specified, then defaultValue is + * returned. + * + * @param name the property name. + * @param defaultValue default value. + * @return property value as a Class[], + * or defaultValue. + */ + public Class[] getClasses(String name, Class... defaultValue) { + String[] classnames = getStrings(name); + if (classnames == null) + return defaultValue; + try { + Class[] classes = new Class[classnames.length]; + for (int i = 0; i < classnames.length; i++) { + classes[i] = getClassByName(classnames[i]); + } + return classes; + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + /** + * Get the value of the name property as a Class. + * If no such property is specified, then defaultValue is + * returned. + * + * @param name the class name. + * @param defaultValue default value. + * @return property value as a Class, + * or defaultValue. + */ + public Class getClass(String name, Class defaultValue) { + String valueString = get(name); + if (valueString == null) + return defaultValue; + try { + return getClassByName(valueString); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + /** + * Get the value of the name property as a Class + * implementing the interface specified by xface. + * + * If no such property is specified, then defaultValue is + * returned. + * + * An exception is thrown if the returned class does not implement the named + * interface. + * + * @param name the class name. + * @param defaultValue default value. + * @param xface the interface implemented by the named class. + * @return property value as a Class, + * or defaultValue. + */ + public Class getClass(String name, Class defaultValue, Class xface) { + try { + Class theClass = getClass(name, defaultValue); + if (theClass != null && !xface.isAssignableFrom(theClass)) + throw new RuntimeException(theClass + " not " + xface.getName()); + else if (theClass != null) + return theClass.asSubclass(xface); + else + return null; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Get the value of the name property as a List + * of objects implementing the interface specified by xface. + * + * An exception is thrown if any of the classes does not exist, or if it does + * not implement the named interface. + * + * @param name the property name. + * @param xface the interface implemented by the classes named by + * name. + * @return a List of objects implementing xface. + */ + @SuppressWarnings("unchecked") + public List getInstances(String name, Class xface) { + List ret = new ArrayList<>(); + Class[] classes = getClasses(name); + for (Class cl : classes) { + if (!xface.isAssignableFrom(cl)) { + throw new RuntimeException(cl + " does not implement " + xface); + } + ret.add((U) ReflectionUtils.newInstance(cl, this)); + } + return ret; + } + + /** + * Set the value of the name property to the name of a + * theClass implementing the given interface xface. + * + * An exception is thrown if theClass does not implement the + * interface xface. + * + * @param name property name. + * @param theClass property value. + * @param xface the interface implemented by the named class. + */ + public void setClass(String name, Class theClass, Class xface) { + if (!xface.isAssignableFrom(theClass)) + throw new RuntimeException(theClass + " not " + xface.getName()); + set(name, theClass.getName()); + } + + + + /** + * Get a local file name under a directory named in dirsProp with + * the given path. If dirsProp contains multiple directories, + * then one is chosen based on path's hash code. If the selected + * directory does not exist, an attempt is made to create it. + * + * @param dirsProp directory in which to locate the file. + * @param path file-path. + * @return local file under the directory with the given path. + */ + public File getFile(String dirsProp, String path) throws IOException { + String[] dirs = getStrings(dirsProp); + int hashCode = path.hashCode(); + for (int i = 0; i < dirs.length; i++) { // try each local dir + int index = (hashCode + i & Integer.MAX_VALUE) % dirs.length; + File file = new File(dirs[index], path); + File dir = file.getParentFile(); + if (dir.exists() || dir.mkdirs()) { + return file; + } + } + throw new IOException("No valid local directories in property: " + dirsProp); + } + + /** + * Get the {@link URL} for the named resource. + * + * @param name resource name. + * @return the url for the named resource. + */ + public URL getResource(String name) { + return classLoader.getResource(name); + } + + /** + * Get an input stream attached to the configuration resource with the + * given name. + * + * @param name configuration resource name. + * @return an input stream attached to the resource. + */ + public InputStream getConfResourceAsInputStream(String name) { + try { + URL url = getResource(name); + + if (url == null) { + LOG.info(name + " not found"); + return null; + } else { + LOG.info("found resource " + name + " at " + url); + } + + return url.openStream(); + } catch (Exception e) { + return null; + } + } + + /** + * Get a {@link Reader} attached to the configuration resource with the + * given name. + * + * @param name configuration resource name. + * @return a reader attached to the resource. + */ + public Reader getConfResourceAsReader(String name) { + try { + URL url = getResource(name); + + if (url == null) { + LOG.info(name + " not found"); + return null; + } else { + LOG.info("found resource " + name + " at " + url); + } + + return new InputStreamReader(url.openStream()); + } catch (Exception e) { + return null; + } + } + + private synchronized Properties getProps() { + if (properties == null) { + properties = new Properties(); + loadResources(properties, resources, quietmode); + if (overlay != null) { + properties.putAll(overlay); + if (storeResource) { + for (Map.Entry item : overlay.entrySet()) { + updatingResource.put((String) item.getKey(), "Unknown"); + } + } + } + } + return properties; + } + + /** + * Return the number of keys in the configuration. + * + * @return number of keys in the configuration. + */ + public int size() { + return getProps().size(); + } + + /** + * Clears all keys from the configuration. + */ + public void clear() { + getProps().clear(); + getOverlay().clear(); + } + + /** + * Get an {@link Iterator} to go through the list of String + * key-value pairs in the configuration. + * + * @return an iterator over the entries. + */ + public Iterator> iterator() { + // Get a copy of just the string to string pairs. After the old object + // methods that allow non-strings to be put into configurations are removed, + // we could replace properties with a Map and get rid of this + // code. + Map result = new HashMap<>(); + for (Map.Entry item : getProps().entrySet()) { + if (item.getKey() instanceof String && item.getValue() instanceof String) { + result.put((String) item.getKey(), (String) item.getValue()); + } + } + return result.entrySet().iterator(); + } + + private void loadResources(Properties properties, ArrayList resources, boolean quiet) { + if (loadDefaults) { + // To avoid addResource causing a ConcurrentModificationException + ArrayList toLoad; + synchronized (Configuration.class) { + toLoad = new ArrayList<>(defaultResources); + } + for (String resource : toLoad) { + loadResource(properties, resource, quiet); + } + + //support the hadoop-site.xml as a deprecated case + if (getResource("hadoop-site.xml") != null) { + loadResource(properties, "hadoop-site.xml", quiet); + } + } + + for (Object resource : resources) { + loadResource(properties, resource, quiet); + } + } + + private void loadResource(Properties properties, Object name, boolean quiet) { + try { + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); + //ignore all comments inside the xml file + docBuilderFactory.setIgnoringComments(true); + + //allow includes in the xml file + docBuilderFactory.setNamespaceAware(true); + try { + docBuilderFactory.setXIncludeAware(true); + } catch (UnsupportedOperationException e) { + LOG.error("Failed to set setXIncludeAware(true) for parser " + docBuilderFactory + ":" + e, e); + } + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = null; + Element root = null; + + if (name instanceof URL) { // an URL resource + URL url = (URL) name; + if (url != null) { + if (!quiet) { + LOG.info("parsing " + url); + } + doc = builder.parse(url.toString()); + } + } else if (name instanceof String) { // a CLASSPATH resource + URL url = getResource((String) name); + if (url != null) { + if (!quiet) { + LOG.info("parsing " + url); + } + doc = builder.parse(url.toString()); + } + } else if (name instanceof InputStream) { + try { + doc = builder.parse((InputStream) name); + } finally { + ((InputStream) name).close(); + } + } else if (name instanceof Element) { + root = (Element) name; + } + + if (doc == null && root == null) { + if (quiet) + return; + throw new RuntimeException(name + " not found"); + } + + if (root == null) { + root = doc.getDocumentElement(); + } + if (!"configuration".equals(root.getTagName())) + LOG.error("bad conf file: top-level element not "); + NodeList props = root.getChildNodes(); + for (int i = 0; i < props.getLength(); i++) { + Node propNode = props.item(i); + if (!(propNode instanceof Element)) + continue; + Element prop = (Element) propNode; + if ("configuration".equals(prop.getTagName())) { + loadResource(properties, prop, quiet); + continue; + } + if (!"property".equals(prop.getTagName())) + LOG.warn("bad conf file: element not "); + NodeList fields = prop.getChildNodes(); + String attr = null; + String value = null; + boolean finalParameter = false; + for (int j = 0; j < fields.getLength(); j++) { + Node fieldNode = fields.item(j); + if (!(fieldNode instanceof Element)) + continue; + Element field = (Element) fieldNode; + if ("name".equals(field.getTagName()) && field.hasChildNodes()) + attr = ((Text) field.getFirstChild()).getData().trim(); + if ("value".equals(field.getTagName()) && field.hasChildNodes()) + value = ((Text) field.getFirstChild()).getData(); + if ("final".equals(field.getTagName()) && field.hasChildNodes()) + finalParameter = "true".equals(((Text) field.getFirstChild()).getData()); + } + + // Ignore this parameter if it has already been marked as 'final' + if (attr != null && value != null) { + if (!finalParameters.contains(attr)) { + properties.setProperty(attr, value); + if (storeResource) { + updatingResource.put(attr, name.toString()); + } + if (finalParameter) + finalParameters.add(attr); + } else { + LOG.warn(name + ":a attempt to override final parameter: " + attr + "; Ignoring."); + } + } + } + + } catch (IOException | ParserConfigurationException | SAXException | DOMException e) { + LOG.error("error parsing conf file: " + e); + throw new RuntimeException(e); + } + } + + /** + * Write out the non-default properties in this configuration to the give + * {@link OutputStream}. + * + * @param out the output stream to write to. + */ + public void writeXml(OutputStream out) throws IOException { + Properties properties = getProps(); + try { + Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); + Element conf = doc.createElement("configuration"); + doc.appendChild(conf); + conf.appendChild(doc.createTextNode("\n")); + for (Enumeration e = properties.keys(); e.hasMoreElements();) { + String name = (String) e.nextElement(); + Object object = properties.get(name); + String value; + if (object instanceof String) { + value = (String) object; + } else { + continue; + } + Element propNode = doc.createElement("property"); + conf.appendChild(propNode); + + Element nameNode = doc.createElement("name"); + nameNode.appendChild(doc.createTextNode(name)); + propNode.appendChild(nameNode); + + Element valueNode = doc.createElement("value"); + valueNode.appendChild(doc.createTextNode(value)); + propNode.appendChild(valueNode); + + conf.appendChild(doc.createTextNode("\n")); + } + + DOMSource source = new DOMSource(doc); + StreamResult result = new StreamResult(out); + TransformerFactory transFactory = TransformerFactory.newInstance(); + Transformer transformer = transFactory.newTransformer(); + transformer.transform(source, result); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Writes out all the parameters and their properties (final and resource) to + * the given {@link Writer} + * The format of the output would be + * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2, + * key2.isFinal,key2.resource}... ] } + * It does not output the parameters of the configuration object which is + * loaded from an input stream. + * @param out the Writer to write to + * @throws IOException + */ + public static void dumpConfiguration(Configuration conf, Writer out) throws IOException { + Configuration config = new Configuration(conf, true); + config.reloadConfiguration(); + JsonFactory dumpFactory = new JsonFactory(); + JsonGenerator dumpGenerator = dumpFactory.createGenerator(out); + dumpGenerator.writeStartObject(); + dumpGenerator.writeFieldName("properties"); + dumpGenerator.writeStartArray(); + dumpGenerator.flush(); + for (Map.Entry item : config.getProps().entrySet()) { + dumpGenerator.writeStartObject(); + dumpGenerator.writeStringField("key", (String) item.getKey()); + dumpGenerator.writeStringField("value", config.get((String) item.getKey())); + dumpGenerator.writeBooleanField("isFinal", config.finalParameters.contains(item.getKey())); + dumpGenerator.writeStringField("resource", config.updatingResource.get(item.getKey())); + dumpGenerator.writeEndObject(); + } + dumpGenerator.writeEndArray(); + dumpGenerator.writeEndObject(); + dumpGenerator.flush(); + } + + /** + * Get the {@link ClassLoader} for this job. + * + * @return the correct class loader. + */ + public ClassLoader getClassLoader() { + return classLoader; + } + + /** + * Set the class loader that will be used to load the various objects. + * + * @param classLoader the new class loader. + */ + public void setClassLoader(ClassLoader classLoader) { + this.classLoader = classLoader; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Configuration: "); + if (loadDefaults) { + synchronized (Configuration.class) { + toString(defaultResources, sb); + } + if (resources.size() > 0) { + sb.append(", "); + } + } + toString(resources, sb); + return sb.toString(); + } + + private void toString(List resources, StringBuilder sb) { + ListIterator i = resources.listIterator(); + while (i.hasNext()) { + if (i.nextIndex() != 0) { + sb.append(", "); + } + sb.append(i.next()); + } + } + + /** + * Set the quietness-mode. + * + * In the quiet-mode, error and informational messages might not be logged. + * + * @param quietmode true to set quiet-mode on, false + * to turn it off. + */ + public synchronized void setQuietMode(boolean quietmode) { + this.quietmode = quietmode; + } + + /** For debugging. List non-default properties to the terminal and exit. */ + public static void main(String[] args) throws Exception { + new Configuration().writeXml(System.out); + } + + + @Override + public double toDouble() { + throw new UnsupportedOperationException(); + } + + @Override + public float toFloat() { + throw new UnsupportedOperationException(); + } + + @Override + public int toInt() { + throw new UnsupportedOperationException(); + } + + @Override + public long toLong() { + throw new UnsupportedOperationException(); + } + + @Override + public WritableType getType() { + throw new UnsupportedOperationException(); + } + + @Override + public void writeType(DataOutput out) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/conf/Configured.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/conf/Configured.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/conf/Configured.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/conf/Configured.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/exceptions/DataVecException.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/exceptions/DataVecException.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/exceptions/DataVecException.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/exceptions/DataVecException.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/exceptions/UnknownFormatException.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/exceptions/UnknownFormatException.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/exceptions/UnknownFormatException.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/exceptions/UnknownFormatException.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/input/BaseInputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/BaseInputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/input/BaseInputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/BaseInputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/input/InputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/InputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/input/InputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/InputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/CSVInputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/CSVInputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/CSVInputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/CSVInputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/LibSvmInputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/LibSvmInputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/LibSvmInputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/LibSvmInputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/LineInputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/LineInputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/LineInputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/LineInputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/ListStringInputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/ListStringInputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/ListStringInputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/ListStringInputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/MatlabInputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/MatlabInputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/MatlabInputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/MatlabInputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/SVMLightInputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/SVMLightInputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/input/impl/SVMLightInputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/input/impl/SVMLightInputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/output/OutputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/OutputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/output/OutputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/OutputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/CSVOutputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/impl/CSVOutputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/CSVOutputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/impl/CSVOutputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/LibSvmOutputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/impl/LibSvmOutputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/LibSvmOutputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/impl/LibSvmOutputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/LineOutputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/impl/LineOutputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/LineOutputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/impl/LineOutputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/SVMLightOutputFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/impl/SVMLightOutputFormat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/formats/output/impl/SVMLightOutputFormat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/formats/output/impl/SVMLightOutputFormat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/BinaryComparable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/BinaryComparable.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/BinaryComparable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/BinaryComparable.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/DataInputBuffer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/DataInputBuffer.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/DataInputBuffer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/DataInputBuffer.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/DataOutputBuffer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/DataOutputBuffer.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/DataOutputBuffer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/DataOutputBuffer.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/RawComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/RawComparator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/RawComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/RawComparator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/WritableComparable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableComparable.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/WritableComparable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableComparable.java diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableComparator.java new file mode 100644 index 000000000..16cc4f35b --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableComparator.java @@ -0,0 +1,230 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.io; + + +import org.datavec.api.util.ReflectionUtils; +import org.datavec.api.writable.Writable; + +import java.io.DataInput; +import java.io.IOException; +import java.util.HashMap; + + +public class WritableComparator implements RawComparator { + + private static HashMap comparators = new HashMap<>(); // registry + + /** Get a comparator for a {@link WritableComparable} implementation. */ + public static synchronized WritableComparator get(Class c) { + WritableComparator comparator = comparators.get(c); + if (comparator == null) { + // force the static initializers to run + forceInit(c); + // look to see if it is defined now + comparator = comparators.get(c); + // if not, use the generic one + if (comparator == null) { + comparator = new WritableComparator(c, true); + comparators.put(c, comparator); + } + } + return comparator; + } + + /** + * Force initialization of the static members. + * As of Java 5, referencing a class doesn't force it to initialize. Since + * this class requires that the classes be initialized to declare their + * comparators, we force that initialization to happen. + * @param cls the class to initialize + */ + private static void forceInit(Class cls) { + try { + Class.forName(cls.getName(), true, cls.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Can't initialize class " + cls, e); + } + } + + /** Register an optimized comparator for a {@link WritableComparable} + * implementation. */ + public static synchronized void define(Class c, WritableComparator comparator) { + comparators.put(c, comparator); + } + + + private final Class keyClass; + private final WritableComparable key1; + private final WritableComparable key2; + private final DataInputBuffer buffer; + + /** Construct for a {@link WritableComparable} implementation. */ + protected WritableComparator(Class keyClass) { + this(keyClass, false); + } + + protected WritableComparator(Class keyClass, boolean createInstances) { + this.keyClass = keyClass; + if (createInstances) { + key1 = newKey(); + key2 = newKey(); + buffer = new DataInputBuffer(); + } else { + key1 = key2 = null; + buffer = null; + } + } + + /** Returns the WritableComparable implementation class. */ + public Class getKeyClass() { + return keyClass; + } + + /** Construct a new {@link WritableComparable} instance. */ + public WritableComparable newKey() { + return ReflectionUtils.newInstance(keyClass, null); + } + + /** Optimization hook. Override this to make SequenceFile.Sorter's scream. + * + *

The default implementation reads the data into two {@link + * WritableComparable}s (using {@link + * Writable#readFields(DataInput)}, then calls {@link + * #compare(WritableComparable,WritableComparable)}. + */ + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + try { + buffer.reset(b1, s1, l1); // parse key1 + key1.readFields(buffer); + + buffer.reset(b2, s2, l2); // parse key2 + key2.readFields(buffer); + + } catch (IOException e) { + throw new RuntimeException(e); + } + + return compare(key1, key2); // compare them + } + + /** Compare two WritableComparables. + * + *

The default implementation uses the natural ordering, calling {@link + * Comparable#compareTo(Object)}. */ + @SuppressWarnings("unchecked") + public int compare(WritableComparable a, WritableComparable b) { + return a.compareTo(b); + } + + public int compare(Object a, Object b) { + return compare((WritableComparable) a, (WritableComparable) b); + } + + /** Lexicographic order of binary data. */ + public static int compareBytes(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + int end1 = s1 + l1; + int end2 = s2 + l2; + for (int i = s1, j = s2; i < end1 && j < end2; i++, j++) { + int a = (b1[i] & 0xff); + int b = (b2[j] & 0xff); + if (a != b) { + return a - b; + } + } + return l1 - l2; + } + + /** Compute hash for binary data. */ + public static int hashBytes(byte[] bytes, int offset, int length) { + int hash = 1; + for (int i = offset; i < offset + length; i++) + hash = (31 * hash) + (int) bytes[i]; + return hash; + } + + /** Compute hash for binary data. */ + public static int hashBytes(byte[] bytes, int length) { + return hashBytes(bytes, 0, length); + } + + /** Parse an unsigned short from a byte array. */ + public static int readUnsignedShort(byte[] bytes, int start) { + return (((bytes[start] & 0xff) << 8) + ((bytes[start + 1] & 0xff))); + } + + /** Parse an integer from a byte array. */ + public static int readInt(byte[] bytes, int start) { + return (((bytes[start] & 0xff) << 24) + ((bytes[start + 1] & 0xff) << 16) + ((bytes[start + 2] & 0xff) << 8) + + ((bytes[start + 3] & 0xff))); + + } + + /** Parse a float from a byte array. */ + public static float readFloat(byte[] bytes, int start) { + return Float.intBitsToFloat(readInt(bytes, start)); + } + + /** Parse a long from a byte array. */ + public static long readLong(byte[] bytes, int start) { + return ((long) (readInt(bytes, start)) << 32) + (readInt(bytes, start + 4) & 0xFFFFFFFFL); + } + + /** Parse a double from a byte array. */ + public static double readDouble(byte[] bytes, int start) { + return Double.longBitsToDouble(readLong(bytes, start)); + } + + /** + * Reads a zero-compressed encoded long from a byte array and returns it. + * @param bytes byte array with decode long + * @param start starting index + * @throws IOException + * @return deserialized long + */ + public static long readVLong(byte[] bytes, int start) throws IOException { + int len = bytes[start]; + if (len >= -112) { + return len; + } + boolean isNegative = (len < -120); + len = isNegative ? -(len + 120) : -(len + 112); + if (start + 1 + len > bytes.length) + throw new IOException("Not enough number of bytes for a zero-compressed integer"); + long i = 0; + for (int idx = 0; idx < len; idx++) { + i = i << 8; + i = i | (bytes[start + 1 + idx] & 0xFF); + } + return (isNegative ? (~i) : i); + } + + /** + * Reads a zero-compressed encoded integer from a byte array and returns it. + * @param bytes byte array with the encoded integer + * @param start start index + * @throws IOException + * @return deserialized integer + */ + public static int readVInt(byte[] bytes, int start) throws IOException { + return (int) readVLong(bytes, start); + } +} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/WritableConverter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableConverter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/WritableConverter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableConverter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/WritableUtils.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableUtils.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/WritableUtils.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableUtils.java index d35d69d4f..ebac8c856 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/io/WritableUtils.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/WritableUtils.java @@ -240,7 +240,7 @@ public final class WritableUtils { * * @param stream Binary output stream * @param i Integer to be serialized - * @throws java.io.IOException + * @throws IOException */ public static void writeVInt(DataOutput stream, int i) throws IOException { writeVLong(stream, i); @@ -259,7 +259,7 @@ public final class WritableUtils { * * @param stream Binary output stream * @param i Long to be serialized - * @throws java.io.IOException + * @throws IOException */ public static void writeVLong(DataOutput stream, long i) throws IOException { if (i >= -112 && i <= 127) { @@ -294,7 +294,7 @@ public final class WritableUtils { /** * Reads a zero-compressed encoded long from input stream and returns it. * @param stream Binary input stream - * @throws java.io.IOException + * @throws IOException * @return deserialized long from stream. */ public static long readVLong(DataInput stream) throws IOException { @@ -315,7 +315,7 @@ public final class WritableUtils { /** * Reads a zero-compressed encoded integer from input stream and returns it. * @param stream Binary input stream - * @throws java.io.IOException + * @throws IOException * @return deserialized integer from stream. */ public static int readVInt(DataInput stream) throws IOException { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/converters/DoubleWritableConverter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/DoubleWritableConverter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/converters/DoubleWritableConverter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/DoubleWritableConverter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/converters/FloatWritableConverter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/FloatWritableConverter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/converters/FloatWritableConverter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/FloatWritableConverter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/converters/LabelWriterConverter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/LabelWriterConverter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/converters/LabelWriterConverter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/LabelWriterConverter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/converters/SelfWritableConverter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/SelfWritableConverter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/converters/SelfWritableConverter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/SelfWritableConverter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/converters/WritableConverterException.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/WritableConverterException.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/converters/WritableConverterException.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/converters/WritableConverterException.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/filters/BalancedPathFilter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/filters/BalancedPathFilter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/filters/BalancedPathFilter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/filters/BalancedPathFilter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/filters/PathFilter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/filters/PathFilter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/filters/PathFilter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/filters/PathFilter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/filters/RandomPathFilter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/filters/RandomPathFilter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/filters/RandomPathFilter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/filters/RandomPathFilter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/labels/ParentPathLabelGenerator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/labels/ParentPathLabelGenerator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/labels/ParentPathLabelGenerator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/labels/ParentPathLabelGenerator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PathLabelGenerator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/labels/PathLabelGenerator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PathLabelGenerator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/labels/PathLabelGenerator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PathMultiLabelGenerator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/labels/PathMultiLabelGenerator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PathMultiLabelGenerator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/labels/PathMultiLabelGenerator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PatternPathLabelGenerator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/labels/PatternPathLabelGenerator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/labels/PatternPathLabelGenerator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/labels/PatternPathLabelGenerator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Deserializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/serializers/Deserializer.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Deserializer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/serializers/Deserializer.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Serialization.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/serializers/Serialization.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Serialization.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/serializers/Serialization.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/SerializationFactory.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/serializers/SerializationFactory.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/SerializationFactory.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/serializers/SerializationFactory.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Serializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/serializers/Serializer.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/io/serializers/Serializer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/io/serializers/Serializer.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/Buffer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/Buffer.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/Buffer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/Buffer.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/IOUtils.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/IOUtils.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/IOUtils.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/IOUtils.java index f1fdbb39b..2a4793ada 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/records/IOUtils.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/IOUtils.java @@ -137,7 +137,7 @@ public class IOUtils { /** * * @param s - * @throws java.io.IOException + * @throws IOException * @return */ static String fromCSVString(String s) throws IOException { @@ -186,7 +186,7 @@ public class IOUtils { /** * * @param s - * @throws java.io.IOException + * @throws IOException * @return */ static Buffer fromXMLBuffer(String s) throws IOException { @@ -218,7 +218,7 @@ public class IOUtils { * Converts a CSV-serialized representation of buffer to a new * Buffer * @param s CSV-serialized representation of buffer - * @throws java.io.IOException + * @throws IOException * @return Deserialized Buffer */ static Buffer fromCSVBuffer(String s) throws IOException { @@ -393,7 +393,7 @@ public class IOUtils { * Reads a zero-compressed encoded long from a byte array and returns it. * @param bytes byte array with decode long * @param start starting index - * @throws java.io.IOException + * @throws IOException * @return deserialized long */ public static long readVLong(byte[] bytes, int start) throws IOException { @@ -404,7 +404,7 @@ public class IOUtils { * Reads a zero-compressed encoded integer from a byte array and returns it. * @param bytes byte array with the encoded integer * @param start start index - * @throws java.io.IOException + * @throws IOException * @return deserialized integer */ public static int readVInt(byte[] bytes, int start) throws IOException { @@ -414,7 +414,7 @@ public class IOUtils { /** * Reads a zero-compressed encoded long from a stream and return it. * @param in input stream - * @throws java.io.IOException + * @throws IOException * @return deserialized long */ public static long readVLong(DataInput in) throws IOException { @@ -424,7 +424,7 @@ public class IOUtils { /** * Reads a zero-compressed encoded integer from a stream and returns it. * @param in input stream - * @throws java.io.IOException + * @throws IOException * @return deserialized integer */ public static int readVInt(DataInput in) throws IOException { @@ -452,7 +452,7 @@ public class IOUtils { * * @param stream Binary output stream * @param i Long to be serialized - * @throws java.io.IOException + * @throws IOException */ public static void writeVLong(DataOutput stream, long i) throws IOException { WritableUtils.writeVLong(stream, i); @@ -463,7 +463,7 @@ public class IOUtils { * * @param stream Binary output stream * @param i int to be serialized - * @throws java.io.IOException + * @throws IOException */ public static void writeVInt(DataOutput stream, int i) throws IOException { WritableUtils.writeVInt(stream, i); diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/Index.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/Index.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/Index.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/Index.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/Record.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/Record.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/Record.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/Record.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/SequenceRecord.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/SequenceRecord.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/SequenceRecord.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/SequenceRecord.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/converter/RecordReaderConverter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/converter/RecordReaderConverter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/converter/RecordReaderConverter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/converter/RecordReaderConverter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/impl/Record.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/impl/Record.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/impl/Record.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/impl/Record.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/impl/SequenceRecord.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/impl/SequenceRecord.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/impl/SequenceRecord.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/impl/SequenceRecord.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/listener/RecordListener.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/listener/RecordListener.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/listener/RecordListener.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/listener/RecordListener.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/listener/impl/LogRecordListener.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/listener/impl/LogRecordListener.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/listener/impl/LogRecordListener.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/listener/impl/LogRecordListener.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/mapper/RecordMapper.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/mapper/RecordMapper.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/mapper/RecordMapper.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/mapper/RecordMapper.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaData.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaData.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposable.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposable.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposableMap.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposableMap.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposableMap.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataComposableMap.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataImageURI.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataImageURI.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataImageURI.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataImageURI.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataIndex.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataIndex.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataIndex.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataIndex.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataInterval.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataInterval.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataInterval.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataInterval.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLine.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLine.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLine.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLine.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLineInterval.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLineInterval.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLineInterval.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataLineInterval.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataURI.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataURI.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataURI.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/metadata/RecordMetaDataURI.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/BaseRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java index a3dfbbd70..84c80a439 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/RecordReader.java @@ -47,7 +47,7 @@ public interface RecordReader extends Closeable, Serializable, Configurable { * Called once at initialization. * * @param split the split that defines the range of records to read - * @throws java.io.IOException + * @throws IOException * @throws InterruptedException */ void initialize(InputSplit split) throws IOException, InterruptedException; @@ -57,7 +57,7 @@ public interface RecordReader extends Closeable, Serializable, Configurable { * * @param conf a configuration for initialization * @param split the split that defines the range of records to read - * @throws java.io.IOException + * @throws IOException * @throws InterruptedException */ void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/SequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/SequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/SequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/SequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordReaderFactory.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordReaderFactory.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordReaderFactory.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordReaderFactory.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordWriterFactory.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordWriterFactory.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordWriterFactory.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/factory/RecordWriterFactory.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ComposableRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/ConcatenatingRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/FileRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/FileRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/FileRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/FileRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/LineRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/LineRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/LineRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/LineRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/CollectionSequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/collection/ListStringRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVLineSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVLineSequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVLineSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVLineSequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVMultiSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVMultiSequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVMultiSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVMultiSequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVNLinesSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVNLinesSequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVNLinesSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVNLinesSequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRegexRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRegexRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRegexRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVRegexRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVSequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVSequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVVariableSlidingWindowRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVVariableSlidingWindowRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVVariableSlidingWindowRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/CSVVariableSlidingWindowRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/SerializableCSVParser.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/SerializableCSVParser.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/SerializableCSVParser.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/SerializableCSVParser.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/filebatch/FileBatchSequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemoryRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemoryRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemoryRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemoryRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemorySequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemorySequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemorySequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/inmemory/InMemorySequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/FieldSelection.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/FieldSelection.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/FieldSelection.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/FieldSelection.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineRecordReader.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineRecordReader.java index 848795be9..17f348e54 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineRecordReader.java @@ -25,7 +25,7 @@ import java.util.List; import org.datavec.api.records.reader.impl.LineRecordReader; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; public class JacksonLineRecordReader extends LineRecordReader { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineSequenceRecordReader.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineSequenceRecordReader.java index 0c67ef64f..7b27cae0f 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineSequenceRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonLineSequenceRecordReader.java @@ -28,7 +28,7 @@ import org.datavec.api.records.metadata.RecordMetaDataURI; import org.datavec.api.records.reader.SequenceRecordReader; import org.datavec.api.records.reader.impl.FileRecordReader; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.*; import java.net.URI; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonReaderUtils.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonReaderUtils.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonReaderUtils.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonReaderUtils.java index 4009477d0..8626188bd 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonReaderUtils.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonReaderUtils.java @@ -22,8 +22,8 @@ package org.datavec.api.records.reader.impl.jackson; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.core.type.TypeReference; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.ArrayList; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonRecordReader.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonRecordReader.java index e11be0722..8e5e571e7 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/jackson/JacksonRecordReader.java @@ -33,8 +33,8 @@ import org.datavec.api.records.reader.BaseRecordReader; import org.datavec.api.split.FileSplit; import org.datavec.api.split.InputSplit; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.core.type.TypeReference; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.*; import java.net.URI; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/LibSvmRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/LibSvmRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/LibSvmRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/LibSvmRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/MatlabRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/MatlabRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/MatlabRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/MatlabRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/SVMLightRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/SVMLightRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/SVMLightRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/misc/SVMLightRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexLineRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexLineRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexLineRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexLineRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexSequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/regex/RegexSequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/reader/impl/transform/TransformProcessSequenceRecordReader.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/writer/RecordWriter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/RecordWriter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/writer/RecordWriter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/RecordWriter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/writer/SequenceRecordWriter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/SequenceRecordWriter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/writer/SequenceRecordWriter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/SequenceRecordWriter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/FileRecordWriter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/FileRecordWriter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/FileRecordWriter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/FileRecordWriter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/LineRecordWriter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/LineRecordWriter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/LineRecordWriter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/LineRecordWriter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/csv/CSVRecordWriter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/csv/CSVRecordWriter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/csv/CSVRecordWriter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/csv/CSVRecordWriter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/LibSvmRecordWriter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/LibSvmRecordWriter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/LibSvmRecordWriter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/LibSvmRecordWriter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/MatlabRecordWriter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/MatlabRecordWriter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/MatlabRecordWriter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/MatlabRecordWriter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/SVMLightRecordWriter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/SVMLightRecordWriter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/SVMLightRecordWriter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/records/writer/impl/misc/SVMLightRecordWriter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/BaseInputSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/BaseInputSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/BaseInputSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/BaseInputSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/CollectionInputSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/CollectionInputSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/CollectionInputSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/CollectionInputSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/FileSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/FileSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/FileSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/FileSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/InputSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/InputSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/InputStreamInputSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputStreamInputSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/InputStreamInputSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/InputStreamInputSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/ListStringSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/ListStringSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/ListStringSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/ListStringSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/NumberedFileInputSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/NumberedFileInputSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/NumberedFileInputSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/NumberedFileInputSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/OutputStreamInputSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/OutputStreamInputSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/OutputStreamInputSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/OutputStreamInputSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/StreamInputSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/StreamInputSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/StreamInputSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/StreamInputSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/StringSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/StringSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/StringSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/StringSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/TransformSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/partition/NumberOfRecordsPartitioner.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/partition/NumberOfRecordsPartitioner.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/partition/NumberOfRecordsPartitioner.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/partition/NumberOfRecordsPartitioner.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/partition/PartitionMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/partition/PartitionMetaData.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/partition/PartitionMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/partition/PartitionMetaData.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/partition/Partitioner.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/partition/Partitioner.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/partition/Partitioner.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/partition/Partitioner.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/split/streams/FileStreamCreatorFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/streams/FileStreamCreatorFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/split/streams/FileStreamCreatorFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/split/streams/FileStreamCreatorFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/timeseries/util/TimeSeriesWritableUtils.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/timeseries/util/TimeSeriesWritableUtils.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/timeseries/util/TimeSeriesWritableUtils.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/timeseries/util/TimeSeriesWritableUtils.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ColumnOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ColumnOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ColumnOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ColumnOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ColumnType.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ColumnType.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ColumnType.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ColumnType.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/DataAction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/DataAction.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/DataAction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/DataAction.java index b03be4a25..d959961ff 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/DataAction.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/DataAction.java @@ -28,7 +28,7 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.sequence.ConvertFromSequence; import org.datavec.api.transform.sequence.ConvertToSequence; import org.datavec.api.transform.sequence.SequenceSplit; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.io.Serializable; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/Distance.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/Distance.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/Distance.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/Distance.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/MathFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/MathFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/MathFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/MathFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/MathOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/MathOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/MathOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/MathOp.java diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/Operation.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/Operation.java new file mode 100644 index 000000000..5be624a4d --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/Operation.java @@ -0,0 +1,24 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ +package org.datavec.api.transform; + +public interface Operation { + TOut transform(TIn input); +} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ReduceOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ReduceOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ReduceOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ReduceOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/StringReduceOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/StringReduceOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/StringReduceOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/StringReduceOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/Transform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/Transform.java similarity index 94% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/Transform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/Transform.java index 5edafa32f..96c7fdc8a 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/Transform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/Transform.java @@ -21,8 +21,8 @@ package org.datavec.api.transform; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/TransformProcess.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/TransformProcess.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/TransformProcess.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/TransformProcess.java index 8a2400172..dfd848ec3 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/TransformProcess.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/TransformProcess.java @@ -69,9 +69,9 @@ import org.datavec.api.writable.*; import org.datavec.api.writable.comparator.WritableComparator; import org.joda.time.DateTimeZone; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.exc.InvalidTypeIdException; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; import java.io.IOException; import java.io.Serializable; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/AnalysisCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/AnalysisCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/AnalysisCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/AnalysisCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/DataAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/DataAnalysis.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/DataAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/DataAnalysis.java index 7308aa942..7504247c3 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/DataAnalysis.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/DataAnalysis.java @@ -30,11 +30,11 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.serde.JsonMappers; import org.datavec.api.transform.serde.JsonSerializer; import org.datavec.api.transform.serde.YamlSerializer; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.exc.InvalidTypeIdException; -import org.nd4j.shade.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.io.IOException; import java.io.Serializable; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/DataVecAnalysisUtils.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/DataVecAnalysisUtils.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/DataVecAnalysisUtils.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/DataVecAnalysisUtils.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/SequenceDataAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/SequenceDataAnalysis.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/SequenceDataAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/SequenceDataAnalysis.java index 4be7977f4..dfddad2af 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/SequenceDataAnalysis.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/SequenceDataAnalysis.java @@ -28,7 +28,7 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.serde.JsonMappers; import org.datavec.api.transform.serde.JsonSerializer; import org.datavec.api.transform.serde.YamlSerializer; -import org.nd4j.shade.jackson.databind.exc.InvalidTypeIdException; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; import java.io.IOException; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/BytesAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/BytesAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/BytesAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/BytesAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/CategoricalAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/CategoricalAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/CategoricalAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/CategoricalAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/ColumnAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/ColumnAnalysis.java similarity index 92% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/ColumnAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/ColumnAnalysis.java index 296df8ef1..6acdf052f 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/ColumnAnalysis.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/ColumnAnalysis.java @@ -21,8 +21,8 @@ package org.datavec.api.transform.analysis.columns; import org.datavec.api.transform.ColumnType; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/DoubleAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/DoubleAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/DoubleAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/DoubleAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/IntegerAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/IntegerAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/IntegerAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/IntegerAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/LongAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/LongAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/LongAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/LongAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NDArrayAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NDArrayAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NDArrayAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NDArrayAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NumericalColumnAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NumericalColumnAnalysis.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NumericalColumnAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NumericalColumnAnalysis.java index 752deb550..14a727274 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NumericalColumnAnalysis.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/NumericalColumnAnalysis.java @@ -25,8 +25,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.analysis.json.TDigestDeserializer; import org.datavec.api.transform.analysis.json.TDigestSerializer; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Data @EqualsAndHashCode(exclude = {"digest"}) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/StringAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/StringAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/StringAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/StringAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/TimeAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/TimeAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/TimeAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/columns/TimeAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/BytesAnalysisCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/BytesAnalysisCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/BytesAnalysisCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/BytesAnalysisCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/CategoricalAnalysisCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/CategoricalAnalysisCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/CategoricalAnalysisCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/CategoricalAnalysisCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/DoubleAnalysisCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/DoubleAnalysisCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/DoubleAnalysisCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/DoubleAnalysisCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/IntegerAnalysisCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/IntegerAnalysisCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/IntegerAnalysisCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/IntegerAnalysisCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/LongAnalysisCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/LongAnalysisCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/LongAnalysisCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/LongAnalysisCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/NDArrayAnalysisCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/NDArrayAnalysisCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/NDArrayAnalysisCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/NDArrayAnalysisCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/StatCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/StatCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/StatCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/StatCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/StringAnalysisCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/StringAnalysisCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/StringAnalysisCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/counter/StringAnalysisCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/CategoricalHistogramCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/CategoricalHistogramCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/CategoricalHistogramCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/CategoricalHistogramCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/DoubleHistogramCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/DoubleHistogramCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/DoubleHistogramCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/DoubleHistogramCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/HistogramCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/HistogramCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/HistogramCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/HistogramCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/NDArrayHistogramCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/NDArrayHistogramCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/NDArrayHistogramCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/NDArrayHistogramCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/StringHistogramCounter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/StringHistogramCounter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/StringHistogramCounter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/histogram/StringHistogramCounter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestDeserializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestDeserializer.java similarity index 86% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestDeserializer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestDeserializer.java index c1c1a9f2f..dd4289906 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestDeserializer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestDeserializer.java @@ -22,11 +22,11 @@ package org.datavec.api.transform.analysis.json; import com.tdunning.math.stats.TDigest; import org.apache.commons.codec.binary.Base64; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestSerializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestSerializer.java similarity index 88% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestSerializer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestSerializer.java index 2208173d2..c3bd4517a 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestSerializer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/json/TDigestSerializer.java @@ -22,10 +22,10 @@ package org.datavec.api.transform.analysis.json; import com.tdunning.math.stats.TDigest; import org.apache.commons.codec.binary.Base64; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisAddFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisAddFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisAddFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisAddFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisCombineFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisCombineFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisCombineFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisCombineFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisState.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisState.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisState.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/QualityAnalysisState.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/bytes/BytesQualityAnalysisState.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/bytes/BytesQualityAnalysisState.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/bytes/BytesQualityAnalysisState.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/bytes/BytesQualityAnalysisState.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityAddFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityAddFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityAddFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityAddFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityAnalysisState.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityAnalysisState.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityAnalysisState.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityAnalysisState.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityMergeFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityMergeFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityMergeFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/categorical/CategoricalQualityMergeFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityAddFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityAddFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityAddFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityAddFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityAnalysisState.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityAnalysisState.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityAnalysisState.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityAnalysisState.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityMergeFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityMergeFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityMergeFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/integer/IntegerQualityMergeFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityAddFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityAddFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityAddFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityAddFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityAnalysisState.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityAnalysisState.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityAnalysisState.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityAnalysisState.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityMergeFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityMergeFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityMergeFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/longq/LongQualityMergeFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityAddFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityAddFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityAddFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityAddFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityAnalysisState.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityAnalysisState.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityAnalysisState.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityAnalysisState.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityMergeFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityMergeFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityMergeFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/real/RealQualityMergeFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityAddFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityAddFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityAddFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityAddFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityAnalysisState.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityAnalysisState.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityAnalysisState.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityAnalysisState.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityMergeFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityMergeFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityMergeFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/string/StringQualityMergeFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityAddFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityAddFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityAddFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityAddFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityAnalysisState.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityAnalysisState.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityAnalysisState.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityAnalysisState.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityMergeFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityMergeFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityMergeFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/quality/time/TimeQualityMergeFunction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/sequence/SequenceLengthAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/sequence/SequenceLengthAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/analysis/sequence/SequenceLengthAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/analysis/sequence/SequenceLengthAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/BooleanCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/BooleanCondition.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/BooleanCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/BooleanCondition.java index a9db25af3..6e128bc66 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/BooleanCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/BooleanCondition.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/Condition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/Condition.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/Condition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/Condition.java index cba0c6930..5928881f7 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/Condition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/Condition.java @@ -23,8 +23,8 @@ package org.datavec.api.transform.condition; import org.datavec.api.transform.ColumnOp; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/ConditionOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/ConditionOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/ConditionOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/ConditionOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/SequenceConditionMode.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/SequenceConditionMode.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/SequenceConditionMode.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/SequenceConditionMode.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/BaseColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/BaseColumnCondition.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/BaseColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/BaseColumnCondition.java index 1333e9c87..f35b28240 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/BaseColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/BaseColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/BooleanColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/BooleanColumnCondition.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/BooleanColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/BooleanColumnCondition.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/CategoricalColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/CategoricalColumnCondition.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/CategoricalColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/CategoricalColumnCondition.java index b5039de78..d10ee29f3 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/CategoricalColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/CategoricalColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.ConditionOp; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Set; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/ColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/ColumnCondition.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/ColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/ColumnCondition.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/DoubleColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/DoubleColumnCondition.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/DoubleColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/DoubleColumnCondition.java index 2f8ac7cba..a749678db 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/DoubleColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/DoubleColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.ConditionOp; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Set; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/FloatColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/FloatColumnCondition.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/FloatColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/FloatColumnCondition.java index f9182bd30..be8ab40e6 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/FloatColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/FloatColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.ConditionOp; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Set; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/InfiniteColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/InfiniteColumnCondition.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/InfiniteColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/InfiniteColumnCondition.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/IntegerColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/IntegerColumnCondition.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/IntegerColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/IntegerColumnCondition.java index 1e1afd7d6..bd55caed5 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/IntegerColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/IntegerColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.ConditionOp; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Set; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/InvalidValueColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/InvalidValueColumnCondition.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/InvalidValueColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/InvalidValueColumnCondition.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/LongColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/LongColumnCondition.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/LongColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/LongColumnCondition.java index 2cc748c74..5855628fa 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/LongColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/LongColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.ConditionOp; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Set; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/NaNColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/NaNColumnCondition.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/NaNColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/NaNColumnCondition.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/NullWritableColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/NullWritableColumnCondition.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/NullWritableColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/NullWritableColumnCondition.java index 3ec14f6f9..6c4819efb 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/NullWritableColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/NullWritableColumnCondition.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.writable.NullWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode(callSuper = true) @Data diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/StringColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/StringColumnCondition.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/StringColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/StringColumnCondition.java index 8cda6223f..c5bee1731 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/StringColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/StringColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.ConditionOp; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Set; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/TimeColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/TimeColumnCondition.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/TimeColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/TimeColumnCondition.java index c654518a5..590ef4522 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/TimeColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/TimeColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.ConditionOp; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Set; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/TrivialColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/TrivialColumnCondition.java similarity index 93% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/TrivialColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/TrivialColumnCondition.java index b06f381de..52a9a6040 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/column/TrivialColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/column/TrivialColumnCondition.java @@ -23,8 +23,8 @@ package org.datavec.api.transform.condition.column; import lombok.Data; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/sequence/SequenceLengthCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/sequence/SequenceLengthCondition.java similarity index 95% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/sequence/SequenceLengthCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/sequence/SequenceLengthCondition.java index 9fe595ee7..15d60608f 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/sequence/SequenceLengthCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/sequence/SequenceLengthCondition.java @@ -26,9 +26,9 @@ import org.datavec.api.transform.condition.Condition; import org.datavec.api.transform.condition.ConditionOp; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import java.util.Set; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/string/StringRegexColumnCondition.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/string/StringRegexColumnCondition.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/string/StringRegexColumnCondition.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/string/StringRegexColumnCondition.java index eff39ff68..4c44c8356 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/condition/string/StringRegexColumnCondition.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/condition/string/StringRegexColumnCondition.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.SequenceConditionMode; import org.datavec.api.transform.condition.column.BaseColumnCondition; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode(callSuper = true) @Data diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/BaseColumnFilter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/BaseColumnFilter.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/BaseColumnFilter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/BaseColumnFilter.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/ConditionFilter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/ConditionFilter.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/ConditionFilter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/ConditionFilter.java index 92e6ecb4e..cc7d24e9e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/ConditionFilter.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/ConditionFilter.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.condition.Condition; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/Filter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/Filter.java similarity index 95% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/Filter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/Filter.java index 8b4cd1e75..ccccc6656 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/Filter.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/Filter.java @@ -23,8 +23,8 @@ package org.datavec.api.transform.filter; import org.datavec.api.transform.ColumnOp; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/FilterInvalidValues.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/FilterInvalidValues.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/FilterInvalidValues.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/FilterInvalidValues.java index fb06f4f5f..54b6cfe07 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/FilterInvalidValues.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/FilterInvalidValues.java @@ -26,7 +26,7 @@ import lombok.ToString; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.*; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/InvalidNumColumns.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/InvalidNumColumns.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/filter/InvalidNumColumns.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/filter/InvalidNumColumns.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/join/Join.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/join/Join.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/join/Join.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/join/Join.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BaseColumnMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BaseColumnMetaData.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BaseColumnMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BaseColumnMetaData.java index ecdda2f07..911d13555 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BaseColumnMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BaseColumnMetaData.java @@ -21,8 +21,10 @@ package org.datavec.api.transform.metadata; import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; @EqualsAndHashCode +@NoArgsConstructor public abstract class BaseColumnMetaData implements ColumnMetaData { protected String name; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BinaryMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BinaryMetaData.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BinaryMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BinaryMetaData.java index 0236b0b48..3acb56ded 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BinaryMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BinaryMetaData.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = true) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BooleanMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BooleanMetaData.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BooleanMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BooleanMetaData.java index 85da6773a..5fae67985 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/BooleanMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/BooleanMetaData.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = true) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/CategoricalMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/CategoricalMetaData.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/CategoricalMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/CategoricalMetaData.java index 1a8773241..95004405d 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/CategoricalMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/CategoricalMetaData.java @@ -24,8 +24,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.HashSet; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/ColumnMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/ColumnMetaData.java similarity index 95% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/ColumnMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/ColumnMetaData.java index b65831496..f13bad69e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/ColumnMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/ColumnMetaData.java @@ -22,8 +22,8 @@ package org.datavec.api.transform.metadata; import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/DoubleMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/DoubleMetaData.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/DoubleMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/DoubleMetaData.java index 41884c388..6a3aee77c 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/DoubleMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/DoubleMetaData.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = true) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/FloatMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/FloatMetaData.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/FloatMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/FloatMetaData.java index 0fa76d8df..69f087433 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/FloatMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/FloatMetaData.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = true) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/IntegerMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/IntegerMetaData.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/IntegerMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/IntegerMetaData.java index 327af31d3..2bf3a2bdc 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/IntegerMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/IntegerMetaData.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = true) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/LongMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/LongMetaData.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/LongMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/LongMetaData.java index 974926186..66a49874d 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/LongMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/LongMetaData.java @@ -26,7 +26,7 @@ import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = true) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/NDArrayMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/NDArrayMetaData.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/NDArrayMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/NDArrayMetaData.java index 13fcbbee8..9449eb780 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/NDArrayMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/NDArrayMetaData.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.NDArrayWritable; import org.datavec.api.writable.Writable; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/StringMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/StringMetaData.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/StringMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/StringMetaData.java index 4c81cba01..bf78d97e8 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/StringMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/StringMetaData.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode(callSuper = true) @Data diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/TimeMetaData.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/TimeMetaData.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/TimeMetaData.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/TimeMetaData.java index e51e3b7d2..c339ffe21 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/metadata/TimeMetaData.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/metadata/TimeMetaData.java @@ -26,7 +26,7 @@ import org.datavec.api.transform.ColumnType; import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.Writable; import org.joda.time.DateTimeZone; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.TimeZone; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java index 19e047812..8be648fa0 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayColumnsMathOpTransform.java @@ -31,7 +31,7 @@ import org.datavec.api.writable.NDArrayWritable; import org.datavec.api.writable.Writable; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayDistanceTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayDistanceTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayDistanceTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayDistanceTransform.java index 447c80f62..3bba47c92 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayDistanceTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayDistanceTransform.java @@ -32,7 +32,7 @@ import org.datavec.api.writable.NDArrayWritable; import org.datavec.api.writable.Writable; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.ops.transforms.Transforms; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java index 1b13438d3..ca5b7921c 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayMathFunctionTransform.java @@ -29,7 +29,7 @@ import org.datavec.api.writable.Writable; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.ops.transforms.Transforms; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class NDArrayMathFunctionTransform extends BaseColumnTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java index 7cd8ed567..4f4dcc4c3 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ndarray/NDArrayScalarOpTransform.java @@ -30,7 +30,7 @@ import org.datavec.api.writable.Writable; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.ops.transforms.Transforms; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class NDArrayScalarOpTransform extends BaseColumnTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/AggregableCheckingOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/AggregableCheckingOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/AggregableCheckingOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/AggregableCheckingOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/AggregableMultiOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/AggregableMultiOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/AggregableMultiOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/AggregableMultiOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/AggregatorImpls.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/AggregatorImpls.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/AggregatorImpls.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/AggregatorImpls.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/ByteWritableOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/ByteWritableOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/ByteWritableOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/ByteWritableOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchWithConditionOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchWithConditionOp.java similarity index 93% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchWithConditionOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchWithConditionOp.java index 3adf79cbe..6f44cac42 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchWithConditionOp.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/DispatchWithConditionOp.java @@ -27,8 +27,8 @@ import org.datavec.api.writable.Writable; import java.util.List; -import static org.nd4j.shade.guava.base.Preconditions.checkArgument; -import static org.nd4j.shade.guava.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; public class DispatchWithConditionOp extends DispatchOp implements IAggregableReduceOp, List> { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/DoubleWritableOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/DoubleWritableOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/DoubleWritableOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/DoubleWritableOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/FloatWritableOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/FloatWritableOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/FloatWritableOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/FloatWritableOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/IAggregableReduceOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/IAggregableReduceOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/IAggregableReduceOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/IAggregableReduceOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/IntWritableOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/IntWritableOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/IntWritableOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/IntWritableOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/LongWritableOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/LongWritableOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/LongWritableOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/LongWritableOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/StringAggregatorImpls.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/StringAggregatorImpls.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/StringAggregatorImpls.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/StringAggregatorImpls.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/StringWritableOp.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/StringWritableOp.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ops/StringWritableOp.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ops/StringWritableOp.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/DataQualityAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/DataQualityAnalysis.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/DataQualityAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/DataQualityAnalysis.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/BytesQuality.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/BytesQuality.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/BytesQuality.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/BytesQuality.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/CategoricalQuality.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/CategoricalQuality.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/CategoricalQuality.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/CategoricalQuality.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/ColumnQuality.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/ColumnQuality.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/ColumnQuality.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/ColumnQuality.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/DoubleQuality.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/DoubleQuality.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/DoubleQuality.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/DoubleQuality.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/IntegerQuality.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/IntegerQuality.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/IntegerQuality.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/IntegerQuality.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/LongQuality.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/LongQuality.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/LongQuality.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/LongQuality.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/StringQuality.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/StringQuality.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/StringQuality.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/StringQuality.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/TimeQuality.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/TimeQuality.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/quality/columns/TimeQuality.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/quality/columns/TimeQuality.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/rank/CalculateSortedRank.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/rank/CalculateSortedRank.java similarity index 95% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/rank/CalculateSortedRank.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/rank/CalculateSortedRank.java index d9469094f..619151051 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/rank/CalculateSortedRank.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/rank/CalculateSortedRank.java @@ -28,10 +28,10 @@ import org.datavec.api.transform.metadata.LongMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.schema.SequenceSchema; import org.datavec.api.writable.comparator.WritableComparator; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.ArrayList; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/AggregableColumnReduction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/AggregableColumnReduction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/AggregableColumnReduction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/AggregableColumnReduction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/AggregableReductionUtils.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/AggregableReductionUtils.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/AggregableReductionUtils.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/AggregableReductionUtils.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/ColumnReduction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/ColumnReduction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/ColumnReduction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/ColumnReduction.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/IAssociativeReducer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/IAssociativeReducer.java similarity index 92% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/IAssociativeReducer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/IAssociativeReducer.java index 677176788..e36830f65 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/IAssociativeReducer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/IAssociativeReducer.java @@ -23,9 +23,9 @@ package org.datavec.api.transform.reduce; import org.datavec.api.transform.ops.IAggregableReduceOp; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/Reducer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/Reducer.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/Reducer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/Reducer.java index 3ef9e5b7c..8536198f9 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/Reducer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/Reducer.java @@ -32,8 +32,8 @@ import org.datavec.api.transform.metadata.*; import org.datavec.api.transform.ops.*; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/impl/GeographicMidpointReduction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/impl/GeographicMidpointReduction.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/impl/GeographicMidpointReduction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/impl/GeographicMidpointReduction.java index 110da950c..27933596f 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/reduce/impl/GeographicMidpointReduction.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/reduce/impl/GeographicMidpointReduction.java @@ -29,7 +29,7 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; import org.nd4j.common.base.Preconditions; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Collections; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/InferredSchema.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/InferredSchema.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/InferredSchema.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/InferredSchema.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java index 4cb692744..003b212b6 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java @@ -27,14 +27,14 @@ import org.datavec.api.transform.metadata.*; import org.datavec.api.transform.serde.JsonMappers; import org.datavec.api.writable.*; import org.joda.time.DateTimeZone; -import org.nd4j.shade.jackson.annotation.*; -import org.nd4j.shade.jackson.core.JsonFactory; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; -import org.nd4j.shade.jackson.databind.exc.InvalidTypeIdException; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; -import org.nd4j.shade.jackson.datatype.joda.JodaModule; +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.joda.JodaModule; import java.io.IOException; import java.io.Serializable; @@ -851,7 +851,7 @@ public class Schema implements Serializable { * @return the infered schema */ public static Schema infer(List record) { - Schema.Builder builder = new Schema.Builder(); + Builder builder = new Builder(); for (int i = 0; i < record.size(); i++) { if (record.get(i) instanceof DoubleWritable) builder.addColumnDouble(String.valueOf(i)); diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/SequenceSchema.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/SequenceSchema.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/SequenceSchema.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/SequenceSchema.java index 2c0aec43e..1a2cfa245 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/SequenceSchema.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/SequenceSchema.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.writable.*; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; @@ -128,7 +128,7 @@ public class SequenceSchema extends Schema { * */ public static SequenceSchema inferSequenceMulti(List>> record) { - SequenceSchema.Builder builder = new SequenceSchema.Builder(); + Builder builder = new Builder(); int minSequenceLength = record.get(0).size(); int maxSequenceLength = record.get(0).size(); for (int i = 0; i < record.size(); i++) { @@ -160,7 +160,7 @@ public class SequenceSchema extends Schema { * */ public static SequenceSchema inferSequence(List> record) { - SequenceSchema.Builder builder = new SequenceSchema.Builder(); + Builder builder = new Builder(); for (int i = 0; i < record.size(); i++) { if (record.get(i) instanceof DoubleWritable) builder.addColumnDouble(String.valueOf(i)); diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/conversion/TypeConversion.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/conversion/TypeConversion.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/conversion/TypeConversion.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/schema/conversion/TypeConversion.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertFromSequence.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertFromSequence.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertFromSequence.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertFromSequence.java index 39ac08534..1920ca3f4 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertFromSequence.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertFromSequence.java @@ -26,7 +26,7 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.schema.SequenceSchema; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertToSequence.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertToSequence.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertToSequence.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertToSequence.java index 15147d3ea..fd98d3b64 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertToSequence.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ConvertToSequence.java @@ -25,8 +25,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.schema.SequenceSchema; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.Collection; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ReduceSequenceTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ReduceSequenceTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ReduceSequenceTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ReduceSequenceTransform.java index 02227ac7a..bb61f9ae9 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/ReduceSequenceTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/ReduceSequenceTransform.java @@ -29,8 +29,8 @@ import org.datavec.api.transform.reduce.IAssociativeReducer; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.schema.SequenceSchema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Collections; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceComparator.java similarity index 92% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceComparator.java index feba082d1..c8616ecda 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceComparator.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceComparator.java @@ -22,8 +22,8 @@ package org.datavec.api.transform.sequence; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.Comparator; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceSplit.java similarity index 94% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceSplit.java index f1fc2fb71..a1a4c4312 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceSplit.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/SequenceSplit.java @@ -22,8 +22,8 @@ package org.datavec.api.transform.sequence; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/BaseColumnComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/BaseColumnComparator.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/BaseColumnComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/BaseColumnComparator.java index a39134034..02f0209fd 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/BaseColumnComparator.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/BaseColumnComparator.java @@ -26,7 +26,7 @@ import org.datavec.api.transform.ColumnOp; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.sequence.SequenceComparator; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/NumericalColumnComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/NumericalColumnComparator.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/NumericalColumnComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/NumericalColumnComparator.java index 9d33fbc3e..419f68e78 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/NumericalColumnComparator.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/NumericalColumnComparator.java @@ -25,8 +25,8 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.ColumnType; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties({"columnType", "schema", "columnIdx"}) @EqualsAndHashCode(callSuper = true, exclude = {"columnType"}) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/StringComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/StringComparator.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/StringComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/StringComparator.java index cf35c5062..9c173b9b6 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/StringComparator.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/comparator/StringComparator.java @@ -22,7 +22,7 @@ package org.datavec.api.transform.sequence.comparator; import lombok.Data; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class StringComparator extends BaseColumnComparator { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/expansion/BaseSequenceExpansionTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/expansion/BaseSequenceExpansionTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/expansion/BaseSequenceExpansionTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/expansion/BaseSequenceExpansionTransform.java index 1a8ff176c..276ff5dff 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/expansion/BaseSequenceExpansionTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/expansion/BaseSequenceExpansionTransform.java @@ -25,8 +25,8 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/merge/SequenceMerge.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/merge/SequenceMerge.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/merge/SequenceMerge.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/merge/SequenceMerge.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SequenceSplitTimeSeparation.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SequenceSplitTimeSeparation.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SequenceSplitTimeSeparation.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SequenceSplitTimeSeparation.java index 890641e97..1ffb60477 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SequenceSplitTimeSeparation.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SequenceSplitTimeSeparation.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.ColumnType; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.sequence.SequenceSplit; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SplitMaxLengthSequence.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SplitMaxLengthSequence.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SplitMaxLengthSequence.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SplitMaxLengthSequence.java index 38e2b1e20..2dca4077e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SplitMaxLengthSequence.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/split/SplitMaxLengthSequence.java @@ -25,8 +25,8 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.sequence.SequenceSplit; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Collections; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimToLengthTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimToLengthTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimToLengthTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimToLengthTransform.java index fb128587f..df873a753 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimToLengthTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimToLengthTransform.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; import org.nd4j.common.base.Preconditions; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimTransform.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimTransform.java index 2b2f15752..ca9cc060b 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/trim/SequenceTrimTransform.java @@ -25,8 +25,8 @@ import lombok.EqualsAndHashCode; import org.datavec.api.transform.Transform; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Collections; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/OverlappingTimeWindowFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/OverlappingTimeWindowFunction.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/OverlappingTimeWindowFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/OverlappingTimeWindowFunction.java index 48096f750..98dbc0c51 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/OverlappingTimeWindowFunction.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/OverlappingTimeWindowFunction.java @@ -30,8 +30,8 @@ import org.datavec.api.transform.schema.SequenceSchema; import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.Writable; import org.joda.time.DateTimeZone; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/ReduceSequenceByWindowTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/ReduceSequenceByWindowTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/ReduceSequenceByWindowTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/ReduceSequenceByWindowTransform.java index e24585aa9..77474f0e6 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/ReduceSequenceByWindowTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/ReduceSequenceByWindowTransform.java @@ -29,8 +29,8 @@ import org.datavec.api.transform.reduce.IAssociativeReducer; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.schema.SequenceSchema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/TimeWindowFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/TimeWindowFunction.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/TimeWindowFunction.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/TimeWindowFunction.java diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/WindowFunction.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/WindowFunction.java new file mode 100644 index 000000000..45e00109f --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/sequence/window/WindowFunction.java @@ -0,0 +1,62 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.sequence.window; + +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.Writable; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface WindowFunction extends Serializable { + + /** + * Apply the windowing function to the given sequence + * @param sequence the input sequence + * @return the sequence with the window function applied + */ + List>> applyToSequence(List> sequence); + + /** + * + * @param schema + */ + void setInputSchema(Schema schema); + + /** + * + * @return + */ + Schema getInputSchema(); + + /** Get the output schema, given the input schema. Typically the output schema is the same as the input schema, + * but not necessarily (for example, if the window function adds columns for the window start/end times) + * @param inputSchema Schema of the input data + * @return Schema of the output windows + */ + Schema transform(Schema inputSchema); + + +} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/BaseSerializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/BaseSerializer.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/BaseSerializer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/BaseSerializer.java index 2f9391543..169b2b174 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/BaseSerializer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/BaseSerializer.java @@ -28,8 +28,8 @@ import org.datavec.api.transform.rank.CalculateSortedRank; import org.datavec.api.transform.reduce.IAssociativeReducer; import org.datavec.api.transform.sequence.SequenceComparator; import org.datavec.api.transform.sequence.SequenceSplit; -import org.nd4j.shade.jackson.core.type.TypeReference; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Arrays; import java.util.List; diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/JsonMappers.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/JsonMappers.java new file mode 100644 index 000000000..7b28c2991 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/JsonMappers.java @@ -0,0 +1,81 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.serde; + +import lombok.extern.slf4j.Slf4j; +import org.datavec.api.transform.serde.legacy.LegacyJsonFormat; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.joda.JodaModule; + +@Slf4j +public class JsonMappers { + + private static ObjectMapper jsonMapper; + private static ObjectMapper yamlMapper; + private static ObjectMapper legacyMapper; //For 1.0.0-alpha and earlier TransformProcess etc + + static { + jsonMapper = new ObjectMapper(); + yamlMapper = new ObjectMapper(new YAMLFactory()); + configureMapper(jsonMapper); + configureMapper(yamlMapper); + } + + public static synchronized ObjectMapper getLegacyMapper(){ + if(legacyMapper == null){ + legacyMapper = LegacyJsonFormat.legacyMapper(); + configureMapper(legacyMapper); + } + return legacyMapper; + } + + /** + * @return The default/primary ObjectMapper for deserializing JSON network configurations in DL4J + */ + public static ObjectMapper getMapper(){ + return jsonMapper; + } + + /** + * @return The default/primary ObjectMapper for deserializing network configurations in DL4J (YAML format) + */ + public static ObjectMapper getMapperYaml() { + return yamlMapper; + } + + private static void configureMapper(ObjectMapper ret) { + ret.registerModule(new JodaModule()); + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); + ret.enable(SerializationFeature.INDENT_OUTPUT); + ret.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); + ret.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + ret.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY); //Need this otherwise JsonProperty annotations on constructors won't be seen + } + +} diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/JsonSerializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/JsonSerializer.java new file mode 100644 index 000000000..90d36ec1c --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/JsonSerializer.java @@ -0,0 +1,37 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.serde; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class JsonSerializer extends BaseSerializer { + + private ObjectMapper om; + + public JsonSerializer() { + this.om = JsonMappers.getMapper(); + } + + @Override + public ObjectMapper getObjectMapper() { + return om; + } +} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/ListWrappers.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/ListWrappers.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/ListWrappers.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/ListWrappers.java index e44b62e7e..8e3b2ac56 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/ListWrappers.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/ListWrappers.java @@ -27,7 +27,7 @@ import org.datavec.api.transform.condition.Condition; import org.datavec.api.transform.filter.Filter; import org.datavec.api.transform.reduce.IAssociativeReducer; import org.datavec.api.transform.sequence.SequenceComparator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/YamlSerializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/YamlSerializer.java similarity index 95% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/YamlSerializer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/YamlSerializer.java index 6e5f75a91..2afe02937 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/YamlSerializer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/YamlSerializer.java @@ -20,7 +20,7 @@ package org.datavec.api.transform.serde; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; public class YamlSerializer extends BaseSerializer { diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/legacy/LegacyJsonFormat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/legacy/LegacyJsonFormat.java new file mode 100644 index 000000000..299434430 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/serde/legacy/LegacyJsonFormat.java @@ -0,0 +1,279 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.serde.legacy; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.datavec.api.transform.Transform; +import org.datavec.api.transform.analysis.columns.*; +import org.datavec.api.transform.condition.BooleanCondition; +import org.datavec.api.transform.condition.Condition; +import org.datavec.api.transform.condition.column.*; +import org.datavec.api.transform.condition.sequence.SequenceLengthCondition; +import org.datavec.api.transform.condition.string.StringRegexColumnCondition; +import org.datavec.api.transform.filter.ConditionFilter; +import org.datavec.api.transform.filter.Filter; +import org.datavec.api.transform.filter.FilterInvalidValues; +import org.datavec.api.transform.filter.InvalidNumColumns; +import org.datavec.api.transform.metadata.*; +import org.datavec.api.transform.ndarray.NDArrayColumnsMathOpTransform; +import org.datavec.api.transform.ndarray.NDArrayDistanceTransform; +import org.datavec.api.transform.ndarray.NDArrayMathFunctionTransform; +import org.datavec.api.transform.ndarray.NDArrayScalarOpTransform; +import org.datavec.api.transform.rank.CalculateSortedRank; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.schema.SequenceSchema; +import org.datavec.api.transform.sequence.ReduceSequenceTransform; +import org.datavec.api.transform.sequence.SequenceComparator; +import org.datavec.api.transform.sequence.SequenceSplit; +import org.datavec.api.transform.sequence.comparator.NumericalColumnComparator; +import org.datavec.api.transform.sequence.comparator.StringComparator; +import org.datavec.api.transform.sequence.split.SequenceSplitTimeSeparation; +import org.datavec.api.transform.sequence.split.SplitMaxLengthSequence; +import org.datavec.api.transform.sequence.trim.SequenceTrimTransform; +import org.datavec.api.transform.sequence.window.OverlappingTimeWindowFunction; +import org.datavec.api.transform.sequence.window.ReduceSequenceByWindowTransform; +import org.datavec.api.transform.sequence.window.TimeWindowFunction; +import org.datavec.api.transform.sequence.window.WindowFunction; +import org.datavec.api.transform.stringreduce.IStringReducer; +import org.datavec.api.transform.stringreduce.StringReducer; +import org.datavec.api.transform.transform.categorical.*; +import org.datavec.api.transform.transform.column.*; +import org.datavec.api.transform.transform.condition.ConditionalCopyValueTransform; +import org.datavec.api.transform.transform.condition.ConditionalReplaceValueTransform; +import org.datavec.api.transform.transform.condition.ConditionalReplaceValueTransformWithDefault; +import org.datavec.api.transform.transform.doubletransform.*; +import org.datavec.api.transform.transform.integer.*; +import org.datavec.api.transform.transform.longtransform.LongColumnsMathOpTransform; +import org.datavec.api.transform.transform.longtransform.LongMathOpTransform; +import org.datavec.api.transform.transform.nlp.TextToCharacterIndexTransform; +import org.datavec.api.transform.transform.parse.ParseDoubleTransform; +import org.datavec.api.transform.transform.sequence.SequenceDifferenceTransform; +import org.datavec.api.transform.transform.sequence.SequenceMovingWindowReduceTransform; +import org.datavec.api.transform.transform.sequence.SequenceOffsetTransform; +import org.datavec.api.transform.transform.string.*; +import org.datavec.api.transform.transform.time.DeriveColumnsFromTimeTransform; +import org.datavec.api.transform.transform.time.StringToTimeTransform; +import org.datavec.api.transform.transform.time.TimeMathOpTransform; +import org.datavec.api.writable.*; +import org.datavec.api.writable.comparator.*; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class LegacyJsonFormat { + + private LegacyJsonFormat(){ } + + /** + * Get a mapper (minus general config) suitable for loading old format JSON - 1.0.0-alpha and before + * @return Object mapper + */ + public static ObjectMapper legacyMapper(){ + ObjectMapper om = new ObjectMapper(); + om.addMixIn(Schema.class, SchemaMixin.class); + om.addMixIn(ColumnMetaData.class, ColumnMetaDataMixin.class); + om.addMixIn(Transform.class, TransformMixin.class); + om.addMixIn(Condition.class, ConditionMixin.class); + om.addMixIn(Writable.class, WritableMixin.class); + om.addMixIn(Filter.class, FilterMixin.class); + om.addMixIn(SequenceComparator.class, SequenceComparatorMixin.class); + om.addMixIn(SequenceSplit.class, SequenceSplitMixin.class); + om.addMixIn(WindowFunction.class, WindowFunctionMixin.class); + om.addMixIn(CalculateSortedRank.class, CalculateSortedRankMixin.class); + om.addMixIn(WritableComparator.class, WritableComparatorMixin.class); + om.addMixIn(ColumnAnalysis.class, ColumnAnalysisMixin.class); + om.addMixIn(IStringReducer.class, IStringReducerMixin.class); + return om; + } + + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes({@JsonSubTypes.Type(value = Schema.class, name = "Schema"), + @JsonSubTypes.Type(value = SequenceSchema.class, name = "SequenceSchema")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class SchemaMixin { } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes({@JsonSubTypes.Type(value = BinaryMetaData.class, name = "Binary"), + @JsonSubTypes.Type(value = BooleanMetaData.class, name = "Boloean"), + @JsonSubTypes.Type(value = CategoricalMetaData.class, name = "Categorical"), + @JsonSubTypes.Type(value = DoubleMetaData.class, name = "Double"), + @JsonSubTypes.Type(value = FloatMetaData.class, name = "Float"), + @JsonSubTypes.Type(value = IntegerMetaData.class, name = "Integer"), + @JsonSubTypes.Type(value = LongMetaData.class, name = "Long"), + @JsonSubTypes.Type(value = NDArrayMetaData.class, name = "NDArray"), + @JsonSubTypes.Type(value = StringMetaData.class, name = "String"), + @JsonSubTypes.Type(value = TimeMetaData.class, name = "Time")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class ColumnMetaDataMixin { } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = CalculateSortedRank.class, name = "CalculateSortedRank"), + @JsonSubTypes.Type(value = CategoricalToIntegerTransform.class, name = "CategoricalToIntegerTransform"), + @JsonSubTypes.Type(value = CategoricalToOneHotTransform.class, name = "CategoricalToOneHotTransform"), + @JsonSubTypes.Type(value = IntegerToCategoricalTransform.class, name = "IntegerToCategoricalTransform"), + @JsonSubTypes.Type(value = StringToCategoricalTransform.class, name = "StringToCategoricalTransform"), + @JsonSubTypes.Type(value = DuplicateColumnsTransform.class, name = "DuplicateColumnsTransform"), + @JsonSubTypes.Type(value = RemoveColumnsTransform.class, name = "RemoveColumnsTransform"), + @JsonSubTypes.Type(value = RenameColumnsTransform.class, name = "RenameColumnsTransform"), + @JsonSubTypes.Type(value = ReorderColumnsTransform.class, name = "ReorderColumnsTransform"), + @JsonSubTypes.Type(value = ConditionalCopyValueTransform.class, name = "ConditionalCopyValueTransform"), + @JsonSubTypes.Type(value = ConditionalReplaceValueTransform.class, name = "ConditionalReplaceValueTransform"), + @JsonSubTypes.Type(value = ConditionalReplaceValueTransformWithDefault.class, name = "ConditionalReplaceValueTransformWithDefault"), + @JsonSubTypes.Type(value = DoubleColumnsMathOpTransform.class, name = "DoubleColumnsMathOpTransform"), + @JsonSubTypes.Type(value = DoubleMathOpTransform.class, name = "DoubleMathOpTransform"), + @JsonSubTypes.Type(value = Log2Normalizer.class, name = "Log2Normalizer"), + @JsonSubTypes.Type(value = MinMaxNormalizer.class, name = "MinMaxNormalizer"), + @JsonSubTypes.Type(value = StandardizeNormalizer.class, name = "StandardizeNormalizer"), + @JsonSubTypes.Type(value = SubtractMeanNormalizer.class, name = "SubtractMeanNormalizer"), + @JsonSubTypes.Type(value = IntegerColumnsMathOpTransform.class, name = "IntegerColumnsMathOpTransform"), + @JsonSubTypes.Type(value = IntegerMathOpTransform.class, name = "IntegerMathOpTransform"), + @JsonSubTypes.Type(value = ReplaceEmptyIntegerWithValueTransform.class, name = "ReplaceEmptyIntegerWithValueTransform"), + @JsonSubTypes.Type(value = ReplaceInvalidWithIntegerTransform.class, name = "ReplaceInvalidWithIntegerTransform"), + @JsonSubTypes.Type(value = LongColumnsMathOpTransform.class, name = "LongColumnsMathOpTransform"), + @JsonSubTypes.Type(value = LongMathOpTransform.class, name = "LongMathOpTransform"), + @JsonSubTypes.Type(value = MapAllStringsExceptListTransform.class, name = "MapAllStringsExceptListTransform"), + @JsonSubTypes.Type(value = RemoveWhiteSpaceTransform.class, name = "RemoveWhiteSpaceTransform"), + @JsonSubTypes.Type(value = ReplaceEmptyStringTransform.class, name = "ReplaceEmptyStringTransform"), + @JsonSubTypes.Type(value = ReplaceStringTransform.class, name = "ReplaceStringTransform"), + @JsonSubTypes.Type(value = StringListToCategoricalSetTransform.class, name = "StringListToCategoricalSetTransform"), + @JsonSubTypes.Type(value = StringMapTransform.class, name = "StringMapTransform"), + @JsonSubTypes.Type(value = DeriveColumnsFromTimeTransform.class, name = "DeriveColumnsFromTimeTransform"), + @JsonSubTypes.Type(value = StringToTimeTransform.class, name = "StringToTimeTransform"), + @JsonSubTypes.Type(value = TimeMathOpTransform.class, name = "TimeMathOpTransform"), + @JsonSubTypes.Type(value = ReduceSequenceByWindowTransform.class, name = "ReduceSequenceByWindowTransform"), + @JsonSubTypes.Type(value = DoubleMathFunctionTransform.class, name = "DoubleMathFunctionTransform"), + @JsonSubTypes.Type(value = AddConstantColumnTransform.class, name = "AddConstantColumnTransform"), + @JsonSubTypes.Type(value = RemoveAllColumnsExceptForTransform.class, name = "RemoveAllColumnsExceptForTransform"), + @JsonSubTypes.Type(value = ParseDoubleTransform.class, name = "ParseDoubleTransform"), + @JsonSubTypes.Type(value = ConvertToString.class, name = "ConvertToStringTransform"), + @JsonSubTypes.Type(value = AppendStringColumnTransform.class, name = "AppendStringColumnTransform"), + @JsonSubTypes.Type(value = SequenceDifferenceTransform.class, name = "SequenceDifferenceTransform"), + @JsonSubTypes.Type(value = ReduceSequenceTransform.class, name = "ReduceSequenceTransform"), + @JsonSubTypes.Type(value = SequenceMovingWindowReduceTransform.class, name = "SequenceMovingWindowReduceTransform"), + @JsonSubTypes.Type(value = IntegerToOneHotTransform.class, name = "IntegerToOneHotTransform"), + @JsonSubTypes.Type(value = SequenceTrimTransform.class, name = "SequenceTrimTransform"), + @JsonSubTypes.Type(value = SequenceOffsetTransform.class, name = "SequenceOffsetTransform"), + @JsonSubTypes.Type(value = NDArrayColumnsMathOpTransform.class, name = "NDArrayColumnsMathOpTransform"), + @JsonSubTypes.Type(value = NDArrayDistanceTransform.class, name = "NDArrayDistanceTransform"), + @JsonSubTypes.Type(value = NDArrayMathFunctionTransform.class, name = "NDArrayMathFunctionTransform"), + @JsonSubTypes.Type(value = NDArrayScalarOpTransform.class, name = "NDArrayScalarOpTransform"), + @JsonSubTypes.Type(value = ChangeCaseStringTransform.class, name = "ChangeCaseStringTransform"), + @JsonSubTypes.Type(value = ConcatenateStringColumns.class, name = "ConcatenateStringColumns"), + @JsonSubTypes.Type(value = StringListToCountsNDArrayTransform.class, name = "StringListToCountsNDArrayTransform"), + @JsonSubTypes.Type(value = StringListToIndicesNDArrayTransform.class, name = "StringListToIndicesNDArrayTransform"), + @JsonSubTypes.Type(value = PivotTransform.class, name = "PivotTransform"), + @JsonSubTypes.Type(value = TextToCharacterIndexTransform.class, name = "TextToCharacterIndexTransform")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class TransformMixin { } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = TrivialColumnCondition.class, name = "TrivialColumnCondition"), + @JsonSubTypes.Type(value = CategoricalColumnCondition.class, name = "CategoricalColumnCondition"), + @JsonSubTypes.Type(value = DoubleColumnCondition.class, name = "DoubleColumnCondition"), + @JsonSubTypes.Type(value = IntegerColumnCondition.class, name = "IntegerColumnCondition"), + @JsonSubTypes.Type(value = LongColumnCondition.class, name = "LongColumnCondition"), + @JsonSubTypes.Type(value = NullWritableColumnCondition.class, name = "NullWritableColumnCondition"), + @JsonSubTypes.Type(value = StringColumnCondition.class, name = "StringColumnCondition"), + @JsonSubTypes.Type(value = TimeColumnCondition.class, name = "TimeColumnCondition"), + @JsonSubTypes.Type(value = StringRegexColumnCondition.class, name = "StringRegexColumnCondition"), + @JsonSubTypes.Type(value = BooleanCondition.class, name = "BooleanCondition"), + @JsonSubTypes.Type(value = NaNColumnCondition.class, name = "NaNColumnCondition"), + @JsonSubTypes.Type(value = InfiniteColumnCondition.class, name = "InfiniteColumnCondition"), + @JsonSubTypes.Type(value = SequenceLengthCondition.class, name = "SequenceLengthCondition")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class ConditionMixin { } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = ArrayWritable.class, name = "ArrayWritable"), + @JsonSubTypes.Type(value = BooleanWritable.class, name = "BooleanWritable"), + @JsonSubTypes.Type(value = ByteWritable.class, name = "ByteWritable"), + @JsonSubTypes.Type(value = DoubleWritable.class, name = "DoubleWritable"), + @JsonSubTypes.Type(value = FloatWritable.class, name = "FloatWritable"), + @JsonSubTypes.Type(value = IntWritable.class, name = "IntWritable"), + @JsonSubTypes.Type(value = LongWritable.class, name = "LongWritable"), + @JsonSubTypes.Type(value = NullWritable.class, name = "NullWritable"), + @JsonSubTypes.Type(value = Text.class, name = "Text"), + @JsonSubTypes.Type(value = BytesWritable.class, name = "BytesWritable")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class WritableMixin { } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = ConditionFilter.class, name = "ConditionFilter"), + @JsonSubTypes.Type(value = FilterInvalidValues.class, name = "FilterInvalidValues"), + @JsonSubTypes.Type(value = InvalidNumColumns.class, name = "InvalidNumCols")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class FilterMixin { } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = NumericalColumnComparator.class, name = "NumericalColumnComparator"), + @JsonSubTypes.Type(value = StringComparator.class, name = "StringComparator")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class SequenceComparatorMixin { } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = SequenceSplitTimeSeparation.class, name = "SequenceSplitTimeSeparation"), + @JsonSubTypes.Type(value = SplitMaxLengthSequence.class, name = "SplitMaxLengthSequence")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class SequenceSplitMixin { } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = TimeWindowFunction.class, name = "TimeWindowFunction"), + @JsonSubTypes.Type(value = OverlappingTimeWindowFunction.class, name = "OverlappingTimeWindowFunction")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class WindowFunctionMixin { } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = CalculateSortedRank.class, name = "CalculateSortedRank")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class CalculateSortedRankMixin { } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = DoubleWritableComparator.class, name = "DoubleWritableComparator"), + @JsonSubTypes.Type(value = FloatWritableComparator.class, name = "FloatWritableComparator"), + @JsonSubTypes.Type(value = IntWritableComparator.class, name = "IntWritableComparator"), + @JsonSubTypes.Type(value = LongWritableComparator.class, name = "LongWritableComparator"), + @JsonSubTypes.Type(value = TextWritableComparator.class, name = "TextWritableComparator")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class WritableComparatorMixin { } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = BytesAnalysis.class, name = "BytesAnalysis"), + @JsonSubTypes.Type(value = CategoricalAnalysis.class, name = "CategoricalAnalysis"), + @JsonSubTypes.Type(value = DoubleAnalysis.class, name = "DoubleAnalysis"), + @JsonSubTypes.Type(value = IntegerAnalysis.class, name = "IntegerAnalysis"), + @JsonSubTypes.Type(value = LongAnalysis.class, name = "LongAnalysis"), + @JsonSubTypes.Type(value = StringAnalysis.class, name = "StringAnalysis"), + @JsonSubTypes.Type(value = TimeAnalysis.class, name = "TimeAnalysis")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class ColumnAnalysisMixin{ } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = StringReducer.class, name = "StringReducer")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class IStringReducerMixin{ } +} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/split/RandomSplit.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/split/RandomSplit.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/split/RandomSplit.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/split/RandomSplit.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/split/SplitStrategy.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/split/SplitStrategy.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/split/SplitStrategy.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/split/SplitStrategy.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/stringreduce/IStringReducer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/stringreduce/IStringReducer.java similarity index 94% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/stringreduce/IStringReducer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/stringreduce/IStringReducer.java index 571762920..3cebf70a1 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/stringreduce/IStringReducer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/stringreduce/IStringReducer.java @@ -22,8 +22,8 @@ package org.datavec.api.transform.stringreduce; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/stringreduce/StringReducer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/stringreduce/StringReducer.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/stringreduce/StringReducer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/stringreduce/StringReducer.java index a487dc09c..907bd7d0c 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/stringreduce/StringReducer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/stringreduce/StringReducer.java @@ -30,8 +30,8 @@ import org.datavec.api.transform.reduce.ColumnReduction; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnTransform.java index f943e4618..6bea20d6c 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnTransform.java @@ -26,7 +26,7 @@ import org.datavec.api.transform.ColumnOp; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.ArrayList; import java.util.Iterator; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnsMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnsMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnsMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnsMathOpTransform.java index 59d3ed4ac..d0bda6912 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnsMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseColumnsMathOpTransform.java @@ -31,8 +31,8 @@ import org.datavec.api.transform.transform.doubletransform.DoubleMathOpTransform import org.datavec.api.transform.transform.integer.IntegerMathOpTransform; import org.datavec.api.transform.transform.longtransform.LongMathOpTransform; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseTransform.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseTransform.java index 8be70ab74..7c5735364 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/BaseTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/BaseTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import org.datavec.api.transform.Transform; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToIntegerTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToIntegerTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToIntegerTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToIntegerTransform.java index b2e00596f..5afd6564e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToIntegerTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToIntegerTransform.java @@ -28,8 +28,8 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.BaseTransform; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToOneHotTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToOneHotTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToOneHotTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToOneHotTransform.java index abbfc4169..56687431c 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToOneHotTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/CategoricalToOneHotTransform.java @@ -28,8 +28,8 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.BaseTransform; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/FirstDigitTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/FirstDigitTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/FirstDigitTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/FirstDigitTransform.java index 3284ed980..8c09737a4 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/FirstDigitTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/FirstDigitTransform.java @@ -27,8 +27,8 @@ import org.datavec.api.transform.transform.BaseTransform; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; import org.nd4j.common.base.Preconditions; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; @@ -57,7 +57,7 @@ public class FirstDigitTransform extends BaseTransform { /** * @param inputColumn Input column name * @param outputColumn Output column name. If same as input, input column is replaced - * @param mode See {@link FirstDigitTransform.Mode} + * @param mode See {@link Mode} */ public FirstDigitTransform(@JsonProperty("inputColumn") String inputColumn, @JsonProperty("outputColumn") String outputColumn, @JsonProperty("mode") Mode mode){ diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/IntegerToCategoricalTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/IntegerToCategoricalTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/IntegerToCategoricalTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/IntegerToCategoricalTransform.java index 97f0cf5bd..e4f9debf9 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/IntegerToCategoricalTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/IntegerToCategoricalTransform.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/PivotTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/PivotTransform.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/PivotTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/PivotTransform.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/StringToCategoricalTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/StringToCategoricalTransform.java similarity index 95% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/StringToCategoricalTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/StringToCategoricalTransform.java index 3d6a9bccd..6e3dd7172 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/StringToCategoricalTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/categorical/StringToCategoricalTransform.java @@ -25,8 +25,8 @@ import org.datavec.api.transform.metadata.CategoricalMetaData; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/AddConstantColumnTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/AddConstantColumnTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/AddConstantColumnTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/AddConstantColumnTransform.java index 79cb0c3ca..48a191f35 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/AddConstantColumnTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/AddConstantColumnTransform.java @@ -26,7 +26,7 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/DuplicateColumnsTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/DuplicateColumnsTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/DuplicateColumnsTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/DuplicateColumnsTransform.java index 0e436591a..41f857c1a 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/DuplicateColumnsTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/DuplicateColumnsTransform.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.HashSet; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveAllColumnsExceptForTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveAllColumnsExceptForTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveAllColumnsExceptForTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveAllColumnsExceptForTransform.java index 6b9111a8c..f71ab0d99 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveAllColumnsExceptForTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveAllColumnsExceptForTransform.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.BaseTransform; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveColumnsTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveColumnsTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveColumnsTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveColumnsTransform.java index dda21ff4c..d5177a055 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveColumnsTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RemoveColumnsTransform.java @@ -27,8 +27,8 @@ import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.BaseTransform; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RenameColumnsTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RenameColumnsTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RenameColumnsTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RenameColumnsTransform.java index 7aa35d91d..d50e52a70 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/RenameColumnsTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/RenameColumnsTransform.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Collections; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/ReorderColumnsTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/ReorderColumnsTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/ReorderColumnsTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/ReorderColumnsTransform.java index 393230b92..0d0deb76e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/column/ReorderColumnsTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/column/ReorderColumnsTransform.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalCopyValueTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalCopyValueTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalCopyValueTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalCopyValueTransform.java index 9a0b89d65..809a11457 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalCopyValueTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalCopyValueTransform.java @@ -27,8 +27,8 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.condition.Condition; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransform.java index f37bdc5a3..435eee6e7 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransform.java @@ -27,8 +27,8 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.condition.Condition; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransformWithDefault.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransformWithDefault.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransformWithDefault.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransformWithDefault.java index 8df7a4131..e64b84e89 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransformWithDefault.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/condition/ConditionalReplaceValueTransformWithDefault.java @@ -27,8 +27,8 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.condition.Condition; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/BaseDoubleTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/BaseDoubleTransform.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/BaseDoubleTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/BaseDoubleTransform.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/ConvertToDouble.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/ConvertToDouble.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/ConvertToDouble.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/ConvertToDouble.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleColumnsMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleColumnsMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleColumnsMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleColumnsMathOpTransform.java index 2137a9f6b..596eb737e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleColumnsMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleColumnsMathOpTransform.java @@ -28,7 +28,7 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.BaseColumnsMathOpTransform; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathFunctionTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathFunctionTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathFunctionTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathFunctionTransform.java index 814f7aefe..e06426643 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathFunctionTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathFunctionTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import org.datavec.api.transform.MathFunction; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class DoubleMathFunctionTransform extends BaseDoubleTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathOpTransform.java index 3def70370..a2c6e1821 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/DoubleMathOpTransform.java @@ -27,7 +27,7 @@ import org.datavec.api.transform.metadata.DoubleMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/Log2Normalizer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/Log2Normalizer.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/Log2Normalizer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/Log2Normalizer.java index d1a22f615..c00a2be58 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/Log2Normalizer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/Log2Normalizer.java @@ -25,7 +25,7 @@ import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.metadata.DoubleMetaData; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class Log2Normalizer extends BaseDoubleTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/MinMaxNormalizer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/MinMaxNormalizer.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/MinMaxNormalizer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/MinMaxNormalizer.java index 879db28c6..79a3dc02c 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/MinMaxNormalizer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/MinMaxNormalizer.java @@ -25,8 +25,8 @@ import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.metadata.DoubleMetaData; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @JsonIgnoreProperties({"ratio", "inputSchema", "columnNumber"}) diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/StandardizeNormalizer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/StandardizeNormalizer.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/StandardizeNormalizer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/StandardizeNormalizer.java index 9899f6d2d..bb12b6541 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/StandardizeNormalizer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/StandardizeNormalizer.java @@ -23,7 +23,7 @@ package org.datavec.api.transform.transform.doubletransform; import lombok.Data; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class StandardizeNormalizer extends BaseDoubleTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/SubtractMeanNormalizer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/SubtractMeanNormalizer.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/SubtractMeanNormalizer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/SubtractMeanNormalizer.java index 32d61ac5c..5f2deb32d 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/SubtractMeanNormalizer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/doubletransform/SubtractMeanNormalizer.java @@ -23,7 +23,7 @@ package org.datavec.api.transform.transform.doubletransform; import lombok.Data; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class SubtractMeanNormalizer extends BaseDoubleTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/BaseFloatTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/BaseFloatTransform.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/BaseFloatTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/BaseFloatTransform.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/ConvertToFloat.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/ConvertToFloat.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/ConvertToFloat.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/ConvertToFloat.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatColumnsMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatColumnsMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatColumnsMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatColumnsMathOpTransform.java index ec0930a76..b45fa9f82 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatColumnsMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatColumnsMathOpTransform.java @@ -29,7 +29,7 @@ import org.datavec.api.transform.transform.BaseColumnsMathOpTransform; import org.datavec.api.transform.transform.floattransform.FloatMathOpTransform; import org.datavec.api.writable.FloatWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathFunctionTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathFunctionTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathFunctionTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathFunctionTransform.java index 7ab36e9d9..0054750f5 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathFunctionTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathFunctionTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import org.datavec.api.transform.MathFunction; import org.datavec.api.writable.FloatWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class FloatMathFunctionTransform extends BaseFloatTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathOpTransform.java index bab756de0..a980c289a 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/floattransform/FloatMathOpTransform.java @@ -28,7 +28,7 @@ import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.transform.transform.floattransform.FloatColumnsMathOpTransform; import org.datavec.api.writable.FloatWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/BaseIntegerTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/BaseIntegerTransform.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/BaseIntegerTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/BaseIntegerTransform.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ConvertToInteger.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ConvertToInteger.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ConvertToInteger.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ConvertToInteger.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerColumnsMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerColumnsMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerColumnsMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerColumnsMathOpTransform.java index d8b497444..878123df0 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerColumnsMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerColumnsMathOpTransform.java @@ -29,7 +29,7 @@ import org.datavec.api.transform.transform.BaseColumnsMathOpTransform; import org.datavec.api.transform.transform.doubletransform.DoubleColumnsMathOpTransform; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerMathOpTransform.java index 19d1d005e..1eac7f2db 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerMathOpTransform.java @@ -27,7 +27,7 @@ import org.datavec.api.transform.metadata.IntegerMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class IntegerMathOpTransform extends BaseColumnTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerToOneHotTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerToOneHotTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerToOneHotTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerToOneHotTransform.java index edcc6a006..ca27348ae 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerToOneHotTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/IntegerToOneHotTransform.java @@ -28,8 +28,8 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.BaseTransform; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Iterator; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceEmptyIntegerWithValueTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceEmptyIntegerWithValueTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceEmptyIntegerWithValueTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceEmptyIntegerWithValueTransform.java index bb0c79662..8e3e44412 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceEmptyIntegerWithValueTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceEmptyIntegerWithValueTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode(callSuper = true) @Data diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceInvalidWithIntegerTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceInvalidWithIntegerTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceInvalidWithIntegerTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceInvalidWithIntegerTransform.java index 7aaeb5ae4..cde5b1182 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceInvalidWithIntegerTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/integer/ReplaceInvalidWithIntegerTransform.java @@ -23,7 +23,7 @@ package org.datavec.api.transform.transform.integer; import lombok.Data; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class ReplaceInvalidWithIntegerTransform extends BaseIntegerTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongColumnsMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongColumnsMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongColumnsMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongColumnsMathOpTransform.java index 3c3ab0383..cf1211dc7 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongColumnsMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongColumnsMathOpTransform.java @@ -29,7 +29,7 @@ import org.datavec.api.transform.transform.BaseColumnsMathOpTransform; import org.datavec.api.transform.transform.doubletransform.DoubleColumnsMathOpTransform; import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongMathOpTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongMathOpTransform.java index 54c1bdbea..365edefa6 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/longtransform/LongMathOpTransform.java @@ -27,7 +27,7 @@ import org.datavec.api.transform.metadata.LongMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class LongMathOpTransform extends BaseColumnTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToCharacterIndexTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToCharacterIndexTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToCharacterIndexTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToCharacterIndexTransform.java index 82b3139eb..c882a76a2 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToCharacterIndexTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToCharacterIndexTransform.java @@ -27,7 +27,7 @@ import org.datavec.api.transform.metadata.IntegerMetaData; import org.datavec.api.transform.sequence.expansion.BaseSequenceExpansionTransform; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToTermIndexSequenceTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToTermIndexSequenceTransform.java similarity index 96% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToTermIndexSequenceTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToTermIndexSequenceTransform.java index 1603bba82..9adbf1771 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToTermIndexSequenceTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/nlp/TextToTermIndexSequenceTransform.java @@ -27,9 +27,9 @@ import org.datavec.api.transform.metadata.IntegerMetaData; import org.datavec.api.transform.sequence.expansion.BaseSequenceExpansionTransform; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/normalize/Normalize.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/normalize/Normalize.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/normalize/Normalize.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/normalize/Normalize.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/parse/ParseDoubleTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/parse/ParseDoubleTransform.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/parse/ParseDoubleTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/parse/ParseDoubleTransform.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceDifferenceTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceDifferenceTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceDifferenceTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceDifferenceTransform.java index 0861dd22b..61bc30796 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceDifferenceTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceDifferenceTransform.java @@ -27,8 +27,8 @@ import org.datavec.api.transform.metadata.*; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.schema.SequenceSchema; import org.datavec.api.writable.*; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceMovingWindowReduceTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceMovingWindowReduceTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceMovingWindowReduceTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceMovingWindowReduceTransform.java index 4cca4de30..11895d47f 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceMovingWindowReduceTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceMovingWindowReduceTransform.java @@ -32,9 +32,9 @@ import org.datavec.api.transform.reduce.AggregableReductionUtils; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.schema.SequenceSchema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Collections; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceOffsetTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceOffsetTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceOffsetTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceOffsetTransform.java index 47397acda..eeba657c5 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceOffsetTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/sequence/SequenceOffsetTransform.java @@ -27,9 +27,9 @@ import org.datavec.api.transform.Transform; import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/AppendStringColumnTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/AppendStringColumnTransform.java similarity index 95% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/AppendStringColumnTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/AppendStringColumnTransform.java index ca3069b45..2b7aa6de2 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/AppendStringColumnTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/AppendStringColumnTransform.java @@ -26,8 +26,8 @@ import org.datavec.api.transform.metadata.StringMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties({"inputSchema", "columnNumber"}) @Data diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/BaseStringTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/BaseStringTransform.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/BaseStringTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/BaseStringTransform.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ChangeCaseStringTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ChangeCaseStringTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ChangeCaseStringTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ChangeCaseStringTransform.java index 3fe18e354..cf55bddfc 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ChangeCaseStringTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ChangeCaseStringTransform.java @@ -23,7 +23,7 @@ package org.datavec.api.transform.transform.string; import lombok.Data; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class ChangeCaseStringTransform extends BaseStringTransform { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConcatenateStringColumns.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConcatenateStringColumns.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConcatenateStringColumns.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConcatenateStringColumns.java index c1d1b5a87..6e0ae78fa 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConcatenateStringColumns.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConcatenateStringColumns.java @@ -28,8 +28,8 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.BaseTransform; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConvertToString.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConvertToString.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConvertToString.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ConvertToString.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/MapAllStringsExceptListTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/MapAllStringsExceptListTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/MapAllStringsExceptListTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/MapAllStringsExceptListTransform.java index 9421a191b..6bc6e0898 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/MapAllStringsExceptListTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/MapAllStringsExceptListTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.HashSet; import java.util.List; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/RemoveWhiteSpaceTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/RemoveWhiteSpaceTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/RemoveWhiteSpaceTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/RemoveWhiteSpaceTransform.java index 5530b7826..ae16a92a5 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/RemoveWhiteSpaceTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/RemoveWhiteSpaceTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode(callSuper = true) @Data diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceEmptyStringTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceEmptyStringTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceEmptyStringTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceEmptyStringTransform.java index bd5741812..1023e7e22 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceEmptyStringTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceEmptyStringTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode(callSuper = true) @Data diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceStringTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceStringTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceStringTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceStringTransform.java index 93b480189..a10056749 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceStringTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/ReplaceStringTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Map; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCategoricalSetTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCategoricalSetTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCategoricalSetTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCategoricalSetTransform.java index a8b7b159e..83d56fd7e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCategoricalSetTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCategoricalSetTransform.java @@ -29,8 +29,8 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.BaseTransform; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.*; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCountsNDArrayTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCountsNDArrayTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCountsNDArrayTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCountsNDArrayTransform.java index 7eda354d0..9f3ff0dcf 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCountsNDArrayTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToCountsNDArrayTransform.java @@ -32,8 +32,8 @@ import org.datavec.api.writable.NDArrayWritable; import org.datavec.api.writable.Writable; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.File; import java.io.IOException; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java index 4818cc915..19a03ce83 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringListToIndicesNDArrayTransform.java @@ -23,7 +23,7 @@ package org.datavec.api.transform.transform.string; import lombok.Data; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Collection; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringMapTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringMapTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringMapTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringMapTransform.java index 45971c629..7a39aa577 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringMapTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/string/StringMapTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Map; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/DeriveColumnsFromTimeTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/DeriveColumnsFromTimeTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/DeriveColumnsFromTimeTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/DeriveColumnsFromTimeTransform.java index e8674cc04..d1e290f7a 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/DeriveColumnsFromTimeTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/DeriveColumnsFromTimeTransform.java @@ -39,11 +39,11 @@ import org.joda.time.DateTimeFieldType; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.IOException; import java.io.ObjectInputStream; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/StringToTimeTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/StringToTimeTransform.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/StringToTimeTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/StringToTimeTransform.java index fc73926be..e5141f2c2 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/StringToTimeTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/StringToTimeTransform.java @@ -30,8 +30,8 @@ import org.datavec.api.writable.Writable; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.IOException; import java.io.ObjectInputStream; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/TimeMathOpTransform.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/TimeMathOpTransform.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/TimeMathOpTransform.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/TimeMathOpTransform.java index 2268281d5..1c5e5fb0c 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/transform/time/TimeMathOpTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/transform/time/TimeMathOpTransform.java @@ -27,8 +27,8 @@ import org.datavec.api.transform.metadata.TimeMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.concurrent.TimeUnit; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/DivObject.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/DivObject.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/DivObject.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/DivObject.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlAnalysis.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlAnalysis.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlAnalysis.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlAnalysis.java index b4fdc3a97..d1f272d0e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlAnalysis.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlAnalysis.java @@ -36,10 +36,10 @@ import org.datavec.api.transform.ui.components.RenderableComponentTable; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import java.io.File; import java.io.StringWriter; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlSequencePlotting.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlSequencePlotting.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlSequencePlotting.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlSequencePlotting.java index b9baa5165..f7b40bdbb 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlSequencePlotting.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/HtmlSequencePlotting.java @@ -35,10 +35,10 @@ import org.datavec.api.writable.Writable; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import java.io.File; import java.io.StringWriter; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponent.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponent.java similarity index 94% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponent.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponent.java index 82efb103b..32f60546a 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponent.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponent.java @@ -21,8 +21,8 @@ package org.datavec.api.transform.ui.components; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) @JsonSubTypes(value = { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentHistogram.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentHistogram.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentHistogram.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentHistogram.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentLineChart.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentLineChart.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentLineChart.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentLineChart.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentTable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentTable.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentTable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/transform/ui/components/RenderableComponentTable.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/ClassPathResource.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ClassPathResource.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/ClassPathResource.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ClassPathResource.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/RecordUtils.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/RecordUtils.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/RecordUtils.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/RecordUtils.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/ReflectionUtils.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ReflectionUtils.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/ReflectionUtils.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ReflectionUtils.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/files/FileFromPathIterator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/files/FileFromPathIterator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/files/FileFromPathIterator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/files/FileFromPathIterator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/files/ShuffledListIterator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/files/ShuffledListIterator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/files/ShuffledListIterator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/files/ShuffledListIterator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/files/URIUtil.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/files/URIUtil.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/files/URIUtil.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/files/URIUtil.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/files/UriFromPathIterator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/files/UriFromPathIterator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/files/UriFromPathIterator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/files/UriFromPathIterator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeDeserializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeDeserializer.java similarity index 93% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeDeserializer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeDeserializer.java index 937ef8955..198b49755 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeDeserializer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeDeserializer.java @@ -21,11 +21,11 @@ package org.datavec.api.util.jackson; import org.joda.time.DateTimeFieldType; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; import java.util.HashMap; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeSerializer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeSerializer.java similarity index 86% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeSerializer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeSerializer.java index 4c1e3a96b..ef286d5e1 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeSerializer.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/jackson/DateTimeFieldTypeSerializer.java @@ -21,10 +21,10 @@ package org.datavec.api.util.jackson; import org.joda.time.DateTimeFieldType; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/DataInputWrapperStream.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ndarray/DataInputWrapperStream.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/DataInputWrapperStream.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ndarray/DataInputWrapperStream.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/DataOutputWrapperStream.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ndarray/DataOutputWrapperStream.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/DataOutputWrapperStream.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ndarray/DataOutputWrapperStream.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java index 67c810526..98cc7d339 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java @@ -20,7 +20,7 @@ package org.datavec.api.util.ndarray; -import org.nd4j.shade.guava.base.Preconditions; +import com.google.common.base.Preconditions; import it.unimi.dsi.fastutil.doubles.DoubleArrayList; import lombok.NonNull; import org.datavec.api.timeseries.util.TimeSeriesWritableUtils; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/vector/Vectorizer.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/vector/Vectorizer.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/vector/Vectorizer.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/vector/Vectorizer.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/ArrayWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/ArrayWritable.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/ArrayWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/ArrayWritable.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/BooleanWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/BooleanWritable.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/BooleanWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/BooleanWritable.java index f2abcbae9..bec23e5e2 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/BooleanWritable.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/BooleanWritable.java @@ -23,7 +23,7 @@ package org.datavec.api.writable; import org.datavec.api.io.WritableComparable; import org.datavec.api.io.WritableComparator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.DataInput; import java.io.DataOutput; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/ByteWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/ByteWritable.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/ByteWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/ByteWritable.java index d594a48f7..f2f098cd8 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/ByteWritable.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/ByteWritable.java @@ -21,10 +21,10 @@ package org.datavec.api.writable; -import org.nd4j.shade.guava.math.DoubleMath; +import com.google.common.math.DoubleMath; import org.datavec.api.io.WritableComparable; import org.datavec.api.io.WritableComparator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.DataInput; import java.io.DataOutput; @@ -33,7 +33,6 @@ import java.io.IOException; public class ByteWritable implements WritableComparable { private byte value; - public ByteWritable() {} public ByteWritable(@JsonProperty("value") byte value) { diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/BytesWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/BytesWritable.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/BytesWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/BytesWritable.java index 67cf12d53..1caa52031 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/BytesWritable.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/BytesWritable.java @@ -53,7 +53,7 @@ public class BytesWritable extends ArrayWritable { * Convert the underlying contents of this {@link Writable} * to an nd4j {@link DataBuffer}. Note that this is a *copy* * of the underlying buffer. - * Also note that {@link java.nio.ByteBuffer#allocateDirect(int)} + * Also note that {@link ByteBuffer#allocateDirect(int)} * is used for allocation. * This should be considered an expensive operation. * diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/DoubleWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/DoubleWritable.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/DoubleWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/DoubleWritable.java index 62e63025c..ed795e958 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/DoubleWritable.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/DoubleWritable.java @@ -21,10 +21,10 @@ package org.datavec.api.writable; -import org.nd4j.shade.guava.math.DoubleMath; +import com.google.common.math.DoubleMath; import org.datavec.api.io.WritableComparable; import org.datavec.api.io.WritableComparator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.DataInput; import java.io.DataOutput; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/FloatWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/FloatWritable.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/FloatWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/FloatWritable.java index 5e663f745..c98bc78f3 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/FloatWritable.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/FloatWritable.java @@ -21,10 +21,10 @@ package org.datavec.api.writable; -import org.nd4j.shade.guava.math.DoubleMath; +import com.google.common.math.DoubleMath; import org.datavec.api.io.WritableComparable; import org.datavec.api.io.WritableComparator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.DataInput; import java.io.DataOutput; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/IntWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/IntWritable.java similarity index 97% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/IntWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/IntWritable.java index 22d8748a0..37d74df2f 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/IntWritable.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/IntWritable.java @@ -21,10 +21,10 @@ package org.datavec.api.writable; -import org.nd4j.shade.guava.math.DoubleMath; +import com.google.common.math.DoubleMath; import org.datavec.api.io.WritableComparable; import org.datavec.api.io.WritableComparator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.DataInput; import java.io.DataOutput; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/LongWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/LongWritable.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/LongWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/LongWritable.java index 228d089a4..4b7dc3d35 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/LongWritable.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/LongWritable.java @@ -21,10 +21,10 @@ package org.datavec.api.writable; -import org.nd4j.shade.guava.math.DoubleMath; +import com.google.common.math.DoubleMath; import org.datavec.api.io.WritableComparable; import org.datavec.api.io.WritableComparator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.DataInput; import java.io.DataOutput; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/NDArrayWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/NDArrayWritable.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/NDArrayWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/NDArrayWritable.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/NullWritable.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/NullWritable.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/NullWritable.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/NullWritable.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/Text.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/Text.java similarity index 99% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/Text.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/Text.java index 0d64b2142..43dc58036 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/Text.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/Text.java @@ -237,7 +237,7 @@ public class Text extends BinaryComparable implements WritableComparableFor efficiency, implementations should attempt to re-use storage in the * existing object where possible.

* - * @param in DataInput to deseriablize this object from. + * @param in DataInput to deserialize this object from. * @throws IOException */ void readFields(DataInput in) throws IOException; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/WritableFactory.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/WritableFactory.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/WritableFactory.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/WritableFactory.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/WritableType.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/WritableType.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/WritableType.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/WritableType.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/batch/AbstractTimeSeriesWritableRecordBatch.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/batch/AbstractTimeSeriesWritableRecordBatch.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/batch/AbstractTimeSeriesWritableRecordBatch.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/batch/AbstractTimeSeriesWritableRecordBatch.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/batch/AbstractWritableRecordBatch.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/batch/AbstractWritableRecordBatch.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/batch/AbstractWritableRecordBatch.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/batch/AbstractWritableRecordBatch.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/batch/NDArrayRecordBatch.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/batch/NDArrayRecordBatch.java similarity index 98% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/batch/NDArrayRecordBatch.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/batch/NDArrayRecordBatch.java index d1f092d0e..0a8d58d0c 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/batch/NDArrayRecordBatch.java +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/batch/NDArrayRecordBatch.java @@ -20,7 +20,7 @@ package org.datavec.api.writable.batch; -import org.nd4j.shade.guava.base.Preconditions; +import com.google.common.base.Preconditions; import lombok.Data; import lombok.NonNull; import org.datavec.api.writable.NDArrayWritable; diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/Comparators.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/Comparators.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/Comparators.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/Comparators.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/DoubleWritableComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/DoubleWritableComparator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/DoubleWritableComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/DoubleWritableComparator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/FloatWritableComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/FloatWritableComparator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/FloatWritableComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/FloatWritableComparator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/IntWritableComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/IntWritableComparator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/IntWritableComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/IntWritableComparator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/LongWritableComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/LongWritableComparator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/LongWritableComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/LongWritableComparator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/ReverseComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/ReverseComparator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/ReverseComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/ReverseComparator.java diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/TextWritableComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/TextWritableComparator.java similarity index 100% rename from datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/TextWritableComparator.java rename to cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/TextWritableComparator.java diff --git a/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/WritableComparator.java b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/WritableComparator.java new file mode 100644 index 000000000..044a27c0e --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/main/java/org/datavec/api/writable/comparator/WritableComparator.java @@ -0,0 +1,32 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.writable.comparator; + +import org.datavec.api.writable.Writable; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; +import java.util.Comparator; + +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface WritableComparator extends Comparator, Serializable { + +} diff --git a/datavec/datavec-api/src/main/resources/templates/analysis.ftl b/cavis-datavec/cavis-datavec-api/src/main/resources/templates/analysis.ftl similarity index 100% rename from datavec/datavec-api/src/main/resources/templates/analysis.ftl rename to cavis-datavec/cavis-datavec-api/src/main/resources/templates/analysis.ftl diff --git a/datavec/datavec-api/src/main/resources/templates/sequenceplot.ftl b/cavis-datavec/cavis-datavec-api/src/main/resources/templates/sequenceplot.ftl similarity index 100% rename from datavec/datavec-api/src/main/resources/templates/sequenceplot.ftl rename to cavis-datavec/cavis-datavec-api/src/main/resources/templates/sequenceplot.ftl diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..e56a02369 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/AssertTestsExtendBaseClass.java @@ -0,0 +1,55 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ +package org.datavec.api; + +import lombok.extern.slf4j.Slf4j; +import org.datavec.api.transform.serde.testClasses.CustomCondition; +import org.datavec.api.transform.serde.testClasses.CustomFilter; +import org.datavec.api.transform.serde.testClasses.CustomTransform; +import org.junit.jupiter.api.TestInstance; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; + +@Slf4j +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + Set> res = new HashSet<>(); + res.add(CustomCondition.class); + res.add(CustomFilter.class); + res.add(CustomTransform.class); + return res; + } + + @Override + protected String getPackageName() { + return "org.datavec.api"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/CalculatorTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/CalculatorTest.java new file mode 100644 index 000000000..89d69155a --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/CalculatorTest.java @@ -0,0 +1,39 @@ + +/* + * Copyright 2015-2018 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * http://www.eclipse.org/legal/epl-v20.html + */ + +package org.datavec.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class CalculatorTests { + + @Test + @DisplayName("1 + 1 = 2") + void addsTwoNumbers() { + assertEquals(2, 1+1, "1 + 1 should equal 2"); + } + + @ParameterizedTest(name = "{0} + {1} = {2}") + @CsvSource({ + "0, 1, 1", + "1, 2, 3", + "49, 51, 100", + "1, 100, 101" + }) + void add(int first, int second, int expectedResult) { + assertEquals(expectedResult, first+second, () -> first + " + " + second + " should equal " + expectedResult); + } +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVLineSequenceRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVLineSequenceRecordReaderTest.java new file mode 100644 index 000000000..0817973a7 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVLineSequenceRecordReaderTest.java @@ -0,0 +1,87 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import org.apache.commons.io.FileUtils; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVLineSequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.tests.BaseND4JTest; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CSVLineSequenceRecordReaderTest extends BaseND4JTest { + + @TempDir + File f; + + @Test + public void test() throws Exception { + File source = new File(f, "temp.csv"); + String str = "a,b,c\n1,2,3,4"; + FileUtils.writeStringToFile(source, str, StandardCharsets.UTF_8); + + SequenceRecordReader rr = new CSVLineSequenceRecordReader(); + rr.initialize(new FileSplit(source)); + + List> exp0 = Arrays.asList( + Collections.singletonList(new Text("a")), + Collections.singletonList(new Text("b")), + Collections.singletonList(new Text("c"))); + + List> exp1 = Arrays.asList( + Collections.singletonList(new Text("1")), + Collections.singletonList(new Text("2")), + Collections.singletonList(new Text("3")), + Collections.singletonList(new Text("4"))); + + for( int i=0; i<3; i++ ) { + int count = 0; + while (rr.hasNext()) { + List> next = rr.sequenceRecord(); + if (count++ == 0) { + assertEquals(exp0, next); + } else { + assertEquals(exp1, next); + } + } + + assertEquals(2, count); + + rr.reset(); + } + } + + @Override + public long getTimeoutMilliseconds() { + return Long.MAX_VALUE; + } +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVMultiSequenceRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVMultiSequenceRecordReaderTest.java similarity index 80% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVMultiSequenceRecordReaderTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVMultiSequenceRecordReaderTest.java index ab61550d0..882fc628f 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVMultiSequenceRecordReaderTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVMultiSequenceRecordReaderTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.reader.impl; import org.apache.commons.io.FileUtils; @@ -25,41 +26,32 @@ import org.datavec.api.records.reader.impl.csv.CSVMultiSequenceRecordReader; import org.datavec.api.split.FileSplit; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.tests.BaseND4JTest; + import java.io.File; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Csv Multi Sequence Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class CSVMultiSequenceRecordReaderTest extends BaseND4JTest { +public class CSVMultiSequenceRecordReaderTest extends BaseND4JTest { @TempDir - public Path testDir; + public File testDir; @Test - @DisplayName("Test Concat Mode") - @Disabled - void testConcatMode() throws Exception { - for (int i = 0; i < 3; i++) { + public void testConcatMode() throws Exception { + for( int i=0; i<3; i++ ) { + String seqSep; String seqSepRegex; - switch(i) { + switch (i){ case 0: seqSep = ""; seqSepRegex = "^$"; @@ -75,23 +67,31 @@ class CSVMultiSequenceRecordReaderTest extends BaseND4JTest { default: throw new RuntimeException(); } + String str = "a,b,c\n1,2,3,4\nx,y\n" + seqSep + "\nA,B,C"; - File f = testDir.toFile(); + File f = testDir; FileUtils.writeStringToFile(f, str, StandardCharsets.UTF_8); + SequenceRecordReader seqRR = new CSVMultiSequenceRecordReader(seqSepRegex, CSVMultiSequenceRecordReader.Mode.CONCAT); seqRR.initialize(new FileSplit(f)); + + List> exp0 = new ArrayList<>(); for (String s : "a,b,c,1,2,3,4,x,y".split(",")) { exp0.add(Collections.singletonList(new Text(s))); } + List> exp1 = new ArrayList<>(); for (String s : "A,B,C".split(",")) { exp1.add(Collections.singletonList(new Text(s))); } + assertEquals(exp0, seqRR.sequenceRecord()); assertEquals(exp1, seqRR.sequenceRecord()); assertFalse(seqRR.hasNext()); + seqRR.reset(); + assertEquals(exp0, seqRR.sequenceRecord()); assertEquals(exp1, seqRR.sequenceRecord()); assertFalse(seqRR.hasNext()); @@ -99,13 +99,13 @@ class CSVMultiSequenceRecordReaderTest extends BaseND4JTest { } @Test - @DisplayName("Test Equal Length") - @Disabled - void testEqualLength() throws Exception { - for (int i = 0; i < 3; i++) { + public void testEqualLength() throws Exception { + + for( int i=0; i<3; i++ ) { + String seqSep; String seqSepRegex; - switch(i) { + switch (i) { case 0: seqSep = ""; seqSepRegex = "^$"; @@ -121,17 +121,27 @@ class CSVMultiSequenceRecordReaderTest extends BaseND4JTest { default: throw new RuntimeException(); } + String str = "a,b\n1,2\nx,y\n" + seqSep + "\nA\nB\nC"; - File f = testDir.toFile(); + File f = testDir; FileUtils.writeStringToFile(f, str, StandardCharsets.UTF_8); + SequenceRecordReader seqRR = new CSVMultiSequenceRecordReader(seqSepRegex, CSVMultiSequenceRecordReader.Mode.EQUAL_LENGTH); seqRR.initialize(new FileSplit(f)); - List> exp0 = Arrays.asList(Arrays.asList(new Text("a"), new Text("1"), new Text("x")), Arrays.asList(new Text("b"), new Text("2"), new Text("y"))); + + + List> exp0 = Arrays.asList( + Arrays.asList(new Text("a"), new Text("1"), new Text("x")), + Arrays.asList(new Text("b"), new Text("2"), new Text("y"))); + List> exp1 = Collections.singletonList(Arrays.asList(new Text("A"), new Text("B"), new Text("C"))); + assertEquals(exp0, seqRR.sequenceRecord()); assertEquals(exp1, seqRR.sequenceRecord()); assertFalse(seqRR.hasNext()); + seqRR.reset(); + assertEquals(exp0, seqRR.sequenceRecord()); assertEquals(exp1, seqRR.sequenceRecord()); assertFalse(seqRR.hasNext()); @@ -139,13 +149,13 @@ class CSVMultiSequenceRecordReaderTest extends BaseND4JTest { } @Test - @DisplayName("Test Padding") - @Disabled - void testPadding() throws Exception { - for (int i = 0; i < 3; i++) { + public void testPadding() throws Exception { + + for( int i=0; i<3; i++ ) { + String seqSep; String seqSepRegex; - switch(i) { + switch (i) { case 0: seqSep = ""; seqSepRegex = "^$"; @@ -161,17 +171,27 @@ class CSVMultiSequenceRecordReaderTest extends BaseND4JTest { default: throw new RuntimeException(); } + String str = "a,b\n1\nx\n" + seqSep + "\nA\nB\nC"; - File f = testDir.toFile(); + File f = testDir; FileUtils.writeStringToFile(f, str, StandardCharsets.UTF_8); + SequenceRecordReader seqRR = new CSVMultiSequenceRecordReader(seqSepRegex, CSVMultiSequenceRecordReader.Mode.PAD, new Text("PAD")); seqRR.initialize(new FileSplit(f)); - List> exp0 = Arrays.asList(Arrays.asList(new Text("a"), new Text("1"), new Text("x")), Arrays.asList(new Text("b"), new Text("PAD"), new Text("PAD"))); + + + List> exp0 = Arrays.asList( + Arrays.asList(new Text("a"), new Text("1"), new Text("x")), + Arrays.asList(new Text("b"), new Text("PAD"), new Text("PAD"))); + List> exp1 = Collections.singletonList(Arrays.asList(new Text("A"), new Text("B"), new Text("C"))); + assertEquals(exp0, seqRR.sequenceRecord()); assertEquals(exp1, seqRR.sequenceRecord()); assertFalse(seqRR.hasNext()); + seqRR.reset(); + assertEquals(exp0, seqRR.sequenceRecord()); assertEquals(exp1, seqRR.sequenceRecord()); assertFalse(seqRR.hasNext()); diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVNLinesSequenceRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVNLinesSequenceRecordReaderTest.java similarity index 85% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVNLinesSequenceRecordReaderTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVNLinesSequenceRecordReaderTest.java index dc494c76c..8096c391c 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVNLinesSequenceRecordReaderTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVNLinesSequenceRecordReaderTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.reader.impl; import org.datavec.api.records.SequenceRecord; @@ -26,57 +27,61 @@ import org.datavec.api.records.reader.impl.csv.CSVNLinesSequenceRecordReader; import org.datavec.api.records.reader.impl.csv.CSVRecordReader; import org.datavec.api.split.FileSplit; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.common.io.ClassPathResource; + import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Csvn Lines Sequence Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class CSVNLinesSequenceRecordReaderTest extends BaseND4JTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CSVNLinesSequenceRecordReaderTest extends BaseND4JTest { @Test - @DisplayName("Test CSV Lines Sequence Record Reader") - void testCSVNLinesSequenceRecordReader() throws Exception { + public void testCSVNLinesSequenceRecordReader() throws Exception { int nLinesPerSequence = 10; + SequenceRecordReader seqRR = new CSVNLinesSequenceRecordReader(nLinesPerSequence); seqRR.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + CSVRecordReader rr = new CSVRecordReader(); rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + int count = 0; while (seqRR.hasNext()) { List> next = seqRR.sequenceRecord(); + List> expected = new ArrayList<>(); for (int i = 0; i < nLinesPerSequence; i++) { expected.add(rr.next()); } + assertEquals(10, next.size()); assertEquals(expected, next); + count++; } + assertEquals(150 / nLinesPerSequence, count); } @Test - @DisplayName("Test CSV Nlines Sequence Record Reader Meta Data") - void testCSVNlinesSequenceRecordReaderMetaData() throws Exception { + public void testCSVNlinesSequenceRecordReaderMetaData() throws Exception { int nLinesPerSequence = 10; + SequenceRecordReader seqRR = new CSVNLinesSequenceRecordReader(nLinesPerSequence); seqRR.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + CSVRecordReader rr = new CSVRecordReader(); rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + List>> out = new ArrayList<>(); while (seqRR.hasNext()) { List> next = seqRR.sequenceRecord(); out.add(next); } + seqRR.reset(); List>> out2 = new ArrayList<>(); List out3 = new ArrayList<>(); @@ -87,8 +92,11 @@ class CSVNLinesSequenceRecordReaderTest extends BaseND4JTest { meta.add(seq.getMetaData()); out3.add(seq); } + assertEquals(out, out2); + List out4 = seqRR.loadSequenceFromMetaData(meta); assertEquals(out3, out4); } + } diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVRecordReaderTest.java new file mode 100644 index 000000000..0b54b7147 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVRecordReaderTest.java @@ -0,0 +1,350 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import org.apache.commons.io.FileUtils; +import org.datavec.api.records.Record; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRegexRecordReader; +import org.datavec.api.records.writer.impl.FileRecordWriter; +import org.datavec.api.records.writer.impl.csv.CSVRecordWriter; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputStreamInputSplit; +import org.datavec.api.split.StringSplit; +import org.datavec.api.split.partition.NumberOfRecordsPartitioner; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; + +import static org.junit.jupiter.api.Assertions.*; + +public class CSVRecordReaderTest extends BaseND4JTest { + @Test + public void testNext() throws Exception { + CSVRecordReader reader = new CSVRecordReader(); + reader.initialize(new StringSplit("1,1,8.0,,,,14.0,,,,15.0,,,,,,,,,,,,1")); + while (reader.hasNext()) { + List vals = reader.next(); + List arr = new ArrayList<>(vals); + + assertEquals( 23, vals.size(), "Entry count"); + Text lastEntry = (Text) arr.get(arr.size() - 1); + assertEquals( 1, lastEntry.getLength(), "Last entry garbage"); + } + } + + @Test + public void testEmptyEntries() throws Exception { + CSVRecordReader reader = new CSVRecordReader(); + reader.initialize(new StringSplit("1,1,8.0,,,,14.0,,,,15.0,,,,,,,,,,,,")); + while (reader.hasNext()) { + List vals = reader.next(); + assertEquals( 23, vals.size(), "Entry count"); + } + } + + @Test + public void testReset() throws Exception { + CSVRecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + + int nResets = 5; + for (int i = 0; i < nResets; i++) { + + int lineCount = 0; + while (rr.hasNext()) { + List line = rr.next(); + assertEquals(5, line.size()); + lineCount++; + } + assertFalse(rr.hasNext()); + assertEquals(150, lineCount); + rr.reset(); + } + } + + @Test + public void testResetWithSkipLines() throws Exception { + CSVRecordReader rr = new CSVRecordReader(10, ','); + rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + int lineCount = 0; + while (rr.hasNext()) { + rr.next(); + ++lineCount; + } + assertEquals(140, lineCount); + rr.reset(); + lineCount = 0; + while (rr.hasNext()) { + rr.next(); + ++lineCount; + } + assertEquals(140, lineCount); + } + + @Test + public void testWrite() throws Exception { + List> list = new ArrayList<>(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 10; i++) { + List temp = new ArrayList<>(); + for (int j = 0; j < 3; j++) { + int v = 100 * i + j; + temp.add(new IntWritable(v)); + sb.append(v); + if (j < 2) + sb.append(","); + else if (i != 9) + sb.append("\n"); + } + list.add(temp); + } + + String expected = sb.toString(); + + Path p = Files.createTempFile("csvwritetest", "csv"); + p.toFile().deleteOnExit(); + + FileRecordWriter writer = new CSVRecordWriter(); + FileSplit fileSplit = new FileSplit(p.toFile()); + writer.initialize(fileSplit,new NumberOfRecordsPartitioner()); + for (List c : list) { + writer.write(c); + } + writer.close(); + + //Read file back in; compare + String fileContents = FileUtils.readFileToString(p.toFile(), FileRecordWriter.DEFAULT_CHARSET.name()); + + // System.out.println(expected); + // System.out.println("----------"); + // System.out.println(fileContents); + + assertEquals(expected, fileContents); + } + + @Test + public void testTabsAsSplit1() throws Exception { + + CSVRecordReader reader = new CSVRecordReader(0, '\t'); + reader.initialize(new FileSplit(new ClassPathResource("datavec-api/tabbed.txt").getFile())); + while (reader.hasNext()) { + List list = new ArrayList<>(reader.next()); + + assertEquals(2, list.size()); + } + } + + @Test + public void testPipesAsSplit() throws Exception { + + CSVRecordReader reader = new CSVRecordReader(0, '|'); + reader.initialize(new FileSplit(new ClassPathResource("datavec-api/issue414.csv").getFile())); + int lineidx = 0; + List sixthColumn = Arrays.asList(13, 95, 15, 25); + while (reader.hasNext()) { + List list = new ArrayList<>(reader.next()); + + assertEquals(10, list.size()); + assertEquals((long)sixthColumn.get(lineidx), list.get(5).toInt()); + lineidx++; + } + } + + + @Test + public void testWithQuotes() throws Exception { + CSVRecordReader reader = new CSVRecordReader(0, ',', '\"'); + reader.initialize(new StringSplit("1,0,3,\"Braund, Mr. Owen Harris\",male,\"\"\"\"")); + while (reader.hasNext()) { + List vals = reader.next(); + assertEquals(6, vals.size(), "Entry count"); + assertEquals("1", vals.get(0).toString()); + assertEquals("0", vals.get(1).toString()); + assertEquals("3", vals.get(2).toString()); + assertEquals("Braund, Mr. Owen Harris", vals.get(3).toString()); + assertEquals("male", vals.get(4).toString()); + assertEquals("\"", vals.get(5).toString()); + } + } + + + @Test + public void testMeta() throws Exception { + CSVRecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + + int lineCount = 0; + List metaList = new ArrayList<>(); + List> writables = new ArrayList<>(); + while (rr.hasNext()) { + Record r = rr.nextRecord(); + assertEquals(5, r.getRecord().size()); + lineCount++; + RecordMetaData meta = r.getMetaData(); + // System.out.println(r.getRecord() + "\t" + meta.getLocation() + "\t" + meta.getURI()); + + metaList.add(meta); + writables.add(r.getRecord()); + } + assertFalse(rr.hasNext()); + assertEquals(150, lineCount); + rr.reset(); + + + System.out.println("\n\n\n--------------------------------"); + List contents = rr.loadFromMetaData(metaList); + assertEquals(150, contents.size()); + // for(Record r : contents ){ + // System.out.println(r); + // } + + List meta2 = new ArrayList<>(); + meta2.add(metaList.get(100)); + meta2.add(metaList.get(90)); + meta2.add(metaList.get(80)); + meta2.add(metaList.get(70)); + meta2.add(metaList.get(60)); + + List contents2 = rr.loadFromMetaData(meta2); + assertEquals(writables.get(100), contents2.get(0).getRecord()); + assertEquals(writables.get(90), contents2.get(1).getRecord()); + assertEquals(writables.get(80), contents2.get(2).getRecord()); + assertEquals(writables.get(70), contents2.get(3).getRecord()); + assertEquals(writables.get(60), contents2.get(4).getRecord()); + } + + @Test + public void testRegex() throws Exception { + CSVRecordReader reader = new CSVRegexRecordReader(0, ",", null, new String[] {null, "(.+) (.+) (.+)"}); + reader.initialize(new StringSplit("normal,1.2.3.4 space separator")); + while (reader.hasNext()) { + List vals = reader.next(); + assertEquals( 4, vals.size(), "Entry count"); + assertEquals("normal", vals.get(0).toString()); + assertEquals("1.2.3.4", vals.get(1).toString()); + assertEquals("space", vals.get(2).toString()); + assertEquals("separator", vals.get(3).toString()); + } + } + + @Test + public void testCsvSkipAllLines() throws IOException, InterruptedException { + Assertions.assertThrows(NoSuchElementException.class, () -> { + final int numLines = 4; + + final List lineList = Arrays.asList((Writable) new IntWritable(numLines - 1), + (Writable) new Text("one"), (Writable) new Text("two"), (Writable) new Text("three")); + String header = ",one,two,three"; + List lines = new ArrayList<>(); + for (int i = 0; i < numLines; i++) + lines.add(Integer.toString(i) + header); + File tempFile = File.createTempFile("csvSkipLines", ".csv"); + FileUtils.writeLines(tempFile, lines); + + CSVRecordReader rr = new CSVRecordReader(numLines, ','); + rr.initialize(new FileSplit(tempFile)); + rr.reset(); + assertTrue(!rr.hasNext()); + rr.next(); + }); + } + + @Test + public void testCsvSkipAllButOneLine() throws IOException, InterruptedException { + final int numLines = 4; + final List lineList = Arrays.asList(new Text(Integer.toString(numLines - 1)), + new Text("one"), new Text("two"), new Text("three")); + String header = ",one,two,three"; + List lines = new ArrayList<>(); + for (int i = 0; i < numLines; i++) + lines.add(Integer.toString(i) + header); + File tempFile = File.createTempFile("csvSkipLines", ".csv"); + FileUtils.writeLines(tempFile, lines); + + CSVRecordReader rr = new CSVRecordReader(numLines - 1, ','); + rr.initialize(new FileSplit(tempFile)); + rr.reset(); + assertTrue(rr.hasNext()); + assertEquals(rr.next(), lineList); + } + + + @Test + public void testStreamReset() throws Exception { + CSVRecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new InputStreamInputSplit(new ClassPathResource("datavec-api/iris.dat").getInputStream())); + + int count = 0; + while(rr.hasNext()){ + assertNotNull(rr.next()); + count++; + } + assertEquals(150, count); + + assertFalse(rr.resetSupported()); + + try{ + rr.reset(); + fail("Expected exception"); + } catch (Exception e){ + String msg = e.getMessage(); + String msg2 = e.getCause().getMessage(); + assertTrue(msg.contains("Error during LineRecordReader reset"), msg); + assertTrue(msg2.contains("Reset not supported from streams"),msg2); +// e.printStackTrace(); + } + } + + @Test + public void testUsefulExceptionNoInit(){ + + CSVRecordReader rr = new CSVRecordReader(0, ','); + + try{ + rr.hasNext(); + fail("Expected exception"); + } catch (Exception e){ + assertTrue( e.getMessage().contains("initialized"), e.getMessage()); + } + + try{ + rr.next(); + fail("Expected exception"); + } catch (Exception e){ + assertTrue(e.getMessage().contains("initialized"), e.getMessage()); + } + } +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java similarity index 84% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java index 248722652..8fdce2165 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVSequenceRecordReaderTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.reader.impl; import org.datavec.api.records.SequenceRecord; @@ -26,12 +27,11 @@ import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; import org.datavec.api.split.InputSplit; import org.datavec.api.split.NumberedFileInputSplit; import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.common.io.ClassPathResource; + import java.io.File; import java.io.InputStream; import java.io.OutputStream; @@ -40,30 +40,25 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Csv Sequence Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class CSVSequenceRecordReaderTest extends BaseND4JTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CSVSequenceRecordReaderTest extends BaseND4JTest { @TempDir - public Path tempDir; + public File tempDir; @Test - @DisplayName("Test") - void test() throws Exception { + public void test() throws Exception { + CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ","); seqReader.initialize(new TestInputSplit()); + int sequenceCount = 0; while (seqReader.hasNext()) { List> sequence = seqReader.sequenceRecord(); - // 4 lines, plus 1 header line - assertEquals(4, sequence.size()); + assertEquals(4, sequence.size()); //4 lines, plus 1 header line + Iterator> timeStepIter = sequence.iterator(); int lineCount = 0; while (timeStepIter.hasNext()) { @@ -84,18 +79,19 @@ class CSVSequenceRecordReaderTest extends BaseND4JTest { } @Test - @DisplayName("Test Reset") - void testReset() throws Exception { + public void testReset() throws Exception { CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ","); seqReader.initialize(new TestInputSplit()); + int nTests = 5; for (int i = 0; i < nTests; i++) { seqReader.reset(); + int sequenceCount = 0; while (seqReader.hasNext()) { List> sequence = seqReader.sequenceRecord(); - // 4 lines, plus 1 header line - assertEquals(4, sequence.size()); + assertEquals(4, sequence.size()); //4 lines, plus 1 header line + Iterator> timeStepIter = sequence.iterator(); int lineCount = 0; while (timeStepIter.hasNext()) { @@ -110,15 +106,15 @@ class CSVSequenceRecordReaderTest extends BaseND4JTest { } @Test - @DisplayName("Test Meta Data") - void testMetaData() throws Exception { + public void testMetaData() throws Exception { CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ","); seqReader.initialize(new TestInputSplit()); + List>> l = new ArrayList<>(); while (seqReader.hasNext()) { List> sequence = seqReader.sequenceRecord(); - // 4 lines, plus 1 header line - assertEquals(4, sequence.size()); + assertEquals(4, sequence.size()); //4 lines, plus 1 header line + Iterator> timeStepIter = sequence.iterator(); int lineCount = 0; while (timeStepIter.hasNext()) { @@ -126,8 +122,10 @@ class CSVSequenceRecordReaderTest extends BaseND4JTest { lineCount++; } assertEquals(4, lineCount); + l.add(sequence); } + List l2 = new ArrayList<>(); List meta = new ArrayList<>(); seqReader.reset(); @@ -137,6 +135,7 @@ class CSVSequenceRecordReaderTest extends BaseND4JTest { meta.add(sr.getMetaData()); } assertEquals(3, l2.size()); + List fromMeta = seqReader.loadSequenceFromMetaData(meta); for (int i = 0; i < 3; i++) { assertEquals(l.get(i), l2.get(i).getSequenceRecord()); @@ -144,8 +143,8 @@ class CSVSequenceRecordReaderTest extends BaseND4JTest { } } - @DisplayName("Test Input Split") - private static class TestInputSplit implements InputSplit { + private static class + TestInputSplit implements InputSplit { @Override public boolean canWriteToLocation(URI location) { @@ -164,6 +163,7 @@ class CSVSequenceRecordReaderTest extends BaseND4JTest { @Override public void updateSplitLocations(boolean reset) { + } @Override @@ -173,6 +173,7 @@ class CSVSequenceRecordReaderTest extends BaseND4JTest { @Override public void bootStrapForWrite() { + } @Override @@ -220,30 +221,38 @@ class CSVSequenceRecordReaderTest extends BaseND4JTest { @Override public void reset() { - // No op + //No op } @Override public boolean resetSupported() { return true; } + + + + } + @Test - @DisplayName("Test Csv Seq And Numbered File Split") - void testCsvSeqAndNumberedFileSplit(@TempDir Path tempDir) throws Exception { - File baseDir = tempDir.toFile(); - // Simple sanity check unit test + public void testCsvSeqAndNumberedFileSplit() throws Exception { + File baseDir = tempDir; + //Simple sanity check unit test for (int i = 0; i < 3; i++) { new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(baseDir); } - // Load time series from CSV sequence files; compare to SequenceRecordReaderDataSetIterator + + //Load time series from CSV sequence files; compare to SequenceRecordReaderDataSetIterator ClassPathResource resource = new ClassPathResource("csvsequence_0.txt"); String featuresPath = new File(baseDir, "csvsequence_%d.txt").getAbsolutePath(); + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - while (featureReader.hasNext()) { + + while(featureReader.hasNext()){ featureReader.nextSequence(); } + } } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVVariableSlidingWindowRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVVariableSlidingWindowRecordReaderTest.java similarity index 75% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVVariableSlidingWindowRecordReaderTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVVariableSlidingWindowRecordReaderTest.java index bee315176..315016932 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVVariableSlidingWindowRecordReaderTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVVariableSlidingWindowRecordReaderTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.reader.impl; import org.datavec.api.records.reader.SequenceRecordReader; @@ -24,91 +25,94 @@ import org.datavec.api.records.reader.impl.csv.CSVRecordReader; import org.datavec.api.records.reader.impl.csv.CSVVariableSlidingWindowRecordReader; import org.datavec.api.split.FileSplit; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.common.io.ClassPathResource; + import java.util.LinkedList; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Csv Variable Sliding Window Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class CSVVariableSlidingWindowRecordReaderTest extends BaseND4JTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CSVVariableSlidingWindowRecordReaderTest extends BaseND4JTest { @Test - @DisplayName("Test CSV Variable Sliding Window Record Reader") - void testCSVVariableSlidingWindowRecordReader() throws Exception { + public void testCSVVariableSlidingWindowRecordReader() throws Exception { int maxLinesPerSequence = 3; + SequenceRecordReader seqRR = new CSVVariableSlidingWindowRecordReader(maxLinesPerSequence); seqRR.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + CSVRecordReader rr = new CSVRecordReader(); rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + int count = 0; while (seqRR.hasNext()) { List> next = seqRR.sequenceRecord(); - if (count == maxLinesPerSequence - 1) { + + if(count==maxLinesPerSequence-1) { LinkedList> expected = new LinkedList<>(); for (int i = 0; i < maxLinesPerSequence; i++) { expected.addFirst(rr.next()); } assertEquals(expected, next); + } - if (count == maxLinesPerSequence) { + if(count==maxLinesPerSequence) { assertEquals(maxLinesPerSequence, next.size()); } - if (count == 0) { - // first seq should be length 1 + if(count==0) { // first seq should be length 1 assertEquals(1, next.size()); } - if (count > 151) { - // last seq should be length 1 + if(count>151) { // last seq should be length 1 assertEquals(1, next.size()); } + count++; } + assertEquals(152, count); } @Test - @DisplayName("Test CSV Variable Sliding Window Record Reader Stride") - void testCSVVariableSlidingWindowRecordReaderStride() throws Exception { + public void testCSVVariableSlidingWindowRecordReaderStride() throws Exception { int maxLinesPerSequence = 3; int stride = 2; + SequenceRecordReader seqRR = new CSVVariableSlidingWindowRecordReader(maxLinesPerSequence, stride); seqRR.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + CSVRecordReader rr = new CSVRecordReader(); rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + int count = 0; while (seqRR.hasNext()) { List> next = seqRR.sequenceRecord(); - if (count == maxLinesPerSequence - 1) { + + if(count==maxLinesPerSequence-1) { LinkedList> expected = new LinkedList<>(); - for (int s = 0; s < stride; s++) { + for(int s = 0; s < stride; s++) { expected = new LinkedList<>(); for (int i = 0; i < maxLinesPerSequence; i++) { expected.addFirst(rr.next()); } } assertEquals(expected, next); + } - if (count == maxLinesPerSequence) { + if(count==maxLinesPerSequence) { assertEquals(maxLinesPerSequence, next.size()); } - if (count == 0) { - // first seq should be length 2 + if(count==0) { // first seq should be length 2 assertEquals(2, next.size()); } - if (count > 151) { - // last seq should be length 1 + if(count>151) { // last seq should be length 1 assertEquals(1, next.size()); } + count++; } + assertEquals(76, count); } } diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileBatchRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileBatchRecordReaderTest.java new file mode 100644 index 000000000..809295f6d --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileBatchRecordReaderTest.java @@ -0,0 +1,126 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import org.apache.commons.io.FileUtils; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; +import org.datavec.api.records.reader.impl.filebatch.FileBatchRecordReader; +import org.datavec.api.records.reader.impl.filebatch.FileBatchSequenceRecordReader; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.loader.FileBatch; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class FileBatchRecordReaderTest extends BaseND4JTest { + + @TempDir + public File testDir; + + @Test + public void testCsv() throws Exception { + + //This is an unrealistic use case - one line/record per CSV + File baseDir = testDir; + + List fileList = new ArrayList<>(); + for( int i=0; i<10; i++ ){ + String s = "file_" + i + "," + i + "," + i; + File f = new File(baseDir, "origFile" + i + ".csv"); + FileUtils.writeStringToFile(f, s, StandardCharsets.UTF_8); + fileList.add(f); + } + + FileBatch fb = FileBatch.forFiles(fileList); + + RecordReader rr = new CSVRecordReader(); + FileBatchRecordReader fbrr = new FileBatchRecordReader(rr, fb); + + + for( int test=0; test<3; test++) { + for (int i = 0; i < 10; i++) { + assertTrue(fbrr.hasNext()); + List next = fbrr.next(); + assertEquals(3, next.size()); + String s1 = "file_" + i; + assertEquals(s1, next.get(0).toString()); + assertEquals(String.valueOf(i), next.get(1).toString()); + assertEquals(String.valueOf(i), next.get(2).toString()); + } + assertFalse(fbrr.hasNext()); + assertTrue(fbrr.resetSupported()); + fbrr.reset(); + } + } + + @Test + public void testCsvSequence() throws Exception { + //CSV sequence - 3 lines per file, 10 files + File baseDir = testDir; + + List fileList = new ArrayList<>(); + for( int i=0; i<10; i++ ){ + StringBuilder sb = new StringBuilder(); + for( int j=0; j<3; j++ ){ + if(j > 0) + sb.append("\n"); + sb.append("file_" + i + "," + i + "," + j); + } + File f = new File(baseDir, "origFile" + i + ".csv"); + FileUtils.writeStringToFile(f, sb.toString(), StandardCharsets.UTF_8); + fileList.add(f); + } + + FileBatch fb = FileBatch.forFiles(fileList); + SequenceRecordReader rr = new CSVSequenceRecordReader(); + FileBatchSequenceRecordReader fbrr = new FileBatchSequenceRecordReader(rr, fb); + + + for( int test=0; test<3; test++) { + for (int i = 0; i < 10; i++) { + assertTrue(fbrr.hasNext()); + List> next = fbrr.sequenceRecord(); + assertEquals(3, next.size()); + int count = 0; + for(List step : next ){ + String s1 = "file_" + i; + assertEquals(s1, step.get(0).toString()); + assertEquals(String.valueOf(i), step.get(1).toString()); + assertEquals(String.valueOf(count++), step.get(2).toString()); + } + } + assertFalse(fbrr.hasNext()); + assertTrue(fbrr.resetSupported()); + fbrr.reset(); + } + } + +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileRecordReaderTest.java similarity index 88% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileRecordReaderTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileRecordReaderTest.java index fa9a5b42a..680254125 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileRecordReaderTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileRecordReaderTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.reader.impl; import org.datavec.api.records.Record; @@ -25,32 +26,28 @@ import org.datavec.api.split.CollectionInputSplit; import org.datavec.api.split.FileSplit; import org.datavec.api.split.InputSplit; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.common.io.ClassPathResource; + import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("File Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class FileRecordReaderTest extends BaseND4JTest { +public class FileRecordReaderTest extends BaseND4JTest { @Test - @DisplayName("Test Reset") - void testReset() throws Exception { + public void testReset() throws Exception { FileRecordReader rr = new FileRecordReader(); rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); + int nResets = 5; for (int i = 0; i < nResets; i++) { + int lineCount = 0; while (rr.hasNext()) { List line = rr.next(); @@ -64,20 +61,25 @@ class FileRecordReaderTest extends BaseND4JTest { } @Test - @DisplayName("Test Meta") - void testMeta() throws Exception { + public void testMeta() throws Exception { FileRecordReader rr = new FileRecordReader(); + + URI[] arr = new URI[3]; arr[0] = new ClassPathResource("datavec-api/csvsequence_0.txt").getFile().toURI(); arr[1] = new ClassPathResource("datavec-api/csvsequence_1.txt").getFile().toURI(); arr[2] = new ClassPathResource("datavec-api/csvsequence_2.txt").getFile().toURI(); + InputSplit is = new CollectionInputSplit(Arrays.asList(arr)); rr.initialize(is); + List> out = new ArrayList<>(); while (rr.hasNext()) { out.add(rr.next()); } + assertEquals(3, out.size()); + rr.reset(); List> out2 = new ArrayList<>(); List out3 = new ArrayList<>(); @@ -88,10 +90,13 @@ class FileRecordReaderTest extends BaseND4JTest { out2.add(r.getRecord()); out3.add(r); meta.add(r.getMetaData()); + assertEquals(arr[count++], r.getMetaData().getURI()); } + assertEquals(out, out2); List fromMeta = rr.loadFromMetaData(meta); assertEquals(out3, fromMeta); } + } diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonLineRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonLineRecordReaderTest.java new file mode 100644 index 000000000..883f0e0d4 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonLineRecordReaderTest.java @@ -0,0 +1,123 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.jackson.FieldSelection; +import org.datavec.api.records.reader.impl.jackson.JacksonLineRecordReader; +import org.datavec.api.records.reader.impl.jackson.JacksonLineSequenceRecordReader; +import org.datavec.api.split.CollectionInputSplit; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.io.ClassPathResource; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class JacksonLineRecordReaderTest extends BaseND4JTest { + + @TempDir + public File testDir; + + public JacksonLineRecordReaderTest() { + } + + private static FieldSelection getFieldSelection() { + return new FieldSelection.Builder().addField("value1"). + addField("value2"). + addField("value3"). + addField("value4"). + addField("value5"). + addField("value6"). + addField("value7"). + addField("value8"). + addField("value9"). + addField("value10").build(); + } + + @Test + public void testReadJSON() throws Exception { + + RecordReader rr = new JacksonLineRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory())); + rr.initialize(new FileSplit(new ClassPathResource("datavec-api/json/json_test_3.txt").getFile())); + + testJacksonRecordReader(rr); + } + + private static void testJacksonRecordReader(RecordReader rr) { + while (rr.hasNext()) { + List json0 = rr.next(); + //System.out.println(json0); + assert(json0.size() > 0); + } + } + + + @Test + public void testJacksonLineSequenceRecordReader() throws Exception { + File dir = testDir; + new ClassPathResource("datavec-api/JacksonLineSequenceRecordReaderTest/").copyDirectory(dir); + + FieldSelection f = new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b") + .addField(new Text("MISSING_CX"), "c", "x").build(); + + JacksonLineSequenceRecordReader rr = new JacksonLineSequenceRecordReader(f, new ObjectMapper(new JsonFactory())); + File[] files = dir.listFiles(); + Arrays.sort(files); + URI[] u = new URI[files.length]; + for( int i=0; i> expSeq0 = new ArrayList<>(); + expSeq0.add(Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0"))); + expSeq0.add(Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1"))); + expSeq0.add(Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX"))); + + List> expSeq1 = new ArrayList<>(); + expSeq1.add(Arrays.asList((Writable) new Text("aValue3"), new Text("bValue3"), new Text("cxValue3"))); + + + int count = 0; + while(rr.hasNext()){ + List> next = rr.sequenceRecord(); + if(count++ == 0){ + assertEquals(expSeq0, next); + } else { + assertEquals(expSeq1, next); + } + } + + assertEquals(2, count); + } +} diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonRecordReaderTest.java new file mode 100644 index 000000000..b6f13adbd --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonRecordReaderTest.java @@ -0,0 +1,261 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import org.apache.commons.io.FileUtils; +import org.datavec.api.io.labels.PathLabelGenerator; +import org.datavec.api.records.Record; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.jackson.FieldSelection; +import org.datavec.api.records.reader.impl.jackson.JacksonRecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.api.split.NumberedFileInputSplit; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.io.ClassPathResource; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.xml.XmlFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import java.io.File; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +public class JacksonRecordReaderTest extends BaseND4JTest { + + + + + @Test + public void testReadingJson() throws Exception { + //Load 3 values from 3 JSON files + //stricture: a:value, b:value, c:x:value, c:y:value + //And we want to load only a:value, b:value and c:x:value + //For first JSON file: all values are present + //For second JSON file: b:value is missing + //For third JSON file: c:x:value is missing + + ClassPathResource cpr = new ClassPathResource("datavec-api/json/"); + File f = new File(FileUtils.getTempDirectoryPath()+File.separatorChar+ UUID.randomUUID().toString()); + FileUtils.forceMkdir(f); + cpr.copyDirectory(f); + String path = new File(f, "json_test_%d.txt").getAbsolutePath(); + + InputSplit is = new NumberedFileInputSplit(path, 0, 2); + + RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory())); + rr.initialize(is); + + testJacksonRecordReader(rr); + } + + @Test + public void testReadingYaml() throws Exception { + //Exact same information as JSON format, but in YAML format + + ClassPathResource cpr = new ClassPathResource("datavec-api/yaml/"); + File f = new File(FileUtils.getTempDirectoryPath()+File.separatorChar+ UUID.randomUUID().toString()); + FileUtils.forceMkdir(f); + cpr.copyDirectory(f); + String path = new File(f, "yaml_test_%d.txt").getAbsolutePath(); + + + InputSplit is = new NumberedFileInputSplit(path, 0, 2); + + RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new YAMLFactory())); + rr.initialize(is); + + testJacksonRecordReader(rr); + } + + @Test + public void testReadingXml() throws Exception { + //Exact same information as JSON format, but in XML format + + ClassPathResource cpr = new ClassPathResource("datavec-api/xml/"); + File f = new File(FileUtils.getTempDirectoryPath()+File.separatorChar+ UUID.randomUUID().toString()); + FileUtils.forceMkdir(f); + cpr.copyDirectory(f); + String path = new File(f, "xml_test_%d.txt").getAbsolutePath(); + + InputSplit is = new NumberedFileInputSplit(path, 0, 2); + + RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new XmlFactory())); + rr.initialize(is); + + testJacksonRecordReader(rr); + } + + + private static FieldSelection getFieldSelection() { + return new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b") + .addField(new Text("MISSING_CX"), "c", "x").build(); + } + + + + private static void testJacksonRecordReader(RecordReader rr) { + + List json0 = rr.next(); + List exp0 = Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0")); + assertEquals(exp0, json0); + + List json1 = rr.next(); + List exp1 = + Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1")); + assertEquals(exp1, json1); + + List json2 = rr.next(); + List exp2 = + Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX")); + assertEquals(exp2, json2); + + assertFalse(rr.hasNext()); + + //Test reset + rr.reset(); + assertEquals(exp0, rr.next()); + assertEquals(exp1, rr.next()); + assertEquals(exp2, rr.next()); + assertFalse(rr.hasNext()); + } + + @Test + public void testAppendingLabels() throws Exception { + + ClassPathResource cpr = new ClassPathResource("datavec-api/json/"); + File f = new File(FileUtils.getTempDirectoryPath()+File.separatorChar+ UUID.randomUUID().toString()); + FileUtils.forceMkdir(f); + cpr.copyDirectory(f); + String path = new File(f, "json_test_%d.txt").getAbsolutePath(); + + InputSplit is = new NumberedFileInputSplit(path, 0, 2); + + //Insert at the end: + RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1, + new LabelGen()); + rr.initialize(is); + + List exp0 = Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0"), + new IntWritable(0)); + assertEquals(exp0, rr.next()); + + List exp1 = Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1"), + new IntWritable(1)); + assertEquals(exp1, rr.next()); + + List exp2 = Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX"), + new IntWritable(2)); + assertEquals(exp2, rr.next()); + + //Insert at position 0: + rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1, + new LabelGen(), 0); + rr.initialize(is); + + exp0 = Arrays.asList((Writable) new IntWritable(0), new Text("aValue0"), new Text("bValue0"), + new Text("cxValue0")); + assertEquals(exp0, rr.next()); + + exp1 = Arrays.asList((Writable) new IntWritable(1), new Text("aValue1"), new Text("MISSING_B"), + new Text("cxValue1")); + assertEquals(exp1, rr.next()); + + exp2 = Arrays.asList((Writable) new IntWritable(2), new Text("aValue2"), new Text("bValue2"), + new Text("MISSING_CX")); + assertEquals(exp2, rr.next()); + } + + @Test + public void testAppendingLabelsMetaData() throws Exception { + ClassPathResource cpr = new ClassPathResource("datavec-api/json/"); + File f = new File(FileUtils.getTempDirectoryPath()+File.separatorChar+ UUID.randomUUID().toString()); + FileUtils.forceMkdir(f); + cpr.copyDirectory(f); + String path = new File(f, "json_test_%d.txt").getAbsolutePath(); + + InputSplit is = new NumberedFileInputSplit(path, 0, 2); + + //Insert at the end: + RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1, + new LabelGen()); + rr.initialize(is); + + List> out = new ArrayList<>(); + while (rr.hasNext()) { + out.add(rr.next()); + } + assertEquals(3, out.size()); + + rr.reset(); + + List> out2 = new ArrayList<>(); + List outRecord = new ArrayList<>(); + List meta = new ArrayList<>(); + while (rr.hasNext()) { + Record r = rr.nextRecord(); + out2.add(r.getRecord()); + outRecord.add(r); + meta.add(r.getMetaData()); + } + + assertEquals(out, out2); + + List fromMeta = rr.loadFromMetaData(meta); + assertEquals(outRecord, fromMeta); + } + + + private static class LabelGen implements PathLabelGenerator { + + @Override + public Writable getLabelForPath(String path) { + if (path.endsWith("0.txt")) + return new IntWritable(0); + else if (path.endsWith("1.txt")) + return new IntWritable(1); + else + return new IntWritable(2); + } + + @Override + public Writable getLabelForPath(URI uri) { + return getLabelForPath(uri.getPath()); + } + + @Override + public boolean inferLabelClasses() { + return true; + } + } + +} diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/LibSvmRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/LibSvmRecordReaderTest.java new file mode 100644 index 000000000..13560a907 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/LibSvmRecordReaderTest.java @@ -0,0 +1,439 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.reader.impl.misc.LibSvmRecordReader; +import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.io.ClassPathResource; + +import java.io.IOException; +import java.util.*; + +import static org.datavec.api.records.reader.impl.misc.LibSvmRecordReader.*; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class LibSvmRecordReaderTest extends BaseND4JTest { + + @Test + public void testBasicRecord() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 7 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5), + new IntWritable(7))); + // 2 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO, + new IntWritable(2))); + // 33 + correct.put(2, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + new IntWritable(33))); + + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testNoAppendLabel() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 7 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5))); + // 2 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO)); + // 33 + correct.put(2, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO)); + + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + config.setBoolean(SVMLightRecordReader.APPEND_LABEL, false); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testNoLabel() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5))); + // qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO)); + // 1:1.0 + correct.put(2, Arrays.asList(new DoubleWritable(1.0), ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO)); + // + correct.put(3, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO)); + + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + config.setBoolean(SVMLightRecordReader.APPEND_LABEL, true); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/noLabels.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testMultioutputRecord() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 7 2.45,9 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5), + new IntWritable(7), new DoubleWritable(2.45), + new IntWritable(9))); + // 2,3,4 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO, + new IntWritable(2), new IntWritable(3), + new IntWritable(4))); + // 33,32.0,31.9 + correct.put(2, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + new IntWritable(33), new DoubleWritable(32.0), + new DoubleWritable(31.9))); + + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multioutput.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + + @Test + public void testMultilabelRecord() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 1,3 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5), + LABEL_ONE, LABEL_ZERO, + LABEL_ONE, LABEL_ZERO)); + // 2 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO, + LABEL_ZERO, LABEL_ONE, + LABEL_ZERO, LABEL_ZERO)); + // 1,2,4 + correct.put(2, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ONE, LABEL_ONE, + LABEL_ZERO, LABEL_ONE)); + // 1:1.0 + correct.put(3, Arrays.asList(new DoubleWritable(1.0), ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO)); + // + correct.put(4, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO)); + + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); + config.setBoolean(LibSvmRecordReader.MULTILABEL, true); + config.setInt(LibSvmRecordReader.NUM_LABELS, 4); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testZeroBasedIndexing() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 1,3 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, + ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5), + LABEL_ZERO, + LABEL_ONE, LABEL_ZERO, + LABEL_ONE, LABEL_ZERO)); + // 2 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(ZERO, + new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO, + LABEL_ZERO, + LABEL_ZERO, LABEL_ONE, + LABEL_ZERO, LABEL_ZERO)); + // 1,2,4 + correct.put(2, Arrays.asList(ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, + LABEL_ONE, LABEL_ONE, + LABEL_ZERO, LABEL_ONE)); + // 1:1.0 + correct.put(3, Arrays.asList(ZERO, + new DoubleWritable(1.0), ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO)); + // + correct.put(4, Arrays.asList(ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO)); + + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + // Zero-based indexing is default + config.setBoolean(SVMLightRecordReader.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD! + config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 11); + config.setBoolean(LibSvmRecordReader.MULTILABEL, true); + config.setInt(LibSvmRecordReader.NUM_LABELS, 5); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testNoSuchElementException() throws Exception { + Assertions.assertThrows(NoSuchElementException.class, () -> { + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 11); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + while (rr.hasNext()) + rr.next(); + rr.next(); + }); + } + + @Test + public void failedToSetNumFeaturesException() throws Exception { + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + while (rr.hasNext()) + rr.next(); + }); + } + + @Test + public void testInconsistentNumLabelsException() throws Exception { + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/inconsistentNumLabels.txt").getFile())); + while (rr.hasNext()) + rr.next(); + }); + } + + @Test + public void testInconsistentNumMultiabelsException() throws Exception { + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(LibSvmRecordReader.MULTILABEL, false); + config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); + while (rr.hasNext()) + rr.next(); + }); + } + + @Test + public void testFeatureIndexExceedsNumFeatures() throws Exception { + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> { + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 9); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + rr.next(); + }); + } + + @Test + public void testLabelIndexExceedsNumLabels() throws Exception { + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> { + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); + config.setInt(LibSvmRecordReader.NUM_LABELS, 6); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + rr.next(); + }); + } + + @Test + public void testZeroIndexFeatureWithoutUsingZeroIndexing() throws Exception { + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> { + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexFeature.txt").getFile())); + rr.next(); + }); + } + + @Test + public void testZeroIndexLabelWithoutUsingZeroIndexing() throws Exception { + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> { + LibSvmRecordReader rr = new LibSvmRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); + config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); + config.setBoolean(LibSvmRecordReader.MULTILABEL, true); + config.setInt(LibSvmRecordReader.NUM_LABELS, 2); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexLabel.txt").getFile())); + rr.next(); + }); + } +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/LineReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/LineReaderTest.java similarity index 85% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/LineReaderTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/LineReaderTest.java index 3d197405c..73c6053dc 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/LineReaderTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/LineReaderTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.reader.impl; import org.apache.commons.io.FileUtils; @@ -29,11 +30,10 @@ import org.datavec.api.split.FileSplit; import org.datavec.api.split.InputSplit; import org.datavec.api.split.InputStreamInputSplit; import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.tests.BaseND4JTest; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -44,34 +44,34 @@ import java.util.Arrays; import java.util.List; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Line Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class LineReaderTest extends BaseND4JTest { +public class LineReaderTest extends BaseND4JTest { + @TempDir + public File testDir; @Test - @DisplayName("Test Line Reader") - void testLineReader(@TempDir Path tmpDir) throws Exception { - File tmpdir = tmpDir.toFile(); + public void testLineReader() throws Exception { + File tmpdir = testDir; if (tmpdir.exists()) tmpdir.delete(); tmpdir.mkdir(); + File tmp1 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp1.txt")); File tmp2 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp2.txt")); File tmp3 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp3.txt")); + FileUtils.writeLines(tmp1, Arrays.asList("1", "2", "3")); FileUtils.writeLines(tmp2, Arrays.asList("4", "5", "6")); FileUtils.writeLines(tmp3, Arrays.asList("7", "8", "9")); + InputSplit split = new FileSplit(tmpdir); + RecordReader reader = new LineRecordReader(); reader.initialize(split); + int count = 0; List> list = new ArrayList<>(); while (reader.hasNext()) { @@ -80,27 +80,34 @@ class LineReaderTest extends BaseND4JTest { list.add(l); count++; } + assertEquals(9, count); } @Test - @DisplayName("Test Line Reader Meta Data") - void testLineReaderMetaData(@TempDir Path tmpDir) throws Exception { - File tmpdir = tmpDir.toFile(); + public void testLineReaderMetaData() throws Exception { + File tmpdir = testDir; + File tmp1 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp1.txt")); File tmp2 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp2.txt")); File tmp3 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp3.txt")); + FileUtils.writeLines(tmp1, Arrays.asList("1", "2", "3")); FileUtils.writeLines(tmp2, Arrays.asList("4", "5", "6")); FileUtils.writeLines(tmp3, Arrays.asList("7", "8", "9")); + InputSplit split = new FileSplit(tmpdir); + RecordReader reader = new LineRecordReader(); reader.initialize(split); + List> list = new ArrayList<>(); while (reader.hasNext()) { list.add(reader.next()); } assertEquals(9, list.size()); + + List> out2 = new ArrayList<>(); List out3 = new ArrayList<>(); List meta = new ArrayList<>(); @@ -116,10 +123,13 @@ class LineReaderTest extends BaseND4JTest { assertEquals(uri, split.locations()[fileIdx]); count++; } + assertEquals(list, out2); + List fromMeta = reader.loadFromMetaData(meta); assertEquals(out3, fromMeta); - // try: second line of second and third files only... + + //try: second line of second and third files only... List subsetMeta = new ArrayList<>(); subsetMeta.add(meta.get(4)); subsetMeta.add(meta.get(7)); @@ -130,22 +140,27 @@ class LineReaderTest extends BaseND4JTest { } @Test - @DisplayName("Test Line Reader With Input Stream Input Split") - void testLineReaderWithInputStreamInputSplit(@TempDir Path testDir) throws Exception { - File tmpdir = testDir.toFile(); + public void testLineReaderWithInputStreamInputSplit() throws Exception { + File tmpdir = testDir; + File tmp1 = new File(tmpdir, "tmp1.txt.gz"); + OutputStream os = new GZIPOutputStream(new FileOutputStream(tmp1, false)); IOUtils.writeLines(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9"), null, os); os.flush(); os.close(); + InputSplit split = new InputStreamInputSplit(new GZIPInputStream(new FileInputStream(tmp1))); + RecordReader reader = new LineRecordReader(); reader.initialize(split); + int count = 0; while (reader.hasNext()) { assertEquals(1, reader.next().size()); count++; } + assertEquals(9, count); } } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/RegexRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/RegexRecordReaderTest.java similarity index 77% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/RegexRecordReaderTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/RegexRecordReaderTest.java index fb0200ab5..a2d6622b3 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/RegexRecordReaderTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/RegexRecordReaderTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.reader.impl; import org.datavec.api.records.Record; @@ -32,45 +33,43 @@ import org.datavec.api.split.InputSplit; import org.datavec.api.split.NumberedFileInputSplit; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.common.io.ClassPathResource; + import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Regex Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class RegexRecordReaderTest extends BaseND4JTest { +public class RegexRecordReaderTest extends BaseND4JTest { @TempDir - public Path testDir; + File testDir; @Test - @DisplayName("Test Regex Line Record Reader") - void testRegexLineRecordReader() throws Exception { + public void testRegexLineRecordReader() throws Exception { String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)"; + RecordReader rr = new RegexLineRecordReader(regex, 1); rr.initialize(new FileSplit(new ClassPathResource("datavec-api/logtestdata/logtestfile0.txt").getFile())); - List exp0 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.001"), new Text("1"), new Text("DEBUG"), new Text("First entry message!")); - List exp1 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.002"), new Text("2"), new Text("INFO"), new Text("Second entry message!")); - List exp2 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.003"), new Text("3"), new Text("WARN"), new Text("Third entry message!")); + + List exp0 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.001"), new Text("1"), + new Text("DEBUG"), new Text("First entry message!")); + List exp1 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.002"), new Text("2"), + new Text("INFO"), new Text("Second entry message!")); + List exp2 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.003"), new Text("3"), + new Text("WARN"), new Text("Third entry message!")); assertEquals(exp0, rr.next()); assertEquals(exp1, rr.next()); assertEquals(exp2, rr.next()); assertFalse(rr.hasNext()); - // Test reset: + + //Test reset: rr.reset(); assertEquals(exp0, rr.next()); assertEquals(exp1, rr.next()); @@ -79,57 +78,74 @@ class RegexRecordReaderTest extends BaseND4JTest { } @Test - @DisplayName("Test Regex Line Record Reader Meta") - void testRegexLineRecordReaderMeta() throws Exception { + public void testRegexLineRecordReaderMeta() throws Exception { String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)"; + RecordReader rr = new RegexLineRecordReader(regex, 1); rr.initialize(new FileSplit(new ClassPathResource("datavec-api/logtestdata/logtestfile0.txt").getFile())); + List> list = new ArrayList<>(); while (rr.hasNext()) { list.add(rr.next()); } assertEquals(3, list.size()); + List list2 = new ArrayList<>(); List> list3 = new ArrayList<>(); List meta = new ArrayList<>(); rr.reset(); - // Start by skipping 1 line - int count = 1; + int count = 1; //Start by skipping 1 line while (rr.hasNext()) { Record r = rr.nextRecord(); list2.add(r); list3.add(r.getRecord()); meta.add(r.getMetaData()); + assertEquals(count++, ((RecordMetaDataLine) r.getMetaData()).getLineNumber()); } + List fromMeta = rr.loadFromMetaData(meta); + assertEquals(list, list3); assertEquals(list2, fromMeta); } @Test - @DisplayName("Test Regex Sequence Record Reader") - void testRegexSequenceRecordReader(@TempDir Path testDir) throws Exception { + public void testRegexSequenceRecordReader() throws Exception { String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)"; + ClassPathResource cpr = new ClassPathResource("datavec-api/logtestdata/"); - File f = testDir.toFile(); + File f = testDir; cpr.copyDirectory(f); String path = new File(f, "logtestfile%d.txt").getAbsolutePath(); + InputSplit is = new NumberedFileInputSplit(path, 0, 1); + SequenceRecordReader rr = new RegexSequenceRecordReader(regex, 1); rr.initialize(is); + List> exp0 = new ArrayList<>(); - exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.001"), new Text("1"), new Text("DEBUG"), new Text("First entry message!"))); - exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.002"), new Text("2"), new Text("INFO"), new Text("Second entry message!"))); - exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.003"), new Text("3"), new Text("WARN"), new Text("Third entry message!"))); + exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.001"), new Text("1"), new Text("DEBUG"), + new Text("First entry message!"))); + exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.002"), new Text("2"), new Text("INFO"), + new Text("Second entry message!"))); + exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.003"), new Text("3"), new Text("WARN"), + new Text("Third entry message!"))); + + List> exp1 = new ArrayList<>(); - exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.011"), new Text("11"), new Text("DEBUG"), new Text("First entry message!"))); - exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.012"), new Text("12"), new Text("INFO"), new Text("Second entry message!"))); - exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.013"), new Text("13"), new Text("WARN"), new Text("Third entry message!"))); + exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.011"), new Text("11"), new Text("DEBUG"), + new Text("First entry message!"))); + exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.012"), new Text("12"), new Text("INFO"), + new Text("Second entry message!"))); + exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.013"), new Text("13"), new Text("WARN"), + new Text("Third entry message!"))); + assertEquals(exp0, rr.sequenceRecord()); assertEquals(exp1, rr.sequenceRecord()); assertFalse(rr.hasNext()); - // Test resetting: + + //Test resetting: rr.reset(); assertEquals(exp0, rr.sequenceRecord()); assertEquals(exp1, rr.sequenceRecord()); @@ -137,20 +153,24 @@ class RegexRecordReaderTest extends BaseND4JTest { } @Test - @DisplayName("Test Regex Sequence Record Reader Meta") - void testRegexSequenceRecordReaderMeta(@TempDir Path testDir) throws Exception { + public void testRegexSequenceRecordReaderMeta() throws Exception { String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)"; + ClassPathResource cpr = new ClassPathResource("datavec-api/logtestdata/"); - File f = testDir.toFile(); + File f = testDir; cpr.copyDirectory(f); String path = new File(f, "logtestfile%d.txt").getAbsolutePath(); + InputSplit is = new NumberedFileInputSplit(path, 0, 1); + SequenceRecordReader rr = new RegexSequenceRecordReader(regex, 1); rr.initialize(is); + List>> out = new ArrayList<>(); while (rr.hasNext()) { out.add(rr.sequenceRecord()); } + assertEquals(2, out.size()); List>> out2 = new ArrayList<>(); List out3 = new ArrayList<>(); @@ -162,8 +182,11 @@ class RegexRecordReaderTest extends BaseND4JTest { out3.add(seqr); meta.add(seqr.getMetaData()); } + List fromMeta = rr.loadSequenceFromMetaData(meta); + assertEquals(out, out2); assertEquals(out3, fromMeta); } + } diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/SVMLightRecordReaderTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/SVMLightRecordReaderTest.java new file mode 100644 index 000000000..e091c6945 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/SVMLightRecordReaderTest.java @@ -0,0 +1,452 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.io.ClassPathResource; + +import java.io.IOException; +import java.util.*; + +import static org.datavec.api.records.reader.impl.misc.SVMLightRecordReader.*; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SVMLightRecordReaderTest extends BaseND4JTest { + + @Test + public void testBasicRecord() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 7 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5), + new IntWritable(7))); + // 2 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO, + new IntWritable(2))); + // 33 + correct.put(2, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + new IntWritable(33))); + + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testNoAppendLabel() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 7 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5))); + // 2 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO)); + // 33 + correct.put(2, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO)); + + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + config.setBoolean(SVMLightRecordReader.APPEND_LABEL, false); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testNoLabel() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5))); + // qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO)); + // 1:1.0 + correct.put(2, Arrays.asList(new DoubleWritable(1.0), ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO)); + // + correct.put(3, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO)); + + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + config.setBoolean(SVMLightRecordReader.APPEND_LABEL, true); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/noLabels.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testMultioutputRecord() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 7 2.45,9 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5), + new IntWritable(7), new DoubleWritable(2.45), + new IntWritable(9))); + // 2,3,4 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO, + new IntWritable(2), new IntWritable(3), + new IntWritable(4))); + // 33,32.0,31.9 + correct.put(2, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + new IntWritable(33), new DoubleWritable(32.0), + new DoubleWritable(31.9))); + + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multioutput.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + + @Test + public void testMultilabelRecord() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 1,3 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5), + LABEL_ONE, LABEL_ZERO, + LABEL_ONE, LABEL_ZERO)); + // 2 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO, + LABEL_ZERO, LABEL_ONE, + LABEL_ZERO, LABEL_ZERO)); + // 1,2,4 + correct.put(2, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ONE, LABEL_ONE, + LABEL_ZERO, LABEL_ONE)); + // 1:1.0 + correct.put(3, Arrays.asList(new DoubleWritable(1.0), ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO)); + // + correct.put(4, Arrays.asList(ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO)); + + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + config.setBoolean(SVMLightRecordReader.MULTILABEL, true); + config.setInt(SVMLightRecordReader.NUM_LABELS, 4); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testZeroBasedIndexing() throws IOException, InterruptedException { + Map> correct = new HashMap<>(); + // 1,3 2:1 4:2 6:3 8:4 10:5 + correct.put(0, Arrays.asList(ZERO, + ZERO, ONE, + ZERO, new DoubleWritable(2), + ZERO, new DoubleWritable(3), + ZERO, new DoubleWritable(4), + ZERO, new DoubleWritable(5), + LABEL_ZERO, + LABEL_ONE, LABEL_ZERO, + LABEL_ONE, LABEL_ZERO)); + // 2 qid:42 1:0.1 2:2 6:6.6 8:80 + correct.put(1, Arrays.asList(ZERO, + new DoubleWritable(0.1), new DoubleWritable(2), + ZERO, ZERO, + ZERO, new DoubleWritable(6.6), + ZERO, new DoubleWritable(80), + ZERO, ZERO, + LABEL_ZERO, + LABEL_ZERO, LABEL_ONE, + LABEL_ZERO, LABEL_ZERO)); + // 1,2,4 + correct.put(2, Arrays.asList(ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, + LABEL_ONE, LABEL_ONE, + LABEL_ZERO, LABEL_ONE)); + // 1:1.0 + correct.put(3, Arrays.asList(ZERO, + new DoubleWritable(1.0), ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO)); + // + correct.put(4, Arrays.asList(ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + ZERO, ZERO, + LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO, + LABEL_ZERO, LABEL_ZERO)); + + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + // Zero-based indexing is default + config.setBoolean(SVMLightRecordReader.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD! + config.setInt(SVMLightRecordReader.NUM_FEATURES, 11); + config.setBoolean(SVMLightRecordReader.MULTILABEL, true); + config.setInt(SVMLightRecordReader.NUM_LABELS, 5); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); + int i = 0; + while (rr.hasNext()) { + List record = rr.next(); + assertEquals(correct.get(i), record); + i++; + } + assertEquals(i, correct.size()); + } + + @Test + public void testNextRecord() throws IOException, InterruptedException { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + config.setBoolean(SVMLightRecordReader.APPEND_LABEL, false); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + + Record record = rr.nextRecord(); + List recordList = record.getRecord(); + assertEquals(new DoubleWritable(1.0), recordList.get(1)); + assertEquals(new DoubleWritable(3.0), recordList.get(5)); + assertEquals(new DoubleWritable(4.0), recordList.get(7)); + + record = rr.nextRecord(); + recordList = record.getRecord(); + assertEquals(new DoubleWritable(0.1), recordList.get(0)); + assertEquals(new DoubleWritable(6.6), recordList.get(5)); + assertEquals(new DoubleWritable(80.0), recordList.get(7)); + } + + @Test + public void testNoSuchElementException() throws Exception { + Assertions.assertThrows(NoSuchElementException.class, () -> { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 11); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + while (rr.hasNext()) + rr.next(); + rr.next(); + }); + } + + @Test + public void failedToSetNumFeaturesException() throws Exception { + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + while (rr.hasNext()) + rr.next(); + }); + } + + @Test + public void testInconsistentNumLabelsException() throws Exception { + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/inconsistentNumLabels.txt").getFile())); + while (rr.hasNext()) + rr.next(); + }); + } + + @Test + public void failedToSetNumMultiabelsException() throws Exception { + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); + while (rr.hasNext()) + rr.next(); + }); + } + + @Test + public void testFeatureIndexExceedsNumFeatures() throws Exception { + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 9); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + rr.next(); + }); + } + + @Test + public void testLabelIndexExceedsNumLabels() throws Exception { + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + config.setInt(SVMLightRecordReader.NUM_LABELS, 6); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); + rr.next(); + }); + } + + @Test + public void testZeroIndexFeatureWithoutUsingZeroIndexing() throws Exception { + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexFeature.txt").getFile())); + rr.next(); + }); + } + + @Test + public void testZeroIndexLabelWithoutUsingZeroIndexing() throws Exception { + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> { + SVMLightRecordReader rr = new SVMLightRecordReader(); + Configuration config = new Configuration(); + config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); + config.setBoolean(SVMLightRecordReader.MULTILABEL, true); + config.setInt(SVMLightRecordReader.NUM_LABELS, 2); + rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexLabel.txt").getFile())); + rr.next(); + }); + } +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestCollectionRecordReaders.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestCollectionRecordReaders.java similarity index 96% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestCollectionRecordReaders.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestCollectionRecordReaders.java index acaba9ccb..decbf0275 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestCollectionRecordReaders.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestCollectionRecordReaders.java @@ -26,18 +26,15 @@ import org.datavec.api.records.reader.SequenceRecordReader; import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestCollectionRecordReaders extends BaseND4JTest { @Test diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestConcatenatingRecordReader.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestConcatenatingRecordReader.java similarity index 93% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestConcatenatingRecordReader.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestConcatenatingRecordReader.java index 04056f6e0..b39a678ce 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestConcatenatingRecordReader.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestConcatenatingRecordReader.java @@ -23,15 +23,12 @@ package org.datavec.api.records.reader.impl; import org.datavec.api.records.reader.RecordReader; import org.datavec.api.records.reader.impl.csv.CSVRecordReader; import org.datavec.api.split.FileSplit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestConcatenatingRecordReader extends BaseND4JTest { @Test diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestSerialization.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestSerialization.java new file mode 100644 index 000000000..c6de5ebcb --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestSerialization.java @@ -0,0 +1,124 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.csv.*; +import org.datavec.api.records.reader.impl.jackson.FieldSelection; +import org.datavec.api.records.reader.impl.jackson.JacksonLineRecordReader; +import org.datavec.api.records.reader.impl.jackson.JacksonRecordReader; +import org.datavec.api.records.reader.impl.misc.LibSvmRecordReader; +import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader; +import org.datavec.api.records.reader.impl.regex.RegexLineRecordReader; +import org.datavec.api.records.reader.impl.regex.RegexSequenceRecordReader; +import org.datavec.api.records.reader.impl.transform.TransformProcessRecordReader; +import org.datavec.api.records.reader.impl.transform.TransformProcessSequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.transform.MathFunction; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.io.ClassPathResource; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestSerialization extends BaseND4JTest { + + @Test + public void testRR() throws Exception { + + List rrs = new ArrayList<>(); + + rrs.add(new CSVNLinesSequenceRecordReader(10)); + rrs.add(new CSVRecordReader(10, ',')); + rrs.add(new CSVSequenceRecordReader(1, ",")); + rrs.add(new CSVVariableSlidingWindowRecordReader(5)); + rrs.add(new CSVRegexRecordReader(0, ",", null, new String[] {null, "(.+) (.+) (.+)"})); + rrs.add(new JacksonRecordReader(new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b") + .addField(new Text("MISSING_CX"), "c", "x").build(), new ObjectMapper(new JsonFactory()))); + rrs.add(new JacksonLineRecordReader(new FieldSelection.Builder().addField("value1") + .addField("value2").build(), new ObjectMapper(new JsonFactory()))); + rrs.add(new LibSvmRecordReader()); + rrs.add(new SVMLightRecordReader()); + rrs.add(new RegexLineRecordReader("(.+) (.+) (.+)", 0)); + rrs.add(new RegexSequenceRecordReader("(.+) (.+) (.+)", 0)); + rrs.add(new TransformProcessRecordReader(new CSVRecordReader(), getTp())); + rrs.add(new TransformProcessSequenceRecordReader(new CSVSequenceRecordReader(), getTp())); + rrs.add(new LineRecordReader()); + + for(RecordReader r : rrs){ + System.out.println(r.getClass().getName()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream os = new ObjectOutputStream(baos); + os.writeObject(r); + byte[] bytes = baos.toByteArray(); + + ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); + + RecordReader r2 = (RecordReader) ois.readObject(); + } + } + + private static TransformProcess getTp(){ + Schema s = new Schema.Builder().addColumnDouble("d").build(); + TransformProcess tp = new TransformProcess.Builder(s) + .doubleMathFunction("d", MathFunction.ABS) + .build(); + return tp; + } + + @Test + public void testCsvRRSerializationResults() throws Exception { + int skipLines = 3; + RecordReader r1 = new CSVRecordReader(skipLines, '\t'); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream os = new ObjectOutputStream(baos); + os.writeObject(r1); + byte[] bytes = baos.toByteArray(); + ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); + RecordReader r2 = (RecordReader) ois.readObject(); + + File f = new ClassPathResource("datavec-api/iris_tab_delim.txt").getFile(); + + r1.initialize(new FileSplit(f)); + r2.initialize(new FileSplit(f)); + + int count = 0; + while(r1.hasNext()){ + List n1 = r1.next(); + List n2 = r2.next(); + assertEquals(n1, n2); + count++; + } + + assertEquals(150-skipLines, count); + } + +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReaderTests.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReaderTests.java similarity index 90% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReaderTests.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReaderTests.java index 7782d8a52..ee2c9b091 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReaderTests.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/reader/impl/transform/TransformProcessRecordReaderTests.java @@ -30,11 +30,9 @@ import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.Writable; import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; import java.util.ArrayList; import java.util.Arrays; @@ -43,8 +41,6 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) public class TransformProcessRecordReaderTests extends BaseND4JTest { @Test @@ -78,11 +74,11 @@ public class TransformProcessRecordReaderTests extends BaseND4JTest { public void simpleTransformTestSequence() { List> sequence = new ArrayList<>(); //First window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0), + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1), + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2), + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2), new IntWritable(0))); Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/CSVRecordWriterTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/CSVRecordWriterTest.java similarity index 83% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/CSVRecordWriterTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/CSVRecordWriterTest.java index f9057f98f..9b90e9221 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/CSVRecordWriterTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/CSVRecordWriterTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.writer.impl; import org.datavec.api.records.reader.impl.csv.CSVRecordReader; @@ -26,45 +27,43 @@ import org.datavec.api.split.partition.NumberOfRecordsPartitioner; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; + import java.io.File; import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Csv Record Writer Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class CSVRecordWriterTest extends BaseND4JTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CSVRecordWriterTest extends BaseND4JTest { @BeforeEach - void setUp() throws Exception { + public void setUp() throws Exception { + } @Test - @DisplayName("Test Write") - void testWrite() throws Exception { + public void testWrite() throws Exception { File tempFile = File.createTempFile("datavec", "writer"); tempFile.deleteOnExit(); FileSplit fileSplit = new FileSplit(tempFile); CSVRecordWriter writer = new CSVRecordWriter(); - writer.initialize(fileSplit, new NumberOfRecordsPartitioner()); + writer.initialize(fileSplit,new NumberOfRecordsPartitioner()); List collection = new ArrayList<>(); collection.add(new Text("12")); collection.add(new Text("13")); collection.add(new Text("14")); + writer.write(collection); + CSVRecordReader reader = new CSVRecordReader(0); reader.initialize(new FileSplit(tempFile)); int cnt = 0; while (reader.hasNext()) { List line = new ArrayList<>(reader.next()); assertEquals(3, line.size()); + assertEquals(12, line.get(0).toInt()); assertEquals(13, line.get(1).toInt()); assertEquals(14, line.get(2).toInt()); diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/LibSvmRecordWriterTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/LibSvmRecordWriterTest.java similarity index 76% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/LibSvmRecordWriterTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/LibSvmRecordWriterTest.java index 3d1ea3090..885a75ec0 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/LibSvmRecordWriterTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/LibSvmRecordWriterTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.writer.impl; import org.apache.commons.io.FileUtils; @@ -29,94 +30,94 @@ import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.NDArrayWritable; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.io.ClassPathResource; + import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.jupiter.api.Assertions.assertThrows; -@DisplayName("Lib Svm Record Writer Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class LibSvmRecordWriterTest extends BaseND4JTest { +public class LibSvmRecordWriterTest extends BaseND4JTest { @Test - @DisplayName("Test Basic") - void testBasic() throws Exception { + public void testBasic() throws Exception { Configuration configWriter = new Configuration(); + Configuration configReader = new Configuration(); configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 10); configReader.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + File inputFile = new ClassPathResource("datavec-api/svmlight/basic.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @Test - @DisplayName("Test No Label") - void testNoLabel() throws Exception { + public void testNoLabel() throws Exception { Configuration configWriter = new Configuration(); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 9); + Configuration configReader = new Configuration(); configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 10); configReader.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + File inputFile = new ClassPathResource("datavec-api/svmlight/basic.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @Test - @DisplayName("Test Multioutput Record") - void testMultioutputRecord() throws Exception { + public void testMultioutputRecord() throws Exception { Configuration configWriter = new Configuration(); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 9); + Configuration configReader = new Configuration(); configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 10); configReader.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + File inputFile = new ClassPathResource("datavec-api/svmlight/multioutput.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @Test - @DisplayName("Test Multilabel Record") - void testMultilabelRecord() throws Exception { + public void testMultilabelRecord() throws Exception { Configuration configWriter = new Configuration(); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 9); configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true); + Configuration configReader = new Configuration(); configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 10); configReader.setBoolean(LibSvmRecordReader.MULTILABEL, true); configReader.setInt(LibSvmRecordReader.NUM_LABELS, 4); configReader.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); + File inputFile = new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @Test - @DisplayName("Test Zero Based Indexing") - void testZeroBasedIndexing() throws Exception { + public void testZeroBasedIndexing() throws Exception { Configuration configWriter = new Configuration(); configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_INDEXING, true); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 10); configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true); + Configuration configReader = new Configuration(); configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 11); configReader.setBoolean(LibSvmRecordReader.MULTILABEL, true); configReader.setInt(LibSvmRecordReader.NUM_LABELS, 5); + File inputFile = new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @@ -127,9 +128,10 @@ class LibSvmRecordWriterTest extends BaseND4JTest { tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { - FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + FileSplit outputSplit = new FileSplit(tempFile); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); LibSvmRecordReader rr = new LibSvmRecordReader(); rr.initialize(configReader, new FileSplit(inputFile)); while (rr.hasNext()) { @@ -137,6 +139,7 @@ class LibSvmRecordWriterTest extends BaseND4JTest { writer.write(record); } } + Pattern p = Pattern.compile(String.format("%s:\\d+ ", LibSvmRecordReader.QID_PREFIX)); List linesOriginal = new ArrayList<>(); for (String line : FileUtils.readLines(inputFile)) { @@ -157,8 +160,7 @@ class LibSvmRecordWriterTest extends BaseND4JTest { } @Test - @DisplayName("Test ND Array Writables") - void testNDArrayWritables() throws Exception { + public void testNDArrayWritables() throws Exception { INDArray arr2 = Nd4j.zeros(2); arr2.putScalar(0, 11); arr2.putScalar(1, 12); @@ -166,28 +168,35 @@ class LibSvmRecordWriterTest extends BaseND4JTest { arr3.putScalar(0, 13); arr3.putScalar(1, 14); arr3.putScalar(2, 15); - List record = Arrays.asList((Writable) new DoubleWritable(1), new NDArrayWritable(arr2), new IntWritable(2), new DoubleWritable(3), new NDArrayWritable(arr3), new IntWritable(4)); + List record = Arrays.asList((Writable) new DoubleWritable(1), + new NDArrayWritable(arr2), + new IntWritable(2), + new DoubleWritable(3), + new NDArrayWritable(arr3), + new IntWritable(4)); File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + String lineOriginal = "13.0,14.0,15.0,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0"; + try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 3); FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } + String lineNew = FileUtils.readFileToString(tempFile).trim(); assertEquals(lineOriginal, lineNew); } @Test - @DisplayName("Test ND Array Writables Multilabel") - void testNDArrayWritablesMultilabel() throws Exception { + public void testNDArrayWritablesMultilabel() throws Exception { INDArray arr2 = Nd4j.zeros(2); arr2.putScalar(0, 11); arr2.putScalar(1, 12); @@ -195,29 +204,36 @@ class LibSvmRecordWriterTest extends BaseND4JTest { arr3.putScalar(0, 0); arr3.putScalar(1, 1); arr3.putScalar(2, 0); - List record = Arrays.asList((Writable) new DoubleWritable(1), new NDArrayWritable(arr2), new IntWritable(2), new DoubleWritable(3), new NDArrayWritable(arr3), new DoubleWritable(1)); + List record = Arrays.asList((Writable) new DoubleWritable(1), + new NDArrayWritable(arr2), + new IntWritable(2), + new DoubleWritable(3), + new NDArrayWritable(arr3), + new DoubleWritable(1)); File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + String lineOriginal = "2,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0"; + try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 3); FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } + String lineNew = FileUtils.readFileToString(tempFile).trim(); assertEquals(lineOriginal, lineNew); } @Test - @DisplayName("Test ND Array Writables Zero Index") - void testNDArrayWritablesZeroIndex() throws Exception { + public void testNDArrayWritablesZeroIndex() throws Exception { INDArray arr2 = Nd4j.zeros(2); arr2.putScalar(0, 11); arr2.putScalar(1, 12); @@ -225,60 +241,70 @@ class LibSvmRecordWriterTest extends BaseND4JTest { arr3.putScalar(0, 0); arr3.putScalar(1, 1); arr3.putScalar(2, 0); - List record = Arrays.asList((Writable) new DoubleWritable(1), new NDArrayWritable(arr2), new IntWritable(2), new DoubleWritable(3), new NDArrayWritable(arr3), new DoubleWritable(1)); + List record = Arrays.asList((Writable) new DoubleWritable(1), + new NDArrayWritable(arr2), + new IntWritable(2), + new DoubleWritable(3), + new NDArrayWritable(arr3), + new DoubleWritable(1)); File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + String lineOriginal = "1,3 0:1.0 1:11.0 2:12.0 3:2.0 4:3.0"; + try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { Configuration configWriter = new Configuration(); - // NOT STANDARD! - configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_INDEXING, true); - // NOT STANDARD! - configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_LABEL_INDEXING, true); + configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_INDEXING, true); // NOT STANDARD! + configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD! configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 3); FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } + String lineNew = FileUtils.readFileToString(tempFile).trim(); assertEquals(lineOriginal, lineNew); } @Test - @DisplayName("Test Non Integer But Valid Multilabel") - void testNonIntegerButValidMultilabel() throws Exception { - List record = Arrays.asList((Writable) new IntWritable(3), new IntWritable(2), new DoubleWritable(1.0)); + public void testNonIntegerButValidMultilabel() throws Exception { + List record = Arrays.asList((Writable) new IntWritable(3), + new IntWritable(2), + new DoubleWritable(1.0)); File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 1); configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true); FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } } @Test - @DisplayName("Non Integer Multilabel") - void nonIntegerMultilabel() { - assertThrows(NumberFormatException.class, () -> { - List record = Arrays.asList((Writable) new IntWritable(3), new IntWritable(2), new DoubleWritable(1.2)); + public void nonIntegerMultilabel() throws Exception { + Assertions.assertThrows(NumberFormatException.class, () -> { + List record = Arrays.asList((Writable) new IntWritable(3), + new IntWritable(2), + new DoubleWritable(1.2)); File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); @@ -292,24 +318,24 @@ class LibSvmRecordWriterTest extends BaseND4JTest { } @Test - @DisplayName("Non Binary Multilabel") - void nonBinaryMultilabel() { - assertThrows(NumberFormatException.class, () -> { - List record = Arrays.asList((Writable) new IntWritable(0), new IntWritable(1), new IntWritable(2)); - File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); - tempFile.setWritable(true); - tempFile.deleteOnExit(); - if (tempFile.exists()) - tempFile.delete(); - try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { - Configuration configWriter = new Configuration(); - configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); - configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 1); - configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true); - FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); - writer.write(record); - } - }); + public void nonBinaryMultilabel() throws Exception { + List record = Arrays.asList((Writable) new IntWritable(0), + new IntWritable(1), + new IntWritable(2)); + File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); + tempFile.setWritable(true); + tempFile.deleteOnExit(); + if (tempFile.exists()) + tempFile.delete(); + + try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { + Configuration configWriter = new Configuration(); + configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN,0); + configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN,1); + configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL,true); + FileSplit outputSplit = new FileSplit(tempFile); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); + writer.write(record); + } } } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/SVMLightRecordWriterTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/SVMLightRecordWriterTest.java similarity index 77% rename from datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/SVMLightRecordWriterTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/SVMLightRecordWriterTest.java index 5bab04d45..d38611cc4 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/writer/impl/SVMLightRecordWriterTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/records/writer/impl/SVMLightRecordWriterTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.records.writer.impl; import org.apache.commons.io.FileUtils; @@ -26,94 +27,94 @@ import org.datavec.api.records.writer.impl.misc.SVMLightRecordWriter; import org.datavec.api.split.FileSplit; import org.datavec.api.split.partition.NumberOfRecordsPartitioner; import org.datavec.api.writable.*; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.io.ClassPathResource; + import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.jupiter.api.Assertions.assertThrows; -@DisplayName("Svm Light Record Writer Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class SVMLightRecordWriterTest extends BaseND4JTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SVMLightRecordWriterTest extends BaseND4JTest { @Test - @DisplayName("Test Basic") - void testBasic() throws Exception { + public void testBasic() throws Exception { Configuration configWriter = new Configuration(); + Configuration configReader = new Configuration(); configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 10); configReader.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + File inputFile = new ClassPathResource("datavec-api/svmlight/basic.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @Test - @DisplayName("Test No Label") - void testNoLabel() throws Exception { + public void testNoLabel() throws Exception { Configuration configWriter = new Configuration(); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 9); + Configuration configReader = new Configuration(); configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 10); configReader.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + File inputFile = new ClassPathResource("datavec-api/svmlight/noLabels.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @Test - @DisplayName("Test Multioutput Record") - void testMultioutputRecord() throws Exception { + public void testMultioutputRecord() throws Exception { Configuration configWriter = new Configuration(); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 9); + Configuration configReader = new Configuration(); configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 10); configReader.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + File inputFile = new ClassPathResource("datavec-api/svmlight/multioutput.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @Test - @DisplayName("Test Multilabel Record") - void testMultilabelRecord() throws Exception { + public void testMultilabelRecord() throws Exception { Configuration configWriter = new Configuration(); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 9); configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true); + Configuration configReader = new Configuration(); configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 10); configReader.setBoolean(SVMLightRecordReader.MULTILABEL, true); configReader.setInt(SVMLightRecordReader.NUM_LABELS, 4); configReader.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); + File inputFile = new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @Test - @DisplayName("Test Zero Based Indexing") - void testZeroBasedIndexing() throws Exception { + public void testZeroBasedIndexing() throws Exception { Configuration configWriter = new Configuration(); configWriter.setBoolean(SVMLightRecordWriter.ZERO_BASED_INDEXING, true); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 10); configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true); + Configuration configReader = new Configuration(); configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 11); configReader.setBoolean(SVMLightRecordReader.MULTILABEL, true); configReader.setInt(SVMLightRecordReader.NUM_LABELS, 5); + File inputFile = new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile(); executeTest(configWriter, configReader, inputFile); } @@ -124,9 +125,10 @@ class SVMLightRecordWriterTest extends BaseND4JTest { tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); SVMLightRecordReader rr = new SVMLightRecordReader(); rr.initialize(configReader, new FileSplit(inputFile)); while (rr.hasNext()) { @@ -134,6 +136,7 @@ class SVMLightRecordWriterTest extends BaseND4JTest { writer.write(record); } } + Pattern p = Pattern.compile(String.format("%s:\\d+ ", SVMLightRecordReader.QID_PREFIX)); List linesOriginal = new ArrayList<>(); for (String line : FileUtils.readLines(inputFile)) { @@ -154,8 +157,7 @@ class SVMLightRecordWriterTest extends BaseND4JTest { } @Test - @DisplayName("Test ND Array Writables") - void testNDArrayWritables() throws Exception { + public void testNDArrayWritables() throws Exception { INDArray arr2 = Nd4j.zeros(2); arr2.putScalar(0, 11); arr2.putScalar(1, 12); @@ -163,28 +165,35 @@ class SVMLightRecordWriterTest extends BaseND4JTest { arr3.putScalar(0, 13); arr3.putScalar(1, 14); arr3.putScalar(2, 15); - List record = Arrays.asList((Writable) new DoubleWritable(1), new NDArrayWritable(arr2), new IntWritable(2), new DoubleWritable(3), new NDArrayWritable(arr3), new IntWritable(4)); + List record = Arrays.asList((Writable) new DoubleWritable(1), + new NDArrayWritable(arr2), + new IntWritable(2), + new DoubleWritable(3), + new NDArrayWritable(arr3), + new IntWritable(4)); File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + String lineOriginal = "13.0,14.0,15.0,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0"; + try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 3); FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } + String lineNew = FileUtils.readFileToString(tempFile).trim(); assertEquals(lineOriginal, lineNew); } @Test - @DisplayName("Test ND Array Writables Multilabel") - void testNDArrayWritablesMultilabel() throws Exception { + public void testNDArrayWritablesMultilabel() throws Exception { INDArray arr2 = Nd4j.zeros(2); arr2.putScalar(0, 11); arr2.putScalar(1, 12); @@ -192,29 +201,36 @@ class SVMLightRecordWriterTest extends BaseND4JTest { arr3.putScalar(0, 0); arr3.putScalar(1, 1); arr3.putScalar(2, 0); - List record = Arrays.asList((Writable) new DoubleWritable(1), new NDArrayWritable(arr2), new IntWritable(2), new DoubleWritable(3), new NDArrayWritable(arr3), new DoubleWritable(1)); + List record = Arrays.asList((Writable) new DoubleWritable(1), + new NDArrayWritable(arr2), + new IntWritable(2), + new DoubleWritable(3), + new NDArrayWritable(arr3), + new DoubleWritable(1)); File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + String lineOriginal = "2,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0"; + try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 3); FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } + String lineNew = FileUtils.readFileToString(tempFile).trim(); assertEquals(lineOriginal, lineNew); } @Test - @DisplayName("Test ND Array Writables Zero Index") - void testNDArrayWritablesZeroIndex() throws Exception { + public void testNDArrayWritablesZeroIndex() throws Exception { INDArray arr2 = Nd4j.zeros(2); arr2.putScalar(0, 11); arr2.putScalar(1, 12); @@ -222,60 +238,70 @@ class SVMLightRecordWriterTest extends BaseND4JTest { arr3.putScalar(0, 0); arr3.putScalar(1, 1); arr3.putScalar(2, 0); - List record = Arrays.asList((Writable) new DoubleWritable(1), new NDArrayWritable(arr2), new IntWritable(2), new DoubleWritable(3), new NDArrayWritable(arr3), new DoubleWritable(1)); + List record = Arrays.asList((Writable) new DoubleWritable(1), + new NDArrayWritable(arr2), + new IntWritable(2), + new DoubleWritable(3), + new NDArrayWritable(arr3), + new DoubleWritable(1)); File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + String lineOriginal = "1,3 0:1.0 1:11.0 2:12.0 3:2.0 4:3.0"; + try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { Configuration configWriter = new Configuration(); - // NOT STANDARD! - configWriter.setBoolean(SVMLightRecordWriter.ZERO_BASED_INDEXING, true); - // NOT STANDARD! - configWriter.setBoolean(SVMLightRecordWriter.ZERO_BASED_LABEL_INDEXING, true); + configWriter.setBoolean(SVMLightRecordWriter.ZERO_BASED_INDEXING, true); // NOT STANDARD! + configWriter.setBoolean(SVMLightRecordWriter.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD! configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 3); FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } + String lineNew = FileUtils.readFileToString(tempFile).trim(); assertEquals(lineOriginal, lineNew); } @Test - @DisplayName("Test Non Integer But Valid Multilabel") - void testNonIntegerButValidMultilabel() throws Exception { - List record = Arrays.asList((Writable) new IntWritable(3), new IntWritable(2), new DoubleWritable(1.0)); + public void testNonIntegerButValidMultilabel() throws Exception { + List record = Arrays.asList((Writable) new IntWritable(3), + new IntWritable(2), + new DoubleWritable(1.0)); File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 1); configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true); FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } } @Test - @DisplayName("Non Integer Multilabel") - void nonIntegerMultilabel() { - assertThrows(NumberFormatException.class, () -> { - List record = Arrays.asList((Writable) new IntWritable(3), new IntWritable(2), new DoubleWritable(1.2)); + public void nonIntegerMultilabel() throws Exception { + Assertions.assertThrows(NumberFormatException.class, () -> { + List record = Arrays.asList((Writable) new IntWritable(3), + new IntWritable(2), + new DoubleWritable(1.2)); File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); + try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); @@ -289,24 +315,27 @@ class SVMLightRecordWriterTest extends BaseND4JTest { } @Test - @DisplayName("Non Binary Multilabel") - void nonBinaryMultilabel() { - assertThrows(NumberFormatException.class, () -> { - List record = Arrays.asList((Writable) new IntWritable(0), new IntWritable(1), new IntWritable(2)); - File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); - tempFile.setWritable(true); - tempFile.deleteOnExit(); - if (tempFile.exists()) - tempFile.delete(); - try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { - Configuration configWriter = new Configuration(); - configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); - configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 1); - configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true); - FileSplit outputSplit = new FileSplit(tempFile); - writer.initialize(configWriter, outputSplit, new NumberOfRecordsPartitioner()); - writer.write(record); - } - }); + public void nonBinaryMultilabel() throws Exception { + Assertions.assertThrows(NumberFormatException.class, () -> { + List record = Arrays.asList((Writable) new IntWritable(0), + new IntWritable(1), + new IntWritable(2)); + + File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); + tempFile.setWritable(true); + tempFile.deleteOnExit(); + if (tempFile.exists()) + tempFile.delete(); + + try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { + Configuration configWriter = new Configuration(); + configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); + configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 1); + configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true); + FileSplit outputSplit = new FileSplit(tempFile); + writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); + writer.write(record); + } + }); } } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java similarity index 93% rename from datavec/datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java index 0841dfb89..f7c413d34 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/InputSplitTests.java @@ -20,10 +20,9 @@ package org.datavec.api.split; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.guava.io.Files; +import com.google.common.io.Files; import org.datavec.api.io.filters.BalancedPathFilter; import org.datavec.api.io.filters.RandomPathFilter; import org.datavec.api.io.labels.ParentPathLabelGenerator; @@ -36,16 +35,12 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Random; - import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * * @author saudet */ -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) public class InputSplitTests extends BaseND4JTest { @Test @@ -135,9 +130,9 @@ public class InputSplitTests extends BaseND4JTest { public void testFileSplitBootstrap() { File tmpDir = Files.createTempDir(); FileSplit boostrap = new FileSplit(tmpDir); - assertTrue(boostrap.needsBootstrapForWrite()); + Assertions.assertTrue(boostrap.needsBootstrapForWrite()); boostrap.bootStrapForWrite(); - assertTrue(tmpDir.listFiles() != null); + Assertions.assertTrue(tmpDir.listFiles() != null); } @Test @@ -158,7 +153,7 @@ public class InputSplitTests extends BaseND4JTest { notOnlyFirstLabel = true; } } - assertTrue(notOnlyFirstLabel); + Assertions.assertTrue(notOnlyFirstLabel); } } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/split/NumberedFileInputSplitTests.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/NumberedFileInputSplitTests.java similarity index 75% rename from datavec/datavec-api/src/test/java/org/datavec/api/split/NumberedFileInputSplitTests.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/NumberedFileInputSplitTests.java index 2677c58cc..ac612a979 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/split/NumberedFileInputSplitTests.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/NumberedFileInputSplitTests.java @@ -20,16 +20,15 @@ package org.datavec.api.split; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.net.URI; -import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class NumberedFileInputSplitTests extends BaseND4JTest { @Test public void testNumberedFileInputSplitBasic() { @@ -71,81 +70,74 @@ public class NumberedFileInputSplitTests extends BaseND4JTest { runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); } - @Test() + @Test public void testNumberedFileInputSplitWithLeadingSpaces() { - assertThrows(IllegalArgumentException.class,() -> { + Assertions.assertThrows(IllegalArgumentException.class, () -> { String baseString = "/path/to/files/prefix-%5d.suffix"; int minIdx = 0; int maxIdx = 10; runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); }); - } - @Test() + @Test public void testNumberedFileInputSplitWithNoLeadingZeroInPadding() { - assertThrows(IllegalArgumentException.class, () -> { + Assertions.assertThrows(IllegalArgumentException.class, () -> { String baseString = "/path/to/files/prefix%5d.suffix"; int minIdx = 0; int maxIdx = 10; runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); }); - } - @Test() + @Test public void testNumberedFileInputSplitWithLeadingPlusInPadding() { - assertThrows(IllegalArgumentException.class,() -> { + Assertions.assertThrows(IllegalArgumentException.class, () -> { String baseString = "/path/to/files/prefix%+5d.suffix"; int minIdx = 0; int maxIdx = 10; runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); }); - } - @Test() + @Test public void testNumberedFileInputSplitWithLeadingMinusInPadding() { - assertThrows(IllegalArgumentException.class,() -> { - String baseString = "/path/to/files/prefix%-5d.suffix"; - int minIdx = 0; - int maxIdx = 10; + String baseString = "/path/to/files/prefix%-5d.suffix"; + int minIdx = 0; + int maxIdx = 10; + Assertions.assertThrows(IllegalArgumentException.class, () -> { runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); }); - } - @Test() + @Test public void testNumberedFileInputSplitWithTwoDigitsInPadding() { - assertThrows(IllegalArgumentException.class,() -> { - String baseString = "/path/to/files/prefix%011d.suffix"; - int minIdx = 0; - int maxIdx = 10; - runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); - }); - + String baseString = "/path/to/files/prefix%011d.suffix"; + int minIdx = 0; + int maxIdx = 10; + Assertions.assertThrows(IllegalArgumentException.class, ()-> { + runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); + }); } - @Test() + @Test public void testNumberedFileInputSplitWithInnerZerosInPadding() { - assertThrows(IllegalArgumentException.class,() -> { - String baseString = "/path/to/files/prefix%101d.suffix"; - int minIdx = 0; - int maxIdx = 10; + String baseString = "/path/to/files/prefix%101d.suffix"; + int minIdx = 0; + int maxIdx = 10; + Assertions.assertThrows(IllegalArgumentException.class, () -> { runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); }); - } - @Test() + @Test public void testNumberedFileInputSplitWithRepeatInnerZerosInPadding() { - assertThrows(IllegalArgumentException.class,() -> { - String baseString = "/path/to/files/prefix%0505d.suffix"; - int minIdx = 0; - int maxIdx = 10; + String baseString = "/path/to/files/prefix%0505d.suffix"; + int minIdx = 0; + int maxIdx = 10; + Assertions.assertThrows(IllegalArgumentException.class, () -> { runNumberedFileInputSplitTest(baseString, minIdx, maxIdx); }); - } @@ -158,7 +150,7 @@ public class NumberedFileInputSplitTests extends BaseND4JTest { String path = locs[j++].getPath(); String exp = String.format(baseString, i); String msg = exp + " vs " + path; - assertTrue(path.endsWith(exp),msg); //Note: on Windows, Java can prepend drive to path - "/C:/" + assertTrue(path.endsWith(exp), msg); //Note: on Windows, Java can prepend drive to path - "/C:/" } } } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/split/TestStreamInputSplit.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/TestStreamInputSplit.java similarity index 93% rename from datavec/datavec-api/src/test/java/org/datavec/api/split/TestStreamInputSplit.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/TestStreamInputSplit.java index 5c845d155..09b01cf8d 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/split/TestStreamInputSplit.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/TestStreamInputSplit.java @@ -25,14 +25,10 @@ import org.datavec.api.records.reader.impl.csv.CSVRecordReader; import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.common.function.Function; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import java.io.FileInputStream; @@ -40,7 +36,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -48,15 +43,15 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestStreamInputSplit extends BaseND4JTest { - + @TempDir + public File testDir; @Test - public void testCsvSimple(@TempDir Path testDir) throws Exception { - File dir = testDir.toFile(); + public void testCsvSimple() throws Exception { + File dir = testDir; File f1 = new File(dir, "file1.txt"); File f2 = new File(dir, "file2.txt"); @@ -97,9 +92,9 @@ public class TestStreamInputSplit extends BaseND4JTest { @Test - public void testCsvSequenceSimple(@TempDir Path testDir) throws Exception { + public void testCsvSequenceSimple() throws Exception { - File dir = testDir.toFile(); + File dir = testDir; File f1 = new File(dir, "file1.txt"); File f2 = new File(dir, "file2.txt"); @@ -141,8 +136,8 @@ public class TestStreamInputSplit extends BaseND4JTest { } @Test - public void testShuffle(@TempDir Path testDir) throws Exception { - File dir = testDir.toFile(); + public void testShuffle() throws Exception { + File dir = testDir; File f1 = new File(dir, "file1.txt"); File f2 = new File(dir, "file2.txt"); File f3 = new File(dir, "file3.txt"); diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/TransformSplitTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/TransformSplitTest.java new file mode 100644 index 000000000..49c1f7d62 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/TransformSplitTest.java @@ -0,0 +1,60 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.split; + +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collection; + +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +/** + * @author Ede Meijer + */ +public class TransformSplitTest extends BaseND4JTest { + @Test + public void testTransform() throws URISyntaxException { + Collection inputFiles = asList(new URI("file:///foo/bar/../0.csv"), new URI("file:///foo/1.csv")); + + InputSplit SUT = new TransformSplit(new CollectionInputSplit(inputFiles), new TransformSplit.URITransform() { + @Override + public URI apply(URI uri) throws URISyntaxException { + return uri.normalize(); + } + }); + + assertArrayEquals(new URI[] {new URI("file:///foo/0.csv"), new URI("file:///foo/1.csv")}, SUT.locations()); + } + + @Test + public void testSearchReplace() throws URISyntaxException { + Collection inputFiles = asList(new URI("file:///foo/1-in.csv"), new URI("file:///foo/2-in.csv")); + + InputSplit SUT = TransformSplit.ofSearchReplace(new CollectionInputSplit(inputFiles), "-in.csv", "-out.csv"); + + assertArrayEquals(new URI[] {new URI("file:///foo/1-out.csv"), new URI("file:///foo/2-out.csv")}, + SUT.locations()); + } +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java similarity index 92% rename from datavec/datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java index d721d5a62..f8e2c99f1 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/split/parittion/PartitionerTests.java @@ -20,10 +20,8 @@ package org.datavec.api.split.parittion; -import org.junit.jupiter.api.Tag; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.guava.io.Files; +import com.google.common.io.Files; import org.datavec.api.conf.Configuration; import org.datavec.api.split.FileSplit; import org.datavec.api.split.partition.NumberOfRecordsPartitioner; @@ -34,9 +32,10 @@ import org.junit.jupiter.api.Test; import java.io.File; import java.io.OutputStream; -import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class PartitionerTests extends BaseND4JTest { @Test public void testRecordsPerFilePartition() { diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/TestTransformProcess.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/TestTransformProcess.java similarity index 97% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/TestTransformProcess.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/TestTransformProcess.java index b9d0e4c1e..7a968ddfe 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/TestTransformProcess.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/TestTransformProcess.java @@ -29,16 +29,13 @@ import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestTransformProcess extends BaseND4JTest { @Test diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/condition/TestConditions.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/condition/TestConditions.java similarity index 99% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/condition/TestConditions.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/condition/TestConditions.java index fd2eb7862..f49e0c4d4 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/condition/TestConditions.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/condition/TestConditions.java @@ -27,17 +27,14 @@ import org.datavec.api.transform.condition.string.StringRegexColumnCondition; import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.transform.TestTransforms; import org.datavec.api.writable.*; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.*; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestConditions extends BaseND4JTest { @Test diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/filter/TestFilters.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/filter/TestFilters.java similarity index 97% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/filter/TestFilters.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/filter/TestFilters.java index 65b4ee3e3..0b339bffa 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/filter/TestFilters.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/filter/TestFilters.java @@ -27,10 +27,8 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.ArrayList; import java.util.Arrays; @@ -40,8 +38,7 @@ import java.util.List; import static java.util.Arrays.asList; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestFilters extends BaseND4JTest { diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/join/TestJoin.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/join/TestJoin.java new file mode 100644 index 000000000..c41ebb165 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/join/TestJoin.java @@ -0,0 +1,123 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.join; + +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.NullWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestJoin extends BaseND4JTest { + + @Test + public void testJoin() { + + Schema firstSchema = + new Schema.Builder().addColumnString("keyColumn").addColumnsInteger("first0", "first1").build(); + + Schema secondSchema = new Schema.Builder().addColumnString("keyColumn").addColumnsInteger("second0").build(); + + List> first = new ArrayList<>(); + first.add(Arrays.asList((Writable) new Text("key0"), new IntWritable(0), new IntWritable(1))); + first.add(Arrays.asList((Writable) new Text("key1"), new IntWritable(10), new IntWritable(11))); + + List> second = new ArrayList<>(); + second.add(Arrays.asList((Writable) new Text("key0"), new IntWritable(100))); + second.add(Arrays.asList((Writable) new Text("key1"), new IntWritable(110))); + + Join join = new Join.Builder(Join.JoinType.Inner).setJoinColumns("keyColumn") + .setSchemas(firstSchema, secondSchema).build(); + + List> expected = new ArrayList<>(); + expected.add(Arrays.asList((Writable) new Text("key0"), new IntWritable(0), new IntWritable(1), + new IntWritable(100))); + expected.add(Arrays.asList((Writable) new Text("key1"), new IntWritable(10), new IntWritable(11), + new IntWritable(110))); + + + //Check schema: + Schema joinedSchema = join.getOutputSchema(); + assertEquals(4, joinedSchema.numColumns()); + assertEquals(Arrays.asList("keyColumn", "first0", "first1", "second0"), joinedSchema.getColumnNames()); + assertEquals(Arrays.asList(ColumnType.String, ColumnType.Integer, ColumnType.Integer, ColumnType.Integer), + joinedSchema.getColumnTypes()); + + + //Check joining with null values: + expected = new ArrayList<>(); + expected.add(Arrays.asList((Writable) new Text("key0"), new IntWritable(0), new IntWritable(1), + NullWritable.INSTANCE)); + expected.add(Arrays.asList((Writable) new Text("key1"), new IntWritable(10), new IntWritable(11), + NullWritable.INSTANCE)); + for (int i = 0; i < first.size(); i++) { + List out = join.joinExamples(first.get(i), null); + assertEquals(expected.get(i), out); + } + + expected = new ArrayList<>(); + expected.add(Arrays.asList((Writable) new Text("key0"), NullWritable.INSTANCE, NullWritable.INSTANCE, + new IntWritable(100))); + expected.add(Arrays.asList((Writable) new Text("key1"), NullWritable.INSTANCE, NullWritable.INSTANCE, + new IntWritable(110))); + for (int i = 0; i < first.size(); i++) { + List out = join.joinExamples(null, second.get(i)); + assertEquals(expected.get(i), out); + } + } + + + @Test + public void testJoinValidation() { + Assertions.assertThrows(IllegalArgumentException.class, () -> { + Schema firstSchema = new Schema.Builder().addColumnString("keyColumn1").addColumnsInteger("first0", "first1") + .build(); + + Schema secondSchema = new Schema.Builder().addColumnString("keyColumn2").addColumnsInteger("second0").build(); + + new Join.Builder(Join.JoinType.Inner).setJoinColumns("keyColumn1", "thisDoesntExist") + .setSchemas(firstSchema, secondSchema).build(); + }); + } + + @Test + public void testJoinValidation2() { + Assertions.assertThrows(IllegalArgumentException.class, () -> { + Schema firstSchema = new Schema.Builder().addColumnString("keyColumn1").addColumnsInteger("first0", "first1") + .build(); + + Schema secondSchema = new Schema.Builder().addColumnString("keyColumn2").addColumnsInteger("second0").build(); + + new Join.Builder(Join.JoinType.Inner).setJoinColumns("keyColumn1").setSchemas(firstSchema, secondSchema) + .build(); + }); + } +} diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpArchTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpArchTest.java new file mode 100644 index 000000000..242138c42 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpArchTest.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.ops; + +import com.tngtech.archunit.core.importer.ImportOption; +import com.tngtech.archunit.junit.AnalyzeClasses; +import com.tngtech.archunit.junit.ArchTest; +import com.tngtech.archunit.lang.ArchRule; +import org.nd4j.common.tests.BaseND4JTest; + +import java.io.Serializable; + +import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes; + +@AnalyzeClasses(packages = "org.datavec.api.transform.ops", importOptions = {ImportOption.DoNotIncludeTests.class}) +public class AggregableMultiOpArchTest extends BaseND4JTest { + + @ArchTest + public static final ArchRule ALL_AGGREGATE_OPS_MUST_BE_SERIALIZABLE = classes() + .that().resideInAPackage("org.datavec.api.transform.ops") + .and().doNotHaveSimpleName("AggregatorImpls") + .and().doNotHaveSimpleName("IAggregableReduceOp") + .and().doNotHaveSimpleName("StringAggregatorImpls") + .and().doNotHaveFullyQualifiedName("org.datavec.api.transform.ops.StringAggregatorImpls$1") + .should().implement(Serializable.class) + .because("All aggregate ops must be serializable."); +} \ No newline at end of file diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpTest.java similarity index 91% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpTest.java index acd2971ac..caadceb15 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpTest.java @@ -17,46 +17,52 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.transform.ops; import org.datavec.api.writable.Writable; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import java.util.*; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Aggregable Multi Op Test") -class AggregableMultiOpTest extends BaseND4JTest { +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AggregableMultiOpTest extends BaseND4JTest { private List intList = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); @Test - @DisplayName("Test Multi") - void testMulti() throws Exception { + public void testMulti() throws Exception { AggregatorImpls.AggregableFirst af = new AggregatorImpls.AggregableFirst<>(); AggregatorImpls.AggregableSum as = new AggregatorImpls.AggregableSum<>(); AggregableMultiOp multi = new AggregableMultiOp<>(Arrays.asList(af, as)); + assertTrue(multi.getOperations().size() == 2); for (int i = 0; i < intList.size(); i++) { multi.accept(intList.get(i)); } + // mutablility assertTrue(as.get().toDouble() == 45D); assertTrue(af.get().toInt() == 1); + List res = multi.get(); assertTrue(res.get(1).toDouble() == 45D); assertTrue(res.get(0).toInt() == 1); + AggregatorImpls.AggregableFirst rf = new AggregatorImpls.AggregableFirst<>(); AggregatorImpls.AggregableSum rs = new AggregatorImpls.AggregableSum<>(); AggregableMultiOp reverse = new AggregableMultiOp<>(Arrays.asList(rf, rs)); + for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); } + List revRes = reverse.get(); assertTrue(revRes.get(1).toDouble() == 45D); assertTrue(revRes.get(0).toInt() == 9); + multi.combine(reverse); List combinedRes = multi.get(); assertTrue(combinedRes.get(1).toDouble() == 90D); diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregatorImplsTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregatorImplsTest.java similarity index 80% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregatorImplsTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregatorImplsTest.java index fa1d82279..8cfd5e979 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregatorImplsTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/AggregatorImplsTest.java @@ -17,39 +17,40 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.transform.ops; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.junit.jupiter.api.DisplayName; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.*; - -@DisplayName("Aggregator Impls Test") -class AggregatorImplsTest extends BaseND4JTest { +public class AggregatorImplsTest extends BaseND4JTest { private List intList = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); - private List stringList = new ArrayList<>(Arrays.asList("arakoa", "abracadabra", "blast", "acceptance")); @Test - @DisplayName("Aggregable First Test") - void aggregableFirstTest() { + public void aggregableFirstTest() { AggregatorImpls.AggregableFirst first = new AggregatorImpls.AggregableFirst<>(); for (int i = 0; i < intList.size(); i++) { first.accept(intList.get(i)); } assertEquals(1, first.get().toInt()); + AggregatorImpls.AggregableFirst firstS = new AggregatorImpls.AggregableFirst<>(); for (int i = 0; i < stringList.size(); i++) { firstS.accept(stringList.get(i)); } assertTrue(firstS.get().toString().equals("arakoa")); + + AggregatorImpls.AggregableFirst reverse = new AggregatorImpls.AggregableFirst<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); @@ -58,19 +59,22 @@ class AggregatorImplsTest extends BaseND4JTest { assertEquals(1, first.get().toInt()); } + @Test - @DisplayName("Aggregable Last Test") - void aggregableLastTest() { + public void aggregableLastTest() { AggregatorImpls.AggregableLast last = new AggregatorImpls.AggregableLast<>(); for (int i = 0; i < intList.size(); i++) { last.accept(intList.get(i)); } assertEquals(9, last.get().toInt()); + AggregatorImpls.AggregableLast lastS = new AggregatorImpls.AggregableLast<>(); for (int i = 0; i < stringList.size(); i++) { lastS.accept(stringList.get(i)); } assertTrue(lastS.get().toString().equals("acceptance")); + + AggregatorImpls.AggregableLast reverse = new AggregatorImpls.AggregableLast<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); @@ -80,18 +84,20 @@ class AggregatorImplsTest extends BaseND4JTest { } @Test - @DisplayName("Aggregable Count Test") - void aggregableCountTest() { + public void aggregableCountTest() { AggregatorImpls.AggregableCount cnt = new AggregatorImpls.AggregableCount<>(); for (int i = 0; i < intList.size(); i++) { cnt.accept(intList.get(i)); } assertEquals(9, cnt.get().toInt()); + AggregatorImpls.AggregableCount lastS = new AggregatorImpls.AggregableCount<>(); for (int i = 0; i < stringList.size(); i++) { lastS.accept(stringList.get(i)); } assertEquals(4, lastS.get().toInt()); + + AggregatorImpls.AggregableCount reverse = new AggregatorImpls.AggregableCount<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); @@ -101,13 +107,14 @@ class AggregatorImplsTest extends BaseND4JTest { } @Test - @DisplayName("Aggregable Max Test") - void aggregableMaxTest() { + public void aggregableMaxTest() { AggregatorImpls.AggregableMax mx = new AggregatorImpls.AggregableMax<>(); for (int i = 0; i < intList.size(); i++) { mx.accept(intList.get(i)); } assertEquals(9, mx.get().toInt()); + + AggregatorImpls.AggregableMax reverse = new AggregatorImpls.AggregableMax<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); @@ -116,14 +123,16 @@ class AggregatorImplsTest extends BaseND4JTest { assertEquals(9, mx.get().toInt()); } + @Test - @DisplayName("Aggregable Range Test") - void aggregableRangeTest() { + public void aggregableRangeTest() { AggregatorImpls.AggregableRange mx = new AggregatorImpls.AggregableRange<>(); for (int i = 0; i < intList.size(); i++) { mx.accept(intList.get(i)); } assertEquals(8, mx.get().toInt()); + + AggregatorImpls.AggregableRange reverse = new AggregatorImpls.AggregableRange<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1) + 9); @@ -133,13 +142,14 @@ class AggregatorImplsTest extends BaseND4JTest { } @Test - @DisplayName("Aggregable Min Test") - void aggregableMinTest() { + public void aggregableMinTest() { AggregatorImpls.AggregableMin mn = new AggregatorImpls.AggregableMin<>(); for (int i = 0; i < intList.size(); i++) { mn.accept(intList.get(i)); } assertEquals(1, mn.get().toInt()); + + AggregatorImpls.AggregableMin reverse = new AggregatorImpls.AggregableMin<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); @@ -149,13 +159,14 @@ class AggregatorImplsTest extends BaseND4JTest { } @Test - @DisplayName("Aggregable Sum Test") - void aggregableSumTest() { + public void aggregableSumTest() { AggregatorImpls.AggregableSum sm = new AggregatorImpls.AggregableSum<>(); for (int i = 0; i < intList.size(); i++) { sm.accept(intList.get(i)); } assertEquals(45, sm.get().toInt()); + + AggregatorImpls.AggregableSum reverse = new AggregatorImpls.AggregableSum<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); @@ -164,15 +175,17 @@ class AggregatorImplsTest extends BaseND4JTest { assertEquals(90, sm.get().toInt()); } + @Test - @DisplayName("Aggregable Mean Test") - void aggregableMeanTest() { + public void aggregableMeanTest() { AggregatorImpls.AggregableMean mn = new AggregatorImpls.AggregableMean<>(); for (int i = 0; i < intList.size(); i++) { mn.accept(intList.get(i)); } assertEquals(9l, (long) mn.getCount()); assertEquals(5D, mn.get().toDouble(), 0.001); + + AggregatorImpls.AggregableMean reverse = new AggregatorImpls.AggregableMean<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); @@ -183,73 +196,80 @@ class AggregatorImplsTest extends BaseND4JTest { } @Test - @DisplayName("Aggregable Std Dev Test") - void aggregableStdDevTest() { + public void aggregableStdDevTest() { AggregatorImpls.AggregableStdDev sd = new AggregatorImpls.AggregableStdDev<>(); for (int i = 0; i < intList.size(); i++) { sd.accept(intList.get(i)); } assertTrue(Math.abs(sd.get().toDouble() - 2.7386) < 0.0001); + + AggregatorImpls.AggregableStdDev reverse = new AggregatorImpls.AggregableStdDev<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); } sd.combine(reverse); - assertTrue(Math.abs(sd.get().toDouble() - 1.8787) < 0.0001,"" + sd.get().toDouble()); + assertTrue(Math.abs(sd.get().toDouble() - 1.8787) < 0.0001, "" + sd.get().toDouble()); } @Test - @DisplayName("Aggregable Variance") - void aggregableVariance() { + public void aggregableVariance() { AggregatorImpls.AggregableVariance sd = new AggregatorImpls.AggregableVariance<>(); for (int i = 0; i < intList.size(); i++) { sd.accept(intList.get(i)); } assertTrue(Math.abs(sd.get().toDouble() - 60D / 8) < 0.0001); + + AggregatorImpls.AggregableVariance reverse = new AggregatorImpls.AggregableVariance<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); } sd.combine(reverse); - assertTrue(Math.abs(sd.get().toDouble() - 3.5294) < 0.0001,"" + sd.get().toDouble()); + assertTrue( Math.abs(sd.get().toDouble() - 3.5294) < 0.0001, "" + sd.get().toDouble()); } @Test - @DisplayName("Aggregable Uncorrected Std Dev Test") - void aggregableUncorrectedStdDevTest() { + public void aggregableUncorrectedStdDevTest() { AggregatorImpls.AggregableUncorrectedStdDev sd = new AggregatorImpls.AggregableUncorrectedStdDev<>(); for (int i = 0; i < intList.size(); i++) { sd.accept(intList.get(i)); } assertTrue(Math.abs(sd.get().toDouble() - 2.582) < 0.0001); - AggregatorImpls.AggregableUncorrectedStdDev reverse = new AggregatorImpls.AggregableUncorrectedStdDev<>(); + + + AggregatorImpls.AggregableUncorrectedStdDev reverse = + new AggregatorImpls.AggregableUncorrectedStdDev<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); } sd.combine(reverse); - assertTrue(Math.abs(sd.get().toDouble() - 1.8257) < 0.0001,"" + sd.get().toDouble()); + assertTrue( Math.abs(sd.get().toDouble() - 1.8257) < 0.0001, "" + sd.get().toDouble()); } + @Test - @DisplayName("Aggregable Population Variance") - void aggregablePopulationVariance() { + public void aggregablePopulationVariance() { AggregatorImpls.AggregablePopulationVariance sd = new AggregatorImpls.AggregablePopulationVariance<>(); for (int i = 0; i < intList.size(); i++) { sd.accept(intList.get(i)); } assertTrue(Math.abs(sd.get().toDouble() - 60D / 9) < 0.0001); - AggregatorImpls.AggregablePopulationVariance reverse = new AggregatorImpls.AggregablePopulationVariance<>(); + + + AggregatorImpls.AggregablePopulationVariance reverse = + new AggregatorImpls.AggregablePopulationVariance<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); } sd.combine(reverse); - assertTrue(Math.abs(sd.get().toDouble() - 30D / 9) < 0.0001,"" + sd.get().toDouble()); + assertTrue( Math.abs(sd.get().toDouble() - 30D / 9) < 0.0001, "" + sd.get().toDouble()); } @Test - @DisplayName("Aggregable Count Unique Test") - void aggregableCountUniqueTest() { + public void aggregableCountUniqueTest() { // at this low range, it's linear counting + AggregatorImpls.AggregableCountUnique cu = new AggregatorImpls.AggregableCountUnique<>(); for (int i = 0; i < intList.size(); i++) { cu.accept(intList.get(i)); @@ -257,6 +277,7 @@ class AggregatorImplsTest extends BaseND4JTest { assertEquals(9, cu.get().toInt()); cu.accept(1); assertEquals(9, cu.get().toInt()); + AggregatorImpls.AggregableCountUnique reverse = new AggregatorImpls.AggregableCountUnique<>(); for (int i = 0; i < intList.size(); i++) { reverse.accept(intList.get(intList.size() - i - 1)); @@ -268,22 +289,20 @@ class AggregatorImplsTest extends BaseND4JTest { @Test - @DisplayName("Incompatible Aggregator Test") - void incompatibleAggregatorTest() { - assertThrows(UnsupportedOperationException.class,() -> { - AggregatorImpls.AggregableSum sm = new AggregatorImpls.AggregableSum<>(); - for (int i = 0; i < intList.size(); i++) { - sm.accept(intList.get(i)); - } - assertEquals(45, sm.get().toInt()); - AggregatorImpls.AggregableMean reverse = new AggregatorImpls.AggregableMean<>(); - for (int i = 0; i < intList.size(); i++) { - reverse.accept(intList.get(intList.size() - i - 1)); - } + public void incompatibleAggregatorTest() { + AggregatorImpls.AggregableSum sm = new AggregatorImpls.AggregableSum<>(); + for (int i = 0; i < intList.size(); i++) { + sm.accept(intList.get(i)); + } + assertEquals(45, sm.get().toInt()); - sm.combine(reverse); - assertEquals(45, sm.get().toInt()); - }); + AggregatorImpls.AggregableMean reverse = new AggregatorImpls.AggregableMean<>(); + for (int i = 0; i < intList.size(); i++) { + reverse.accept(intList.get(intList.size() - i - 1)); + } + sm.combine(reverse); + assertEquals(45, sm.get().toInt()); } + } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/DispatchOpTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/DispatchOpTest.java similarity index 79% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/DispatchOpTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/DispatchOpTest.java index 6a444923d..a04d6f57a 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/DispatchOpTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ops/DispatchOpTest.java @@ -17,65 +17,77 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.transform.ops; import org.datavec.api.writable.Writable; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Dispatch Op Test") -class DispatchOpTest extends BaseND4JTest { +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DispatchOpTest extends BaseND4JTest { private List intList = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); - private List stringList = new ArrayList<>(Arrays.asList("arakoa", "abracadabra", "blast", "acceptance")); @Test - @DisplayName("Test Dispatch Simple") - void testDispatchSimple() { + public void testDispatchSimple() { AggregatorImpls.AggregableFirst af = new AggregatorImpls.AggregableFirst<>(); AggregatorImpls.AggregableSum as = new AggregatorImpls.AggregableSum<>(); - AggregableMultiOp multiaf = new AggregableMultiOp<>(Collections.>singletonList(af)); - AggregableMultiOp multias = new AggregableMultiOp<>(Collections.>singletonList(as)); - DispatchOp parallel = new DispatchOp<>(Arrays.>>asList(multiaf, multias)); + AggregableMultiOp multiaf = + new AggregableMultiOp<>(Collections.>singletonList(af)); + AggregableMultiOp multias = + new AggregableMultiOp<>(Collections.>singletonList(as)); + + DispatchOp parallel = + new DispatchOp<>(Arrays.>>asList(multiaf, multias)); + assertTrue(multiaf.getOperations().size() == 1); assertTrue(multias.getOperations().size() == 1); assertTrue(parallel.getOperations().size() == 2); for (int i = 0; i < intList.size(); i++) { parallel.accept(Arrays.asList(intList.get(i), intList.get(i))); } + List res = parallel.get(); assertTrue(res.get(1).toDouble() == 45D); assertTrue(res.get(0).toInt() == 1); + } @Test - @DisplayName("Test Dispatch Flat Map") - void testDispatchFlatMap() { + public void testDispatchFlatMap() { AggregatorImpls.AggregableFirst af = new AggregatorImpls.AggregableFirst<>(); AggregatorImpls.AggregableSum as = new AggregatorImpls.AggregableSum<>(); AggregableMultiOp multi = new AggregableMultiOp<>(Arrays.asList(af, as)); + AggregatorImpls.AggregableLast al = new AggregatorImpls.AggregableLast<>(); AggregatorImpls.AggregableMax amax = new AggregatorImpls.AggregableMax<>(); AggregableMultiOp otherMulti = new AggregableMultiOp<>(Arrays.asList(al, amax)); - DispatchOp parallel = new DispatchOp<>(Arrays.>>asList(multi, otherMulti)); + + + DispatchOp parallel = new DispatchOp<>( + Arrays.>>asList(multi, otherMulti)); + assertTrue(multi.getOperations().size() == 2); assertTrue(otherMulti.getOperations().size() == 2); assertTrue(parallel.getOperations().size() == 2); for (int i = 0; i < intList.size(); i++) { parallel.accept(Arrays.asList(intList.get(i), intList.get(i))); } + List res = parallel.get(); assertTrue(res.get(1).toDouble() == 45D); assertTrue(res.get(0).toInt() == 1); assertTrue(res.get(3).toDouble() == 9); assertTrue(res.get(2).toInt() == 9); + } + } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/reduce/TestMultiOpReduce.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/reduce/TestMultiOpReduce.java similarity index 82% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/reduce/TestMultiOpReduce.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/reduce/TestMultiOpReduce.java index 1361f4ff0..80d7d7eee 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/reduce/TestMultiOpReduce.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/reduce/TestMultiOpReduce.java @@ -32,28 +32,24 @@ import org.datavec.api.transform.ops.AggregableMultiOp; import org.datavec.api.transform.ops.IAggregableReduceOp; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.*; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestMultiOpReduce extends BaseND4JTest { @Test public void testMultiOpReducerDouble() { List> inputs = new ArrayList<>(); - inputs.add(Arrays.asList(new Text("someKey"), new DoubleWritable(0))); - inputs.add(Arrays.asList(new Text("someKey"), new DoubleWritable(1))); - inputs.add(Arrays.asList(new Text("someKey"), new DoubleWritable(2))); - inputs.add(Arrays.asList(new Text("someKey"), new DoubleWritable(2))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new DoubleWritable(0))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new DoubleWritable(1))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new DoubleWritable(2))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new DoubleWritable(2))); Map exp = new LinkedHashMap<>(); exp.put(ReduceOp.Min, 0.0); @@ -86,7 +82,7 @@ public class TestMultiOpReduce extends BaseND4JTest { assertEquals(out.get(0), new Text("someKey")); String msg = op.toString(); - assertEquals(exp.get(op), out.get(1).toDouble(), 1e-5,msg); + assertEquals( exp.get(op), out.get(1).toDouble(), 1e-5, msg); } } @@ -94,10 +90,10 @@ public class TestMultiOpReduce extends BaseND4JTest { public void testReducerInteger() { List> inputs = new ArrayList<>(); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(0))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(0))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2))); Map exp = new LinkedHashMap<>(); exp.put(ReduceOp.Min, 0.0); @@ -130,18 +126,19 @@ public class TestMultiOpReduce extends BaseND4JTest { assertEquals(out.get(0), new Text("someKey")); String msg = op.toString(); - assertEquals(exp.get(op), out.get(1).toDouble(), 1e-5,msg); + assertEquals(exp.get(op), out.get(1).toDouble(), 1e-5, msg); } } @Test public void testReduceString() { + List> inputs = new ArrayList<>(); - inputs.add(Arrays.asList(new Text("someKey"), new Text("1"))); - inputs.add(Arrays.asList(new Text("someKey"), new Text("2"))); - inputs.add(Arrays.asList(new Text("someKey"), new Text("3"))); - inputs.add(Arrays.asList(new Text("someKey"), new Text("4"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("1"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("2"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("3"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("4"))); Map exp = new LinkedHashMap<>(); exp.put(ReduceOp.Append, "1234"); @@ -166,7 +163,7 @@ public class TestMultiOpReduce extends BaseND4JTest { assertEquals(out.get(0), new Text("someKey")); String msg = op.toString(); - assertEquals(exp.get(op), out.get(1).toString(),msg); + assertEquals(exp.get(op), out.get(1).toString(), msg); } } @@ -174,12 +171,12 @@ public class TestMultiOpReduce extends BaseND4JTest { public void testReduceIntegerIgnoreInvalidValues() { List> inputs = new ArrayList<>(); - inputs.add(Arrays.asList(new Text("someKey"), new Text("0"))); - inputs.add(Arrays.asList(new Text("someKey"), new Text("1"))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2))); - inputs.add(Arrays.asList(new Text("someKey"), new Text("ignore me"))); - inputs.add(Arrays.asList(new Text("someKey"), new Text("also ignore me"))); - inputs.add(Arrays.asList(new Text("someKey"), new Text("2"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("0"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("1"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("ignore me"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("also ignore me"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("2"))); Map exp = new LinkedHashMap<>(); @@ -213,7 +210,7 @@ public class TestMultiOpReduce extends BaseND4JTest { assertEquals(out.get(0), new Text("someKey")); String msg = op.toString(); - assertEquals(exp.get(op), out.get(1).toDouble(), 1e-5,msg); + assertEquals(exp.get(op), out.get(1).toDouble(), 1e-5, msg); } for (ReduceOp op : Arrays.asList(ReduceOp.Min, ReduceOp.Max, ReduceOp.Range, ReduceOp.Sum, ReduceOp.Mean, @@ -241,16 +238,16 @@ public class TestMultiOpReduce extends BaseND4JTest { public void testCustomReductions() { List> inputs = new ArrayList<>(); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("zero"), + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1), new Text("zero"), new DoubleWritable(0))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("one"), + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2), new Text("one"), new DoubleWritable(1))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("two"), + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(3), new Text("two"), new DoubleWritable(2))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("three"), + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(4), new Text("three"), new DoubleWritable(3))); - List expected = Arrays.asList(new Text("someKey"), new IntWritable(10), new Text("one"), + List expected = Arrays.asList((Writable) new Text("someKey"), new IntWritable(10), new Text("one"), new DoubleWritable(1)); @@ -291,16 +288,16 @@ public class TestMultiOpReduce extends BaseND4JTest { public void testCustomReductionsWithCondition() { List> inputs = new ArrayList<>(); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("zero"), + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(1), new Text("zero"), new DoubleWritable(0))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("one"), + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(2), new Text("one"), new DoubleWritable(1))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("two"), + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(3), new Text("two"), new DoubleWritable(2))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("three"), + inputs.add(Arrays.asList((Writable) new Text("someKey"), new IntWritable(4), new Text("three"), new DoubleWritable(3))); - List expected = Arrays.asList(new Text("someKey"), new IntWritable(10), new IntWritable(3), + List expected = Arrays.asList((Writable) new Text("someKey"), new IntWritable(10), new IntWritable(3), new DoubleWritable(1)); @@ -344,7 +341,7 @@ public class TestMultiOpReduce extends BaseND4JTest { public IAggregableReduceOp> reduceOp() { //For testing: let's take the second value return new AggregableMultiOp<>(Collections - .>singletonList(new AggregableSecond<>())); + .>singletonList(new AggregableSecond())); } @Override @@ -486,12 +483,12 @@ public class TestMultiOpReduce extends BaseND4JTest { .addColumnString("filterCol").addColumnString("textCol").build(); List> inputs = new ArrayList<>(); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("a"), new Text("zero"))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("b"), new Text("one"))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("a"), new Text("two"))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("b"), new Text("three"))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(5), new Text("a"), new Text("three"))); - inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(6), new Text("b"), new Text("three"))); + inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(1), new Text("a"), new Text("zero"))); + inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(2), new Text("b"), new Text("one"))); + inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(3), new Text("a"), new Text("two"))); + inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(4), new Text("b"), new Text("three"))); + inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(5), new Text("a"), new Text("three"))); + inputs.add(Arrays.asList(new Text("someKey"), new IntWritable(6), new Text("b"), new Text("three"))); Condition condition = new StringColumnCondition("filterCol", ConditionOp.Equal, "a"); diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/reduce/TestReductions.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/reduce/TestReductions.java similarity index 96% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/reduce/TestReductions.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/reduce/TestReductions.java index 2199b1f2a..f7aa89170 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/reduce/TestReductions.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/reduce/TestReductions.java @@ -24,17 +24,14 @@ import org.datavec.api.transform.ops.IAggregableReduceOp; import org.datavec.api.transform.reduce.impl.GeographicMidpointReduction; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestReductions extends BaseND4JTest { @Test diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/schema/TestJsonYaml.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/schema/TestJsonYaml.java new file mode 100644 index 000000000..0f9263bb4 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/schema/TestJsonYaml.java @@ -0,0 +1,142 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.schema; + +import org.datavec.api.transform.metadata.ColumnMetaData; +import org.joda.time.DateTimeZone; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestJsonYaml extends BaseND4JTest { + + @Test + public void testToFromJsonYaml() { + + Schema schema = new Schema.Builder() + .addColumnCategorical("Cat", "State1", "State2") + .addColumnDouble("Dbl") + .addColumnDouble("Dbl2", null, 100.0, true, false) + .addColumnInteger("Int") + .addColumnInteger("Int2", 0, 10) + .addColumnLong("Long") + .addColumnLong("Long2", -100L, null) + .addColumnString("Str") + .addColumnString("Str2", "someregexhere", 1, null) + .addColumnTime("TimeCol", DateTimeZone.UTC) + .addColumnTime("TimeCol2", DateTimeZone.UTC, null, 1000L) + .addColumnNDArray("ndarray", new long[]{1, 10}) + .addColumnBoolean("boolean") + .addColumnFloat("float") + .addColumnFloat("float2", -100f, 100f, true, false) + .build(); + + String asJson = schema.toJson(); + // System.out.println(asJson); + + Schema schema2 = Schema.fromJson(asJson); + + int count = schema.numColumns(); + for (int i = 0; i < count; i++) { + ColumnMetaData c1 = schema.getMetaData(i); + ColumnMetaData c2 = schema2.getMetaData(i); + assertEquals(c1, c2); + } + assertEquals(schema, schema2); + + + String asYaml = schema.toYaml(); + // System.out.println(asYaml); + + Schema schema3 = Schema.fromYaml(asYaml); + for (int i = 0; i < schema.numColumns(); i++) { + ColumnMetaData c1 = schema.getMetaData(i); + ColumnMetaData c3 = schema3.getMetaData(i); + assertEquals(c1, c3); + } + assertEquals(schema, schema3); + } + + @Test + public void testMissingPrimitives() { + + Schema schema = new Schema.Builder().addColumnDouble("Dbl2", null, 100.0, false, false).build(); + //Legacy format JSON + String strJson = "{\n" + " \"Schema\" : {\n" + + " \"columns\" : [ {\n" + " \"Double\" : {\n" + + " \"name\" : \"Dbl2\",\n" + " \"maxAllowedValue\" : 100.0\n" + + //" \"allowNaN\" : false,\n" + //Normally included: but exclude here to test + //" \"allowInfinite\" : false\n" + //Normally included: but exclude here to test + " }\n" + " } ]\n" + " }\n" + "}"; + + Schema schema2 = Schema.fromJson(strJson); + assertEquals(schema, schema2); + + + + String strYaml = "--- !\n" + "columns:\n" + "- !\n" + " name: \"Dbl2\"\n" + + " maxAllowedValue: 100.0"; + //" allowNaN: false\n" + //Normally included: but exclude here to test + //" allowInfinite: false"; //Normally included: but exclude here to test + +// Schema schema2a = Schema.fromYaml(strYaml); +// assertEquals(schema, schema2a); + } + + @Test + public void testToFromJsonYamlSequence() { + + Schema schema = new SequenceSchema.Builder().addColumnCategorical("Cat", "State1", "State2") + .addColumnDouble("Dbl").addColumnDouble("Dbl2", null, 100.0, true, false) + .addColumnInteger("Int").addColumnInteger("Int2", 0, 10).addColumnLong("Long") + .addColumnLong("Long2", -100L, null).addColumnString("Str") + .addColumnString("Str2", "someregexhere", 1, null).addColumnTime("TimeCol", DateTimeZone.UTC) + .addColumnTime("TimeCol2", DateTimeZone.UTC, null, 1000L).build(); + + String asJson = schema.toJson(); + // System.out.println(asJson); + + Schema schema2 = Schema.fromJson(asJson); + + int count = schema.numColumns(); + for (int i = 0; i < count; i++) { + ColumnMetaData c1 = schema.getMetaData(i); + ColumnMetaData c2 = schema2.getMetaData(i); + assertEquals(c1, c2); + } + assertEquals(schema, schema2); + + + String asYaml = schema.toYaml(); + // System.out.println(asYaml); + + Schema schema3 = Schema.fromYaml(asYaml); + for (int i = 0; i < schema.numColumns(); i++) { + ColumnMetaData c1 = schema.getMetaData(i); + ColumnMetaData c3 = schema3.getMetaData(i); + assertEquals(c1, c3); + } + assertEquals(schema, schema3); + + } + +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/schema/TestSchemaMethods.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/schema/TestSchemaMethods.java similarity index 94% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/schema/TestSchemaMethods.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/schema/TestSchemaMethods.java index f3cb22ee1..1439cfc40 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/schema/TestSchemaMethods.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/schema/TestSchemaMethods.java @@ -21,14 +21,11 @@ package org.datavec.api.transform.schema; import org.datavec.api.transform.ColumnType; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestSchemaMethods extends BaseND4JTest { @Test diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestReduceSequenceByWindowFunction.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestReduceSequenceByWindowFunction.java similarity index 97% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestReduceSequenceByWindowFunction.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestReduceSequenceByWindowFunction.java index 6e5b800ea..1bb9ae62a 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestReduceSequenceByWindowFunction.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestReduceSequenceByWindowFunction.java @@ -33,10 +33,8 @@ import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.NullWritable; import org.datavec.api.writable.Writable; import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.ArrayList; import java.util.Arrays; @@ -44,8 +42,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestReduceSequenceByWindowFunction extends BaseND4JTest { @Test diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestSequenceSplit.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestSequenceSplit.java new file mode 100644 index 000000000..c26eaec61 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestSequenceSplit.java @@ -0,0 +1,77 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.sequence; + +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.schema.SequenceSchema; +import org.datavec.api.transform.sequence.split.SequenceSplitTimeSeparation; +import org.datavec.api.writable.LongWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.joda.time.DateTimeZone; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestSequenceSplit extends BaseND4JTest { + + @Test + public void testSequenceSplitTimeSeparation() { + + Schema schema = new SequenceSchema.Builder().addColumnTime("time", DateTimeZone.UTC).addColumnString("text") + .build(); + + List> inputSequence = new ArrayList<>(); + inputSequence.add(Arrays.asList((Writable) new LongWritable(0), new Text("t0"))); + inputSequence.add(Arrays.asList((Writable) new LongWritable(1000), new Text("t1"))); + //Second split: 74 seconds later + inputSequence.add(Arrays.asList((Writable) new LongWritable(75000), new Text("t2"))); + inputSequence.add(Arrays.asList((Writable) new LongWritable(100000), new Text("t3"))); + //Third split: 1 minute and 1 milliseconds later + inputSequence.add(Arrays.asList((Writable) new LongWritable(160001), new Text("t4"))); + + SequenceSplit seqSplit = new SequenceSplitTimeSeparation("time", 1, TimeUnit.MINUTES); + seqSplit.setInputSchema(schema); + + List>> splits = seqSplit.split(inputSequence); + assertEquals(3, splits.size()); + + List> exp0 = new ArrayList<>(); + exp0.add(Arrays.asList((Writable) new LongWritable(0), new Text("t0"))); + exp0.add(Arrays.asList((Writable) new LongWritable(1000), new Text("t1"))); + List> exp1 = new ArrayList<>(); + exp1.add(Arrays.asList((Writable) new LongWritable(75000), new Text("t2"))); + exp1.add(Arrays.asList((Writable) new LongWritable(100000), new Text("t3"))); + List> exp2 = new ArrayList<>(); + exp2.add(Arrays.asList((Writable) new LongWritable(160001), new Text("t4"))); + + assertEquals(exp0, splits.get(0)); + assertEquals(exp1, splits.get(1)); + assertEquals(exp2, splits.get(2)); + } + +} diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestWindowFunctions.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestWindowFunctions.java new file mode 100644 index 000000000..ff45a3f3e --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/sequence/TestWindowFunctions.java @@ -0,0 +1,312 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.transform.sequence; + +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.schema.SequenceSchema; +import org.datavec.api.transform.sequence.window.OverlappingTimeWindowFunction; +import org.datavec.api.transform.sequence.window.TimeWindowFunction; +import org.datavec.api.transform.sequence.window.WindowFunction; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.LongWritable; +import org.datavec.api.writable.Writable; +import org.joda.time.DateTimeZone; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestWindowFunctions extends BaseND4JTest { + + @Test + public void testTimeWindowFunction() { + + //Time windowing: 1 second (1000 milliseconds) window + + //Create some data. + List> sequence = new ArrayList<>(); + //First window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2))); + //Second window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 1000L), new IntWritable(3))); + //Third window: empty + //Fourth window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3000L), new IntWritable(4))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3100L), new IntWritable(5))); + + Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) + .addColumnInteger("intcolumn").build(); + + WindowFunction wf = new TimeWindowFunction("timecolumn", 1, TimeUnit.SECONDS); + wf.setInputSchema(schema); + + List>> windows = wf.applyToSequence(sequence); + + assertEquals(4, windows.size()); + assertEquals(3, windows.get(0).size()); + assertEquals(1, windows.get(1).size()); + assertEquals(0, windows.get(2).size()); + assertEquals(2, windows.get(3).size()); + + List> exp0 = new ArrayList<>(); + exp0.add(sequence.get(0)); + exp0.add(sequence.get(1)); + exp0.add(sequence.get(2)); + assertEquals(exp0, windows.get(0)); + + List> exp1 = new ArrayList<>(); + exp1.add(sequence.get(3)); + assertEquals(exp1, windows.get(1)); + + List> exp2 = new ArrayList<>(); + assertEquals(exp2, windows.get(2)); + + List> exp3 = new ArrayList<>(); + exp3.add(sequence.get(4)); + exp3.add(sequence.get(5)); + assertEquals(exp3, windows.get(3)); + } + + @Test + public void testTimeWindowFunctionExcludeEmpty() { + + //Time windowing: 1 second (1000 milliseconds) window + + //Create some data. + List> sequence = new ArrayList<>(); + //First window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2))); + //Second window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 1000L), new IntWritable(3))); + //Third window: empty + //Fourth window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3000L), new IntWritable(4))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3100L), new IntWritable(5))); + + Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) + .addColumnInteger("intcolumn").build(); + + WindowFunction wf = new TimeWindowFunction.Builder().timeColumn("timecolumn").windowSize(1, TimeUnit.SECONDS) + .excludeEmptyWindows(true).build(); + + wf.setInputSchema(schema); + + List>> windows = wf.applyToSequence(sequence); + + assertEquals(3, windows.size()); + assertEquals(3, windows.get(0).size()); + assertEquals(1, windows.get(1).size()); + assertEquals(2, windows.get(2).size()); + + List> exp0 = new ArrayList<>(); + exp0.add(sequence.get(0)); + exp0.add(sequence.get(1)); + exp0.add(sequence.get(2)); + assertEquals(exp0, windows.get(0)); + + List> exp1 = new ArrayList<>(); + exp1.add(sequence.get(3)); + assertEquals(exp1, windows.get(1)); + + List> exp2 = new ArrayList<>(); + exp2.add(sequence.get(4)); + exp2.add(sequence.get(5)); + assertEquals(exp2, windows.get(2)); + } + + @Test + public void testOverlappingTimeWindowFunctionSimple() { + //Compare Overlapping and standard window functions where the window separation is equal to the window size + // In this case, we should get exactly the same results from both. + //Time windowing: 1 second (1000 milliseconds) window + + //Create some data. + List> sequence = new ArrayList<>(); + //First window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2))); + //Second window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 1000L), new IntWritable(3))); + //Third window: empty + //Fourth window: + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3000L), new IntWritable(4))); + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 3100L), new IntWritable(5))); + + Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) + .addColumnInteger("intcolumn").build(); + + WindowFunction wf = new TimeWindowFunction("timecolumn", 1, TimeUnit.SECONDS); + wf.setInputSchema(schema); + + WindowFunction wf2 = new OverlappingTimeWindowFunction("timecolumn", 1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); + wf2.setInputSchema(schema); + + List>> windowsExp = wf.applyToSequence(sequence); + List>> windowsAct = wf2.applyToSequence(sequence); + + int[] expSizes = {3, 1, 0, 2}; + assertEquals(4, windowsExp.size()); + assertEquals(4, windowsAct.size()); + for (int i = 0; i < 4; i++) { + assertEquals(expSizes[i], windowsExp.get(i).size()); + assertEquals(expSizes[i], windowsAct.get(i).size()); + + assertEquals(windowsExp.get(i), windowsAct.get(i)); + } + } + + @Test + public void testOverlappingTimeWindowFunction() { + //Create some data. + List> sequence = new ArrayList<>(); + //First window: + sequence.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0))); + sequence.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1))); + sequence.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2))); + sequence.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3))); + sequence.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4))); + sequence.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5))); + sequence.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7))); + + + Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) + .addColumnInteger("intcolumn").build(); + //Window size: 2 seconds; calculated every 1 second + WindowFunction wf2 = new OverlappingTimeWindowFunction("timecolumn", 2, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); + wf2.setInputSchema(schema); + + List>> windowsAct = wf2.applyToSequence(sequence); + + //First window: -1000 to 1000 + List> exp0 = new ArrayList<>(); + exp0.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0))); + exp0.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1))); + exp0.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2))); + //Second window: 0 to 2000 + List> exp1 = new ArrayList<>(); + exp1.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0))); + exp1.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1))); + exp1.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2))); + exp1.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3))); + exp1.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4))); + //Third window: 1000 to 3000 + List> exp2 = new ArrayList<>(); + exp2.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3))); + exp2.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4))); + exp2.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5))); + //Fourth window: 2000 to 4000 + List> exp3 = new ArrayList<>(); + exp3.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5))); + //Fifth window: 3000 to 5000 + List> exp4 = new ArrayList<>(); + //Sixth window: 4000 to 6000 + List> exp5 = new ArrayList<>(); + exp5.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7))); + //Seventh window: 5000 to 7000 + List> exp6 = new ArrayList<>(); + exp6.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7))); + + List>> windowsExp = Arrays.asList(exp0, exp1, exp2, exp3, exp4, exp5, exp6); + + assertEquals(7, windowsAct.size()); + for (int i = 0; i < 7; i++) { + List> exp = windowsExp.get(i); + List> act = windowsAct.get(i); + + assertEquals(exp, act); + } + } + + @Test + public void testOverlappingTimeWindowFunctionExcludeEmpty() { + //Create some data. + List> sequence = new ArrayList<>(); + //First window: + sequence.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0))); + sequence.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1))); + sequence.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2))); + sequence.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3))); + sequence.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4))); + sequence.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5))); + sequence.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7))); + + + Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) + .addColumnInteger("intcolumn").build(); + //Window size: 2 seconds; calculated every 1 second + // WindowFunction wf2 = new OverlappingTimeWindowFunction("timecolumn",2,TimeUnit.SECONDS,1,TimeUnit.SECONDS); + WindowFunction wf2 = new OverlappingTimeWindowFunction.Builder().timeColumn("timecolumn") + .windowSize(2, TimeUnit.SECONDS).windowSeparation(1, TimeUnit.SECONDS).excludeEmptyWindows(true) + .build(); + wf2.setInputSchema(schema); + + List>> windowsAct = wf2.applyToSequence(sequence); + + //First window: -1000 to 1000 + List> exp0 = new ArrayList<>(); + exp0.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0))); + exp0.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1))); + exp0.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2))); + //Second window: 0 to 2000 + List> exp1 = new ArrayList<>(); + exp1.add(Arrays.asList((Writable) new LongWritable(0), new IntWritable(0))); + exp1.add(Arrays.asList((Writable) new LongWritable(100), new IntWritable(1))); + exp1.add(Arrays.asList((Writable) new LongWritable(200), new IntWritable(2))); + exp1.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3))); + exp1.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4))); + //Third window: 1000 to 3000 + List> exp2 = new ArrayList<>(); + exp2.add(Arrays.asList((Writable) new LongWritable(1000), new IntWritable(3))); + exp2.add(Arrays.asList((Writable) new LongWritable(1500), new IntWritable(4))); + exp2.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5))); + //Fourth window: 2000 to 4000 + List> exp3 = new ArrayList<>(); + exp3.add(Arrays.asList((Writable) new LongWritable(2000), new IntWritable(5))); + //Fifth window: 3000 to 5000 -> Empty: excluded + //Sixth window: 4000 to 6000 + List> exp5 = new ArrayList<>(); + exp5.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7))); + //Seventh window: 5000 to 7000 + List> exp6 = new ArrayList<>(); + exp6.add(Arrays.asList((Writable) new LongWritable(5000), new IntWritable(7))); + + List>> windowsExp = Arrays.asList(exp0, exp1, exp2, exp3, exp5, exp6); + + assertEquals(6, windowsAct.size()); + for (int i = 0; i < 6; i++) { + List> exp = windowsExp.get(i); + List> act = windowsAct.get(i); + + assertEquals(exp, act); + } + } + +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/TestCustomTransformJsonYaml.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/TestCustomTransformJsonYaml.java similarity index 92% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/TestCustomTransformJsonYaml.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/TestCustomTransformJsonYaml.java index b95ffe18b..53b63bb49 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/TestCustomTransformJsonYaml.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/TestCustomTransformJsonYaml.java @@ -26,17 +26,11 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.transform.serde.testClasses.CustomCondition; import org.datavec.api.transform.serde.testClasses.CustomFilter; import org.datavec.api.transform.serde.testClasses.CustomTransform; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JACKSON_SERDE) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) public class TestCustomTransformJsonYaml extends BaseND4JTest { @Test diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/TestYamlJsonSerde.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/TestYamlJsonSerde.java similarity index 99% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/TestYamlJsonSerde.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/TestYamlJsonSerde.java index 2a96158ee..84da1c272 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/TestYamlJsonSerde.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/TestYamlJsonSerde.java @@ -64,19 +64,14 @@ import org.datavec.api.transform.transform.time.TimeMathOpTransform; import org.datavec.api.writable.comparator.DoubleWritableComparator; import org.joda.time.DateTimeFieldType; import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.*; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JACKSON_SERDE) public class TestYamlJsonSerde extends BaseND4JTest { public static YamlSerializer y = new YamlSerializer(); diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomCondition.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomCondition.java similarity index 100% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomCondition.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomCondition.java diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomFilter.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomFilter.java similarity index 100% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomFilter.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomFilter.java diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomTransform.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomTransform.java similarity index 97% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomTransform.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomTransform.java index dba9b30c4..d9a157a06 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomTransform.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/serde/testClasses/CustomTransform.java @@ -24,7 +24,7 @@ import org.datavec.api.transform.metadata.ColumnMetaData; import org.datavec.api.transform.metadata.DoubleMetaData; import org.datavec.api.transform.transform.BaseColumnTransform; import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; public class CustomTransform extends BaseColumnTransform { diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/stringreduce/TestReduce.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/stringreduce/TestReduce.java similarity index 86% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/stringreduce/TestReduce.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/stringreduce/TestReduce.java index 0517e0c3f..f7eaa85ad 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/stringreduce/TestReduce.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/stringreduce/TestReduce.java @@ -24,26 +24,22 @@ import org.datavec.api.transform.StringReduceOp; import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) public class TestReduce extends BaseND4JTest { @Test public void testReducerDouble() { List> inputs = new ArrayList<>(); - inputs.add(Arrays.asList(new Text("1"), new Text("2"))); - inputs.add(Arrays.asList(new Text("1"), new Text("2"))); - inputs.add(Arrays.asList(new Text("1"), new Text("2"))); + inputs.add(Arrays.asList((Writable) new Text("1"), new Text("2"))); + inputs.add(Arrays.asList((Writable) new Text("1"), new Text("2"))); + inputs.add(Arrays.asList((Writable) new Text("1"), new Text("2"))); Map exp = new LinkedHashMap<>(); exp.put(StringReduceOp.MERGE, "12"); diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/RegressionTestJson.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/RegressionTestJson.java similarity index 100% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/RegressionTestJson.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/RegressionTestJson.java diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/TestJsonYaml.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/TestJsonYaml.java similarity index 100% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/TestJsonYaml.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/TestJsonYaml.java diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/TestTransforms.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/TestTransforms.java similarity index 96% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/TestTransforms.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/TestTransforms.java index 1d9d72189..c0468b916 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/TestTransforms.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/TestTransforms.java @@ -20,7 +20,6 @@ package org.datavec.api.transform.transform; -import junit.framework.TestCase; import org.datavec.api.transform.*; import org.datavec.api.transform.condition.Condition; import org.datavec.api.transform.condition.ConditionOp; @@ -58,7 +57,7 @@ import org.datavec.api.transform.transform.time.TimeMathOpTransform; import org.datavec.api.writable.*; import org.joda.time.DateTimeFieldType; import org.joda.time.DateTimeZone; - +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.linalg.api.ndarray.INDArray; @@ -71,7 +70,6 @@ import java.io.ObjectOutputStream; import java.util.*; import java.util.concurrent.TimeUnit; - import static org.junit.jupiter.api.Assertions.*; public class TestTransforms extends BaseND4JTest { @@ -116,7 +114,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(0).getColumnType()); IntegerMetaData meta = (IntegerMetaData) out.getMetaData(0); assertNotNull(meta.getMinAllowedValue()); assertEquals(0, (int) meta.getMinAllowedValue()); @@ -139,7 +137,7 @@ public class TestTransforms extends BaseND4JTest { assertEquals(3, out.getColumnMetaData().size()); for (int i = 0; i < 3; i++) { - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(i).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(i).getColumnType()); IntegerMetaData meta = (IntegerMetaData) out.getMetaData(i); assertNotNull(meta.getMinAllowedValue()); assertEquals(0, (int) meta.getMinAllowedValue()); @@ -246,7 +244,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Categorical, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Categorical, out.getMetaData(0).getColumnType()); CategoricalMetaData meta = (CategoricalMetaData) out.getMetaData(0); assertEquals(Arrays.asList("zero", "one", "two"), meta.getStateNames()); @@ -348,8 +346,8 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(2, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(1).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(1).getColumnType()); assertEquals(Arrays.asList(new Text("one"), new IntWritable(1)), transform.map(Arrays.asList((Writable) new DoubleWritable(1.0), new Text("one"), @@ -367,8 +365,8 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(2, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(1).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(1).getColumnType()); assertEquals(Arrays.asList(new Text("one"), new IntWritable(1)), transform.map(Arrays.asList((Writable) new DoubleWritable(1.0), new Text("one"), @@ -385,7 +383,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new IntWritable(0)), transform.map(Collections.singletonList((Writable) new IntWritable(0)))); @@ -404,7 +402,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new IntWritable(0)), transform.map(Collections.singletonList((Writable) new IntWritable(0)))); @@ -428,7 +426,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); DoubleMetaData meta = (DoubleMetaData) out.getMetaData(0); assertNotNull(meta.getMinAllowedValue()); assertEquals(0, meta.getMinAllowedValue(), 1e-6); @@ -459,7 +457,7 @@ public class TestTransforms extends BaseND4JTest { Schema out2 = transform2.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); DoubleMetaData meta = (DoubleMetaData) out.getMetaData(0); DoubleMetaData meta2 = (DoubleMetaData) out2.getMetaData(0); assertEquals(0, meta.getMinAllowedValue(), 1e-6); @@ -500,7 +498,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); DoubleMetaData meta = (DoubleMetaData) out.getMetaData(0); assertNull(meta.getMinAllowedValue()); assertNull(meta.getMaxAllowedValue()); @@ -528,7 +526,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); DoubleMetaData meta = (DoubleMetaData) out.getMetaData(0); assertNull(meta.getMinAllowedValue()); assertNull(meta.getMaxAllowedValue()); @@ -551,7 +549,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new Text("one")), transform.map(Collections.singletonList((Writable) new Text("one")))); @@ -570,7 +568,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new Text("one")), transform.map(Collections.singletonList((Writable) new Text("one ")))); @@ -591,7 +589,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new Text("one")), transform.map(Collections.singletonList((Writable) new Text("one")))); @@ -610,7 +608,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new Text("one_AppendThis")), transform.map(Collections.singletonList((Writable) new Text("one")))); @@ -633,7 +631,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(3, out.getColumnMetaData().size()); for (int i = 0; i < 3; i++) { - TestCase.assertEquals(ColumnType.Categorical, out.getType(i)); + assertEquals(ColumnType.Categorical, out.getType(i)); CategoricalMetaData meta = (CategoricalMetaData) out.getMetaData(i); assertEquals(Arrays.asList("true", "false"), meta.getStateNames()); } @@ -664,7 +662,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new Text("ONE")), transform.map(Collections.singletonList((Writable) new Text("one")))); @@ -714,7 +712,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Time, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.Time, out.getMetaData(0).getColumnType()); String in1 = "2016-01-01 12:30:45"; long out1 = 1451651445000L; @@ -760,12 +758,12 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(6, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Time, out.getMetaData(0).getColumnType()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(1).getColumnType()); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(2).getColumnType()); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(3).getColumnType()); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(4).getColumnType()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(5).getColumnType()); + assertEquals(ColumnType.Time, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(1).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(2).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(3).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(4).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(5).getColumnType()); assertEquals("column", out.getName(0)); assertEquals("otherColumn", out.getName(1)); @@ -838,7 +836,7 @@ public class TestTransforms extends BaseND4JTest { ColumnType.Long, ColumnType.Long); for (int i = 0; i < 5; i++) { assertEquals(expOutNames.get(i), out.getName(i)); - TestCase.assertEquals(expOutTypes.get(i), out.getType(i)); + assertEquals(expOutTypes.get(i), out.getType(i)); } List inList = Arrays.asList((Writable) new Text("one"), new IntWritable(2), new LongWritable(3L)); @@ -857,7 +855,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Integer, out.getType(0)); + assertEquals(ColumnType.Integer, out.getType(0)); IntegerMetaData meta = (IntegerMetaData) out.getMetaData(0); assertEquals(-5, (int) meta.getMinAllowedValue()); assertEquals(5, (int) meta.getMaxAllowedValue()); @@ -904,7 +902,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Long, out.getType(0)); + assertEquals(ColumnType.Long, out.getType(0)); LongMetaData meta = (LongMetaData) out.getMetaData(0); assertEquals(-5, (long) meta.getMinAllowedValue()); assertEquals(5, (long) meta.getMaxAllowedValue()); @@ -951,7 +949,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Time, out.getType(0)); + assertEquals(ColumnType.Time, out.getType(0)); assertEquals(Collections.singletonList((Writable) new LongWritable(1000 + 43200000)), transform.map(Collections.singletonList((Writable) new LongWritable(1000)))); @@ -968,7 +966,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Double, out.getType(0)); + assertEquals(ColumnType.Double, out.getType(0)); DoubleMetaData meta = (DoubleMetaData) out.getMetaData(0); assertEquals(-5.0, meta.getMinAllowedValue(), 1e-6); assertEquals(5.0, meta.getMaxAllowedValue(), 1e-6); @@ -1039,9 +1037,9 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(3, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(1).getColumnType()); - TestCase.assertEquals(ColumnType.Integer, out.getMetaData(2).getColumnType()); + assertEquals(ColumnType.Double, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(1).getColumnType()); + assertEquals(ColumnType.Integer, out.getMetaData(2).getColumnType()); assertEquals("column1", out.getName(0)); assertEquals("col2", out.getName(1)); @@ -1201,7 +1199,7 @@ public class TestTransforms extends BaseND4JTest { Schema out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new Text("BoneConeTone")), transform.map(Collections.singletonList((Writable) new Text("B1midT3")))); @@ -1214,7 +1212,7 @@ public class TestTransforms extends BaseND4JTest { out = transform.transform(schema); assertEquals(1, out.getColumnMetaData().size()); - TestCase.assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); assertEquals(Collections.singletonList((Writable) new Text("4.25")), transform.map(Collections.singletonList((Writable) new Text(" 4.25 ")))); diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/ndarray/TestNDArrayWritableTransforms.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/ndarray/TestNDArrayWritableTransforms.java similarity index 100% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/ndarray/TestNDArrayWritableTransforms.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/ndarray/TestNDArrayWritableTransforms.java diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/ndarray/TestYamlJsonSerde.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/ndarray/TestYamlJsonSerde.java similarity index 100% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/ndarray/TestYamlJsonSerde.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/ndarray/TestYamlJsonSerde.java diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/parse/ParseDoubleTransformTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/parse/ParseDoubleTransformTest.java similarity index 85% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/parse/ParseDoubleTransformTest.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/parse/ParseDoubleTransformTest.java index a42b273e2..e531d040f 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/transform/parse/ParseDoubleTransformTest.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/transform/parse/ParseDoubleTransformTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.api.transform.transform.parse; import org.datavec.api.writable.DoubleWritable; @@ -24,22 +25,21 @@ import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Parse Double Transform Test") -class ParseDoubleTransformTest extends BaseND4JTest { +public class ParseDoubleTransformTest extends BaseND4JTest { @Test - @DisplayName("Test Double Transform") - void testDoubleTransform() { + public void testDoubleTransform() { List record = new ArrayList<>(); record.add(new Text("0.0")); List transformed = Arrays.asList(new DoubleWritable(0.0)); assertEquals(transformed, new ParseDoubleTransform().map(record)); } + + } diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ui/TestUI.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ui/TestUI.java similarity index 95% rename from datavec/datavec-api/src/test/java/org/datavec/api/transform/ui/TestUI.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ui/TestUI.java index 8233308f8..7b13b03d7 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ui/TestUI.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/transform/ui/TestUI.java @@ -35,30 +35,24 @@ import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Disabled; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) + public class TestUI extends BaseND4JTest { + @TempDir + public File testDir; @Test - public void testUI(@TempDir Path testDir) throws Exception { + public void testUI() throws Exception { Schema schema = new Schema.Builder().addColumnString("StringColumn").addColumnInteger("IntColumn") .addColumnInteger("IntColumn2").addColumnInteger("IntColumn3") .addColumnTime("TimeColumn", DateTimeZone.UTC).build(); @@ -96,7 +90,7 @@ public class TestUI extends BaseND4JTest { DataAnalysis da = new DataAnalysis(schema, list); - File fDir = testDir.toFile(); + File fDir = testDir; String tempDir = fDir.getAbsolutePath(); String outPath = FilenameUtils.concat(tempDir, "datavec_transform_UITest.html"); System.out.println(outPath); @@ -147,7 +141,7 @@ public class TestUI extends BaseND4JTest { @Test - @Disabled + ////@Ignore public void testSequencePlot() throws Exception { Schema schema = new SequenceSchema.Builder().addColumnDouble("sinx") diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/util/ClassPathResourceTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/util/ClassPathResourceTest.java new file mode 100644 index 000000000..d656ddebd --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/util/ClassPathResourceTest.java @@ -0,0 +1,132 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.util; + + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.hamcrest.MatcherAssert.assertThat; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.anyOf; + +public class ClassPathResourceTest extends BaseND4JTest { + + private boolean isWindows = false; //File sizes are reported slightly different on Linux vs. Windows + + @BeforeEach + public void setUp() throws Exception { + String osname = System.getProperty("os.name"); + if (osname != null && osname.toLowerCase().contains("win")) { + isWindows = true; + } + } + + @Test + public void testGetFile1() throws Exception { + File intFile = new ClassPathResource("datavec-api/iris.dat").getFile(); + + assertTrue(intFile.exists()); + if (isWindows) { + assertThat(intFile.length(), anyOf(equalTo(2700L), equalTo(2850L))); + } else { + assertEquals(2700, intFile.length()); + } + } + + @Test + public void testGetFileSlash1() throws Exception { + File intFile = new ClassPathResource("datavec-api/iris.dat").getFile(); + + assertTrue(intFile.exists()); + if (isWindows) { + assertThat(intFile.length(), anyOf(equalTo(2700L), equalTo(2850L))); + } else { + assertEquals(2700, intFile.length()); + } + } + + @Test + public void testGetFileWithSpace1() throws Exception { + File intFile = new ClassPathResource("datavec-api/csvsequence test.txt").getFile(); + + assertTrue(intFile.exists()); + + if (isWindows) { + assertThat(intFile.length(), anyOf(equalTo(60L), equalTo(64L))); + } else { + assertEquals(60, intFile.length()); + } + } + + @Test + public void testInputStream() throws Exception { + ClassPathResource resource = new ClassPathResource("datavec-api/csvsequence_1.txt"); + File intFile = resource.getFile(); + + if (isWindows) { + assertThat(intFile.length(), anyOf(equalTo(60L), equalTo(64L))); + } else { + assertEquals(60, intFile.length()); + } + + InputStream stream = resource.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + String line = ""; + int cnt = 0; + while ((line = reader.readLine()) != null) { + cnt++; + } + + assertEquals(5, cnt); + } + + @Test + public void testInputStreamSlash() throws Exception { + ClassPathResource resource = new ClassPathResource("datavec-api/csvsequence_1.txt"); + File intFile = resource.getFile(); + + if (isWindows) { + assertThat(intFile.length(), anyOf(equalTo(60L), equalTo(64L))); + } else { + assertEquals(60, intFile.length()); + } + + InputStream stream = resource.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + String line = ""; + int cnt = 0; + while ((line = reader.readLine()) != null) { + cnt++; + } + + assertEquals(5, cnt); + } +} diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/util/TimeSeriesUtilsTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/util/TimeSeriesUtilsTest.java new file mode 100644 index 000000000..17ffa9ea9 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/util/TimeSeriesUtilsTest.java @@ -0,0 +1,60 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.util; + +import org.datavec.api.timeseries.util.TimeSeriesWritableUtils; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +public class TimeSeriesUtilsTest extends BaseND4JTest { + + @Test + public void testTimeSeriesCreation() { + List>> test = new ArrayList<>(); + List> timeStep = new ArrayList<>(); + for(int i = 0; i < 5; i++) { + timeStep.add(getRecord(5)); + } + + test.add(timeStep); + + INDArray arr = TimeSeriesWritableUtils.convertWritablesSequence(test).getFirst(); + assertArrayEquals(new long[]{1,5,5},arr.shape()); + } + + private List getRecord(int length) { + List ret = new ArrayList<>(); + for(int i = 0; i < length; i++) { + ret.add(new DoubleWritable(1.0)); + } + + return ret; + } + +} diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/RecordConverterTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/RecordConverterTest.java new file mode 100644 index 000000000..ed9c01793 --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/RecordConverterTest.java @@ -0,0 +1,143 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.writable; + +import org.nd4j.common.tests.BaseND4JTest; +import com.google.common.collect.Lists; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.util.ndarray.RecordConverter; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Arrays; +import java.util.List; +import java.util.TimeZone; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RecordConverterTest extends BaseND4JTest { + @Test + public void toRecords_PassInClassificationDataSet_ExpectNDArrayAndIntWritables() { + INDArray feature1 = Nd4j.create(new double[]{4, -5.7, 10, -0.1}, new long[]{1, 4}, DataType.FLOAT); + INDArray feature2 = Nd4j.create(new double[]{11, .7, -1.3, 4}, new long[]{1, 4}, DataType.FLOAT); + INDArray label1 = Nd4j.create(new double[]{0, 0, 1, 0}, new long[]{1, 4}, DataType.FLOAT); + INDArray label2 = Nd4j.create(new double[]{0, 1, 0, 0}, new long[]{1, 4}, DataType.FLOAT); + DataSet dataSet = new DataSet(Nd4j.vstack(Lists.newArrayList(feature1, feature2)), + Nd4j.vstack(Lists.newArrayList(label1, label2))); + + List> writableList = RecordConverter.toRecords(dataSet); + + assertEquals(2, writableList.size()); + testClassificationWritables(feature1, 2, writableList.get(0)); + testClassificationWritables(feature2, 1, writableList.get(1)); + } + + @Test + public void toRecords_PassInRegressionDataSet_ExpectNDArrayAndDoubleWritables() { + INDArray feature = Nd4j.create(new double[]{4, -5.7, 10, -0.1}, new long[]{1, 4}, DataType.FLOAT); + INDArray label = Nd4j.create(new double[]{.5, 2, 3, .5}, new long[]{1, 4}, DataType.FLOAT); + DataSet dataSet = new DataSet(feature, label); + + List> writableList = RecordConverter.toRecords(dataSet); + List results = writableList.get(0); + NDArrayWritable ndArrayWritable = (NDArrayWritable) results.get(0); + + assertEquals(1, writableList.size()); + assertEquals(5, results.size()); + assertEquals(feature, ndArrayWritable.get()); + for (int i = 0; i < label.shape()[1]; i++) { + DoubleWritable doubleWritable = (DoubleWritable) results.get(i + 1); + assertEquals(label.getDouble(i), doubleWritable.get(), 0); + } + } + + private void testClassificationWritables(INDArray expectedFeatureVector, int expectLabelIndex, + List writables) { + NDArrayWritable ndArrayWritable = (NDArrayWritable) writables.get(0); + IntWritable intWritable = (IntWritable) writables.get(1); + + assertEquals(2, writables.size()); + assertEquals(expectedFeatureVector, ndArrayWritable.get()); + assertEquals(expectLabelIndex, intWritable.get()); + } + + + @Test + public void testNDArrayWritableConcat() { + List l = Arrays.asList(new DoubleWritable(1), + new NDArrayWritable(Nd4j.create(new double[]{2, 3, 4}, new long[]{1, 3}, DataType.FLOAT)), new DoubleWritable(5), + new NDArrayWritable(Nd4j.create(new double[]{6, 7, 8}, new long[]{1, 3}, DataType.FLOAT)), new IntWritable(9), + new IntWritable(1)); + + INDArray exp = Nd4j.create(new double[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 1}, new long[]{1, 10}, DataType.FLOAT); + INDArray act = RecordConverter.toArray(DataType.FLOAT, l); + + assertEquals(exp, act); + } + + @Test + public void testNDArrayWritableConcatToMatrix(){ + + List l1 = Arrays.asList(new DoubleWritable(1), new NDArrayWritable(Nd4j.create(new double[]{2, 3, 4}, new long[]{1,3}, DataType.FLOAT)), new DoubleWritable(5)); + List l2 = Arrays.asList(new DoubleWritable(6), new NDArrayWritable(Nd4j.create(new double[]{7, 8, 9}, new long[]{1,3}, DataType.FLOAT)), new DoubleWritable(10)); + + INDArray exp = Nd4j.create(new double[][]{ + {1,2,3,4,5}, + {6,7,8,9,10}}).castTo(DataType.FLOAT); + + INDArray act = RecordConverter.toMatrix(DataType.FLOAT, Arrays.asList(l1,l2)); + + assertEquals(exp, act); + } + + @Test + public void testToRecordWithListOfObject(){ + final List list = Arrays.asList((Object)3, 7.0f, "Foo", "Bar", 1.0, 3f, 3L, 7, 0L); + final Schema schema = new Schema.Builder() + .addColumnInteger("a") + .addColumnFloat("b") + .addColumnString("c") + .addColumnCategorical("d", "Bar", "Baz") + .addColumnDouble("e") + .addColumnFloat("f") + .addColumnLong("g") + .addColumnInteger("h") + .addColumnTime("i", TimeZone.getDefault()) + .build(); + + final List record = RecordConverter.toRecord(schema, list); + + assertEquals(record.get(0).toInt(), 3); + assertEquals(record.get(1).toFloat(), 7f, 1e-6); + assertEquals(record.get(2).toString(), "Foo"); + assertEquals(record.get(3).toString(), "Bar"); + assertEquals(record.get(4).toDouble(), 1.0, 1e-6); + assertEquals(record.get(5).toFloat(), 3f, 1e-6); + assertEquals(record.get(6).toLong(), 3L); + assertEquals(record.get(7).toInt(), 7); + assertEquals(record.get(8).toLong(), 0); + + + } +} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/writable/TestNDArrayWritableAndSerialization.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/TestNDArrayWritableAndSerialization.java similarity index 97% rename from datavec/datavec-api/src/test/java/org/datavec/api/writable/TestNDArrayWritableAndSerialization.java rename to cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/TestNDArrayWritableAndSerialization.java index b56dc9192..71149b9b2 100644 --- a/datavec/datavec-api/src/test/java/org/datavec/api/writable/TestNDArrayWritableAndSerialization.java +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/TestNDArrayWritableAndSerialization.java @@ -21,18 +21,15 @@ package org.datavec.api.writable; import org.datavec.api.transform.metadata.NDArrayMetaData; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import java.io.*; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class TestNDArrayWritableAndSerialization extends BaseND4JTest { @Test diff --git a/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/WritableTest.java b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/WritableTest.java new file mode 100644 index 000000000..767742e4a --- /dev/null +++ b/cavis-datavec/cavis-datavec-api/src/test/java/org/datavec/api/writable/WritableTest.java @@ -0,0 +1,183 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.writable; + +import org.datavec.api.writable.batch.NDArrayRecordBatch; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class WritableTest extends BaseND4JTest { + + @Test + public void testWritableEqualityReflexive() { + assertEquals(new IntWritable(1), new IntWritable(1)); + assertEquals(new LongWritable(1), new LongWritable(1)); + assertEquals(new DoubleWritable(1), new DoubleWritable(1)); + assertEquals(new FloatWritable(1), new FloatWritable(1)); + assertEquals(new Text("Hello"), new Text("Hello")); + assertEquals(new BytesWritable("Hello".getBytes()),new BytesWritable("Hello".getBytes())); + INDArray ndArray = Nd4j.rand(new int[]{1, 100}); + + assertEquals(new NDArrayWritable(ndArray), new NDArrayWritable(ndArray)); + assertEquals(new NullWritable(), new NullWritable()); + assertEquals(new BooleanWritable(true), new BooleanWritable(true)); + byte b = 0; + assertEquals(new ByteWritable(b), new ByteWritable(b)); + } + + + @Test + public void testBytesWritableIndexing() { + byte[] doubleWrite = new byte[16]; + ByteBuffer wrapped = ByteBuffer.wrap(doubleWrite); + Buffer buffer = (Buffer) wrapped; + wrapped.putDouble(1.0); + wrapped.putDouble(2.0); + buffer.rewind(); + BytesWritable byteWritable = new BytesWritable(doubleWrite); + assertEquals(2,byteWritable.getDouble(1),1e-1); + DataBuffer dataBuffer = Nd4j.createBuffer(new double[] {1,2}); + double[] d1 = dataBuffer.asDouble(); + double[] d2 = byteWritable.asNd4jBuffer(DataType.DOUBLE,8).asDouble(); + assertArrayEquals(d1, d2, 0.0); + } + + @Test + public void testByteWritable() { + byte b = 0xfffffffe; + assertEquals(new IntWritable(-2), new ByteWritable(b)); + assertEquals(new LongWritable(-2), new ByteWritable(b)); + assertEquals(new ByteWritable(b), new IntWritable(-2)); + assertEquals(new ByteWritable(b), new LongWritable(-2)); + + // those would cast to the same Int + byte minus126 = 0xffffff82; + assertNotEquals(new ByteWritable(minus126), new IntWritable(130)); + } + + @Test + public void testIntLongWritable() { + assertEquals(new IntWritable(1), new LongWritable(1l)); + assertEquals(new LongWritable(2l), new IntWritable(2)); + + long l = 1L << 34; + // those would cast to the same Int + assertNotEquals(new LongWritable(l), new IntWritable(4)); + } + + + @Test + public void testDoubleFloatWritable() { + assertEquals(new DoubleWritable(1d), new FloatWritable(1f)); + assertEquals(new FloatWritable(2f), new DoubleWritable(2d)); + + // we defer to Java equality for Floats + assertNotEquals(new DoubleWritable(1.1d), new FloatWritable(1.1f)); + // same idea as above + assertNotEquals(new DoubleWritable(1.1d), new FloatWritable((float)1.1d)); + + assertNotEquals(new DoubleWritable((double)Float.MAX_VALUE + 1), new FloatWritable(Float.POSITIVE_INFINITY)); + } + + + @Test + public void testFuzzies() { + assertTrue(new DoubleWritable(1.1d).fuzzyEquals(new FloatWritable(1.1f), 1e-6d)); + assertTrue(new FloatWritable(1.1f).fuzzyEquals(new DoubleWritable(1.1d), 1e-6d)); + byte b = 0xfffffffe; + assertTrue(new ByteWritable(b).fuzzyEquals(new DoubleWritable(-2.0), 1e-6d)); + assertFalse(new IntWritable(1).fuzzyEquals(new FloatWritable(1.1f), 1e-2d)); + assertTrue(new IntWritable(1).fuzzyEquals(new FloatWritable(1.05f), 1e-1d)); + assertTrue(new LongWritable(1).fuzzyEquals(new DoubleWritable(1.05f), 1e-1d)); + } + + + @Test + public void testNDArrayRecordBatch(){ + Nd4j.getRandom().setSeed(12345); + + List> orig = new ArrayList<>(); //Outer list over writables/columns, inner list over examples + for( int i=0; i<3; i++ ){ + orig.add(new ArrayList()); + } + + for( int i=0; i<5; i++ ){ + orig.get(0).add(Nd4j.rand(1,10)); + orig.get(1).add(Nd4j.rand(new int[]{1,5,6})); + orig.get(2).add(Nd4j.rand(new int[]{1,3,4,5})); + } + + List> origByExample = new ArrayList<>(); //Outer list over examples, inner list over writables + for( int i=0; i<5; i++ ){ + origByExample.add(Arrays.asList(orig.get(0).get(i), orig.get(1).get(i), orig.get(2).get(i))); + } + + List batched = new ArrayList<>(); + for(List l : orig){ + batched.add(Nd4j.concat(0, l.toArray(new INDArray[5]))); + } + + NDArrayRecordBatch batch = new NDArrayRecordBatch(batched); + assertEquals(5, batch.size()); + for( int i=0; i<5; i++ ){ + List act = batch.get(i); + List unboxed = new ArrayList<>(); + for(Writable w : act){ + unboxed.add(((NDArrayWritable)w).get()); + } + List exp = origByExample.get(i); + assertEquals(exp.size(), unboxed.size()); + for( int j=0; j> iter = batch.iterator(); + int count = 0; + while(iter.hasNext()){ + List next = iter.next(); + List unboxed = new ArrayList<>(); + for(Writable w : next){ + unboxed.add(((NDArrayWritable)w).get()); + } + List exp = origByExample.get(count++); + assertEquals(exp.size(), unboxed.size()); + for( int j=0; j> ret = new ArrayList<>(numRows); + for(int i = 0; i < numRows; i++) { + ret.add(Arrays.asList(new NDArrayWritable(Nd4j.linspace(1,4,4).reshape(1, 4)))); + } + + List fieldVectors = ArrowConverter.toArrowColumns(bufferAllocator, schema, ret); + ArrowWritableRecordBatch arrowWritableRecordBatch = new ArrowWritableRecordBatch(fieldVectors,schema); + INDArray array = ArrowConverter.toArray(arrowWritableRecordBatch); + assertArrayEquals(new long[]{4,4},array.shape()); + + INDArray assertion = Nd4j.repeat(Nd4j.linspace(1,4,4),4).reshape(4,4); + assertEquals(assertion,array); + } + + @Test + public void testArrowColumnINDArray() { + Schema.Builder schema = new Schema.Builder(); + List single = new ArrayList<>(); + int numCols = 2; + INDArray arr = Nd4j.linspace(1,4,4); + for(int i = 0; i < numCols; i++) { + schema.addColumnNDArray(String.valueOf(i),new long[]{1,4}); + single.add(String.valueOf(i)); + } + + Schema buildSchema = schema.build(); + List> list = new ArrayList<>(); + List firstRow = new ArrayList<>(); + for(int i = 0 ; i < numCols; i++) { + firstRow.add(new NDArrayWritable(arr)); + } + + list.add(firstRow); + + List fieldVectors = ArrowConverter.toArrowColumns(bufferAllocator, buildSchema, list); + assertEquals(numCols,fieldVectors.size()); + assertEquals(1,fieldVectors.get(0).getValueCount()); + assertFalse(fieldVectors.get(0).isNull(0)); + + ArrowWritableRecordBatch arrowWritableRecordBatch = ArrowConverter.toArrowWritables(fieldVectors, buildSchema); + assertEquals(1,arrowWritableRecordBatch.size()); + + Writable writable = arrowWritableRecordBatch.get(0).get(0); + assertTrue(writable instanceof NDArrayWritable); + NDArrayWritable ndArrayWritable = (NDArrayWritable) writable; + assertEquals(arr,ndArrayWritable.get()); + + Writable writable1 = ArrowConverter.fromEntry(0, fieldVectors.get(0), ColumnType.NDArray); + NDArrayWritable ndArrayWritablewritable1 = (NDArrayWritable) writable1; + System.out.println(ndArrayWritablewritable1.get()); + + } + + @Test + public void testArrowColumnString() { + Schema.Builder schema = new Schema.Builder(); + List single = new ArrayList<>(); + for(int i = 0; i < 2; i++) { + schema.addColumnInteger(String.valueOf(i)); + single.add(String.valueOf(i)); + } + + + List fieldVectors = ArrowConverter.toArrowColumnsStringSingle(bufferAllocator, schema.build(), single); + List> records = ArrowConverter.toArrowWritables(fieldVectors, schema.build()); + List> assertion = new ArrayList<>(); + assertion.add(Arrays.asList(new IntWritable(0),new IntWritable(1))); + assertEquals(assertion,records); + + List> batch = new ArrayList<>(); + for(int i = 0; i < 2; i++) { + batch.add(Arrays.asList(String.valueOf(i),String.valueOf(i))); + } + + List fieldVectorsBatch = ArrowConverter.toArrowColumnsString(bufferAllocator, schema.build(), batch); + List> batchRecords = ArrowConverter.toArrowWritables(fieldVectorsBatch, schema.build()); + + List> assertionBatch = new ArrayList<>(); + assertionBatch.add(Arrays.asList(new IntWritable(0),new IntWritable(0))); + assertionBatch.add(Arrays.asList(new IntWritable(1),new IntWritable(1))); + assertEquals(assertionBatch,batchRecords); + + + } + + + + @Test + public void testArrowBatchSetTime() { + Schema.Builder schema = new Schema.Builder(); + List single = new ArrayList<>(); + for(int i = 0; i < 2; i++) { + schema.addColumnTime(String.valueOf(i),TimeZone.getDefault()); + single.add(String.valueOf(i)); + } + + List> input = Arrays.asList( + Arrays.asList(new LongWritable(0),new LongWritable(1)), + Arrays.asList(new LongWritable(2),new LongWritable(3)) + ); + + List fieldVector = ArrowConverter.toArrowColumns(bufferAllocator,schema.build(),input); + ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector,schema.build()); + List assertion = Arrays.asList(new LongWritable(4), new LongWritable(5)); + writableRecordBatch.set(1, Arrays.asList(new LongWritable(4),new LongWritable(5))); + List recordTest = writableRecordBatch.get(1); + assertEquals(assertion,recordTest); + } + + @Test + public void testArrowBatchSet() { + Schema.Builder schema = new Schema.Builder(); + List single = new ArrayList<>(); + for(int i = 0; i < 2; i++) { + schema.addColumnInteger(String.valueOf(i)); + single.add(String.valueOf(i)); + } + + List> input = Arrays.asList( + Arrays.asList(new IntWritable(0),new IntWritable(1)), + Arrays.asList(new IntWritable(2),new IntWritable(3)) + ); + + List fieldVector = ArrowConverter.toArrowColumns(bufferAllocator,schema.build(),input); + ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector,schema.build()); + List assertion = Arrays.asList(new IntWritable(4), new IntWritable(5)); + writableRecordBatch.set(1, Arrays.asList(new IntWritable(4),new IntWritable(5))); + List recordTest = writableRecordBatch.get(1); + assertEquals(assertion,recordTest); + } + + @Test + public void testArrowColumnsStringTimeSeries() { + Schema.Builder schema = new Schema.Builder(); + List>> entries = new ArrayList<>(); + for(int i = 0; i < 3; i++) { + schema.addColumnInteger(String.valueOf(i)); + } + + for(int i = 0; i < 5; i++) { + List> arr = Arrays.asList(Arrays.asList(String.valueOf(i), String.valueOf(i), String.valueOf(i))); + entries.add(arr); + } + + List fieldVectors = ArrowConverter.toArrowColumnsStringTimeSeries(bufferAllocator, schema.build(), entries); + assertEquals(3,fieldVectors.size()); + assertEquals(5,fieldVectors.get(0).getValueCount()); + + + INDArray exp = Nd4j.create(5, 3); + for( int i = 0; i < 5; i++) { + exp.getRow(i).assign(i); + } + //Convert to ArrowWritableRecordBatch - note we can't do this in general with time series... + ArrowWritableRecordBatch wri = ArrowConverter.toArrowWritables(fieldVectors, schema.build()); + INDArray arr = ArrowConverter.toArray(wri); + assertArrayEquals(new long[] {5,3}, arr.shape()); + + + assertEquals(exp, arr); + } + + @Test + public void testConvertVector() { + Schema.Builder schema = new Schema.Builder(); + List>> entries = new ArrayList<>(); + for(int i = 0; i < 3; i++) { + schema.addColumnInteger(String.valueOf(i)); + } + + for(int i = 0; i < 5; i++) { + List> arr = Arrays.asList(Arrays.asList(String.valueOf(i), String.valueOf(i), String.valueOf(i))); + entries.add(arr); + } + + List fieldVectors = ArrowConverter.toArrowColumnsStringTimeSeries(bufferAllocator, schema.build(), entries); + INDArray arr = ArrowConverter.convertArrowVector(fieldVectors.get(0),schema.build().getType(0)); + assertEquals(5,arr.length()); + } + + @Test + public void testCreateNDArray() throws Exception { + val recordsToWrite = recordToWrite(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),byteArrayOutputStream); + + File f = testDir; + + File tmpFile = new File(f, "tmp-arrow-file-" + UUID.randomUUID().toString() + ".arrorw"); + FileOutputStream outputStream = new FileOutputStream(tmpFile); + tmpFile.deleteOnExit(); + ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),outputStream); + outputStream.flush(); + outputStream.close(); + + Pair schemaArrowWritableRecordBatchPair = ArrowConverter.readFromFile(tmpFile); + assertEquals(recordsToWrite.getFirst(),schemaArrowWritableRecordBatchPair.getFirst()); + assertEquals(recordsToWrite.getRight(),schemaArrowWritableRecordBatchPair.getRight().toArrayList()); + + byte[] arr = byteArrayOutputStream.toByteArray(); + val read = ArrowConverter.readFromBytes(arr); + assertEquals(recordsToWrite,read); + + //send file + File tmp = tmpDataFile(recordsToWrite); + ArrowRecordReader recordReader = new ArrowRecordReader(); + + recordReader.initialize(new FileSplit(tmp)); + + recordReader.next(); + ArrowWritableRecordBatch currentBatch = recordReader.getCurrentBatch(); + INDArray arr2 = ArrowConverter.toArray(currentBatch); + assertEquals(2,arr2.rows()); + assertEquals(2,arr2.columns()); + } + + + @Test + public void testConvertToArrowVectors() { + INDArray matrix = Nd4j.linspace(1,4,4).reshape(2,2); + val vectors = ArrowConverter.convertToArrowVector(matrix,Arrays.asList("test","test2"), ColumnType.Double,bufferAllocator); + assertEquals(matrix.rows(),vectors.size()); + + INDArray vector = Nd4j.linspace(1,4,4); + val vectors2 = ArrowConverter.convertToArrowVector(vector,Arrays.asList("test"), ColumnType.Double,bufferAllocator); + assertEquals(1,vectors2.size()); + assertEquals(matrix.length(),vectors2.get(0).getValueCount()); + + } + + @Test + public void testSchemaConversionBasic() { + Schema.Builder schemaBuilder = new Schema.Builder(); + for(int i = 0; i < 2; i++) { + schemaBuilder.addColumnDouble("test-" + i); + schemaBuilder.addColumnInteger("testi-" + i); + schemaBuilder.addColumnLong("testl-" + i); + schemaBuilder.addColumnFloat("testf-" + i); + } + + + Schema schema = schemaBuilder.build(); + val schema2 = ArrowConverter.toArrowSchema(schema); + assertEquals(8,schema2.getFields().size()); + val convertedSchema = ArrowConverter.toDatavecSchema(schema2); + assertEquals(schema,convertedSchema); + } + + @Test + public void testReadSchemaAndRecordsFromByteArray() throws Exception { + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + + int valueCount = 3; + List fields = new ArrayList<>(); + fields.add(ArrowConverter.field("field1",new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE))); + fields.add(ArrowConverter.intField("field2")); + + List fieldVectors = new ArrayList<>(); + fieldVectors.add(ArrowConverter.vectorFor(allocator,"field1",new float[] {1,2,3})); + fieldVectors.add(ArrowConverter.vectorFor(allocator,"field2",new int[] {1,2,3})); + + + org.apache.arrow.vector.types.pojo.Schema schema = new org.apache.arrow.vector.types.pojo.Schema(fields); + + VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, valueCount); + VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1); + vectorUnloader.getRecordBatch(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try(ArrowFileWriter arrowFileWriter = new ArrowFileWriter(schemaRoot1,null,newChannel(byteArrayOutputStream))) { + arrowFileWriter.writeBatch(); + } catch (IOException e) { + log.error("",e); + } + + byte[] arr = byteArrayOutputStream.toByteArray(); + val arr2 = ArrowConverter.readFromBytes(arr); + assertEquals(2,arr2.getFirst().numColumns()); + assertEquals(3,arr2.getRight().size()); + + val arrowCols = ArrowConverter.toArrowColumns(allocator,arr2.getFirst(),arr2.getRight()); + assertEquals(2,arrowCols.size()); + assertEquals(valueCount,arrowCols.get(0).getValueCount()); + } + + + @Test + public void testVectorForEdgeCases() { + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + val vector = ArrowConverter.vectorFor(allocator,"field1",new float[]{Float.MIN_VALUE,Float.MAX_VALUE}); + assertEquals(Float.MIN_VALUE,vector.get(0),1e-2); + assertEquals(Float.MAX_VALUE,vector.get(1),1e-2); + + val vectorInt = ArrowConverter.vectorFor(allocator,"field1",new int[]{Integer.MIN_VALUE,Integer.MAX_VALUE}); + assertEquals(Integer.MIN_VALUE,vectorInt.get(0),1e-2); + assertEquals(Integer.MAX_VALUE,vectorInt.get(1),1e-2); + + } + + @Test + public void testVectorFor() { + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + + val vector = ArrowConverter.vectorFor(allocator,"field1",new float[]{1,2,3}); + assertEquals(3,vector.getValueCount()); + assertEquals(1,vector.get(0),1e-2); + assertEquals(2,vector.get(1),1e-2); + assertEquals(3,vector.get(2),1e-2); + + val vectorLong = ArrowConverter.vectorFor(allocator,"field1",new long[]{1,2,3}); + assertEquals(3,vectorLong.getValueCount()); + assertEquals(1,vectorLong.get(0),1e-2); + assertEquals(2,vectorLong.get(1),1e-2); + assertEquals(3,vectorLong.get(2),1e-2); + + + val vectorInt = ArrowConverter.vectorFor(allocator,"field1",new int[]{1,2,3}); + assertEquals(3,vectorInt.getValueCount()); + assertEquals(1,vectorInt.get(0),1e-2); + assertEquals(2,vectorInt.get(1),1e-2); + assertEquals(3,vectorInt.get(2),1e-2); + + val vectorDouble = ArrowConverter.vectorFor(allocator,"field1",new double[]{1,2,3}); + assertEquals(3,vectorDouble.getValueCount()); + assertEquals(1,vectorDouble.get(0),1e-2); + assertEquals(2,vectorDouble.get(1),1e-2); + assertEquals(3,vectorDouble.get(2),1e-2); + + + val vectorBool = ArrowConverter.vectorFor(allocator,"field1",new boolean[]{true,true,false}); + assertEquals(3,vectorBool.getValueCount()); + assertEquals(1,vectorBool.get(0),1e-2); + assertEquals(1,vectorBool.get(1),1e-2); + assertEquals(0,vectorBool.get(2),1e-2); + } + + @Test + public void testRecordReaderAndWriteFile() throws Exception { + val recordsToWrite = recordToWrite(); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),byteArrayOutputStream); + byte[] arr = byteArrayOutputStream.toByteArray(); + val read = ArrowConverter.readFromBytes(arr); + assertEquals(recordsToWrite,read); + + //send file + File tmp = tmpDataFile(recordsToWrite); + RecordReader recordReader = new ArrowRecordReader(); + + recordReader.initialize(new FileSplit(tmp)); + + List record = recordReader.next(); + assertEquals(2,record.size()); + + } + + @Test + public void testRecordReaderMetaDataList() throws Exception { + val recordsToWrite = recordToWrite(); + //send file + File tmp = tmpDataFile(recordsToWrite); + RecordReader recordReader = new ArrowRecordReader(); + RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class); + recordReader.loadFromMetaData(Arrays.asList(recordMetaDataIndex)); + + Record record = recordReader.nextRecord(); + assertEquals(2,record.getRecord().size()); + + } + + @Test + public void testDates() { + Date now = new Date(); + BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE); + TimeStampMilliVector timeStampMilliVector = ArrowConverter.vectorFor(bufferAllocator, "col1", new Date[]{now}); + assertEquals(now.getTime(),timeStampMilliVector.get(0)); + } + + + @Test + public void testRecordReaderMetaData() throws Exception { + val recordsToWrite = recordToWrite(); + //send file + File tmp = tmpDataFile(recordsToWrite); + RecordReader recordReader = new ArrowRecordReader(); + RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class); + recordReader.loadFromMetaData(recordMetaDataIndex); + + Record record = recordReader.nextRecord(); + assertEquals(2,record.getRecord().size()); + } + + private File tmpDataFile(Pair>> recordsToWrite) throws IOException { + + File f = testDir; + + //send file + File tmp = new File(f,"tmp-file-" + UUID.randomUUID().toString()); + tmp.mkdirs(); + File tmpFile = new File(tmp,"data.arrow"); + tmpFile.deleteOnExit(); + FileOutputStream bufferedOutputStream = new FileOutputStream(tmpFile); + ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),bufferedOutputStream); + bufferedOutputStream.flush(); + bufferedOutputStream.close(); + return tmp; + } + + private Pair>> recordToWrite() { + List> records = new ArrayList<>(); + records.add(Arrays.asList(new DoubleWritable(0.0),new DoubleWritable(0.0))); + records.add(Arrays.asList(new DoubleWritable(0.0),new DoubleWritable(0.0))); + Schema.Builder schemaBuilder = new Schema.Builder(); + for(int i = 0; i < 2; i++) { + schemaBuilder.addColumnFloat("col-" + i); + } + + return Pair.of(schemaBuilder.build(),records); + } + + + + +} diff --git a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-arrow/src/test/java/org/datavec/arrow/AssertTestsExtendBaseClass.java similarity index 100% rename from datavec/datavec-arrow/src/test/java/org/datavec/arrow/AssertTestsExtendBaseClass.java rename to cavis-datavec/cavis-datavec-arrow/src/test/java/org/datavec/arrow/AssertTestsExtendBaseClass.java diff --git a/cavis-datavec/cavis-datavec-arrow/src/test/java/org/datavec/arrow/RecordMapperTest.java b/cavis-datavec/cavis-datavec-arrow/src/test/java/org/datavec/arrow/RecordMapperTest.java new file mode 100644 index 000000000..9666c18d7 --- /dev/null +++ b/cavis-datavec/cavis-datavec-arrow/src/test/java/org/datavec/arrow/RecordMapperTest.java @@ -0,0 +1,190 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.arrow; + +import lombok.val; +import org.apache.commons.io.FileUtils; +import org.datavec.api.records.mapper.RecordMapper; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.records.writer.impl.csv.CSVRecordWriter; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.split.partition.NumberOfRecordsPartitioner; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Writable; +import org.datavec.arrow.recordreader.ArrowRecordReader; +import org.datavec.arrow.recordreader.ArrowRecordWriter; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.primitives.Triple; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RecordMapperTest extends BaseND4JTest { + + @Test + public void testMultiWrite() throws Exception { + val recordsPair = records(); + + Path p = Files.createTempFile("arrowwritetest", ".arrow"); + FileUtils.write(p.toFile(),recordsPair.getFirst()); + p.toFile().deleteOnExit(); + + int numReaders = 2; + RecordReader[] readers = new RecordReader[numReaders]; + InputSplit[] splits = new InputSplit[numReaders]; + for(int i = 0; i < readers.length; i++) { + FileSplit split = new FileSplit(p.toFile()); + ArrowRecordReader arrowRecordReader = new ArrowRecordReader(); + readers[i] = arrowRecordReader; + splits[i] = split; + } + + ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(recordsPair.getMiddle()); + FileSplit split = new FileSplit(p.toFile()); + arrowRecordWriter.initialize(split,new NumberOfRecordsPartitioner()); + arrowRecordWriter.writeBatch(recordsPair.getRight()); + + + CSVRecordWriter csvRecordWriter = new CSVRecordWriter(); + Path p2 = Files.createTempFile("arrowwritetest", ".csv"); + FileUtils.write(p2.toFile(),recordsPair.getFirst()); + p.toFile().deleteOnExit(); + FileSplit outputCsv = new FileSplit(p2.toFile()); + + RecordMapper mapper = RecordMapper.builder().batchSize(10).inputUrl(split) + .outputUrl(outputCsv) + .partitioner(new NumberOfRecordsPartitioner()).readersToConcat(readers) + .splitPerReader(splits) + .recordWriter(csvRecordWriter) + .build(); + mapper.copy(); + + + } + + + @Test + public void testCopyFromArrowToCsv() throws Exception { + val recordsPair = records(); + + Path p = Files.createTempFile("arrowwritetest", ".arrow"); + FileUtils.write(p.toFile(),recordsPair.getFirst()); + p.toFile().deleteOnExit(); + + ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(recordsPair.getMiddle()); + FileSplit split = new FileSplit(p.toFile()); + arrowRecordWriter.initialize(split,new NumberOfRecordsPartitioner()); + arrowRecordWriter.writeBatch(recordsPair.getRight()); + + + ArrowRecordReader arrowRecordReader = new ArrowRecordReader(); + arrowRecordReader.initialize(split); + + + CSVRecordWriter csvRecordWriter = new CSVRecordWriter(); + Path p2 = Files.createTempFile("arrowwritetest", ".csv"); + FileUtils.write(p2.toFile(),recordsPair.getFirst()); + p.toFile().deleteOnExit(); + FileSplit outputCsv = new FileSplit(p2.toFile()); + + RecordMapper mapper = RecordMapper.builder().batchSize(10).inputUrl(split) + .outputUrl(outputCsv) + .partitioner(new NumberOfRecordsPartitioner()) + .recordReader(arrowRecordReader).recordWriter(csvRecordWriter) + .build(); + mapper.copy(); + + CSVRecordReader recordReader = new CSVRecordReader(); + recordReader.initialize(outputCsv); + + + List> loadedCSvRecords = recordReader.next(10); + assertEquals(10,loadedCSvRecords.size()); + } + + + @Test + public void testCopyFromCsvToArrow() throws Exception { + val recordsPair = records(); + + Path p = Files.createTempFile("csvwritetest", ".csv"); + FileUtils.write(p.toFile(),recordsPair.getFirst()); + p.toFile().deleteOnExit(); + + + CSVRecordReader recordReader = new CSVRecordReader(); + FileSplit fileSplit = new FileSplit(p.toFile()); + + ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(recordsPair.getMiddle()); + File outputFile = Files.createTempFile("outputarrow","arrow").toFile(); + FileSplit outputFileSplit = new FileSplit(outputFile); + RecordMapper mapper = RecordMapper.builder().batchSize(10).inputUrl(fileSplit) + .outputUrl(outputFileSplit).partitioner(new NumberOfRecordsPartitioner()) + .recordReader(recordReader).recordWriter(arrowRecordWriter) + .build(); + mapper.copy(); + + ArrowRecordReader arrowRecordReader = new ArrowRecordReader(); + arrowRecordReader.initialize(outputFileSplit); + List> next = arrowRecordReader.next(10); + System.out.println(next); + assertEquals(10,next.size()); + + } + + private Triple>> records() { + List> list = new ArrayList<>(); + StringBuilder sb = new StringBuilder(); + int numColumns = 3; + for (int i = 0; i < 10; i++) { + List temp = new ArrayList<>(); + for (int j = 0; j < numColumns; j++) { + int v = 100 * i + j; + temp.add(new IntWritable(v)); + sb.append(v); + if (j < 2) + sb.append(","); + else if (i != 9) + sb.append("\n"); + } + list.add(temp); + } + + + Schema.Builder schemaBuilder = new Schema.Builder(); + for(int i = 0; i < numColumns; i++) { + schemaBuilder.addColumnInteger(String.valueOf(i)); + } + + return Triple.of(sb.toString(),schemaBuilder.build(),list); + } + + +} diff --git a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/recordreader/ArrowWritableRecordTimeSeriesBatchTests.java b/cavis-datavec/cavis-datavec-arrow/src/test/java/org/datavec/arrow/recordreader/ArrowWritableRecordTimeSeriesBatchTests.java similarity index 78% rename from datavec/datavec-arrow/src/test/java/org/datavec/arrow/recordreader/ArrowWritableRecordTimeSeriesBatchTests.java rename to cavis-datavec/cavis-datavec-arrow/src/test/java/org/datavec/arrow/recordreader/ArrowWritableRecordTimeSeriesBatchTests.java index 362330262..b39b88013 100644 --- a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/recordreader/ArrowWritableRecordTimeSeriesBatchTests.java +++ b/cavis-datavec/cavis-datavec-arrow/src/test/java/org/datavec/arrow/recordreader/ArrowWritableRecordTimeSeriesBatchTests.java @@ -29,11 +29,9 @@ import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; import org.datavec.arrow.ArrowConverter; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; import java.util.ArrayList; import java.util.Arrays; @@ -41,16 +39,13 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) + public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest { private static BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE); @Test - @Tag(TagNames.NEEDS_VERIFY) - @Disabled public void testBasicIndexing() { Schema.Builder schema = new Schema.Builder(); for(int i = 0; i < 3; i++) { @@ -59,9 +54,9 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest { List> timeStep = Arrays.asList( - Arrays.asList(new IntWritable(0),new IntWritable(1),new IntWritable(2)), - Arrays.asList(new IntWritable(1),new IntWritable(2),new IntWritable(3)), - Arrays.asList(new IntWritable(4),new IntWritable(5),new IntWritable(6)) + Arrays.asList(new IntWritable(0),new IntWritable(1),new IntWritable(2)), + Arrays.asList(new IntWritable(1),new IntWritable(2),new IntWritable(3)), + Arrays.asList(new IntWritable(4),new IntWritable(5),new IntWritable(6)) ); int numTimeSteps = 5; @@ -74,7 +69,7 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest { assertEquals(3,fieldVectors.size()); for(FieldVector fieldVector : fieldVectors) { for(int i = 0; i < fieldVector.getValueCount(); i++) { - assertFalse( fieldVector.isNull(i),"Index " + i + " was null for field vector " + fieldVector); + assertFalse( fieldVector.isNull(i), "Index " + i + " was null for field vector " + fieldVector); } } @@ -83,9 +78,8 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest { } @Test - @Tag(TagNames.NEEDS_VERIFY) - @Disabled //not worried about this till after next release + //@Ignore public void testVariableLengthTS() { Schema.Builder schema = new Schema.Builder() .addColumnString("str") @@ -93,13 +87,13 @@ public class ArrowWritableRecordTimeSeriesBatchTests extends BaseND4JTest { .addColumnDouble("dbl"); List> firstSeq = Arrays.asList( - Arrays.asList(new Text("00"),new IntWritable(0),new DoubleWritable(2.0)), - Arrays.asList(new Text("01"),new IntWritable(1),new DoubleWritable(2.1)), - Arrays.asList(new Text("02"),new IntWritable(2),new DoubleWritable(2.2))); + Arrays.asList(new Text("00"),new IntWritable(0),new DoubleWritable(2.0)), + Arrays.asList(new Text("01"),new IntWritable(1),new DoubleWritable(2.1)), + Arrays.asList(new Text("02"),new IntWritable(2),new DoubleWritable(2.2))); List> secondSeq = Arrays.asList( - Arrays.asList(new Text("10"),new IntWritable(10),new DoubleWritable(12.0)), - Arrays.asList(new Text("11"),new IntWritable(11),new DoubleWritable(12.1))); + Arrays.asList(new Text("10"),new IntWritable(10),new DoubleWritable(12.0)), + Arrays.asList(new Text("11"),new IntWritable(11),new DoubleWritable(12.1))); List>> sequences = Arrays.asList(firstSeq, secondSeq); diff --git a/cavis-datavec/cavis-datavec-data/build.gradle b/cavis-datavec/cavis-datavec-data/build.gradle new file mode 100644 index 000000000..aa06d6f8e --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/build.gradle @@ -0,0 +1,3 @@ +subprojects { + group = group + ".data" +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/build.gradle b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/build.gradle new file mode 100644 index 000000000..9e6b9ef76 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/build.gradle @@ -0,0 +1,32 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisDatavec.cavisDatavecApi + implementation "org.bytedeco:javacpp" + implementation "org.bytedeco:javacv" + implementation "com.github.wendykierp:JTransforms:3.1:with-dependencies" + implementation "org.slf4j:slf4j-api" + implementation "commons-io:commons-io" + testImplementation projects.cavisNd4j.cavisNd4jCommonTests +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/Wave.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/Wave.java new file mode 100644 index 000000000..db75a546f --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/Wave.java @@ -0,0 +1,330 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio; + + +import org.datavec.audio.extension.NormalizedSampleAmplitudes; +import org.datavec.audio.extension.Spectrogram; +import org.datavec.audio.fingerprint.FingerprintManager; +import org.datavec.audio.fingerprint.FingerprintSimilarity; +import org.datavec.audio.fingerprint.FingerprintSimilarityComputer; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; + +/** + * Read WAVE headers and data from wave input stream + * + * @author Jacquet Wong + */ +public class Wave implements Serializable { + + private static final long serialVersionUID = 1L; + private WaveHeader waveHeader; + private byte[] data; // little endian + private byte[] fingerprint; + + /** + * Constructor + * + */ + public Wave() { + this.waveHeader = new WaveHeader(); + this.data = new byte[0]; + } + + /** + * Constructor + * + * @param filename + * Wave file + */ + public Wave(String filename) { + try { + InputStream inputStream = new FileInputStream(filename); + initWaveWithInputStream(inputStream); + inputStream.close(); + } catch (IOException e) { + System.out.println(e.toString()); + } + } + + /** + * Constructor + * + * @param inputStream + * Wave file input stream + */ + public Wave(InputStream inputStream) { + initWaveWithInputStream(inputStream); + } + + /** + * Constructor + * + * @param waveHeader + * @param data + */ + public Wave(WaveHeader waveHeader, byte[] data) { + this.waveHeader = waveHeader; + this.data = data; + } + + private void initWaveWithInputStream(InputStream inputStream) { + // reads the first 44 bytes for header + waveHeader = new WaveHeader(inputStream); + + if (waveHeader.isValid()) { + // load data + try { + data = new byte[inputStream.available()]; + inputStream.read(data); + } catch (IOException e) { + System.err.println(e.toString()); + } + // end load data + } else { + System.err.println("Invalid Wave Header"); + } + } + + /** + * Trim the wave data + * + * @param leftTrimNumberOfSample + * Number of sample trimmed from beginning + * @param rightTrimNumberOfSample + * Number of sample trimmed from ending + */ + public void trim(int leftTrimNumberOfSample, int rightTrimNumberOfSample) { + + long chunkSize = waveHeader.getChunkSize(); + long subChunk2Size = waveHeader.getSubChunk2Size(); + + long totalTrimmed = leftTrimNumberOfSample + rightTrimNumberOfSample; + + if (totalTrimmed > subChunk2Size) { + leftTrimNumberOfSample = (int) subChunk2Size; + } + + // update wav info + chunkSize -= totalTrimmed; + subChunk2Size -= totalTrimmed; + + if (chunkSize >= 0 && subChunk2Size >= 0) { + waveHeader.setChunkSize(chunkSize); + waveHeader.setSubChunk2Size(subChunk2Size); + + byte[] trimmedData = new byte[(int) subChunk2Size]; + System.arraycopy(data, (int) leftTrimNumberOfSample, trimmedData, 0, (int) subChunk2Size); + data = trimmedData; + } else { + System.err.println("Trim error: Negative length"); + } + } + + /** + * Trim the wave data from beginning + * + * @param numberOfSample + * numberOfSample trimmed from beginning + */ + public void leftTrim(int numberOfSample) { + trim(numberOfSample, 0); + } + + /** + * Trim the wave data from ending + * + * @param numberOfSample + * numberOfSample trimmed from ending + */ + public void rightTrim(int numberOfSample) { + trim(0, numberOfSample); + } + + /** + * Trim the wave data + * + * @param leftTrimSecond + * Seconds trimmed from beginning + * @param rightTrimSecond + * Seconds trimmed from ending + */ + public void trim(double leftTrimSecond, double rightTrimSecond) { + + int sampleRate = waveHeader.getSampleRate(); + int bitsPerSample = waveHeader.getBitsPerSample(); + int channels = waveHeader.getChannels(); + + int leftTrimNumberOfSample = (int) (sampleRate * bitsPerSample / 8 * channels * leftTrimSecond); + int rightTrimNumberOfSample = (int) (sampleRate * bitsPerSample / 8 * channels * rightTrimSecond); + + trim(leftTrimNumberOfSample, rightTrimNumberOfSample); + } + + /** + * Trim the wave data from beginning + * + * @param second + * Seconds trimmed from beginning + */ + public void leftTrim(double second) { + trim(second, 0); + } + + /** + * Trim the wave data from ending + * + * @param second + * Seconds trimmed from ending + */ + public void rightTrim(double second) { + trim(0, second); + } + + /** + * Get the wave header + * + * @return waveHeader + */ + public WaveHeader getWaveHeader() { + return waveHeader; + } + + /** + * Get the wave spectrogram + * + * @return spectrogram + */ + public Spectrogram getSpectrogram() { + return new Spectrogram(this); + } + + /** + * Get the wave spectrogram + * + * @param fftSampleSize number of sample in fft, the value needed to be a number to power of 2 + * @param overlapFactor 1/overlapFactor overlapping, e.g. 1/4=25% overlapping, 0 for no overlapping + * + * @return spectrogram + */ + public Spectrogram getSpectrogram(int fftSampleSize, int overlapFactor) { + return new Spectrogram(this, fftSampleSize, overlapFactor); + } + + /** + * Get the wave data in bytes + * + * @return wave data + */ + public byte[] getBytes() { + return data; + } + + /** + * Data byte size of the wave excluding header size + * + * @return byte size of the wave + */ + public int size() { + return data.length; + } + + /** + * Length of the wave in second + * + * @return length in second + */ + public float length() { + return (float) waveHeader.getSubChunk2Size() / waveHeader.getByteRate(); + } + + /** + * Timestamp of the wave length + * + * @return timestamp + */ + public String timestamp() { + float totalSeconds = this.length(); + float second = totalSeconds % 60; + int minute = (int) totalSeconds / 60 % 60; + int hour = (int) (totalSeconds / 3600); + + StringBuilder sb = new StringBuilder(); + if (hour > 0) { + sb.append(hour + ":"); + } + if (minute > 0) { + sb.append(minute + ":"); + } + sb.append(second); + + return sb.toString(); + } + + /** + * Get the amplitudes of the wave samples (depends on the header) + * + * @return amplitudes array (signed 16-bit) + */ + public short[] getSampleAmplitudes() { + int bytePerSample = waveHeader.getBitsPerSample() / 8; + int numSamples = data.length / bytePerSample; + short[] amplitudes = new short[numSamples]; + + int pointer = 0; + for (int i = 0; i < numSamples; i++) { + short amplitude = 0; + for (int byteNumber = 0; byteNumber < bytePerSample; byteNumber++) { + // little endian + amplitude |= (short) ((data[pointer++] & 0xFF) << (byteNumber * 8)); + } + amplitudes[i] = amplitude; + } + + return amplitudes; + } + + public String toString() { + StringBuilder sb = new StringBuilder(waveHeader.toString()); + sb.append("\n"); + sb.append("length: " + timestamp()); + return sb.toString(); + } + + public double[] getNormalizedAmplitudes() { + NormalizedSampleAmplitudes amplitudes = new NormalizedSampleAmplitudes(this); + return amplitudes.getNormalizedAmplitudes(); + } + + public byte[] getFingerprint() { + if (fingerprint == null) { + FingerprintManager fingerprintManager = new FingerprintManager(); + fingerprint = fingerprintManager.extractFingerprint(this); + } + return fingerprint; + } + + public FingerprintSimilarity getFingerprintSimilarity(Wave wave) { + FingerprintSimilarityComputer fingerprintSimilarityComputer = + new FingerprintSimilarityComputer(this.getFingerprint(), wave.getFingerprint()); + return fingerprintSimilarityComputer.getFingerprintsSimilarity(); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/WaveFileManager.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/WaveFileManager.java new file mode 100644 index 000000000..877447787 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/WaveFileManager.java @@ -0,0 +1,94 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio; + +import lombok.extern.slf4j.Slf4j; + +import java.io.FileOutputStream; +import java.io.IOException; + +@Slf4j +public class WaveFileManager { + + private Wave wave; + + public WaveFileManager() { + wave = new Wave(); + } + + public WaveFileManager(Wave wave) { + setWave(wave); + } + + /** + * Save the wave file + * + * @param filename + * filename to be saved + * + * @see Wave file saved + */ + public void saveWaveAsFile(String filename) { + + WaveHeader waveHeader = wave.getWaveHeader(); + + int byteRate = waveHeader.getByteRate(); + int audioFormat = waveHeader.getAudioFormat(); + int sampleRate = waveHeader.getSampleRate(); + int bitsPerSample = waveHeader.getBitsPerSample(); + int channels = waveHeader.getChannels(); + long chunkSize = waveHeader.getChunkSize(); + long subChunk1Size = waveHeader.getSubChunk1Size(); + long subChunk2Size = waveHeader.getSubChunk2Size(); + int blockAlign = waveHeader.getBlockAlign(); + + try { + FileOutputStream fos = new FileOutputStream(filename); + fos.write(WaveHeader.RIFF_HEADER.getBytes()); + // little endian + fos.write(new byte[] {(byte) (chunkSize), (byte) (chunkSize >> 8), (byte) (chunkSize >> 16), + (byte) (chunkSize >> 24)}); + fos.write(WaveHeader.WAVE_HEADER.getBytes()); + fos.write(WaveHeader.FMT_HEADER.getBytes()); + fos.write(new byte[] {(byte) (subChunk1Size), (byte) (subChunk1Size >> 8), (byte) (subChunk1Size >> 16), + (byte) (subChunk1Size >> 24)}); + fos.write(new byte[] {(byte) (audioFormat), (byte) (audioFormat >> 8)}); + fos.write(new byte[] {(byte) (channels), (byte) (channels >> 8)}); + fos.write(new byte[] {(byte) (sampleRate), (byte) (sampleRate >> 8), (byte) (sampleRate >> 16), + (byte) (sampleRate >> 24)}); + fos.write(new byte[] {(byte) (byteRate), (byte) (byteRate >> 8), (byte) (byteRate >> 16), + (byte) (byteRate >> 24)}); + fos.write(new byte[] {(byte) (blockAlign), (byte) (blockAlign >> 8)}); + fos.write(new byte[] {(byte) (bitsPerSample), (byte) (bitsPerSample >> 8)}); + fos.write(WaveHeader.DATA_HEADER.getBytes()); + fos.write(new byte[] {(byte) (subChunk2Size), (byte) (subChunk2Size >> 8), (byte) (subChunk2Size >> 16), + (byte) (subChunk2Size >> 24)}); + fos.write(wave.getBytes()); + fos.close(); + } catch (IOException e) { + log.error("",e); + } + } + + public Wave getWave() { + return wave; + } + + public void setWave(Wave wave) { + this.wave = wave; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/WaveHeader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/WaveHeader.java new file mode 100644 index 000000000..3f7af014a --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/WaveHeader.java @@ -0,0 +1,283 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.io.InputStream; + +/** + * WAV File Specification + * https://ccrma.stanford.edu/courses/422/projects/WaveFormat/ + * + * @author Jacquet Wong + */ +@Slf4j +public class WaveHeader { + + public static final String RIFF_HEADER = "RIFF"; + public static final String WAVE_HEADER = "WAVE"; + public static final String FMT_HEADER = "fmt "; + public static final String DATA_HEADER = "data"; + public static final int HEADER_BYTE_LENGTH = 44; // 44 bytes for header + + private boolean valid; + private String chunkId; // 4 bytes + private long chunkSize; // unsigned 4 bytes, little endian + private String format; // 4 bytes + private String subChunk1Id; // 4 bytes + private long subChunk1Size; // unsigned 4 bytes, little endian + private int audioFormat; // unsigned 2 bytes, little endian + private int channels; // unsigned 2 bytes, little endian + private long sampleRate; // unsigned 4 bytes, little endian + private long byteRate; // unsigned 4 bytes, little endian + private int blockAlign; // unsigned 2 bytes, little endian + private int bitsPerSample; // unsigned 2 bytes, little endian + private String subChunk2Id; // 4 bytes + private long subChunk2Size; // unsigned 4 bytes, little endian + + public WaveHeader() { + // init a 8k 16bit mono wav + chunkSize = 36; + subChunk1Size = 16; + audioFormat = 1; + channels = 1; + sampleRate = 8000; + byteRate = 16000; + blockAlign = 2; + bitsPerSample = 16; + subChunk2Size = 0; + valid = true; + } + + public WaveHeader(InputStream inputStream) { + valid = loadHeader(inputStream); + } + + private boolean loadHeader(InputStream inputStream) { + + byte[] headerBuffer = new byte[HEADER_BYTE_LENGTH]; + try { + inputStream.read(headerBuffer); + + // read header + int pointer = 0; + chunkId = new String(new byte[] {headerBuffer[pointer++], headerBuffer[pointer++], headerBuffer[pointer++], + headerBuffer[pointer++]}); + // little endian + chunkSize = (long) (headerBuffer[pointer++] & 0xff) | (long) (headerBuffer[pointer++] & 0xff) << 8 + | (long) (headerBuffer[pointer++] & 0xff) << 16 + | (long) (headerBuffer[pointer++] & 0xff << 24); + format = new String(new byte[] {headerBuffer[pointer++], headerBuffer[pointer++], headerBuffer[pointer++], + headerBuffer[pointer++]}); + subChunk1Id = new String(new byte[] {headerBuffer[pointer++], headerBuffer[pointer++], + headerBuffer[pointer++], headerBuffer[pointer++]}); + subChunk1Size = (long) (headerBuffer[pointer++] & 0xff) | (long) (headerBuffer[pointer++] & 0xff) << 8 + | (long) (headerBuffer[pointer++] & 0xff) << 16 + | (long) (headerBuffer[pointer++] & 0xff) << 24; + audioFormat = (int) ((headerBuffer[pointer++] & 0xff) | (headerBuffer[pointer++] & 0xff) << 8); + channels = (int) ((headerBuffer[pointer++] & 0xff) | (headerBuffer[pointer++] & 0xff) << 8); + sampleRate = (long) (headerBuffer[pointer++] & 0xff) | (long) (headerBuffer[pointer++] & 0xff) << 8 + | (long) (headerBuffer[pointer++] & 0xff) << 16 + | (long) (headerBuffer[pointer++] & 0xff) << 24; + byteRate = (long) (headerBuffer[pointer++] & 0xff) | (long) (headerBuffer[pointer++] & 0xff) << 8 + | (long) (headerBuffer[pointer++] & 0xff) << 16 + | (long) (headerBuffer[pointer++] & 0xff) << 24; + blockAlign = (int) ((headerBuffer[pointer++] & 0xff) | (headerBuffer[pointer++] & 0xff) << 8); + bitsPerSample = (int) ((headerBuffer[pointer++] & 0xff) | (headerBuffer[pointer++] & 0xff) << 8); + subChunk2Id = new String(new byte[] {headerBuffer[pointer++], headerBuffer[pointer++], + headerBuffer[pointer++], headerBuffer[pointer++]}); + subChunk2Size = (long) (headerBuffer[pointer++] & 0xff) | (long) (headerBuffer[pointer++] & 0xff) << 8 + | (long) (headerBuffer[pointer++] & 0xff) << 16 + | (long) (headerBuffer[pointer++] & 0xff) << 24; + // end read header + + // the inputStream should be closed outside this method + + // dis.close(); + + } catch (IOException e) { + log.error("",e); + return false; + } + + if (bitsPerSample != 8 && bitsPerSample != 16) { + System.err.println("WaveHeader: only supports bitsPerSample 8 or 16"); + return false; + } + + // check the format is support + if (chunkId.toUpperCase().equals(RIFF_HEADER) && format.toUpperCase().equals(WAVE_HEADER) && audioFormat == 1) { + return true; + } else { + System.err.println("WaveHeader: Unsupported header format"); + } + + return false; + } + + public boolean isValid() { + return valid; + } + + public String getChunkId() { + return chunkId; + } + + public long getChunkSize() { + return chunkSize; + } + + public String getFormat() { + return format; + } + + public String getSubChunk1Id() { + return subChunk1Id; + } + + public long getSubChunk1Size() { + return subChunk1Size; + } + + public int getAudioFormat() { + return audioFormat; + } + + public int getChannels() { + return channels; + } + + public int getSampleRate() { + return (int) sampleRate; + } + + public int getByteRate() { + return (int) byteRate; + } + + public int getBlockAlign() { + return blockAlign; + } + + public int getBitsPerSample() { + return bitsPerSample; + } + + public String getSubChunk2Id() { + return subChunk2Id; + } + + public long getSubChunk2Size() { + return subChunk2Size; + } + + public void setSampleRate(int sampleRate) { + int newSubChunk2Size = (int) (this.subChunk2Size * sampleRate / this.sampleRate); + // if num bytes for each sample is even, the size of newSubChunk2Size also needed to be in even number + if ((bitsPerSample / 8) % 2 == 0) { + if (newSubChunk2Size % 2 != 0) { + newSubChunk2Size++; + } + } + + this.sampleRate = sampleRate; + this.byteRate = sampleRate * bitsPerSample / 8; + this.chunkSize = newSubChunk2Size + 36; + this.subChunk2Size = newSubChunk2Size; + } + + public void setChunkId(String chunkId) { + this.chunkId = chunkId; + } + + public void setChunkSize(long chunkSize) { + this.chunkSize = chunkSize; + } + + public void setFormat(String format) { + this.format = format; + } + + public void setSubChunk1Id(String subChunk1Id) { + this.subChunk1Id = subChunk1Id; + } + + public void setSubChunk1Size(long subChunk1Size) { + this.subChunk1Size = subChunk1Size; + } + + public void setAudioFormat(int audioFormat) { + this.audioFormat = audioFormat; + } + + public void setChannels(int channels) { + this.channels = channels; + } + + public void setByteRate(long byteRate) { + this.byteRate = byteRate; + } + + public void setBlockAlign(int blockAlign) { + this.blockAlign = blockAlign; + } + + public void setBitsPerSample(int bitsPerSample) { + this.bitsPerSample = bitsPerSample; + } + + public void setSubChunk2Id(String subChunk2Id) { + this.subChunk2Id = subChunk2Id; + } + + public void setSubChunk2Size(long subChunk2Size) { + this.subChunk2Size = subChunk2Size; + } + + public String toString() { + + StringBuilder sb = new StringBuilder(); + sb.append("chunkId: " + chunkId); + sb.append("\n"); + sb.append("chunkSize: " + chunkSize); + sb.append("\n"); + sb.append("format: " + format); + sb.append("\n"); + sb.append("subChunk1Id: " + subChunk1Id); + sb.append("\n"); + sb.append("subChunk1Size: " + subChunk1Size); + sb.append("\n"); + sb.append("audioFormat: " + audioFormat); + sb.append("\n"); + sb.append("channels: " + channels); + sb.append("\n"); + sb.append("sampleRate: " + sampleRate); + sb.append("\n"); + sb.append("byteRate: " + byteRate); + sb.append("\n"); + sb.append("blockAlign: " + blockAlign); + sb.append("\n"); + sb.append("bitsPerSample: " + bitsPerSample); + sb.append("\n"); + sb.append("subChunk2Id: " + subChunk2Id); + sb.append("\n"); + sb.append("subChunk2Size: " + subChunk2Size); + return sb.toString(); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/FastFourierTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/FastFourierTransform.java new file mode 100644 index 000000000..a4f026b4e --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/FastFourierTransform.java @@ -0,0 +1,82 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.dsp; + +import org.jtransforms.fft.DoubleFFT_1D; + +/** + * FFT object, transform amplitudes to frequency intensities + * + * @author Jacquet Wong + */ +public class FastFourierTransform { + + /** + * Get the frequency intensities + * + * @param amplitudes amplitudes of the signal. Format depends on value of complex + * @param complex if true, amplitudes is assumed to be complex interlaced (re = even, im = odd), if false amplitudes + * are assumed to be real valued. + * @return intensities of each frequency unit: mag[frequency_unit]=intensity + */ + public double[] getMagnitudes(double[] amplitudes, boolean complex) { + + final int sampleSize = amplitudes.length; + final int nrofFrequencyBins = sampleSize / 2; + + + // call the fft and transform the complex numbers + if (complex) { + DoubleFFT_1D fft = new DoubleFFT_1D(nrofFrequencyBins); + fft.complexForward(amplitudes); + } else { + DoubleFFT_1D fft = new DoubleFFT_1D(sampleSize); + fft.realForward(amplitudes); + // amplitudes[1] contains re[sampleSize/2] or im[(sampleSize-1) / 2] (depending on whether sampleSize is odd or even) + // Discard it as it is useless without the other part + // im part dc bin is always 0 for real input + amplitudes[1] = 0; + } + // end call the fft and transform the complex numbers + + // even indexes (0,2,4,6,...) are real parts + // odd indexes (1,3,5,7,...) are img parts + double[] mag = new double[nrofFrequencyBins]; + for (int i = 0; i < nrofFrequencyBins; i++) { + final int f = 2 * i; + mag[i] = Math.sqrt(amplitudes[f] * amplitudes[f] + amplitudes[f + 1] * amplitudes[f + 1]); + } + + return mag; + } + + /** + * Get the frequency intensities. Backwards compatible with previous versions w.r.t to number of frequency bins. + * Use getMagnitudes(amplitudes, true) to get all bins. + * + * @param amplitudes complex-valued signal to transform. Even indexes are real and odd indexes are img + * @return intensities of each frequency unit: mag[frequency_unit]=intensity + */ + public double[] getMagnitudes(double[] amplitudes) { + double[] magnitudes = getMagnitudes(amplitudes, true); + + double[] halfOfMagnitudes = new double[magnitudes.length/2]; + System.arraycopy(magnitudes, 0,halfOfMagnitudes, 0, halfOfMagnitudes.length); + return halfOfMagnitudes; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/LinearInterpolation.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/LinearInterpolation.java new file mode 100644 index 000000000..bb4c3f789 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/LinearInterpolation.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.dsp; + +/** + * Construct new data points within the range of a discrete set of known data points by linear equation + * + * @author Jacquet Wong + */ +public class LinearInterpolation { + + public LinearInterpolation() { + + } + + /** + * Do interpolation on the samples according to the original and destinated sample rates + * + * @param oldSampleRate sample rate of the original samples + * @param newSampleRate sample rate of the interpolated samples + * @param samples original samples + * @return interpolated samples + */ + public short[] interpolate(int oldSampleRate, int newSampleRate, short[] samples) { + + if (oldSampleRate == newSampleRate) { + return samples; + } + + int newLength = Math.round(((float) samples.length / oldSampleRate * newSampleRate)); + float lengthMultiplier = (float) newLength / samples.length; + short[] interpolatedSamples = new short[newLength]; + + // interpolate the value by the linear equation y=mx+c + for (int i = 0; i < newLength; i++) { + + // get the nearest positions for the interpolated point + float currentPosition = i / lengthMultiplier; + int nearestLeftPosition = (int) currentPosition; + int nearestRightPosition = nearestLeftPosition + 1; + if (nearestRightPosition >= samples.length) { + nearestRightPosition = samples.length - 1; + } + + float slope = samples[nearestRightPosition] - samples[nearestLeftPosition]; // delta x is 1 + float positionFromLeft = currentPosition - nearestLeftPosition; + + interpolatedSamples[i] = (short) (slope * positionFromLeft + samples[nearestLeftPosition]); // y=mx+c + } + + return interpolatedSamples; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/Resampler.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/Resampler.java new file mode 100644 index 000000000..e0d79215f --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/Resampler.java @@ -0,0 +1,86 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.dsp; + +/** + * Resample signal data (base on bytes) + * + * @author jacquet + * + */ +public class Resampler { + + public Resampler() {} + + /** + * Do resampling. Currently the amplitude is stored by short such that maximum bitsPerSample is 16 (bytePerSample is 2) + * + * @param sourceData The source data in bytes + * @param bitsPerSample How many bits represents one sample (currently supports max. bitsPerSample=16) + * @param sourceRate Sample rate of the source data + * @param targetRate Sample rate of the target data + * @return re-sampled data + */ + public byte[] reSample(byte[] sourceData, int bitsPerSample, int sourceRate, int targetRate) { + + // make the bytes to amplitudes first + int bytePerSample = bitsPerSample / 8; + int numSamples = sourceData.length / bytePerSample; + short[] amplitudes = new short[numSamples]; // 16 bit, use a short to store + + int pointer = 0; + for (int i = 0; i < numSamples; i++) { + short amplitude = 0; + for (int byteNumber = 0; byteNumber < bytePerSample; byteNumber++) { + // little endian + amplitude |= (short) ((sourceData[pointer++] & 0xFF) << (byteNumber * 8)); + } + amplitudes[i] = amplitude; + } + // end make the amplitudes + + // do interpolation + LinearInterpolation reSample = new LinearInterpolation(); + short[] targetSample = reSample.interpolate(sourceRate, targetRate, amplitudes); + int targetLength = targetSample.length; + // end do interpolation + + // TODO: Remove the high frequency signals with a digital filter, leaving a signal containing only half-sample-rated frequency information, but still sampled at a rate of target sample rate. Usually FIR is used + + // end resample the amplitudes + + // convert the amplitude to bytes + byte[] bytes; + if (bytePerSample == 1) { + bytes = new byte[targetLength]; + for (int i = 0; i < targetLength; i++) { + bytes[i] = (byte) targetSample[i]; + } + } else { + // suppose bytePerSample==2 + bytes = new byte[targetLength * 2]; + for (int i = 0; i < targetSample.length; i++) { + // little endian + bytes[i * 2] = (byte) (targetSample[i] & 0xff); + bytes[i * 2 + 1] = (byte) ((targetSample[i] >> 8) & 0xff); + } + } + // end convert the amplitude to bytes + + return bytes; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/WindowFunction.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/WindowFunction.java new file mode 100644 index 000000000..c0d7b6253 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/WindowFunction.java @@ -0,0 +1,97 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.dsp; + +/** + * Window functions generator + * + * @author Jacquet Wong + * + */ +public class WindowFunction { + + public static final int RECTANGULAR = 0; + public static final int BARTLETT = 1; + public static final int HANNING = 2; + public static final int HAMMING = 3; + public static final int BLACKMAN = 4; + + int windowType = 0; // defaults to rectangular window + + public WindowFunction() {} + + public void setWindowType(int wt) { + windowType = wt; + } + + public void setWindowType(String w) { + if (w.toUpperCase().equals("RECTANGULAR")) + windowType = RECTANGULAR; + if (w.toUpperCase().equals("BARTLETT")) + windowType = BARTLETT; + if (w.toUpperCase().equals("HANNING")) + windowType = HANNING; + if (w.toUpperCase().equals("HAMMING")) + windowType = HAMMING; + if (w.toUpperCase().equals("BLACKMAN")) + windowType = BLACKMAN; + } + + public int getWindowType() { + return windowType; + } + + /** + * Generate a window + * + * @param nSamples size of the window + * @return window in array + */ + public double[] generate(int nSamples) { + // generate nSamples window function values + // for index values 0 .. nSamples - 1 + int m = nSamples / 2; + double r; + double pi = Math.PI; + double[] w = new double[nSamples]; + switch (windowType) { + case BARTLETT: // Bartlett (triangular) window + for (int n = 0; n < nSamples; n++) + w[n] = 1.0f - Math.abs(n - m) / m; + break; + case HANNING: // Hanning window + r = pi / (m + 1); + for (int n = -m; n < m; n++) + w[m + n] = 0.5f + 0.5f * Math.cos(n * r); + break; + case HAMMING: // Hamming window + r = pi / m; + for (int n = -m; n < m; n++) + w[m + n] = 0.54f + 0.46f * Math.cos(n * r); + break; + case BLACKMAN: // Blackman window + r = pi / m; + for (int n = -m; n < m; n++) + w[m + n] = 0.42f + 0.5f * Math.cos(n * r) + 0.08f * Math.cos(2 * n * r); + break; + default: // Rectangular window function + for (int n = 0; n < nSamples; n++) + w[n] = 1.0f; + } + return w; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/package-info.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/package-info.java new file mode 100644 index 000000000..269644ad8 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/dsp/package-info.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +/** + * Originally derived from musicg. Importing relevant snippets for working with basic audio data. + * + * https://code.google.com/p/musicg/ + * + * + */ +package org.datavec.audio.dsp; diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/extension/NormalizedSampleAmplitudes.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/extension/NormalizedSampleAmplitudes.java new file mode 100644 index 000000000..9a8eaba58 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/extension/NormalizedSampleAmplitudes.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.extension; + + +import org.datavec.audio.Wave; + +/** + * Handles the wave data in amplitude-time domain. + * + * @author Jacquet Wong + */ +public class NormalizedSampleAmplitudes { + + private Wave wave; + private double[] normalizedAmplitudes; // normalizedAmplitudes[sampleNumber]=normalizedAmplitudeInTheFrame + + public NormalizedSampleAmplitudes(Wave wave) { + this.wave = wave; + } + + /** + * + * Get normalized amplitude of each frame + * + * @return array of normalized amplitudes(signed 16 bit): normalizedAmplitudes[frame]=amplitude + */ + public double[] getNormalizedAmplitudes() { + + if (normalizedAmplitudes == null) { + + boolean signed = true; + + // usually 8bit is unsigned + if (wave.getWaveHeader().getBitsPerSample() == 8) { + signed = false; + } + + short[] amplitudes = wave.getSampleAmplitudes(); + int numSamples = amplitudes.length; + int maxAmplitude = 1 << (wave.getWaveHeader().getBitsPerSample() - 1); + + if (!signed) { // one more bit for unsigned value + maxAmplitude <<= 1; + } + + normalizedAmplitudes = new double[numSamples]; + for (int i = 0; i < numSamples; i++) { + normalizedAmplitudes[i] = (double) amplitudes[i] / maxAmplitude; + } + } + return normalizedAmplitudes; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/extension/Spectrogram.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/extension/Spectrogram.java new file mode 100644 index 000000000..fdc680e1d --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/extension/Spectrogram.java @@ -0,0 +1,215 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.extension; + + +import org.datavec.audio.Wave; +import org.datavec.audio.dsp.FastFourierTransform; +import org.datavec.audio.dsp.WindowFunction; + +/** + * Handles the wave data in frequency-time domain. + * + * @author Jacquet Wong + */ +public class Spectrogram { + + public static final int SPECTROGRAM_DEFAULT_FFT_SAMPLE_SIZE = 1024; + public static final int SPECTROGRAM_DEFAULT_OVERLAP_FACTOR = 0; // 0 for no overlapping + + private Wave wave; + private double[][] spectrogram; // relative spectrogram + private double[][] absoluteSpectrogram; // absolute spectrogram + private int fftSampleSize; // number of sample in fft, the value needed to be a number to power of 2 + private int overlapFactor; // 1/overlapFactor overlapping, e.g. 1/4=25% overlapping + private int numFrames; // number of frames of the spectrogram + private int framesPerSecond; // frame per second of the spectrogram + private int numFrequencyUnit; // number of y-axis unit + private double unitFrequency; // frequency per y-axis unit + + /** + * Constructor + * + * @param wave + */ + public Spectrogram(Wave wave) { + this.wave = wave; + // default + this.fftSampleSize = SPECTROGRAM_DEFAULT_FFT_SAMPLE_SIZE; + this.overlapFactor = SPECTROGRAM_DEFAULT_OVERLAP_FACTOR; + buildSpectrogram(); + } + + /** + * Constructor + * + * @param wave + * @param fftSampleSize number of sample in fft, the value needed to be a number to power of 2 + * @param overlapFactor 1/overlapFactor overlapping, e.g. 1/4=25% overlapping, 0 for no overlapping + */ + public Spectrogram(Wave wave, int fftSampleSize, int overlapFactor) { + this.wave = wave; + + if (Integer.bitCount(fftSampleSize) == 1) { + this.fftSampleSize = fftSampleSize; + } else { + System.err.print("The input number must be a power of 2"); + this.fftSampleSize = SPECTROGRAM_DEFAULT_FFT_SAMPLE_SIZE; + } + + this.overlapFactor = overlapFactor; + + buildSpectrogram(); + } + + /** + * Build spectrogram + */ + private void buildSpectrogram() { + + short[] amplitudes = wave.getSampleAmplitudes(); + int numSamples = amplitudes.length; + + int pointer = 0; + // overlapping + if (overlapFactor > 1) { + int numOverlappedSamples = numSamples * overlapFactor; + int backSamples = fftSampleSize * (overlapFactor - 1) / overlapFactor; + short[] overlapAmp = new short[numOverlappedSamples]; + pointer = 0; + for (int i = 0; i < amplitudes.length; i++) { + overlapAmp[pointer++] = amplitudes[i]; + if (pointer % fftSampleSize == 0) { + // overlap + i -= backSamples; + } + } + numSamples = numOverlappedSamples; + amplitudes = overlapAmp; + } + // end overlapping + + numFrames = numSamples / fftSampleSize; + framesPerSecond = (int) (numFrames / wave.length()); + + // set signals for fft + WindowFunction window = new WindowFunction(); + window.setWindowType("Hamming"); + double[] win = window.generate(fftSampleSize); + + double[][] signals = new double[numFrames][]; + for (int f = 0; f < numFrames; f++) { + signals[f] = new double[fftSampleSize]; + int startSample = f * fftSampleSize; + for (int n = 0; n < fftSampleSize; n++) { + signals[f][n] = amplitudes[startSample + n] * win[n]; + } + } + // end set signals for fft + + absoluteSpectrogram = new double[numFrames][]; + // for each frame in signals, do fft on it + FastFourierTransform fft = new FastFourierTransform(); + for (int i = 0; i < numFrames; i++) { + absoluteSpectrogram[i] = fft.getMagnitudes(signals[i], false); + } + + if (absoluteSpectrogram.length > 0) { + + numFrequencyUnit = absoluteSpectrogram[0].length; + unitFrequency = (double) wave.getWaveHeader().getSampleRate() / 2 / numFrequencyUnit; // frequency could be caught within the half of nSamples according to Nyquist theory + + // normalization of absoultSpectrogram + spectrogram = new double[numFrames][numFrequencyUnit]; + + // set max and min amplitudes + double maxAmp = Double.MIN_VALUE; + double minAmp = Double.MAX_VALUE; + for (int i = 0; i < numFrames; i++) { + for (int j = 0; j < numFrequencyUnit; j++) { + if (absoluteSpectrogram[i][j] > maxAmp) { + maxAmp = absoluteSpectrogram[i][j]; + } else if (absoluteSpectrogram[i][j] < minAmp) { + minAmp = absoluteSpectrogram[i][j]; + } + } + } + // end set max and min amplitudes + + // normalization + // avoiding divided by zero + double minValidAmp = 0.00000000001F; + if (minAmp == 0) { + minAmp = minValidAmp; + } + + double diff = Math.log10(maxAmp / minAmp); // perceptual difference + for (int i = 0; i < numFrames; i++) { + for (int j = 0; j < numFrequencyUnit; j++) { + if (absoluteSpectrogram[i][j] < minValidAmp) { + spectrogram[i][j] = 0; + } else { + spectrogram[i][j] = (Math.log10(absoluteSpectrogram[i][j] / minAmp)) / diff; + } + } + } + // end normalization + } + } + + /** + * Get spectrogram: spectrogram[time][frequency]=intensity + * + * @return logarithm normalized spectrogram + */ + public double[][] getNormalizedSpectrogramData() { + return spectrogram; + } + + /** + * Get spectrogram: spectrogram[time][frequency]=intensity + * + * @return absolute spectrogram + */ + public double[][] getAbsoluteSpectrogramData() { + return absoluteSpectrogram; + } + + public int getNumFrames() { + return numFrames; + } + + public int getFramesPerSecond() { + return framesPerSecond; + } + + public int getNumFrequencyUnit() { + return numFrequencyUnit; + } + + public double getUnitFrequency() { + return unitFrequency; + } + + public int getFftSampleSize() { + return fftSampleSize; + } + + public int getOverlapFactor() { + return overlapFactor; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintManager.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintManager.java new file mode 100644 index 000000000..efa481a91 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintManager.java @@ -0,0 +1,274 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + + +import lombok.extern.slf4j.Slf4j; +import org.datavec.audio.Wave; +import org.datavec.audio.WaveHeader; +import org.datavec.audio.dsp.Resampler; +import org.datavec.audio.extension.Spectrogram; +import org.datavec.audio.processor.TopManyPointsProcessorChain; +import org.datavec.audio.properties.FingerprintProperties; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +/** + * Audio fingerprint manager, handle fingerprint operations + * + * @author jacquet + * + */ +@Slf4j +public class FingerprintManager { + + private FingerprintProperties fingerprintProperties = FingerprintProperties.getInstance(); + private int sampleSizePerFrame = fingerprintProperties.getSampleSizePerFrame(); + private int overlapFactor = fingerprintProperties.getOverlapFactor(); + private int numRobustPointsPerFrame = fingerprintProperties.getNumRobustPointsPerFrame(); + private int numFilterBanks = fingerprintProperties.getNumFilterBanks(); + + /** + * Constructor + */ + public FingerprintManager() { + + } + + /** + * Extract fingerprint from Wave object + * + * @param wave Wave Object to be extracted fingerprint + * @return fingerprint in bytes + */ + public byte[] extractFingerprint(Wave wave) { + + int[][] coordinates; // coordinates[x][0..3]=y0..y3 + byte[] fingerprint = new byte[0]; + + // resample to target rate + Resampler resampler = new Resampler(); + int sourceRate = wave.getWaveHeader().getSampleRate(); + int targetRate = fingerprintProperties.getSampleRate(); + + byte[] resampledWaveData = resampler.reSample(wave.getBytes(), wave.getWaveHeader().getBitsPerSample(), + sourceRate, targetRate); + + // update the wave header + WaveHeader resampledWaveHeader = wave.getWaveHeader(); + resampledWaveHeader.setSampleRate(targetRate); + + // make resampled wave + Wave resampledWave = new Wave(resampledWaveHeader, resampledWaveData); + // end resample to target rate + + // get spectrogram's data + Spectrogram spectrogram = resampledWave.getSpectrogram(sampleSizePerFrame, overlapFactor); + double[][] spectorgramData = spectrogram.getNormalizedSpectrogramData(); + + List[] pointsLists = getRobustPointList(spectorgramData); + int numFrames = pointsLists.length; + + // prepare fingerprint bytes + coordinates = new int[numFrames][numRobustPointsPerFrame]; + + for (int x = 0; x < numFrames; x++) { + if (pointsLists[x].size() == numRobustPointsPerFrame) { + Iterator pointsListsIterator = pointsLists[x].iterator(); + for (int y = 0; y < numRobustPointsPerFrame; y++) { + coordinates[x][y] = pointsListsIterator.next(); + } + } else { + // use -1 to fill the empty byte + for (int y = 0; y < numRobustPointsPerFrame; y++) { + coordinates[x][y] = -1; + } + } + } + // end make fingerprint + + // for each valid coordinate, append with its intensity + List byteList = new LinkedList(); + for (int i = 0; i < numFrames; i++) { + for (int j = 0; j < numRobustPointsPerFrame; j++) { + if (coordinates[i][j] != -1) { + // first 2 bytes is x + byteList.add((byte) (i >> 8)); + byteList.add((byte) i); + + // next 2 bytes is y + int y = coordinates[i][j]; + byteList.add((byte) (y >> 8)); + byteList.add((byte) y); + + // next 4 bytes is intensity + int intensity = (int) (spectorgramData[i][y] * Integer.MAX_VALUE); // spectorgramData is ranged from 0~1 + byteList.add((byte) (intensity >> 24)); + byteList.add((byte) (intensity >> 16)); + byteList.add((byte) (intensity >> 8)); + byteList.add((byte) intensity); + } + } + } + // end for each valid coordinate, append with its intensity + + fingerprint = new byte[byteList.size()]; + Iterator byteListIterator = byteList.iterator(); + int pointer = 0; + while (byteListIterator.hasNext()) { + fingerprint[pointer++] = byteListIterator.next(); + } + + return fingerprint; + } + + /** + * Get bytes from fingerprint file + * + * @param fingerprintFile fingerprint filename + * @return fingerprint in bytes + */ + public byte[] getFingerprintFromFile(String fingerprintFile) { + byte[] fingerprint = null; + try { + InputStream fis = new FileInputStream(fingerprintFile); + fingerprint = getFingerprintFromInputStream(fis); + fis.close(); + } catch (IOException e) { + log.error("",e); + } + return fingerprint; + } + + /** + * Get bytes from fingerprint inputstream + * + * @param inputStream fingerprint inputstream + * @return fingerprint in bytes + */ + public byte[] getFingerprintFromInputStream(InputStream inputStream) { + byte[] fingerprint = null; + try { + fingerprint = new byte[inputStream.available()]; + inputStream.read(fingerprint); + } catch (IOException e) { + log.error("",e); + } + return fingerprint; + } + + /** + * Save fingerprint to a file + * + * @param fingerprint fingerprint bytes + * @param filename fingerprint filename + * @see FingerprintManager file saved + */ + public void saveFingerprintAsFile(byte[] fingerprint, String filename) { + + FileOutputStream fileOutputStream; + try { + fileOutputStream = new FileOutputStream(filename); + fileOutputStream.write(fingerprint); + fileOutputStream.close(); + } catch (IOException e) { + log.error("",e); + } + } + + // robustLists[x]=y1,y2,y3,... + private List[] getRobustPointList(double[][] spectrogramData) { + + int numX = spectrogramData.length; + int numY = spectrogramData[0].length; + + double[][] allBanksIntensities = new double[numX][numY]; + int bandwidthPerBank = numY / numFilterBanks; + + for (int b = 0; b < numFilterBanks; b++) { + + double[][] bankIntensities = new double[numX][bandwidthPerBank]; + + for (int i = 0; i < numX; i++) { + System.arraycopy(spectrogramData[i], b * bandwidthPerBank, bankIntensities[i], 0, bandwidthPerBank); + } + + // get the most robust point in each filter bank + TopManyPointsProcessorChain processorChain = new TopManyPointsProcessorChain(bankIntensities, 1); + double[][] processedIntensities = processorChain.getIntensities(); + + for (int i = 0; i < numX; i++) { + System.arraycopy(processedIntensities[i], 0, allBanksIntensities[i], b * bandwidthPerBank, + bandwidthPerBank); + } + } + + List robustPointList = new LinkedList(); + + // find robust points + for (int i = 0; i < allBanksIntensities.length; i++) { + for (int j = 0; j < allBanksIntensities[i].length; j++) { + if (allBanksIntensities[i][j] > 0) { + + int[] point = new int[] {i, j}; + //System.out.println(i+","+frequency); + robustPointList.add(point); + } + } + } + // end find robust points + + List[] robustLists = new LinkedList[spectrogramData.length]; + for (int i = 0; i < robustLists.length; i++) { + robustLists[i] = new LinkedList<>(); + } + + // robustLists[x]=y1,y2,y3,... + for (int[] coor : robustPointList) { + robustLists[coor[0]].add(coor[1]); + } + + // return the list per frame + return robustLists; + } + + /** + * Number of frames in a fingerprint + * Each frame lengths 8 bytes + * Usually there is more than one point in each frame, so it cannot simply divide the bytes length by 8 + * Last 8 byte of thisFingerprint is the last frame of this wave + * First 2 byte of the last 8 byte is the x position of this wave, i.e. (number_of_frames-1) of this wave + * + * @param fingerprint fingerprint bytes + * @return number of frames of the fingerprint + */ + public static int getNumFrames(byte[] fingerprint) { + + if (fingerprint.length < 8) { + return 0; + } + + // get the last x-coordinate (length-8&length-7)bytes from fingerprint + return ((fingerprint[fingerprint.length - 8] & 0xff) << 8 | (fingerprint[fingerprint.length - 7] & 0xff)) + 1; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintSimilarity.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintSimilarity.java new file mode 100644 index 000000000..c76756310 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintSimilarity.java @@ -0,0 +1,109 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + + +import org.datavec.audio.properties.FingerprintProperties; + +/** + * A class for fingerprint's similarity + * + * @author jacquet + * + */ +public class FingerprintSimilarity { + + private FingerprintProperties fingerprintProperties = FingerprintProperties.getInstance(); + private int mostSimilarFramePosition; + private float score; + private float similarity; + + /** + * Constructor + */ + public FingerprintSimilarity() { + mostSimilarFramePosition = Integer.MIN_VALUE; + score = -1; + similarity = -1; + } + + /** + * Get the most similar position in terms of frame number + * + * @return most similar frame position + */ + public int getMostSimilarFramePosition() { + return mostSimilarFramePosition; + } + + /** + * Set the most similar position in terms of frame number + * + * @param mostSimilarFramePosition + */ + public void setMostSimilarFramePosition(int mostSimilarFramePosition) { + this.mostSimilarFramePosition = mostSimilarFramePosition; + } + + /** + * Get the similarity of the fingerprints + * similarity from 0~1, which 0 means no similar feature is found and 1 means in average there is at least one match in every frame + * + * @return fingerprints similarity + */ + public float getSimilarity() { + return similarity; + } + + /** + * Set the similarity of the fingerprints + * + * @param similarity similarity + */ + public void setSimilarity(float similarity) { + this.similarity = similarity; + } + + /** + * Get the similarity score of the fingerprints + * Number of features found in the fingerprints per frame + * + * @return fingerprints similarity score + */ + public float getScore() { + return score; + } + + /** + * Set the similarity score of the fingerprints + * + * @param score + */ + public void setScore(float score) { + this.score = score; + } + + /** + * Get the most similar position in terms of time in second + * + * @return most similar starting time + */ + public float getsetMostSimilarTimePosition() { + return (float) mostSimilarFramePosition / fingerprintProperties.getNumRobustPointsPerFrame() + / fingerprintProperties.getFps(); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintSimilarityComputer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintSimilarityComputer.java new file mode 100644 index 000000000..222bb5e67 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/FingerprintSimilarityComputer.java @@ -0,0 +1,136 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +import java.util.HashMap; +import java.util.List; + +/** + * Compute the similarity of two fingerprints + * + * @author jacquet + * + */ +public class FingerprintSimilarityComputer { + + private FingerprintSimilarity fingerprintSimilarity; + byte[] fingerprint1, fingerprint2; + + /** + * Constructor, ready to compute the similarity of two fingerprints + * + * @param fingerprint1 + * @param fingerprint2 + */ + public FingerprintSimilarityComputer(byte[] fingerprint1, byte[] fingerprint2) { + + this.fingerprint1 = fingerprint1; + this.fingerprint2 = fingerprint2; + + fingerprintSimilarity = new FingerprintSimilarity(); + } + + /** + * Get fingerprint similarity of inout fingerprints + * + * @return fingerprint similarity object + */ + public FingerprintSimilarity getFingerprintsSimilarity() { + HashMap offset_Score_Table = new HashMap<>(); // offset_Score_Table + int numFrames; + float score = 0; + int mostSimilarFramePosition = Integer.MIN_VALUE; + + // one frame may contain several points, use the shorter one be the denominator + if (fingerprint1.length > fingerprint2.length) { + numFrames = FingerprintManager.getNumFrames(fingerprint2); + } else { + numFrames = FingerprintManager.getNumFrames(fingerprint1); + } + + // get the pairs + PairManager pairManager = new PairManager(); + HashMap> this_Pair_PositionList_Table = + pairManager.getPair_PositionList_Table(fingerprint1); + HashMap> compareWave_Pair_PositionList_Table = + pairManager.getPair_PositionList_Table(fingerprint2); + + for (Integer compareWaveHashNumber : compareWave_Pair_PositionList_Table.keySet()) { + // if the compareWaveHashNumber doesn't exist in both tables, no need to compare + if (!this_Pair_PositionList_Table.containsKey(compareWaveHashNumber) + || !compareWave_Pair_PositionList_Table.containsKey(compareWaveHashNumber)) { + continue; + } + + // for each compare hash number, get the positions + List wavePositionList = this_Pair_PositionList_Table.get(compareWaveHashNumber); + List compareWavePositionList = compareWave_Pair_PositionList_Table.get(compareWaveHashNumber); + + for (Integer thisPosition : wavePositionList) { + for (Integer compareWavePosition : compareWavePositionList) { + int offset = thisPosition - compareWavePosition; + if (offset_Score_Table.containsKey(offset)) { + offset_Score_Table.put(offset, offset_Score_Table.get(offset) + 1); + } else { + offset_Score_Table.put(offset, 1); + } + } + } + } + + // map rank + MapRank mapRank = new MapRankInteger(offset_Score_Table, false); + + // get the most similar positions and scores + List orderedKeyList = mapRank.getOrderedKeyList(100, true); + if (orderedKeyList.size() > 0) { + int key = orderedKeyList.get(0); + // get the highest score position + mostSimilarFramePosition = key; + score = offset_Score_Table.get(key); + + // accumulate the scores from neighbours + if (offset_Score_Table.containsKey(key - 1)) { + score += offset_Score_Table.get(key - 1) / 2; + } + if (offset_Score_Table.containsKey(key + 1)) { + score += offset_Score_Table.get(key + 1) / 2; + } + } + + /* + Iterator orderedKeyListIterator=orderedKeyList.iterator(); + while (orderedKeyListIterator.hasNext()){ + int offset=orderedKeyListIterator.next(); + System.out.println(offset+": "+offset_Score_Table.get(offset)); + } + */ + + score /= numFrames; + float similarity = score; + // similarity >1 means in average there is at least one match in every frame + if (similarity > 1) { + similarity = 1; + } + + fingerprintSimilarity.setMostSimilarFramePosition(mostSimilarFramePosition); + fingerprintSimilarity.setScore(score); + fingerprintSimilarity.setSimilarity(similarity); + + return fingerprintSimilarity; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRank.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRank.java new file mode 100644 index 000000000..3378f5c09 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRank.java @@ -0,0 +1,23 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +import java.util.List; + +public interface MapRank { + public List getOrderedKeyList(int numKeys, boolean sharpLimit); +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRankDouble.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRankDouble.java new file mode 100644 index 000000000..a24ba0959 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRankDouble.java @@ -0,0 +1,175 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +import java.util.*; +import java.util.Map.Entry; + +public class MapRankDouble implements MapRank { + + private Map map; + private boolean acsending = true; + + public MapRankDouble(Map map, boolean acsending) { + this.map = map; + this.acsending = acsending; + } + + public List getOrderedKeyList(int numKeys, boolean sharpLimit) { // if sharp limited, will return sharp numKeys, otherwise will return until the values not equals the exact key's value + + Set mapEntrySet = map.entrySet(); + List keyList = new LinkedList(); + + // if the numKeys is larger than map size, limit it + if (numKeys > map.size()) { + numKeys = map.size(); + } + // end if the numKeys is larger than map size, limit it + + if (map.size() > 0) { + double[] array = new double[map.size()]; + int count = 0; + + // get the pass values + Iterator mapIterator = mapEntrySet.iterator(); + while (mapIterator.hasNext()) { + Entry entry = mapIterator.next(); + array[count++] = (Double) entry.getValue(); + } + // end get the pass values + + int targetindex; + if (acsending) { + targetindex = numKeys; + } else { + targetindex = array.length - numKeys; + } + + double passValue = getOrderedValue(array, targetindex); // this value is the value of the numKey-th element + // get the passed keys and values + Map passedMap = new HashMap(); + List valueList = new LinkedList(); + mapIterator = mapEntrySet.iterator(); + + while (mapIterator.hasNext()) { + Entry entry = mapIterator.next(); + double value = (Double) entry.getValue(); + if ((acsending && value <= passValue) || (!acsending && value >= passValue)) { + passedMap.put(entry.getKey(), value); + valueList.add(value); + } + } + // end get the passed keys and values + + // sort the value list + Double[] listArr = new Double[valueList.size()]; + valueList.toArray(listArr); + Arrays.sort(listArr); + // end sort the value list + + // get the list of keys + int resultCount = 0; + int index; + if (acsending) { + index = 0; + } else { + index = listArr.length - 1; + } + + if (!sharpLimit) { + numKeys = listArr.length; + } + + while (true) { + double targetValue = (Double) listArr[index]; + Iterator passedMapIterator = passedMap.entrySet().iterator(); + while (passedMapIterator.hasNext()) { + Entry entry = passedMapIterator.next(); + if ((Double) entry.getValue() == targetValue) { + keyList.add(entry.getKey()); + passedMapIterator.remove(); + resultCount++; + break; + } + } + + if (acsending) { + index++; + } else { + index--; + } + + if (resultCount >= numKeys) { + break; + } + } + // end get the list of keys + } + + return keyList; + } + + private double getOrderedValue(double[] array, int index) { + locate(array, 0, array.length - 1, index); + return array[index]; + } + + // sort the partitions by quick sort, and locate the target index + private void locate(double[] array, int left, int right, int index) { + + int mid = (left + right) / 2; + //System.out.println(left+" to "+right+" ("+mid+")"); + + if (right == left) { + //System.out.println("* "+array[targetIndex]); + //result=array[targetIndex]; + return; + } + + if (left < right) { + double s = array[mid]; + int i = left - 1; + int j = right + 1; + + while (true) { + while (array[++i] < s); + while (array[--j] > s); + if (i >= j) + break; + swap(array, i, j); + } + + //System.out.println("2 parts: "+left+"-"+(i-1)+" and "+(j+1)+"-"+right); + + if (i > index) { + // the target index in the left partition + //System.out.println("left partition"); + locate(array, left, i - 1, index); + } else { + // the target index in the right partition + //System.out.println("right partition"); + locate(array, j + 1, right, index); + } + } + } + + private void swap(double[] array, int i, int j) { + double t = array[i]; + array[i] = array[j]; + array[j] = t; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRankInteger.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRankInteger.java new file mode 100644 index 000000000..befbcbe00 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/MapRankInteger.java @@ -0,0 +1,175 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +import java.util.*; +import java.util.Map.Entry; + +public class MapRankInteger implements MapRank { + + private Map map; + private boolean acsending = true; + + public MapRankInteger(Map map, boolean acsending) { + this.map = map; + this.acsending = acsending; + } + + public List getOrderedKeyList(int numKeys, boolean sharpLimit) { // if sharp limited, will return sharp numKeys, otherwise will return until the values not equals the exact key's value + + Set mapEntrySet = map.entrySet(); + List keyList = new LinkedList(); + + // if the numKeys is larger than map size, limit it + if (numKeys > map.size()) { + numKeys = map.size(); + } + // end if the numKeys is larger than map size, limit it + + if (map.size() > 0) { + int[] array = new int[map.size()]; + int count = 0; + + // get the pass values + Iterator mapIterator = mapEntrySet.iterator(); + while (mapIterator.hasNext()) { + Entry entry = mapIterator.next(); + array[count++] = (Integer) entry.getValue(); + } + // end get the pass values + + int targetindex; + if (acsending) { + targetindex = numKeys; + } else { + targetindex = array.length - numKeys; + } + + int passValue = getOrderedValue(array, targetindex); // this value is the value of the numKey-th element + // get the passed keys and values + Map passedMap = new HashMap(); + List valueList = new LinkedList(); + mapIterator = mapEntrySet.iterator(); + + while (mapIterator.hasNext()) { + Entry entry = mapIterator.next(); + int value = (Integer) entry.getValue(); + if ((acsending && value <= passValue) || (!acsending && value >= passValue)) { + passedMap.put(entry.getKey(), value); + valueList.add(value); + } + } + // end get the passed keys and values + + // sort the value list + Integer[] listArr = new Integer[valueList.size()]; + valueList.toArray(listArr); + Arrays.sort(listArr); + // end sort the value list + + // get the list of keys + int resultCount = 0; + int index; + if (acsending) { + index = 0; + } else { + index = listArr.length - 1; + } + + if (!sharpLimit) { + numKeys = listArr.length; + } + + while (true) { + int targetValue = (Integer) listArr[index]; + Iterator passedMapIterator = passedMap.entrySet().iterator(); + while (passedMapIterator.hasNext()) { + Entry entry = passedMapIterator.next(); + if ((Integer) entry.getValue() == targetValue) { + keyList.add(entry.getKey()); + passedMapIterator.remove(); + resultCount++; + break; + } + } + + if (acsending) { + index++; + } else { + index--; + } + + if (resultCount >= numKeys) { + break; + } + } + // end get the list of keys + } + + return keyList; + } + + private int getOrderedValue(int[] array, int index) { + locate(array, 0, array.length - 1, index); + return array[index]; + } + + // sort the partitions by quick sort, and locate the target index + private void locate(int[] array, int left, int right, int index) { + + int mid = (left + right) / 2; + //System.out.println(left+" to "+right+" ("+mid+")"); + + if (right == left) { + //System.out.println("* "+array[targetIndex]); + //result=array[targetIndex]; + return; + } + + if (left < right) { + int s = array[mid]; + int i = left - 1; + int j = right + 1; + + while (true) { + while (array[++i] < s); + while (array[--j] > s); + if (i >= j) + break; + swap(array, i, j); + } + + //System.out.println("2 parts: "+left+"-"+(i-1)+" and "+(j+1)+"-"+right); + + if (i > index) { + // the target index in the left partition + //System.out.println("left partition"); + locate(array, left, i - 1, index); + } else { + // the target index in the right partition + //System.out.println("right partition"); + locate(array, j + 1, right, index); + } + } + } + + private void swap(int[] array, int i, int j) { + int t = array[i]; + array[i] = array[j]; + array[j] = t; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/PairManager.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/PairManager.java new file mode 100644 index 000000000..ff18c34c9 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/PairManager.java @@ -0,0 +1,232 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + + + +import org.datavec.audio.properties.FingerprintProperties; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; + +/** + * Make pairs for the audio fingerprints, which a pair is used to group the same features together + * + * @author jacquet + * + */ +public class PairManager { + + FingerprintProperties fingerprintProperties = FingerprintProperties.getInstance(); + private int numFilterBanks = fingerprintProperties.getNumFilterBanks(); + private int bandwidthPerBank = fingerprintProperties.getNumFrequencyUnits() / numFilterBanks; + private int anchorPointsIntervalLength = fingerprintProperties.getAnchorPointsIntervalLength(); + private int numAnchorPointsPerInterval = fingerprintProperties.getNumAnchorPointsPerInterval(); + private int maxTargetZoneDistance = fingerprintProperties.getMaxTargetZoneDistance(); + private int numFrequencyUnits = fingerprintProperties.getNumFrequencyUnits(); + + private int maxPairs; + private boolean isReferencePairing; + private HashMap stopPairTable = new HashMap<>(); + + /** + * Constructor + */ + public PairManager() { + maxPairs = fingerprintProperties.getRefMaxActivePairs(); + isReferencePairing = true; + } + + /** + * Constructor, number of pairs of robust points depends on the parameter isReferencePairing + * no. of pairs of reference and sample can be different due to environmental influence of source + * @param isReferencePairing + */ + public PairManager(boolean isReferencePairing) { + if (isReferencePairing) { + maxPairs = fingerprintProperties.getRefMaxActivePairs(); + } else { + maxPairs = fingerprintProperties.getSampleMaxActivePairs(); + } + this.isReferencePairing = isReferencePairing; + } + + /** + * Get a pair-positionList table + * It's a hash map which the key is the hashed pair, and the value is list of positions + * That means the table stores the positions which have the same hashed pair + * + * @param fingerprint fingerprint bytes + * @return pair-positionList HashMap + */ + public HashMap> getPair_PositionList_Table(byte[] fingerprint) { + + List pairPositionList = getPairPositionList(fingerprint); + + // table to store pair:pos,pos,pos,...;pair2:pos,pos,pos,.... + HashMap> pair_positionList_table = new HashMap<>(); + + // get all pair_positions from list, use a table to collect the data group by pair hashcode + for (int[] pair_position : pairPositionList) { + //System.out.println(pair_position[0]+","+pair_position[1]); + + // group by pair-hashcode, i.e.: > + if (pair_positionList_table.containsKey(pair_position[0])) { + pair_positionList_table.get(pair_position[0]).add(pair_position[1]); + } else { + List positionList = new LinkedList<>(); + positionList.add(pair_position[1]); + pair_positionList_table.put(pair_position[0], positionList); + } + // end group by pair-hashcode, i.e.: > + } + // end get all pair_positions from list, use a table to collect the data group by pair hashcode + + return pair_positionList_table; + } + + // this return list contains: int[0]=pair_hashcode, int[1]=position + private List getPairPositionList(byte[] fingerprint) { + + int numFrames = FingerprintManager.getNumFrames(fingerprint); + + // table for paired frames + byte[] pairedFrameTable = new byte[numFrames / anchorPointsIntervalLength + 1]; // each second has numAnchorPointsPerSecond pairs only + // end table for paired frames + + List pairList = new LinkedList<>(); + List sortedCoordinateList = getSortedCoordinateList(fingerprint); + + for (int[] anchorPoint : sortedCoordinateList) { + int anchorX = anchorPoint[0]; + int anchorY = anchorPoint[1]; + int numPairs = 0; + + for (int[] aSortedCoordinateList : sortedCoordinateList) { + + if (numPairs >= maxPairs) { + break; + } + + if (isReferencePairing && pairedFrameTable[anchorX + / anchorPointsIntervalLength] >= numAnchorPointsPerInterval) { + break; + } + + int targetX = aSortedCoordinateList[0]; + int targetY = aSortedCoordinateList[1]; + + if (anchorX == targetX && anchorY == targetY) { + continue; + } + + // pair up the points + int x1, y1, x2, y2; // x2 always >= x1 + if (targetX >= anchorX) { + x2 = targetX; + y2 = targetY; + x1 = anchorX; + y1 = anchorY; + } else { + x2 = anchorX; + y2 = anchorY; + x1 = targetX; + y1 = targetY; + } + + // check target zone + if ((x2 - x1) > maxTargetZoneDistance) { + continue; + } + // end check target zone + + // check filter bank zone + if (!(y1 / bandwidthPerBank == y2 / bandwidthPerBank)) { + continue; // same filter bank should have equal value + } + // end check filter bank zone + + int pairHashcode = (x2 - x1) * numFrequencyUnits * numFrequencyUnits + y2 * numFrequencyUnits + y1; + + // stop list applied on sample pairing only + if (!isReferencePairing && stopPairTable.containsKey(pairHashcode)) { + numPairs++; // no reservation + continue; // escape this point only + } + // end stop list applied on sample pairing only + + // pass all rules + pairList.add(new int[] {pairHashcode, anchorX}); + pairedFrameTable[anchorX / anchorPointsIntervalLength]++; + numPairs++; + // end pair up the points + } + } + + return pairList; + } + + private List getSortedCoordinateList(byte[] fingerprint) { + // each point data is 8 bytes + // first 2 bytes is x + // next 2 bytes is y + // next 4 bytes is intensity + + // get all intensities + int numCoordinates = fingerprint.length / 8; + int[] intensities = new int[numCoordinates]; + for (int i = 0; i < numCoordinates; i++) { + int pointer = i * 8 + 4; + int intensity = (fingerprint[pointer] & 0xff) << 24 | (fingerprint[pointer + 1] & 0xff) << 16 + | (fingerprint[pointer + 2] & 0xff) << 8 | (fingerprint[pointer + 3] & 0xff); + intensities[i] = intensity; + } + + QuickSortIndexPreserved quicksort = new QuickSortIndexPreserved(intensities); + int[] sortIndexes = quicksort.getSortIndexes(); + + List sortedCoordinateList = new LinkedList<>(); + for (int i = sortIndexes.length - 1; i >= 0; i--) { + int pointer = sortIndexes[i] * 8; + int x = (fingerprint[pointer] & 0xff) << 8 | (fingerprint[pointer + 1] & 0xff); + int y = (fingerprint[pointer + 2] & 0xff) << 8 | (fingerprint[pointer + 3] & 0xff); + sortedCoordinateList.add(new int[] {x, y}); + } + return sortedCoordinateList; + } + + /** + * Convert hashed pair to bytes + * + * @param pairHashcode hashed pair + * @return byte array + */ + public static byte[] pairHashcodeToBytes(int pairHashcode) { + return new byte[] {(byte) (pairHashcode >> 8), (byte) pairHashcode}; + } + + /** + * Convert bytes to hased pair + * + * @param pairBytes + * @return hashed pair + */ + public static int pairBytesToHashcode(byte[] pairBytes) { + return (pairBytes[0] & 0xFF) << 8 | (pairBytes[1] & 0xFF); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSort.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSort.java new file mode 100644 index 000000000..aebb362d2 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSort.java @@ -0,0 +1,21 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +public abstract class QuickSort { + public abstract int[] getSortIndexes(); +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortDouble.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortDouble.java new file mode 100644 index 000000000..258cbc888 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortDouble.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +public class QuickSortDouble extends QuickSort { + + private int[] indexes; + private double[] array; + + public QuickSortDouble(double[] array) { + this.array = array; + indexes = new int[array.length]; + for (int i = 0; i < indexes.length; i++) { + indexes[i] = i; + } + } + + public int[] getSortIndexes() { + sort(); + return indexes; + } + + private void sort() { + quicksort(array, indexes, 0, indexes.length - 1); + } + + // quicksort a[left] to a[right] + private void quicksort(double[] a, int[] indexes, int left, int right) { + if (right <= left) + return; + int i = partition(a, indexes, left, right); + quicksort(a, indexes, left, i - 1); + quicksort(a, indexes, i + 1, right); + } + + // partition a[left] to a[right], assumes left < right + private int partition(double[] a, int[] indexes, int left, int right) { + int i = left - 1; + int j = right; + while (true) { + while (a[indexes[++i]] < a[indexes[right]]); // find item on left to swap, a[right] acts as sentinel + while (a[indexes[right]] < a[indexes[--j]]) { // find item on right to swap + if (j == left) + break; // don't go out-of-bounds + } + if (i >= j) + break; // check if pointers cross + swap(a, indexes, i, j); // swap two elements into place + } + swap(a, indexes, i, right); // swap with partition element + return i; + } + + // exchange a[i] and a[j] + private void swap(double[] a, int[] indexes, int i, int j) { + int swap = indexes[i]; + indexes[i] = indexes[j]; + indexes[j] = swap; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortIndexPreserved.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortIndexPreserved.java new file mode 100644 index 000000000..61e391d71 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortIndexPreserved.java @@ -0,0 +1,39 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +public class QuickSortIndexPreserved { + + private QuickSort quickSort; + + public QuickSortIndexPreserved(int[] array) { + quickSort = new QuickSortInteger(array); + } + + public QuickSortIndexPreserved(double[] array) { + quickSort = new QuickSortDouble(array); + } + + public QuickSortIndexPreserved(short[] array) { + quickSort = new QuickSortShort(array); + } + + public int[] getSortIndexes() { + return quickSort.getSortIndexes(); + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortInteger.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortInteger.java new file mode 100644 index 000000000..178553865 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortInteger.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +public class QuickSortInteger extends QuickSort { + + private int[] indexes; + private int[] array; + + public QuickSortInteger(int[] array) { + this.array = array; + indexes = new int[array.length]; + for (int i = 0; i < indexes.length; i++) { + indexes[i] = i; + } + } + + public int[] getSortIndexes() { + sort(); + return indexes; + } + + private void sort() { + quicksort(array, indexes, 0, indexes.length - 1); + } + + // quicksort a[left] to a[right] + private void quicksort(int[] a, int[] indexes, int left, int right) { + if (right <= left) + return; + int i = partition(a, indexes, left, right); + quicksort(a, indexes, left, i - 1); + quicksort(a, indexes, i + 1, right); + } + + // partition a[left] to a[right], assumes left < right + private int partition(int[] a, int[] indexes, int left, int right) { + int i = left - 1; + int j = right; + while (true) { + while (a[indexes[++i]] < a[indexes[right]]); // find item on left to swap, a[right] acts as sentinel + while (a[indexes[right]] < a[indexes[--j]]) { // find item on right to swap + if (j == left) + break; // don't go out-of-bounds + } + if (i >= j) + break; // check if pointers cross + swap(a, indexes, i, j); // swap two elements into place + } + swap(a, indexes, i, right); // swap with partition element + return i; + } + + // exchange a[i] and a[j] + private void swap(int[] a, int[] indexes, int i, int j) { + int swap = indexes[i]; + indexes[i] = indexes[j]; + indexes[j] = swap; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortShort.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortShort.java new file mode 100644 index 000000000..8b4324b7e --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/fingerprint/QuickSortShort.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.fingerprint; + +public class QuickSortShort extends QuickSort { + + private int[] indexes; + private short[] array; + + public QuickSortShort(short[] array) { + this.array = array; + indexes = new int[array.length]; + for (int i = 0; i < indexes.length; i++) { + indexes[i] = i; + } + } + + public int[] getSortIndexes() { + sort(); + return indexes; + } + + private void sort() { + quicksort(array, indexes, 0, indexes.length - 1); + } + + // quicksort a[left] to a[right] + private void quicksort(short[] a, int[] indexes, int left, int right) { + if (right <= left) + return; + int i = partition(a, indexes, left, right); + quicksort(a, indexes, left, i - 1); + quicksort(a, indexes, i + 1, right); + } + + // partition a[left] to a[right], assumes left < right + private int partition(short[] a, int[] indexes, int left, int right) { + int i = left - 1; + int j = right; + while (true) { + while (a[indexes[++i]] < a[indexes[right]]); // find item on left to swap, a[right] acts as sentinel + while (a[indexes[right]] < a[indexes[--j]]) { // find item on right to swap + if (j == left) + break; // don't go out-of-bounds + } + if (i >= j) + break; // check if pointers cross + swap(a, indexes, i, j); // swap two elements into place + } + swap(a, indexes, i, right); // swap with partition element + return i; + } + + // exchange a[i] and a[j] + private void swap(short[] a, int[] indexes, int i, int j) { + int swap = indexes[i]; + indexes[i] = indexes[j]; + indexes[j] = swap; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/formats/input/WavInputFormat.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/formats/input/WavInputFormat.java new file mode 100644 index 000000000..6b51e3c44 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/formats/input/WavInputFormat.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.formats.input; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.formats.input.BaseInputFormat; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.audio.recordreader.WavFileRecordReader; + +import java.io.IOException; + +/** + * + * Wave file input format + * + * @author Adam Gibson + */ +public class WavInputFormat extends BaseInputFormat { + @Override + public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException { + return createReader(split); + } + + @Override + public RecordReader createReader(InputSplit split) throws IOException, InterruptedException { + RecordReader waveRecordReader = new WavFileRecordReader(); + waveRecordReader.initialize(split); + return waveRecordReader; + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/formats/output/WaveOutputFormat.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/formats/output/WaveOutputFormat.java new file mode 100644 index 000000000..be9d17bd4 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/formats/output/WaveOutputFormat.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.formats.output; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.exceptions.DataVecException; +import org.datavec.api.formats.output.OutputFormat; +import org.datavec.api.records.writer.RecordWriter; + +/** + * @author Adam Gibson + */ +public class WaveOutputFormat implements OutputFormat { + @Override + public RecordWriter createWriter(Configuration conf) throws DataVecException { + return null; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/ArrayRankDouble.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/ArrayRankDouble.java new file mode 100644 index 000000000..a4fcbbc6d --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/ArrayRankDouble.java @@ -0,0 +1,135 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.processor; + +public class ArrayRankDouble { + + /** + * Get the index position of maximum value the given array + * @param array an array + * @return index of the max value in array + */ + public int getMaxValueIndex(double[] array) { + + int index = 0; + double max = Integer.MIN_VALUE; + + for (int i = 0; i < array.length; i++) { + if (array[i] > max) { + max = array[i]; + index = i; + } + } + + return index; + } + + /** + * Get the index position of minimum value in the given array + * @param array an array + * @return index of the min value in array + */ + public int getMinValueIndex(double[] array) { + + int index = 0; + double min = Integer.MAX_VALUE; + + for (int i = 0; i < array.length; i++) { + if (array[i] < min) { + min = array[i]; + index = i; + } + } + + return index; + } + + /** + * Get the n-th value in the array after sorted + * @param array an array + * @param n position in array + * @param ascending is ascending order or not + * @return value at nth position of array + */ + public double getNthOrderedValue(double[] array, int n, boolean ascending) { + + if (n > array.length) { + n = array.length; + } + + int targetindex; + if (ascending) { + targetindex = n; + } else { + targetindex = array.length - n; + } + + // this value is the value of the numKey-th element + + return getOrderedValue(array, targetindex); + } + + private double getOrderedValue(double[] array, int index) { + locate(array, 0, array.length - 1, index); + return array[index]; + } + + // sort the partitions by quick sort, and locate the target index + private void locate(double[] array, int left, int right, int index) { + + int mid = (left + right) / 2; + // System.out.println(left+" to "+right+" ("+mid+")"); + + if (right == left) { + // System.out.println("* "+array[targetIndex]); + // result=array[targetIndex]; + return; + } + + if (left < right) { + double s = array[mid]; + int i = left - 1; + int j = right + 1; + + while (true) { + while (array[++i] < s); + while (array[--j] > s); + if (i >= j) + break; + swap(array, i, j); + } + + // System.out.println("2 parts: "+left+"-"+(i-1)+" and "+(j+1)+"-"+right); + + if (i > index) { + // the target index in the left partition + // System.out.println("left partition"); + locate(array, left, i - 1, index); + } else { + // the target index in the right partition + // System.out.println("right partition"); + locate(array, j + 1, right, index); + } + } + } + + private void swap(double[] array, int i, int j) { + double t = array[i]; + array[i] = array[j]; + array[j] = t; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/IntensityProcessor.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/IntensityProcessor.java new file mode 100644 index 000000000..083ac4765 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/IntensityProcessor.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.processor; + +public interface IntensityProcessor { + + public void execute(); + + public double[][] getIntensities(); +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/ProcessorChain.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/ProcessorChain.java new file mode 100644 index 000000000..c081e309a --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/ProcessorChain.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.processor; + +import java.util.LinkedList; +import java.util.List; + +public class ProcessorChain { + + private double[][] intensities; + List processorList = new LinkedList(); + + public ProcessorChain(double[][] intensities) { + this.intensities = intensities; + RobustIntensityProcessor robustProcessor = new RobustIntensityProcessor(intensities, 1); + processorList.add(robustProcessor); + process(); + } + + private void process() { + for (IntensityProcessor processor : processorList) { + processor.execute(); + intensities = processor.getIntensities(); + } + } + + public double[][] getIntensities() { + return intensities; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/RobustIntensityProcessor.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/RobustIntensityProcessor.java new file mode 100644 index 000000000..1d884855c --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/RobustIntensityProcessor.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.processor; + + +public class RobustIntensityProcessor implements IntensityProcessor { + + private double[][] intensities; + private int numPointsPerFrame; + + public RobustIntensityProcessor(double[][] intensities, int numPointsPerFrame) { + this.intensities = intensities; + this.numPointsPerFrame = numPointsPerFrame; + } + + public void execute() { + + int numX = intensities.length; + int numY = intensities[0].length; + double[][] processedIntensities = new double[numX][numY]; + + for (int i = 0; i < numX; i++) { + double[] tmpArray = new double[numY]; + System.arraycopy(intensities[i], 0, tmpArray, 0, numY); + + // pass value is the last some elements in sorted array + ArrayRankDouble arrayRankDouble = new ArrayRankDouble(); + double passValue = arrayRankDouble.getNthOrderedValue(tmpArray, numPointsPerFrame, false); + + // only passed elements will be assigned a value + for (int j = 0; j < numY; j++) { + if (intensities[i][j] >= passValue) { + processedIntensities[i][j] = intensities[i][j]; + } + } + } + intensities = processedIntensities; + } + + public double[][] getIntensities() { + return intensities; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/TopManyPointsProcessorChain.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/TopManyPointsProcessorChain.java new file mode 100644 index 000000000..e5a742b01 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/processor/TopManyPointsProcessorChain.java @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.processor; + +import java.util.LinkedList; +import java.util.List; + + +public class TopManyPointsProcessorChain { + + private double[][] intensities; + List processorList = new LinkedList<>(); + + public TopManyPointsProcessorChain(double[][] intensities, int numPoints) { + this.intensities = intensities; + RobustIntensityProcessor robustProcessor = new RobustIntensityProcessor(intensities, numPoints); + processorList.add(robustProcessor); + process(); + } + + private void process() { + for (IntensityProcessor processor : processorList) { + processor.execute(); + intensities = processor.getIntensities(); + } + } + + public double[][] getIntensities() { + return intensities; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/properties/FingerprintProperties.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/properties/FingerprintProperties.java new file mode 100644 index 000000000..db69a0b36 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/properties/FingerprintProperties.java @@ -0,0 +1,117 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.properties; + +public class FingerprintProperties { + + protected static FingerprintProperties instance = null; + + private int numRobustPointsPerFrame = 4; // number of points in each frame, i.e. top 4 intensities in fingerprint + private int sampleSizePerFrame = 2048; // number of audio samples in a frame, it is suggested to be the FFT Size + private int overlapFactor = 4; // 8 means each move 1/8 nSample length. 1 means no overlap, better 1,2,4,8 ... 32 + private int numFilterBanks = 4; + + private int upperBoundedFrequency = 1500; // low pass + private int lowerBoundedFrequency = 400; // high pass + private int fps = 5; // in order to have 5fps with 2048 sampleSizePerFrame, wave's sample rate need to be 10240 (sampleSizePerFrame*fps) + private int sampleRate = sampleSizePerFrame * fps; // the audio's sample rate needed to resample to this in order to fit the sampleSizePerFrame and fps + private int numFramesInOneSecond = overlapFactor * fps; // since the overlap factor affects the actual number of fps, so this value is used to evaluate how many frames in one second eventually + + private int refMaxActivePairs = 1; // max. active pairs per anchor point for reference songs + private int sampleMaxActivePairs = 10; // max. active pairs per anchor point for sample clip + private int numAnchorPointsPerInterval = 10; + private int anchorPointsIntervalLength = 4; // in frames (5fps,4 overlap per second) + private int maxTargetZoneDistance = 4; // in frame (5fps,4 overlap per second) + + private int numFrequencyUnits = (upperBoundedFrequency - lowerBoundedFrequency + 1) / fps + 1; // num frequency units + + public static FingerprintProperties getInstance() { + if (instance == null) { + synchronized (FingerprintProperties.class) { + if (instance == null) { + instance = new FingerprintProperties(); + } + } + } + return instance; + } + + public int getNumRobustPointsPerFrame() { + return numRobustPointsPerFrame; + } + + public int getSampleSizePerFrame() { + return sampleSizePerFrame; + } + + public int getOverlapFactor() { + return overlapFactor; + } + + public int getNumFilterBanks() { + return numFilterBanks; + } + + public int getUpperBoundedFrequency() { + return upperBoundedFrequency; + } + + public int getLowerBoundedFrequency() { + return lowerBoundedFrequency; + } + + public int getFps() { + return fps; + } + + public int getRefMaxActivePairs() { + return refMaxActivePairs; + } + + public int getSampleMaxActivePairs() { + return sampleMaxActivePairs; + } + + public int getNumAnchorPointsPerInterval() { + return numAnchorPointsPerInterval; + } + + public int getAnchorPointsIntervalLength() { + return anchorPointsIntervalLength; + } + + public int getMaxTargetZoneDistance() { + return maxTargetZoneDistance; + } + + public int getNumFrequencyUnits() { + return numFrequencyUnits; + } + + public int getMaxPossiblePairHashcode() { + return maxTargetZoneDistance * numFrequencyUnits * numFrequencyUnits + numFrequencyUnits * numFrequencyUnits + + numFrequencyUnits; + } + + public int getSampleRate() { + return sampleRate; + } + + public int getNumFramesInOneSecond() { + return numFramesInOneSecond; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java new file mode 100644 index 000000000..82c9bb1ce --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/BaseAudioRecordReader.java @@ -0,0 +1,221 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.recordreader; + +import org.apache.commons.io.FileUtils; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.reader.BaseRecordReader; +import org.datavec.api.split.BaseInputSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.split.InputStreamInputSplit; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Writable; + +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * Base audio file loader + * @author Adam Gibson + */ +public abstract class BaseAudioRecordReader extends BaseRecordReader { + private Iterator iter; + private List record; + private boolean hitImage = false; + private boolean appendLabel = false; + private List labels = new ArrayList<>(); + private Configuration conf; + protected InputSplit inputSplit; + + public BaseAudioRecordReader() {} + + public BaseAudioRecordReader(boolean appendLabel, List labels) { + this.appendLabel = appendLabel; + this.labels = labels; + } + + public BaseAudioRecordReader(List labels) { + this.labels = labels; + } + + public BaseAudioRecordReader(boolean appendLabel) { + this.appendLabel = appendLabel; + } + + protected abstract List loadData(File file, InputStream inputStream) throws IOException; + + @Override + public void initialize(InputSplit split) throws IOException, InterruptedException { + inputSplit = split; + if (split instanceof BaseInputSplit) { + URI[] locations = split.locations(); + if (locations != null && locations.length >= 1) { + if (locations.length > 1) { + List allFiles = new ArrayList<>(); + for (URI location : locations) { + File iter = new File(location); + if (iter.isDirectory()) { + Iterator allFiles2 = FileUtils.iterateFiles(iter, null, true); + while (allFiles2.hasNext()) + allFiles.add(allFiles2.next()); + } + + else + allFiles.add(iter); + } + + iter = allFiles.iterator(); + } else { + File curr = new File(locations[0]); + if (curr.isDirectory()) + iter = FileUtils.iterateFiles(curr, null, true); + else + iter = Collections.singletonList(curr).iterator(); + } + } + } + + + else if (split instanceof InputStreamInputSplit) { + record = new ArrayList<>(); + InputStreamInputSplit split2 = (InputStreamInputSplit) split; + InputStream is = split2.getIs(); + URI[] locations = split2.locations(); + if (appendLabel) { + Path path = Paths.get(locations[0]); + String parent = path.getParent().toString(); + record.add(new DoubleWritable(labels.indexOf(parent))); + } + + is.close(); + } + + } + + @Override + public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException { + this.conf = conf; + this.appendLabel = conf.getBoolean(APPEND_LABEL, false); + this.labels = new ArrayList<>(conf.getStringCollection(LABELS)); + initialize(split); + } + + @Override + public List next() { + if (iter != null) { + File next = iter.next(); + invokeListeners(next); + try { + return loadData(next, null); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else if (record != null) { + hitImage = true; + return record; + } + + throw new IllegalStateException("Indeterminant state: record must not be null, or a file iterator must exist"); + } + + @Override + public boolean hasNext() { + if (iter != null) { + return iter.hasNext(); + } else if (record != null) { + return !hitImage; + } + throw new IllegalStateException("Indeterminant state: record must not be null, or a file iterator must exist"); + } + + + @Override + public void close() throws IOException { + + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public List getLabels() { + return null; + } + + + @Override + public void reset() { + if (inputSplit == null) + throw new UnsupportedOperationException("Cannot reset without first initializing"); + try { + initialize(inputSplit); + } catch (Exception e) { + throw new RuntimeException("Error during LineRecordReader reset", e); + } + } + + @Override + public boolean resetSupported(){ + if(inputSplit == null){ + return false; + } + return inputSplit.resetSupported(); + } + + @Override + public List record(URI uri, DataInputStream dataInputStream) throws IOException { + invokeListeners(uri); + try { + return loadData(null, dataInputStream); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public Record nextRecord() { + return new org.datavec.api.records.impl.Record(next(), null); + } + + @Override + public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException { + throw new UnsupportedOperationException("Loading from metadata not yet implemented"); + } + + @Override + public List loadFromMetaData(List recordMetaDatas) throws IOException { + throw new UnsupportedOperationException("Loading from metadata not yet implemented"); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/NativeAudioRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/NativeAudioRecordReader.java new file mode 100644 index 000000000..c2a049b9e --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/NativeAudioRecordReader.java @@ -0,0 +1,72 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.recordreader; + +import org.bytedeco.javacv.FFmpegFrameGrabber; +import org.bytedeco.javacv.Frame; +import org.datavec.api.writable.FloatWritable; +import org.datavec.api.writable.Writable; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.FloatBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.bytedeco.ffmpeg.global.avutil.AV_SAMPLE_FMT_FLT; + +/** + * Native audio file loader using FFmpeg. + * + * @author saudet + */ +public class NativeAudioRecordReader extends BaseAudioRecordReader { + + public NativeAudioRecordReader() {} + + public NativeAudioRecordReader(boolean appendLabel, List labels) { + super(appendLabel, labels); + } + + public NativeAudioRecordReader(List labels) { + super(labels); + } + + public NativeAudioRecordReader(boolean appendLabel) { + super(appendLabel); + } + + protected List loadData(File file, InputStream inputStream) throws IOException { + List ret = new ArrayList<>(); + try (FFmpegFrameGrabber grabber = inputStream != null ? new FFmpegFrameGrabber(inputStream) + : new FFmpegFrameGrabber(file.getAbsolutePath())) { + grabber.setSampleFormat(AV_SAMPLE_FMT_FLT); + grabber.start(); + Frame frame; + while ((frame = grabber.grab()) != null) { + while (frame.samples != null && frame.samples[0].hasRemaining()) { + for (int i = 0; i < frame.samples.length; i++) { + ret.add(new FloatWritable(((FloatBuffer) frame.samples[i]).get())); + } + } + } + } + return ret; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/WavFileRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/WavFileRecordReader.java new file mode 100644 index 000000000..e0fb22bbb --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/main/java/org/datavec/audio/recordreader/WavFileRecordReader.java @@ -0,0 +1,53 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio.recordreader; + +import org.datavec.api.util.RecordUtils; +import org.datavec.api.writable.Writable; +import org.datavec.audio.Wave; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +/** + * Wav file loader + * @author Adam Gibson + */ +public class WavFileRecordReader extends BaseAudioRecordReader { + + public WavFileRecordReader() {} + + public WavFileRecordReader(boolean appendLabel, List labels) { + super(appendLabel, labels); + } + + public WavFileRecordReader(List labels) { + super(labels); + } + + public WavFileRecordReader(boolean appendLabel) { + super(appendLabel); + } + + protected List loadData(File file, InputStream inputStream) throws IOException { + Wave wave = inputStream != null ? new Wave(inputStream) : new Wave(file.getAbsolutePath()); + return RecordUtils.toRecord(wave.getNormalizedAmplitudes()); + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..20cc1ee1f --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/AssertTestsExtendBaseClass.java @@ -0,0 +1,55 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.audio; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + public long getTimeoutMilliseconds() { + return 60000; + } + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.audio"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/AudioReaderTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/AudioReaderTest.java new file mode 100644 index 000000000..126f3566c --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/AudioReaderTest.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio; + +import org.bytedeco.javacv.FFmpegFrameRecorder; +import org.bytedeco.javacv.Frame; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.Writable; +import org.datavec.audio.recordreader.NativeAudioRecordReader; + +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; + +import java.io.File; +import java.nio.ShortBuffer; +import java.util.List; + +import static org.bytedeco.ffmpeg.global.avcodec.AV_CODEC_ID_VORBIS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author saudet + */ +public class AudioReaderTest extends BaseND4JTest { + //@Ignore + @Test + public void testNativeAudioReader() throws Exception { + File tempFile = File.createTempFile("testNativeAudioReader", ".ogg"); + FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(tempFile, 2); + recorder.setAudioCodec(AV_CODEC_ID_VORBIS); + recorder.setSampleRate(44100); + recorder.start(); + Frame audioFrame = new Frame(); + ShortBuffer audioBuffer = ShortBuffer.allocate(64 * 1024); + audioFrame.sampleRate = 44100; + audioFrame.audioChannels = 2; + audioFrame.samples = new ShortBuffer[] {audioBuffer}; + recorder.record(audioFrame); + recorder.stop(); + recorder.release(); + + RecordReader reader = new NativeAudioRecordReader(); + reader.initialize(new FileSplit(tempFile)); + assertTrue(reader.hasNext()); + List record = reader.next(); + assertEquals(audioBuffer.limit(), record.size()); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/TestFastFourierTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/TestFastFourierTransform.java new file mode 100644 index 000000000..7eb32fc3b --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-audio/src/test/java/org/datavec/audio/TestFastFourierTransform.java @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.audio; + +import org.datavec.audio.dsp.FastFourierTransform; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; + +public class TestFastFourierTransform extends BaseND4JTest { + + @Test + public void testFastFourierTransformComplex() { + FastFourierTransform fft = new FastFourierTransform(); + double[] amplitudes = new double[] {3.0, 4.0, 0.5, 7.8, 6.9, -6.5, 8.5, 4.6}; + double[] frequencies = fft.getMagnitudes(amplitudes); + + Assertions.assertEquals(2, frequencies.length); + Assertions.assertArrayEquals(new double[] {21.335, 18.513}, frequencies, 0.005); + } + + @Test + public void testFastFourierTransformComplexLong() { + FastFourierTransform fft = new FastFourierTransform(); + double[] amplitudes = new double[] {3.0, 4.0, 0.5, 7.8, 6.9, -6.5, 8.5, 4.6}; + double[] frequencies = fft.getMagnitudes(amplitudes, true); + + Assertions.assertEquals(4, frequencies.length); + Assertions.assertArrayEquals(new double[] {21.335, 18.5132, 14.927, 7.527}, frequencies, 0.005); + } + + @Test + public void testFastFourierTransformReal() { + FastFourierTransform fft = new FastFourierTransform(); + double[] amplitudes = new double[] {3.0, 4.0, 0.5, 7.8, 6.9, -6.5, 8.5, 4.6}; + double[] frequencies = fft.getMagnitudes(amplitudes, false); + + Assertions.assertEquals(4, frequencies.length); + Assertions.assertArrayEquals(new double[] {28.8, 2.107, 14.927, 19.874}, frequencies, 0.005); + } + + @Test + public void testFastFourierTransformRealOddSize() { + FastFourierTransform fft = new FastFourierTransform(); + double[] amplitudes = new double[] {3.0, 4.0, 0.5, 7.8, 6.9, -6.5, 8.5}; + double[] frequencies = fft.getMagnitudes(amplitudes, false); + + Assertions.assertEquals(3, frequencies.length); + Assertions.assertArrayEquals(new double[] {24.2, 3.861, 16.876}, frequencies, 0.005); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/build.gradle b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/build.gradle new file mode 100644 index 000000000..b5de498e2 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/build.gradle @@ -0,0 +1,47 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDnn.cavisDnnApi + + implementation "org.bytedeco:javacv" + implementation "org.apache.commons:commons-compress" + implementation "org.jcodec:jcodec:0.1.5" + + implementation "org.slf4j:slf4j-api" + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + + /* + + + + */ +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/format/input/CodecInputFormat.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/format/input/CodecInputFormat.java new file mode 100644 index 000000000..118902b70 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/format/input/CodecInputFormat.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.codec.format.input; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.formats.input.BaseInputFormat; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.codec.reader.CodecRecordReader; + +import java.io.IOException; + +/** + * @author Adam Gibson + */ +public class CodecInputFormat extends BaseInputFormat { + @Override + public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException { + RecordReader reader = new CodecRecordReader(); + reader.initialize(conf, split); + return reader; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/BaseCodecRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/BaseCodecRecordReader.java new file mode 100644 index 000000000..e2d136474 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/BaseCodecRecordReader.java @@ -0,0 +1,145 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.codec.reader; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.SequenceRecord; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.metadata.RecordMetaDataURI; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.records.reader.impl.FileRecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.Writable; + +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Codec record reader for parsing videos + * + * @author Adam Gibson + */ +public abstract class BaseCodecRecordReader extends FileRecordReader implements SequenceRecordReader { + protected int startFrame = 0; + protected int numFrames = -1; + protected int totalFrames = -1; + protected double framesPerSecond = -1; + protected double videoLength = -1; + protected int rows = 28, cols = 28; + protected boolean ravel = false; + + public final static String NAME_SPACE = "org.datavec.codec.reader"; + public final static String ROWS = NAME_SPACE + ".rows"; + public final static String COLUMNS = NAME_SPACE + ".columns"; + public final static String START_FRAME = NAME_SPACE + ".startframe"; + public final static String TOTAL_FRAMES = NAME_SPACE + ".frames"; + public final static String TIME_SLICE = NAME_SPACE + ".time"; + public final static String RAVEL = NAME_SPACE + ".ravel"; + public final static String VIDEO_DURATION = NAME_SPACE + ".duration"; + + + @Override + public List> sequenceRecord() { + URI next = locationsIterator.next(); + + try (InputStream s = streamCreatorFn.apply(next)){ + return loadData(null, s); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException { + return loadData(null, dataInputStream); + } + + protected abstract List> loadData(File file, InputStream inputStream) throws IOException; + + + @Override + public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException { + setConf(conf); + initialize(split); + } + + @Override + public List next() { + throw new UnsupportedOperationException("next() not supported for CodecRecordReader (use: sequenceRecord)"); + } + + @Override + public List record(URI uri, DataInputStream dataInputStream) throws IOException { + throw new UnsupportedOperationException("record(URI,DataInputStream) not supported for CodecRecordReader"); + } + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + startFrame = conf.getInt(START_FRAME, 0); + numFrames = conf.getInt(TOTAL_FRAMES, -1); + rows = conf.getInt(ROWS, 28); + cols = conf.getInt(COLUMNS, 28); + framesPerSecond = conf.getFloat(TIME_SLICE, -1); + videoLength = conf.getFloat(VIDEO_DURATION, -1); + ravel = conf.getBoolean(RAVEL, false); + totalFrames = conf.getInt(TOTAL_FRAMES, -1); + } + + @Override + public Configuration getConf() { + return super.getConf(); + } + + @Override + public SequenceRecord nextSequence() { + URI next = locationsIterator.next(); + + List> list; + try (InputStream s = streamCreatorFn.apply(next)){ + list = loadData(null, s); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new org.datavec.api.records.impl.SequenceRecord(list, + new RecordMetaDataURI(next, CodecRecordReader.class)); + } + + @Override + public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException { + return loadSequenceFromMetaData(Collections.singletonList(recordMetaData)).get(0); + } + + @Override + public List loadSequenceFromMetaData(List recordMetaDatas) throws IOException { + List out = new ArrayList<>(); + for (RecordMetaData meta : recordMetaDatas) { + try (InputStream s = streamCreatorFn.apply(meta.getURI())){ + List> list = loadData(null, s); + out.add(new org.datavec.api.records.impl.SequenceRecord(list, meta)); + } + } + + return out; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/CodecRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/CodecRecordReader.java new file mode 100644 index 000000000..9e32b9bc0 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/CodecRecordReader.java @@ -0,0 +1,151 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.codec.reader; + +import org.apache.commons.compress.utils.IOUtils; +import org.datavec.api.conf.Configuration; +import org.datavec.api.util.ndarray.RecordConverter; +import org.datavec.api.writable.Writable; +import org.datavec.image.loader.ImageLoader; +import org.jcodec.api.FrameGrab; +import org.jcodec.api.JCodecException; +import org.jcodec.common.ByteBufferSeekableByteChannel; +import org.jcodec.common.NIOUtils; +import org.jcodec.common.SeekableByteChannel; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * Codec record reader for parsing: + * H.264 ( AVC ) Main profile decoder MP3 decoder/encoder + Apple ProRes decoder and encoder AAC encoder + H264 Baseline profile encoder + Matroska ( MKV ) demuxer and muxer + MP4 ( ISO BMF, QuickTime ) demuxer/muxer and tools + MPEG 1/2 decoder ( supports interlace ) + MPEG PS/TS demuxer + Java player applet + VP8 encoder + MXF demuxer + + Credit to jcodec for the underlying parser + * + * @author Adam Gibson + */ +public class CodecRecordReader extends BaseCodecRecordReader { + + private ImageLoader imageLoader; + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + imageLoader = new ImageLoader(rows, cols); + } + + @Override + protected List> loadData(File file, InputStream inputStream) throws IOException { + SeekableByteChannel seekableByteChannel; + if (inputStream != null) { + //Reading video from DataInputStream: Need data from this stream in a SeekableByteChannel + //Approach used here: load entire video into memory -> ByteBufferSeekableByteChanel + byte[] data = IOUtils.toByteArray(inputStream); + ByteBuffer bb = ByteBuffer.wrap(data); + seekableByteChannel = new FixedByteBufferSeekableByteChannel(bb); + } else { + seekableByteChannel = NIOUtils.readableFileChannel(file); + } + + List> record = new ArrayList<>(); + + if (numFrames >= 1) { + FrameGrab fg; + try { + fg = new FrameGrab(seekableByteChannel); + if (startFrame != 0) + fg.seekToFramePrecise(startFrame); + } catch (JCodecException e) { + throw new RuntimeException(e); + } + + for (int i = startFrame; i < startFrame + numFrames; i++) { + try { + BufferedImage grab = fg.getFrame(); + if (ravel) + record.add(RecordConverter.toRecord(imageLoader.toRaveledTensor(grab))); + else + record.add(RecordConverter.toRecord(imageLoader.asRowVector(grab))); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } else { + if (framesPerSecond < 1) + throw new IllegalStateException("No frames or frame time intervals specified"); + + + else { + for (double i = 0; i < videoLength; i += framesPerSecond) { + try { + BufferedImage grab = FrameGrab.getFrame(seekableByteChannel, i); + if (ravel) + record.add(RecordConverter.toRecord(imageLoader.toRaveledTensor(grab))); + else + record.add(RecordConverter.toRecord(imageLoader.asRowVector(grab))); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + } + + return record; + } + + /** Ugly workaround to a bug in JCodec: https://github.com/jcodec/jcodec/issues/24 */ + private static class FixedByteBufferSeekableByteChannel extends ByteBufferSeekableByteChannel { + private ByteBuffer backing; + + public FixedByteBufferSeekableByteChannel(ByteBuffer backing) { + super(backing); + try { + Field f = this.getClass().getSuperclass().getDeclaredField("maxPos"); + f.setAccessible(true); + f.set(this, backing.limit()); + } catch (Exception e) { + throw new RuntimeException(e); + } + this.backing = backing; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + if (!backing.hasRemaining()) + return -1; + return super.read(dst); + } + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/NativeCodecRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/NativeCodecRecordReader.java new file mode 100644 index 000000000..e6e7844ff --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/main/java/org/datavec/codec/reader/NativeCodecRecordReader.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.codec.reader; + +import org.bytedeco.javacv.FFmpegFrameGrabber; +import org.bytedeco.javacv.Frame; +import org.bytedeco.javacv.OpenCVFrameConverter; +import org.datavec.api.conf.Configuration; +import org.datavec.api.util.ndarray.RecordConverter; +import org.datavec.api.writable.Writable; +import org.datavec.image.loader.NativeImageLoader; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * An implementation of the CodecRecordReader that uses JavaCV and FFmpeg. + * + * @author saudet + */ +public class NativeCodecRecordReader extends BaseCodecRecordReader { + + private OpenCVFrameConverter.ToMat converter; + private NativeImageLoader imageLoader; + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + converter = new OpenCVFrameConverter.ToMat(); + imageLoader = new NativeImageLoader(rows, cols); + } + + @Override + protected List> loadData(File file, InputStream inputStream) throws IOException { + List> record = new ArrayList<>(); + + try (FFmpegFrameGrabber fg = + inputStream != null ? new FFmpegFrameGrabber(inputStream) : new FFmpegFrameGrabber(file)) { + if (numFrames >= 1) { + fg.start(); + if (startFrame != 0) + fg.setFrameNumber(startFrame); + + for (int i = startFrame; i < startFrame + numFrames; i++) { + Frame grab = fg.grabImage(); + record.add(RecordConverter.toRecord(imageLoader.asRowVector(converter.convert(grab)))); + } + } else { + if (framesPerSecond < 1) + throw new IllegalStateException("No frames or frame time intervals specified"); + else { + fg.start(); + + for (double i = 0; i < videoLength; i += framesPerSecond) { + fg.setTimestamp(Math.round(i * 1000000L)); + Frame grab = fg.grabImage(); + record.add(RecordConverter.toRecord(imageLoader.asRowVector(converter.convert(grab)))); + } + } + } + } + + return record; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/test/java/org/datavec/codec/reader/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/test/java/org/datavec/codec/reader/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..611915bb0 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/test/java/org/datavec/codec/reader/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.codec.reader; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.codec.reader"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/test/java/org/datavec/codec/reader/CodecReaderTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/test/java/org/datavec/codec/reader/CodecReaderTest.java new file mode 100644 index 000000000..ca80949ef --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-codec/src/test/java/org/datavec/codec/reader/CodecReaderTest.java @@ -0,0 +1,208 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.codec.reader; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.SequenceRecord; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.ArrayWritable; +import org.datavec.api.writable.Writable; + +import org.junit.jupiter.api.Test; +import org.nd4j.common.io.ClassPathResource; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.util.Iterator; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author Adam Gibson + */ +public class CodecReaderTest { + @Test + public void testCodecReader() throws Exception { + File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile(); + SequenceRecordReader reader = new CodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "160"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "500"); + conf.set(CodecRecordReader.ROWS, "80"); + conf.set(CodecRecordReader.COLUMNS, "46"); + reader.initialize(new FileSplit(file)); + reader.setConf(conf); + assertTrue(reader.hasNext()); + List> record = reader.sequenceRecord(); + // System.out.println(record.size()); + + Iterator> it = record.iterator(); + List first = it.next(); + // System.out.println(first); + + //Expected size: 80x46x3 + assertEquals(1, first.size()); + assertEquals(80 * 46 * 3, ((ArrayWritable) first.iterator().next()).length()); + } + + @Test + public void testCodecReaderMeta() throws Exception { + File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile(); + SequenceRecordReader reader = new CodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "160"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "500"); + conf.set(CodecRecordReader.ROWS, "80"); + conf.set(CodecRecordReader.COLUMNS, "46"); + reader.initialize(new FileSplit(file)); + reader.setConf(conf); + assertTrue(reader.hasNext()); + List> record = reader.sequenceRecord(); + assertEquals(500, record.size()); //500 frames + + reader.reset(); + SequenceRecord seqR = reader.nextSequence(); + assertEquals(record, seqR.getSequenceRecord()); + RecordMetaData meta = seqR.getMetaData(); + // System.out.println(meta); + assertTrue(meta.getURI().toString().endsWith(file.getName())); + + SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta); + assertEquals(seqR, fromMeta); + } + + @Test + public void testViaDataInputStream() throws Exception { + + File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile(); + SequenceRecordReader reader = new CodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "160"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "500"); + conf.set(CodecRecordReader.ROWS, "80"); + conf.set(CodecRecordReader.COLUMNS, "46"); + + Configuration conf2 = new Configuration(conf); + + reader.initialize(new FileSplit(file)); + reader.setConf(conf); + assertTrue(reader.hasNext()); + List> expected = reader.sequenceRecord(); + + + SequenceRecordReader reader2 = new CodecRecordReader(); + reader2.setConf(conf2); + + DataInputStream dataInputStream = new DataInputStream(new FileInputStream(file)); + List> actual = reader2.sequenceRecord(null, dataInputStream); + + assertEquals(expected, actual); + } + + + //@Ignore + @Test + public void testNativeCodecReader() throws Exception { + File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile(); + SequenceRecordReader reader = new NativeCodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "160"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "500"); + conf.set(CodecRecordReader.ROWS, "80"); + conf.set(CodecRecordReader.COLUMNS, "46"); + reader.initialize(new FileSplit(file)); + reader.setConf(conf); + assertTrue(reader.hasNext()); + List> record = reader.sequenceRecord(); + // System.out.println(record.size()); + + Iterator> it = record.iterator(); + List first = it.next(); + // System.out.println(first); + + //Expected size: 80x46x3 + assertEquals(1, first.size()); + assertEquals(80 * 46 * 3, ((ArrayWritable) first.iterator().next()).length()); + } + + //@Ignore + @Test + public void testNativeCodecReaderMeta() throws Exception { + File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile(); + SequenceRecordReader reader = new NativeCodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "160"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "500"); + conf.set(CodecRecordReader.ROWS, "80"); + conf.set(CodecRecordReader.COLUMNS, "46"); + reader.initialize(new FileSplit(file)); + reader.setConf(conf); + assertTrue(reader.hasNext()); + List> record = reader.sequenceRecord(); + assertEquals(500, record.size()); //500 frames + + reader.reset(); + SequenceRecord seqR = reader.nextSequence(); + assertEquals(record, seqR.getSequenceRecord()); + RecordMetaData meta = seqR.getMetaData(); + // System.out.println(meta); + assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4")); + + SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta); + assertEquals(seqR, fromMeta); + } + + //@Ignore + @Test + public void testNativeViaDataInputStream() throws Exception { + + File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile(); + SequenceRecordReader reader = new NativeCodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "160"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "500"); + conf.set(CodecRecordReader.ROWS, "80"); + conf.set(CodecRecordReader.COLUMNS, "46"); + + Configuration conf2 = new Configuration(conf); + + reader.initialize(new FileSplit(file)); + reader.setConf(conf); + assertTrue(reader.hasNext()); + List> expected = reader.sequenceRecord(); + + + SequenceRecordReader reader2 = new NativeCodecRecordReader(); + reader2.setConf(conf2); + + DataInputStream dataInputStream = new DataInputStream(new FileInputStream(file)); + List> actual = reader2.sequenceRecord(null, dataInputStream); + + assertEquals(expected, actual); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/build.gradle b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/build.gradle new file mode 100644 index 000000000..49822b112 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/build.gradle @@ -0,0 +1,31 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDnn.cavisDnnCommon + implementation "com.maxmind.geoip2:geoip2:2.8.1" + implementation "org.slf4j:slf4j-api" + implementation "commons-io:commons-io" + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/geo/LocationType.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/geo/LocationType.java new file mode 100644 index 000000000..cd12f3fc9 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/geo/LocationType.java @@ -0,0 +1,26 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.api.transform.geo; + +/** + * The type of geolocation. + * + * @author saudet + */ +public enum LocationType { + CITY, CITY_ID, CONTINENT, CONTINENT_ID, COUNTRY, COUNTRY_ID, COORDINATES, POSTAL_CODE, SUBDIVISIONS, SUBDIVISIONS_ID +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/reduce/geo/CoordinatesReduction.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/reduce/geo/CoordinatesReduction.java new file mode 100644 index 000000000..d5e9e3439 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/reduce/geo/CoordinatesReduction.java @@ -0,0 +1,196 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.api.transform.reduce.geo; + +import lombok.Getter; +import org.datavec.api.transform.ReduceOp; +import org.datavec.api.transform.metadata.ColumnMetaData; +import org.datavec.api.transform.metadata.StringMetaData; +import org.datavec.api.transform.ops.IAggregableReduceOp; +import org.datavec.api.transform.reduce.AggregableColumnReduction; +import org.datavec.api.transform.reduce.AggregableReductionUtils; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.nd4j.common.function.Supplier; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Applies a ReduceOp to a column of coordinates, for each component independently. + * Basically a dispatchop with n = 2 an integrated coordinate parsing & serialization + * + * @author saudet + */ +public class CoordinatesReduction implements AggregableColumnReduction { + public static final String DEFAULT_COLUMN_NAME = "CoordinatesReduction"; + + public final static String DEFAULT_DELIMITER = ":"; + protected String delimiter = DEFAULT_DELIMITER; + + private final List columnNamesPostReduce; + + private final Supplier>> multiOp(final List ops) { + return new Supplier>>() { + @Override + public IAggregableReduceOp> get() { + return AggregableReductionUtils.reduceDoubleColumn(ops, false, null); + } + }; + } + + public CoordinatesReduction(String columnNamePostReduce, ReduceOp op) { + this(columnNamePostReduce, op, DEFAULT_DELIMITER); + } + + public CoordinatesReduction(List columnNamePostReduce, List op) { + this(columnNamePostReduce, op, DEFAULT_DELIMITER); + } + + public CoordinatesReduction(String columnNamePostReduce, ReduceOp op, String delimiter) { + this(Collections.singletonList(columnNamePostReduce), Collections.singletonList(op), delimiter); + } + + public CoordinatesReduction(List columnNamesPostReduce, List ops, String delimiter) { + this.columnNamesPostReduce = columnNamesPostReduce; + this.reducer = new CoordinateAggregableReduceOp(ops.size(), multiOp(ops), delimiter); + } + + @Override + public List getColumnsOutputName(String columnInputName) { + return columnNamesPostReduce; + } + + @Override + public List getColumnOutputMetaData(List newColumnName, ColumnMetaData columnInputMeta) { + List res = new ArrayList<>(newColumnName.size()); + for (String cn : newColumnName) + res.add(new StringMetaData((cn))); + return res; + } + + @Override + public Schema transform(Schema inputSchema) { + throw new UnsupportedOperationException(); + } + + @Override + public void setInputSchema(Schema inputSchema) { + throw new UnsupportedOperationException(); + } + + @Override + public Schema getInputSchema() { + throw new UnsupportedOperationException(); + } + + @Override + public String outputColumnName() { + throw new UnsupportedOperationException(); + } + + @Override + public String[] outputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + public String[] columnNames() { + throw new UnsupportedOperationException(); + } + + @Override + public String columnName() { + throw new UnsupportedOperationException(); + } + + private IAggregableReduceOp> reducer; + + @Override + public IAggregableReduceOp> reduceOp() { + return reducer; + } + + + public static class CoordinateAggregableReduceOp implements IAggregableReduceOp> { + + + private int nOps; + private Supplier>> initialOpValue; + @Getter + private ArrayList>> perCoordinateOps; // of size coords() + private String delimiter; + + public CoordinateAggregableReduceOp(int n, Supplier>> initialOp, + String delim) { + this.nOps = n; + this.perCoordinateOps = new ArrayList<>(); + this.initialOpValue = initialOp; + this.delimiter = delim; + } + + @Override + public >> void combine(W accu) { + if (accu instanceof CoordinateAggregableReduceOp) { + CoordinateAggregableReduceOp accumulator = (CoordinateAggregableReduceOp) accu; + for (int i = 0; i < Math.min(perCoordinateOps.size(), accumulator.getPerCoordinateOps().size()); i++) { + perCoordinateOps.get(i).combine(accumulator.getPerCoordinateOps().get(i)); + } // the rest is assumed identical + } + } + + @Override + public void accept(Writable writable) { + String[] coordinates = writable.toString().split(delimiter); + for (int i = 0; i < coordinates.length; i++) { + String coordinate = coordinates[i]; + while (perCoordinateOps.size() < i + 1) { + perCoordinateOps.add(initialOpValue.get()); + } + perCoordinateOps.get(i).accept(new DoubleWritable(Double.parseDouble(coordinate))); + } + } + + @Override + public List get() { + List res = new ArrayList<>(nOps); + for (int i = 0; i < nOps; i++) { + res.add(new StringBuilder()); + } + + for (int i = 0; i < perCoordinateOps.size(); i++) { + List resThisCoord = perCoordinateOps.get(i).get(); + for (int j = 0; j < nOps; j++) { + res.get(j).append(resThisCoord.get(j).toString()); + if (i < perCoordinateOps.size() - 1) { + res.get(j).append(delimiter); + } + } + } + + List finalRes = new ArrayList<>(nOps); + for (StringBuilder sb : res) { + finalRes.add(new Text(sb.toString())); + } + return finalRes; + } + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/CoordinatesDistanceTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/CoordinatesDistanceTransform.java new file mode 100644 index 000000000..4595abf1a --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/CoordinatesDistanceTransform.java @@ -0,0 +1,119 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.api.transform.transform.geo; + +import org.datavec.api.transform.MathOp; +import org.datavec.api.transform.metadata.ColumnMetaData; +import org.datavec.api.transform.metadata.DoubleMetaData; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.transform.BaseColumnsMathOpTransform; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Writable; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Computes the Euclidean distance between coordinates found in two columns, divided by an optional third for normalization purposes. + * A new column (with the specified name) is added as the final column of the output. No other columns are modified. + * + * @author saudet + */ +public class CoordinatesDistanceTransform extends BaseColumnsMathOpTransform { + + public final static String DEFAULT_DELIMITER = ":"; + protected String delimiter = DEFAULT_DELIMITER; + + public CoordinatesDistanceTransform(String newColumnName, String firstColumn, String secondColumn, + String stdevColumn) { + this(newColumnName, firstColumn, secondColumn, stdevColumn, DEFAULT_DELIMITER); + } + + public CoordinatesDistanceTransform(@JsonProperty("newColumnName") String newColumnName, + @JsonProperty("firstColumn") String firstColumn, @JsonProperty("secondColumn") String secondColumn, + @JsonProperty("stdevColumn") String stdevColumn, @JsonProperty("delimiter") String delimiter) { + super(newColumnName, MathOp.Add /* dummy op */, + stdevColumn != null ? new String[] {firstColumn, secondColumn, stdevColumn} + : new String[] {firstColumn, secondColumn}); + this.delimiter = delimiter; + } + + @Override + protected ColumnMetaData derivedColumnMetaData(String newColumnName, Schema inputSchema) { + return new DoubleMetaData(newColumnName); + } + + @Override + protected Writable doOp(Writable... input) { + String[] first = input[0].toString().split(delimiter); + String[] second = input[1].toString().split(delimiter); + String[] stdev = columns.length > 2 ? input[2].toString().split(delimiter) : null; + + double dist = 0; + for (int i = 0; i < first.length; i++) { + double d = Double.parseDouble(first[i]) - Double.parseDouble(second[i]); + double s = stdev != null ? Double.parseDouble(stdev[i]) : 1; + dist += (d * d) / (s * s); + } + return new DoubleWritable(Math.sqrt(dist)); + } + + @Override + public String toString() { + return "CoordinatesDistanceTransform(newColumnName=\"" + newColumnName + "\",columns=" + + Arrays.toString(columns) + ",delimiter=" + delimiter + ")"; + } + + /** + * Transform an object + * in to another object + * + * @param input the record to transform + * @return the transformed writable + */ + @Override + public Object map(Object input) { + List row = (List) input; + String[] first = row.get(0).toString().split(delimiter); + String[] second = row.get(1).toString().split(delimiter); + String[] stdev = columns.length > 2 ? row.get(2).toString().split(delimiter) : null; + + double dist = 0; + for (int i = 0; i < first.length; i++) { + double d = Double.parseDouble(first[i]) - Double.parseDouble(second[i]); + double s = stdev != null ? Double.parseDouble(stdev[i]) : 1; + dist += (d * d) / (s * s); + } + return Math.sqrt(dist); + } + + /** + * Transform a sequence + * + * @param sequence + */ + @Override + public Object mapSequence(Object sequence) { + List seq = (List) sequence; + List ret = new ArrayList<>(); + for (Object step : seq) + ret.add((Double) map(step)); + return ret; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/GeoIPFetcher.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/GeoIPFetcher.java new file mode 100644 index 000000000..7868b0044 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/GeoIPFetcher.java @@ -0,0 +1,72 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.api.transform.transform.geo; + +import org.apache.commons.io.FileUtils; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.util.ArchiveUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +/** + * Downloads and caches the GeoLite2 City database created by MaxMind, available from + * http://www.maxmind.com or uses one already available on system. + * + * @author saudet + */ +public class GeoIPFetcher { + protected static final Logger log = LoggerFactory.getLogger(GeoIPFetcher.class); + + /** Default directory for http://dev.maxmind.com/geoip/geoipupdate/ */ + public static final String GEOIP_DIR = "/usr/local/share/GeoIP/"; + public static final String GEOIP_DIR2 = System.getProperty("user.home") + "/.datavec-geoip"; + + public static final String CITY_DB = "GeoIP2-City.mmdb"; + public static final String CITY_LITE_DB = "GeoLite2-City.mmdb"; + + public static final String CITY_LITE_URL = + "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"; + + public static synchronized File fetchCityDB() throws IOException { + File cityFile = new File(GEOIP_DIR, CITY_DB); + if (cityFile.isFile()) { + return cityFile; + } + cityFile = new File(GEOIP_DIR, CITY_LITE_DB); + if (cityFile.isFile()) { + return cityFile; + } + cityFile = new File(GEOIP_DIR2, CITY_LITE_DB); + if (cityFile.isFile()) { + return cityFile; + } + + log.info("Downloading GeoLite2 City database..."); + File archive = new File(GEOIP_DIR2, CITY_LITE_DB + ".gz"); + File dir = new File(GEOIP_DIR2); + dir.mkdirs(); + FileUtils.copyURLToFile(new URL(CITY_LITE_URL), archive); + ArchiveUtils.unzipFileTo(archive.getAbsolutePath(), dir.getAbsolutePath()); + Preconditions.checkState(cityFile.isFile(), "Error extracting files: expected city file does not exist after extraction"); + + return cityFile; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/IPAddressToCoordinatesTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/IPAddressToCoordinatesTransform.java new file mode 100644 index 000000000..f52dab981 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/IPAddressToCoordinatesTransform.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.api.transform.transform.geo; + +import org.datavec.api.transform.geo.LocationType; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.IOException; + +/** + * Uses GeoIP2 from http://www.maxmind.com + * to convert IP addresses to (approximate) coordinates (latitude:longitude). + * For example, "128.101.101.101" becomes something like "44.9733:-93.2323". + * + * @author saudet + */ +public class IPAddressToCoordinatesTransform extends IPAddressToLocationTransform { + + public IPAddressToCoordinatesTransform(@JsonProperty("columnName") String columnName) throws IOException { + this(columnName, DEFAULT_DELIMITER); + } + + public IPAddressToCoordinatesTransform(@JsonProperty("columnName") String columnName, + @JsonProperty("delimiter") String delimiter) throws IOException { + super(columnName, LocationType.COORDINATES, delimiter); + } + + @Override + public String toString() { + return "IPAddressToCoordinatesTransform"; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/IPAddressToLocationTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/IPAddressToLocationTransform.java new file mode 100644 index 000000000..e878619fe --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/main/java/org/datavec/api/transform/transform/geo/IPAddressToLocationTransform.java @@ -0,0 +1,188 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.api.transform.transform.geo; + +import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.exception.GeoIp2Exception; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.record.Location; +import com.maxmind.geoip2.record.Subdivision; +import lombok.extern.slf4j.Slf4j; +import org.datavec.api.transform.geo.LocationType; +import org.datavec.api.transform.metadata.ColumnMetaData; +import org.datavec.api.transform.metadata.StringMetaData; +import org.datavec.api.transform.transform.BaseColumnTransform; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.net.InetAddress; + +/** + * Uses GeoIP2 from http://www.maxmind.com + * to convert IP addresses to (approximate) locations. + * + * @see LocationType + * + * @author saudet + */ +@Slf4j +public class IPAddressToLocationTransform extends BaseColumnTransform { + /** + * Name of the system property to use when configuring the GeoIP database file.
+ * Most users don't need to set this - typically used for testing purposes.
+ * Set with the full local path, like: "C:/datavec-geo/GeoIP2-City-Test.mmdb" + */ + public static final String GEOIP_FILE_PROPERTY = "org.datavec.geoip.file"; + + private static File database; + private static DatabaseReader reader; + + public final static String DEFAULT_DELIMITER = ":"; + protected String delimiter = DEFAULT_DELIMITER; + protected LocationType locationType; + + private static synchronized void init() throws IOException { + // A File object pointing to your GeoIP2 or GeoLite2 database: + // http://dev.maxmind.com/geoip/geoip2/geolite2/ + if (database == null) { + String s = System.getProperty(GEOIP_FILE_PROPERTY); + if(s != null && !s.isEmpty()){ + //Use user-specified GEOIP file - mainly for testing purposes + File f = new File(s); + if(f.exists() && f.isFile()){ + database = f; + } else { + log.warn("GeoIP file (system property {}) is set to \"{}\" but this is not a valid file, using default database", GEOIP_FILE_PROPERTY, s); + database = GeoIPFetcher.fetchCityDB(); + } + } else { + database = GeoIPFetcher.fetchCityDB(); + } + } + + // This creates the DatabaseReader object, which should be reused across lookups. + if (reader == null) { + reader = new DatabaseReader.Builder(database).build(); + } + } + + public IPAddressToLocationTransform(String columnName) throws IOException { + this(columnName, LocationType.CITY); + } + + public IPAddressToLocationTransform(String columnName, LocationType locationType) throws IOException { + this(columnName, locationType, DEFAULT_DELIMITER); + } + + public IPAddressToLocationTransform(@JsonProperty("columnName") String columnName, + @JsonProperty("delimiter") LocationType locationType, @JsonProperty("delimiter") String delimiter) + throws IOException { + super(columnName); + this.delimiter = delimiter; + this.locationType = locationType; + init(); + } + + @Override + public ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType) { + return new StringMetaData(newName); //Output after transform: String (Text) + } + + @Override + public Writable map(Writable columnWritable) { + try { + InetAddress ipAddress = InetAddress.getByName(columnWritable.toString()); + CityResponse response = reader.city(ipAddress); + String text = ""; + switch (locationType) { + case CITY: + text = response.getCity().getName(); + break; + case CITY_ID: + text = response.getCity().getGeoNameId().toString(); + break; + case CONTINENT: + text = response.getContinent().getName(); + break; + case CONTINENT_ID: + text = response.getContinent().getGeoNameId().toString(); + break; + case COUNTRY: + text = response.getCountry().getName(); + break; + case COUNTRY_ID: + text = response.getCountry().getGeoNameId().toString(); + break; + case COORDINATES: + Location location = response.getLocation(); + text = location.getLatitude() + delimiter + location.getLongitude(); + break; + case POSTAL_CODE: + text = response.getPostal().getCode(); + break; + case SUBDIVISIONS: + for (Subdivision s : response.getSubdivisions()) { + if (text.length() > 0) { + text += delimiter; + } + text += s.getName(); + } + break; + case SUBDIVISIONS_ID: + for (Subdivision s : response.getSubdivisions()) { + if (text.length() > 0) { + text += delimiter; + } + text += s.getGeoNameId().toString(); + } + break; + default: + assert false; + } + if(text == null) + text = ""; + return new Text(text); + } catch (GeoIp2Exception | IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() { + return "IPAddressToLocationTransform"; + } + + //Custom serialization methods, because GeoIP2 doesn't allow DatabaseReader objects to be serialized :( + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + init(); + } + + @Override + public Object map(Object input) { + return null; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..3a01a1c4b --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.api.transform; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseND4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.api.transform"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/reduce/TestGeoReduction.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/reduce/TestGeoReduction.java new file mode 100644 index 000000000..55fd5855a --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/reduce/TestGeoReduction.java @@ -0,0 +1,76 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.api.transform.reduce; + +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.ReduceOp; +import org.datavec.api.transform.ops.IAggregableReduceOp; +import org.datavec.api.transform.reduce.geo.CoordinatesReduction; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author saudet + */ +public class TestGeoReduction { + + @Test + public void testCustomReductions() { + + List> inputs = new ArrayList<>(); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("1#5"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("2#6"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("3#7"))); + inputs.add(Arrays.asList((Writable) new Text("someKey"), new Text("4#8"))); + + List expected = Arrays.asList((Writable) new Text("someKey"), new Text("10.0#26.0")); + + Schema schema = new Schema.Builder().addColumnString("key").addColumnString("coord").build(); + + Reducer reducer = new Reducer.Builder(ReduceOp.Count).keyColumns("key") + .customReduction("coord", new CoordinatesReduction("coordSum", ReduceOp.Sum, "#")).build(); + + reducer.setInputSchema(schema); + + IAggregableReduceOp, List> aggregableReduceOp = reducer.aggregableReducer(); + for (List l : inputs) + aggregableReduceOp.accept(l); + List out = aggregableReduceOp.get(); + + assertEquals(2, out.size()); + assertEquals(expected, out); + + //Check schema: + String[] expNames = new String[] {"key", "coordSum"}; + ColumnType[] expTypes = new ColumnType[] {ColumnType.String, ColumnType.String}; + Schema outSchema = reducer.transform(schema); + + assertEquals(2, outSchema.numColumns()); + for (int i = 0; i < 2; i++) { + assertEquals(expNames[i], outSchema.getName(i)); + assertEquals(expTypes[i], outSchema.getType(i)); + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/transform/TestGeoTransforms.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/transform/TestGeoTransforms.java new file mode 100644 index 000000000..d91d34b95 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-geo/src/test/java/org/datavec/api/transform/transform/TestGeoTransforms.java @@ -0,0 +1,150 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.api.transform.transform; + +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.Transform; +import org.datavec.api.transform.geo.LocationType; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.transform.geo.CoordinatesDistanceTransform; +import org.datavec.api.transform.transform.geo.IPAddressToCoordinatesTransform; +import org.datavec.api.transform.transform.geo.IPAddressToLocationTransform; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.nd4j.common.io.ClassPathResource; + +import java.io.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author saudet + */ +public class TestGeoTransforms { + + @BeforeAll + public static void beforeClass() throws Exception { + //Use test resources version to avoid tests suddenly failing due to IP/Location DB content changing + File f = new ClassPathResource("datavec-geo/GeoIP2-City-Test.mmdb").getFile(); + System.setProperty(IPAddressToLocationTransform.GEOIP_FILE_PROPERTY, f.getPath()); + } + + @AfterAll + public static void afterClass(){ + System.setProperty(IPAddressToLocationTransform.GEOIP_FILE_PROPERTY, ""); + } + + @Test + public void testCoordinatesDistanceTransform() throws Exception { + Schema schema = new Schema.Builder().addColumnString("point").addColumnString("mean").addColumnString("stddev") + .build(); + + Transform transform = new CoordinatesDistanceTransform("dist", "point", "mean", "stddev", "\\|"); + transform.setInputSchema(schema); + + Schema out = transform.transform(schema); + assertEquals(4, out.numColumns()); + assertEquals(Arrays.asList("point", "mean", "stddev", "dist"), out.getColumnNames()); + assertEquals(Arrays.asList(ColumnType.String, ColumnType.String, ColumnType.String, ColumnType.Double), + out.getColumnTypes()); + + assertEquals(Arrays.asList((Writable) new Text("-30"), new Text("20"), new Text("10"), new DoubleWritable(5.0)), + transform.map(Arrays.asList((Writable) new Text("-30"), new Text("20"), new Text("10")))); + assertEquals(Arrays.asList((Writable) new Text("50|40"), new Text("10|-20"), new Text("10|5"), + new DoubleWritable(Math.sqrt(160))), + transform.map(Arrays.asList((Writable) new Text("50|40"), new Text("10|-20"), + new Text("10|5")))); + } + + @Test + public void testIPAddressToCoordinatesTransform() throws Exception { + Schema schema = new Schema.Builder().addColumnString("column").build(); + + Transform transform = new IPAddressToCoordinatesTransform("column", "CUSTOM_DELIMITER"); + transform.setInputSchema(schema); + + Schema out = transform.transform(schema); + + assertEquals(1, out.getColumnMetaData().size()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + + String in = "81.2.69.160"; + double latitude = 51.5142; + double longitude = -0.0931; + + List writables = transform.map(Collections.singletonList((Writable) new Text(in))); + assertEquals(1, writables.size()); + String[] coordinates = writables.get(0).toString().split("CUSTOM_DELIMITER"); + assertEquals(2, coordinates.length); + assertEquals(latitude, Double.parseDouble(coordinates[0]), 0.1); + assertEquals(longitude, Double.parseDouble(coordinates[1]), 0.1); + + //Check serialization: things like DatabaseReader etc aren't serializable, hence we need custom serialization :/ + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(transform); + + byte[] bytes = baos.toByteArray(); + + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bais); + + Transform deserialized = (Transform) ois.readObject(); + writables = deserialized.map(Collections.singletonList((Writable) new Text(in))); + assertEquals(1, writables.size()); + coordinates = writables.get(0).toString().split("CUSTOM_DELIMITER"); + //System.out.println(Arrays.toString(coordinates)); + assertEquals(2, coordinates.length); + assertEquals(latitude, Double.parseDouble(coordinates[0]), 0.1); + assertEquals(longitude, Double.parseDouble(coordinates[1]), 0.1); + } + + @Test + public void testIPAddressToLocationTransform() throws Exception { + Schema schema = new Schema.Builder().addColumnString("column").build(); + LocationType[] locationTypes = LocationType.values(); + String in = "81.2.69.160"; + String[] locations = {"London", "2643743", "Europe", "6255148", "United Kingdom", "2635167", + "51.5142:-0.0931", "", "England", "6269131"}; //Note: no postcode in this test DB for this record + + for (int i = 0; i < locationTypes.length; i++) { + LocationType locationType = locationTypes[i]; + String location = locations[i]; + + Transform transform = new IPAddressToLocationTransform("column", locationType); + transform.setInputSchema(schema); + + Schema out = transform.transform(schema); + + assertEquals(1, out.getColumnMetaData().size()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + + List writables = transform.map(Collections.singletonList((Writable) new Text(in))); + assertEquals(1, writables.size()); + assertEquals(location, writables.get(0).toString()); + //System.out.println(location); + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/build.gradle b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/build.gradle new file mode 100644 index 000000000..5fc090231 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/build.gradle @@ -0,0 +1,48 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDnn.cavisDnnCommon + + implementation 'io.netty:netty-all' + compileOnly("org.apache.hadoop:hadoop-common:3.2.0") { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + exclude group: 'jdk.tools', module: 'jdk.tools' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + } + testCompileOnly("org.apache.hadoop:hadoop-common:3.2.0") { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + exclude group: 'jdk.tools', module: 'jdk.tools' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + } + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testImplementation projects.cavisDnn.cavisDnnApi + + testImplementation "org.slf4j:slf4j-api" + testImplementation "org.apache.hadoop:hadoop-common:3.2.0" + testImplementation "com.google.guava:guava" + + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" + +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/conf/ConfigurationUtil.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/conf/ConfigurationUtil.java new file mode 100644 index 000000000..01d5b2f84 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/conf/ConfigurationUtil.java @@ -0,0 +1,58 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +/** + * Notes + * + * https://linuxjunkies.wordpress.com/2011/11/21/a-hdfsclient-for-hadoop-using-the-native-java-api-a-tutorial/ + * + * Design Ideas + * + * - Need a DataVec Conf entry: + * - hadoop.configuration.path + * - example: hadoop.configuration.path=/home/hadoop/hadoop/conf/ + * + * + * @author josh + * + */ +public class ConfigurationUtil { + + public static Configuration generateConfig(String baseConfPath) { + + String baseConfPathTrimmed = baseConfPath.trim(); + + if (false == "/".equals(baseConfPathTrimmed.endsWith("/"))) { + + baseConfPathTrimmed += "/"; + + } + + Configuration conf = new Configuration(); + conf.addResource(new Path(baseConfPathTrimmed + "core-site.xml")); + conf.addResource(new Path(baseConfPathTrimmed + "hdfs-site.xml")); + conf.addResource(new Path(baseConfPathTrimmed + "mapred-site.xml")); + + return conf; + + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/IndexToKey.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/IndexToKey.java new file mode 100644 index 000000000..56a751953 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/IndexToKey.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader.mapfile; + +import org.apache.hadoop.io.MapFile; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.nd4j.common.primitives.Pair; + +import java.io.IOException; +import java.util.List; + +/** + * An interface to handle Index to key conversion, for use in {@link MapFileReader} + * + * @author Alex Black + */ +public interface IndexToKey { + + /** + * Initialise the instance, and return the first and last record indexes (inclusive) for each reader + * + * @param readers The underlying map file readers + */ + List> initialize(MapFile.Reader[] readers, Class valueClass) + throws IOException; + + /** + * Get the key for the given index + * + * @param index 0 to getNumRecords(reader) + * @return The key for the given index + */ + WritableComparable getKeyForIndex(long index); + + /** + * Getter infer the number of records in the given map file(s) + * + * @return Number of records in the map file(s) + */ + long getNumRecords() throws IOException; + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileReader.java new file mode 100644 index 000000000..f5b28847e --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileReader.java @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader.mapfile; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.MapFile; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.util.ReflectionUtils; +import org.datavec.hadoop.records.reader.mapfile.index.LongIndexToKey; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; +import org.nd4j.common.primitives.Pair; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * A wrapper around a Hadoop {@link MapFile.Reader}, used in {@link MapFileRecordReader} and {@link MapFileSequenceRecordReader} + * + * Note: This also handles multiple map files, such as the output from Spark, which gives a set of map files + * in directories like /part-r-00000, /part-r-00001 + * + * @author Alex Black + */ +public class MapFileReader implements Closeable { + + private MapFile.Reader[] readers; + private IndexToKey indexToKey; + private Class recordClass; + private List> recordIndexesEachReader; + private Long numRecords; + + + public MapFileReader(String path) throws Exception { + this(path, new LongIndexToKey(), RecordWritable.class); + } + + /** + * @param path Path (directory) of the MapFile + * @param indexToKey Instance used to convert long indices to key values. This allows for lookup by key + * @param recordClass Class of the records in the MapFile + * @throws IOException If an error occurs during opening or initialisation + */ + public MapFileReader(String path, IndexToKey indexToKey, Class recordClass) throws IOException { + this(Collections.singletonList(path), indexToKey, recordClass); + } + + public MapFileReader(List paths, IndexToKey indexToKey, Class recordClass) + throws IOException { + + this.indexToKey = indexToKey; + this.recordClass = recordClass; + this.readers = new MapFile.Reader[paths.size()]; + + SequenceFile.Reader.Option[] opts = new SequenceFile.Reader.Option[0]; + + Configuration config = new Configuration(); + for (int i = 0; i < paths.size(); i++) { + readers[i] = new MapFile.Reader(new Path(paths.get(i)), config, opts); + if (readers[i].getValueClass() != recordClass) { + throw new UnsupportedOperationException("MapFile record class: " + readers[i].getValueClass() + + ", but got class " + recordClass + ", path = " + paths.get(i)); + } + } + + recordIndexesEachReader = indexToKey.initialize(readers, recordClass); + } + + /** + * Determine the total number of records in the map file, using the {@link IndexToKey} instance + * + * @return Total number of records and the map file + */ + public long numRecords() { + if (numRecords == null) { + try { + numRecords = indexToKey.getNumRecords(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return numRecords; + } + + /** + * It a single record from the map file for the given index + * + * @param index Index, between 0 and numRecords()-1 + * @return Value from the MapFile + * @throws IOException If an error occurs during reading + */ + public V getRecord(long index) throws IOException { + //First: determine which reader to read from... + int readerIdx = -1; + for (int i = 0; i < recordIndexesEachReader.size(); i++) { + Pair p = recordIndexesEachReader.get(i); + if (index >= p.getFirst() && index <= p.getSecond()) { + readerIdx = i; + break; + } + } + if (readerIdx == -1) { + throw new IllegalStateException("Index not found in any reader: " + index); + } + + WritableComparable key = indexToKey.getKeyForIndex(index); + Writable value = ReflectionUtils.newInstance(recordClass, null); + + V v = (V) readers[readerIdx].get(key, value); + return v; + } + + + @Override + public void close() throws IOException { + for (MapFile.Reader r : readers) { + r.close(); + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileRecordReader.java new file mode 100644 index 000000000..df649f8e4 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileRecordReader.java @@ -0,0 +1,297 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader.mapfile; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.listener.RecordListener; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.metadata.RecordMetaDataIndex; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.Writable; +import org.datavec.hadoop.records.reader.mapfile.index.LongIndexToKey; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; +import org.nd4j.common.util.MathUtils; + +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.*; + +/** + * A {@link RecordReader} implementation for reading from a Hadoop {@link org.apache.hadoop.io.MapFile}
+ *

+ * A typical use case is with {@link org.datavec.api.transform.TransformProcess} executed on Spark (perhaps Spark + * local), followed by non-distributed training on a single machine. For example: + * 

+ *  {@code
+ *  JavaRDD> myRDD = ...;
+ *  String mapFilePath = ...;
+ *  SparkStorageUtils.saveMapFile( mapFilePath, myRDD );
+ *
+ *  RecordReader rr = new MapFileRecordReader();
+ *  rr.initialize( new FileSplit( new File( mapFilePath ) ) );
+ *  //Pass to DataSetIterator or similar
+ *  }
+ *
+ * + * Alternatively, use {@link org.datavec.hadoop.records.writer.mapfile.MapFileRecordWriter}.
+ * Note that this record reader supports optional randomisation of order. + * + * @author Alex Black + */ +public class MapFileRecordReader implements RecordReader { + private static final Class recordClass = RecordWritable.class; + + private final IndexToKey indexToKey; + private MapFileReader mapFileReader; + private URI baseDirUri; + private List listeners; + + private long numRecords; + private long position; + private Random rng; + private int[] order; + + /** + * Create a MapFileRecordReader with no randomisation, and assuming MapFile keys are {@link org.apache.hadoop.io.LongWritable} + * values + */ + public MapFileRecordReader() throws Exception { + this(new LongIndexToKey(), null); + } + + /** + * Create a MapFileRecordReader with optional randomisation, and assuming MapFile keys are + * {@link org.apache.hadoop.io.LongWritable} values + * + * @param rng If non-null, will be used to randomize the order of examples + * + */ + public MapFileRecordReader(Random rng) { + this(new LongIndexToKey(), rng); + } + + /** + * Create a MapFileRecordReader with optional randomisation, with a custom {@link IndexToKey} instance to + * handle MapFile keys + * + * @param indexToKey Handles conversion between long indices and key values (see for example {@link LongIndexToKey} + * @param rng If non-null, will be used to randomize the order of examples + * + */ + public MapFileRecordReader(IndexToKey indexToKey, Random rng) { + this.indexToKey = indexToKey; + this.rng = rng; + } + + @Override + public void initialize(InputSplit split) throws IOException, InterruptedException { + initialize(null, split); + } + + @Override + public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException { + URI[] uris = split.locations(); + + //First: work out whether we have a single MapFile or multiple parts + int dataCount = 0; + int indexCount = 0; + List dataUris = new ArrayList<>(); + for (URI u : uris) { + String p = u.getPath(); + if (p.endsWith("data")) { + dataCount++; + dataUris.add(u); + } else if (p.endsWith("index")) { + indexCount++; + } + } + + //Check URIs are correct: we expect one or more /data and /index files... + if (dataCount == 0 || indexCount == 0) { + throw new IllegalStateException("Cannot initialize MapFileSequenceRecordReader: could not find data and " + + "index files in input split"); + } + if (dataCount != indexCount) { + throw new IllegalStateException("Invalid input: found " + dataCount + " data files but " + indexCount + + " index files. Expect equal number of both for map files"); + } + + List mapFilePartRootDirectories = new ArrayList<>(dataUris.size()); + for (URI u : dataUris) { + File partRootDir = new File(u).getParentFile(); + mapFilePartRootDirectories.add(partRootDir.getAbsolutePath()); + } + + //Sort the paths so we iterate over multi-part MapFiles like part-r-00000, part-r-00001, etc when not randomized + Collections.sort(mapFilePartRootDirectories); + + + if (dataUris.size() == 1) { + //Just parent of /data + baseDirUri = new File(dataUris.get(0)).getParentFile().toURI(); + } else { + //Multiple parts -> up 2 levels from data + //so, /baseDir/part-r-00000/data -> /baseDir + baseDirUri = new File(dataUris.get(0)).getParentFile().getParentFile().toURI(); + } + + if (mapFileReader != null) { + mapFileReader.close(); + } + + this.mapFileReader = new MapFileReader<>(mapFilePartRootDirectories, indexToKey, recordClass); + this.numRecords = mapFileReader.numRecords(); + + if (rng != null) { + order = new int[(int) numRecords]; + for (int i = 0; i < order.length; i++) { + order[i] = i; + } + MathUtils.shuffleArray(order, rng); + } + } + + @Override + public void setConf(Configuration conf) { + + } + + @Override + public Configuration getConf() { + return null; + } + + @Override + public boolean batchesSupported() { + return false; + } + + @Override + public List> next(int num) { + throw new UnsupportedOperationException(); + } + + @Override + public List next() { + return next(false).getRecord(); + } + + @Override + public boolean hasNext() { + return position < numRecords; + } + + @Override + public List getLabels() { + return null; + } + + @Override + public void reset() { + position = 0; + if (order != null) { + MathUtils.shuffleArray(order, rng); + } + } + + @Override + public boolean resetSupported() { + return true; + } + + @Override + public List record(URI uri, DataInputStream dataInputStream) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Record nextRecord() { + return next(true); + } + + @Override + public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public List loadFromMetaData(List recordMetaDatas) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public List getListeners() { + return listeners; + } + + @Override + public void setListeners(RecordListener... listeners) { + this.listeners = Arrays.asList(listeners); + } + + @Override + public void setListeners(Collection listeners) { + this.listeners = new ArrayList<>(listeners); + } + + @Override + public void close() throws IOException { + if (mapFileReader != null) { + mapFileReader.close(); + } + } + + + private Record next(boolean withMetadata) { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + RecordWritable rec; + long currIdx; + if (order != null) { + currIdx = order[(int) position++]; + } else { + currIdx = position++; + } + + try { + rec = mapFileReader.getRecord(currIdx); + } catch (IOException e) { + throw new RuntimeException(e); + } + + RecordMetaData meta; + if (withMetadata) { + meta = new RecordMetaDataIndex(currIdx, baseDirUri, MapFileRecordReader.class); + } else { + meta = null; + } + + if (listeners != null && !listeners.isEmpty()) { + for (RecordListener l : listeners) { + l.recordRead(this, rec); + } + } + + return new org.datavec.api.records.impl.Record(rec.getRecord(), meta); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileSequenceRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileSequenceRecordReader.java new file mode 100644 index 000000000..3a0513132 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/MapFileSequenceRecordReader.java @@ -0,0 +1,330 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader.mapfile; + +import lombok.NonNull; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.SequenceRecord; +import org.datavec.api.records.listener.RecordListener; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.metadata.RecordMetaDataIndex; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.Writable; +import org.datavec.hadoop.records.reader.mapfile.index.LongIndexToKey; +import org.datavec.hadoop.records.reader.mapfile.record.SequenceRecordWritable; +import org.nd4j.common.util.MathUtils; + +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.*; + +/** + * A {@link SequenceRecordReader} implementation for reading from a Hadoop {@link org.apache.hadoop.io.MapFile}
+ *

+ * A typical use case is with {@link org.datavec.api.transform.TransformProcess} executed on Spark (perhaps Spark + * local), followed by non-distributed training on a single machine. For example: + * 

+ *  {@code
+ *  JavaRDD>> myRDD = ...;
+ *  String mapFilePath = ...;
+ *  SparkStorageUtils.saveMapFileSequences( mapFilePath, myRDD );
+ *
+ *  SequenceRecordReader rr = new MapFileSequenceRecordReader();
+ *  rr.initialize( new FileSplit( new File( mapFilePath ) ) );
+ *  //Pass to DataSetIterator or similar
+ *  }
+ *
+ * + * Alternatively, use {@link org.datavec.hadoop.records.writer.mapfile.MapFileSequenceRecordWriter}.
+ * Note that this sequence record reader supports optional randomisation of order. + * + * @author Alex Black + */ +public class MapFileSequenceRecordReader implements SequenceRecordReader { + private static final Class recordClass = SequenceRecordWritable.class; + + private final IndexToKey indexToKey; + private MapFileReader mapFileReader; + private URI baseDirUri; + private List listeners; + + private long numSequences; + private long position; + private Random rng; + private int[] order; + + /** + * Create a MapFileSequenceRecordReader with no randomisation, and assuming MapFile keys are {@link org.apache.hadoop.io.LongWritable} + * values + */ + public MapFileSequenceRecordReader() { + this(new LongIndexToKey(), null); + } + + /** + * Create a MapFileSequenceRecordReader with optional randomisation, and assuming MapFile keys are + * {@link org.apache.hadoop.io.LongWritable} values + * + * @param rng If non-null, will be used to randomize the order of examples + * + */ + public MapFileSequenceRecordReader(Random rng) { + this(new LongIndexToKey(), rng); + } + + /** + * Create a MapFileSequenceRecordReader with optional randomisation, with a custom {@link IndexToKey} instance to + * handle MapFile keys + * + * @param indexToKey Handles conversion between long indices and key values (see for example {@link LongIndexToKey} + * @param rng If non-null, will be used to randomize the order of examples + * + */ + public MapFileSequenceRecordReader(IndexToKey indexToKey, Random rng) { + this.indexToKey = indexToKey; + this.rng = rng; + } + + @Override + public void initialize(InputSplit split) throws IOException, InterruptedException { + initialize(null, split); + } + + @Override + public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException { + URI[] uris = split.locations(); + + //First: work out whether we have a single MapFile or multiple parts + int dataCount = 0; + int indexCount = 0; + List dataUris = new ArrayList<>(); + for (URI u : uris) { + String p = u.getPath(); + if (p.endsWith("data")) { + dataCount++; + dataUris.add(u); + } else if (p.endsWith("index")) { + indexCount++; + } + } + + //Check URIs are correct: we expect one or more /data and /index files... + if (dataCount == 0 || indexCount == 0) { + throw new IllegalStateException("Cannot initialize MapFileSequenceRecordReader: could not find data and " + + "index files in input split"); + } + if (dataCount != indexCount) { + throw new IllegalStateException("Invalid input: found " + dataCount + " data files but " + indexCount + + " index files. Expect equal number of both for map files"); + } + + List mapFilePartRootDirectories = new ArrayList<>(dataUris.size()); + for (URI u : dataUris) { + File partRootDir = new File(u).getParentFile(); + mapFilePartRootDirectories.add(partRootDir.getAbsolutePath()); + } + + //Sort the paths so we iterate over multi-part MapFiles like part-r-00000, part-r-00001, etc when not randomized + Collections.sort(mapFilePartRootDirectories); + + + if (dataUris.size() == 1) { + //Just parent of /data + baseDirUri = new File(dataUris.get(0)).getParentFile().toURI(); + } else { + //Multiple parts -> up 2 levels from data + //so, /baseDir/part-r-00000/data -> /baseDir + baseDirUri = new File(dataUris.get(0)).getParentFile().getParentFile().toURI(); + } + + if (mapFileReader != null) { + mapFileReader.close(); + } + + this.mapFileReader = new MapFileReader<>(mapFilePartRootDirectories, indexToKey, recordClass); + this.numSequences = mapFileReader.numRecords(); + + if (rng != null) { + order = new int[(int) numSequences]; + for (int i = 0; i < order.length; i++) { + order[i] = i; + } + MathUtils.shuffleArray(order, rng); + } + } + + @Override + public void setConf(Configuration conf) { + + } + + @Override + public Configuration getConf() { + return null; + } + + @Override + public List> sequenceRecord() { + return nextSequence(false).getSequenceRecord(); + } + + @Override + public List> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException { + throw new UnsupportedOperationException("MapFileSequenceRecordReader: does not support reading from streams"); + } + + @Override + public SequenceRecord nextSequence() { + return nextSequence(true); + } + + private SequenceRecord nextSequence(boolean withMetadata) { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + SequenceRecordWritable seq; + long currIdx; + if (order != null) { + currIdx = order[(int) position++]; + } else { + currIdx = position++; + } + + try { + seq = mapFileReader.getRecord(currIdx); + } catch (IOException e) { + throw new RuntimeException(e); + } + + RecordMetaData meta; + if (withMetadata) { + meta = new RecordMetaDataIndex(currIdx, baseDirUri, MapFileSequenceRecordReader.class); + } else { + meta = null; + } + + if (listeners != null && !listeners.isEmpty()) { + for (RecordListener l : listeners) { + l.recordRead(this, seq); + } + } + + return new org.datavec.api.records.impl.SequenceRecord(seq.getSequenceRecord(), meta); + } + + @Override + public SequenceRecord loadSequenceFromMetaData(@NonNull RecordMetaData recordMetaData) throws IOException { + long idx = ((RecordMetaDataIndex) recordMetaData).getIndex(); + return new org.datavec.api.records.impl.SequenceRecord(mapFileReader.getRecord(idx).getSequenceRecord(), + recordMetaData); + } + + @Override + public List loadSequenceFromMetaData(@NonNull List recordMetaDatas) + throws IOException { + List out = new ArrayList<>(recordMetaDatas.size()); + for (RecordMetaData r : recordMetaDatas) { + out.add(loadSequenceFromMetaData(r)); + } + return out; + } + + @Override + public boolean batchesSupported() { + return false; + } + + @Override + public List> next(int num) { + throw new UnsupportedOperationException(); + } + + @Override + public List next() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasNext() { + return position < numSequences; + } + + @Override + public List getLabels() { + return null; + } + + @Override + public void reset() { + position = 0; + if (order != null) { + MathUtils.shuffleArray(order, rng); + } + } + + @Override + public boolean resetSupported() { + return true; + } + + @Override + public List record(URI uri, DataInputStream dataInputStream) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Record nextRecord() { + throw new UnsupportedOperationException(); + } + + @Override + public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public List loadFromMetaData(List recordMetaDatas) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public List getListeners() { + return listeners; + } + + @Override + public void setListeners(RecordListener... listeners) { + this.listeners = Arrays.asList(listeners); + } + + @Override + public void setListeners(Collection listeners) { + this.listeners = new ArrayList<>(listeners); + } + + @Override + public void close() throws IOException { + if (mapFileReader != null) { + mapFileReader.close(); + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/index/LongIndexToKey.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/index/LongIndexToKey.java new file mode 100644 index 000000000..6e9225a4a --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/index/LongIndexToKey.java @@ -0,0 +1,132 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader.mapfile.index; + +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.MapFile; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.ReflectionUtils; +import org.datavec.hadoop.records.reader.mapfile.IndexToKey; +import org.nd4j.common.primitives.Pair; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * A default implementation of {@link IndexToKey} that assumes (strictly requires) keys that are + * {@link LongWritable} values, where all values are both unique and contiguous (0 to numRecords()-1)
+ * This allows for easy inference of the number of records, and identify mapping between indexes and keys. + * + * @author Alex Black + */ +public class LongIndexToKey implements IndexToKey { + + private List> readerIndices; + + @Override + public List> initialize(MapFile.Reader[] readers, Class valueClass) + throws IOException { + + List> l = new ArrayList<>(readers.length); + for (MapFile.Reader r : readers) { + //Get the first and last keys: + long first = -1; + long last = -1; + + //First key: no method for this for some inexplicable reason :/ + LongWritable k = new LongWritable(); + Writable v = ReflectionUtils.newInstance(valueClass, null); + boolean hasNext = r.next(k, v); + if(!hasNext){ + //This map file is empty - no data + l.add(new Pair<>(-1L, -1L)); + continue; + } + first = k.get(); + + //Last key: easy + r.reset(); + r.finalKey(k); + last = k.get(); + + l.add(new Pair<>(first, last)); + } + + //Check that things are actually contiguous: + List> sorted = new ArrayList<>(l.size()); + for(Pair p : l){ + if(p.getLeft() >= 0){ + sorted.add(p); + } + } + Collections.sort(sorted, new Comparator>() { + @Override + public int compare(Pair o1, Pair o2) { + return Long.compare(o1.getFirst(), o2.getFirst()); + } + }); + + if (sorted.size() == 0){ + throw new IllegalStateException("Map file is empty - no data available"); + } + if (sorted.get(0).getFirst() != 0L) { + throw new UnsupportedOperationException("Minimum key value is not 0: got " + sorted.get(0).getFirst()); + } + + for (int i = 0; i < sorted.size() - 1; i++) { + long currLast = sorted.get(i).getSecond(); + long nextFirst = sorted.get(i + 1).getFirst(); + + if(nextFirst == -1){ + //Skip empty map file + continue; + } + + if (currLast + 1 != nextFirst) { + throw new IllegalStateException( + "Keys are not contiguous between readers: first/last indices (inclusive) " + "are " + + sorted + + ".\n LongIndexKey assumes unique and contiguous LongWritable keys"); + } + } + + readerIndices = l; + return readerIndices; + } + + @Override + public LongWritable getKeyForIndex(long index) { + return new LongWritable(index); + } + + @Override + public long getNumRecords() throws IOException { + long max = -1; + for (Pair p : readerIndices) { + max = Math.max(max, p.getSecond()); + } + + if (max <= 0) { + throw new IllegalStateException("Invalid number of keys found: " + max); + } + + return max + 1; //Zero indexed + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/record/RecordWritable.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/record/RecordWritable.java new file mode 100644 index 000000000..139f28ce9 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/record/RecordWritable.java @@ -0,0 +1,59 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader.mapfile.record; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.hadoop.io.Writable; +import org.datavec.api.writable.WritableFactory; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Created by Alex on 29/05/2017. + */ +@AllArgsConstructor +@NoArgsConstructor +@Data +public class RecordWritable implements Writable { + private List record; + + @Override + public void write(DataOutput out) throws IOException { + WritableFactory wf = WritableFactory.getInstance(); + out.writeInt(record.size()); + for (org.datavec.api.writable.Writable w : record) { + wf.writeWithType(w, out); + } + } + + @Override + public void readFields(DataInput in) throws IOException { + WritableFactory wf = WritableFactory.getInstance(); + int numRecords = in.readInt(); + + record = new ArrayList<>(numRecords); + for (int i = 0; i < numRecords; i++) { + record.add(wf.readWithType(in)); + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/record/SequenceRecordWritable.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/record/SequenceRecordWritable.java new file mode 100644 index 000000000..1511de990 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/reader/mapfile/record/SequenceRecordWritable.java @@ -0,0 +1,82 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader.mapfile.record; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.hadoop.io.Writable; +import org.datavec.api.writable.WritableFactory; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Created by Alex on 29/05/2017. + */ +@AllArgsConstructor +@NoArgsConstructor +@Data +public class SequenceRecordWritable implements Writable { + private List> sequenceRecord; + + @Override + public void write(DataOutput out) throws IOException { + WritableFactory wf = WritableFactory.getInstance(); + //Assumption: each step in each record is the same size + out.writeInt(sequenceRecord.size()); + if (sequenceRecord.size() > 0) { + int valuesPerStep = sequenceRecord.get(0).size(); + out.writeInt(valuesPerStep); + + for (List step : sequenceRecord) { + if (step.size() != valuesPerStep) { + throw new IllegalStateException( + "Number of values per time step vary: " + valuesPerStep + " vs. " + step.size()); + } + for (org.datavec.api.writable.Writable w : step) { + wf.writeWithType(w, out); + } + } + } + } + + @Override + public void readFields(DataInput in) throws IOException { + WritableFactory wf = WritableFactory.getInstance(); + int numSteps = in.readInt(); + if (numSteps > 0) { + int valuesPerStep = in.readInt(); + List> out = new ArrayList<>(numSteps); + + for (int i = 0; i < numSteps; i++) { + List currStep = new ArrayList<>(valuesPerStep); + for (int j = 0; j < valuesPerStep; j++) { + currStep.add(wf.readWithType(in)); + } + out.add(currStep); + } + sequenceRecord = out; + } else { + sequenceRecord = Collections.emptyList(); + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/AbstractMapFileWriter.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/AbstractMapFileWriter.java new file mode 100644 index 000000000..b87db7101 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/AbstractMapFileWriter.java @@ -0,0 +1,280 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.writer.mapfile; + +import lombok.NonNull; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.MapFile; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.WritableComparable; +import org.datavec.api.conf.Configuration; +import org.datavec.api.split.partition.PartitionMetaData; +import org.datavec.api.writable.*; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * An abstract class For creating Hadoop map files, that underlies {@link MapFileRecordWriter} and + * {@link MapFileSequenceRecordWriter}. + * + * @author Alex Black + */ +public abstract class AbstractMapFileWriter { + + public static final String DEFAULT_FILENAME_PATTERN = "part-r-%1$05d"; + public static final Class KEY_CLASS = org.apache.hadoop.io.LongWritable.class; + + /** + * Configuration key for the map file interval. + * This is defined in MapFile.Writer.INDEX_INTERVAL but unfortunately that field is private, hence cannot be + * referenced here. + */ + public static final String MAP_FILE_INDEX_INTERVAL_KEY = "io.map.index.interval"; + + public static final int DEFAULT_MAP_FILE_SPLIT_SIZE = -1; + public static final int DEFAULT_INDEX_INTERVAL = 1; + + protected final File outputDir; + protected final int mapFileSplitSize; + protected final WritableType convertTextTo; + protected final int indexInterval; + protected final String filenamePattern; + protected org.apache.hadoop.conf.Configuration hadoopConfiguration; + + protected final AtomicLong counter = new AtomicLong(); + protected final AtomicBoolean isClosed = new AtomicBoolean(); + + protected List outputFiles = new ArrayList<>(); + protected List writers = new ArrayList<>(); + + + + protected SequenceFile.Writer.Option[] opts; + + + /** + * Constructor for all default values. Single output MapFile, no text writable conversion, default index + * interval (1), default naming pattern. + * + * @param outputDir Output directory for the map file(s) + */ + public AbstractMapFileWriter(File outputDir) { + this(outputDir, DEFAULT_MAP_FILE_SPLIT_SIZE); + } + + /** + * + * Constructor for most default values. Specified number of output MapFile s, no text writable conversion, default + * index interval (1), default naming pattern. + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize. + * This can be used to avoid having a single multi gigabyte map file, which may be + * undesirable in some cases (transfer across the network, for example) + */ + public AbstractMapFileWriter(@NonNull File outputDir, int mapFileSplitSize) { + this(outputDir, mapFileSplitSize, null); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + */ + public AbstractMapFileWriter(@NonNull File outputDir, WritableType convertTextTo) { + this(outputDir, DEFAULT_MAP_FILE_SPLIT_SIZE, convertTextTo); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize. + * This can be used to avoid having a single multi gigabyte map file, which may be + * undesirable in some cases (transfer across the network, for example) + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + */ + public AbstractMapFileWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo) { + this(outputDir, mapFileSplitSize, convertTextTo, DEFAULT_INDEX_INTERVAL, new org.apache.hadoop.conf.Configuration()); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize. + * This can be used to avoid having a single multi gigabyte map file, which may be + * undesirable in some cases (transfer across the network, for example) + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + * @param indexInterval Index interval for the Map file. Defaults to 1, which is suitable for most cases + * @param hadoopConfiguration Hadoop configuration. + */ + public AbstractMapFileWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo, + int indexInterval, org.apache.hadoop.conf.Configuration hadoopConfiguration) { + this(outputDir, mapFileSplitSize, convertTextTo, indexInterval, DEFAULT_FILENAME_PATTERN, hadoopConfiguration); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize. + * This can be used to avoid having a single multi gigabyte map file, which may be + * undesirable in some cases (transfer across the network, for example) + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + * @param indexInterval Index interval for the Map file. Defaults to 1, which is suitable for most cases + * @param filenamePattern The naming pattern for the map files. Used with String.format(pattern, int) + * @param hadoopConfiguration Hadoop configuration. + */ + public AbstractMapFileWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo, + int indexInterval, String filenamePattern, + org.apache.hadoop.conf.Configuration hadoopConfiguration) { + if(indexInterval <= 0){ + throw new UnsupportedOperationException("Index interval: must be >= 0 (got: " + indexInterval + ")"); + } + this.outputDir = outputDir; + this.mapFileSplitSize = mapFileSplitSize; + if (convertTextTo == WritableType.Text) { + convertTextTo = null; + } + this.convertTextTo = convertTextTo; + this.indexInterval = indexInterval; + this.filenamePattern = filenamePattern; + + this.hadoopConfiguration = hadoopConfiguration; + if(this.hadoopConfiguration.get(MAP_FILE_INDEX_INTERVAL_KEY) != null){ + this.hadoopConfiguration.set(MAP_FILE_INDEX_INTERVAL_KEY, String.valueOf(indexInterval)); + } + + opts = new SequenceFile.Writer.Option[]{MapFile.Writer.keyClass(KEY_CLASS), + SequenceFile.Writer.valueClass(getValueClass())}; + + } + + protected abstract Class getValueClass(); + + + public void setConf(Configuration conf) { + + } + + + public Configuration getConf() { + return null; + } + + protected abstract org.apache.hadoop.io.Writable getHadoopWritable(T input); + + protected List convertTextWritables(List record) { + List newList; + if (convertTextTo != null) { + newList = new ArrayList<>(record.size()); + for (Writable writable : record) { + Writable newWritable; + if (writable.getType() == WritableType.Text) { + switch (convertTextTo) { + case Byte: + newWritable = new ByteWritable((byte) writable.toInt()); + break; + case Double: + newWritable = new DoubleWritable(writable.toDouble()); + break; + case Float: + newWritable = new FloatWritable(writable.toFloat()); + break; + case Int: + newWritable = new IntWritable(writable.toInt()); + break; + case Long: + newWritable = new org.datavec.api.writable.LongWritable(writable.toLong()); + break; + default: + throw new UnsupportedOperationException("Cannot convert text to: " + convertTextTo); + } + } else { + newWritable = writable; + } + newList.add(newWritable); + } + } else { + newList = record; + } + + return newList; + } + + public PartitionMetaData write(T record) throws IOException { + if (isClosed.get()) { + throw new UnsupportedOperationException("Cannot write to MapFileRecordReader that has already been closed"); + } + + if (counter.get() == 0) { + //Initialize first writer + String filename = String.format(DEFAULT_FILENAME_PATTERN, 0); + outputFiles.add(new File(outputDir, filename)); + writers.add(new MapFile.Writer(hadoopConfiguration, new Path(outputFiles.get(0).getAbsolutePath()), opts)); + } + + long key = counter.getAndIncrement(); + MapFile.Writer w; + if (mapFileSplitSize <= 0) { + w = writers.get(0); + } else { + int splitIdx = (int) (key / mapFileSplitSize); + if (writers.size() <= splitIdx) { + //Initialize new writer - next split + String filename = String.format(DEFAULT_FILENAME_PATTERN, splitIdx); + outputFiles.add(new File(outputDir, filename)); + writers.add(new MapFile.Writer(hadoopConfiguration, new Path(outputFiles.get(splitIdx).getAbsolutePath()), opts)); + } + w = writers.get(splitIdx); + } + + org.apache.hadoop.io.Writable hadoopWritable = getHadoopWritable(record); + + w.append(new org.apache.hadoop.io.LongWritable(key), hadoopWritable); + + return PartitionMetaData.builder().numRecordsUpdated(1).build(); + } + + + public void close() { + try { + for (MapFile.Writer w : writers) { + w.close(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + isClosed.set(true); + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/MapFileRecordWriter.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/MapFileRecordWriter.java new file mode 100644 index 000000000..bf0479805 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/MapFileRecordWriter.java @@ -0,0 +1,184 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.writer.mapfile; + +import lombok.NonNull; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.writer.RecordWriter; +import org.datavec.api.split.InputSplit; +import org.datavec.api.split.partition.PartitionMetaData; +import org.datavec.api.split.partition.Partitioner; +import org.datavec.api.writable.Writable; +import org.datavec.api.writable.WritableType; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * MapFileRecordWriter is used to write values to a Hadoop MapFile, that can then be read by: + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader} + * + * @author Alex Black + * @see org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader + */ +public class MapFileRecordWriter extends AbstractMapFileWriter> implements RecordWriter { + + /** + * Constructor for all default values. Single output MapFile, no text writable conversion, default index + * interval (1), default naming pattern. + * + * @param outputDir Output directory for the map file(s) + */ + public MapFileRecordWriter(File outputDir) { + super(outputDir); + } + + /** + * + * Constructor for most default values. Specified number of output MapFile s, no text writable conversion, default + * index interval (1), default naming pattern. + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + */ + public MapFileRecordWriter(@NonNull File outputDir, int mapFileSplitSize){ + this(outputDir, mapFileSplitSize, null); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + */ + public MapFileRecordWriter(@NonNull File outputDir, WritableType convertTextTo) { + this(outputDir, DEFAULT_MAP_FILE_SPLIT_SIZE, convertTextTo); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + */ + public MapFileRecordWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo) { + super(outputDir, mapFileSplitSize, convertTextTo); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + * @param hadoopConfiguration Hadoop configuration. + */ + public MapFileRecordWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo, + org.apache.hadoop.conf.Configuration hadoopConfiguration) { + super(outputDir, mapFileSplitSize, convertTextTo, DEFAULT_INDEX_INTERVAL, hadoopConfiguration); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + * @param indexInterval Index interval for the Map file. Defaults to 1, which is suitable for most cases + * @param hadoopConfiguration Hadoop configuration. + */ + public MapFileRecordWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo, + int indexInterval, org.apache.hadoop.conf.Configuration hadoopConfiguration) { + super(outputDir, mapFileSplitSize, convertTextTo, indexInterval, hadoopConfiguration); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + * @param indexInterval Index interval for the Map file. Defaults to 1, which is suitable for most cases + * @param filenamePattern The naming pattern for the map files. Used with String.format(pattern, int) + * @param hadoopConfiguration Hadoop configuration. + */ + public MapFileRecordWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo, + int indexInterval, String filenamePattern, + org.apache.hadoop.conf.Configuration hadoopConfiguration) { + super(outputDir, mapFileSplitSize, convertTextTo, indexInterval, filenamePattern, hadoopConfiguration); + } + + @Override + protected Class getValueClass() { + return RecordWritable.class; + } + + @Override + protected org.apache.hadoop.io.Writable getHadoopWritable(List input) { + if(convertTextTo != null){ + input = convertTextWritables(input); + } + + return new RecordWritable(input); + } + + @Override + public boolean supportsBatch() { + return true; + } + + @Override + public void initialize(InputSplit inputSplit, Partitioner partitioner) throws Exception { + + } + + @Override + public void initialize(Configuration configuration, InputSplit split, Partitioner partitioner) throws Exception { + + } + + @Override + public PartitionMetaData writeBatch(List> batch) throws IOException { + for (List record : batch) { + write(record); + } + return PartitionMetaData.builder().numRecordsUpdated(batch.size()).build(); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/MapFileSequenceRecordWriter.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/MapFileSequenceRecordWriter.java new file mode 100644 index 000000000..878bd4348 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/main/java/org/datavec/hadoop/records/writer/mapfile/MapFileSequenceRecordWriter.java @@ -0,0 +1,161 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.writer.mapfile; + +import lombok.NonNull; +import org.datavec.api.records.writer.SequenceRecordWriter; +import org.datavec.api.writable.Writable; +import org.datavec.api.writable.WritableType; +import org.datavec.hadoop.records.reader.mapfile.record.SequenceRecordWritable; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +/** + * MapFileSequenceRecordWriter is used to write sequence values to a Hadoop MapFile, that can then be read by: + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader} + * + * @author Alex Black + * @see org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader + */ +public class MapFileSequenceRecordWriter extends AbstractMapFileWriter>> implements SequenceRecordWriter { + + /** + * Constructor for all default values. Single output MapFile, no text writable conversion, default index + * interval (1), default naming pattern. + * + * @param outputDir Output directory for the map file(s) + */ + public MapFileSequenceRecordWriter(File outputDir) { + super(outputDir); + } + + /** + * + * Constructor for most default values. Specified number of output MapFile s, no text writable conversion, default + * index interval (1), default naming pattern. + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + */ + public MapFileSequenceRecordWriter(@NonNull File outputDir, int mapFileSplitSize){ + this(outputDir, mapFileSplitSize, null); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + */ + public MapFileSequenceRecordWriter(@NonNull File outputDir, WritableType convertTextTo) { + this(outputDir, DEFAULT_MAP_FILE_SPLIT_SIZE, convertTextTo); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + */ + public MapFileSequenceRecordWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo) { + super(outputDir, mapFileSplitSize, convertTextTo); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + * @param hadoopConfiguration Hadoop configuration. + */ + public MapFileSequenceRecordWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo, + org.apache.hadoop.conf.Configuration hadoopConfiguration) { + super(outputDir, mapFileSplitSize, convertTextTo, DEFAULT_INDEX_INTERVAL, hadoopConfiguration); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + * @param indexInterval Index interval for the Map file. Defaults to 1, which is suitable for most cases + * @param hadoopConfiguration Hadoop configuration. + */ + public MapFileSequenceRecordWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo, + int indexInterval, org.apache.hadoop.conf.Configuration hadoopConfiguration) { + super(outputDir, mapFileSplitSize, convertTextTo, indexInterval, hadoopConfiguration); + } + + /** + * + * @param outputDir Output directory for the map file(s) + * @param mapFileSplitSize Split size for the map file: if 0, use a single map file for all output. If > 0, + * multiple map files will be used: each will contain a maximum of mapFileSplitSize + * examples. This can be used to avoid having a single multi gigabyte map file, which may + * be undesirable in some cases (transfer across the network, for example). + * @param convertTextTo If null: Make no changes to Text writable objects. If non-null, Text writable instances + * will be converted to this type. This is useful, when would rather store numerical values + * even if the original record reader produces strings/text. + * @param indexInterval Index interval for the Map file. Defaults to 1, which is suitable for most cases + * @param filenamePattern The naming pattern for the map files. Used with String.format(pattern, int) + * @param hadoopConfiguration Hadoop configuration. + */ + public MapFileSequenceRecordWriter(@NonNull File outputDir, int mapFileSplitSize, WritableType convertTextTo, + int indexInterval, String filenamePattern, + org.apache.hadoop.conf.Configuration hadoopConfiguration) { + super(outputDir, mapFileSplitSize, convertTextTo, indexInterval, filenamePattern, hadoopConfiguration); + } + + @Override + protected Class getValueClass() { + return SequenceRecordWritable.class; + } + + @Override + protected org.apache.hadoop.io.Writable getHadoopWritable(List> input) { + if(convertTextTo != null){ + List> newSeq = new ArrayList<>(input.size()); + for(List l : input){ + newSeq.add(convertTextWritables(l)); + } + input = newSeq; + } + + return new SequenceRecordWritable(input); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..7464b95b6 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/AssertTestsExtendBaseClass.java @@ -0,0 +1,49 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.hadoop; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseND4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.hadoop"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/conf/TestConfigurationUtil.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/conf/TestConfigurationUtil.java new file mode 100644 index 000000000..ff44aa118 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/conf/TestConfigurationUtil.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.conf; + +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; + +public class TestConfigurationUtil { + + @Test + public void testLoadHadoopConfFiles() { + + // this would come from the properties file + String confPath = "src/test/resources/conf/example_conf/"; + + Configuration conf = ConfigurationUtil.generateConfig(confPath); + + System.out.println(" works? " + conf.get("fs.default.name")); + + + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReader.java new file mode 100644 index 000000000..d5595e53d --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReader.java @@ -0,0 +1,247 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader; + +import org.junit.jupiter.api.AfterAll; +import org.nd4j.common.util.MathUtils; +import com.google.common.io.Files; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.*; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Text; +import org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader; +import org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; +import org.datavec.hadoop.records.reader.mapfile.record.SequenceRecordWritable; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URI; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Created by Alex on 29/05/2017. + */ +public class TestMapFileRecordReader { + + private static File tempDirSeq; + private static File tempDir; + private static Path seqMapFilePath; + private static Path mapFilePath; + private static Map seqMap; + private static Map recordMap; + + @BeforeAll + public static void buildMapFiles() throws IOException { + + //----- Sequence RR setup ----- + + Configuration c = new Configuration(); + Class keyClass = LongWritable.class; + Class valueClass = SequenceRecordWritable.class; + + SequenceFile.Writer.Option[] opts = new SequenceFile.Writer.Option[] {MapFile.Writer.keyClass(keyClass), + SequenceFile.Writer.valueClass(valueClass)}; + + tempDirSeq = Files.createTempDir(); + seqMapFilePath = new Path("file:///" + tempDirSeq.getAbsolutePath()); + + MapFile.Writer writer = new MapFile.Writer(c, seqMapFilePath, opts); + + seqMap = new HashMap<>(); + seqMap.put(new LongWritable(0), new SequenceRecordWritable(Arrays.asList( + Arrays.asList(new Text("zero"), new IntWritable(0), + new DoubleWritable(0), new NDArrayWritable(Nd4j.valueArrayOf(10, 0.0))), + Arrays.asList(new Text("one"), new IntWritable(1), + new DoubleWritable(1.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 1.0))), + Arrays.asList(new Text("two"), new IntWritable(2), + new DoubleWritable(2.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 2.0)))))); + + seqMap.put(new LongWritable(1), new SequenceRecordWritable(Arrays.asList( + Arrays.asList(new Text("Bzero"), new IntWritable(10), + new DoubleWritable(10), new NDArrayWritable(Nd4j.valueArrayOf(10, 10.0))), + Arrays.asList(new Text("Bone"), new IntWritable(11), + new DoubleWritable(11.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 11.0))), + Arrays.asList(new Text("Btwo"), new IntWritable(12), + new DoubleWritable(12.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 12.0)))))); + + seqMap.put(new LongWritable(2), new SequenceRecordWritable(Arrays.asList( + Arrays.asList(new Text("Czero"), new IntWritable(20), + new DoubleWritable(20), new NDArrayWritable(Nd4j.valueArrayOf(10, 20.0))), + Arrays.asList(new Text("Cone"), new IntWritable(21), + new DoubleWritable(21.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 21.0))), + Arrays.asList(new Text("Ctwo"), new IntWritable(22), + new DoubleWritable(22.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 22.0)))))); + + + //Need to write in order + for (int i = 0; i <= 2; i++) { + LongWritable key = new LongWritable(i); + SequenceRecordWritable value = seqMap.get(key); + + writer.append(key, value); + } + writer.close(); + + + //----- Standard RR setup ----- + + valueClass = RecordWritable.class; + + opts = new SequenceFile.Writer.Option[] {MapFile.Writer.keyClass(keyClass), + SequenceFile.Writer.valueClass(valueClass)}; + + tempDir = Files.createTempDir(); + mapFilePath = new Path("file:///" + tempDir.getAbsolutePath()); + + writer = new MapFile.Writer(c, mapFilePath, opts); + + recordMap = new HashMap<>(); + recordMap.put(new LongWritable(0), + new RecordWritable(Arrays.asList(new Text("zero"), + new IntWritable(0), new DoubleWritable(0), + new NDArrayWritable(Nd4j.valueArrayOf(10, 0.0))))); + + recordMap.put(new LongWritable(1), + new RecordWritable(Arrays.asList(new Text("one"), + new IntWritable(11), new DoubleWritable(11.0), + new NDArrayWritable(Nd4j.valueArrayOf(10, 11.0))))); + + recordMap.put(new LongWritable(2), + new RecordWritable(Arrays.asList(new Text("two"), + new IntWritable(22), new DoubleWritable(22.0), + new NDArrayWritable(Nd4j.valueArrayOf(10, 22.0))))); + + + //Need to write in order + for (int i = 0; i <= 2; i++) { + LongWritable key = new LongWritable(i); + RecordWritable value = recordMap.get(key); + + writer.append(key, value); + } + writer.close(); + + } + + @AfterAll + public static void destroyMapFiles() { + tempDirSeq.delete(); + tempDirSeq = null; + seqMapFilePath = null; + seqMap = null; + + tempDir.delete(); + tempDir = null; + mapFilePath = null; + seqMap = null; + } + + @Test + public void testSequenceRecordReader() throws Exception { + SequenceRecordReader seqRR = new MapFileSequenceRecordReader(); + URI uri = seqMapFilePath.toUri(); + InputSplit is = new FileSplit(new File(uri)); + seqRR.initialize(is); + + assertTrue(seqRR.hasNext()); + int count = 0; + while (seqRR.hasNext()) { + List> l = seqRR.sequenceRecord(); + + assertEquals(seqMap.get(new LongWritable(count)).getSequenceRecord(), l); + + count++; + } + assertEquals(seqMap.size(), count); + + seqRR.close(); + + //Try the same thing, but with random order + seqRR = new MapFileSequenceRecordReader(new Random(12345)); + seqRR.initialize(is); + + Field f = MapFileSequenceRecordReader.class.getDeclaredField("order"); + f.setAccessible(true); + int[] order = (int[]) f.get(seqRR); + assertNotNull(order); + int[] expOrder = new int[]{0,1,2}; + MathUtils.shuffleArray(expOrder, new Random(12345)); + assertArrayEquals(expOrder, order); + + count = 0; + while (seqRR.hasNext()) { + List> l = seqRR.sequenceRecord(); + assertEquals(seqMap.get(new LongWritable(expOrder[count])).getSequenceRecord(), l); + count++; + } + } + + @Test + public void testRecordReader() throws Exception { + RecordReader rr = new MapFileRecordReader(); + URI uri = mapFilePath.toUri(); + InputSplit is = new FileSplit(new File(uri)); + rr.initialize(is); + + assertTrue(rr.hasNext()); + int count = 0; + while (rr.hasNext()) { + List l = rr.next(); + + assertEquals(recordMap.get(new LongWritable(count)).getRecord(), l); + + count++; + } + assertEquals(recordMap.size(), count); + + rr.close(); + + //Try the same thing, but with random order + rr = new MapFileRecordReader(new Random(12345)); + rr.initialize(is); + + Field f = MapFileRecordReader.class.getDeclaredField("order"); + f.setAccessible(true); + int[] order = (int[]) f.get(rr); + assertNotNull(order); + + int[] expOrder = new int[]{0,1,2}; + MathUtils.shuffleArray(expOrder, new Random(12345)); + assertArrayEquals(expOrder, order); + + count = 0; + while (rr.hasNext()) { + List l = rr.next(); + assertEquals(recordMap.get(new LongWritable(expOrder[count])).getRecord(), l); + count++; + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReaderMultipleParts.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReaderMultipleParts.java new file mode 100644 index 000000000..7b50373c8 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReaderMultipleParts.java @@ -0,0 +1,298 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader; + +import org.junit.jupiter.api.AfterAll; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.util.MathUtils; +import com.google.common.io.Files; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.*; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Text; +import org.datavec.hadoop.records.reader.mapfile.IndexToKey; +import org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader; +import org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader; +import org.datavec.hadoop.records.reader.mapfile.index.LongIndexToKey; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; +import org.datavec.hadoop.records.reader.mapfile.record.SequenceRecordWritable; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URI; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Basically the same as TestMapfileRecordReader, but we have multiple parts as per say a Spark save operation + * Paths are like + * /part-r-00000/data + * /part-r-00000/index + * /part-r-00001/data + * /part-r-00001/index + * /part-r-00002/data + * /part-r-00002/index + */ +public class TestMapFileRecordReaderMultipleParts { + + private static File tempDirSeq; + private static File tempDir; + private static Path seqMapFilePath; + private static Path mapFilePath; + private static Map seqMap; + private static Map recordMap; + + @BeforeAll + public static void buildMapFiles() throws IOException { + + //----- Sequence RR setup ----- + + Configuration c = new Configuration(); + Class keyClass = LongWritable.class; + Class valueClass = SequenceRecordWritable.class; + + SequenceFile.Writer.Option[] opts = new SequenceFile.Writer.Option[] {MapFile.Writer.keyClass(keyClass), + SequenceFile.Writer.valueClass(valueClass)}; + + tempDirSeq = Files.createTempDir(); + File[] subdirs = new File[3]; + Path[] paths = new Path[subdirs.length]; + MapFile.Writer[] writers = new MapFile.Writer[subdirs.length]; + for (int i = 0; i < subdirs.length; i++) { + subdirs[i] = new File(tempDirSeq, "part-r-0000" + i); + subdirs[i].mkdir(); + paths[i] = new Path("file:///" + subdirs[i].getAbsolutePath()); + writers[i] = new MapFile.Writer(c, paths[i], opts); + } + seqMapFilePath = new Path("file:///" + tempDirSeq.getAbsolutePath()); + + + + seqMap = new HashMap<>(); + + for (int i = 0; i < 9; i++) { + seqMap.put(new LongWritable(i), new SequenceRecordWritable(Arrays.asList( + Arrays.asList(new Text(i + "-0"), new IntWritable(3 * i), + new DoubleWritable(3 * i)), + Arrays.asList(new Text(i + "-1"), + new IntWritable(3 * i + 1), new DoubleWritable(3 * i + 1.0)), + Arrays.asList(new Text(i + "-2"), + new IntWritable(3 * i + 2), new DoubleWritable(3 * i + 2.0))))); + } + + + //Need to write in order, to different map files separately + for (int i = 0; i < seqMap.size(); i++) { + int mapFileIdx = i / writers.length; + + LongWritable key = new LongWritable(i); + SequenceRecordWritable value = seqMap.get(key); + + writers[mapFileIdx].append(key, value); + } + + for (MapFile.Writer m : writers) { + m.close(); + } + + + //----- Standard RR setup ----- + + valueClass = RecordWritable.class; + + opts = new SequenceFile.Writer.Option[] {MapFile.Writer.keyClass(keyClass), + SequenceFile.Writer.valueClass(valueClass)}; + + tempDir = Files.createTempDir(); + subdirs = new File[3]; + paths = new Path[subdirs.length]; + writers = new MapFile.Writer[subdirs.length]; + for (int i = 0; i < subdirs.length; i++) { + subdirs[i] = new File(tempDir, "part-r-0000" + i); + subdirs[i].mkdir(); + paths[i] = new Path("file:///" + subdirs[i].getAbsolutePath()); + writers[i] = new MapFile.Writer(c, paths[i], opts); + } + mapFilePath = new Path("file:///" + tempDir.getAbsolutePath()); + + recordMap = new HashMap<>(); + for (int i = 0; i < 9; i++) { + recordMap.put(new LongWritable(i), new RecordWritable(Arrays.asList( + new Text(String.valueOf(i)), new IntWritable(i), new DoubleWritable(i)))); + } + + + //Need to write in order + for (int i = 0; i < recordMap.size(); i++) { + int mapFileIdx = i / writers.length; + LongWritable key = new LongWritable(i); + RecordWritable value = recordMap.get(key); + + writers[mapFileIdx].append(key, value); + } + + for (MapFile.Writer m : writers) { + m.close(); + } + + } + + @AfterAll + public static void destroyMapFiles() { + tempDirSeq.delete(); + tempDirSeq = null; + seqMapFilePath = null; + seqMap = null; + + tempDir.delete(); + tempDir = null; + mapFilePath = null; + seqMap = null; + } + + @Test + public void testSequenceRecordReader() throws Exception { + SequenceRecordReader seqRR = new MapFileSequenceRecordReader(); + URI uri = seqMapFilePath.toUri(); + InputSplit is = new FileSplit(new File(uri)); + seqRR.initialize(is); + + //Check number of records calculation + Field f = MapFileSequenceRecordReader.class.getDeclaredField("indexToKey"); + f.setAccessible(true); + IndexToKey itk = (IndexToKey) f.get(seqRR); + assertEquals(seqMap.size(), itk.getNumRecords()); + + //Check indices for each map file + List> expReaderExampleIdxs = new ArrayList<>(); + expReaderExampleIdxs.add(new Pair<>(0L, 2L)); + expReaderExampleIdxs.add(new Pair<>(3L, 5L)); + expReaderExampleIdxs.add(new Pair<>(6L, 8L)); + + f = LongIndexToKey.class.getDeclaredField("readerIndices"); + f.setAccessible(true); + assertEquals(expReaderExampleIdxs, f.get(itk)); + // System.out.println(f.get(itk)); + + //Check standard iteration order (no randomization) + assertTrue(seqRR.hasNext()); + int count = 0; + while (seqRR.hasNext()) { + List> l = seqRR.sequenceRecord(); + + assertEquals(seqMap.get(new LongWritable(count)).getSequenceRecord(), l); + + count++; + } + assertFalse(seqRR.hasNext()); + assertEquals(seqMap.size(), count); + + seqRR.close(); + + //Try the same thing, but with random order + seqRR = new MapFileSequenceRecordReader(new Random(12345)); + seqRR.initialize(is); + + //Check order is defined and as expected + f = MapFileSequenceRecordReader.class.getDeclaredField("order"); + f.setAccessible(true); + int[] order = (int[]) f.get(seqRR); + assertNotNull(order); + + int[] expOrder = new int[9]; + for (int i = 0; i < expOrder.length; i++) { + expOrder[i] = i; + } + MathUtils.shuffleArray(expOrder, new Random(12345)); + assertArrayEquals(expOrder, order); + // System.out.println(Arrays.toString(expOrder)); + + count = 0; + while (seqRR.hasNext()) { + List> l = seqRR.sequenceRecord(); + assertEquals(seqMap.get(new LongWritable(expOrder[count])).getSequenceRecord(), l); + count++; + } + } + + @Test + public void testRecordReaderMultipleParts() throws Exception { + RecordReader rr = new MapFileRecordReader(); + URI uri = mapFilePath.toUri(); + InputSplit is = new FileSplit(new File(uri)); + rr.initialize(is); + + //Check number of records calculation + Field f = MapFileRecordReader.class.getDeclaredField("indexToKey"); + f.setAccessible(true); + IndexToKey itk = (IndexToKey) f.get(rr); + assertEquals(seqMap.size(), itk.getNumRecords()); + + //Check indices for each map file + List> expReaderExampleIdxs = new ArrayList<>(); + expReaderExampleIdxs.add(new Pair<>(0L, 2L)); + expReaderExampleIdxs.add(new Pair<>(3L, 5L)); + expReaderExampleIdxs.add(new Pair<>(6L, 8L)); + + f = LongIndexToKey.class.getDeclaredField("readerIndices"); + f.setAccessible(true); + assertEquals(expReaderExampleIdxs, f.get(itk)); + + assertTrue(rr.hasNext()); + int count = 0; + while (rr.hasNext()) { + List l = rr.next(); + assertEquals(recordMap.get(new LongWritable(count)).getRecord(), l); + count++; + } + assertEquals(recordMap.size(), count); + + rr.close(); + + //Try the same thing, but with random order + rr = new MapFileRecordReader(new Random(12345)); + rr.initialize(is); + + f = MapFileRecordReader.class.getDeclaredField("order"); + f.setAccessible(true); + int[] order = (int[]) f.get(rr); + assertNotNull(order); + int[] expOrder = new int[9]; + for (int i = 0; i < expOrder.length; i++) { + expOrder[i] = i; + } + MathUtils.shuffleArray(expOrder, new Random(12345)); + assertArrayEquals(expOrder, order); + + count = 0; + while (rr.hasNext()) { + List l = rr.next(); + assertEquals(recordMap.get(new LongWritable(expOrder[count])).getRecord(), l); + count++; + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReaderMultiplePartsSomeEmpty.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReaderMultiplePartsSomeEmpty.java new file mode 100644 index 000000000..ff420241b --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/reader/TestMapFileRecordReaderMultiplePartsSomeEmpty.java @@ -0,0 +1,309 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.reader; + +import org.junit.jupiter.api.AfterAll; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.util.MathUtils; +import com.google.common.io.Files; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.*; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Text; +import org.datavec.hadoop.records.reader.mapfile.IndexToKey; +import org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader; +import org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader; +import org.datavec.hadoop.records.reader.mapfile.index.LongIndexToKey; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; +import org.datavec.hadoop.records.reader.mapfile.record.SequenceRecordWritable; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URI; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Basically the same as TestMapfileRecordReader, but we have multiple parts as per say a Spark save operation + * Paths are like + * /part-r-00000/data + * /part-r-00000/index + * /part-r-00001/data + * /part-r-00001/index + * /part-r-00002/data + * /part-r-00002/index + */ +public class TestMapFileRecordReaderMultiplePartsSomeEmpty { + + private static File tempDirSeq; + private static File tempDir; + private static Path seqMapFilePath; + private static Path mapFilePath; + private static Map seqMap; + private static Map recordMap; + + @BeforeAll + public static void buildMapFiles() throws IOException { + + //----- Sequence RR setup ----- + + Configuration c = new Configuration(); + Class keyClass = LongWritable.class; + Class valueClass = SequenceRecordWritable.class; + + SequenceFile.Writer.Option[] opts = new SequenceFile.Writer.Option[] {MapFile.Writer.keyClass(keyClass), + SequenceFile.Writer.valueClass(valueClass)}; + + tempDirSeq = Files.createTempDir(); + File[] subdirs = new File[3]; + Path[] paths = new Path[subdirs.length]; + MapFile.Writer[] writers = new MapFile.Writer[subdirs.length]; + for (int i = 0; i < subdirs.length; i++) { + subdirs[i] = new File(tempDirSeq, "part-r-0000" + i); + subdirs[i].mkdir(); + paths[i] = new Path("file:///" + subdirs[i].getAbsolutePath()); + writers[i] = new MapFile.Writer(c, paths[i], opts); + } + seqMapFilePath = new Path("file:///" + tempDirSeq.getAbsolutePath()); + + + + seqMap = new HashMap<>(); + + for (int i = 0; i < 6; i++) { + seqMap.put(new LongWritable(i), new SequenceRecordWritable(Arrays.asList( + Arrays.asList(new Text(i + "-0"), new IntWritable(3 * i), + new DoubleWritable(3 * i)), + Arrays.asList(new Text(i + "-1"), + new IntWritable(3 * i + 1), new DoubleWritable(3 * i + 1.0)), + Arrays.asList(new Text(i + "-2"), + new IntWritable(3 * i + 2), new DoubleWritable(3 * i + 2.0))))); + } + + + //Need to write in order, to different map files separately + for (int i = 0; i < seqMap.size(); i++) { + int mapFileIdx; + if(i < 3){ + mapFileIdx = 0; + } else { + mapFileIdx = 2; + } + + LongWritable key = new LongWritable(i); + SequenceRecordWritable value = seqMap.get(key); + + writers[mapFileIdx].append(key, value); + } + + for (MapFile.Writer m : writers) { + m.close(); + } + + + //----- Standard RR setup ----- + + valueClass = RecordWritable.class; + + opts = new SequenceFile.Writer.Option[] {MapFile.Writer.keyClass(keyClass), + SequenceFile.Writer.valueClass(valueClass)}; + + tempDir = Files.createTempDir(); + subdirs = new File[3]; + paths = new Path[subdirs.length]; + writers = new MapFile.Writer[subdirs.length]; + for (int i = 0; i < subdirs.length; i++) { + subdirs[i] = new File(tempDir, "part-r-0000" + i); + subdirs[i].mkdir(); + paths[i] = new Path("file:///" + subdirs[i].getAbsolutePath()); + writers[i] = new MapFile.Writer(c, paths[i], opts); + } + mapFilePath = new Path("file:///" + tempDir.getAbsolutePath()); + + recordMap = new HashMap<>(); + for (int i = 0; i < 6; i++) { + recordMap.put(new LongWritable(i), new RecordWritable(Arrays.asList( + new Text(String.valueOf(i)), new IntWritable(i), new DoubleWritable(i)))); + } + + + //Need to write in order + for (int i = 0; i < recordMap.size(); i++) { + int mapFileIdx; + if(i < 3){ + mapFileIdx = 0; + } else { + mapFileIdx = 2; + } + + LongWritable key = new LongWritable(i); + RecordWritable value = recordMap.get(key); + + writers[mapFileIdx].append(key, value); + } + + for (MapFile.Writer m : writers) { + m.close(); + } + + } + + @AfterAll + public static void destroyMapFiles() { + tempDirSeq.delete(); + tempDirSeq = null; + seqMapFilePath = null; + seqMap = null; + +// tempDir.delete(); +// tempDir = null; +// mapFilePath = null; +// seqMap = null; + } + + @Test + public void testSequenceRecordReader() throws Exception { + SequenceRecordReader seqRR = new MapFileSequenceRecordReader(); + URI uri = seqMapFilePath.toUri(); + InputSplit is = new FileSplit(new File(uri)); + seqRR.initialize(is); + + //Check number of records calculation + Field f = MapFileSequenceRecordReader.class.getDeclaredField("indexToKey"); + f.setAccessible(true); + IndexToKey itk = (IndexToKey) f.get(seqRR); + assertEquals(seqMap.size(), itk.getNumRecords()); + + //Check indices for each map file + List> expReaderExampleIdxs = new ArrayList<>(); + expReaderExampleIdxs.add(new Pair<>(0L, 2L)); + expReaderExampleIdxs.add(new Pair<>(-1L, -1L)); + expReaderExampleIdxs.add(new Pair<>(3L, 5L)); + + f = LongIndexToKey.class.getDeclaredField("readerIndices"); + f.setAccessible(true); + assertEquals(expReaderExampleIdxs, f.get(itk)); + // System.out.println(f.get(itk)); + + //Check standard iteration order (no randomization) + assertTrue(seqRR.hasNext()); + int count = 0; + while (seqRR.hasNext()) { + List> l = seqRR.sequenceRecord(); + + assertEquals(seqMap.get(new LongWritable(count)).getSequenceRecord(), l); + + count++; + } + assertFalse(seqRR.hasNext()); + assertEquals(seqMap.size(), count); + + seqRR.close(); + + //Try the same thing, but with random order + seqRR = new MapFileSequenceRecordReader(new Random(12345)); + seqRR.initialize(is); + + //Check order is defined and as expected + f = MapFileSequenceRecordReader.class.getDeclaredField("order"); + f.setAccessible(true); + int[] order = (int[]) f.get(seqRR); + assertNotNull(order); + + int[] expOrder = new int[6]; + for (int i = 0; i < expOrder.length; i++) { + expOrder[i] = i; + } + MathUtils.shuffleArray(expOrder, new Random(12345)); + assertArrayEquals(expOrder, order); + // System.out.println(Arrays.toString(expOrder)); + + count = 0; + while (seqRR.hasNext()) { + List> l = seqRR.sequenceRecord(); + assertEquals(seqMap.get(new LongWritable(expOrder[count])).getSequenceRecord(), l); + count++; + } + } + + @Test + public void testRecordReaderMultipleParts() throws Exception { + RecordReader rr = new MapFileRecordReader(); + URI uri = mapFilePath.toUri(); + InputSplit is = new FileSplit(new File(uri)); + rr.initialize(is); + + //Check number of records calculation + Field f = MapFileRecordReader.class.getDeclaredField("indexToKey"); + f.setAccessible(true); + IndexToKey itk = (IndexToKey) f.get(rr); + assertEquals(recordMap.size(), itk.getNumRecords()); + + //Check indices for each map file + List> expReaderExampleIdxs = new ArrayList<>(); + expReaderExampleIdxs.add(new Pair<>(0L, 2L)); + expReaderExampleIdxs.add(new Pair<>(-1L, -1L)); //Empty + expReaderExampleIdxs.add(new Pair<>(3L, 5L)); + + f = LongIndexToKey.class.getDeclaredField("readerIndices"); + f.setAccessible(true); + assertEquals(expReaderExampleIdxs, f.get(itk)); + + assertTrue(rr.hasNext()); + int count = 0; + while (rr.hasNext()) { + List l = rr.next(); + assertEquals(recordMap.get(new LongWritable(count)).getRecord(), l); + count++; + } + assertEquals(recordMap.size(), count); + + rr.close(); + + //Try the same thing, but with random order + rr = new MapFileRecordReader(new Random(12345)); + rr.initialize(is); + + f = MapFileRecordReader.class.getDeclaredField("order"); + f.setAccessible(true); + int[] order = (int[]) f.get(rr); + assertNotNull(order); + int[] expOrder = new int[recordMap.size()]; + for (int i = 0; i < expOrder.length; i++) { + expOrder[i] = i; + } + MathUtils.shuffleArray(expOrder, new Random(12345)); + assertArrayEquals(expOrder, order); + + count = 0; + while (rr.hasNext()) { + List l = rr.next(); + assertEquals(recordMap.get(new LongWritable(expOrder[count])).getRecord(), l); + count++; + } + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/writer/TestMapFileRecordWriter.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/writer/TestMapFileRecordWriter.java new file mode 100644 index 000000000..71dd9d7a6 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/java/org/datavec/hadoop/records/writer/TestMapFileRecordWriter.java @@ -0,0 +1,235 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.hadoop.records.writer; + +import com.google.common.io.Files; +import org.datavec.api.records.converter.RecordReaderConverter; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVNLinesSequenceRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.records.writer.RecordWriter; +import org.datavec.api.records.writer.SequenceRecordWriter; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.FloatWritable; +import org.datavec.api.writable.Writable; +import org.datavec.api.writable.WritableType; +import org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader; +import org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader; +import org.datavec.hadoop.records.writer.mapfile.MapFileRecordWriter; +import org.datavec.hadoop.records.writer.mapfile.MapFileSequenceRecordWriter; +import org.junit.jupiter.api.Test; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Created by Alex on 07/07/2017. + */ +public class TestMapFileRecordWriter { + + @Test + public void testWriter() throws Exception { + + for(boolean convertWritables : new boolean[]{false, true}) { + + File tempDirSingle = Files.createTempDir(); + File tempDirMultiple = Files.createTempDir(); + File tempDirBatch = Files.createTempDir(); + + tempDirSingle.deleteOnExit(); + tempDirMultiple.deleteOnExit(); + tempDirBatch.deleteOnExit(); + + WritableType textWritablesTo = convertWritables ? WritableType.Float : null; + + RecordWriter singlePartWriter = new MapFileRecordWriter(tempDirSingle, -1, textWritablesTo); + RecordWriter multiPartWriter = new MapFileRecordWriter(tempDirMultiple, 30, textWritablesTo); + RecordWriter multiPartBatch = new MapFileRecordWriter(tempDirBatch, 30, textWritablesTo); + + RecordReader rr = new CSVRecordReader(); + ClassPathResource cpr = new ClassPathResource("iris.dat"); + rr.initialize(new FileSplit(cpr.getFile())); + + RecordReaderConverter.convert(rr, singlePartWriter); + rr.reset(); + RecordReaderConverter.convert(rr, multiPartWriter); + + rr.reset(); + List> allLines = new ArrayList<>(); + while(rr.hasNext()){allLines.add(rr.next());} + multiPartBatch.writeBatch(allLines); + + singlePartWriter.close(); + multiPartWriter.close(); + multiPartBatch.close(); + + RecordReader rr1 = new MapFileRecordReader(); + RecordReader rr2 = new MapFileRecordReader(); + RecordReader rr3 = new MapFileRecordReader(); + rr1.initialize(new FileSplit(tempDirSingle)); + rr2.initialize(new FileSplit(tempDirMultiple)); + rr3.initialize(new FileSplit(tempDirBatch)); + + List> exp = new ArrayList<>(); + List> s1 = new ArrayList<>(); + List> s2 = new ArrayList<>(); + List> s3 = new ArrayList<>(); + + rr.reset(); + while (rr.hasNext()) { + exp.add(rr.next()); + } + + while (rr1.hasNext()) { + s1.add(rr1.next()); + } + + while (rr2.hasNext()) { + s2.add(rr2.next()); + } + + while (rr3.hasNext()) { + s3.add(rr3.next()); + } + + assertEquals(150, exp.size()); + + if(convertWritables){ + List> asFloat = new ArrayList<>(); + for(List l : exp ){ + List newList = new ArrayList<>(); + for(Writable w : l){ + newList.add(new FloatWritable(w.toFloat())); + } + asFloat.add(newList); + } + + exp = asFloat; + } + + assertEquals(exp, s1); + assertEquals(exp, s2); + assertEquals(exp, s3); + + + //By default: we won't be doing any conversion of text types. CsvRecordReader outputs Text writables + for (List l : s1) { + for (Writable w : l) { + if(convertWritables){ + assertEquals(WritableType.Float, w.getType()); + } else { + assertEquals(WritableType.Text, w.getType()); + } + } + } + } + } + + + @Test + public void testSequenceWriter() throws Exception { + + for(boolean convertWritables : new boolean[]{false, true}) { + + File tempDirSingle = Files.createTempDir(); + File tempDirMultiple = Files.createTempDir(); + + tempDirSingle.deleteOnExit(); + tempDirMultiple.deleteOnExit(); + + WritableType textWritablesTo = convertWritables ? WritableType.Float : null; + + SequenceRecordWriter singlePartWriter = new MapFileSequenceRecordWriter(tempDirSingle, -1, textWritablesTo); + SequenceRecordWriter multiPartWriter = new MapFileSequenceRecordWriter(tempDirMultiple, 10, textWritablesTo); + + SequenceRecordReader rr = new CSVNLinesSequenceRecordReader(5); + ClassPathResource cpr = new ClassPathResource("iris.dat"); + rr.initialize(new FileSplit(cpr.getFile())); + + RecordReaderConverter.convert(rr, singlePartWriter); + rr.reset(); + RecordReaderConverter.convert(rr, multiPartWriter); + + singlePartWriter.close(); + multiPartWriter.close(); + + SequenceRecordReader rr1 = new MapFileSequenceRecordReader(); + SequenceRecordReader rr2 = new MapFileSequenceRecordReader(); + rr1.initialize(new FileSplit(tempDirSingle)); + rr2.initialize(new FileSplit(tempDirMultiple)); + + List>> exp = new ArrayList<>(); + List>> s1 = new ArrayList<>(); + List>> s2 = new ArrayList<>(); + + rr.reset(); + while (rr.hasNext()) { + exp.add(rr.sequenceRecord()); + } + + while (rr1.hasNext()) { + s1.add(rr1.sequenceRecord()); + } + + while (rr2.hasNext()) { + s2.add(rr2.sequenceRecord()); + } + + assertEquals(150/5, exp.size()); + + if(convertWritables){ + List>> asFloat = new ArrayList<>(); + for(List> sequence : exp ){ + List> newSeq = new ArrayList<>(); + for(List step : sequence ){ + List newStep = new ArrayList<>(); + for(Writable w : step){ + newStep.add(new FloatWritable(w.toFloat())); + } + newSeq.add(newStep); + } + asFloat.add(newSeq); + } + exp = asFloat; + } + + assertEquals(exp, s1); + assertEquals(exp, s2); + + + //By default: we won't be doing any conversion of text types. CsvRecordReader outputs Text writables + for(List> seq : s1) { + for (List l : seq) { + for (Writable w : l) { + if (convertWritables) { + assertEquals(WritableType.Float, w.getType()); + } else { + assertEquals(WritableType.Text, w.getType()); + } + } + } + } + } + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/resources/log4j.properties b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/resources/log4j.properties new file mode 100644 index 000000000..00eaf12f4 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/resources/log4j.properties @@ -0,0 +1,40 @@ +################################################################################ +# Copyright (c) 2015-2018 Skymind, Inc. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + +log4j.rootLogger=ERROR, Console +log4j.logger.play=DEBUG +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n + +log4j.appender.org.springframework=DEBUG +log4j.appender.org.nd4j=INFO +log4j.appender.org.canova=INFO +log4j.appender.org.datavec=INFO +log4j.appender.org.deeplearning4j=INFO +log4j.appender.opennlp.uima=OFF +log4j.appender.org.apache.uima=OFF +log4j.appender.org.cleartk=OFF + +log4j.logger.org.springframework=INFO +log4j.logger.org.nd4j=INFO +log4j.logger.org.canova=INFO +log4j.logger.org.datavec=INFO +log4j.logger.org.apache.spark=WARN +log4j.logger.org.deeplearning4j=INFO +log4j.logger.opennlp.uima.util=OFF +log4j.logger.org.apache.uima=OFF +log4j.logger.org.cleartk=OFF \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/resources/logback.xml b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/resources/logback.xml new file mode 100644 index 000000000..2087d615c --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-hadoop/src/test/resources/logback.xml @@ -0,0 +1,49 @@ + + + + + + logs/application.log + + %date - [%level] - from %logger in %thread + %n%message%n%xException%n + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/build.gradle b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/build.gradle new file mode 100644 index 000000000..4fa3217e3 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/build.gradle @@ -0,0 +1,50 @@ +plugins { + id 'java-library' + id 'maven-publish' + id 'signing' +} + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + + + + + +dependencies { + implementation project(":cavis-datavec:cavis-datavec-api") + implementation project(":cavis-dnn:cavis-dnn-api") + implementation project(":cavis-dnn:cavis-dnn-common") + + implementation 'com.github.jai-imageio:jai-imageio-core' + implementation 'com.twelvemonkeys.imageio:imageio-jpeg' + implementation 'com.twelvemonkeys.imageio:imageio-tiff' + implementation 'com.twelvemonkeys.imageio:imageio-psd' + implementation 'com.twelvemonkeys.imageio:imageio-bmp' + implementation('com.google.android:android:4.1.1.4') { + // optional = true (optional is not supported for dependency with closure) + transitive = false + } + implementation "org.bytedeco:javacpp" + implementation "org.bytedeco:javacv" + implementation "org.bytedeco:opencv" + implementation group: "org.bytedeco", name: "opencv", classifier: buildTarget + implementation "org.bytedeco:leptonica-platform" + implementation "org.bytedeco:hdf5-platform" + + implementation "commons-io:commons-io" + + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" + + implementation "org.slf4j:slf4j-api" + + implementation "com.google.guava:guava" + + testImplementation project(":cavis-nd4j:cavis-nd4j-common-tests") + testImplementation project(":cavis-native:cavis-native-blas") + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" + + +} + diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/data/Image.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/data/Image.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/data/Image.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/data/Image.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/data/ImageWritable.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/data/ImageWritable.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/data/ImageWritable.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/data/ImageWritable.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/format/ImageInputFormat.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/format/ImageInputFormat.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/format/ImageInputFormat.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/format/ImageInputFormat.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/AndroidNativeImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/AndroidNativeImageLoader.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/AndroidNativeImageLoader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/AndroidNativeImageLoader.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/BaseImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/BaseImageLoader.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/BaseImageLoader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/BaseImageLoader.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/CifarLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/CifarLoader.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/CifarLoader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/CifarLoader.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/ImageLoader.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/Java2DNativeImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/Java2DNativeImageLoader.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/Java2DNativeImageLoader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/Java2DNativeImageLoader.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/LFWLoader.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java similarity index 98% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java index 600e87cca..cf3e4abbe 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/loader/NativeImageLoader.java @@ -57,17 +57,14 @@ public class NativeImageLoader extends BaseImageLoader { "png", "tif", "tiff", "exr", "webp", "BMP", "GIF", "JPG", "JPEG", "JP2", "PBM", "PGM", "PPM", "PNM", "PNG", "TIF", "TIFF", "EXR", "WEBP"}; - protected OpenCVFrameConverter.ToMat converter; + protected OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); boolean direct = !Loader.getPlatform().startsWith("android"); /** * Loads images with no scaling or conversion. */ - public NativeImageLoader() { - converter = new OpenCVFrameConverter.ToMat(); - - } + public NativeImageLoader() {} /** * Instantiate an image with the given @@ -79,7 +76,6 @@ public class NativeImageLoader extends BaseImageLoader { public NativeImageLoader(long height, long width) { this.height = height; this.width = width; - converter = new OpenCVFrameConverter.ToMat(); } @@ -94,7 +90,6 @@ public class NativeImageLoader extends BaseImageLoader { this.height = height; this.width = width; this.channels = channels; - converter = new OpenCVFrameConverter.ToMat(); } /** @@ -108,7 +103,6 @@ public class NativeImageLoader extends BaseImageLoader { public NativeImageLoader(long height, long width, long channels, boolean centerCropIfNeeded) { this(height, width, channels); this.centerCropIfNeeded = centerCropIfNeeded; - converter = new OpenCVFrameConverter.ToMat(); } /** @@ -122,7 +116,6 @@ public class NativeImageLoader extends BaseImageLoader { public NativeImageLoader(long height, long width, long channels, ImageTransform imageTransform) { this(height, width, channels); this.imageTransform = imageTransform; - converter = new OpenCVFrameConverter.ToMat(); } /** @@ -136,7 +129,6 @@ public class NativeImageLoader extends BaseImageLoader { public NativeImageLoader(long height, long width, long channels, MultiPageMode mode) { this(height, width, channels); this.multiPageMode = mode; - converter = new OpenCVFrameConverter.ToMat(); } protected NativeImageLoader(NativeImageLoader other) { @@ -146,7 +138,6 @@ public class NativeImageLoader extends BaseImageLoader { this.centerCropIfNeeded = other.centerCropIfNeeded; this.imageTransform = other.imageTransform; this.multiPageMode = other.multiPageMode; - converter = new OpenCVFrameConverter.ToMat(); } @Override diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistDbFile.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistDbFile.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistDbFile.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistDbFile.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistFetcher.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistFetcher.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistFetcher.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistFetcher.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistImageFile.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistImageFile.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistImageFile.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistImageFile.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistLabelFile.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistLabelFile.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistLabelFile.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistLabelFile.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistManager.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistManager.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/MnistManager.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/MnistManager.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/draw/DrawMnist.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/draw/DrawMnist.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/draw/DrawMnist.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/draw/DrawMnist.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/draw/DrawReconstruction.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/draw/DrawReconstruction.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/mnist/draw/DrawReconstruction.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/mnist/draw/DrawReconstruction.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java similarity index 99% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java index fa36f5e60..86a6a59c1 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/BaseImageRecordReader.java @@ -20,7 +20,7 @@ package org.datavec.image.recordreader; -import org.nd4j.shade.guava.base.Preconditions; +import com.google.common.base.Preconditions; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/ImageRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/ImageRecordReader.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/ImageRecordReader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/ImageRecordReader.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ImageObject.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ImageObject.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ImageObject.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ImageObject.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ImageObjectLabelProvider.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ImageObjectLabelProvider.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ImageObjectLabelProvider.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ImageObjectLabelProvider.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ObjectDetectionRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ObjectDetectionRecordReader.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ObjectDetectionRecordReader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/ObjectDetectionRecordReader.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/impl/SvhnLabelProvider.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/impl/SvhnLabelProvider.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/impl/SvhnLabelProvider.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/impl/SvhnLabelProvider.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/impl/VocLabelProvider.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/impl/VocLabelProvider.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/impl/VocLabelProvider.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/recordreader/objdetect/impl/VocLabelProvider.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/BaseImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BaseImageTransform.java similarity index 97% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/BaseImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BaseImageTransform.java index 650c70c42..ff6389868 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/BaseImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BaseImageTransform.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.bytedeco.javacv.FrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java similarity index 95% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java index efaa12fbb..76dc8a798 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/BoxImageTransform.java @@ -26,9 +26,9 @@ import lombok.Setter; import lombok.experimental.Accessors; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ColorConversionTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ColorConversionTransform.java similarity index 98% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ColorConversionTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ColorConversionTransform.java index ee431d8b1..fe83aef22 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ColorConversionTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ColorConversionTransform.java @@ -23,7 +23,7 @@ package org.datavec.image.transform; import lombok.Data; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/CropImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/CropImageTransform.java similarity index 97% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/CropImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/CropImageTransform.java index 5770469dd..5fe2a3a3b 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/CropImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/CropImageTransform.java @@ -23,8 +23,8 @@ package org.datavec.image.transform; import lombok.Data; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/EqualizeHistTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/EqualizeHistTransform.java similarity index 96% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/EqualizeHistTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/EqualizeHistTransform.java index 40c556d8b..3704d97d8 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/EqualizeHistTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/EqualizeHistTransform.java @@ -23,8 +23,8 @@ package org.datavec.image.transform; import lombok.Data; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/FilterImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FilterImageTransform.java similarity index 94% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/FilterImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FilterImageTransform.java index 3d087f41d..1120bcef6 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/FilterImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FilterImageTransform.java @@ -24,9 +24,9 @@ import lombok.Data; import org.bytedeco.javacv.FFmpegFrameFilter; import org.bytedeco.javacv.FrameFilter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java similarity index 98% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java index 74d586607..c1e00ce35 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/FlipImageTransform.java @@ -23,7 +23,7 @@ package org.datavec.image.transform; import lombok.Data; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransform.java similarity index 95% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransform.java index 3c98ad969..bc4b842fb 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransform.java @@ -22,8 +22,8 @@ package org.datavec.image.transform; import org.datavec.api.transform.Operation; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransformProcess.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransformProcess.java similarity index 99% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransformProcess.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransformProcess.java index b841e5e7c..788a26581 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransformProcess.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ImageTransformProcess.java @@ -28,7 +28,7 @@ import org.datavec.api.writable.Writable; import org.datavec.image.data.ImageWritable; import org.datavec.image.loader.NativeImageLoader; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.File; import java.io.IOException; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/LargestBlobCropTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/LargestBlobCropTransform.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/LargestBlobCropTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/LargestBlobCropTransform.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/MultiImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/MultiImageTransform.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/MultiImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/MultiImageTransform.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/PipelineImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/PipelineImageTransform.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/PipelineImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/PipelineImageTransform.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java similarity index 95% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java index 5d8a2d029..8d76ecb2c 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RandomCropTransform.java @@ -24,9 +24,9 @@ import lombok.Data; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ResizeImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ResizeImageTransform.java similarity index 96% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ResizeImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ResizeImageTransform.java index 18702fc16..565bd0d32 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ResizeImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ResizeImageTransform.java @@ -23,8 +23,8 @@ package org.datavec.image.transform; import lombok.Data; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/RotateImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RotateImageTransform.java similarity index 96% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/RotateImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RotateImageTransform.java index 25b8f53be..8be9359ed 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/RotateImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/RotateImageTransform.java @@ -27,9 +27,9 @@ import lombok.experimental.Accessors; import org.bytedeco.javacpp.FloatPointer; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.nio.FloatBuffer; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ScaleImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ScaleImageTransform.java similarity index 96% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ScaleImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ScaleImageTransform.java index 998bc8799..c2c70a874 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ScaleImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ScaleImageTransform.java @@ -23,8 +23,8 @@ package org.datavec.image.transform; import lombok.Data; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ShowImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ShowImageTransform.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/ShowImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/ShowImageTransform.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/WarpImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/WarpImageTransform.java similarity index 96% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/WarpImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/WarpImageTransform.java index cf51f8603..f55369a6e 100644 --- a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/transform/WarpImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/transform/WarpImageTransform.java @@ -27,9 +27,9 @@ import lombok.experimental.Accessors; import org.bytedeco.javacpp.FloatPointer; import org.bytedeco.javacv.OpenCVFrameConverter; import org.datavec.image.data.ImageWritable; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.nio.FloatBuffer; import java.util.Random; diff --git a/datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/util/ImageUtils.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/util/ImageUtils.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/java/org/datavec/image/util/ImageUtils.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/java/org/datavec/image/util/ImageUtils.java diff --git a/datavec/datavec-data/datavec-data-image/src/main/resources/META-INF/services/javax.imageio.spi.ImageReaderSpi b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/resources/META-INF/services/javax.imageio.spi.ImageReaderSpi similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/resources/META-INF/services/javax.imageio.spi.ImageReaderSpi rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/resources/META-INF/services/javax.imageio.spi.ImageReaderSpi diff --git a/datavec/datavec-data/datavec-data-image/src/main/resources/META-INF/services/javax.imageio.spi.ImageWriterSpi b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/resources/META-INF/services/javax.imageio.spi.ImageWriterSpi similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/main/resources/META-INF/services/javax.imageio.spi.ImageWriterSpi rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/main/resources/META-INF/services/javax.imageio.spi.ImageWriterSpi diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/AssertTestsExtendBaseClass.java similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/AssertTestsExtendBaseClass.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/AssertTestsExtendBaseClass.java diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/LabelGeneratorTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/LabelGeneratorTest.java similarity index 78% rename from datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/LabelGeneratorTest.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/LabelGeneratorTest.java index 6a26fcd2d..e539c7fa8 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/LabelGeneratorTest.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/LabelGeneratorTest.java @@ -17,46 +17,40 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.image; import org.apache.commons.io.FileUtils; import org.datavec.api.io.labels.ParentPathLabelGenerator; import org.datavec.api.split.FileSplit; import org.datavec.image.recordreader.ImageRecordReader; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; + import java.io.File; import java.util.Arrays; import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import java.util.UUID; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Label Generator Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class LabelGeneratorTest { +public class LabelGeneratorTest { + @TempDir + public File testDir; @Test - @DisplayName("Test Parent Path Label Generator") - void testParentPathLabelGenerator(@TempDir Path testDir) throws Exception { + public void testParentPathLabelGenerator() throws Exception { + //https://github.com/deeplearning4j/DataVec/issues/273 File orig = new ClassPathResource("datavec-data-image/testimages/class0/0.jpg").getFile(); - for (String dirPrefix : new String[] { "m.", "m" }) { - File f = testDir.resolve("new-dir-" + UUID.randomUUID().toString()).toFile(); - f.mkdirs(); + + for(String dirPrefix : new String[]{"m.", "m"}) { + File f = testDir; + int numDirs = 3; int filesPerDir = 4; + for (int i = 0; i < numDirs; i++) { File currentLabelDir = new File(f, dirPrefix + i); currentLabelDir.mkdirs(); @@ -66,11 +60,14 @@ class LabelGeneratorTest { assertTrue(f3.exists()); } } + ImageRecordReader rr = new ImageRecordReader(28, 28, 1, new ParentPathLabelGenerator()); rr.initialize(new FileSplit(f)); + List labelsAct = rr.getLabels(); List labelsExp = Arrays.asList(dirPrefix + "0", dirPrefix + "1", dirPrefix + "2"); assertEquals(labelsExp, labelsAct); + int expCount = numDirs * filesPerDir; int actCount = 0; while (rr.hasNext()) { diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java similarity index 90% rename from datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java index 425acba58..b9b4cf3a2 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/loader/LoaderTests.java @@ -22,12 +22,7 @@ package org.datavec.image.loader; import org.apache.commons.io.FilenameUtils; import org.datavec.api.records.reader.RecordReader; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Tags; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.DataSet; import java.io.File; @@ -43,10 +38,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * */ -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) public class LoaderTests { private static void ensureDataAvailable(){ @@ -89,7 +80,7 @@ public class LoaderTests { String subDir = "cifar/cifar-10-batches-bin/data_batch_1.bin"; String path = FilenameUtils.concat(System.getProperty("user.home"), subDir); byte[] fullDataExpected = new byte[3073]; - FileInputStream inExpected = new FileInputStream(path); + FileInputStream inExpected = new FileInputStream(new File(path)); inExpected.read(fullDataExpected); byte[] fullDataActual = new byte[3073]; @@ -102,7 +93,7 @@ public class LoaderTests { subDir = "cifar/cifar-10-batches-bin/test_batch.bin"; path = FilenameUtils.concat(System.getProperty("user.home"), subDir); fullDataExpected = new byte[3073]; - inExpected = new FileInputStream(path); + inExpected = new FileInputStream(new File(path)); inExpected.read(fullDataExpected); fullDataActual = new byte[3073]; @@ -190,7 +181,7 @@ public class LoaderTests { } - @Disabled // Use when confirming data is getting stored + ////@Ignore // Use when confirming data is getting stored @Test public void testProcessCifar() { int row = 32; @@ -216,15 +207,15 @@ public class LoaderTests { int minibatch = 100; int nMinibatches = 50000 / minibatch; - for( int i=0; i < nMinibatches; i++) { + for( int i=0; i c = new ArrayList<>(FileUtils.listFiles(extractedSourceDir, null, true)); + assertEquals(6, c.size()); + + Collections.sort(c, new Comparator() { + @Override + public int compare(File o1, File o2) { + return o1.getPath().compareTo(o2.getPath()); + } + }); + + + FileBatch fb = FileBatch.forFiles(c); + File saveFile = new File(baseDir, "saved.zip"); + fb.writeAsZip(saveFile); + fb = FileBatch.readFromZip(saveFile); + + PathLabelGenerator labelMaker = new ParentPathLabelGenerator(); + ImageRecordReader rr = new ImageRecordReader(32, 32, 1, labelMaker); + rr.setLabels(Arrays.asList("class0", "class1")); + FileBatchRecordReader fbrr = new FileBatchRecordReader(rr, fb); + + + NativeImageLoader il = new NativeImageLoader(32, 32, 1); + for( int test=0; test<3; test++) { + for (int i = 0; i < 6; i++) { + assertTrue(fbrr.hasNext()); + List next = fbrr.next(); + assertEquals(2, next.size()); + + INDArray exp; + switch (i){ + case 0: + exp = il.asMatrix(new File(extractedSourceDir, "class0/0.jpg")); + break; + case 1: + exp = il.asMatrix(new File(extractedSourceDir, "class0/1.png")); + break; + case 2: + exp = il.asMatrix(new File(extractedSourceDir, "class0/2.jpg")); + break; + case 3: + exp = il.asMatrix(new File(extractedSourceDir, "class1/A.jpg")); + break; + case 4: + exp = il.asMatrix(new File(extractedSourceDir, "class1/B.png")); + break; + case 5: + exp = il.asMatrix(new File(extractedSourceDir, "class1/C.jpg")); + break; + default: + throw new RuntimeException(); + } + Writable expLabel = (i < 3 ? new IntWritable(0) : new IntWritable(1)); + + assertEquals(((NDArrayWritable)next.get(0)).get(), exp); + assertEquals(expLabel, next.get(1)); + } + assertFalse(fbrr.hasNext()); + assertTrue(fbrr.resetSupported()); + fbrr.reset(); + } + } + +} diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/TestImageRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestImageRecordReader.java similarity index 93% rename from datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/TestImageRecordReader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestImageRecordReader.java index e8ca38e79..e075e8c5d 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/TestImageRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestImageRecordReader.java @@ -36,13 +36,9 @@ import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.NDArrayWritable; import org.datavec.api.writable.Writable; import org.datavec.api.writable.batch.NDArrayRecordBatch; - -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -50,31 +46,29 @@ import org.nd4j.common.io.ClassPathResource; import java.io.*; import java.net.URI; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.FILE_IO) + public class TestImageRecordReader { + @TempDir + public File testDir; - @Test() + @Test public void testEmptySplit() throws IOException { - assertThrows(IllegalArgumentException.class,() -> { - InputSplit data = new CollectionInputSplit(new ArrayList<>()); + InputSplit data = new CollectionInputSplit(new ArrayList()); + Assertions.assertThrows(IllegalArgumentException.class, () -> { new ImageRecordReader().initialize(data, null); }); - } @Test - public void testMetaData(@TempDir Path testDir) throws IOException { - - File parentDir = testDir.toFile(); + public void testMetaData() throws IOException { + File parentDir = testDir; new ClassPathResource("datavec-data-image/testimages/").copyDirectory(parentDir); // System.out.println(f.getAbsolutePath()); // System.out.println(f.getParentFile().getParentFile().getAbsolutePath()); @@ -111,11 +105,11 @@ public class TestImageRecordReader { } @Test - public void testImageRecordReaderLabelsOrder(@TempDir Path testDir) throws Exception { + public void testImageRecordReaderLabelsOrder() throws Exception { //Labels order should be consistent, regardless of file iteration order //Idea: labels order should be consistent regardless of input file order - File f = testDir.toFile(); + File f = testDir; new ClassPathResource("datavec-data-image/testimages/").copyDirectory(f); File f0 = new File(f, "/class0/0.jpg"); File f1 = new File(f, "/class1/A.jpg"); @@ -142,11 +136,11 @@ public class TestImageRecordReader { @Test - public void testImageRecordReaderRandomization(@TempDir Path testDir) throws Exception { + public void testImageRecordReaderRandomization() throws Exception { //Order of FileSplit+ImageRecordReader should be different after reset //Idea: labels order should be consistent regardless of input file order - File f0 = testDir.toFile(); + File f0 = testDir; new ClassPathResource("datavec-data-image/testimages/").copyDirectory(f0); FileSplit fs = new FileSplit(f0, new Random(12345)); @@ -196,13 +190,13 @@ public class TestImageRecordReader { @Test - public void testImageRecordReaderRegression(@TempDir Path testDir) throws Exception { + public void testImageRecordReaderRegression() throws Exception { PathLabelGenerator regressionLabelGen = new TestRegressionLabelGen(); ImageRecordReader rr = new ImageRecordReader(28, 28, 3, regressionLabelGen); - File rootDir = testDir.toFile(); + File rootDir = testDir; new ClassPathResource("datavec-data-image/testimages/").copyDirectory(rootDir); FileSplit fs = new FileSplit(rootDir); rr.initialize(fs); @@ -251,10 +245,10 @@ public class TestImageRecordReader { } @Test - public void testListenerInvocationBatch(@TempDir Path testDir) throws IOException { + public void testListenerInvocationBatch() throws IOException { ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator(); ImageRecordReader rr = new ImageRecordReader(32, 32, 3, labelMaker); - File f = testDir.toFile(); + File f = testDir; new ClassPathResource("datavec-data-image/testimages/").copyDirectory(f); File parent = f; @@ -267,10 +261,10 @@ public class TestImageRecordReader { } @Test - public void testListenerInvocationSingle(@TempDir Path testDir) throws IOException { + public void testListenerInvocationSingle() throws IOException { ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator(); ImageRecordReader rr = new ImageRecordReader(32, 32, 3, labelMaker); - File parent = testDir.toFile(); + File parent = testDir; new ClassPathResource("datavec-data-image/testimages/class0/").copyDirectory(parent); int numFiles = parent.list().length; rr.initialize(new FileSplit(parent)); @@ -322,7 +316,7 @@ public class TestImageRecordReader { @Test - public void testImageRecordReaderPathMultiLabelGenerator(@TempDir Path testDir) throws Exception { + public void testImageRecordReaderPathMultiLabelGenerator() throws Exception { Nd4j.setDataType(DataType.FLOAT); //Assumption: 2 multi-class (one hot) classification labels: 2 and 3 classes respectively // PLUS single value (Writable) regression label @@ -331,7 +325,7 @@ public class TestImageRecordReader { ImageRecordReader rr = new ImageRecordReader(28, 28, 3, multiLabelGen); - File rootDir = testDir.toFile(); + File rootDir = testDir; new ClassPathResource("datavec-data-image/testimages/").copyDirectory(rootDir); FileSplit fs = new FileSplit(rootDir); rr.initialize(fs); @@ -478,9 +472,9 @@ public class TestImageRecordReader { @Test - public void testNCHW_NCHW(@TempDir Path testDir) throws Exception { + public void testNCHW_NCHW() throws Exception { //Idea: labels order should be consistent regardless of input file order - File f0 = testDir.toFile(); + File f0 = testDir; new ClassPathResource("datavec-data-image/testimages/").copyDirectory(f0); FileSplit fs0 = new FileSplit(f0, new Random(12345)); diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java similarity index 97% rename from datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java index 8ab4f431a..81d667f78 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/TestObjectDetectionRecordReader.java @@ -35,13 +35,8 @@ import org.datavec.image.transform.FlipImageTransform; import org.datavec.image.transform.ImageTransform; import org.datavec.image.transform.PipelineImageTransform; import org.datavec.image.transform.ResizeImageTransform; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.indexing.BooleanIndexing; @@ -50,27 +45,24 @@ import org.nd4j.common.io.ClassPathResource; import java.io.File; import java.net.URI; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) + public class TestObjectDetectionRecordReader { - + @TempDir + public File testDir; @Test - public void test(@TempDir Path testDir) throws Exception { + public void test() throws Exception { for(boolean nchw : new boolean[]{true, false}) { ImageObjectLabelProvider lp = new TestImageObjectDetectionLabelProvider(); - File f = testDir.toFile(); + File f = testDir; new ClassPathResource("datavec-data-image/objdetect/").copyDirectory(f); String path = new File(f, "000012.jpg").getParent(); diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/objdetect/TestVocLabelProvider.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/objdetect/TestVocLabelProvider.java similarity index 88% rename from datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/objdetect/TestVocLabelProvider.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/objdetect/TestVocLabelProvider.java index ca68ff797..9c4094964 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/objdetect/TestVocLabelProvider.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/recordreader/objdetect/TestVocLabelProvider.java @@ -21,31 +21,26 @@ package org.datavec.image.recordreader.objdetect; import org.datavec.image.recordreader.objdetect.impl.VocLabelProvider; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) + public class TestVocLabelProvider { + @TempDir + public File testDir; @Test - public void testVocLabelProvider(@TempDir Path testDir) throws Exception { + public void testVocLabelProvider() throws Exception { - File f = testDir.toFile(); + File f = testDir; new ClassPathResource("datavec-data-image/voc/2007/").copyDirectory(f); String path = f.getAbsolutePath(); //new ClassPathResource("voc/2007/JPEGImages/000005.jpg").getFile().getParentFile().getParent(); diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/JsonYamlTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/JsonYamlTest.java new file mode 100644 index 000000000..f1d194769 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/JsonYamlTest.java @@ -0,0 +1,122 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.image.transform; + +import org.datavec.image.data.ImageWritable; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class JsonYamlTest { + @Test + public void testJsonYamlImageTransformProcess() throws IOException { + int seed = 12345; + Random random = new Random(seed); + + //from org.bytedeco.javacpp.opencv_imgproc + int COLOR_BGR2Luv = 50; + int CV_BGR2GRAY = 6; + + + ImageTransformProcess itp = new ImageTransformProcess.Builder().colorConversionTransform(COLOR_BGR2Luv) + .cropImageTransform(10).equalizeHistTransform(CV_BGR2GRAY).flipImageTransform(0) + .resizeImageTransform(300, 300).rotateImageTransform(30).scaleImageTransform(3) + .warpImageTransform((float) 0.5) + + // Note : since randomCropTransform use random value + // the results from each case(json, yaml, ImageTransformProcess) + // can be different + // don't use the below line + // if you uncomment it, you will get fail from below assertions + // .randomCropTransform(seed, 50, 50) + + // Note : you will get "java.lang.NoClassDefFoundError: Could not initialize class org.bytedeco.javacpp.avutil" + // it needs to add the below dependency + // + // org.bytedeco + // ffmpeg-platform + // + // FFmpeg has license issues, be careful to use it + //.filterImageTransform("noise=alls=20:allf=t+u,format=rgba", 100, 100, 4) + + .build(); + + String asJson = itp.toJson(); + String asYaml = itp.toYaml(); + +// System.out.println(asJson); +// System.out.println("\n\n\n"); +// System.out.println(asYaml); + + ImageWritable img = TestImageTransform.makeRandomImage(0, 0, 3); + ImageWritable imgJson = new ImageWritable(img.getFrame().clone()); + ImageWritable imgYaml = new ImageWritable(img.getFrame().clone()); + ImageWritable imgAll = new ImageWritable(img.getFrame().clone()); + + ImageTransformProcess itpFromJson = ImageTransformProcess.fromJson(asJson); + ImageTransformProcess itpFromYaml = ImageTransformProcess.fromYaml(asYaml); + + List transformList = itp.getTransformList(); + List transformListJson = itpFromJson.getTransformList(); + List transformListYaml = itpFromYaml.getTransformList(); + + for (int i = 0; i < transformList.size(); i++) { + ImageTransform it = transformList.get(i); + ImageTransform itJson = transformListJson.get(i); + ImageTransform itYaml = transformListYaml.get(i); + + System.out.println(i + "\t" + it); + + img = it.transform(img); + imgJson = itJson.transform(imgJson); + imgYaml = itYaml.transform(imgYaml); + + if (it instanceof RandomCropTransform) { + assertTrue(img.getFrame().imageHeight == imgJson.getFrame().imageHeight); + assertTrue(img.getFrame().imageWidth == imgJson.getFrame().imageWidth); + + assertTrue(img.getFrame().imageHeight == imgYaml.getFrame().imageHeight); + assertTrue(img.getFrame().imageWidth == imgYaml.getFrame().imageWidth); + } else if (it instanceof FilterImageTransform) { + assertEquals(img.getFrame().imageHeight, imgJson.getFrame().imageHeight); + assertEquals(img.getFrame().imageWidth, imgJson.getFrame().imageWidth); + assertEquals(img.getFrame().imageChannels, imgJson.getFrame().imageChannels); + + assertEquals(img.getFrame().imageHeight, imgYaml.getFrame().imageHeight); + assertEquals(img.getFrame().imageWidth, imgYaml.getFrame().imageWidth); + assertEquals(img.getFrame().imageChannels, imgYaml.getFrame().imageChannels); + } else { + assertEquals(img, imgJson); + + assertEquals(img, imgYaml); + } + } + + imgAll = itp.execute(imgAll); + + assertEquals(imgAll, img); + } +} diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/ResizeImageTransformTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/ResizeImageTransformTest.java similarity index 77% rename from datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/ResizeImageTransformTest.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/ResizeImageTransformTest.java index b0c1de47d..2cf86459c 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/ResizeImageTransformTest.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/ResizeImageTransformTest.java @@ -17,55 +17,56 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.image.transform; import org.bytedeco.javacv.Frame; import org.datavec.image.data.ImageWritable; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Resize Image Transform Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class ResizeImageTransformTest { +public class ResizeImageTransformTest { @BeforeEach - void setUp() throws Exception { + public void setUp() throws Exception { + } @Test - @DisplayName("Test Resize Upscale 1") - void testResizeUpscale1() throws Exception { + public void testResizeUpscale1() throws Exception { ImageWritable srcImg = TestImageTransform.makeRandomImage(32, 32, 3); + ResizeImageTransform transform = new ResizeImageTransform(200, 200); + ImageWritable dstImg = transform.transform(srcImg); + Frame f = dstImg.getFrame(); assertEquals(f.imageWidth, 200); assertEquals(f.imageHeight, 200); - float[] coordinates = { 100, 200 }; + + float[] coordinates = {100, 200}; float[] transformed = transform.query(coordinates); assertEquals(200f * 100 / 32, transformed[0], 0); assertEquals(200f * 200 / 32, transformed[1], 0); } @Test - @DisplayName("Test Resize Downscale") - void testResizeDownscale() throws Exception { + public void testResizeDownscale() throws Exception { ImageWritable srcImg = TestImageTransform.makeRandomImage(571, 443, 3); + ResizeImageTransform transform = new ResizeImageTransform(200, 200); + ImageWritable dstImg = transform.transform(srcImg); + Frame f = dstImg.getFrame(); assertEquals(f.imageWidth, 200); assertEquals(f.imageHeight, 200); - float[] coordinates = { 300, 400 }; + + float[] coordinates = {300, 400}; float[] transformed = transform.query(coordinates); assertEquals(200f * 300 / 443, transformed[0], 0); assertEquals(200f * 400 / 571, transformed[1], 0); } + } diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/TestImageTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/TestImageTransform.java similarity index 98% rename from datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/TestImageTransform.java rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/TestImageTransform.java index 4c8d4cc32..ee713e091 100644 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/TestImageTransform.java +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/java/org/datavec/image/transform/TestImageTransform.java @@ -24,12 +24,10 @@ import org.bytedeco.javacpp.indexer.UByteIndexer; import org.bytedeco.javacv.CanvasFrame; import org.bytedeco.javacv.Frame; import org.bytedeco.javacv.OpenCVFrameConverter; -import org.junit.jupiter.api.Tag; import org.nd4j.common.io.ClassPathResource; import org.nd4j.common.primitives.Pair; import org.datavec.image.data.ImageWritable; import org.datavec.image.loader.NativeImageLoader; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.awt.*; @@ -38,8 +36,6 @@ import java.util.List; import java.util.Random; import org.bytedeco.opencv.opencv_core.*; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.bytedeco.opencv.global.opencv_core.*; import static org.bytedeco.opencv.global.opencv_imgproc.*; @@ -49,8 +45,6 @@ import static org.junit.jupiter.api.Assertions.*; * * @author saudet */ -@NativeTag -@Tag(TagNames.FILE_IO) public class TestImageTransform { static final long seed = 10; static final Random rng = new Random(seed); @@ -260,6 +254,7 @@ public class TestImageTransform { assertEquals(22, transformed[1], 0); } + ////@Ignore @Test public void testFilterImageTransform() throws Exception { ImageWritable writable = makeRandomImage(0, 0, 4); diff --git a/datavec/datavec-data/datavec-data-image/src/test/resources/logback.xml b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/resources/logback.xml similarity index 100% rename from datavec/datavec-data/datavec-data-image/src/test/resources/logback.xml rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-image/src/test/resources/logback.xml diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/build.gradle b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/build.gradle new file mode 100644 index 000000000..364c79b76 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/build.gradle @@ -0,0 +1,40 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + + +dependencies { + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisDnn.cavisDnnApi + + implementation "org.cleartk:cleartk-snowball:2.0.0" + implementation "org.cleartk:cleartk-opennlp-tools:2.0.0" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "org.apache.commons:commons-lang3" + implementation "org.slf4j:slf4j-api" + + + testImplementation projects.cavisDatavec.cavisDatavecLocal + testImplementation projects.cavisNd4j.cavisNd4jCommonTests +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/PoStagger.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/PoStagger.java new file mode 100644 index 000000000..d071a42a4 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/PoStagger.java @@ -0,0 +1,233 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.annotator; + +import opennlp.tools.postag.POSModel; +import opennlp.tools.postag.POSTaggerME; +import opennlp.uima.postag.POSModelResource; +import opennlp.uima.postag.POSModelResourceImpl; +import opennlp.uima.util.AnnotationComboIterator; +import opennlp.uima.util.AnnotationIteratorPair; +import opennlp.uima.util.AnnotatorUtil; +import opennlp.uima.util.UimaUtil; +import org.apache.uima.UimaContext; +import org.apache.uima.analysis_engine.AnalysisEngineDescription; +import org.apache.uima.analysis_engine.AnalysisEngineProcessException; +import org.apache.uima.cas.CAS; +import org.apache.uima.cas.Feature; +import org.apache.uima.cas.Type; +import org.apache.uima.cas.TypeSystem; +import org.apache.uima.cas.text.AnnotationFS; +import org.apache.uima.fit.component.CasAnnotator_ImplBase; +import org.apache.uima.fit.factory.AnalysisEngineFactory; +import org.apache.uima.fit.factory.ExternalResourceFactory; +import org.apache.uima.resource.ResourceAccessException; +import org.apache.uima.resource.ResourceInitializationException; +import org.apache.uima.util.Level; +import org.apache.uima.util.Logger; +import org.cleartk.token.type.Sentence; +import org.cleartk.token.type.Token; +import org.datavec.nlp.movingwindow.Util; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + + +public class PoStagger extends CasAnnotator_ImplBase { + + static { + //UIMA logging + Util.disableLogging(); + } + + private POSTaggerME posTagger; + + private Type sentenceType; + + private Type tokenType; + + private Feature posFeature; + + private Feature probabilityFeature; + + private UimaContext context; + + private Logger logger; + + /** + * Initializes a new instance. + * + * Note: Use {@link #initialize(org.apache.uima.UimaContext) } to initialize this instance. Not use the + * constructor. + */ + public PoStagger() { + // must not be implemented ! + } + + /** + * Initializes the current instance with the given context. + * + * Note: Do all initialization in this method, do not use the constructor. + */ + @Override + public void initialize(UimaContext context) throws ResourceInitializationException { + + super.initialize(context); + + this.context = context; + + this.logger = context.getLogger(); + + if (this.logger.isLoggable(Level.INFO)) { + this.logger.log(Level.INFO, "Initializing the OpenNLP " + "Part of Speech annotator."); + } + + POSModel model; + + try { + POSModelResource modelResource = (POSModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER); + + model = modelResource.getModel(); + } catch (ResourceAccessException e) { + throw new ResourceInitializationException(e); + } + + Integer beamSize = AnnotatorUtil.getOptionalIntegerParameter(context, UimaUtil.BEAM_SIZE_PARAMETER); + + if (beamSize == null) + beamSize = POSTaggerME.DEFAULT_BEAM_SIZE; + + this.posTagger = new POSTaggerME(model, beamSize, 0); + } + + /** + * Initializes the type system. + */ + @Override + public void typeSystemInit(TypeSystem typeSystem) throws AnalysisEngineProcessException { + + // sentence type + this.sentenceType = AnnotatorUtil.getRequiredTypeParameter(this.context, typeSystem, + UimaUtil.SENTENCE_TYPE_PARAMETER); + + // token type + this.tokenType = AnnotatorUtil.getRequiredTypeParameter(this.context, typeSystem, + UimaUtil.TOKEN_TYPE_PARAMETER); + + // pos feature + this.posFeature = AnnotatorUtil.getRequiredFeatureParameter(this.context, this.tokenType, + UimaUtil.POS_FEATURE_PARAMETER, CAS.TYPE_NAME_STRING); + + this.probabilityFeature = AnnotatorUtil.getOptionalFeatureParameter(this.context, this.tokenType, + UimaUtil.PROBABILITY_FEATURE_PARAMETER, CAS.TYPE_NAME_DOUBLE); + } + + /** + * Performs pos-tagging on the given tcas object. + */ + @Override + public synchronized void process(CAS tcas) { + + final AnnotationComboIterator comboIterator = + new AnnotationComboIterator(tcas, this.sentenceType, this.tokenType); + + for (AnnotationIteratorPair annotationIteratorPair : comboIterator) { + + final List sentenceTokenAnnotationList = new LinkedList(); + + final List sentenceTokenList = new LinkedList(); + + for (AnnotationFS tokenAnnotation : annotationIteratorPair.getSubIterator()) { + + sentenceTokenAnnotationList.add(tokenAnnotation); + + sentenceTokenList.add(tokenAnnotation.getCoveredText()); + } + + final List posTags = this.posTagger.tag(sentenceTokenList); + + double posProbabilities[] = null; + + if (this.probabilityFeature != null) { + posProbabilities = this.posTagger.probs(); + } + + final Iterator posTagIterator = posTags.iterator(); + final Iterator sentenceTokenIterator = sentenceTokenAnnotationList.iterator(); + + int index = 0; + while (posTagIterator.hasNext() && sentenceTokenIterator.hasNext()) { + final String posTag = posTagIterator.next(); + final AnnotationFS tokenAnnotation = sentenceTokenIterator.next(); + + tokenAnnotation.setStringValue(this.posFeature, posTag); + + if (posProbabilities != null) { + tokenAnnotation.setDoubleValue(this.posFeature, posProbabilities[index]); + } + + index++; + } + + // log tokens with pos + if (this.logger.isLoggable(Level.FINER)) { + + final StringBuilder sentenceWithPos = new StringBuilder(); + + sentenceWithPos.append("\""); + + for (final Iterator it = sentenceTokenAnnotationList.iterator(); it.hasNext();) { + final AnnotationFS token = it.next(); + sentenceWithPos.append(token.getCoveredText()); + sentenceWithPos.append('\\'); + sentenceWithPos.append(token.getStringValue(this.posFeature)); + sentenceWithPos.append(' '); + } + + // delete last whitespace + if (sentenceWithPos.length() > 1) // not 0 because it contains already the " char + sentenceWithPos.setLength(sentenceWithPos.length() - 1); + + sentenceWithPos.append("\""); + + this.logger.log(Level.FINER, sentenceWithPos.toString()); + } + } + } + + /** + * Releases allocated resources. + */ + @Override + public void destroy() { + this.posTagger = null; + } + + + public static AnalysisEngineDescription getDescription(String languageCode) throws ResourceInitializationException { + String modelPath = String.format("/models/%s-pos-maxent.bin", languageCode); + return AnalysisEngineFactory.createEngineDescription(PoStagger.class, UimaUtil.MODEL_PARAMETER, + ExternalResourceFactory.createExternalResourceDescription(POSModelResourceImpl.class, + PoStagger.class.getResource(modelPath).toString()), + UimaUtil.SENTENCE_TYPE_PARAMETER, Sentence.class.getName(), UimaUtil.TOKEN_TYPE_PARAMETER, + Token.class.getName(), UimaUtil.POS_FEATURE_PARAMETER, "pos"); + } + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/SentenceAnnotator.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/SentenceAnnotator.java new file mode 100644 index 000000000..4ba7e8532 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/SentenceAnnotator.java @@ -0,0 +1,48 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.annotator; + +import org.apache.uima.analysis_engine.AnalysisEngineDescription; +import org.apache.uima.analysis_engine.AnalysisEngineProcessException; +import org.apache.uima.fit.factory.AnalysisEngineFactory; +import org.apache.uima.jcas.JCas; +import org.apache.uima.resource.ResourceInitializationException; +import org.cleartk.util.ParamUtil; +import org.datavec.nlp.movingwindow.Util; + +public class SentenceAnnotator extends org.cleartk.opennlp.tools.SentenceAnnotator { + + static { + //UIMA logging + Util.disableLogging(); + } + + public static AnalysisEngineDescription getDescription() throws ResourceInitializationException { + return AnalysisEngineFactory.createEngineDescription(SentenceAnnotator.class, PARAM_SENTENCE_MODEL_PATH, + ParamUtil.getParameterValue(PARAM_SENTENCE_MODEL_PATH, "/models/en-sent.bin"), + PARAM_WINDOW_CLASS_NAMES, ParamUtil.getParameterValue(PARAM_WINDOW_CLASS_NAMES, null)); + } + + + @Override + public synchronized void process(JCas jCas) throws AnalysisEngineProcessException { + super.process(jCas); + } + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/StemmerAnnotator.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/StemmerAnnotator.java new file mode 100644 index 000000000..253eebd9f --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/StemmerAnnotator.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.annotator; + +import org.apache.uima.analysis_engine.AnalysisEngineDescription; +import org.apache.uima.analysis_engine.AnalysisEngineProcessException; +import org.apache.uima.fit.factory.AnalysisEngineFactory; +import org.apache.uima.jcas.JCas; +import org.apache.uima.resource.ResourceInitializationException; +import org.cleartk.snowball.SnowballStemmer; +import org.cleartk.token.type.Token; + + +public class StemmerAnnotator extends SnowballStemmer { + + public static AnalysisEngineDescription getDescription() throws ResourceInitializationException { + return getDescription("English"); + } + + + public static AnalysisEngineDescription getDescription(String language) throws ResourceInitializationException { + return AnalysisEngineFactory.createEngineDescription(StemmerAnnotator.class, SnowballStemmer.PARAM_STEMMER_NAME, + language); + } + + + @SuppressWarnings("unchecked") + @Override + public synchronized void process(JCas jCas) throws AnalysisEngineProcessException { + super.process(jCas); + } + + + + @Override + public void setStem(Token token, String stem) { + token.setStem(stem); + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/TokenizerAnnotator.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/TokenizerAnnotator.java new file mode 100644 index 000000000..844a8554f --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/annotator/TokenizerAnnotator.java @@ -0,0 +1,66 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.annotator; + + +import opennlp.uima.tokenize.TokenizerModelResourceImpl; +import org.apache.uima.analysis_engine.AnalysisEngineDescription; +import org.apache.uima.fit.factory.AnalysisEngineFactory; +import org.apache.uima.fit.factory.ExternalResourceFactory; +import org.apache.uima.resource.ResourceInitializationException; +import org.cleartk.opennlp.tools.Tokenizer; +import org.cleartk.token.type.Sentence; +import org.cleartk.token.type.Token; +import org.datavec.nlp.movingwindow.Util; +import org.datavec.nlp.tokenization.tokenizer.ConcurrentTokenizer; + + +/** + * Overrides OpenNLP tokenizer to be thread safe + */ +public class TokenizerAnnotator extends Tokenizer { + + static { + //UIMA logging + Util.disableLogging(); + } + + public static AnalysisEngineDescription getDescription(String languageCode) throws ResourceInitializationException { + String modelPath = String.format("/models/%s-token.bin", languageCode); + return AnalysisEngineFactory.createEngineDescription(ConcurrentTokenizer.class, + opennlp.uima.util.UimaUtil.MODEL_PARAMETER, + ExternalResourceFactory.createExternalResourceDescription(TokenizerModelResourceImpl.class, + ConcurrentTokenizer.class.getResource(modelPath).toString()), + opennlp.uima.util.UimaUtil.SENTENCE_TYPE_PARAMETER, Sentence.class.getName(), + opennlp.uima.util.UimaUtil.TOKEN_TYPE_PARAMETER, Token.class.getName()); + } + + + + public static AnalysisEngineDescription getDescription() throws ResourceInitializationException { + String modelPath = String.format("/models/%s-token.bin", "en"); + return AnalysisEngineFactory.createEngineDescription(ConcurrentTokenizer.class, + opennlp.uima.util.UimaUtil.MODEL_PARAMETER, + ExternalResourceFactory.createExternalResourceDescription(TokenizerModelResourceImpl.class, + ConcurrentTokenizer.class.getResource(modelPath).toString()), + opennlp.uima.util.UimaUtil.SENTENCE_TYPE_PARAMETER, Sentence.class.getName(), + opennlp.uima.util.UimaUtil.TOKEN_TYPE_PARAMETER, Token.class.getName()); + } + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/input/TextInputFormat.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/input/TextInputFormat.java new file mode 100644 index 000000000..a120a3727 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/input/TextInputFormat.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.input; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.formats.input.BaseInputFormat; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.nlp.reader.TfidfRecordReader; + +import java.io.IOException; + +/** + * @author Adam Gibson + */ +public class TextInputFormat extends BaseInputFormat { + @Override + public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException { + RecordReader reader = new TfidfRecordReader(); + reader.initialize(conf, split); + return reader; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/metadata/DefaultVocabCache.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/metadata/DefaultVocabCache.java new file mode 100644 index 000000000..16dff8e5f --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/metadata/DefaultVocabCache.java @@ -0,0 +1,150 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.metadata; + +import org.nd4j.common.primitives.Counter; +import org.datavec.api.conf.Configuration; +import org.datavec.nlp.vectorizer.TextVectorizer; +import org.nd4j.common.util.MathUtils; +import org.nd4j.common.util.Index; + +/** + * Vocab cache used for storing information + * about vocab + * + * @author Adam Gibson + */ +public class DefaultVocabCache implements VocabCache { + + private Counter wordFrequencies = new Counter<>(); + private Counter docFrequencies = new Counter<>(); + private int minWordFrequency; + private Index vocabWords = new Index(); + private double numDocs = 0; + + /** + * Instantiate with a given min word frequency + * @param minWordFrequency + */ + public DefaultVocabCache(int minWordFrequency) { + this.minWordFrequency = minWordFrequency; + } + + /* + * Constructor for use with initialize() + */ + public DefaultVocabCache() { + } + + @Override + public void incrementNumDocs(double by) { + numDocs += by; + } + + @Override + public double numDocs() { + return numDocs; + } + + @Override + public String wordAt(int i) { + return vocabWords.get(i).toString(); + } + + @Override + public int wordIndex(String word) { + return vocabWords.indexOf(word); + } + + @Override + public void initialize(Configuration conf) { + minWordFrequency = conf.getInt(TextVectorizer.MIN_WORD_FREQUENCY, 5); + } + + @Override + public double wordFrequency(String word) { + return wordFrequencies.getCount(word); + } + + @Override + public int minWordFrequency() { + return minWordFrequency; + } + + @Override + public Index vocabWords() { + return vocabWords; + } + + @Override + public void incrementDocCount(String word) { + incrementDocCount(word, 1.0); + } + + @Override + public void incrementDocCount(String word, double by) { + docFrequencies.incrementCount(word, by); + + } + + @Override + public void incrementCount(String word) { + incrementCount(word, 1.0); + } + + @Override + public void incrementCount(String word, double by) { + wordFrequencies.incrementCount(word, by); + if (wordFrequencies.getCount(word) >= minWordFrequency && vocabWords.indexOf(word) < 0) + vocabWords.add(word); + } + + @Override + public double idf(String word) { + return docFrequencies.getCount(word); + } + + @Override + public double tfidf(String word, double frequency, boolean smoothIdf) { + double tf = tf((int) frequency); + double docFreq = docFrequencies.getCount(word); + + double idf = idf(numDocs, docFreq, smoothIdf); + double tfidf = MathUtils.tfidf(tf, idf); + return tfidf; + } + + public double idf(double totalDocs, double numTimesWordAppearedInADocument, boolean smooth) { + if(smooth){ + return Math.log((1 + totalDocs) / (1 + numTimesWordAppearedInADocument)) + 1.0; + } else { + return Math.log(totalDocs / numTimesWordAppearedInADocument) + 1.0; + } + } + + public static double tf(int count) { + return count; + } + + public int getMinWordFrequency() { + return minWordFrequency; + } + + public void setMinWordFrequency(int minWordFrequency) { + this.minWordFrequency = minWordFrequency; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/metadata/VocabCache.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/metadata/VocabCache.java new file mode 100644 index 000000000..64fb1e8cb --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/metadata/VocabCache.java @@ -0,0 +1,122 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.metadata; + + +import org.datavec.api.conf.Configuration; +import org.nd4j.common.util.Index; + +/** + * Track metadata about vocabs + * + * @author Adam Gibson + */ +public interface VocabCache { + + + /** + * Increment the number of documents + * @param by + */ + void incrementNumDocs(double by); + + /** + * Number of documents + * @return the number of documents + */ + double numDocs(); + + /** + * Returns a word in the vocab at a particular index + * @param i the index to get + * @return the word at that index in the vocab + */ + String wordAt(int i); + + int wordIndex(String word); + + /** + * Configuration for initializing + * @param conf the configuration to initialize with + */ + void initialize(Configuration conf); + + /** + * Get the word frequency for a word + * @param word the word to get frequency for + * @return the frequency for a given word + */ + double wordFrequency(String word); + + /** + * The min word frequency + * needed to be included in the vocab + * (default 5) + * @return the min word frequency to + * be included in the vocab + */ + int minWordFrequency(); + + /** + * All of the vocab words (ordered) + * note that these are not all the possible tokens + * @return the list of vocab words + */ + Index vocabWords(); + + + /** + * Increment the doc count for a word by 1 + * @param word the word to increment the count for + */ + void incrementDocCount(String word); + + /** + * Increment the document count for a particular word + * @param word the word to increment the count for + * @param by the amount to increment by + */ + void incrementDocCount(String word, double by); + + /** + * Increment a word count by 1 + * @param word the word to increment the count for + */ + void incrementCount(String word); + + /** + * Increment count for a word + * @param word the word to increment the count for + * @param by the amount to increment by + */ + void incrementCount(String word, double by); + + /** + * Number of documents word has occurred in + * @param word the word to get the idf for + */ + double idf(String word); + + /** + * Calculate the tfidf of the word given the document frequency + * @param word the word to get frequency for + * @param frequency the frequency + * @return the tfidf for a word + */ + double tfidf(String word, double frequency, boolean smoothIdf); + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/ContextLabelRetriever.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/ContextLabelRetriever.java new file mode 100644 index 000000000..76b0244bd --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/ContextLabelRetriever.java @@ -0,0 +1,126 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.movingwindow; + + +import org.apache.commons.lang3.StringUtils; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.collection.MultiDimensionalMap; +import org.nd4j.common.primitives.Pair; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; +import org.datavec.nlp.tokenization.tokenizerfactory.TokenizerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * Context Label Retriever + * + * @author Adam Gibson + */ +public class ContextLabelRetriever { + + + private static String BEGIN_LABEL = "<([A-Za-z]+|\\d+)>"; + private static String END_LABEL = ""; + + + private ContextLabelRetriever() {} + + + /** + * Returns a stripped sentence with the indices of words + * with certain kinds of labels. + * + * @param sentence the sentence to process + * @return a pair of a post processed sentence + * with labels stripped and the spans of + * the labels + */ + public static Pair> stringWithLabels(String sentence, + TokenizerFactory tokenizerFactory) { + MultiDimensionalMap map = MultiDimensionalMap.newHashBackedMap(); + Tokenizer t = tokenizerFactory.create(sentence); + List currTokens = new ArrayList<>(); + String currLabel = null; + String endLabel = null; + List>> tokensWithSameLabel = new ArrayList<>(); + while (t.hasMoreTokens()) { + String token = t.nextToken(); + if (token.matches(BEGIN_LABEL)) { + currLabel = token; + + //no labels; add these as NONE and begin the new label + if (!currTokens.isEmpty()) { + tokensWithSameLabel.add(new Pair<>("NONE", (List) new ArrayList<>(currTokens))); + currTokens.clear(); + + } + + } else if (token.matches(END_LABEL)) { + if (currLabel == null) + throw new IllegalStateException("Found an ending label with no matching begin label"); + endLabel = token; + } else + currTokens.add(token); + + if (currLabel != null && endLabel != null) { + currLabel = currLabel.replaceAll("[<>/]", ""); + endLabel = endLabel.replaceAll("[<>/]", ""); + Preconditions.checkState(!currLabel.isEmpty(), "Current label is empty!"); + Preconditions.checkState(!endLabel.isEmpty(), "End label is empty!"); + Preconditions.checkState(currLabel.equals(endLabel), "Current label begin and end did not match for the parse. Was: %s ending with %s", + currLabel, endLabel); + + tokensWithSameLabel.add(new Pair<>(currLabel, (List) new ArrayList<>(currTokens))); + currTokens.clear(); + + + //clear out the tokens + currLabel = null; + endLabel = null; + } + + + } + + //no labels; add these as NONE and begin the new label + if (!currTokens.isEmpty()) { + tokensWithSameLabel.add(new Pair<>("none", (List) new ArrayList<>(currTokens))); + currTokens.clear(); + + } + + //now join the output + StringBuilder strippedSentence = new StringBuilder(); + for (Pair> tokensWithLabel : tokensWithSameLabel) { + String joinedSentence = StringUtils.join(tokensWithLabel.getSecond(), " "); + //spaces between separate parts of the sentence + if (!(strippedSentence.length() < 1)) + strippedSentence.append(" "); + strippedSentence.append(joinedSentence); + int begin = strippedSentence.toString().indexOf(joinedSentence); + int end = begin + joinedSentence.length(); + map.put(begin, end, tokensWithLabel.getFirst()); + } + + + return new Pair<>(strippedSentence.toString(), map); + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Util.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Util.java new file mode 100644 index 000000000..225b2190c --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Util.java @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.movingwindow; + + +import org.nd4j.common.primitives.Counter; + +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + + +public class Util { + + /** + * Returns a thread safe counter + * + * @return + */ + public static Counter parallelCounter() { + return new Counter<>(); + } + + public static boolean matchesAnyStopWord(List stopWords, String word) { + for (String s : stopWords) + if (s.equalsIgnoreCase(word)) + return true; + return false; + } + + public static Level disableLogging() { + Logger logger = Logger.getLogger("org.apache.uima"); + while (logger.getLevel() == null) { + logger = logger.getParent(); + } + Level level = logger.getLevel(); + logger.setLevel(Level.OFF); + return level; + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Window.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Window.java new file mode 100644 index 000000000..a75b37fd0 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Window.java @@ -0,0 +1,179 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.movingwindow; + +import org.apache.commons.lang3.StringUtils; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + + +/** + * A representation of a sliding window. + * This is used for creating training examples. + * @author Adam Gibson + * + */ +public class Window implements Serializable { + /** + * + */ + private static final long serialVersionUID = 6359906393699230579L; + private List words; + private String label = "NONE"; + private boolean beginLabel; + private boolean endLabel; + private int median; + private static String BEGIN_LABEL = "<([A-Z]+|\\d+)>"; + private static String END_LABEL = ""; + private int begin, end; + + /** + * Creates a window with a context of size 3 + * @param words a collection of strings of size 3 + */ + public Window(Collection words, int begin, int end) { + this(words, 5, begin, end); + + } + + public String asTokens() { + return StringUtils.join(words, " "); + } + + + /** + * Initialize a window with the given size + * @param words the words to use + * @param windowSize the size of the window + * @param begin the begin index for the window + * @param end the end index for the window + */ + public Window(Collection words, int windowSize, int begin, int end) { + if (words == null) + throw new IllegalArgumentException("Words must be a list of size 3"); + + this.words = new ArrayList<>(words); + int windowSize1 = windowSize; + this.begin = begin; + this.end = end; + initContext(); + } + + + private void initContext() { + int median = (int) Math.floor(words.size() / 2); + List begin = words.subList(0, median); + List after = words.subList(median + 1, words.size()); + + + for (String s : begin) { + if (s.matches(BEGIN_LABEL)) { + this.label = s.replaceAll("(<|>)", "").replace("/", ""); + beginLabel = true; + } else if (s.matches(END_LABEL)) { + endLabel = true; + this.label = s.replaceAll("(<|>|/)", "").replace("/", ""); + + } + + } + + for (String s1 : after) { + + if (s1.matches(BEGIN_LABEL)) { + this.label = s1.replaceAll("(<|>)", "").replace("/", ""); + beginLabel = true; + } + + if (s1.matches(END_LABEL)) { + endLabel = true; + this.label = s1.replaceAll("(<|>)", ""); + + } + } + this.median = median; + + } + + + + @Override + public String toString() { + return words.toString(); + } + + public List getWords() { + return words; + } + + public void setWords(List words) { + this.words = words; + } + + public String getWord(int i) { + return words.get(i); + } + + public String getFocusWord() { + return words.get(median); + } + + public boolean isBeginLabel() { + return !label.equals("NONE") && beginLabel; + } + + public boolean isEndLabel() { + return !label.equals("NONE") && endLabel; + } + + public String getLabel() { + return label.replace("/", ""); + } + + public int getWindowSize() { + return words.size(); + } + + public int getMedian() { + return median; + } + + public void setLabel(String label) { + this.label = label; + } + + public int getBegin() { + return begin; + } + + public void setBegin(int begin) { + this.begin = begin; + } + + public int getEnd() { + return end; + } + + public void setEnd(int end) { + this.end = end; + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Windows.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Windows.java new file mode 100644 index 000000000..8791aa524 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/movingwindow/Windows.java @@ -0,0 +1,188 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.movingwindow; + + +import org.apache.commons.lang3.StringUtils; +import org.datavec.nlp.tokenization.tokenizer.DefaultStreamTokenizer; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; +import org.datavec.nlp.tokenization.tokenizerfactory.TokenizerFactory; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + +/** + * Static utility class for textual based windowing functions + * @author Adam Gibson + */ +public class Windows { + + + /** + * Constructs a list of window of size windowSize. + * Note that padding for each window is created as well. + * @param words the words to tokenize and construct windows from + * @param windowSize the window size to generate + * @return the list of windows for the tokenized string + */ + public static List windows(InputStream words, int windowSize) { + Tokenizer tokenizer = new DefaultStreamTokenizer(words); + List list = new ArrayList<>(); + while (tokenizer.hasMoreTokens()) + list.add(tokenizer.nextToken()); + return windows(list, windowSize); + } + + /** + * Constructs a list of window of size windowSize. + * Note that padding for each window is created as well. + * @param words the words to tokenize and construct windows from + * @param tokenizerFactory tokenizer factory to use + * @param windowSize the window size to generate + * @return the list of windows for the tokenized string + */ + public static List windows(InputStream words, TokenizerFactory tokenizerFactory, int windowSize) { + Tokenizer tokenizer = tokenizerFactory.create(words); + List list = new ArrayList<>(); + while (tokenizer.hasMoreTokens()) + list.add(tokenizer.nextToken()); + + if (list.isEmpty()) + throw new IllegalStateException("No tokens found for windows"); + + return windows(list, windowSize); + } + + + /** + * Constructs a list of window of size windowSize. + * Note that padding for each window is created as well. + * @param words the words to tokenize and construct windows from + * @param windowSize the window size to generate + * @return the list of windows for the tokenized string + */ + public static List windows(String words, int windowSize) { + StringTokenizer tokenizer = new StringTokenizer(words); + List list = new ArrayList(); + while (tokenizer.hasMoreTokens()) + list.add(tokenizer.nextToken()); + return windows(list, windowSize); + } + + /** + * Constructs a list of window of size windowSize. + * Note that padding for each window is created as well. + * @param words the words to tokenize and construct windows from + * @param tokenizerFactory tokenizer factory to use + * @param windowSize the window size to generate + * @return the list of windows for the tokenized string + */ + public static List windows(String words, TokenizerFactory tokenizerFactory, int windowSize) { + Tokenizer tokenizer = tokenizerFactory.create(words); + List list = new ArrayList<>(); + while (tokenizer.hasMoreTokens()) + list.add(tokenizer.nextToken()); + + if (list.isEmpty()) + throw new IllegalStateException("No tokens found for windows"); + + return windows(list, windowSize); + } + + + /** + * Constructs a list of window of size windowSize. + * Note that padding for each window is created as well. + * @param words the words to tokenize and construct windows from + * @return the list of windows for the tokenized string + */ + public static List windows(String words) { + StringTokenizer tokenizer = new StringTokenizer(words); + List list = new ArrayList(); + while (tokenizer.hasMoreTokens()) + list.add(tokenizer.nextToken()); + return windows(list, 5); + } + + /** + * Constructs a list of window of size windowSize. + * Note that padding for each window is created as well. + * @param words the words to tokenize and construct windows from + * @param tokenizerFactory tokenizer factory to use + * @return the list of windows for the tokenized string + */ + public static List windows(String words, TokenizerFactory tokenizerFactory) { + Tokenizer tokenizer = tokenizerFactory.create(words); + List list = new ArrayList<>(); + while (tokenizer.hasMoreTokens()) + list.add(tokenizer.nextToken()); + return windows(list, 5); + } + + + /** + * Creates a sliding window from text + * @param windowSize the window size to use + * @param wordPos the position of the word to center + * @param sentence the sentence to createComplex a window for + * @return a window based on the given sentence + */ + public static Window windowForWordInPosition(int windowSize, int wordPos, List sentence) { + List window = new ArrayList<>(); + List onlyTokens = new ArrayList<>(); + int contextSize = (int) Math.floor((windowSize - 1) / 2); + + for (int i = wordPos - contextSize; i <= wordPos + contextSize; i++) { + if (i < 0) + window.add(""); + else if (i >= sentence.size()) + window.add(""); + else { + onlyTokens.add(sentence.get(i)); + window.add(sentence.get(i)); + + } + } + + String wholeSentence = StringUtils.join(sentence); + String window2 = StringUtils.join(onlyTokens); + int begin = wholeSentence.indexOf(window2); + int end = begin + window2.length(); + return new Window(window, begin, end); + + } + + + /** + * Constructs a list of window of size windowSize + * @param words the words to construct windows from + * @return the list of windows for the tokenized string + */ + public static List windows(List words, int windowSize) { + + List ret = new ArrayList<>(); + + for (int i = 0; i < words.size(); i++) + ret.add(windowForWordInPosition(windowSize, i, words)); + + + return ret; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/reader/TfidfRecordReader.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/reader/TfidfRecordReader.java new file mode 100644 index 000000000..8401b640d --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/reader/TfidfRecordReader.java @@ -0,0 +1,191 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.reader; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.metadata.RecordMetaDataURI; +import org.datavec.api.records.reader.impl.FileRecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.api.vector.Vectorizer; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; +import org.datavec.nlp.vectorizer.TfidfVectorizer; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.IOException; +import java.util.*; + +/** + * TFIDF record reader (wraps a tfidf vectorizer + * for delivering labels and conforming to the record reader interface) + * + * @author Adam Gibson + */ +public class TfidfRecordReader extends FileRecordReader { + private TfidfVectorizer tfidfVectorizer; + private List records = new ArrayList<>(); + private Iterator recordIter; + private int numFeatures; + private boolean initialized = false; + + + @Override + public void initialize(InputSplit split) throws IOException, InterruptedException { + initialize(new Configuration(), split); + } + + @Override + public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException { + super.initialize(conf, split); + //train a new one since it hasn't been specified + if (tfidfVectorizer == null) { + tfidfVectorizer = new TfidfVectorizer(); + tfidfVectorizer.initialize(conf); + + //clear out old strings + records.clear(); + + INDArray ret = tfidfVectorizer.fitTransform(this, new Vectorizer.RecordCallBack() { + @Override + public void onRecord(Record fullRecord) { + records.add(fullRecord); + } + }); + + //cache the number of features used for each document + numFeatures = ret.columns(); + recordIter = records.iterator(); + } else { + records = new ArrayList<>(); + + //the record reader has 2 phases, we are skipping the + //document frequency phase and just using the super() to get the file contents + //and pass it to the already existing vectorizer. + while (super.hasNext()) { + Record fileContents = super.nextRecord(); + INDArray transform = tfidfVectorizer.transform(fileContents); + + org.datavec.api.records.impl.Record record = new org.datavec.api.records.impl.Record( + new ArrayList<>(Collections.singletonList(new NDArrayWritable(transform))), + new RecordMetaDataURI(fileContents.getMetaData().getURI(), TfidfRecordReader.class)); + + if (appendLabel) + record.getRecord().add(fileContents.getRecord().get(fileContents.getRecord().size() - 1)); + + records.add(record); + } + + recordIter = records.iterator(); + } + + this.initialized = true; + } + + @Override + public void reset() { + if (inputSplit == null) + throw new UnsupportedOperationException("Cannot reset without first initializing"); + recordIter = records.iterator(); + } + + @Override + public Record nextRecord() { + if (recordIter == null) + return super.nextRecord(); + return recordIter.next(); + } + + @Override + public List next() { + return nextRecord().getRecord(); + } + + @Override + public boolean hasNext() { + //we aren't done vectorizing yet + if (recordIter == null) + return super.hasNext(); + return recordIter.hasNext(); + } + + @Override + public void close() throws IOException { + + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return conf; + } + + public TfidfVectorizer getTfidfVectorizer() { + return tfidfVectorizer; + } + + public void setTfidfVectorizer(TfidfVectorizer tfidfVectorizer) { + if (initialized) { + throw new IllegalArgumentException( + "Setting TfidfVectorizer after TfidfRecordReader initialization doesn't have an effect"); + } + this.tfidfVectorizer = tfidfVectorizer; + } + + public int getNumFeatures() { + return numFeatures; + } + + public void shuffle() { + this.shuffle(new Random()); + } + + public void shuffle(Random random) { + Collections.shuffle(this.records, random); + this.reset(); + } + + @Override + public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException { + return loadFromMetaData(Collections.singletonList(recordMetaData)).get(0); + } + + @Override + public List loadFromMetaData(List recordMetaDatas) throws IOException { + List out = new ArrayList<>(); + + for (Record fileContents : super.loadFromMetaData(recordMetaDatas)) { + INDArray transform = tfidfVectorizer.transform(fileContents); + + org.datavec.api.records.impl.Record record = new org.datavec.api.records.impl.Record( + new ArrayList<>(Collections.singletonList(new NDArrayWritable(transform))), + new RecordMetaDataURI(fileContents.getMetaData().getURI(), TfidfRecordReader.class)); + + if (appendLabel) + record.getRecord().add(fileContents.getRecord().get(fileContents.getRecord().size() - 1)); + out.add(record); + } + + return out; + } +} + diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/stopwords/StopWords.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/stopwords/StopWords.java new file mode 100644 index 000000000..abff8f999 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/stopwords/StopWords.java @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.stopwords; + +import org.apache.commons.io.IOUtils; + +import java.io.IOException; +import java.util.List; + +/** + * Loads stop words from the class path + * @author Adam Gibson + * + */ +public class StopWords { + + private static List stopWords; + + @SuppressWarnings("unchecked") + public static List getStopWords() { + + try { + if (stopWords == null) + stopWords = IOUtils.readLines(StopWords.class.getResourceAsStream("/stopwords")); + } catch (IOException e) { + throw new RuntimeException(e); + } + return stopWords; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/ConcurrentTokenizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/ConcurrentTokenizer.java new file mode 100644 index 000000000..d46e68790 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/ConcurrentTokenizer.java @@ -0,0 +1,140 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer; + +import opennlp.tools.tokenize.TokenizerME; +import opennlp.tools.tokenize.TokenizerModel; +import opennlp.tools.util.Span; +import opennlp.uima.tokenize.AbstractTokenizer; +import opennlp.uima.tokenize.TokenizerModelResource; +import opennlp.uima.util.AnnotatorUtil; +import opennlp.uima.util.UimaUtil; +import org.apache.uima.UimaContext; +import org.apache.uima.analysis_engine.AnalysisEngineProcessException; +import org.apache.uima.cas.CAS; +import org.apache.uima.cas.Feature; +import org.apache.uima.cas.TypeSystem; +import org.apache.uima.cas.text.AnnotationFS; +import org.apache.uima.resource.ResourceAccessException; +import org.apache.uima.resource.ResourceInitializationException; + +/** + * OpenNLP Tokenizer annotator. + *

+ * Mandatory parameters + * + * + * + * + * + *
Type Name Description
String opennlp.uima.ModelName The name of the model file
String opennlp.uima.SentenceType The full name of the sentence type
String opennlp.uima.TokenType The full name of the token type
+ *

+ * Optional parameters + * + * + * + *
Type Name Description
String opennlp.uima.ProbabilityFeature The name of the double + * probability feature (not applyTransformToDestination by default)
+ * @see {@link TokenizerME} + */ +public class ConcurrentTokenizer extends AbstractTokenizer { + + /** + * The OpenNLP tokenizer. + */ + private TokenizerME tokenizer; + + private Feature probabilityFeature; + + @Override + public synchronized void process(CAS cas) throws AnalysisEngineProcessException { + super.process(cas); + } + + /** + * Initializes a new instance. + * + * Note: Use {@link #initialize(UimaContext) } to initialize + * this instance. Not use the constructor. + */ + public ConcurrentTokenizer() { + super("OpenNLP Tokenizer"); + + // must not be implemented ! + } + + /** + * Initializes the current instance with the given context. + * + * Note: Do all initialization in this method, do not use the constructor. + */ + public void initialize(UimaContext context) throws ResourceInitializationException { + + super.initialize(context); + + TokenizerModel model; + + try { + TokenizerModelResource modelResource = + (TokenizerModelResource) context.getResourceObject(UimaUtil.MODEL_PARAMETER); + + model = modelResource.getModel(); + } catch (ResourceAccessException e) { + throw new ResourceInitializationException(e); + } + + tokenizer = new TokenizerME(model); + } + + /** + * Initializes the type system. + */ + public void typeSystemInit(TypeSystem typeSystem) throws AnalysisEngineProcessException { + + super.typeSystemInit(typeSystem); + + probabilityFeature = AnnotatorUtil.getOptionalFeatureParameter(context, tokenType, + UimaUtil.PROBABILITY_FEATURE_PARAMETER, CAS.TYPE_NAME_DOUBLE); + } + + + @Override + protected Span[] tokenize(CAS cas, AnnotationFS sentence) { + return tokenizer.tokenizePos(sentence.getCoveredText()); + } + + @Override + protected void postProcessAnnotations(Span[] tokens, AnnotationFS[] tokenAnnotations) { + // if interest + if (probabilityFeature != null) { + double tokenProbabilties[] = tokenizer.getTokenProbabilities(); + + for (int i = 0; i < tokenAnnotations.length; i++) { + tokenAnnotations[i].setDoubleValue(probabilityFeature, tokenProbabilties[i]); + } + } + } + + /** + * Releases allocated resources. + */ + public void destroy() { + // dereference model to allow garbage collection + tokenizer = null; + } +} + diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/DefaultStreamTokenizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/DefaultStreamTokenizer.java new file mode 100644 index 000000000..9216ed24a --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/DefaultStreamTokenizer.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer; + + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +/** + * Tokenizer based on the {@link java.io.StreamTokenizer} + * @author Adam Gibson + * + */ +public class DefaultStreamTokenizer implements Tokenizer { + + private StreamTokenizer streamTokenizer; + private TokenPreProcess tokenPreProcess; + + + public DefaultStreamTokenizer(InputStream is) { + Reader r = new BufferedReader(new InputStreamReader(is)); + streamTokenizer = new StreamTokenizer(r); + + } + + @Override + public boolean hasMoreTokens() { + if (streamTokenizer.ttype != StreamTokenizer.TT_EOF) { + try { + streamTokenizer.nextToken(); + } catch (IOException e1) { + throw new RuntimeException(e1); + } + } + return streamTokenizer.ttype != StreamTokenizer.TT_EOF && streamTokenizer.ttype != -1; + } + + @Override + public int countTokens() { + return getTokens().size(); + } + + @Override + public String nextToken() { + StringBuilder sb = new StringBuilder(); + + + if (streamTokenizer.ttype == StreamTokenizer.TT_WORD) { + sb.append(streamTokenizer.sval); + } else if (streamTokenizer.ttype == StreamTokenizer.TT_NUMBER) { + sb.append(streamTokenizer.nval); + } else if (streamTokenizer.ttype == StreamTokenizer.TT_EOL) { + try { + while (streamTokenizer.ttype == StreamTokenizer.TT_EOL) + streamTokenizer.nextToken(); + } catch (IOException e) { + throw new RuntimeException(e); + + } + } + + else if (hasMoreTokens()) + return nextToken(); + + + String ret = sb.toString(); + + if (tokenPreProcess != null) + ret = tokenPreProcess.preProcess(ret); + return ret; + + } + + @Override + public List getTokens() { + List tokens = new ArrayList<>(); + while (hasMoreTokens()) { + tokens.add(nextToken()); + } + return tokens; + } + + @Override + public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) { + this.tokenPreProcess = tokenPreProcessor; + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/DefaultTokenizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/DefaultTokenizer.java new file mode 100644 index 000000000..4b393c0d5 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/DefaultTokenizer.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + +/** + * Default tokenizer + * @author Adam Gibson + */ +public class DefaultTokenizer implements Tokenizer { + + public DefaultTokenizer(String tokens) { + tokenizer = new StringTokenizer(tokens); + } + + private StringTokenizer tokenizer; + private TokenPreProcess tokenPreProcess; + + @Override + public boolean hasMoreTokens() { + return tokenizer.hasMoreTokens(); + } + + @Override + public int countTokens() { + return tokenizer.countTokens(); + } + + @Override + public String nextToken() { + String base = tokenizer.nextToken(); + if (tokenPreProcess != null) + base = tokenPreProcess.preProcess(base); + return base; + } + + @Override + public List getTokens() { + List tokens = new ArrayList<>(); + while (hasMoreTokens()) { + tokens.add(nextToken()); + } + return tokens; + } + + @Override + public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) { + this.tokenPreProcess = tokenPreProcessor; + + } + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/PosUimaTokenizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/PosUimaTokenizer.java new file mode 100644 index 000000000..e9e94bb99 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/PosUimaTokenizer.java @@ -0,0 +1,139 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer; + +import org.apache.uima.analysis_engine.AnalysisEngine; +import org.apache.uima.cas.CAS; +import org.apache.uima.fit.factory.AnalysisEngineFactory; +import org.apache.uima.fit.util.JCasUtil; +import org.cleartk.token.type.Sentence; +import org.cleartk.token.type.Token; +import org.datavec.nlp.annotator.PoStagger; +import org.datavec.nlp.annotator.SentenceAnnotator; +import org.datavec.nlp.annotator.StemmerAnnotator; +import org.datavec.nlp.annotator.TokenizerAnnotator; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Filter by part of speech tag. + * Any not valid part of speech tags + * become NONE + * @author Adam Gibson + * + */ +public class PosUimaTokenizer implements Tokenizer { + + private static AnalysisEngine engine; + private List tokens; + private Collection allowedPosTags; + private int index; + private static CAS cas; + + public PosUimaTokenizer(String tokens, AnalysisEngine engine, Collection allowedPosTags) { + if (engine == null) + PosUimaTokenizer.engine = engine; + this.allowedPosTags = allowedPosTags; + this.tokens = new ArrayList<>(); + try { + if (cas == null) + cas = engine.newCAS(); + + cas.reset(); + cas.setDocumentText(tokens); + PosUimaTokenizer.engine.process(cas); + for (Sentence s : JCasUtil.select(cas.getJCas(), Sentence.class)) { + for (Token t : JCasUtil.selectCovered(Token.class, s)) { + //add NONE for each invalid token + if (valid(t)) + if (t.getLemma() != null) + this.tokens.add(t.getLemma()); + else if (t.getStem() != null) + this.tokens.add(t.getStem()); + else + this.tokens.add(t.getCoveredText()); + else + this.tokens.add("NONE"); + } + } + + + + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + private boolean valid(Token token) { + String check = token.getCoveredText(); + if (check.matches("<[A-Z]+>") || check.matches("")) + return false; + else if (token.getPos() != null && !this.allowedPosTags.contains(token.getPos())) + return false; + return true; + } + + + + @Override + public boolean hasMoreTokens() { + return index < tokens.size(); + } + + @Override + public int countTokens() { + return tokens.size(); + } + + @Override + public String nextToken() { + String ret = tokens.get(index); + index++; + return ret; + } + + @Override + public List getTokens() { + List tokens = new ArrayList(); + while (hasMoreTokens()) { + tokens.add(nextToken()); + } + return tokens; + } + + public static AnalysisEngine defaultAnalysisEngine() { + try { + return AnalysisEngineFactory.createEngine(AnalysisEngineFactory.createEngineDescription( + SentenceAnnotator.getDescription(), TokenizerAnnotator.getDescription(), + PoStagger.getDescription("en"), StemmerAnnotator.getDescription("English"))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) { + // TODO Auto-generated method stub + + } + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/TokenPreProcess.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/TokenPreProcess.java new file mode 100644 index 000000000..4390a17c9 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/TokenPreProcess.java @@ -0,0 +1,35 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer; + + +/** + * Token preprocessing + * @author Adam Gibson + * + */ +public interface TokenPreProcess { + + /** + * Pre process a token + * @param token the token to pre process + * @return the preprocessed token + */ + String preProcess(String token); + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/Tokenizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/Tokenizer.java new file mode 100644 index 000000000..37e8ddb63 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/Tokenizer.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer; + +import java.util.List; + +/** + * A representation of a tokenizer. + * Different applications may require + * different kind of tokenization (say rules based vs more formal NLP approaches) + * @author Adam Gibson + * + */ +public interface Tokenizer { + + /** + * An iterator for tracking whether + * more tokens are left in the iterator not + * @return whether there is anymore tokens + * to iterate over + */ + boolean hasMoreTokens(); + + /** + * The number of tokens in the tokenizer + * @return the number of tokens + */ + int countTokens(); + + /** + * The next token (word usually) in the string + * @return the next token in the string if any + */ + String nextToken(); + + /** + * Returns a list of all the tokens + * @return a list of all the tokens + */ + List getTokens(); + + /** + * Set the token pre process + * @param tokenPreProcessor the token pre processor to set + */ + void setTokenPreProcessor(TokenPreProcess tokenPreProcessor); + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/UimaTokenizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/UimaTokenizer.java new file mode 100644 index 000000000..eb14fdabd --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/UimaTokenizer.java @@ -0,0 +1,119 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer; + +import org.apache.uima.cas.CAS; +import org.apache.uima.fit.util.JCasUtil; +import org.cleartk.token.type.Token; +import org.datavec.nlp.uima.UimaResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Tokenizer based on the passed in analysis engine + * @author Adam Gibson + * + */ +public class UimaTokenizer implements Tokenizer { + + private List tokens; + private int index; + private static Logger log = LoggerFactory.getLogger(UimaTokenizer.class); + private boolean checkForLabel; + private TokenPreProcess tokenPreProcessor; + + + public UimaTokenizer(String tokens, UimaResource resource, boolean checkForLabel) { + + this.checkForLabel = checkForLabel; + this.tokens = new ArrayList<>(); + try { + CAS cas = resource.process(tokens); + + Collection tokenList = JCasUtil.select(cas.getJCas(), Token.class); + + for (Token t : tokenList) { + + if (!checkForLabel || valid(t.getCoveredText())) + if (t.getLemma() != null) + this.tokens.add(t.getLemma()); + else if (t.getStem() != null) + this.tokens.add(t.getStem()); + else + this.tokens.add(t.getCoveredText()); + } + + + resource.release(cas); + + + } catch (Exception e) { + log.error("",e); + throw new RuntimeException(e); + } + + } + + private boolean valid(String check) { + if (check.matches("<[A-Z]+>") || check.matches("")) + return false; + return true; + } + + + + @Override + public boolean hasMoreTokens() { + return index < tokens.size(); + } + + @Override + public int countTokens() { + return tokens.size(); + } + + @Override + public String nextToken() { + String ret = tokens.get(index); + index++; + if (tokenPreProcessor != null) { + ret = tokenPreProcessor.preProcess(ret); + } + return ret; + } + + @Override + public List getTokens() { + List tokens = new ArrayList<>(); + while (hasMoreTokens()) { + tokens.add(nextToken()); + } + return tokens; + } + + @Override + public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) { + this.tokenPreProcessor = tokenPreProcessor; + } + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/preprocessor/EndingPreProcessor.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/preprocessor/EndingPreProcessor.java new file mode 100644 index 000000000..7a1ddc044 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/preprocessor/EndingPreProcessor.java @@ -0,0 +1,43 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer.preprocessor; + + +import org.datavec.nlp.tokenization.tokenizer.TokenPreProcess; + +/** + * Gets rid of endings: + * + * ed,ing, ly, s, . + * @author Adam Gibson + */ +public class EndingPreProcessor implements TokenPreProcess { + @Override + public String preProcess(String token) { + if (token.endsWith("s") && !token.endsWith("ss")) + token = token.substring(0, token.length() - 1); + if (token.endsWith(".")) + token = token.substring(0, token.length() - 1); + if (token.endsWith("ed")) + token = token.substring(0, token.length() - 2); + if (token.endsWith("ing")) + token = token.substring(0, token.length() - 3); + if (token.endsWith("ly")) + token = token.substring(0, token.length() - 2); + return token; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/preprocessor/LowerCasePreProcessor.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/preprocessor/LowerCasePreProcessor.java new file mode 100644 index 000000000..69987b14c --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizer/preprocessor/LowerCasePreProcessor.java @@ -0,0 +1,26 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizer.preprocessor; + +import org.datavec.nlp.tokenization.tokenizer.TokenPreProcess; + +public class LowerCasePreProcessor implements TokenPreProcess { + @Override + public String preProcess(String token) { + return token.toLowerCase(); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/DefaultTokenizerFactory.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/DefaultTokenizerFactory.java new file mode 100644 index 000000000..4e70edfe2 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/DefaultTokenizerFactory.java @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizerfactory; + + + +import org.datavec.nlp.tokenization.tokenizer.DefaultStreamTokenizer; +import org.datavec.nlp.tokenization.tokenizer.DefaultTokenizer; +import org.datavec.nlp.tokenization.tokenizer.TokenPreProcess; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; + +import java.io.InputStream; + +/** + * Default tokenizer based on string tokenizer or stream tokenizer + * @author Adam Gibson + */ +public class DefaultTokenizerFactory implements TokenizerFactory { + + private TokenPreProcess tokenPreProcess; + + @Override + public Tokenizer create(String toTokenize) { + DefaultTokenizer t = new DefaultTokenizer(toTokenize); + t.setTokenPreProcessor(tokenPreProcess); + return t; + } + + @Override + public Tokenizer create(InputStream toTokenize) { + Tokenizer t = new DefaultStreamTokenizer(toTokenize); + t.setTokenPreProcessor(tokenPreProcess); + return t; + } + + @Override + public void setTokenPreProcessor(TokenPreProcess preProcessor) { + this.tokenPreProcess = preProcessor; + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/PosUimaTokenizerFactory.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/PosUimaTokenizerFactory.java new file mode 100644 index 000000000..5419bf9ef --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/PosUimaTokenizerFactory.java @@ -0,0 +1,88 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizerfactory; + + +import org.apache.uima.analysis_engine.AnalysisEngine; +import org.datavec.nlp.annotator.PoStagger; +import org.datavec.nlp.annotator.SentenceAnnotator; +import org.datavec.nlp.annotator.StemmerAnnotator; +import org.datavec.nlp.annotator.TokenizerAnnotator; +import org.datavec.nlp.tokenization.tokenizer.PosUimaTokenizer; +import org.datavec.nlp.tokenization.tokenizer.TokenPreProcess; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; + +import java.io.InputStream; +import java.util.Collection; + +import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngine; +import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; + +/** + * Creates a tokenizer that filters by + * part of speech tags + * @see {org.deeplearning4j.text.tokenization.tokenizer.PosUimaTokenizer} + * @author Adam Gibson + * + */ +public class PosUimaTokenizerFactory implements TokenizerFactory { + + private AnalysisEngine tokenizer; + private Collection allowedPoSTags; + private TokenPreProcess tokenPreProcess; + + + public PosUimaTokenizerFactory(Collection allowedPoSTags) { + this(defaultAnalysisEngine(), allowedPoSTags); + } + + public PosUimaTokenizerFactory(AnalysisEngine tokenizer, Collection allowedPosTags) { + this.tokenizer = tokenizer; + this.allowedPoSTags = allowedPosTags; + } + + + public static AnalysisEngine defaultAnalysisEngine() { + try { + return createEngine(createEngineDescription(SentenceAnnotator.getDescription(), + TokenizerAnnotator.getDescription(), PoStagger.getDescription("en"), + StemmerAnnotator.getDescription("English"))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + @Override + public Tokenizer create(String toTokenize) { + PosUimaTokenizer t = new PosUimaTokenizer(toTokenize, tokenizer, allowedPoSTags); + t.setTokenPreProcessor(tokenPreProcess); + return t; + } + + @Override + public Tokenizer create(InputStream toTokenize) { + throw new UnsupportedOperationException(); + } + + @Override + public void setTokenPreProcessor(TokenPreProcess preProcessor) { + this.tokenPreProcess = preProcessor; + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/TokenizerFactory.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/TokenizerFactory.java new file mode 100644 index 000000000..7d1489fdc --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/TokenizerFactory.java @@ -0,0 +1,59 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizerfactory; + + + +import org.datavec.nlp.tokenization.tokenizer.TokenPreProcess; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.InputStream; + +/** + * Generates a tokenizer for a given string + * @author Adam Gibson + * + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface TokenizerFactory { + + /** + * The tokenizer to createComplex + * @param toTokenize the string to createComplex the tokenizer with + * @return the new tokenizer + */ + Tokenizer create(String toTokenize); + + /** + * Create a tokenizer based on an input stream + * @param toTokenize + * @return + */ + Tokenizer create(InputStream toTokenize); + + /** + * Sets a token pre processor to be used + * with every tokenizer + * @param preProcessor the token pre processor to use + */ + void setTokenPreProcessor(TokenPreProcess preProcessor); + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/UimaTokenizerFactory.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/UimaTokenizerFactory.java new file mode 100644 index 000000000..7b24244ac --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/tokenization/tokenizerfactory/UimaTokenizerFactory.java @@ -0,0 +1,134 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.tokenization.tokenizerfactory; + +import org.apache.uima.analysis_engine.AnalysisEngine; +import org.apache.uima.fit.factory.AnalysisEngineFactory; +import org.apache.uima.resource.ResourceInitializationException; +import org.datavec.nlp.annotator.SentenceAnnotator; +import org.datavec.nlp.annotator.TokenizerAnnotator; +import org.datavec.nlp.tokenization.tokenizer.TokenPreProcess; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; +import org.datavec.nlp.tokenization.tokenizer.UimaTokenizer; +import org.datavec.nlp.uima.UimaResource; + +import java.io.InputStream; + + +/** + * Uses a uima {@link AnalysisEngine} to + * tokenize text. + * + * + * @author Adam Gibson + * + */ +public class UimaTokenizerFactory implements TokenizerFactory { + + + private UimaResource uimaResource; + private boolean checkForLabel; + private static AnalysisEngine defaultAnalysisEngine; + private TokenPreProcess preProcess; + + public UimaTokenizerFactory() throws ResourceInitializationException { + this(defaultAnalysisEngine(), true); + } + + + public UimaTokenizerFactory(UimaResource resource) { + this(resource, true); + } + + + public UimaTokenizerFactory(AnalysisEngine tokenizer) { + this(tokenizer, true); + } + + + + public UimaTokenizerFactory(UimaResource resource, boolean checkForLabel) { + this.uimaResource = resource; + this.checkForLabel = checkForLabel; + } + + public UimaTokenizerFactory(boolean checkForLabel) throws ResourceInitializationException { + this(defaultAnalysisEngine(), checkForLabel); + } + + + + public UimaTokenizerFactory(AnalysisEngine tokenizer, boolean checkForLabel) { + super(); + this.checkForLabel = checkForLabel; + try { + this.uimaResource = new UimaResource(tokenizer); + + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + + @Override + public Tokenizer create(String toTokenize) { + if (toTokenize == null || toTokenize.isEmpty()) + throw new IllegalArgumentException("Unable to proceed; on sentence to tokenize"); + Tokenizer ret = new UimaTokenizer(toTokenize, uimaResource, checkForLabel); + ret.setTokenPreProcessor(preProcess); + return ret; + } + + + public UimaResource getUimaResource() { + return uimaResource; + } + + + /** + * Creates a tokenization,/stemming pipeline + * @return a tokenization/stemming pipeline + */ + public static AnalysisEngine defaultAnalysisEngine() { + try { + if (defaultAnalysisEngine == null) + + defaultAnalysisEngine = AnalysisEngineFactory.createEngine( + AnalysisEngineFactory.createEngineDescription(SentenceAnnotator.getDescription(), + TokenizerAnnotator.getDescription())); + + return defaultAnalysisEngine; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + @Override + public Tokenizer create(InputStream toTokenize) { + throw new UnsupportedOperationException(); + } + + @Override + public void setTokenPreProcessor(TokenPreProcess preProcessor) { + this.preProcess = preProcessor; + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/BagOfWordsTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/BagOfWordsTransform.java new file mode 100644 index 000000000..1a1a5eaf2 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/BagOfWordsTransform.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.transforms; + +import org.datavec.api.transform.Transform; +import org.datavec.api.writable.Writable; +import org.nd4j.linalg.api.ndarray.INDArray; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.util.List; + +/** + * A bag of words transform represents taking a list of words + * and converting it to a vector where that vector is + * of length number of vocab words. + * Vocab words are determined by what is passed in to the transform via a constructor generally. + * + * To build a vocab in NLP, you crawl a corpus with a tokenizer tracking word frequencies. + * Any words above a specified frequency are added to an ordered list. + * + * When using this ordered list in NLP pipelines (at least for bag of words) + * you perform a lookup for each word in a string (determined by a tokenizer) + * and fill in the appropriate weight (a word count or tfidf weight generally) + * to represent the word at a particular column. + * + * The column is determined by the ordered list of words. + * + * @author Adam Gibson + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +public interface BagOfWordsTransform extends Transform { + + + /** + * The output shape of the transform (usually 1 x number of words) + * @return + */ + long[] outputShape(); + + /** + * The vocab words in the transform. + * This is the words that were accumulated + * when building a vocabulary. + * (This is generally associated with some form of + * mininmum words frequency scanning to build a vocab + * you then map on to a list of vocab words as a list) + * @return the vocab words for the transform + */ + List vocabWords(); + + /** + * Transform for a list of tokens + * that are objects. This is to allow loose + * typing for tokens that are unique (non string) + * @param tokens the token objects to transform + * @return the output {@link INDArray} (a tokens.size() by {@link #vocabWords()}.size() array) + */ + INDArray transformFromObject(List> tokens); + + + /** + * Transform for a list of tokens + * that are {@link Writable} (Generally {@link org.datavec.api.writable.Text} + * @param tokens the token objects to transform + * @return the output {@link INDArray} (a tokens.size() by {@link #vocabWords()}.size() array) + */ + INDArray transformFrom(List> tokens); + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/BaseWordMapTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/BaseWordMapTransform.java new file mode 100644 index 000000000..92500f8f6 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/BaseWordMapTransform.java @@ -0,0 +1,20 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.transforms; + +public class BaseWordMapTransform { +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/GazeteerTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/GazeteerTransform.java new file mode 100644 index 000000000..1c774e904 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/GazeteerTransform.java @@ -0,0 +1,153 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.transforms; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.datavec.api.transform.metadata.ColumnMetaData; +import org.datavec.api.transform.metadata.NDArrayMetaData; +import org.datavec.api.transform.transform.BaseColumnTransform; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * A gazeteer is a work lookup table + * based on a word list. + * A 0 or 1 is returned if the word is in the list. + * A word list is also needed to represent the vocab words + * that go along side the vector creation. + * For more on this process, please see the {@link BagOfWordsTransform} + * interface docs. + * + * @author Adam Gibson + */ +@Data +@EqualsAndHashCode(callSuper = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties({"gazeteer"}) +public class GazeteerTransform extends BaseColumnTransform implements BagOfWordsTransform { + + private String newColumnName; + private List wordList; + private Set gazeteer; + + @JsonCreator + public GazeteerTransform(@JsonProperty("columnName") String columnName, + @JsonProperty("newColumnName")String newColumnName, + @JsonProperty("wordList") List wordList) { + super(columnName); + this.newColumnName = newColumnName; + this.wordList = wordList; + this.gazeteer = new HashSet<>(wordList); + } + + @Override + public ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType) { + return new NDArrayMetaData(newName,new long[]{wordList.size()}); + } + + @Override + public Writable map(Writable columnWritable) { + throw new UnsupportedOperationException(); + } + + @Override + public Object mapSequence(Object sequence) { + List> sequenceInput = (List>) sequence; + INDArray ret = Nd4j.create(DataType.FLOAT, wordList.size()); + + for(List list : sequenceInput) { + for(Object token : list) { + String s = token.toString(); + if(gazeteer.contains(s)) { + ret.putScalar(wordList.indexOf(s),1); + } + } + } + return ret; + } + + + + @Override + public List> mapSequence(List> sequence) { + INDArray arr = (INDArray) mapSequence((Object) sequence); + return Collections.singletonList(Collections.singletonList(new NDArrayWritable(arr))); + } + + @Override + public String toString() { + return newColumnName; + } + + @Override + public Object map(Object input) { + return gazeteer.contains(input.toString()); + } + + @Override + public String outputColumnName() { + return newColumnName; + } + + @Override + public String[] outputColumnNames() { + return new String[]{newColumnName}; + } + + @Override + public String[] columnNames() { + return new String[]{columnName()}; + } + + @Override + public String columnName() { + return columnName; + } + + @Override + public long[] outputShape() { + return new long[]{wordList.size()}; + } + + @Override + public List vocabWords() { + return wordList; + } + + @Override + public INDArray transformFromObject(List> tokens) { + return (INDArray) mapSequence(tokens); + } + + @Override + public INDArray transformFrom(List> tokens) { + return (INDArray) mapSequence((Object) tokens); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/MultiNlpTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/MultiNlpTransform.java new file mode 100644 index 000000000..b702422e8 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/MultiNlpTransform.java @@ -0,0 +1,154 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.datavec.nlp.transforms; + +import org.datavec.api.transform.metadata.ColumnMetaData; +import org.datavec.api.transform.metadata.NDArrayMetaData; +import org.datavec.api.transform.transform.BaseColumnTransform; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.list.NDArrayList; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collections; +import java.util.List; + +/** + * A multi NLP transform takes in 1 or more bag of words transforms as a pipeline + * and runs them in sequence. + * This transform takes in a column name and 1 or more bag of words transforms to run. + * Lastly, a new column name is specified. + * + * @author Adam Gibson + */ +public class MultiNlpTransform extends BaseColumnTransform implements BagOfWordsTransform { + + private BagOfWordsTransform[] transforms; + private String newColumnName; + private List vocabWords; + + /** + * + * @param columnName + * @param transforms + * @param newColumnName + */ + @JsonCreator + public MultiNlpTransform(@JsonProperty("columnName") String columnName, + @JsonProperty("transforms") BagOfWordsTransform[] transforms, + @JsonProperty("newColumnName") String newColumnName) { + super(columnName); + this.transforms = transforms; + this.vocabWords = transforms[0].vocabWords(); + if(transforms.length > 1) { + for(int i = 1; i < transforms.length; i++) { + if(!transforms[i].vocabWords().equals(vocabWords)) { + throw new IllegalArgumentException("Vocab words not consistent across transforms!"); + } + } + } + + this.newColumnName = newColumnName; + } + + @Override + public Object mapSequence(Object sequence) { + NDArrayList ndArrayList = new NDArrayList(); + for(BagOfWordsTransform bagofWordsTransform : transforms) { + ndArrayList.addAll(new NDArrayList(bagofWordsTransform.transformFromObject((List>) sequence))); + } + + return ndArrayList.array(); + } + + @Override + public List> mapSequence(List> sequence) { + return Collections.singletonList(Collections.singletonList(new NDArrayWritable(transformFrom(sequence)))); + } + + @Override + public ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType) { + return new NDArrayMetaData(newName,outputShape()); + } + + @Override + public Writable map(Writable columnWritable) { + throw new UnsupportedOperationException("Only able to add for time series"); + } + + @Override + public String toString() { + return newColumnName; + } + + @Override + public Object map(Object input) { + throw new UnsupportedOperationException("Only able to add for time series"); + } + + @Override + public long[] outputShape() { + long[] ret = new long[transforms[0].outputShape().length]; + int validatedRank = transforms[0].outputShape().length; + for(int i = 1; i < transforms.length; i++) { + if(transforms[i].outputShape().length != validatedRank) { + throw new IllegalArgumentException("Inconsistent shape length at transform " + i + " , should have been: " + validatedRank); + } + } + for(int i = 0; i < transforms.length; i++) { + for(int j = 0; j < validatedRank; j++) + ret[j] += transforms[i].outputShape()[j]; + } + + return ret; + } + + @Override + public List vocabWords() { + return vocabWords; + } + + @Override + public INDArray transformFromObject(List> tokens) { + NDArrayList ndArrayList = new NDArrayList(); + for(BagOfWordsTransform bagofWordsTransform : transforms) { + INDArray arr2 = bagofWordsTransform.transformFromObject(tokens); + arr2 = arr2.reshape(arr2.length()); + NDArrayList newList = new NDArrayList(arr2,(int) arr2.length()); + ndArrayList.addAll(newList); } + + return ndArrayList.array(); + } + + @Override + public INDArray transformFrom(List> tokens) { + NDArrayList ndArrayList = new NDArrayList(); + for(BagOfWordsTransform bagofWordsTransform : transforms) { + INDArray arr2 = bagofWordsTransform.transformFrom(tokens); + arr2 = arr2.reshape(arr2.length()); + NDArrayList newList = new NDArrayList(arr2,(int) arr2.length()); + ndArrayList.addAll(newList); + } + + return ndArrayList.array(); + } + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/TokenizerBagOfWordsTermSequenceIndexTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/TokenizerBagOfWordsTermSequenceIndexTransform.java new file mode 100644 index 000000000..29cdf9153 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/transforms/TokenizerBagOfWordsTermSequenceIndexTransform.java @@ -0,0 +1,245 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.transforms; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.datavec.api.transform.metadata.ColumnMetaData; +import org.datavec.api.transform.metadata.NDArrayMetaData; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.transform.BaseColumnTransform; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.nlp.tokenization.tokenizer.TokenPreProcess; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; +import org.datavec.nlp.tokenization.tokenizerfactory.DefaultTokenizerFactory; +import org.datavec.nlp.tokenization.tokenizerfactory.TokenizerFactory; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Counter; +import org.nd4j.common.util.MathUtils; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * This transform takes in a list of words + * and outputs a single vector where that vector is of size + * number of words in the vocab. + * + * For more information on vocab, see {@link BagOfWordsTransform} + * + * For definition of a vocab, it is generated using a {@link TokenizerFactory} + * This transform will use {@link DefaultTokenizerFactory} + * for the tokenizer factory if one is not specified. + * Otherwise, one can specify a custom {@link TokenizerFactory} + * with a default constructor. + * + * The other components that need to be specified are: + * a word index map representing what words go in what columns + * an inverse document frequency map representing the weighting of inverse document frequencies + * for each word (this is for tfidf calculation) + * + * This is typically used for non english languages. + * + * + * @author Adam Gibson + */ +@Data +@EqualsAndHashCode(callSuper = true, exclude = {"tokenizerFactory"}) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class TokenizerBagOfWordsTermSequenceIndexTransform extends BaseColumnTransform { + + private String newColumName; + private Map wordIndexMap; + private Map weightMap; + private boolean exceptionOnUnknown; + private String tokenizerFactoryClass; + private String preprocessorClass; + private TokenizerFactory tokenizerFactory; + + @JsonCreator + public TokenizerBagOfWordsTermSequenceIndexTransform(@JsonProperty("columnName") String columnName, + @JsonProperty("newColumnName") String newColumnName, + @JsonProperty("wordIndexMap") Map wordIndexMap, + @JsonProperty("idfMap") Map idfMap, + @JsonProperty("exceptionOnUnknown") boolean exceptionOnUnknown, + @JsonProperty("tokenizerFactoryClass") String tokenizerFactoryClass, + @JsonProperty("preprocessorClass") String preprocessorClass) { + super(columnName); + this.newColumName = newColumnName; + this.wordIndexMap = wordIndexMap; + this.exceptionOnUnknown = exceptionOnUnknown; + this.weightMap = idfMap; + this.tokenizerFactoryClass = tokenizerFactoryClass; + this.preprocessorClass = preprocessorClass; + if(this.tokenizerFactoryClass == null) { + this.tokenizerFactoryClass = DefaultTokenizerFactory.class.getName(); + } + try { + tokenizerFactory = (TokenizerFactory) Class.forName(this.tokenizerFactoryClass).newInstance(); + } catch (Exception e) { + throw new IllegalStateException("Unable to instantiate tokenizer factory with empty constructor. Does the tokenizer factory class contain a default empty constructor?"); + } + + if(preprocessorClass != null){ + try { + TokenPreProcess tpp = (TokenPreProcess) Class.forName(this.preprocessorClass).newInstance(); + tokenizerFactory.setTokenPreProcessor(tpp); + } catch (Exception e){ + throw new IllegalStateException("Unable to instantiate preprocessor factory with empty constructor. Does the tokenizer factory class contain a default empty constructor?"); + } + } + + } + + + + @Override + public List map(List writables) { + Text text = (Text) writables.get(inputSchema.getIndexOfColumn(columnName)); + List ret = new ArrayList<>(writables); + ret.set(inputSchema.getIndexOfColumn(columnName),new NDArrayWritable(convert(text.toString()))); + return ret; + } + + @Override + public Object map(Object input) { + return convert(input.toString()); + } + + @Override + public Object mapSequence(Object sequence) { + return convert(sequence.toString()); + } + + @Override + public Schema transform(Schema inputSchema) { + Schema.Builder newSchema = new Schema.Builder(); + for(int i = 0; i < inputSchema.numColumns(); i++) { + if(inputSchema.getName(i).equals(this.columnName)) { + newSchema.addColumnNDArray(newColumName,new long[]{1,wordIndexMap.size()}); + } + else { + newSchema.addColumn(inputSchema.getMetaData(i)); + } + } + + return newSchema.build(); + } + + + /** + * Convert the given text + * in to an {@link INDArray} + * using the {@link TokenizerFactory} + * specified in the constructor. + * @param text the text to transform + * @return the created {@link INDArray} + * based on the {@link #wordIndexMap} for the column indices + * of the word. + */ + public INDArray convert(String text) { + Tokenizer tokenizer = tokenizerFactory.create(text); + List tokens = tokenizer.getTokens(); + INDArray create = Nd4j.create(1,wordIndexMap.size()); + Counter tokenizedCounter = new Counter<>(); + + for(int i = 0; i < tokens.size(); i++) { + tokenizedCounter.incrementCount(tokens.get(i),1.0); + } + + for(int i = 0; i < tokens.size(); i++) { + if(wordIndexMap.containsKey(tokens.get(i))) { + int idx = wordIndexMap.get(tokens.get(i)); + int count = (int) tokenizedCounter.getCount(tokens.get(i)); + double weight = tfidfWord(tokens.get(i),count,tokens.size()); + create.putScalar(idx,weight); + } + } + + return create; + } + + + /** + * Calculate the tifdf for a word + * given the word, word count, and document length + * @param word the word to calculate + * @param wordCount the word frequency + * @param documentLength the number of words in the document + * @return the tfidf weight for a given word + */ + public double tfidfWord(String word, long wordCount, long documentLength) { + double tf = tfForWord(wordCount, documentLength); + double idf = idfForWord(word); + return MathUtils.tfidf(tf, idf); + } + + /** + * Calculate the weight term frequency for a given + * word normalized by the dcoument length + * @param wordCount the word frequency + * @param documentLength the number of words in the edocument + * @return + */ + private double tfForWord(long wordCount, long documentLength) { + return wordCount; + } + + private double idfForWord(String word) { + if(weightMap.containsKey(word)) + return weightMap.get(word); + return 0; + } + + + @Override + public ColumnMetaData getNewColumnMetaData(String newName, ColumnMetaData oldColumnType) { + return new NDArrayMetaData(outputColumnName(),new long[]{1,wordIndexMap.size()}); + } + + @Override + public String outputColumnName() { + return newColumName; + } + + @Override + public String[] outputColumnNames() { + return new String[]{newColumName}; + } + + @Override + public String[] columnNames() { + return new String[]{columnName()}; + } + + @Override + public String columnName() { + return columnName; + } + + @Override + public Writable map(Writable columnWritable) { + return new NDArrayWritable(convert(columnWritable.toString())); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/uima/UimaResource.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/uima/UimaResource.java new file mode 100644 index 000000000..9c1c9546e --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/uima/UimaResource.java @@ -0,0 +1,108 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.uima; + +import org.apache.uima.analysis_engine.AnalysisEngine; +import org.apache.uima.analysis_engine.AnalysisEngineProcessException; +import org.apache.uima.cas.CAS; +import org.apache.uima.resource.ResourceInitializationException; +import org.apache.uima.util.CasPool; + +/** + * Resource holder for uima + * @author Adam Gibson + * + */ +public class UimaResource { + + private AnalysisEngine analysisEngine; + private CasPool casPool; + + public UimaResource(AnalysisEngine analysisEngine) throws ResourceInitializationException { + this.analysisEngine = analysisEngine; + this.casPool = new CasPool(Runtime.getRuntime().availableProcessors() * 10, analysisEngine); + + } + + public UimaResource(AnalysisEngine analysisEngine, CasPool casPool) { + this.analysisEngine = analysisEngine; + this.casPool = casPool; + + } + + + public AnalysisEngine getAnalysisEngine() { + return analysisEngine; + } + + + public void setAnalysisEngine(AnalysisEngine analysisEngine) { + this.analysisEngine = analysisEngine; + } + + + public CasPool getCasPool() { + return casPool; + } + + + public void setCasPool(CasPool casPool) { + this.casPool = casPool; + } + + + /** + * Use the given analysis engine and process the given text + * You must release the return cas yourself + * @param text the text to rpocess + * @return the processed cas + */ + public CAS process(String text) { + CAS cas = retrieve(); + + cas.setDocumentText(text); + try { + analysisEngine.process(cas); + } catch (AnalysisEngineProcessException e) { + if (text != null && !text.isEmpty()) + return process(text); + throw new RuntimeException(e); + } + + return cas; + + + } + + + public CAS retrieve() { + CAS ret = casPool.getCas(); + try { + return ret == null ? analysisEngine.newCAS() : ret; + } catch (ResourceInitializationException e) { + throw new RuntimeException(e); + } + } + + + public void release(CAS cas) { + casPool.releaseCas(cas); + } + + + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/AbstractTfidfVectorizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/AbstractTfidfVectorizer.java new file mode 100644 index 000000000..228774142 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/AbstractTfidfVectorizer.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.vectorizer; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.nlp.tokenization.tokenizer.TokenPreProcess; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; +import org.datavec.nlp.tokenization.tokenizerfactory.DefaultTokenizerFactory; +import org.datavec.nlp.tokenization.tokenizerfactory.TokenizerFactory; + +import java.util.HashSet; +import java.util.Set; + +/** + * Tf idf vectorizer + * @author Adam Gibson + */ +public abstract class AbstractTfidfVectorizer extends TextVectorizer { + + @Override + public void doWithTokens(Tokenizer tokenizer) { + Set seen = new HashSet<>(); + while (tokenizer.hasMoreTokens()) { + String token = tokenizer.nextToken(); + if (!stopWords.contains(token)) { + cache.incrementCount(token); + if (!seen.contains(token)) { + cache.incrementDocCount(token); + } + seen.add(token); + } + } + } + + @Override + public TokenizerFactory createTokenizerFactory(Configuration conf) { + String clazz = conf.get(TOKENIZER, DefaultTokenizerFactory.class.getName()); + try { + Class tokenizerFactoryClazz = + (Class) Class.forName(clazz); + TokenizerFactory tf = tokenizerFactoryClazz.newInstance(); + String preproc = conf.get(PREPROCESSOR, null); + if(preproc != null){ + TokenPreProcess tpp = (TokenPreProcess) Class.forName(preproc).newInstance(); + tf.setTokenPreProcessor(tpp); + } + return tf; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public abstract VECTOR_TYPE createVector(Object[] args); + + @Override + public abstract VECTOR_TYPE fitTransform(RecordReader reader); + + @Override + public abstract VECTOR_TYPE transform(Record record); +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/TextVectorizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/TextVectorizer.java new file mode 100644 index 000000000..dcf588ce9 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/TextVectorizer.java @@ -0,0 +1,123 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.vectorizer; + +import lombok.Getter; +import org.nd4j.common.primitives.Counter; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.vector.Vectorizer; +import org.datavec.api.writable.Writable; +import org.datavec.nlp.metadata.DefaultVocabCache; +import org.datavec.nlp.metadata.VocabCache; +import org.datavec.nlp.stopwords.StopWords; +import org.datavec.nlp.tokenization.tokenizer.Tokenizer; +import org.datavec.nlp.tokenization.tokenizerfactory.TokenizerFactory; + +import java.util.Collection; + +/** + * Baseline text vectorizer that includes some common elements + * to text analysis such as the tokenizer factory + * + * @author Adam Gibson + */ +public abstract class TextVectorizer implements Vectorizer { + + protected TokenizerFactory tokenizerFactory; + protected int minWordFrequency = 0; + public final static String MIN_WORD_FREQUENCY = "org.nd4j.nlp.minwordfrequency"; + public final static String STOP_WORDS = "org.nd4j.nlp.stopwords"; + public final static String TOKENIZER = "org.datavec.nlp.tokenizerfactory"; + public static final String PREPROCESSOR = "org.datavec.nlp.preprocessor"; + public final static String VOCAB_CACHE = "org.datavec.nlp.vocabcache"; + protected Collection stopWords; + @Getter + protected VocabCache cache; + + @Override + public void initialize(Configuration conf) { + tokenizerFactory = createTokenizerFactory(conf); + minWordFrequency = conf.getInt(MIN_WORD_FREQUENCY, 5); + if(conf.get(STOP_WORDS) != null) + stopWords = conf.getStringCollection(STOP_WORDS); + if (stopWords == null) + stopWords = StopWords.getStopWords(); + + String clazz = conf.get(VOCAB_CACHE, DefaultVocabCache.class.getName()); + try { + Class tokenizerFactoryClazz = (Class) Class.forName(clazz); + cache = tokenizerFactoryClazz.newInstance(); + cache.initialize(conf); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void fit(RecordReader reader) { + fit(reader, null); + } + + @Override + public void fit(RecordReader reader, RecordCallBack callBack) { + while (reader.hasNext()) { + Record record = reader.nextRecord(); + String s = toString(record.getRecord()); + Tokenizer tokenizer = tokenizerFactory.create(s); + doWithTokens(tokenizer); + if (callBack != null) + callBack.onRecord(record); + cache.incrementNumDocs(1); + } + } + + + protected Counter wordFrequenciesForRecord(Collection record) { + String s = toString(record); + Tokenizer tokenizer = tokenizerFactory.create(s); + Counter ret = new Counter<>(); + while (tokenizer.hasMoreTokens()) + ret.incrementCount(tokenizer.nextToken(), 1.0); + return ret; + } + + + protected String toString(Collection record) { + StringBuilder sb = new StringBuilder(); + for(Writable w : record){ + sb.append(w.toString()); + } + return sb.toString(); + } + + + /** + * Increment counts, add to collection,... + * @param tokenizer + */ + public abstract void doWithTokens(Tokenizer tokenizer); + + /** + * Create tokenizer factory based on the configuration + * @param conf the configuration to use + * @return the tokenizer factory based on the configuration + */ + public abstract TokenizerFactory createTokenizerFactory(Configuration conf); + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/TfidfVectorizer.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/TfidfVectorizer.java new file mode 100644 index 000000000..a730bc739 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/java/org/datavec/nlp/vectorizer/TfidfVectorizer.java @@ -0,0 +1,107 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.vectorizer; + + +import org.datavec.api.conf.Configuration; +import org.nd4j.common.primitives.Counter; +import org.datavec.api.records.Record; +import org.datavec.api.records.metadata.RecordMetaDataURI; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.writable.NDArrayWritable; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * + * Nd4j tfidf vectorizer + * + * @author Adam Gibson + */ +public class TfidfVectorizer extends AbstractTfidfVectorizer { + /** + * Default: True.
+ * If true: use idf(d, t) = log [ (1 + n) / (1 + df(d, t)) ] + 1
+ * If false: use idf(t) = log [ n / df(t) ] + 1
+ */ + public static final String SMOOTH_IDF = "org.datavec.nlp.TfidfVectorizer.smooth_idf"; + + protected boolean smooth_idf; + + @Override + public INDArray createVector(Object[] args) { + Counter docFrequencies = (Counter) args[0]; + double[] vector = new double[cache.vocabWords().size()]; + for (int i = 0; i < cache.vocabWords().size(); i++) { + String word = cache.wordAt(i); + double freq = docFrequencies.getCount(word); + vector[i] = cache.tfidf(word, freq, smooth_idf); + } + return Nd4j.create(vector); + } + + @Override + public INDArray fitTransform(RecordReader reader) { + return fitTransform(reader, null); + } + + @Override + public INDArray fitTransform(final RecordReader reader, RecordCallBack callBack) { + final List records = new ArrayList<>(); + fit(reader, new RecordCallBack() { + @Override + public void onRecord(Record record) { + records.add(record); + } + }); + + if (records.isEmpty()) + throw new IllegalStateException("No records found!"); + INDArray ret = Nd4j.create(records.size(), cache.vocabWords().size()); + int i = 0; + for (Record record : records) { + INDArray transformed = transform(record); + org.datavec.api.records.impl.Record transformedRecord = new org.datavec.api.records.impl.Record( + Arrays.asList(new NDArrayWritable(transformed), + record.getRecord().get(record.getRecord().size() - 1)), + new RecordMetaDataURI(record.getMetaData().getURI(), reader.getClass())); + ret.putRow(i++, transformed); + if (callBack != null) { + callBack.onRecord(transformedRecord); + } + } + + return ret; + } + + @Override + public INDArray transform(Record record) { + Counter wordFrequencies = wordFrequenciesForRecord(record.getRecord()); + return createVector(new Object[] {wordFrequencies}); + } + + + @Override + public void initialize(Configuration conf){ + super.initialize(conf); + this.smooth_idf = conf.getBoolean(SMOOTH_IDF, true); + } +} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/stopwords.txt b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/resources/stopwords similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/stopwords.txt rename to cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/main/resources/stopwords diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..533c6b3ad --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.nlp; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.nlp"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/reader/TfidfRecordReaderTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/reader/TfidfRecordReaderTest.java new file mode 100644 index 000000000..b3dba2b96 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/reader/TfidfRecordReaderTest.java @@ -0,0 +1,127 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.reader; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.CollectionInputSplit; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; +import org.datavec.nlp.vectorizer.TfidfVectorizer; + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.net.URI; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Adam Gibson + */ +public class TfidfRecordReaderTest { + + @TempDir + public File testDir; + + @Test + public void testReader() throws Exception { + TfidfVectorizer vectorizer = new TfidfVectorizer(); + Configuration conf = new Configuration(); + conf.setInt(TfidfVectorizer.MIN_WORD_FREQUENCY, 1); + conf.setBoolean(RecordReader.APPEND_LABEL, true); + vectorizer.initialize(conf); + TfidfRecordReader reader = new TfidfRecordReader(); + File f = testDir; + new ClassPathResource("datavec-data-nlp/labeled/").copyDirectory(f); + List u = new ArrayList<>(); + for(File f2 : f.listFiles()){ + if(f2.isDirectory()){ + for(File f3 : f2.listFiles()){ + u.add(f3.toURI()); + } + } else { + u.add(f2.toURI()); + } + } + Collections.sort(u); + CollectionInputSplit c = new CollectionInputSplit(u); + reader.initialize(conf, c); + int count = 0; + int[] labelAssertions = new int[3]; + while (reader.hasNext()) { + Collection record = reader.next(); + Iterator recordIter = record.iterator(); + NDArrayWritable writable = (NDArrayWritable) recordIter.next(); + labelAssertions[count] = recordIter.next().toInt(); + count++; + } + + assertArrayEquals(new int[] {0, 1, 2}, labelAssertions); + assertEquals(3, reader.getLabels().size()); + assertEquals(3, count); + } + + @Test + public void testRecordMetaData() throws Exception { + TfidfVectorizer vectorizer = new TfidfVectorizer(); + Configuration conf = new Configuration(); + conf.setInt(TfidfVectorizer.MIN_WORD_FREQUENCY, 1); + conf.setBoolean(RecordReader.APPEND_LABEL, true); + vectorizer.initialize(conf); + TfidfRecordReader reader = new TfidfRecordReader(); + File f = testDir; + new ClassPathResource("datavec-data-nlp/labeled/").copyDirectory(f); + reader.initialize(conf, new FileSplit(f)); + + while (reader.hasNext()) { + Record record = reader.nextRecord(); + assertNotNull(record.getMetaData().getURI()); + assertEquals(record.getMetaData().getReaderClass(), TfidfRecordReader.class); + } + } + + + @Test + public void testReadRecordFromMetaData() throws Exception { + TfidfVectorizer vectorizer = new TfidfVectorizer(); + Configuration conf = new Configuration(); + conf.setInt(TfidfVectorizer.MIN_WORD_FREQUENCY, 1); + conf.setBoolean(RecordReader.APPEND_LABEL, true); + vectorizer.initialize(conf); + TfidfRecordReader reader = new TfidfRecordReader(); + File f = testDir; + new ClassPathResource("datavec-data-nlp/labeled/").copyDirectory(f); + reader.initialize(conf, new FileSplit(f)); + + Record record = reader.nextRecord(); + + Record reread = reader.loadFromMetaData(record.getMetaData()); + + assertEquals(record.getRecord().size(), 2); + assertEquals(reread.getRecord().size(), 2); + assertEquals(record.getRecord().get(0), reread.getRecord().get(0)); + assertEquals(record.getRecord().get(1), reread.getRecord().get(1)); + assertEquals(record.getMetaData(), reread.getMetaData()); + } +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TestGazeteerTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TestGazeteerTransform.java new file mode 100644 index 000000000..c63ff14c7 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TestGazeteerTransform.java @@ -0,0 +1,92 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.transforms; + +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.SequenceSchema; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.local.transforms.LocalTransformExecutor; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestGazeteerTransform { + + @Test + public void testGazeteerTransform(){ + + String[] corpus = { + "hello I like apple".toLowerCase(), + "cherry date eggplant potato".toLowerCase() + }; + + //Gazeteer transform: basically 0/1 if word is present. Assumes already tokenized input + List words = Arrays.asList("apple", "banana", "cherry", "date", "eggplant"); + + GazeteerTransform t = new GazeteerTransform("words", "out", words); + + SequenceSchema schema = (SequenceSchema) new SequenceSchema.Builder() + .addColumnString("words").build(); + + TransformProcess tp = new TransformProcess.Builder(schema) + .transform(t) + .build(); + + List>> input = new ArrayList<>(); + for(String s : corpus){ + String[] split = s.split(" "); + List> seq = new ArrayList<>(); + for(String s2 : split){ + seq.add(Collections.singletonList(new Text(s2))); + } + input.add(seq); + } + + List>> execute = LocalTransformExecutor.executeSequenceToSequence(input, tp); + + INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); + INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); + + INDArray exp0 = Nd4j.create(new float[]{1, 0, 0, 0, 0}); + INDArray exp1 = Nd4j.create(new float[]{0, 0, 1, 1, 1}); + + assertEquals(exp0, arr0); + assertEquals(exp1, arr1); + + + String json = tp.toJson(); + TransformProcess tp2 = TransformProcess.fromJson(json); + assertEquals(tp, tp2); + + List>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, tp); + INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get(); + INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get(); + + assertEquals(exp0, arr0a); + assertEquals(exp1, arr1a); + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TestMultiNLPTransform.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TestMultiNLPTransform.java new file mode 100644 index 000000000..b0642f2a9 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TestMultiNLPTransform.java @@ -0,0 +1,92 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.transforms; + +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.SequenceSchema; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.local.transforms.LocalTransformExecutor; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestMultiNLPTransform { + + @Test + public void test(){ + + List words = Arrays.asList("apple", "banana", "cherry", "date", "eggplant"); + GazeteerTransform t1 = new GazeteerTransform("words", "out", words); + GazeteerTransform t2 = new GazeteerTransform("out", "out", words); + + + MultiNlpTransform multi = new MultiNlpTransform("text", new BagOfWordsTransform[]{t1, t2}, "out"); + + String[] corpus = { + "hello I like apple".toLowerCase(), + "date eggplant potato".toLowerCase() + }; + + List>> input = new ArrayList<>(); + for(String s : corpus){ + String[] split = s.split(" "); + List> seq = new ArrayList<>(); + for(String s2 : split){ + seq.add(Collections.singletonList(new Text(s2))); + } + input.add(seq); + } + + SequenceSchema schema = (SequenceSchema) new SequenceSchema.Builder() + .addColumnString("text").build(); + + TransformProcess tp = new TransformProcess.Builder(schema) + .transform(multi) + .build(); + + List>> execute = LocalTransformExecutor.executeSequenceToSequence(input, tp); + + INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); + INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); + + INDArray exp0 = Nd4j.create(new float[]{1, 0, 0, 0, 0, 1, 0, 0, 0, 0}); + INDArray exp1 = Nd4j.create(new float[]{0, 0, 0, 1, 1, 0, 0, 0, 1, 1}); + + assertEquals(exp0, arr0); + assertEquals(exp1, arr1); + + + String json = tp.toJson(); + TransformProcess tp2 = TransformProcess.fromJson(json); + assertEquals(tp, tp2); + + List>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, tp); + INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get(); + INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get(); + + assertEquals(exp0, arr0a); + assertEquals(exp1, arr1a); + + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TokenizerBagOfWordsTermSequenceIndexTransformTest.java b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TokenizerBagOfWordsTermSequenceIndexTransformTest.java new file mode 100644 index 000000000..dded0cc06 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/java/org/datavec/nlp/transforms/TokenizerBagOfWordsTermSequenceIndexTransformTest.java @@ -0,0 +1,410 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.nlp.transforms; + +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.reader.impl.collection.CollectionRecordReader; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.SequenceSchema; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.local.transforms.LocalTransformExecutor; +import org.datavec.nlp.metadata.VocabCache; +import org.datavec.nlp.tokenization.tokenizer.preprocessor.LowerCasePreProcessor; +import org.datavec.nlp.tokenization.tokenizerfactory.DefaultTokenizerFactory; +import org.datavec.nlp.vectorizer.TfidfVectorizer; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Triple; + +import java.util.*; + +import static org.datavec.nlp.vectorizer.TextVectorizer.*; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TokenizerBagOfWordsTermSequenceIndexTransformTest { + + @Test + public void testSequenceExecution() { + //credit: https://stackoverflow.com/questions/23792781/tf-idf-feature-weights-using-sklearn-feature-extraction-text-tfidfvectorizer + String[] corpus = { + "This is very strange".toLowerCase(), + "This is very nice".toLowerCase() + }; + //{'is': 1.0, 'nice': 1.4054651081081644, 'strange': 1.4054651081081644, 'this': 1.0, 'very': 1.0} + + /* + ## Reproduce with: + from sklearn.feature_extraction.text import TfidfVectorizer + corpus = ["This is very strange", "This is very nice"] + + ## SMOOTH = FALSE case: + vectorizer = TfidfVectorizer(min_df=0, norm=None, smooth_idf=False) + X = vectorizer.fit_transform(corpus) + idf = vectorizer.idf_ + print(dict(zip(vectorizer.get_feature_names(), idf))) + + newText = ["This is very strange", "This is very nice"] + out = vectorizer.transform(newText) + print(out) + + {'is': 1.0, 'nice': 1.6931471805599454, 'strange': 1.6931471805599454, 'this': 1.0, 'very': 1.0} + (0, 4) 1.0 + (0, 3) 1.0 + (0, 2) 1.6931471805599454 + (0, 0) 1.0 + (1, 4) 1.0 + (1, 3) 1.0 + (1, 1) 1.6931471805599454 + (1, 0) 1.0 + + ## SMOOTH + TRUE case: + {'is': 1.0, 'nice': 1.4054651081081644, 'strange': 1.4054651081081644, 'this': 1.0, 'very': 1.0} + (0, 4) 1.0 + (0, 3) 1.0 + (0, 2) 1.4054651081081644 + (0, 0) 1.0 + (1, 4) 1.0 + (1, 3) 1.0 + (1, 1) 1.4054651081081644 + (1, 0) 1.0 + */ + + List>> input = new ArrayList<>(); + input.add(Arrays.asList(Arrays.asList(new Text(corpus[0])),Arrays.asList(new Text(corpus[1])))); + + // First: Check TfidfVectorizer vs. scikit: + + Map idfMapNoSmooth = new HashMap<>(); + idfMapNoSmooth.put("is",1.0); + idfMapNoSmooth.put("nice",1.6931471805599454); + idfMapNoSmooth.put("strange",1.6931471805599454); + idfMapNoSmooth.put("this",1.0); + idfMapNoSmooth.put("very",1.0); + + Map idfMapSmooth = new HashMap<>(); + idfMapSmooth.put("is",1.0); + idfMapSmooth.put("nice",1.4054651081081644); + idfMapSmooth.put("strange",1.4054651081081644); + idfMapSmooth.put("this",1.0); + idfMapSmooth.put("very",1.0); + + + + TfidfVectorizer tfidfVectorizer = new TfidfVectorizer(); + Configuration configuration = new Configuration(); + configuration.set(TOKENIZER, DefaultTokenizerFactory.class.getName()); + configuration.set(MIN_WORD_FREQUENCY,"1"); + configuration.set(STOP_WORDS,""); + configuration.set(TfidfVectorizer.SMOOTH_IDF, "false"); + + tfidfVectorizer.initialize(configuration); + + CollectionRecordReader collectionRecordReader = new CollectionRecordReader(input.get(0)); + INDArray array = tfidfVectorizer.fitTransform(collectionRecordReader); + + INDArray expNoSmooth = Nd4j.create(DataType.FLOAT, 2, 5); + VocabCache vc = tfidfVectorizer.getCache(); + expNoSmooth.putScalar(0, vc.wordIndex("very"), 1.0); + expNoSmooth.putScalar(0, vc.wordIndex("this"), 1.0); + expNoSmooth.putScalar(0, vc.wordIndex("strange"), 1.6931471805599454); + expNoSmooth.putScalar(0, vc.wordIndex("is"), 1.0); + + expNoSmooth.putScalar(1, vc.wordIndex("very"), 1.0); + expNoSmooth.putScalar(1, vc.wordIndex("this"), 1.0); + expNoSmooth.putScalar(1, vc.wordIndex("nice"), 1.6931471805599454); + expNoSmooth.putScalar(1, vc.wordIndex("is"), 1.0); + + assertEquals(expNoSmooth, array); + + + //------------------------------------------------------------ + //Smooth version: + tfidfVectorizer = new TfidfVectorizer(); + configuration = new Configuration(); + configuration.set(TOKENIZER, DefaultTokenizerFactory.class.getName()); + configuration.set(MIN_WORD_FREQUENCY,"1"); + configuration.set(STOP_WORDS,""); + configuration.set(TfidfVectorizer.SMOOTH_IDF, "true"); + + tfidfVectorizer.initialize(configuration); + + collectionRecordReader.reset(); + array = tfidfVectorizer.fitTransform(collectionRecordReader); + + INDArray expSmooth = Nd4j.create(DataType.FLOAT, 2, 5); + expSmooth.putScalar(0, vc.wordIndex("very"), 1.0); + expSmooth.putScalar(0, vc.wordIndex("this"), 1.0); + expSmooth.putScalar(0, vc.wordIndex("strange"), 1.4054651081081644); + expSmooth.putScalar(0, vc.wordIndex("is"), 1.0); + + expSmooth.putScalar(1, vc.wordIndex("very"), 1.0); + expSmooth.putScalar(1, vc.wordIndex("this"), 1.0); + expSmooth.putScalar(1, vc.wordIndex("nice"), 1.4054651081081644); + expSmooth.putScalar(1, vc.wordIndex("is"), 1.0); + + assertEquals(expSmooth, array); + + + ////////////////////////////////////////////////////////// + + //Second: Check transform vs scikit/TfidfVectorizer + + List vocab = new ArrayList<>(5); //Arrays.asList("is","nice","strange","this","very"); + for( int i=0; i<5; i++ ){ + vocab.add(vc.wordAt(i)); + } + + String inputColumnName = "input"; + String outputColumnName = "output"; + Map wordIndexMap = new HashMap<>(); + for(int i = 0; i < vocab.size(); i++) { + wordIndexMap.put(vocab.get(i),i); + } + + TokenizerBagOfWordsTermSequenceIndexTransform tokenizerBagOfWordsTermSequenceIndexTransform = new TokenizerBagOfWordsTermSequenceIndexTransform( + inputColumnName, + outputColumnName, + wordIndexMap, + idfMapNoSmooth, + false, + null, null); + + SequenceSchema.Builder sequenceSchemaBuilder = new SequenceSchema.Builder(); + sequenceSchemaBuilder.addColumnString("input"); + SequenceSchema schema = sequenceSchemaBuilder.build(); + assertEquals("input",schema.getName(0)); + + TransformProcess transformProcess = new TransformProcess.Builder(schema) + .transform(tokenizerBagOfWordsTermSequenceIndexTransform) + .build(); + + List>> execute = LocalTransformExecutor.executeSequenceToSequence(input, transformProcess); + + + + //System.out.println(execute); + INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); + INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); + + assertEquals(expNoSmooth.getRow(0, true), arr0); + assertEquals(expNoSmooth.getRow(1, true), arr1); + + + //-------------------------------- + //Check smooth: + + tokenizerBagOfWordsTermSequenceIndexTransform = new TokenizerBagOfWordsTermSequenceIndexTransform( + inputColumnName, + outputColumnName, + wordIndexMap, + idfMapSmooth, + false, + null, null); + + schema = (SequenceSchema) new SequenceSchema.Builder().addColumnString("input").build(); + + transformProcess = new TransformProcess.Builder(schema) + .transform(tokenizerBagOfWordsTermSequenceIndexTransform) + .build(); + + execute = LocalTransformExecutor.executeSequenceToSequence(input, transformProcess); + + arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); + arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); + + assertEquals(expSmooth.getRow(0, true), arr0); + assertEquals(expSmooth.getRow(1, true), arr1); + + + + //Test JSON serialization: + + String json = transformProcess.toJson(); + TransformProcess fromJson = TransformProcess.fromJson(json); + assertEquals(transformProcess, fromJson); + List>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, fromJson); + + INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get(); + INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get(); + + assertEquals(expSmooth.getRow(0, true), arr0a); + assertEquals(expSmooth.getRow(1, true), arr1a); + } + + @Test + public void additionalTest(){ + /* + ## To reproduce: + from sklearn.feature_extraction.text import TfidfVectorizer + corpus = [ + 'This is the first document', + 'This document is the second document', + 'And this is the third one', + 'Is this the first document', + ] + vectorizer = TfidfVectorizer(min_df=0, norm=None, smooth_idf=False) + X = vectorizer.fit_transform(corpus) + print(vectorizer.get_feature_names()) + + out = vectorizer.transform(corpus) + print(out) + + ['and', 'document', 'first', 'is', 'one', 'second', 'the', 'third', 'this'] + (0, 8) 1.0 + (0, 6) 1.0 + (0, 3) 1.0 + (0, 2) 1.6931471805599454 + (0, 1) 1.2876820724517808 + (1, 8) 1.0 + (1, 6) 1.0 + (1, 5) 2.386294361119891 + (1, 3) 1.0 + (1, 1) 2.5753641449035616 + (2, 8) 1.0 + (2, 7) 2.386294361119891 + (2, 6) 1.0 + (2, 4) 2.386294361119891 + (2, 3) 1.0 + (2, 0) 2.386294361119891 + (3, 8) 1.0 + (3, 6) 1.0 + (3, 3) 1.0 + (3, 2) 1.6931471805599454 + (3, 1) 1.2876820724517808 + {'and': 2.386294361119891, 'document': 1.2876820724517808, 'first': 1.6931471805599454, 'is': 1.0, 'one': 2.386294361119891, 'second': 2.386294361119891, 'the': 1.0, 'third': 2.386294361119891, 'this': 1.0} + */ + + String[] corpus = { + "This is the first document", + "This document is the second document", + "And this is the third one", + "Is this the first document"}; + + TfidfVectorizer tfidfVectorizer = new TfidfVectorizer(); + Configuration configuration = new Configuration(); + configuration.set(TOKENIZER, DefaultTokenizerFactory.class.getName()); + configuration.set(MIN_WORD_FREQUENCY,"1"); + configuration.set(STOP_WORDS,""); + configuration.set(TfidfVectorizer.SMOOTH_IDF, "false"); + configuration.set(PREPROCESSOR, LowerCasePreProcessor.class.getName()); + + tfidfVectorizer.initialize(configuration); + + List>> input = new ArrayList<>(); + //input.add(Arrays.asList(Arrays.asList(new Text(corpus[0])),Arrays.asList(new Text(corpus[1])))); + List> seq = new ArrayList<>(); + for(String s : corpus){ + seq.add(Collections.singletonList(new Text(s))); + } + input.add(seq); + + CollectionRecordReader crr = new CollectionRecordReader(seq); + INDArray arr = tfidfVectorizer.fitTransform(crr); + + //System.out.println(arr); + assertArrayEquals(new long[]{4, 9}, arr.shape()); + + List pyVocab = Arrays.asList("and", "document", "first", "is", "one", "second", "the", "third", "this"); + List> l = new ArrayList<>(); + l.add(new Triple<>(0, 8, 1.0)); + l.add(new Triple<>(0, 6, 1.0)); + l.add(new Triple<>(0, 3, 1.0)); + l.add(new Triple<>(0, 2, 1.6931471805599454)); + l.add(new Triple<>(0, 1, 1.2876820724517808)); + l.add(new Triple<>(1, 8, 1.0)); + l.add(new Triple<>(1, 6, 1.0)); + l.add(new Triple<>(1, 5, 2.386294361119891)); + l.add(new Triple<>(1, 3, 1.0)); + l.add(new Triple<>(1, 1, 2.5753641449035616)); + l.add(new Triple<>(2, 8, 1.0)); + l.add(new Triple<>(2, 7, 2.386294361119891)); + l.add(new Triple<>(2, 6, 1.0)); + l.add(new Triple<>(2, 4, 2.386294361119891)); + l.add(new Triple<>(2, 3, 1.0)); + l.add(new Triple<>(2, 0, 2.386294361119891)); + l.add(new Triple<>(3, 8, 1.0)); + l.add(new Triple<>(3, 6, 1.0)); + l.add(new Triple<>(3, 3, 1.0)); + l.add(new Triple<>(3, 2, 1.6931471805599454)); + l.add(new Triple<>(3, 1, 1.2876820724517808)); + + INDArray exp = Nd4j.create(DataType.FLOAT, 4, 9); + for(Triple t : l){ + //Work out work index, accounting for different vocab/word orders: + int wIdx = tfidfVectorizer.getCache().wordIndex(pyVocab.get(t.getSecond())); + exp.putScalar(t.getFirst(), wIdx, t.getThird()); + } + + assertEquals(exp, arr); + + + Map idfWeights = new HashMap<>(); + idfWeights.put("and", 2.386294361119891); + idfWeights.put("document", 1.2876820724517808); + idfWeights.put("first", 1.6931471805599454); + idfWeights.put("is", 1.0); + idfWeights.put("one", 2.386294361119891); + idfWeights.put("second", 2.386294361119891); + idfWeights.put("the", 1.0); + idfWeights.put("third", 2.386294361119891); + idfWeights.put("this", 1.0); + + + List vocab = new ArrayList<>(9); //Arrays.asList("is","nice","strange","this","very"); + for( int i=0; i<9; i++ ){ + vocab.add(tfidfVectorizer.getCache().wordAt(i)); + } + + String inputColumnName = "input"; + String outputColumnName = "output"; + Map wordIndexMap = new HashMap<>(); + for(int i = 0; i < vocab.size(); i++) { + wordIndexMap.put(vocab.get(i),i); + } + + TokenizerBagOfWordsTermSequenceIndexTransform transform = new TokenizerBagOfWordsTermSequenceIndexTransform( + inputColumnName, + outputColumnName, + wordIndexMap, + idfWeights, + false, + null, LowerCasePreProcessor.class.getName()); + + SequenceSchema.Builder sequenceSchemaBuilder = new SequenceSchema.Builder(); + sequenceSchemaBuilder.addColumnString("input"); + SequenceSchema schema = sequenceSchemaBuilder.build(); + assertEquals("input",schema.getName(0)); + + TransformProcess transformProcess = new TransformProcess.Builder(schema) + .transform(transform) + .build(); + + List>> execute = LocalTransformExecutor.executeSequenceToSequence(input, transformProcess); + + INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get(); + INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get(); + + assertEquals(exp.getRow(0, true), arr0); + assertEquals(exp.getRow(1, true), arr1); + } + +} diff --git a/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/resources/logback.xml b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/resources/logback.xml new file mode 100644 index 000000000..2087d615c --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/cavis-datavec-data-nlp/src/test/resources/logback.xml @@ -0,0 +1,49 @@ + + + + + + logs/application.log + + %date - [%level] - from %logger in %thread + %n%message%n%xException%n + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/build.gradle b/cavis-datavec/cavis-datavec-data/datavec-excel/build.gradle new file mode 100644 index 000000000..05af1969e --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/datavec-excel/build.gradle @@ -0,0 +1,26 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +dependencies { + implementation projects.cavisDatavec.cavisDatavecApi + implementation "org.apache.poi:poi:3.17" + implementation "org.apache.poi:poi-ooxml:3.17" +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-data/datavec-excel/pom.xml b/cavis-datavec/cavis-datavec-data/datavec-excel/pom.xml new file mode 100644 index 000000000..dfde1d302 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/datavec-excel/pom.xml @@ -0,0 +1,59 @@ + + + + + + 4.0.0 + + + net.brutex.ai + datavec-parent + 1.0.0-SNAPSHOT + + + datavec-excel + + datavec-excel + + + + net.brutex.ai + datavec-api + ${project.version} + + + + org.apache.poi + poi + ${poi.version} + + + + org.apache.poi + poi-ooxml + ${poi.version} + + + + + diff --git a/datavec/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java b/cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java similarity index 99% rename from datavec/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java rename to cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java index 7f6be8d15..6925bc47d 100644 --- a/datavec/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java +++ b/cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordReader.java @@ -181,7 +181,7 @@ public class ExcelRecordReader extends FileRecordReader { List ret = new ArrayList<>(currRow.getLastCellNum()); for(Cell cell: currRow) { String cellValue = dataFormatter.formatCellValue(cell); - switch(cell.getCellType()) { + switch(cell.getCellTypeEnum()) { case BLANK: ret.add(new Text("")); break; case STRING: ret.add(new Text("")); break; case BOOLEAN: ret.add(new BooleanWritable(Boolean.valueOf(cellValue))); break; diff --git a/datavec/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java b/cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java similarity index 100% rename from datavec/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java rename to cavis-datavec/cavis-datavec-data/datavec-excel/src/main/java/org/datavec/poi/excel/ExcelRecordWriter.java diff --git a/datavec/datavec-excel/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java similarity index 100% rename from datavec/datavec-excel/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java rename to cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/AssertTestsExtendBaseClass.java diff --git a/datavec/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java b/cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java similarity index 78% rename from datavec/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java rename to cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java index c7a3fe12c..201d257f9 100644 --- a/datavec/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java +++ b/cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordReaderTest.java @@ -17,39 +17,37 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.poi.excel; import org.datavec.api.records.reader.RecordReader; import org.datavec.api.split.FileSplit; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.io.ClassPathResource; + import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Excel Record Reader Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -class ExcelRecordReaderTest { +public class ExcelRecordReaderTest { @Test - @DisplayName("Test Simple") - void testSimple() throws Exception { + public void testSimple() throws Exception { RecordReader excel = new ExcelRecordReader(); excel.initialize(new FileSplit(new ClassPathResource("datavec-excel/testsheet.xlsx").getFile())); assertTrue(excel.hasNext()); List next = excel.next(); - assertEquals(3, next.size()); + assertEquals(3,next.size()); + RecordReader headerReader = new ExcelRecordReader(1); headerReader.initialize(new FileSplit(new ClassPathResource("datavec-excel/testsheetheader.xlsx").getFile())); assertTrue(excel.hasNext()); List next2 = excel.next(); - assertEquals(3, next2.size()); + assertEquals(3,next2.size()); + + } + } diff --git a/datavec/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java b/cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java similarity index 75% rename from datavec/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java rename to cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java index 08de8b153..e977aa5d4 100644 --- a/datavec/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java +++ b/cavis-datavec/cavis-datavec-data/datavec-excel/src/test/java/org/datavec/poi/excel/ExcelRecordWriterTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.datavec.poi.excel; import lombok.val; @@ -26,48 +27,44 @@ import org.datavec.api.transform.schema.Schema; import org.datavec.api.writable.IntWritable; import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.primitives.Triple; + import java.io.File; import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Excel Record Writer Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -class ExcelRecordWriterTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ExcelRecordWriterTest { @TempDir - public Path testDir; + public File testDir; @Test - @DisplayName("Test Writer") - void testWriter() throws Exception { + public void testWriter() throws Exception { ExcelRecordWriter excelRecordWriter = new ExcelRecordWriter(); val records = records(); - File tmpDir = testDir.toFile(); - File outputFile = new File(tmpDir, "testexcel.xlsx"); + File tmpDir = testDir; + File outputFile = new File(tmpDir,"testexcel.xlsx"); outputFile.deleteOnExit(); FileSplit fileSplit = new FileSplit(outputFile); - excelRecordWriter.initialize(fileSplit, new NumberOfRecordsPartitioner()); + excelRecordWriter.initialize(fileSplit,new NumberOfRecordsPartitioner()); excelRecordWriter.writeBatch(records.getRight()); excelRecordWriter.close(); File parentFile = outputFile.getParentFile(); - assertEquals(1, parentFile.list().length); + assertEquals(1,parentFile.list().length); + ExcelRecordReader excelRecordReader = new ExcelRecordReader(); excelRecordReader.initialize(fileSplit); List> next = excelRecordReader.next(10); - assertEquals(10, next.size()); + assertEquals(10,next.size()); + } - private Triple>> records() { + private Triple>> records() { List> list = new ArrayList<>(); StringBuilder sb = new StringBuilder(); int numColumns = 3; @@ -84,10 +81,13 @@ class ExcelRecordWriterTest { } list.add(temp); } + + Schema.Builder schemaBuilder = new Schema.Builder(); - for (int i = 0; i < numColumns; i++) { + for(int i = 0; i < numColumns; i++) { schemaBuilder.addColumnInteger(String.valueOf(i)); } - return Triple.of(sb.toString(), schemaBuilder.build(), list); + + return Triple.of(sb.toString(),schemaBuilder.build(),list); } } diff --git a/cavis-datavec/cavis-datavec-data/datavec-jdbc/pom.xml b/cavis-datavec/cavis-datavec-data/datavec-jdbc/pom.xml new file mode 100644 index 000000000..c6a4ad65a --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/datavec-jdbc/pom.xml @@ -0,0 +1,66 @@ + + + + + + 4.0.0 + + + net.brutex.ai + datavec-parent + 1.0.0-SNAPSHOT + + + datavec-jdbc + + + 1.7 + 2.4.12 + 10.13.1.1 + + + + + net.brutex.ai + datavec-api + ${project.version} + + + commons-dbutils + commons-dbutils + ${dbutils.version} + + + com.zaxxer + HikariCP-java7 + ${hikaricp.version} + + + org.apache.derby + derby + ${derby.version} + test + + + + diff --git a/datavec/datavec-jdbc/src/main/java/org/datavec/jdbc/records/metadata/RecordMetaDataJdbc.java b/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/main/java/org/datavec/jdbc/records/metadata/RecordMetaDataJdbc.java similarity index 100% rename from datavec/datavec-jdbc/src/main/java/org/datavec/jdbc/records/metadata/RecordMetaDataJdbc.java rename to cavis-datavec/cavis-datavec-data/datavec-jdbc/src/main/java/org/datavec/jdbc/records/metadata/RecordMetaDataJdbc.java diff --git a/datavec/datavec-jdbc/src/main/java/org/datavec/jdbc/records/reader/impl/jdbc/JDBCRecordReader.java b/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/main/java/org/datavec/jdbc/records/reader/impl/jdbc/JDBCRecordReader.java similarity index 100% rename from datavec/datavec-jdbc/src/main/java/org/datavec/jdbc/records/reader/impl/jdbc/JDBCRecordReader.java rename to cavis-datavec/cavis-datavec-data/datavec-jdbc/src/main/java/org/datavec/jdbc/records/reader/impl/jdbc/JDBCRecordReader.java diff --git a/datavec/datavec-jdbc/src/main/java/org/datavec/jdbc/util/JdbcWritableConverter.java b/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/main/java/org/datavec/jdbc/util/JdbcWritableConverter.java similarity index 100% rename from datavec/datavec-jdbc/src/main/java/org/datavec/jdbc/util/JdbcWritableConverter.java rename to cavis-datavec/cavis-datavec-data/datavec-jdbc/src/main/java/org/datavec/jdbc/util/JdbcWritableConverter.java diff --git a/datavec/datavec-jdbc/src/main/java/org/datavec/jdbc/util/ResettableResultSetIterator.java b/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/main/java/org/datavec/jdbc/util/ResettableResultSetIterator.java similarity index 100% rename from datavec/datavec-jdbc/src/main/java/org/datavec/jdbc/util/ResettableResultSetIterator.java rename to cavis-datavec/cavis-datavec-data/datavec-jdbc/src/main/java/org/datavec/jdbc/util/ResettableResultSetIterator.java diff --git a/datavec/datavec-jdbc/src/test/java/org/datavec/api/records/reader/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/test/java/org/datavec/api/records/reader/AssertTestsExtendBaseClass.java similarity index 100% rename from datavec/datavec-jdbc/src/test/java/org/datavec/api/records/reader/AssertTestsExtendBaseClass.java rename to cavis-datavec/cavis-datavec-data/datavec-jdbc/src/test/java/org/datavec/api/records/reader/AssertTestsExtendBaseClass.java diff --git a/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/JDBCRecordReaderTest.java b/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/JDBCRecordReaderTest.java new file mode 100644 index 000000000..bf5cab5b7 --- /dev/null +++ b/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/JDBCRecordReaderTest.java @@ -0,0 +1,286 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.api.records.reader.impl; + +import java.io.File; +import java.net.URI; +import java.sql.Connection; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.commons.dbutils.DbUtils; +import org.apache.derby.jdbc.EmbeddedDataSource; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.listener.RecordListener; +import org.datavec.api.records.listener.impl.LogRecordListener; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.jdbc.records.metadata.RecordMetaDataJdbc; +import org.datavec.api.records.metadata.RecordMetaDataLine; +import org.datavec.jdbc.records.reader.impl.jdbc.JDBCRecordReader; +import org.datavec.api.writable.BooleanWritable; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.FloatWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.LongWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + + +import static org.junit.jupiter.api.Assertions.*; + +public class JDBCRecordReaderTest { + + @TempDir + public File testDir; + + Connection conn; + EmbeddedDataSource dataSource; + + private final String dbName = "datavecTests"; + private final String driverClassName = "org.apache.derby.jdbc.EmbeddedDriver"; + + @BeforeEach + public void setUp() throws Exception { + File f = testDir; + System.setProperty("derby.system.home", f.getAbsolutePath()); + + dataSource = new EmbeddedDataSource(); + dataSource.setDatabaseName(dbName); + dataSource.setCreateDatabase("create"); + conn = dataSource.getConnection(); + + TestDb.dropTables(conn); + TestDb.buildCoffeeTable(conn); + } + + @AfterEach + public void tearDown() throws Exception { + DbUtils.closeQuietly(conn); + } + + @Test + public void testSimpleIter() throws Exception { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + List> records = new ArrayList<>(); + while (reader.hasNext()) { + List values = reader.next(); + records.add(values); + } + + assertFalse(records.isEmpty()); + + List first = records.get(0); + assertEquals(new Text("Bolivian Dark"), first.get(0)); + assertEquals(new Text("14-001"), first.get(1)); + assertEquals(new DoubleWritable(8.95), first.get(2)); + } + } + + @Test + public void testSimpleWithListener() throws Exception { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + RecordListener recordListener = new LogRecordListener(); + reader.setListeners(recordListener); + reader.next(); + + assertTrue(recordListener.invoked()); + } + } + + @Test + public void testReset() throws Exception { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + List> records = new ArrayList<>(); + records.add(reader.next()); + reader.reset(); + records.add(reader.next()); + + assertEquals(2, records.size()); + assertEquals(new Text("Bolivian Dark"), records.get(0).get(0)); + assertEquals(new Text("Bolivian Dark"), records.get(1).get(0)); + } + } + + @Test + public void testLackingDataSourceShouldFail() throws Exception { + assertThrows(IllegalStateException.class, () -> { + try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) { + reader.initialize(null); + } + }); + } + + @Test + public void testConfigurationDataSourceInitialization() throws Exception { + try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) { + Configuration conf = new Configuration(); + conf.set(JDBCRecordReader.JDBC_URL, "jdbc:derby:" + dbName + ";create=true"); + conf.set(JDBCRecordReader.JDBC_DRIVER_CLASS_NAME, driverClassName); + reader.initialize(conf, null); + assertTrue(reader.hasNext()); + } + } + + @Test + public void testInitConfigurationMissingParametersShouldFail() throws Exception { + assertThrows(IllegalArgumentException.class, () -> { + try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) { + Configuration conf = new Configuration(); + conf.set(JDBCRecordReader.JDBC_URL, "should fail anyway"); + reader.initialize(conf, null); + } + }); + } + + @Test + public void testRecordDataInputStreamShouldFail() throws Exception { + assertThrows(UnsupportedOperationException.class, () -> { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + reader.record(null, null); + } + }); + } + + @Test + public void testLoadFromMetaData() throws Exception { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + RecordMetaDataJdbc rmd = new RecordMetaDataJdbc(new URI(conn.getMetaData().getURL()), + "SELECT * FROM Coffee WHERE ProdNum = ?", Collections.singletonList("14-001"), reader.getClass()); + + Record res = reader.loadFromMetaData(rmd); + assertNotNull(res); + assertEquals(new Text("Bolivian Dark"), res.getRecord().get(0)); + assertEquals(new Text("14-001"), res.getRecord().get(1)); + assertEquals(new DoubleWritable(8.95), res.getRecord().get(2)); + } + } + + @Test + public void testNextRecord() throws Exception { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + Record r = reader.nextRecord(); + List fields = r.getRecord(); + RecordMetaData meta = r.getMetaData(); + assertNotNull(r); + assertNotNull(fields); + assertNotNull(meta); + assertEquals(new Text("Bolivian Dark"), fields.get(0)); + assertEquals(new Text("14-001"), fields.get(1)); + assertEquals(new DoubleWritable(8.95), fields.get(2)); + assertEquals(RecordMetaDataJdbc.class, meta.getClass()); + } + } + + @Test + public void testNextRecordAndRecover() throws Exception { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + Record r = reader.nextRecord(); + List fields = r.getRecord(); + RecordMetaData meta = r.getMetaData(); + Record recovered = reader.loadFromMetaData(meta); + List fieldsRecovered = recovered.getRecord(); + assertEquals(fields.size(), fieldsRecovered.size()); + for (int i = 0; i < fields.size(); i++) { + assertEquals(fields.get(i), fieldsRecovered.get(i)); + } + } + } + + // Resetting the record reader when initialized as forward only should fail + @Test + public void testResetForwardOnlyShouldFail() throws Exception { + assertThrows(RuntimeException.class, () -> { + try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee", dataSource)) { + Configuration conf = new Configuration(); + conf.setInt(JDBCRecordReader.JDBC_RESULTSET_TYPE, ResultSet.TYPE_FORWARD_ONLY); + reader.initialize(conf, null); + reader.next(); + reader.reset(); + } + }); + } + + @Test + public void testReadAllTypes() throws Exception { + TestDb.buildAllTypesTable(conn); + try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM AllTypes", dataSource)) { + reader.initialize(null); + List item = reader.next(); + + assertEquals(item.size(), 15); + assertEquals(BooleanWritable.class, item.get(0).getClass()); // boolean to boolean + assertEquals(Text.class, item.get(1).getClass()); // date to text + assertEquals(Text.class, item.get(2).getClass()); // time to text + assertEquals(Text.class, item.get(3).getClass()); // timestamp to text + assertEquals(Text.class, item.get(4).getClass()); // char to text + assertEquals(Text.class, item.get(5).getClass()); // long varchar to text + assertEquals(Text.class, item.get(6).getClass()); // varchar to text + assertEquals(DoubleWritable.class, + item.get(7).getClass()); // float to double (derby's float is an alias of double by default) + assertEquals(FloatWritable.class, item.get(8).getClass()); // real to float + assertEquals(DoubleWritable.class, item.get(9).getClass()); // decimal to double + assertEquals(DoubleWritable.class, item.get(10).getClass()); // numeric to double + assertEquals(DoubleWritable.class, item.get(11).getClass()); // double to double + assertEquals(IntWritable.class, item.get(12).getClass()); // integer to integer + assertEquals(IntWritable.class, item.get(13).getClass()); // small int to integer + assertEquals(LongWritable.class, item.get(14).getClass()); // bigint to long + + } + } + + @Test + public void testNextNoMoreShouldFail() throws Exception { + assertThrows(RuntimeException.class, () -> { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + while (reader.hasNext()) { + reader.next(); + } + reader.next(); + } + }); + } + + @Test + public void testInvalidMetadataShouldFail() throws Exception { + assertThrows(IllegalArgumentException.class, () -> { + try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { + RecordMetaDataLine md = new RecordMetaDataLine(1, new URI("file://test"), JDBCRecordReader.class); + reader.loadFromMetaData(md); + } + }); + } + + private JDBCRecordReader getInitializedReader(String query) throws Exception { + int[] indices = {1}; // ProdNum column + JDBCRecordReader reader = new JDBCRecordReader(query, dataSource, "SELECT * FROM Coffee WHERE ProdNum = ?", + indices); + reader.setTrimStrings(true); + reader.initialize(null); + return reader; + } +} \ No newline at end of file diff --git a/datavec/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/TestDb.java b/cavis-datavec/cavis-datavec-data/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/TestDb.java similarity index 100% rename from datavec/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/TestDb.java rename to cavis-datavec/cavis-datavec-data/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/TestDb.java diff --git a/cavis-datavec/cavis-datavec-local/build.gradle b/cavis-datavec/cavis-datavec-local/build.gradle new file mode 100644 index 000000000..153bf0499 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/build.gradle @@ -0,0 +1,39 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation "com.codepoetics:protonpack:1.15" + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDatavec.cavisDatavecArrow + + implementation projects.cavisDnn.cavisDnnApi + implementation "com.google.guava:guava" + implementation "commons-io:commons-io" + implementation "org.slf4j:slf4j-api" + + testImplementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataGeo + testImplementation projects.cavisDatavec.cavisDatavecPython + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testImplementation "joda-time:joda-time" + + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" +} \ No newline at end of file diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/AnalyzeLocal.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/AnalyzeLocal.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/AnalyzeLocal.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/AnalyzeLocal.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/BaseFlatMapFunctionAdaptee.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/BaseFlatMapFunctionAdaptee.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/BaseFlatMapFunctionAdaptee.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/BaseFlatMapFunctionAdaptee.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformExecutor.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformExecutor.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformExecutor.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformExecutor.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformProcessRecordReader.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformProcessRecordReader.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformProcessRecordReader.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformProcessRecordReader.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformProcessSequenceRecordReader.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformProcessSequenceRecordReader.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformProcessSequenceRecordReader.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformProcessSequenceRecordReader.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/SequenceEmptyRecordFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/SequenceEmptyRecordFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/SequenceEmptyRecordFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/SequenceEmptyRecordFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/analysis/aggregate/AnalysisAddFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/analysis/aggregate/AnalysisAddFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/analysis/aggregate/AnalysisAddFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/analysis/aggregate/AnalysisAddFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/analysis/aggregate/AnalysisCombineFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/analysis/aggregate/AnalysisCombineFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/analysis/aggregate/AnalysisCombineFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/analysis/aggregate/AnalysisCombineFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/analysis/histogram/HistogramAddFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/analysis/histogram/HistogramAddFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/analysis/histogram/HistogramAddFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/analysis/histogram/HistogramAddFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/analysis/histogram/HistogramCombineFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/analysis/histogram/HistogramCombineFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/analysis/histogram/HistogramCombineFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/analysis/histogram/HistogramCombineFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/EmptyRecordFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/EmptyRecordFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/EmptyRecordFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/EmptyRecordFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/FlatMapFunctionAdapter.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/FlatMapFunctionAdapter.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/FlatMapFunctionAdapter.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/FlatMapFunctionAdapter.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/LineRecordReaderFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/LineRecordReaderFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/LineRecordReaderFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/LineRecordReaderFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/RecordReaderFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/RecordReaderFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/RecordReaderFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/RecordReaderFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/SequenceRecordReaderFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/SequenceRecordReaderFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/SequenceRecordReaderFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/SequenceRecordReaderFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/data/FilesAsBytesFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/data/FilesAsBytesFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/data/FilesAsBytesFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/data/FilesAsBytesFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/data/RecordReaderBytesFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/data/RecordReaderBytesFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/data/RecordReaderBytesFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/data/RecordReaderBytesFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/data/SequenceRecordReaderBytesFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/data/SequenceRecordReaderBytesFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/functions/data/SequenceRecordReaderBytesFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/functions/data/SequenceRecordReaderBytesFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunctionAdapter.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunctionAdapter.java similarity index 99% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunctionAdapter.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunctionAdapter.java index a0b283e86..58ed6d10e 100644 --- a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunctionAdapter.java +++ b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/ExecuteJoinFromCoGroupFlatMapFunctionAdapter.java @@ -20,7 +20,7 @@ package org.datavec.local.transforms.join; -import org.nd4j.shade.guava.collect.Iterables; +import com.google.common.collect.Iterables; import org.datavec.api.transform.join.Join; import org.datavec.api.writable.Writable; import org.datavec.local.transforms.functions.FlatMapFunctionAdapter; diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/ExtractKeysFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/ExtractKeysFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/ExtractKeysFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/ExtractKeysFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/FilterAndFlattenJoinedValues.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/FilterAndFlattenJoinedValues.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/FilterAndFlattenJoinedValues.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/FilterAndFlattenJoinedValues.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/FilterAndFlattenJoinedValuesAdapter.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/FilterAndFlattenJoinedValuesAdapter.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/FilterAndFlattenJoinedValuesAdapter.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/FilterAndFlattenJoinedValuesAdapter.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/JoinedValue.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/JoinedValue.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/join/JoinedValue.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/join/JoinedValue.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/ColumnAsKeyPairFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/ColumnAsKeyPairFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/ColumnAsKeyPairFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/ColumnAsKeyPairFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/ColumnToKeyPairTransform.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/ColumnToKeyPairTransform.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/ColumnToKeyPairTransform.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/ColumnToKeyPairTransform.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/NDArrayToWritablesFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/NDArrayToWritablesFunction.java similarity index 95% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/NDArrayToWritablesFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/NDArrayToWritablesFunction.java index e533a823a..eda700c50 100644 --- a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/NDArrayToWritablesFunction.java +++ b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/NDArrayToWritablesFunction.java @@ -21,6 +21,7 @@ package org.datavec.local.transforms.misc; import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.NDArrayWritable; import org.datavec.api.writable.Writable; @@ -31,13 +32,10 @@ import java.util.ArrayList; import java.util.List; @AllArgsConstructor +@NoArgsConstructor public class NDArrayToWritablesFunction implements Function> { private boolean useNdarrayWritable = false; - public NDArrayToWritablesFunction() { - useNdarrayWritable = false; - } - @Override public List apply(INDArray arr) { if (arr.rows() != 1) diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/SequenceMergeFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/SequenceMergeFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/SequenceMergeFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/SequenceMergeFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/SequenceWritablesToStringFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/SequenceWritablesToStringFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/SequenceWritablesToStringFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/SequenceWritablesToStringFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/StringToWritablesFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/StringToWritablesFunction.java similarity index 94% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/StringToWritablesFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/StringToWritablesFunction.java index b5e4e4fee..0270be039 100644 --- a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/StringToWritablesFunction.java +++ b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/StringToWritablesFunction.java @@ -31,6 +31,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +/** + * Converts a string to a list of Writables using a RecordReader (potentially splitting the string) + */ @AllArgsConstructor public class StringToWritablesFunction implements Function> { diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/SumLongsFunction2.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/SumLongsFunction2.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/SumLongsFunction2.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/SumLongsFunction2.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/WritablesToNDArrayFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/WritablesToNDArrayFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/WritablesToNDArrayFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/WritablesToNDArrayFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/WritablesToStringFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/WritablesToStringFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/WritablesToStringFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/WritablesToStringFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/comparator/Tuple2Comparator.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/comparator/Tuple2Comparator.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/misc/comparator/Tuple2Comparator.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/misc/comparator/Tuple2Comparator.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/rank/UnzipForCalculateSortedRankFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/rank/UnzipForCalculateSortedRankFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/rank/UnzipForCalculateSortedRankFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/rank/UnzipForCalculateSortedRankFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/reduce/MapToPairForReducerFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/reduce/MapToPairForReducerFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/reduce/MapToPairForReducerFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/reduce/MapToPairForReducerFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/reduce/ReducerFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/reduce/ReducerFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/reduce/ReducerFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/reduce/ReducerFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/ConvertToSequenceLengthOne.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/ConvertToSequenceLengthOne.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/ConvertToSequenceLengthOne.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/ConvertToSequenceLengthOne.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalGroupToSequenceFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalGroupToSequenceFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalGroupToSequenceFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalGroupToSequenceFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalMapToPairByColumnFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalMapToPairByColumnFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalMapToPairByColumnFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalMapToPairByColumnFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalMapToPairByMultipleColumnsFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalMapToPairByMultipleColumnsFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalMapToPairByMultipleColumnsFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalMapToPairByMultipleColumnsFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalSequenceFilterFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalSequenceFilterFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalSequenceFilterFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalSequenceFilterFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalSequenceTransformFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalSequenceTransformFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalSequenceTransformFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/sequence/LocalSequenceTransformFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformProcessFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformProcessFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformProcessFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformProcessFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformProcessFunctionAdapter.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformProcessFunctionAdapter.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformProcessFunctionAdapter.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/LocalTransformProcessFunctionAdapter.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/SequenceSplitFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/SequenceSplitFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/SequenceSplitFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/SequenceSplitFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/SequenceSplitFunctionAdapter.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/SequenceSplitFunctionAdapter.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/SequenceSplitFunctionAdapter.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/SequenceSplitFunctionAdapter.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/filter/FilterWritablesBySchemaFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/filter/FilterWritablesBySchemaFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/filter/FilterWritablesBySchemaFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/filter/FilterWritablesBySchemaFunction.java diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/filter/LocalFilterFunction.java b/cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/filter/LocalFilterFunction.java similarity index 100% rename from datavec/datavec-local/src/main/java/org/datavec/local/transforms/transform/filter/LocalFilterFunction.java rename to cavis-datavec/cavis-datavec-local/src/main/java/org/datavec/local/transforms/transform/filter/LocalFilterFunction.java diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/AssertTestsExtendBaseClass.java similarity index 100% rename from datavec/datavec-local/src/test/java/org/datavec/local/transforms/AssertTestsExtendBaseClass.java rename to cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/AssertTestsExtendBaseClass.java diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/LocalTransformProcessRecordReaderTests.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/LocalTransformProcessRecordReaderTests.java similarity index 93% rename from datavec/datavec-local/src/test/java/org/datavec/local/transforms/LocalTransformProcessRecordReaderTests.java rename to cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/LocalTransformProcessRecordReaderTests.java index 0360129d2..2ec96607e 100644 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/LocalTransformProcessRecordReaderTests.java +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/LocalTransformProcessRecordReaderTests.java @@ -36,18 +36,15 @@ import org.datavec.api.writable.LongWritable; import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) + public class LocalTransformProcessRecordReaderTests { @Test @@ -67,11 +64,11 @@ public class LocalTransformProcessRecordReaderTests { public void simpleTransformTestSequence() { List> sequence = new ArrayList<>(); //First window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0), + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L), new IntWritable(0), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1), + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 100L), new IntWritable(1), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2), + sequence.add(Arrays.asList((Writable) new LongWritable(1451606400000L + 200L), new IntWritable(2), new IntWritable(0))); Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/analysis/TestAnalyzeLocal.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/analysis/TestAnalyzeLocal.java similarity index 94% rename from datavec/datavec-local/src/test/java/org/datavec/local/transforms/analysis/TestAnalyzeLocal.java rename to cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/analysis/TestAnalyzeLocal.java index 05ad1a662..fcab5efb5 100644 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/analysis/TestAnalyzeLocal.java +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/analysis/TestAnalyzeLocal.java @@ -30,23 +30,23 @@ import org.datavec.api.util.ndarray.RecordConverter; import org.datavec.api.writable.Writable; import org.datavec.local.transforms.AnalyzeLocal; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.io.TempDir; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.io.ClassPathResource; +import java.io.File; import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) + public class TestAnalyzeLocal { - + @TempDir + public File testDir; @Test public void testAnalysisBasic() throws Exception { @@ -74,7 +74,7 @@ public class TestAnalyzeLocal { INDArray mean = arr.mean(0); INDArray std = arr.std(0); - for( int i = 0; i < 5; i++) { + for( int i=0; i<5; i++ ){ double m = ((NumericalColumnAnalysis)da.getColumnAnalysis().get(i)).getMean(); double stddev = ((NumericalColumnAnalysis)da.getColumnAnalysis().get(i)).getSampleStdev(); assertEquals(mean.getDouble(i), m, 1e-3); diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestLineRecordReaderFunction.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestLineRecordReaderFunction.java new file mode 100644 index 000000000..11d4672b1 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestLineRecordReaderFunction.java @@ -0,0 +1,73 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.functions; + +import org.apache.commons.io.FileUtils; + + +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.Writable; + +import org.junit.jupiter.api.Test; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestLineRecordReaderFunction { + + @Test + public void testLineRecordReader() throws Exception { + + File dataFile = new ClassPathResource("iris.dat").getFile(); + List lines = FileUtils.readLines(dataFile); + + List linesRdd = (lines); + + CSVRecordReader rr = new CSVRecordReader(0, ','); + + List> out = linesRdd.stream().map(input -> new LineRecordReaderFunction(rr).apply(input)).collect(Collectors.toList()); + List> outList = out; + + + CSVRecordReader rr2 = new CSVRecordReader(0, ','); + rr2.initialize(new FileSplit(dataFile)); + Set> expectedSet = new HashSet<>(); + int totalCount = 0; + while (rr2.hasNext()) { + expectedSet.add(rr2.next()); + totalCount++; + } + + assertEquals(totalCount, outList.size()); + + for (List line : outList) { + assertTrue(expectedSet.contains(line)); + } + } +} diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestNDArrayToWritablesFunction.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestNDArrayToWritablesFunction.java new file mode 100644 index 000000000..37a86a2f3 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestNDArrayToWritablesFunction.java @@ -0,0 +1,57 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.functions; + +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; + +import org.datavec.local.transforms.misc.NDArrayToWritablesFunction; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestNDArrayToWritablesFunction { + + @Test + public void testNDArrayToWritablesScalars() throws Exception { + INDArray arr = Nd4j.arange(5); + List expected = new ArrayList<>(); + for (int i = 0; i < 5; i++) + expected.add(new DoubleWritable(i)); + List actual = new NDArrayToWritablesFunction().apply(arr); + assertEquals(expected, actual); + } + + @Test + public void testNDArrayToWritablesArray() throws Exception { + INDArray arr = Nd4j.arange(5); + List expected = Arrays.asList((Writable) new NDArrayWritable(arr)); + List actual = new NDArrayToWritablesFunction(true).apply(arr); + assertEquals(expected, actual); + } +} diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToNDArrayFunction.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToNDArrayFunction.java new file mode 100644 index 000000000..1cc2943f8 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToNDArrayFunction.java @@ -0,0 +1,66 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.functions; + +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; + +import org.datavec.local.transforms.misc.WritablesToNDArrayFunction; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestWritablesToNDArrayFunction { + + @Test + public void testWritablesToNDArrayAllScalars() throws Exception { + Nd4j.setDataType(DataType.FLOAT); + List l = new ArrayList<>(); + for (int i = 0; i < 5; i++) + l.add(new IntWritable(i)); + INDArray expected = Nd4j.arange(5f).castTo(DataType.FLOAT).reshape(1, 5); + assertEquals(expected, new WritablesToNDArrayFunction().apply(l)); + } + + @Test + public void testWritablesToNDArrayMixed() throws Exception { + Nd4j.setDataType(DataType.FLOAT); + List l = new ArrayList<>(); + l.add(new IntWritable(0)); + l.add(new IntWritable(1)); + INDArray arr = Nd4j.arange(2, 5).reshape(1, 3); + l.add(new NDArrayWritable(arr)); + l.add(new IntWritable(5)); + arr = Nd4j.arange(6, 9).reshape(1, 3); + l.add(new NDArrayWritable(arr)); + l.add(new IntWritable(9)); + + INDArray expected = Nd4j.arange(10).castTo(DataType.FLOAT).reshape(1, 10); + assertEquals(expected, new WritablesToNDArrayFunction().apply(l)); + } +} diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToStringFunctions.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToStringFunctions.java new file mode 100644 index 000000000..fca45adb1 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToStringFunctions.java @@ -0,0 +1,64 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.functions; + + + + +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; + + +import org.datavec.local.transforms.misc.SequenceWritablesToStringFunction; +import org.datavec.local.transforms.misc.WritablesToStringFunction; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestWritablesToStringFunctions { + + + + @Test + public void testWritablesToString() throws Exception { + + List l = Arrays.asList(new DoubleWritable(1.5), new Text("someValue")); + String expected = l.get(0).toString() + "," + l.get(1).toString(); + + assertEquals(expected, new WritablesToStringFunction(",").apply(l)); + } + + @Test + public void testSequenceWritablesToString() throws Exception { + + List> l = Arrays.asList(Arrays.asList(new DoubleWritable(1.5), new Text("someValue")), + Arrays.asList(new DoubleWritable(2.5), new Text("otherValue"))); + + String expected = l.get(0).get(0).toString() + "," + l.get(0).get(1).toString() + "\n" + + l.get(1).get(0).toString() + "," + l.get(1).get(1).toString(); + + assertEquals(expected, new SequenceWritablesToStringFunction(",").apply(l)); + } +} diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/ExecutionTest.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/ExecutionTest.java new file mode 100644 index 000000000..e265136f8 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/ExecutionTest.java @@ -0,0 +1,285 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.transform; + + +import org.datavec.api.transform.MathFunction; +import org.datavec.api.transform.MathOp; +import org.datavec.api.transform.ReduceOp; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.condition.ConditionOp; +import org.datavec.api.transform.condition.column.DoubleColumnCondition; +import org.datavec.api.transform.reduce.Reducer; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.schema.SequenceSchema; +import org.datavec.api.writable.*; +import org.datavec.python.PythonTransform; + +import org.datavec.local.transforms.LocalTransformExecutor; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.ops.transforms.Transforms; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ExecutionTest { + + @Test + public void testExecutionNdarray() { + Schema schema = new Schema.Builder() + .addColumnNDArray("first",new long[]{1,32577}) + .addColumnNDArray("second",new long[]{1,32577}).build(); + + TransformProcess transformProcess = new TransformProcess.Builder(schema) + .ndArrayMathFunctionTransform("first", MathFunction.SIN) + .ndArrayMathFunctionTransform("second",MathFunction.COS) + .build(); + + List> functions = new ArrayList<>(); + List firstRow = new ArrayList<>(); + INDArray firstArr = Nd4j.linspace(1,4,4); + INDArray secondArr = Nd4j.linspace(1,4,4); + firstRow.add(new NDArrayWritable(firstArr)); + firstRow.add(new NDArrayWritable(secondArr)); + functions.add(firstRow); + + List> execute = LocalTransformExecutor.execute(functions, transformProcess); + INDArray firstResult = ((NDArrayWritable) execute.get(0).get(0)).get(); + INDArray secondResult = ((NDArrayWritable) execute.get(0).get(1)).get(); + + INDArray expected = Transforms.sin(firstArr); + INDArray secondExpected = Transforms.cos(secondArr); + assertEquals(expected,firstResult); + assertEquals(secondExpected,secondResult); + + } + + @Test + public void testExecutionSimple() { + Schema schema = new Schema.Builder().addColumnInteger("col0") + .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2"). + addColumnFloat("col3").build(); + + TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1") + .doubleMathOp("col2", MathOp.Add, 10.0).floatMathOp("col3", MathOp.Add, 5f).build(); + + List> inputData = new ArrayList<>(); + inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1), new FloatWritable(0.3f))); + inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1), new FloatWritable(1.7f))); + inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1), new FloatWritable(3.6f))); + + List> rdd = (inputData); + + List> out = new ArrayList<>(LocalTransformExecutor.execute(rdd, tp)); + + Collections.sort(out, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt()); + } + }); + + List> expected = new ArrayList<>(); + expected.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1), new FloatWritable(5.3f))); + expected.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1), new FloatWritable(6.7f))); + expected.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1), new FloatWritable(8.6f))); + + assertEquals(expected, out); + } + + @Test + public void testFilter() { + Schema filterSchema = new Schema.Builder() + .addColumnDouble("col1").addColumnDouble("col2") + .addColumnDouble("col3").build(); + List> inputData = new ArrayList<>(); + inputData.add(Arrays.asList(new IntWritable(0), new DoubleWritable(1), new DoubleWritable(0.1))); + inputData.add(Arrays.asList(new IntWritable(1), new DoubleWritable(3), new DoubleWritable(1.1))); + inputData.add(Arrays.asList(new IntWritable(2), new DoubleWritable(3), new DoubleWritable(2.1))); + TransformProcess transformProcess = new TransformProcess.Builder(filterSchema) + .filter(new DoubleColumnCondition("col1",ConditionOp.LessThan,1)).build(); + List> execute = LocalTransformExecutor.execute(inputData, transformProcess); + assertEquals(2,execute.size()); + } + + @Test + public void testExecutionSequence() { + + Schema schema = new SequenceSchema.Builder().addColumnInteger("col0") + .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build(); + + TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1") + .doubleMathOp("col2", MathOp.Add, 10.0).build(); + + List>> inputSequences = new ArrayList<>(); + List> seq1 = new ArrayList<>(); + seq1.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); + seq1.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); + seq1.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); + List> seq2 = new ArrayList<>(); + seq2.add(Arrays.asList(new IntWritable(3), new Text("state0"), new DoubleWritable(3.1))); + seq2.add(Arrays.asList(new IntWritable(4), new Text("state1"), new DoubleWritable(4.1))); + + inputSequences.add(seq1); + inputSequences.add(seq2); + + List>> rdd = (inputSequences); + + List>> out = LocalTransformExecutor.executeSequenceToSequence(rdd, tp); + + Collections.sort(out, new Comparator>>() { + @Override + public int compare(List> o1, List> o2) { + return -Integer.compare(o1.size(), o2.size()); + } + }); + + List>> expectedSequence = new ArrayList<>(); + List> seq1e = new ArrayList<>(); + seq1e.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1))); + seq1e.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1))); + seq1e.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1))); + List> seq2e = new ArrayList<>(); + seq2e.add(Arrays.asList(new IntWritable(3), new IntWritable(0), new DoubleWritable(13.1))); + seq2e.add(Arrays.asList(new IntWritable(4), new IntWritable(1), new DoubleWritable(14.1))); + + expectedSequence.add(seq1e); + expectedSequence.add(seq2e); + + assertEquals(expectedSequence, out); + } + + + @Test + public void testReductionGlobal() { + + List> in = Arrays.asList( + Arrays.asList(new Text("first"), new DoubleWritable(3.0)), + Arrays.asList(new Text("second"), new DoubleWritable(5.0)) + ); + + List> inData = in; + + Schema s = new Schema.Builder() + .addColumnString("textCol") + .addColumnDouble("doubleCol") + .build(); + + TransformProcess tp = new TransformProcess.Builder(s) + .reduce(new Reducer.Builder(ReduceOp.TakeFirst) + .takeFirstColumns("textCol") + .meanColumns("doubleCol").build()) + .build(); + + List> outRdd = LocalTransformExecutor.execute(inData, tp); + + List> out = outRdd; + + List> expOut = Collections.singletonList(Arrays.asList(new Text("first"), new DoubleWritable(4.0))); + + assertEquals(expOut, out); + } + + @Test + public void testReductionByKey(){ + + List> in = Arrays.asList( + Arrays.asList(new IntWritable(0), new Text("first"), new DoubleWritable(3.0)), + Arrays.asList(new IntWritable(0), new Text("second"), new DoubleWritable(5.0)), + Arrays.asList(new IntWritable(1), new Text("f"), new DoubleWritable(30.0)), + Arrays.asList(new IntWritable(1), new Text("s"), new DoubleWritable(50.0)) + ); + + List> inData = in; + + Schema s = new Schema.Builder() + .addColumnInteger("intCol") + .addColumnString("textCol") + .addColumnDouble("doubleCol") + .build(); + + TransformProcess tp = new TransformProcess.Builder(s) + .reduce(new Reducer.Builder(ReduceOp.TakeFirst) + .keyColumns("intCol") + .takeFirstColumns("textCol") + .meanColumns("doubleCol").build()) + .build(); + + List> outRdd = LocalTransformExecutor.execute(inData, tp); + + List> out = outRdd; + + List> expOut = Arrays.asList( + Arrays.asList(new IntWritable(0), new Text("first"), new DoubleWritable(4.0)), + Arrays.asList(new IntWritable(1), new Text("f"), new DoubleWritable(40.0))); + + out = new ArrayList<>(out); + Collections.sort( + out, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt()); + } + } + ); + + assertEquals(expOut, out); + } + + @Test @Timeout(60) + //@Ignore("AB 2019/05/21 - Fine locally, timeouts on CI - Issue #7657 and #7771") + public void testPythonExecutionNdarray()throws Exception{ + Schema schema = new Schema.Builder() + .addColumnNDArray("first",new long[]{1,32577}) + .addColumnNDArray("second",new long[]{1,32577}).build(); + + TransformProcess transformProcess = new TransformProcess.Builder(schema) + .transform( + PythonTransform.builder().code( + "import numpy as np\nfirst = np.sin(first)\nsecond = np.cos(second)") + .outputSchema(schema).build()) + .build(); + + List> functions = new ArrayList<>(); + List firstRow = new ArrayList<>(); + INDArray firstArr = Nd4j.linspace(1,4,4); + INDArray secondArr = Nd4j.linspace(1,4,4); + firstRow.add(new NDArrayWritable(firstArr)); + firstRow.add(new NDArrayWritable(secondArr)); + functions.add(firstRow); + + List> execute = LocalTransformExecutor.execute(functions, transformProcess); + INDArray firstResult = ((NDArrayWritable) execute.get(0).get(0)).get(); + INDArray secondResult = ((NDArrayWritable) execute.get(0).get(1)).get(); + + INDArray expected = Transforms.sin(firstArr); + INDArray secondExpected = Transforms.cos(secondArr); + assertEquals(expected,firstResult); + assertEquals(secondExpected,secondResult); + + } + +} diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/TestGeoTransforms.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/TestGeoTransforms.java new file mode 100644 index 000000000..d0e431678 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/TestGeoTransforms.java @@ -0,0 +1,154 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.transform; + +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.Transform; +import org.datavec.api.transform.geo.LocationType; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.transform.geo.CoordinatesDistanceTransform; +import org.datavec.api.transform.transform.geo.IPAddressToCoordinatesTransform; +import org.datavec.api.transform.transform.geo.IPAddressToLocationTransform; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.nd4j.common.io.ClassPathResource; + +import java.io.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author saudet + */ +public class TestGeoTransforms { + + @BeforeAll + public static void beforeClass() throws Exception { + //Use test resources version to avoid tests suddenly failing due to IP/Location DB content changing + File f = new ClassPathResource("datavec-geo/GeoIP2-City-Test.mmdb").getFile(); + System.setProperty(IPAddressToLocationTransform.GEOIP_FILE_PROPERTY, f.getPath()); + } + + @AfterAll + public static void afterClass(){ + System.setProperty(IPAddressToLocationTransform.GEOIP_FILE_PROPERTY, ""); + } + + + @Test + public void testCoordinatesDistanceTransform() throws Exception { + Schema schema = new Schema.Builder().addColumnString("point").addColumnString("mean").addColumnString("stddev") + .build(); + + Transform transform = new CoordinatesDistanceTransform("dist", "point", "mean", "stddev", "\\|"); + transform.setInputSchema(schema); + + Schema out = transform.transform(schema); + assertEquals(4, out.numColumns()); + assertEquals(Arrays.asList("point", "mean", "stddev", "dist"), out.getColumnNames()); + assertEquals(Arrays.asList(ColumnType.String, ColumnType.String, ColumnType.String, ColumnType.Double), + out.getColumnTypes()); + + assertEquals(Arrays.asList((Writable) new Text("-30"), new Text("20"), new Text("10"), new DoubleWritable(5.0)), + transform.map(Arrays.asList((Writable) new Text("-30"), new Text("20"), new Text("10")))); + assertEquals(Arrays.asList((Writable) new Text("50|40"), new Text("10|-20"), new Text("10|5"), + new DoubleWritable(Math.sqrt(160))), + transform.map(Arrays.asList((Writable) new Text("50|40"), new Text("10|-20"), + new Text("10|5")))); + } + + @Test + public void testIPAddressToCoordinatesTransform() throws Exception { + Schema schema = new Schema.Builder().addColumnString("column").build(); + + Transform transform = new IPAddressToCoordinatesTransform("column", "CUSTOM_DELIMITER"); + transform.setInputSchema(schema); + + Schema out = transform.transform(schema); + + assertEquals(1, out.getColumnMetaData().size()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + + String in = "81.2.69.160"; + double latitude = 51.5142; + double longitude = -0.0931; + + List writables = transform.map(Collections.singletonList((Writable) new Text(in))); + assertEquals(1, writables.size()); + String[] coordinates = writables.get(0).toString().split("CUSTOM_DELIMITER"); + assertEquals(2, coordinates.length); + assertEquals(latitude, Double.parseDouble(coordinates[0]), 0.1); + assertEquals(longitude, Double.parseDouble(coordinates[1]), 0.1); + + //Check serialization: things like DatabaseReader etc aren't serializable, hence we need custom serialization :/ + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(transform); + + byte[] bytes = baos.toByteArray(); + + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bais); + + Transform deserialized = (Transform) ois.readObject(); + writables = deserialized.map(Collections.singletonList((Writable) new Text(in))); + assertEquals(1, writables.size()); + coordinates = writables.get(0).toString().split("CUSTOM_DELIMITER"); + //System.out.println(Arrays.toString(coordinates)); + assertEquals(2, coordinates.length); + assertEquals(latitude, Double.parseDouble(coordinates[0]), 0.1); + assertEquals(longitude, Double.parseDouble(coordinates[1]), 0.1); + } + + @Test + public void testIPAddressToLocationTransform() throws Exception { + Schema schema = new Schema.Builder().addColumnString("column").build(); + LocationType[] locationTypes = LocationType.values(); + String in = "81.2.69.160"; + String[] locations = {"London", "2643743", "Europe", "6255148", "United Kingdom", "2635167", + "51.5142:-0.0931", "", "England", "6269131"}; //Note: no postcode in this test DB for this record + + for (int i = 0; i < locationTypes.length; i++) { + LocationType locationType = locationTypes[i]; + String location = locations[i]; + + Transform transform = new IPAddressToLocationTransform("column", locationType); + transform.setInputSchema(schema); + + Schema out = transform.transform(schema); + + assertEquals(1, out.getColumnMetaData().size()); + assertEquals(ColumnType.String, out.getMetaData(0).getColumnType()); + + List writables = transform.map(Collections.singletonList((Writable) new Text(in))); + assertEquals(1, writables.size()); + assertEquals(location, writables.get(0).toString()); + //System.out.println(location); + } + } +} diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/TestPythonTransformProcess.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/TestPythonTransformProcess.java new file mode 100644 index 000000000..21ae33b3f --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/TestPythonTransformProcess.java @@ -0,0 +1,380 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.transform; + +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.condition.Condition; +import org.datavec.api.transform.filter.ConditionFilter; +import org.datavec.api.transform.filter.Filter; +import org.datavec.api.transform.schema.Schema; +import org.datavec.local.transforms.LocalTransformExecutor; + +import org.datavec.api.writable.*; +import org.datavec.python.PythonCondition; +import org.datavec.python.PythonTransform; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import javax.annotation.concurrent.NotThreadSafe; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.datavec.api.transform.schema.Schema.Builder; +import static org.junit.jupiter.api.Assertions.*; + +@NotThreadSafe +public class TestPythonTransformProcess { + + + @Test() + public void testStringConcat() throws Exception{ + Builder schemaBuilder = new Builder(); + schemaBuilder + .addColumnString("col1") + .addColumnString("col2"); + + Schema initialSchema = schemaBuilder.build(); + schemaBuilder.addColumnString("col3"); + Schema finalSchema = schemaBuilder.build(); + + String pythonCode = "col3 = col1 + col2"; + + TransformProcess tp = new TransformProcess.Builder(initialSchema).transform( + PythonTransform.builder().code(pythonCode) + .outputSchema(finalSchema) + .build() + ).build(); + + List inputs = Arrays.asList((Writable)new Text("Hello "), new Text("World!")); + + List outputs = tp.execute(inputs); + assertEquals((outputs.get(0)).toString(), "Hello "); + assertEquals((outputs.get(1)).toString(), "World!"); + assertEquals((outputs.get(2)).toString(), "Hello World!"); + + } + + @Test @Timeout(60) + public void testMixedTypes() throws Exception{ + Builder schemaBuilder = new Builder(); + schemaBuilder + .addColumnInteger("col1") + .addColumnFloat("col2") + .addColumnString("col3") + .addColumnDouble("col4"); + + + Schema initialSchema = schemaBuilder.build(); + schemaBuilder.addColumnInteger("col5"); + Schema finalSchema = schemaBuilder.build(); + + String pythonCode = "col5 = (int(col3) + col1 + int(col2)) * int(col4)"; + + TransformProcess tp = new TransformProcess.Builder(initialSchema).transform( + PythonTransform.builder().code(pythonCode) + .outputSchema(finalSchema) + .inputSchema(initialSchema) + .build() ).build(); + + List inputs = Arrays.asList((Writable)new IntWritable(10), + new FloatWritable(3.5f), + new Text("5"), + new DoubleWritable(2.0) + ); + + List outputs = tp.execute(inputs); + assertEquals(((LongWritable)outputs.get(4)).get(), 36); + } + + @Test @Timeout(60) + public void testNDArray() throws Exception{ + long[] shape = new long[]{3, 2}; + INDArray arr1 = Nd4j.rand(shape); + INDArray arr2 = Nd4j.rand(shape); + + INDArray expectedOutput = arr1.add(arr2); + + Builder schemaBuilder = new Builder(); + schemaBuilder + .addColumnNDArray("col1", shape) + .addColumnNDArray("col2", shape); + + Schema initialSchema = schemaBuilder.build(); + schemaBuilder.addColumnNDArray("col3", shape); + Schema finalSchema = schemaBuilder.build(); + + String pythonCode = "col3 = col1 + col2"; + TransformProcess tp = new TransformProcess.Builder(initialSchema).transform( + PythonTransform.builder().code(pythonCode) + .outputSchema(finalSchema) + .build() ).build(); + + List inputs = Arrays.asList( + (Writable) + new NDArrayWritable(arr1), + new NDArrayWritable(arr2) + ); + + List outputs = tp.execute(inputs); + assertEquals(arr1, ((NDArrayWritable)outputs.get(0)).get()); + assertEquals(arr2, ((NDArrayWritable)outputs.get(1)).get()); + assertEquals(expectedOutput,((NDArrayWritable)outputs.get(2)).get()); + + } + + @Test @Timeout(60) + public void testNDArray2() throws Exception{ + long[] shape = new long[]{3, 2}; + INDArray arr1 = Nd4j.rand(shape); + INDArray arr2 = Nd4j.rand(shape); + + INDArray expectedOutput = arr1.add(arr2); + + Builder schemaBuilder = new Builder(); + schemaBuilder + .addColumnNDArray("col1", shape) + .addColumnNDArray("col2", shape); + + Schema initialSchema = schemaBuilder.build(); + schemaBuilder.addColumnNDArray("col3", shape); + Schema finalSchema = schemaBuilder.build(); + + String pythonCode = "col3 = col1 + col2"; + TransformProcess tp = new TransformProcess.Builder(initialSchema).transform( + PythonTransform.builder().code(pythonCode) + .outputSchema(finalSchema) + .build() ).build(); + + List inputs = Arrays.asList( + (Writable) + new NDArrayWritable(arr1), + new NDArrayWritable(arr2) + ); + + List outputs = tp.execute(inputs); + assertEquals(arr1, ((NDArrayWritable)outputs.get(0)).get()); + assertEquals(arr2, ((NDArrayWritable)outputs.get(1)).get()); + assertEquals(expectedOutput,((NDArrayWritable)outputs.get(2)).get()); + + } + + @Test @Timeout(60) + public void testNDArrayMixed() throws Exception{ + long[] shape = new long[]{3, 2}; + INDArray arr1 = Nd4j.rand(DataType.DOUBLE, shape); + INDArray arr2 = Nd4j.rand(DataType.DOUBLE, shape); + INDArray expectedOutput = arr1.add(arr2.castTo(DataType.DOUBLE)); + + Builder schemaBuilder = new Builder(); + schemaBuilder + .addColumnNDArray("col1", shape) + .addColumnNDArray("col2", shape); + + Schema initialSchema = schemaBuilder.build(); + schemaBuilder.addColumnNDArray("col3", shape); + Schema finalSchema = schemaBuilder.build(); + + String pythonCode = "col3 = col1 + col2"; + TransformProcess tp = new TransformProcess.Builder(initialSchema).transform( + PythonTransform.builder().code(pythonCode) + .outputSchema(finalSchema) + .build() + ).build(); + + List inputs = Arrays.asList( + (Writable) + new NDArrayWritable(arr1), + new NDArrayWritable(arr2) + ); + + List outputs = tp.execute(inputs); + assertEquals(arr1, ((NDArrayWritable)outputs.get(0)).get()); + assertEquals(arr2, ((NDArrayWritable)outputs.get(1)).get()); + assertEquals(expectedOutput,((NDArrayWritable)outputs.get(2)).get()); + + } + + @Test @Timeout(60) + public void testPythonFilter() { + Schema schema = new Builder().addColumnInteger("column").build(); + + Condition condition = new PythonCondition( + "f = lambda: column < 0" + ); + + condition.setInputSchema(schema); + + Filter filter = new ConditionFilter(condition); + + assertFalse(filter.removeExample(Collections.singletonList(new IntWritable(10)))); + assertFalse(filter.removeExample(Collections.singletonList(new IntWritable(1)))); + assertFalse(filter.removeExample(Collections.singletonList(new IntWritable(0)))); + assertTrue(filter.removeExample(Collections.singletonList(new IntWritable(-1)))); + assertTrue(filter.removeExample(Collections.singletonList(new IntWritable(-10)))); + + } + + @Test @Timeout(60) + public void testPythonFilterAndTransform() throws Exception{ + Builder schemaBuilder = new Builder(); + schemaBuilder + .addColumnInteger("col1") + .addColumnFloat("col2") + .addColumnString("col3") + .addColumnDouble("col4"); + + Schema initialSchema = schemaBuilder.build(); + schemaBuilder.addColumnString("col6"); + Schema finalSchema = schemaBuilder.build(); + + Condition condition = new PythonCondition( + "f = lambda: col1 < 0 and col2 > 10.0" + ); + + condition.setInputSchema(initialSchema); + + Filter filter = new ConditionFilter(condition); + + String pythonCode = "col6 = str(col1 + col2)"; + TransformProcess tp = new TransformProcess.Builder(initialSchema).transform( + PythonTransform.builder().code(pythonCode) + .outputSchema(finalSchema) + .build() + ).filter( + filter + ).build(); + + List> inputs = new ArrayList<>(); + inputs.add( + Arrays.asList( + (Writable) + new IntWritable(5), + new FloatWritable(3.0f), + new Text("abcd"), + new DoubleWritable(2.1)) + ); + inputs.add( + Arrays.asList( + (Writable) + new IntWritable(-3), + new FloatWritable(3.0f), + new Text("abcd"), + new DoubleWritable(2.1)) + ); + inputs.add( + Arrays.asList( + (Writable) + new IntWritable(5), + new FloatWritable(11.2f), + new Text("abcd"), + new DoubleWritable(2.1)) + ); + + LocalTransformExecutor.execute(inputs,tp); + } + + + @Test + public void testPythonTransformNoOutputSpecified() throws Exception { + PythonTransform pythonTransform = PythonTransform.builder() + .code("a += 2; b = 'hello world'") + .returnAllInputs(true) + .build(); + List> inputs = new ArrayList<>(); + inputs.add(Arrays.asList((Writable)new IntWritable(1))); + Schema inputSchema = new Builder() + .addColumnInteger("a") + .build(); + + TransformProcess tp = new TransformProcess.Builder(inputSchema) + .transform(pythonTransform) + .build(); + List> execute = LocalTransformExecutor.execute(inputs, tp); + assertEquals(3,execute.get(0).get(0).toInt()); + assertEquals("hello world",execute.get(0).get(1).toString()); + + } + + @Test + public void testNumpyTransform() { + PythonTransform pythonTransform = PythonTransform.builder() + .code("a += 2; b = 'hello world'") + .returnAllInputs(true) + .build(); + + List> inputs = new ArrayList<>(); + inputs.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.scalar(1).reshape(1,1)))); + Schema inputSchema = new Builder() + .addColumnNDArray("a",new long[]{1,1}) + .build(); + + TransformProcess tp = new TransformProcess.Builder(inputSchema) + .transform(pythonTransform) + .build(); + List> execute = LocalTransformExecutor.execute(inputs, tp); + assertFalse(execute.isEmpty()); + assertNotNull(execute.get(0)); + assertNotNull(execute.get(0).get(0)); + assertNotNull(execute.get(0).get(1)); + assertEquals(Nd4j.scalar(3).reshape(1, 1),((NDArrayWritable)execute.get(0).get(0)).get()); + assertEquals("hello world",execute.get(0).get(1).toString()); + } + + @Test + public void testWithSetupRun() throws Exception { + + PythonTransform pythonTransform = PythonTransform.builder() + .code("five=None\n" + + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " c = a + b + five\n"+ + " return {'c':c}\n\n") + .returnAllInputs(true) + .setupAndRun(true) + .build(); + + List> inputs = new ArrayList<>(); + inputs.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.scalar(1).reshape(1,1)), + new NDArrayWritable(Nd4j.scalar(2).reshape(1,1)))); + Schema inputSchema = new Builder() + .addColumnNDArray("a",new long[]{1,1}) + .addColumnNDArray("b", new long[]{1, 1}) + .build(); + + TransformProcess tp = new TransformProcess.Builder(inputSchema) + .transform(pythonTransform) + .build(); + List> execute = LocalTransformExecutor.execute(inputs, tp); + assertFalse(execute.isEmpty()); + assertNotNull(execute.get(0)); + assertNotNull(execute.get(0).get(0)); + assertEquals(Nd4j.scalar(8).reshape(1, 1),((NDArrayWritable)execute.get(0).get(3)).get()); + } + +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/join/TestJoin.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/join/TestJoin.java new file mode 100644 index 000000000..adb511603 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/join/TestJoin.java @@ -0,0 +1,237 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.transform.join; + + +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.join.Join; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.*; + + +import org.datavec.local.transforms.LocalTransformExecutor; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestJoin { + + @Test + public void testJoinOneToMany_ManyToOne() { + + Schema customerInfoSchema = + new Schema.Builder().addColumnLong("customerID").addColumnString("customerName").build(); + + Schema purchasesSchema = new Schema.Builder().addColumnLong("purchaseID").addColumnLong("customerID") + .addColumnDouble("amount").build(); + + List> infoList = new ArrayList<>(); + infoList.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"))); + infoList.add(Arrays.asList(new LongWritable(98765), new Text("Customer98765"))); + infoList.add(Arrays.asList(new LongWritable(50000), new Text("Customer50000"))); + + List> purchaseList = new ArrayList<>(); + purchaseList.add(Arrays.asList(new LongWritable(1000000), new LongWritable(12345), + new DoubleWritable(10.00))); + purchaseList.add(Arrays.asList(new LongWritable(1000001), new LongWritable(12345), + new DoubleWritable(20.00))); + purchaseList.add(Arrays.asList(new LongWritable(1000002), new LongWritable(98765), + new DoubleWritable(30.00))); + + Join join = new Join.Builder(Join.JoinType.RightOuter).setJoinColumns("customerID") + .setSchemas(customerInfoSchema, purchasesSchema).build(); + + List> expected = new ArrayList<>(); + expected.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"), + new LongWritable(1000000), new DoubleWritable(10.00))); + expected.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"), + new LongWritable(1000001), new DoubleWritable(20.00))); + expected.add(Arrays.asList(new LongWritable(98765), new Text("Customer98765"), + new LongWritable(1000002), new DoubleWritable(30.00))); + + + + List> info = (infoList); + List> purchases = (purchaseList); + + List> joined = LocalTransformExecutor.executeJoin(join, info, purchases); + List> joinedList = new ArrayList<>(joined); + //Sort by order ID (column 3, index 2) + Collections.sort(joinedList, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Long.compare(o1.get(2).toLong(), o2.get(2).toLong()); + } + }); + assertEquals(expected, joinedList); + + assertEquals(3, joinedList.size()); + + List expectedColNames = Arrays.asList("customerID", "customerName", "purchaseID", "amount"); + assertEquals(expectedColNames, join.getOutputSchema().getColumnNames()); + + List expectedColTypes = + Arrays.asList(ColumnType.Long, ColumnType.String, ColumnType.Long, ColumnType.Double); + assertEquals(expectedColTypes, join.getOutputSchema().getColumnTypes()); + + + //Test Many to one: same thing, but swap the order... + Join join2 = new Join.Builder(Join.JoinType.LeftOuter).setJoinColumns("customerID") + .setSchemas(purchasesSchema, customerInfoSchema).build(); + + List> expectedManyToOne = new ArrayList<>(); + expectedManyToOne.add(Arrays.asList(new LongWritable(1000000), new LongWritable(12345), + new DoubleWritable(10.00), new Text("Customer12345"))); + expectedManyToOne.add(Arrays.asList(new LongWritable(1000001), new LongWritable(12345), + new DoubleWritable(20.00), new Text("Customer12345"))); + expectedManyToOne.add(Arrays.asList(new LongWritable(1000002), new LongWritable(98765), + new DoubleWritable(30.00), new Text("Customer98765"))); + + List> joined2 = LocalTransformExecutor.executeJoin(join2, purchases, info); + List> joinedList2 = new ArrayList<>(joined2); + //Sort by order ID (column 0) + Collections.sort(joinedList2, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Long.compare(o1.get(0).toLong(), o2.get(0).toLong()); + } + }); + assertEquals(3, joinedList2.size()); + + assertEquals(expectedManyToOne, joinedList2); + + List expectedColNames2 = Arrays.asList("purchaseID", "customerID", "amount", "customerName"); + assertEquals(expectedColNames2, join2.getOutputSchema().getColumnNames()); + + List expectedColTypes2 = + Arrays.asList(ColumnType.Long, ColumnType.Long, ColumnType.Double, ColumnType.String); + assertEquals(expectedColTypes2, join2.getOutputSchema().getColumnTypes()); + } + + + @Test + public void testJoinManyToMany() { + Schema schema1 = new Schema.Builder().addColumnLong("id") + .addColumnCategorical("category", Arrays.asList("cat0", "cat1", "cat2")).build(); + + Schema schema2 = new Schema.Builder().addColumnLong("otherId") + .addColumnCategorical("otherCategory", Arrays.asList("cat0", "cat1", "cat2")).build(); + + List> first = new ArrayList<>(); + first.add(Arrays.asList(new LongWritable(0), new Text("cat0"))); + first.add(Arrays.asList(new LongWritable(1), new Text("cat0"))); + first.add(Arrays.asList(new LongWritable(2), new Text("cat1"))); + + List> second = new ArrayList<>(); + second.add(Arrays.asList(new LongWritable(100), new Text("cat0"))); + second.add(Arrays.asList(new LongWritable(101), new Text("cat0"))); + second.add(Arrays.asList(new LongWritable(102), new Text("cat2"))); + + + + List> expOuterJoin = new ArrayList<>(); + expOuterJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); + expOuterJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); + expOuterJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); + expOuterJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); + expOuterJoin.add(Arrays.asList(new LongWritable(2), new Text("cat1"), new NullWritable())); + expOuterJoin.add(Arrays.asList(new NullWritable(), new Text("cat2"), new LongWritable(102))); + + List> expLeftJoin = new ArrayList<>(); + expLeftJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); + expLeftJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); + expLeftJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); + expLeftJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); + expLeftJoin.add(Arrays.asList(new LongWritable(2), new Text("cat1"), new NullWritable())); + + + List> expRightJoin = new ArrayList<>(); + expRightJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); + expRightJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); + expRightJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); + expRightJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); + expRightJoin.add(Arrays.asList(new NullWritable(), new Text("cat2"), new LongWritable(102))); + + List> expInnerJoin = new ArrayList<>(); + expInnerJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); + expInnerJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); + expInnerJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); + expInnerJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); + + List> firstRDD = (first); + List> secondRDD = (second); + + int count = 0; + for (Join.JoinType jt : Join.JoinType.values()) { + Join join = new Join.Builder(jt).setJoinColumnsLeft("category").setJoinColumnsRight("otherCategory") + .setSchemas(schema1, schema2).build(); + List> out = + new ArrayList<>(LocalTransformExecutor.executeJoin(join, firstRDD, secondRDD)); + + //Sort output by column 0, then column 1, then column 2 for comparison to expected... + Collections.sort(out, new Comparator>() { + @Override + public int compare(List o1, List o2) { + Writable w1 = o1.get(0); + Writable w2 = o2.get(0); + if (w1 instanceof NullWritable) + return 1; + else if (w2 instanceof NullWritable) + return -1; + int c = Long.compare(w1.toLong(), w2.toLong()); + if (c != 0) + return c; + c = o1.get(1).toString().compareTo(o2.get(1).toString()); + if (c != 0) + return c; + w1 = o1.get(2); + w2 = o2.get(2); + if (w1 instanceof NullWritable) + return 1; + else if (w2 instanceof NullWritable) + return -1; + return Long.compare(w1.toLong(), w2.toLong()); + } + }); + + switch (jt) { + case Inner: + assertEquals(expInnerJoin, out); + break; + case LeftOuter: + assertEquals(expLeftJoin, out); + break; + case RightOuter: + assertEquals(expRightJoin, out); + break; + case FullOuter: + assertEquals(expOuterJoin, out); + break; + } + count++; + } + + assertEquals(4, count); + } + +} diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/rank/TestCalculateSortedRank.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/rank/TestCalculateSortedRank.java new file mode 100644 index 000000000..39f3405a9 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/rank/TestCalculateSortedRank.java @@ -0,0 +1,79 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.transform.rank; + + +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.api.writable.comparator.DoubleWritableComparator; + + +import org.datavec.local.transforms.LocalTransformExecutor; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestCalculateSortedRank { + + @Test + public void testCalculateSortedRank() { + + List> data = new ArrayList<>(); + data.add(Arrays.asList((Writable) new Text("0"), new DoubleWritable(0.0))); + data.add(Arrays.asList((Writable) new Text("3"), new DoubleWritable(0.3))); + data.add(Arrays.asList((Writable) new Text("2"), new DoubleWritable(0.2))); + data.add(Arrays.asList((Writable) new Text("1"), new DoubleWritable(0.1))); + + List> rdd = (data); + + Schema schema = new Schema.Builder().addColumnsString("TextCol").addColumnDouble("DoubleCol").build(); + + TransformProcess tp = new TransformProcess.Builder(schema) + .calculateSortedRank("rank", "DoubleCol", new DoubleWritableComparator()).build(); + + Schema outSchema = tp.getFinalSchema(); + assertEquals(3, outSchema.numColumns()); + assertEquals(Arrays.asList("TextCol", "DoubleCol", "rank"), outSchema.getColumnNames()); + assertEquals(Arrays.asList(ColumnType.String, ColumnType.Double, ColumnType.Long), outSchema.getColumnTypes()); + + List> out = LocalTransformExecutor.execute(rdd, tp); + + List> collected = out; + assertEquals(4, collected.size()); + for (int i = 0; i < 4; i++) + assertEquals(3, collected.get(i).size()); + + for (List example : collected) { + int exampleNum = example.get(0).toInt(); + int rank = example.get(2).toInt(); + assertEquals(exampleNum, rank); + } + } + +} diff --git a/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/sequence/TestConvertToSequence.java b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/sequence/TestConvertToSequence.java new file mode 100644 index 000000000..04a4a5c47 --- /dev/null +++ b/cavis-datavec/cavis-datavec-local/src/test/java/org/datavec/local/transforms/transform/sequence/TestConvertToSequence.java @@ -0,0 +1,119 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.local.transforms.transform.sequence; + + +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.sequence.comparator.NumericalColumnComparator; +import org.datavec.api.writable.LongWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; + + +import org.datavec.arrow.recordreader.ArrowWritableRecordTimeSeriesBatch; +import org.datavec.local.transforms.LocalTransformExecutor; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestConvertToSequence { + + @Test + public void testConvertToSequenceCompoundKey() { + + Schema s = new Schema.Builder().addColumnsString("key1", "key2").addColumnLong("time").build(); + + List> allExamples = + Arrays.asList(Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(10)), + Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(10)), + Arrays.asList(new Text("k1a"), new Text("k2a"), + new LongWritable(-10)), + Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(5)), + Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(0))); + + TransformProcess tp = new TransformProcess.Builder(s) + .convertToSequence(Arrays.asList("key1", "key2"), new NumericalColumnComparator("time")) + .build(); + + List> rdd = (allExamples); + + List>> out = LocalTransformExecutor.executeToSequence(rdd, tp); + + assertEquals(2, out.size()); + List> seq0; + List> seq1; + if (out.get(0).size() == 3) { + seq0 = out.get(0); + seq1 = out.get(1); + } else { + seq0 = out.get(1); + seq1 = out.get(0); + } + + List> expSeq0 = Arrays.asList( + Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(-10)), + Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(0)), + Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(10))); + + List> expSeq1 = Arrays.asList( + Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(5)), + Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(10))); + + assertEquals(expSeq0, seq0); + assertEquals(expSeq1, seq1); + } + + @Test + public void testConvertToSequenceLength1() { + + Schema s = new Schema.Builder() + .addColumnsString("string") + .addColumnLong("long") + .build(); + + List> allExamples = Arrays.asList( + Arrays.asList(new Text("a"), new LongWritable(0)), + Arrays.asList(new Text("b"), new LongWritable(1)), + Arrays.asList(new Text("c"), new LongWritable(2))); + + TransformProcess tp = new TransformProcess.Builder(s) + .convertToSequence() + .build(); + + List> rdd = (allExamples); + + ArrowWritableRecordTimeSeriesBatch out = (ArrowWritableRecordTimeSeriesBatch) LocalTransformExecutor.executeToSequence(rdd, tp); + + List>> out2 = out.toArrayList(); + + assertEquals(3, out2.size()); + + for( int i = 0; i < 3; i++) { + assertTrue(out2.contains(Collections.singletonList(allExamples.get(i)))); + } + } +} diff --git a/datavec/datavec-local/src/test/resources/log4j.properties b/cavis-datavec/cavis-datavec-local/src/test/resources/log4j.properties similarity index 100% rename from datavec/datavec-local/src/test/resources/log4j.properties rename to cavis-datavec/cavis-datavec-local/src/test/resources/log4j.properties diff --git a/datavec/datavec-local/src/test/resources/logback.xml b/cavis-datavec/cavis-datavec-local/src/test/resources/logback.xml similarity index 100% rename from datavec/datavec-local/src/test/resources/logback.xml rename to cavis-datavec/cavis-datavec-local/src/test/resources/logback.xml diff --git a/cavis-datavec/cavis-datavec-python/build.gradle b/cavis-datavec/cavis-datavec-python/build.gradle new file mode 100644 index 000000000..2b9292500 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/build.gradle @@ -0,0 +1,39 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation 'org.json:json:20190722' + implementation "org.bytedeco:cpython-platform:3.9.6-1.5.6" + implementation "org.bytedeco:numpy-platform:1.21.1-1.5.6" + implementation 'com.google.code.findbugs:jsr305:3.0.2' + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisNative.cavisNativeBlas + implementation "org.slf4j:slf4j-api" + implementation "org.apache.commons:commons-lang3" + implementation "commons-io:commons-io" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/NumpyArray.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/NumpyArray.java new file mode 100644 index 000000000..708184de7 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/NumpyArray.java @@ -0,0 +1,147 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.ArrayUtils; +import org.bytedeco.javacpp.Pointer; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.concurrency.AffinityManager; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.shape.Shape; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.nativeblas.NativeOps; +import org.nd4j.nativeblas.NativeOpsHolder; +import org.nd4j.linalg.api.buffer.DataType; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.nd4j.linalg.api.buffer.DataType.FLOAT; + + +/** + * Wrapper around INDArray for initializing from numpy array + * + * @author Fariz Rahman + */ +@Getter +@NoArgsConstructor +public class NumpyArray { + + private static NativeOps nativeOps; + private static Map arrayCache; // Avoids re-allocation of device buffer + private long address; + private long[] shape; + private long[] strides; + private DataType dtype; + private INDArray nd4jArray; + + static { + //initialize + Nd4j.scalar(1.0); + nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); + arrayCache = new HashMap<>(); + } + + @Builder + public NumpyArray(long address, long[] shape, long strides[], DataType dtype, boolean copy) { + this.address = address; + this.shape = shape; + this.strides = strides; + this.dtype = dtype; + setND4JArray(); + if (copy) { + nd4jArray = nd4jArray.dup(); + Nd4j.getAffinityManager().ensureLocation(nd4jArray, AffinityManager.Location.HOST); + this.address = nd4jArray.data().address(); + } + } + + + + public NumpyArray copy() { + return new NumpyArray(nd4jArray.dup()); + } + + public NumpyArray(long address, long[] shape, long strides[]) { + this(address, shape, strides, FLOAT, false); + } + + public NumpyArray(long address, long[] shape, long strides[], DataType dtype) { + this(address, shape, strides, dtype, false); + } + + + private void setND4JArray() { + + long size = 1; + for (long d : shape) { + size *= d; + } + + String cacheKey = address + "_" + size + "_" + dtype + "_" + ArrayUtils.toString(strides); + nd4jArray = arrayCache.get(cacheKey); + if (nd4jArray == null) { + Pointer ptr = nativeOps.pointerForAddress(address); + ptr = ptr.limit(size); + ptr = ptr.capacity(size); + DataBuffer buff = Nd4j.createBuffer(ptr, size, dtype); + + int elemSize = buff.getElementSize(); + long[] nd4jStrides = new long[strides.length]; + for (int i = 0; i < strides.length; i++) { + nd4jStrides[i] = strides[i] / elemSize; + } + + nd4jArray = Nd4j.create(buff, shape, nd4jStrides, 0, Shape.getOrder(shape, nd4jStrides, 1), dtype); + arrayCache.put(cacheKey, nd4jArray); + } + else{ + if (!Arrays.equals(nd4jArray.shape(), shape)){ + nd4jArray = nd4jArray.reshape(shape); + } + } + Nd4j.getAffinityManager().ensureLocation(nd4jArray, AffinityManager.Location.HOST); + } + + public INDArray getNd4jArray(){ + Nd4j.getAffinityManager().tagLocation(nd4jArray, AffinityManager.Location.HOST); + return nd4jArray; + } + + public NumpyArray(INDArray nd4jArray) { + Nd4j.getAffinityManager().ensureLocation(nd4jArray, AffinityManager.Location.HOST); + DataBuffer buff = nd4jArray.data(); + address = buff.pointer().address(); + shape = nd4jArray.shape(); + long[] nd4jStrides = nd4jArray.stride(); + strides = new long[nd4jStrides.length]; + int elemSize = buff.getElementSize(); + for (int i = 0; i < strides.length; i++) { + strides[i] = nd4jStrides[i] * elemSize; + } + dtype = nd4jArray.dataType(); + this.nd4jArray = nd4jArray; + String cacheKey = address + "_" + nd4jArray.length() + "_" + dtype + "_" + ArrayUtils.toString(strides); + arrayCache.put(cacheKey, nd4jArray); + } + +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/Python.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/Python.java new file mode 100644 index 000000000..98c9b964c --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/Python.java @@ -0,0 +1,275 @@ + +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + + +import org.bytedeco.cpython.PyObject; + +import static org.bytedeco.cpython.global.python.*; +import static org.bytedeco.numpy.global.numpy.PyArray_EnsureArray; + +/** + * Swift like python wrapper for Java + * + * @author Fariz Rahman + */ + +public class Python { + + /** + * Imports a python module, similar to python import statement. + * @param moduleName name of the module to be imported + * @return reference to the module object + * @throws PythonException + */ + public static PythonObject importModule(String moduleName) throws PythonException{ + PythonObject module = new PythonObject(PyImport_ImportModule(moduleName)); + if (module.isNone()) { + throw new PythonException("Error importing module: " + moduleName); + } + return module; + } + + public static PythonObject attr(String attrName) { + return builtins().attr(attrName); + } + + public static PythonObject len(PythonObject pythonObject) { + return attr("len").call(pythonObject); + } + + public static PythonObject str(PythonObject pythonObject) { + return attr("str").call(pythonObject); + } + + public static PythonObject str() { + return attr("str").call(); + } + + public static PythonObject strType() { + return attr("str"); + } + + public static PythonObject float_(PythonObject pythonObject) { + return attr("float").call(pythonObject); + } + + public static PythonObject float_() { + return attr("float").call(); + } + + public static PythonObject floatType() { + return attr("float"); + } + + public static PythonObject bool(PythonObject pythonObject) { + return attr("bool").call(pythonObject); + } + + public static PythonObject bool() { + return attr("bool").call(); + } + + public static PythonObject boolType() { + return attr("bool"); + } + + public static PythonObject int_(PythonObject pythonObject) { + return attr("int").call(pythonObject); + } + + public static PythonObject int_() { + return attr("int").call(); + } + + public static PythonObject intType() { + return attr("int"); + } + + public static PythonObject list(PythonObject pythonObject) { + return attr("list").call(pythonObject); + } + + public static PythonObject list() { + return attr("list").call(); + } + + public static PythonObject listType() { + return attr("list"); + } + + public static PythonObject dict(PythonObject pythonObject) { + return attr("dict").call(pythonObject); + } + + public static PythonObject dict() { + return attr("dict").call(); + } + + public static PythonObject dictType() { + return attr("dict"); + } + + public static PythonObject set(PythonObject pythonObject) { + return attr("set").call(pythonObject); + } + + public static PythonObject set() { + return attr("set").call(); + } + + public static PythonObject bytearray(PythonObject pythonObject) { + return attr("bytearray").call(pythonObject); + } + + public static PythonObject bytearray() { + return attr("bytearray").call(); + } + + public static PythonObject bytearrayType() { + return attr("bytearray"); + } + + public static PythonObject memoryview(PythonObject pythonObject) { + return attr("memoryview").call(pythonObject); + } + + public static PythonObject memoryviewType() { + return attr("memoryview"); + } + + public static PythonObject bytes(PythonObject pythonObject) { + return attr("bytes").call(pythonObject); + } + + public static PythonObject bytes() { + return attr("bytes").call(); + } + + public static PythonObject bytesType() { + return attr("bytes"); + } + + public static PythonObject tuple(PythonObject pythonObject) { + return attr("tuple").call(pythonObject); + } + + public static PythonObject tuple() { + return attr("tuple").call(); + } + + + public static PythonObject Exception(PythonObject pythonObject) { + return attr("Exception").call(pythonObject); + } + + public static PythonObject Exception() { + return attr("Exception").call(); + } + + public static PythonObject ExceptionType() { + return attr("Exception"); + } + + + public static PythonObject tupleType() { + return attr("tuple"); + } + public static PythonObject globals() { + return new PythonObject(PyModule_GetDict(PyImport_ImportModule("__main__"))); + } + + public static PythonObject type(PythonObject obj) { + return attr("type").call(obj); + } + + public static boolean isinstance(PythonObject obj, PythonObject... type) { + return PyObject_IsInstance(obj.getNativePythonObject(), + PyList_AsTuple(new PythonObject(type).getNativePythonObject())) != 0; + } + + public static PythonObject eval(String code) { + PyObject compiledCode = Py_CompileString(code, "", Py_eval_input); + PyObject globals = globals().getNativePythonObject(); + PyObject locals = Python.dict().getNativePythonObject(); + return new PythonObject(PyEval_EvalCode(compiledCode, globals, locals)); + } + + + public static PythonObject builtins(){ + try{ + return importModule("builtins"); + }catch (PythonException pe){ + throw new IllegalStateException("Unable to import builtins: " + pe); // this should never happen + } + + } + + public static PythonObject None() { + return dict().attr("get").call(0); + } + + public static PythonObject True() { + return boolType().call(1); + } + + public static PythonObject False() { + return boolType().call(0); + + } + + public static PythonObject ndarray(PythonObject pythonObject){ + return new PythonObject(PyArray_EnsureArray(pythonObject.getNativePythonObject())); + } + + public static boolean callable(PythonObject pythonObject) { + return PyCallable_Check(pythonObject.getNativePythonObject()) == 1; + } + + + public static void setContext(String context) throws PythonException{ + PythonContextManager.setContext(context); + } + + public static String getCurrentContext(){ + return PythonContextManager.getCurrentContext(); + } + + public static void deleteContext(String context) throws PythonException{ + PythonContextManager.deleteContext(context); + } + + public static void deleteNonMainContexts(){ + PythonContextManager.deleteNonMainContexts(); + } + + public static void setMainContext(){PythonContextManager.setMainContext();} + + public static void exec(String code)throws PythonException{ + PythonExecutioner.exec(code); + } + public static void exec(String code, PythonVariables inputs, PythonVariables outputs) throws PythonException{ + PythonExecutioner.exec(code, inputs, outputs); + } + + public static PythonGIL lock(){ + return PythonGIL.lock(); + } + + +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonCondition.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonCondition.java new file mode 100644 index 000000000..e94e5a171 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonCondition.java @@ -0,0 +1,162 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import org.datavec.api.transform.condition.Condition; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.*; +import java.util.List; + +import static org.datavec.python.PythonUtils.schemaToPythonVariables; +import static org.nd4j.common.base.Preconditions.checkNotNull; +import static org.nd4j.common.base.Preconditions.checkState; + +/** + * Lets a condition be defined as a python method f that takes no arguments + * and returns a boolean indicating whether or not to filter a row. + * The values of all columns in current row are available as global variables to f. + * + * @author Fariz Rahman + */ +public class PythonCondition implements Condition { + + private Schema inputSchema; + private PythonVariables pyInputs; + private PythonTransform pythonTransform; + private String code; + + + public PythonCondition(String pythonCode) { + checkNotNull("Python code must not be null!", pythonCode); + checkState(!pythonCode.isEmpty(), "Python code must not be empty!"); + code = pythonCode; + } + + + @Override + public void setInputSchema(Schema inputSchema) { + this.inputSchema = inputSchema; + try { + pyInputs = schemaToPythonVariables(inputSchema); + PythonVariables pyOuts = new PythonVariables(); + pyOuts.addInt("out"); + pythonTransform = PythonTransform.builder() + .code(code + "\n\nout=f()\nout=0 if out is None else int(out)") + .inputs(pyInputs) + .outputs(pyOuts) + .build(); + + } catch (Exception e) { + throw new RuntimeException(e); + } + + + } + + @Override + public Schema getInputSchema() { + return inputSchema; + } + + @Override + public String[] outputColumnNames() { + String[] columnNames = new String[inputSchema.numColumns()]; + inputSchema.getColumnNames().toArray(columnNames); + return columnNames; + } + + @Override + public String outputColumnName() { + return outputColumnNames()[0]; + } + + @Override + public String[] columnNames() { + return outputColumnNames(); + } + + @Override + public String columnName() { + return outputColumnName(); + } + + @Override + public Schema transform(Schema inputSchema) { + return inputSchema; + } + + @Override + public boolean condition(List list) { + PythonVariables inputs = getPyInputsFromWritables(list); + try { + pythonTransform.getPythonJob().exec(inputs, pythonTransform.getOutputs()); + boolean ret = pythonTransform.getOutputs().getIntValue("out") != 0; + return ret; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean condition(Object input) { + return condition(input); + } + + @Override + public boolean conditionSequence(List> list) { + throw new UnsupportedOperationException("not supported"); + } + + + @Override + public boolean conditionSequence(Object input) { + throw new UnsupportedOperationException("not supported"); + } + + private PythonVariables getPyInputsFromWritables(List writables) { + PythonVariables ret = new PythonVariables(); + + for (int i = 0; i < inputSchema.numColumns(); i++) { + String name = inputSchema.getName(i); + Writable w = writables.get(i); + PythonType pyType = pyInputs.getType(inputSchema.getName(i)); + switch (pyType.getName()) { + case INT: + if (w instanceof LongWritable) { + ret.addInt(name, ((LongWritable) w).get()); + } else { + ret.addInt(name, ((IntWritable) w).get()); + } + + break; + case FLOAT: + ret.addFloat(name, ((DoubleWritable) w).get()); + break; + case STR: + ret.addStr(name, w.toString()); + break; + case NDARRAY: + ret.addNDArray(name, ((NDArrayWritable) w).get()); + break; + } + } + + return ret; + } + + +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonContextManager.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonContextManager.java new file mode 100644 index 000000000..c3563bfc2 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonContextManager.java @@ -0,0 +1,188 @@ + +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.datavec.python; + + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Emulates multiples interpreters in a single interpreter. + * This works by simply obfuscating/de-obfuscating variable names + * such that only the required subset of the global namespace is "visible" + * at any given time. + * By default, there exists a "main" context emulating the default interpreter + * and cannot be deleted. + * @author Fariz Rahman + */ + + +public class PythonContextManager { + + private static Set contexts = new HashSet<>(); + private static AtomicBoolean init = new AtomicBoolean(false); + private static String currentContext; + private static final String MAIN_CONTEXT = "main"; + static { + init(); + } + + private static void init() { + if (init.get()) return; + new PythonExecutioner(); + init.set(true); + currentContext = MAIN_CONTEXT; + contexts.add(currentContext); + } + + + public static void addContext(String contextName) throws PythonException { + if (!validateContextName(contextName)) { + throw new PythonException("Invalid context name: " + contextName); + } + contexts.add(contextName); + } + + public static boolean hasContext(String contextName) { + return contexts.contains(contextName); + } + + + public static boolean validateContextName(String s) { + if (s.length() == 0) return false; + if (!Character.isJavaIdentifierStart(s.charAt(0))) return false; + for (int i = 1; i < s.length(); i++) + if (!Character.isJavaIdentifierPart(s.charAt(i))) + return false; + return true; + } + + private static String getContextPrefix(String contextName) { + return "__collapsed__" + contextName + "__"; + } + + private static String getCollapsedVarNameForContext(String varName, String contextName) { + return getContextPrefix(contextName) + varName; + } + + private static String expandCollapsedVarName(String varName, String contextName) { + String prefix = "__collapsed__" + contextName + "__"; + return varName.substring(prefix.length()); + + } + + private static void collapseContext(String contextName) { + PythonObject globals = Python.globals(); + PythonObject keysList = Python.list(globals.attr("keys").call()); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (!((keyStr.startsWith("__") && keyStr.endsWith("__")) || keyStr.startsWith("__collapsed_"))) { + String collapsedKey = getCollapsedVarNameForContext(keyStr, contextName); + PythonObject val = globals.attr("pop").call(key); + globals.set(new PythonObject(collapsedKey), val); + } + } + } + + private static void expandContext(String contextName) { + String prefix = getContextPrefix(contextName); + PythonObject globals = Python.globals(); + PythonObject keysList = Python.list(globals.attr("keys").call()); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (keyStr.startsWith(prefix)) { + String expandedKey = expandCollapsedVarName(keyStr, contextName); + PythonObject val = globals.attr("pop").call(key); + globals.set(new PythonObject(expandedKey), val); + } + } + + } + + public static void setContext(String contextName) throws PythonException{ + if (contextName.equals(currentContext)) { + return; + } + if (!hasContext(contextName)) { + addContext(contextName); + } + collapseContext(currentContext); + expandContext(contextName); + currentContext = contextName; + + } + + public static void setMainContext() { + try{ + setContext(MAIN_CONTEXT); + } + catch (PythonException pe){ + throw new RuntimeException(pe); + } + + } + + public static String getCurrentContext() { + return currentContext; + } + + public static void deleteContext(String contextName) throws PythonException { + if (contextName.equals(MAIN_CONTEXT)) { + throw new PythonException("Can not delete main context!"); + } + if (contextName.equals(currentContext)) { + throw new PythonException("Can not delete current context!"); + } + String prefix = getContextPrefix(contextName); + PythonObject globals = Python.globals(); + PythonObject keysList = Python.list(globals.attr("keys").call()); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (keyStr.startsWith(prefix)) { + globals.attr("__delitem__").call(key); + } + } + contexts.remove(contextName); + } + + public static void deleteNonMainContexts() { + try{ + setContext(MAIN_CONTEXT); // will never fail + for (String c : contexts.toArray(new String[0])) { + if (!c.equals(MAIN_CONTEXT)) { + deleteContext(c); // will never fail + } + } + }catch(Exception e){ + throw new RuntimeException(e); + } + } + + public String[] getContexts() { + return contexts.toArray(new String[0]); + } + +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonException.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonException.java new file mode 100644 index 000000000..d66c67a32 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonException.java @@ -0,0 +1,44 @@ + +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +/** + * Thrown when an exception occurs in python land + */ +public class PythonException extends Exception { + public PythonException(String message){ + super(message); + } + private static String getExceptionString(PythonObject exception){ + if (Python.isinstance(exception, Python.ExceptionType())){ + String exceptionClass = Python.type(exception).attr("__name__").toString(); + String message = exception.toString(); + return exceptionClass + ": " + message; + } + return exception.toString(); + } + public PythonException(PythonObject exception){ + this(getExceptionString(exception)); + } + public PythonException(String message, Throwable cause){ + super(message, cause); + } + public PythonException(Throwable cause){ + super(cause); + } +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonExecutioner.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonExecutioner.java new file mode 100644 index 000000000..dd48cb104 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonExecutioner.java @@ -0,0 +1,403 @@ + +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.datavec.python; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.bytedeco.cpython.global.python; +import org.bytedeco.numpy.global.numpy; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.bytedeco.cpython.global.python.*; +import static org.datavec.python.Python.*; + +/** + * Allows execution of python scripts managed by + * an internal interpreter. + * An end user may specify a python script to run + * via any of the execution methods available in this class. + * + * At static initialization time (when the class is first initialized) + * a number of components are setup: + * 1. The python path. A user may over ride this with the system property {@link #DEFAULT_PYTHON_PATH_PROPERTY} + * + * 2. Since this executioner uses javacpp to manage and run python interpreters underneath the covers, + * a user may also over ride the system property {@link #JAVACPP_PYTHON_APPEND_TYPE} with one of the {@link JavaCppPathType} + * values. This will allow the user to determine whether the javacpp default python path is used at all, and if so + * whether it is appended, prepended, or not used. This behavior is useful when you need to use an external + * python distribution such as anaconda. + * + * 3. A proper numpy import for use with javacpp: We call numpy import ourselves to ensure proper loading of + * native libraries needed by numpy are allowed to load in the proper order. If we don't do this, + * it causes a variety of issues with running numpy. (User must still include "import numpy as np" in their scripts). + * + * 4. Various python scripts pre defined on the classpath included right with the java code. + * These are auxillary python scripts used for loading classes, pre defining certain kinds of behavior + * in order for us to manipulate values within the python memory, as well as pulling them out of memory + * for integration within the internal python executioner. + * + * For more information on how this works, please take a look at the {@link #init()} + * method. + * + * Generally, a user defining a python script for use by the python executioner + * will have a set of defined target input values and output values. + * These values should not be present when actually running the script, but just referenced. + * In order to test your python script for execution outside the engine, + * we recommend commenting out a few default values as dummy input values. + * This will allow an end user to test their script before trying to use the server. + * + * In order to get output values out of a python script, all a user has to do + * is define the output variables they want being used in the final output in the actual pipeline. + * For example, if a user wants to return a dictionary, they just have to create a dictionary with that name + * and based on the configured {@link PythonVariables} passed as outputs + * to one of the execution methods, we can pull the values out automatically. + * + * For input definitions, it is similar. You just define the values you want used in + * {@link PythonVariables} and we will automatically generate code for defining those values + * as desired for running. This allows the user to customize values dynamically + * at runtime but reference them by name in a python script. + * + * + * @author Fariz Rahman + * @author Adam Gibson + */ + + +@Slf4j +public class PythonExecutioner { + + + private static AtomicBoolean init = new AtomicBoolean(false); + public final static String DEFAULT_PYTHON_PATH_PROPERTY = "org.datavec.python.path"; + public final static String JAVACPP_PYTHON_APPEND_TYPE = "org.datavec.python.javacpp.path.append"; + public final static String DEFAULT_APPEND_TYPE = "before"; + private final static String PYTHON_EXCEPTION_KEY = "__python_exception__"; + + static { + init(); + } + + + private static synchronized void init() { + if (init.get()) { + return; + } + initPythonPath(); + init.set(true); + log.info("CPython: PyEval_InitThreads()"); + PyEval_InitThreads(); + log.info("CPython: Py_InitializeEx()"); + Py_InitializeEx(0); + numpy._import_array(); + } + + private static synchronized void simpleExec(String code) throws PythonException{ + log.debug(code); + log.info("CPython: PyRun_SimpleStringFlag()"); + + int result = PyRun_SimpleStringFlags(code, null); + if (result != 0) { + throw new PythonException("Execution failed, unable to retrieve python exception."); + } + } + + public static boolean validateVariableName(String s) { + if (s.isEmpty()) return false; + if (!Character.isJavaIdentifierStart(s.charAt(0))) return false; + for (int i = 1; i < s.length(); i++) + if (!Character.isJavaIdentifierPart(s.charAt(i))) + return false; + return true; + } + + + /** + * Sets a variable in the global scope of the current context (See @PythonContextManager). + * This is equivalent to `exec("a = b");` where a is the variable name + * and b is the variable value. + * @param varName Name of the python variable being set. Should be a valid python identifier string + * @param pythonObject Value for the python variable + * @throws Exception + */ + public static void setVariable(String varName, PythonObject pythonObject) throws PythonException{ + if (!validateVariableName(varName)){ + throw new PythonException("Invalid variable name: " + varName); + } + Python.globals().set(new PythonObject(varName), pythonObject); + } + + public static void setVariable(String varName, PythonType varType, Object value) throws PythonException { + PythonObject pythonObject; + switch (varType.getName()) { + case STR: + pythonObject = new PythonObject(PythonType.STR.convert(value)); + break; + case INT: + pythonObject = new PythonObject(PythonType.INT.convert(value)); + break; + case FLOAT: + pythonObject = new PythonObject(PythonType.FLOAT.convert(value)); + break; + case BOOL: + pythonObject = new PythonObject(PythonType.BOOL.convert(value)); + break; + case NDARRAY: + pythonObject = new PythonObject(PythonType.NDARRAY.convert(value)); + break; + case LIST: + pythonObject = new PythonObject(PythonType.LIST.convert(value)); + break; + case DICT: + pythonObject = new PythonObject(PythonType.DICT.convert(value)); + break; + case BYTES: + pythonObject = new PythonObject(PythonType.BYTES.convert(value)); + break; + default: + throw new PythonException("Unsupported type: " + varType); + + } + setVariable(varName, pythonObject); + } + + public static void setVariables(PythonVariables pyVars) throws PythonException{ + if (pyVars == null) return; + for (String varName : pyVars.getVariables()) { + setVariable(varName, pyVars.getType(varName), pyVars.getValue(varName)); + } + } + + public static PythonObject getVariable(String varName) { + return Python.globals().attr("get").call(varName); + } + + public static T getVariable(String varName, PythonType varType) throws PythonException{ + PythonObject pythonObject = getVariable(varName); + return varType.toJava(pythonObject); + } + + public static void getVariables(PythonVariables pyVars) throws PythonException { + if (pyVars == null){ + return; + } + for (String varName : pyVars.getVariables()) { + pyVars.setValue(varName, getVariable(varName, pyVars.getType(varName))); + } + } + + + private static String getWrappedCode(String code) { + try (InputStream is = new ClassPathResource("pythonexec/pythonexec.py").getInputStream()) { + String base = IOUtils.toString(is, Charset.defaultCharset()); + StringBuffer indentedCode = new StringBuffer(); + for (String split : code.split("\n")) { + indentedCode.append(" " + split + "\n"); + + } + + String out = base.replace(" pass", indentedCode); + return out; + } catch (IOException e) { + throw new IllegalStateException("Unable to read python code!", e); + } + + } + + private static void throwIfExecutionFailed() throws PythonException{ + PythonObject ex = getVariable(PYTHON_EXCEPTION_KEY); + if (ex != null && !ex.isNone() && !ex.toString().isEmpty()) { + setVariable(PYTHON_EXCEPTION_KEY, new PythonObject("")); + throw new PythonException(ex); + } + } + + public static void exec(String code) throws PythonException { + simpleExec(getWrappedCode(code)); + throwIfExecutionFailed(); + } + + public static void exec(String code, PythonVariables inputVariables, PythonVariables outputVariables) throws PythonException { + setVariables(inputVariables); + simpleExec(getWrappedCode(code)); + throwIfExecutionFailed(); + getVariables(outputVariables); + } + + public static PythonVariables execAndReturnAllVariables(String code) throws PythonException { + simpleExec(getWrappedCode(code)); + throwIfExecutionFailed(); + PythonVariables out = new PythonVariables(); + PythonObject globals = Python.globals(); + PythonObject keysList = Python.list(globals.attr("keys")); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (!keyStr.startsWith("_")) { + PythonObject val = globals.get(key); + if (Python.isinstance(val, intType())) { + out.addInt(keyStr, val.toInt()); + } else if (Python.isinstance(val, floatType())) { + out.addFloat(keyStr, val.toDouble()); + } else if (Python.isinstance(val, strType())) { + out.addStr(keyStr, val.toString()); + } else if (Python.isinstance(val, boolType())) { + out.addBool(keyStr, val.toBoolean()); + } else if (Python.isinstance(val, listType())) { + out.addList(keyStr, val.toList().toArray(new Object[0])); + } else if (Python.isinstance(val, dictType())) { + out.addDict(keyStr, val.toMap()); + } + } + } + return out; + + } + + public static PythonVariables getAllVariables() throws PythonException{ + PythonVariables out = new PythonVariables(); + PythonObject globals = Python.globals(); + PythonObject keysList = Python.list(globals.attr("keys").call()); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (!keyStr.startsWith("_")) { + PythonObject val = globals.get(key); + if (Python.isinstance(val, intType())) { + out.addInt(keyStr, val.toInt()); + } else if (Python.isinstance(val, floatType())) { + out.addFloat(keyStr, val.toDouble()); + } else if (Python.isinstance(val, strType())) { + out.addStr(keyStr, val.toString()); + } else if (Python.isinstance(val, boolType())) { + out.addBool(keyStr, val.toBoolean()); + } else if (Python.isinstance(val, listType())) { + out.addList(keyStr, val.toList().toArray(new Object[0])); + } else if (Python.isinstance(val, dictType())) { + out.addDict(keyStr, val.toMap()); + } else { + PythonObject np = importModule("numpy"); + if (Python.isinstance(val, np.attr("ndarray"), np.attr("generic"))) { + out.addNDArray(keyStr, val.toNumpy()); + } + } + + } + } + return out; + } + + public static PythonVariables execAndReturnAllVariables(String code, PythonVariables inputs) throws Exception{ + setVariables(inputs); + simpleExec(getWrappedCode(code)); + return getAllVariables(); + } + + /** + * One of a few desired values + * for how we should handle + * using javacpp's python path. + * BEFORE: Prepend the python path alongside a defined one + * AFTER: Append the javacpp python path alongside the defined one + * NONE: Don't use javacpp's python path at all + */ + public enum JavaCppPathType { + BEFORE, AFTER, NONE + } + + /** + * Set the python path. + * Generally you can just use the PYTHONPATH environment variable, + * but if you need to set it from code, this can work as well. + */ + + public static synchronized void initPythonPath() { + if (!init.get()) { + try { + String path = System.getProperty(DEFAULT_PYTHON_PATH_PROPERTY); + if (path == null) { + log.info("Setting python default path"); + File[] packages = numpy.cachePackages(); + + //// TODO: fix in javacpp + File sitePackagesWindows = new File(python.cachePackage(), "site-packages"); + File[] packages2 = new File[packages.length + 1]; + for (int i = 0;i < packages.length; i++){ + //System.out.println(packages[i].getAbsolutePath()); + packages2[i] = packages[i]; + } + packages2[packages.length] = sitePackagesWindows; + //System.out.println(sitePackagesWindows.getAbsolutePath()); + packages = packages2; + ////////// + + Py_SetPath(packages); + } else { + log.info("Setting python path " + path); + StringBuffer sb = new StringBuffer(); + File[] packages = numpy.cachePackages(); + JavaCppPathType pathAppendValue = JavaCppPathType.valueOf(System.getProperty(JAVACPP_PYTHON_APPEND_TYPE, DEFAULT_APPEND_TYPE).toUpperCase()); + switch (pathAppendValue) { + case BEFORE: + for (File cacheDir : packages) { + sb.append(cacheDir); + sb.append(java.io.File.pathSeparator); + } + + sb.append(path); + + log.info("Prepending javacpp python path: {}", sb.toString()); + break; + case AFTER: + sb.append(path); + + for (File cacheDir : packages) { + sb.append(cacheDir); + sb.append(java.io.File.pathSeparator); + } + + log.info("Appending javacpp python path " + sb.toString()); + break; + case NONE: + log.info("Not appending javacpp path"); + sb.append(path); + break; + } + + //prepend the javacpp packages + log.info("Final python path: {}", sb.toString()); + + Py_SetPath(sb.toString()); + } + } catch (IOException e) { + log.error("Failed to set python path.", e); + } + } else { + throw new IllegalStateException("Unable to reset python path. Already initialized."); + } + } + +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonGIL.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonGIL.java new file mode 100644 index 000000000..be8165413 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonGIL.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.datavec.python; + +import lombok.extern.slf4j.Slf4j; +import org.bytedeco.cpython.PyThreadState; + +import static org.bytedeco.cpython.global.python.*; +import static org.bytedeco.cpython.global.python.PyEval_RestoreThread; +import static org.bytedeco.cpython.global.python.PyEval_SaveThread; + + +@Slf4j +public class PythonGIL implements AutoCloseable { + private static PyThreadState mainThreadState; + + static { + //log.debug("CPython: PyThreadState_Get()"); + mainThreadState = PyThreadState_Get(); + } + + private PythonGIL() { + acquire(); + } + + @Override + public void close() { + release(); + } + + public static PythonGIL lock() { + return new PythonGIL(); + } + + private static synchronized void acquire() { + log.debug("acquireGIL()"); + log.debug("CPython: PyEval_SaveThread()"); + mainThreadState = PyEval_SaveThread(); + log.debug("CPython: PyThreadState_New()"); + PyThreadState ts = PyThreadState_New(mainThreadState.interp()); + log.debug("CPython: PyEval_RestoreThread()"); + PyEval_RestoreThread(ts); + log.debug("CPython: PyThreadState_Swap()"); + PyThreadState_Swap(ts); + } + + private static synchronized void release() { + log.debug("CPython: PyEval_SaveThread()"); + PyEval_SaveThread(); + log.debug("CPython: PyEval_RestoreThread()"); + PyEval_RestoreThread(mainThreadState); + } +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonJob.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonJob.java new file mode 100644 index 000000000..c50c9bb9e --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonJob.java @@ -0,0 +1,171 @@ +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.datavec.python; + +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.annotation.Nonnull; +import java.util.HashMap; +import java.util.Map; + + +@Data +@NoArgsConstructor +/** + * PythonJob is the right abstraction for executing multiple python scripts + * in a multi thread stateful environment. The setup-and-run mode allows your + * "setup" code (imports, model loading etc) to be executed only once. + */ +public class PythonJob { + + private String code; + private String name; + private String context; + private boolean setupRunMode; + private PythonObject runF; + + static { + new PythonExecutioner(); + } + + @Builder + /** + * @param name Name for the python job. + * @param code Python code. + * @param setupRunMode If true, the python code is expected to have two methods: setup(), which takes no arguments, + * and run() which takes some or no arguments. setup() method is executed once, + * and the run() method is called with the inputs(if any) per transaction, and is expected to return a dictionary + * mapping from output variable names (str) to output values. + * If false, the full script is run on each transaction and the output variables are obtained from the global namespace + * after execution. + */ + public PythonJob(@Nonnull String name, @Nonnull String code, boolean setupRunMode) throws Exception { + this.name = name; + this.code = code; + this.setupRunMode = setupRunMode; + context = "__job_" + name; + if (PythonContextManager.hasContext(context)) { + throw new PythonException("Unable to create python job " + name + ". Context " + context + " already exists!"); + } + if (setupRunMode) setup(); + } + + + /** + * Clears all variables in current context and calls setup() + */ + public void clearState() throws Exception { + String context = this.context; + PythonContextManager.setContext("main"); + PythonContextManager.deleteContext(context); + this.context = context; + setup(); + } + + public void setup() throws Exception { + try (PythonGIL gil = PythonGIL.lock()) { + PythonContextManager.setContext(context); + PythonObject runF = PythonExecutioner.getVariable("run"); + if (runF.isNone() || !Python.callable(runF)) { + PythonExecutioner.exec(code); + runF = PythonExecutioner.getVariable("run"); + } + if (runF.isNone() || !Python.callable(runF)) { + throw new PythonException("run() method not found! " + + "If a PythonJob is created with 'setup and run' " + + "mode enabled, the associated python code is " + + "expected to contain a run() method " + + "(with or without arguments)."); + } + this.runF = runF; + PythonObject setupF = PythonExecutioner.getVariable("setup"); + if (!setupF.isNone()) { + setupF.call(); + } + } + } + + public void exec(PythonVariables inputs, PythonVariables outputs) throws Exception { + try (PythonGIL gil = PythonGIL.lock()) { + PythonContextManager.setContext(context); + if (!setupRunMode) { + PythonExecutioner.exec(code, inputs, outputs); + return; + } + PythonExecutioner.setVariables(inputs); + + PythonObject inspect = Python.importModule("inspect"); + PythonObject getfullargspec = inspect.attr("getfullargspec"); + PythonObject argspec = getfullargspec.call(runF); + PythonObject argsList = argspec.attr("args"); + PythonObject runargs = Python.dict(); + int argsCount = Python.len(argsList).toInt(); + for (int i = 0; i < argsCount; i++) { + PythonObject arg = argsList.get(i); + PythonObject val = Python.globals().get(arg); + if (val.isNone()) { + throw new PythonException("Input value not received for run() argument: " + arg.toString()); + } + runargs.set(arg, val); + } + PythonObject outDict = runF.callWithKwargs(runargs); + Python.globals().attr("update").call(outDict); + + PythonExecutioner.getVariables(outputs); + inspect.del(); + getfullargspec.del(); + argspec.del(); + runargs.del(); + } + } + + public PythonVariables execAndReturnAllVariables(PythonVariables inputs) throws Exception { + try (PythonGIL gil = PythonGIL.lock()) { + PythonContextManager.setContext(context); + if (!setupRunMode) { + return PythonExecutioner.execAndReturnAllVariables(code, inputs); + } + PythonExecutioner.setVariables(inputs); + PythonObject inspect = Python.importModule("inspect"); + PythonObject getfullargspec = inspect.attr("getfullargspec"); + PythonObject argspec = getfullargspec.call(runF); + PythonObject argsList = argspec.attr("args"); + PythonObject runargs = Python.dict(); + int argsCount = Python.len(argsList).toInt(); + for (int i = 0; i < argsCount; i++) { + PythonObject arg = argsList.get(i); + PythonObject val = Python.globals().get(arg); + if (val.isNone()) { + throw new PythonException("Input value not received for run() argument: " + arg.toString()); + } + runargs.set(arg, val); + } + PythonObject outDict = runF.callWithKwargs(runargs); + Python.globals().attr("update").call(outDict); + inspect.del(); + getfullargspec.del(); + argspec.del(); + runargs.del(); + return PythonExecutioner.getAllVariables(); + } + } + + +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonObject.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonObject.java new file mode 100644 index 000000000..4a6a617d5 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonObject.java @@ -0,0 +1,588 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.datavec.python; + + +import lombok.extern.slf4j.Slf4j; +import org.bytedeco.cpython.PyObject; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.SizeTPointer; +import org.bytedeco.numpy.PyArrayObject; +import org.json.JSONArray; +import org.json.JSONObject; +import org.nd4j.linalg.api.buffer.BaseDataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.nativeblas.NativeOpsHolder; + +import java.util.*; + +import static org.bytedeco.cpython.global.python.*; +import static org.bytedeco.numpy.global.numpy.*; + +/** + * Swift like python wrapper for J + * + * @author Fariz Rahman + */ + +@Slf4j +public class PythonObject { + private PyObject nativePythonObject; + + static { + new PythonExecutioner(); + } + + private static Map _getNDArraySerializer() { + Map ndarraySerializer = new HashMap<>(); + PythonObject lambda = Python.eval( + "lambda x: " + + "{'address':" + + "x.__array_interface__['data'][0]," + + "'shape':x.shape,'strides':x.strides," + + "'dtype': str(x.dtype),'_is_numpy_array': True}" + + " if str(type(x))== \"\" else x"); + ndarraySerializer.put("default", + lambda); + return ndarraySerializer; + + } + + public PythonObject(PyObject pyObject) { + nativePythonObject = pyObject; + } + + public PythonObject(INDArray npArray) { + this(new NumpyArray(npArray)); + } + + public PythonObject(BytePointer bp){ + + long address = bp.address(); + long size = bp.capacity(); + NumpyArray npArr = NumpyArray.builder().address(address).shape(new long[]{size}).strides(new long[]{1}).dtype(DataType.INT8).build(); + nativePythonObject = Python.memoryview(new PythonObject(npArr)).nativePythonObject; + } + + public PythonObject(NumpyArray npArray) { + int numpyType; + INDArray indArray = npArray.getNd4jArray(); + DataType dataType = indArray.dataType(); + + switch (dataType) { + case DOUBLE: + numpyType = NPY_DOUBLE; + break; + case FLOAT: + case BFLOAT16: + numpyType = NPY_FLOAT; + break; + case SHORT: + numpyType = NPY_SHORT; + break; + case INT: + numpyType = NPY_INT; + break; + case LONG: + numpyType = NPY_INT64; + break; + case UINT16: + numpyType = NPY_USHORT; + break; + case UINT32: + numpyType = NPY_UINT; + break; + case UINT64: + numpyType = NPY_UINT64; + break; + case BOOL: + numpyType = NPY_BOOL; + break; + case BYTE: + numpyType = NPY_BYTE; + break; + case UBYTE: + numpyType = NPY_UBYTE; + break; + case HALF: + numpyType = NPY_HALF; + break; + default: + throw new RuntimeException("Unsupported dtype: " + npArray.getDtype()); + } + + long[] shape = indArray.shape(); + INDArray inputArray = indArray; + if(dataType == DataType.BFLOAT16) { + log.warn("\n\nThe given nd4j array \n\n{}\n\n is of BFLOAT16 datatype. " + + "Casting a copy of it to FLOAT and creating the respective numpy array from it.\n", indArray); + inputArray = indArray.castTo(DataType.FLOAT); + } + + //Sync to host memory in the case of CUDA, before passing the host memory pointer to Python + if(inputArray.data() instanceof BaseDataBuffer){ + ((BaseDataBuffer)inputArray.data()).syncToPrimary(); + } + + nativePythonObject = PyArray_New(PyArray_Type(), shape.length, new SizeTPointer(shape), + numpyType, null, + inputArray.data().addressPointer(), + 0, NPY_ARRAY_CARRAY, null); + + } + + /*---primitve constructors---*/ + public PyObject getNativePythonObject() { + return nativePythonObject; + } + + public PythonObject(String data) { + nativePythonObject = PyUnicode_FromString(data); + } + + public PythonObject(int data) { + nativePythonObject = PyLong_FromLong((long) data); + } + + public PythonObject(long data) { + nativePythonObject = PyLong_FromLong(data); + } + + public PythonObject(double data) { + nativePythonObject = PyFloat_FromDouble(data); + } + + public PythonObject(boolean data) { + nativePythonObject = PyBool_FromLong(data ? 1 : 0); + } + + private static PythonObject j2pyObject(Object item) { + if (item instanceof PythonObject) { + return (PythonObject) item; + } else if (item instanceof PyObject) { + return new PythonObject((PyObject) item); + } else if (item instanceof INDArray) { + return new PythonObject((INDArray) item); + } else if (item instanceof NumpyArray) { + return new PythonObject((NumpyArray) item); + } else if (item instanceof List) { + return new PythonObject((List) item); + } else if (item instanceof Object[]) { + return new PythonObject((Object[]) item); + } else if (item instanceof Map) { + return new PythonObject((Map) item); + } else if (item instanceof String) { + return new PythonObject((String) item); + } else if (item instanceof Double) { + return new PythonObject((Double) item); + } else if (item instanceof Float) { + return new PythonObject((Float) item); + } else if (item instanceof Long) { + return new PythonObject((Long) item); + } else if (item instanceof Integer) { + return new PythonObject((Integer) item); + } else if (item instanceof Boolean) { + return new PythonObject((Boolean) item); + } else if (item instanceof Pointer){ + return new PythonObject(new BytePointer((Pointer)item)); + } else { + throw new RuntimeException("Unsupported item in list: " + item); + } + } + + public PythonObject(Object[] data) { + PyObject pyList = PyList_New((long) data.length); + for (int i = 0; i < data.length; i++) { + PyList_SetItem(pyList, i, j2pyObject(data[i]).nativePythonObject); + } + nativePythonObject = pyList; + } + + public PythonObject(List data) { + PyObject pyList = PyList_New((long) data.size()); + for (int i = 0; i < data.size(); i++) { + PyList_SetItem(pyList, i, j2pyObject(data.get(i)).nativePythonObject); + } + nativePythonObject = pyList; + } + + public PythonObject(Map data) { + PyObject pyDict = PyDict_New(); + for (Object k : data.keySet()) { + PythonObject pyKey; + if (k instanceof PythonObject) { + pyKey = (PythonObject) k; + } else if (k instanceof String) { + pyKey = new PythonObject((String) k); + } else if (k instanceof Double) { + pyKey = new PythonObject((Double) k); + } else if (k instanceof Float) { + pyKey = new PythonObject((Float) k); + } else if (k instanceof Long) { + pyKey = new PythonObject((Long) k); + } else if (k instanceof Integer) { + pyKey = new PythonObject((Integer) k); + } else if (k instanceof Boolean) { + pyKey = new PythonObject((Boolean) k); + } else { + throw new RuntimeException("Unsupported key in map: " + k.getClass()); + } + Object v = data.get(k); + PythonObject pyVal; + if (v instanceof PythonObject) { + pyVal = (PythonObject) v; + } else if (v instanceof PyObject) { + pyVal = new PythonObject((PyObject) v); + } else if (v instanceof INDArray) { + pyVal = new PythonObject((INDArray) v); + } else if (v instanceof NumpyArray) { + pyVal = new PythonObject((NumpyArray) v); + } else if (v instanceof Map) { + pyVal = new PythonObject((Map) v); + } else if (v instanceof List) { + pyVal = new PythonObject((List) v); + } else if (v instanceof String) { + pyVal = new PythonObject((String) v); + } else if (v instanceof Double) { + pyVal = new PythonObject((Double) v); + } else if (v instanceof Float) { + pyVal = new PythonObject((Float) v); + } else if (v instanceof Long) { + pyVal = new PythonObject((Long) v); + } else if (v instanceof Integer) { + pyVal = new PythonObject((Integer) v); + } else if (v instanceof Boolean) { + pyVal = new PythonObject((Boolean) v); + } else { + throw new RuntimeException("Unsupported value in map: " + k.getClass()); + } + + PyDict_SetItem(pyDict, pyKey.nativePythonObject, pyVal.nativePythonObject); + + } + nativePythonObject = pyDict; + } + + + /*------*/ + + private static String pyObjectToString(PyObject pyObject) { + PyObject repr = PyObject_Str(pyObject); + PyObject str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + String jstr = PyBytes_AsString(str).getString(); + Py_DecRef(repr); + Py_DecRef(str); + return jstr; + } + + public String toString() { + return pyObjectToString(nativePythonObject); + } + + public double toDouble() { + return PyFloat_AsDouble(nativePythonObject); + } + + public float toFloat() { + return (float) PyFloat_AsDouble(nativePythonObject); + } + + public int toInt() { + return (int) PyLong_AsLong(nativePythonObject); + } + + public long toLong() { + return PyLong_AsLong(nativePythonObject); + } + + public boolean toBoolean() { + if (isNone()) return false; + return toInt() != 0; + } + + public NumpyArray toNumpy() throws PythonException{ + PyObject np = PyImport_ImportModule("numpy"); + PyObject ndarray = PyObject_GetAttrString(np, "ndarray"); + if (PyObject_IsInstance(nativePythonObject, ndarray) != 1){ + throw new PythonException("Object is not a numpy array! Use Python.ndarray() to convert object to a numpy array."); + } + Py_DecRef(ndarray); + Py_DecRef(np); + + Pointer objPtr = new Pointer(nativePythonObject); + PyArrayObject npArr = new PyArrayObject(objPtr); + Pointer ptr = PyArray_DATA(npArr); + long[] shape = new long[PyArray_NDIM(npArr)]; + SizeTPointer shapePtr = PyArray_SHAPE(npArr); + if (shapePtr != null) + shapePtr.get(shape, 0, shape.length); + long[] strides = new long[shape.length]; + SizeTPointer stridesPtr = PyArray_STRIDES(npArr); + if (stridesPtr != null) + stridesPtr.get(strides, 0, strides.length); + int npdtype = PyArray_TYPE(npArr); + + DataType dtype; + switch (npdtype){ + case NPY_DOUBLE: + dtype = DataType.DOUBLE; break; + case NPY_FLOAT: + dtype = DataType.FLOAT; break; + case NPY_SHORT: + dtype = DataType.SHORT; break; + case NPY_INT: + dtype = DataType.INT32; break; + case NPY_LONG: + dtype = DataType.LONG; break; + case NPY_UINT: + dtype = DataType.UINT32; break; + case NPY_BYTE: + dtype = DataType.INT8; break; + case NPY_UBYTE: + dtype = DataType.UINT8; break; + case NPY_BOOL: + dtype = DataType.BOOL; break; + case NPY_HALF: + dtype = DataType.FLOAT16; break; + case NPY_LONGLONG: + dtype = DataType.INT64; break; + case NPY_USHORT: + dtype = DataType.UINT16; break; + case NPY_ULONG: + case NPY_ULONGLONG: + dtype = DataType.UINT64; break; + default: + throw new PythonException("Unsupported array data type: " + npdtype); + } + + return new NumpyArray(ptr.address(), shape, strides, dtype); + + } + + public PythonObject attr(String attr) { + + return new PythonObject(PyObject_GetAttrString(nativePythonObject, attr)); + } + + public PythonObject call(Object... args) { + if (args.length > 0 && args[args.length - 1] instanceof Map) { + List args2 = new ArrayList<>(); + for (int i = 0; i < args.length - 1; i++) { + args2.add(args[i]); + } + return call(args2, (Map) args[args.length - 1]); + } + if (args.length == 0) { + return new PythonObject(PyObject_CallObject(nativePythonObject, null)); + } + PyObject tuple = PyTuple_New(args.length); // leaky; tuple may contain borrowed references, so can not be de-allocated. + for (int i = 0; i < args.length; i++) { + PyTuple_SetItem(tuple, i, j2pyObject(args[i]).nativePythonObject); + } + PythonObject ret = new PythonObject(PyObject_Call(nativePythonObject, tuple, null)); + return ret; + } + + public PythonObject callWithArgs(PythonObject args) { + PyObject tuple = PyList_AsTuple(args.nativePythonObject); + return new PythonObject(PyObject_Call(nativePythonObject, tuple, null)); + } + + public PythonObject callWithKwargs(PythonObject kwargs) { + PyObject tuple = PyTuple_New(0); + return new PythonObject(PyObject_Call(nativePythonObject, tuple, kwargs.nativePythonObject)); + } + + public PythonObject callWithArgsAndKwargs(PythonObject args, PythonObject kwargs) { + PyObject tuple = PyList_AsTuple(args.nativePythonObject); + PyObject dict = kwargs.nativePythonObject; + return new PythonObject(PyObject_Call(nativePythonObject, tuple, dict)); + } + + public PythonObject call(Map kwargs) { + PyObject dict = new PythonObject(kwargs).nativePythonObject; + PyObject tuple = PyTuple_New(0); + return new PythonObject(PyObject_Call(nativePythonObject, tuple, dict)); + } + + public PythonObject call(List args) { + PyObject tuple = PyList_AsTuple(new PythonObject(args).nativePythonObject); + return new PythonObject(PyObject_Call(nativePythonObject, tuple, null)); + } + + public PythonObject call(List args, Map kwargs) { + PyObject tuple = PyList_AsTuple(new PythonObject(args).nativePythonObject); + PyObject dict = new PythonObject(kwargs).nativePythonObject; + return new PythonObject(PyObject_Call(nativePythonObject, tuple, dict)); + } + + private PythonObject get(PyObject key) { + return new PythonObject( + PyObject_GetItem(nativePythonObject, key) + ); + } + + public PythonObject get(PythonObject key) { + return get(key.nativePythonObject); + } + + public PythonObject get(int key) { + return get(PyLong_FromLong((long) key)); + } + + public PythonObject get(long key) { + return new PythonObject( + PyObject_GetItem(nativePythonObject, PyLong_FromLong(key)) + ); + } + + public PythonObject get(double key) { + return new PythonObject( + PyObject_GetItem(nativePythonObject, PyFloat_FromDouble(key)) + ); + } + + public PythonObject get(String key) { + return get(new PythonObject(key)); + } + + public void set(PythonObject key, PythonObject value) { + PyObject_SetItem(nativePythonObject, key.nativePythonObject, value.nativePythonObject); + } + + public void del() { + Py_DecRef(nativePythonObject); + nativePythonObject = null; + } + + public JSONArray toJSONArray() throws PythonException { + PythonObject json = Python.importModule("json"); + PythonObject serialized = json.attr("dumps").call(this, _getNDArraySerializer()); + String jsonString = serialized.toString(); + return new JSONArray(jsonString); + } + + public JSONObject toJSONObject() throws PythonException { + PythonObject json = Python.importModule("json"); + PythonObject serialized = json.attr("dumps").call(this, _getNDArraySerializer()); + String jsonString = serialized.toString(); + return new JSONObject(jsonString); + } + + public List toList() throws PythonException{ + List list = new ArrayList(); + int n = Python.len(this).toInt(); + for (int i = 0; i < n; i++) { + PythonObject o = get(i); + if (Python.isinstance(o, Python.strType())) { + list.add(o.toString()); + } else if (Python.isinstance(o, Python.intType())) { + list.add(o.toLong()); + } else if (Python.isinstance(o, Python.floatType())) { + list.add(o.toDouble()); + } else if (Python.isinstance(o, Python.boolType())) { + list.add(o); + } else if (Python.isinstance(o, Python.listType(), Python.tupleType())) { + list.add(o.toList()); + } else if (Python.isinstance(o, Python.importModule("numpy").attr("ndarray"))) { + list.add(o.toNumpy().getNd4jArray()); + } else if (Python.isinstance(o, Python.dictType())) { + list.add(o.toMap()); + } else { + throw new RuntimeException("Error while converting python" + + " list to java List: Unable to serialize python " + + "object of type " + Python.type(this).toString()); + } + } + + return list; + } + + public Map toMap() throws PythonException{ + Map map = new HashMap(); + List keys = Python.list(attr("keys").call()).toList(); + List values = Python.list(attr("values").call()).toList(); + for (int i = 0; i < keys.size(); i++) { + map.put(keys.get(i), values.get(i)); + } + return map; + } + + public BytePointer toBytePointer() throws PythonException{ + if (Python.isinstance(this, Python.bytesType())){ + PyObject byteArray = PyByteArray_FromObject(nativePythonObject); + return PyByteArray_AsString(byteArray); + + } + else if (Python.isinstance(this, Python.bytearrayType())){ + return PyByteArray_AsString(nativePythonObject); + } + else if (Python.isinstance(this, Python.memoryviewType())){ + +// PyObject np = PyImport_ImportModule("numpy"); +// PyObject array = PyObject_GetAttrString(np, "asarray"); +// PyObject npArr = PyObject_CallObject(array, nativePythonObject); // Doesn't work + // Invoke interpreter: + String tempContext = "temp" + UUID.randomUUID().toString().replace('-', '_'); + String originalContext = Python.getCurrentContext(); + Python.setContext(tempContext); + PythonExecutioner.setVariable("memview", this); + PythonExecutioner.exec("import numpy as np\narr = np.frombuffer(memview, dtype='int8')"); + INDArray arr = PythonExecutioner.getVariable("arr").toNumpy().getNd4jArray(); + if(arr.data() instanceof BaseDataBuffer){ + ((BaseDataBuffer)arr.data()).syncToPrimary(); + } + BytePointer ret = new BytePointer(arr.data().pointer()); + Python.setContext(originalContext); + Python.deleteContext(tempContext); + return ret; + } else { + PyObject ctypes = PyImport_ImportModule("ctypes"); + PyObject cArrType = PyObject_GetAttrString(ctypes, "Array"); + if (PyObject_IsInstance(nativePythonObject, cArrType) != 0){ + PyObject cVoidP = PyObject_GetAttrString(ctypes, "c_void_p"); + PyObject cast = PyObject_GetAttrString(ctypes, "cast"); + PyObject argsTuple = PyTuple_New(2); + PyTuple_SetItem(argsTuple, 0, nativePythonObject); + PyTuple_SetItem(argsTuple, 1, cVoidP); + PyObject voidPtr = PyObject_Call(cast, argsTuple, null); + PyObject pyAddress = PyObject_GetAttrString(voidPtr, "value"); + long address = PyLong_AsLong(pyAddress); + long size = PyObject_Size(nativePythonObject); + Py_DecRef(ctypes); + Py_DecRef(cArrType); + Py_DecRef(argsTuple); + Py_DecRef(voidPtr); + Py_DecRef(pyAddress); + Pointer ptr = NativeOpsHolder.getInstance().getDeviceNativeOps().pointerForAddress(address); + ptr = ptr.limit(size); + ptr = ptr.capacity(size); + return new BytePointer(ptr); + } + else{ + throw new PythonException("Expected bytes, bytearray, memoryview or ctypesArray. Received " + Python.type(this).toString()); + } + } + } + public boolean isNone() { + return (nativePythonObject == null)|| + (toString().equals("None") && Python.type(this).toString().equals("")); + } +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonProcess.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonProcess.java new file mode 100644 index 000000000..a8ee56510 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonProcess.java @@ -0,0 +1,132 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.datavec.python; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.bytedeco.javacpp.Loader; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +@Slf4j +public class PythonProcess { + private static String pythonExecutable = Loader.load(org.bytedeco.cpython.python.class); + public static String runAndReturn(String... arguments)throws IOException, InterruptedException{ + String[] allArgs = new String[arguments.length + 1]; + for (int i = 0; i < arguments.length; i++){ + allArgs[i + 1] = arguments[i]; + } + allArgs[0] = pythonExecutable; + log.info("Executing command: " + Arrays.toString(allArgs)); + ProcessBuilder pb = new ProcessBuilder(allArgs); + Process process = pb.start(); + String out = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8); + process.waitFor(); + return out; + + } + + public static void run(String... arguments)throws IOException, InterruptedException{ + String[] allArgs = new String[arguments.length + 1]; + for (int i = 0; i < arguments.length; i++){ + allArgs[i + 1] = arguments[i]; + } + allArgs[0] = pythonExecutable; + log.info("Executing command: " + Arrays.toString(allArgs)); + ProcessBuilder pb = new ProcessBuilder(allArgs); + pb.inheritIO().start().waitFor(); + } + public static void pipInstall(String packageName) throws PythonException{ + try{ + run("-m", "pip", "install", packageName); + }catch(Exception e){ + throw new PythonException("Error installing package " + packageName, e); + } + + } + + public static void pipInstall(String packageName, String version) throws PythonException{ + pipInstall(packageName + "==" + version); + } + + public static void pipUninstall(String packageName) throws PythonException{ + try{ + run("-m", "pip", "uninstall", packageName); + }catch(Exception e){ + throw new PythonException("Error uninstalling package " + packageName, e); + } + + } + public static void pipInstallFromGit(String gitRepoUrl) throws PythonException{ + if (!gitRepoUrl.contains("://")){ + gitRepoUrl = "git://" + gitRepoUrl; + } + try{ + run("-m", "pip", "install", "git+", gitRepoUrl); + }catch(Exception e){ + throw new PythonException("Error installing package from " + gitRepoUrl, e); + } + + } + + public static String getPackageVersion(String packageName) throws PythonException{ + String out; + try{ + out = runAndReturn("-m", "pip", "show", packageName); + } catch (Exception e){ + throw new PythonException("Error finding version for package " + packageName, e); + } + + if (!out.contains("Version: ")){ + throw new PythonException("Can't find package " + packageName); + } + String pkgVersion = out.split("Version: ")[1].split(System.lineSeparator())[0]; + return pkgVersion; + } + + public static boolean isPackageInstalled(String packageName)throws PythonException{ + try{ + String out = runAndReturn("-m", "pip", "show", packageName); + return !out.isEmpty(); + }catch (Exception e){ + throw new PythonException("Error checking if package is installed: " +packageName, e); + } + + } + + public static void pipInstallFromRequirementsTxt(String path) throws PythonException{ + try{ + run("-m", "pip", "install","-r", path); + }catch (Exception e){ + throw new PythonException("Error installing packages from " + path, e); + } + } + + public static void pipInstallFromSetupScript(String path, boolean inplace) throws PythonException{ + + try{ + run(path, inplace?"develop":"install"); + }catch (Exception e){ + throw new PythonException("Error installing package from " + path, e); + } + + } + +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonTransform.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonTransform.java new file mode 100644 index 000000000..4395078d3 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonTransform.java @@ -0,0 +1,331 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.datavec.api.transform.Transform; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.*; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.holder.ObjectMapperHolder; +import org.nd4j.linalg.api.ndarray.INDArray; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.datavec.python.PythonUtils.schemaToPythonVariables; + +/** + * Row-wise Transform that applies arbitrary python code on each row + * + * @author Fariz Rahman + */ + +@NoArgsConstructor +@Data +public class PythonTransform implements Transform { + + private String code; + private PythonVariables inputs; + private PythonVariables outputs; + private String name = UUID.randomUUID().toString(); + private Schema inputSchema; + private Schema outputSchema; + private String outputDict; + private boolean returnAllVariables; + private boolean setupAndRun = false; + private PythonJob pythonJob; + + + @Builder + public PythonTransform(String code, + PythonVariables inputs, + PythonVariables outputs, + String name, + Schema inputSchema, + Schema outputSchema, + String outputDict, + boolean returnAllInputs, + boolean setupAndRun) { + Preconditions.checkNotNull(code, "No code found to run!"); + this.code = code; + this.returnAllVariables = returnAllInputs; + this.setupAndRun = setupAndRun; + if (inputs != null) + this.inputs = inputs; + if (outputs != null) + this.outputs = outputs; + if (name != null) + this.name = name; + if (outputDict != null) { + this.outputDict = outputDict; + this.outputs = new PythonVariables(); + this.outputs.addDict(outputDict); + } + + try { + if (inputSchema != null) { + this.inputSchema = inputSchema; + if (inputs == null || inputs.isEmpty()) { + this.inputs = schemaToPythonVariables(inputSchema); + } + } + + if (outputSchema != null) { + this.outputSchema = outputSchema; + if (outputs == null || outputs.isEmpty()) { + this.outputs = schemaToPythonVariables(outputSchema); + } + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + try{ + pythonJob = PythonJob.builder() + .name("a" + UUID.randomUUID().toString().replace("-", "_")) + .code(code) + .setupRunMode(setupAndRun) + .build(); + } + catch(Exception e){ + throw new IllegalStateException("Error creating python job: " + e); + } + + } + + + @Override + public void setInputSchema(Schema inputSchema) { + Preconditions.checkNotNull(inputSchema, "No input schema found!"); + this.inputSchema = inputSchema; + try { + inputs = schemaToPythonVariables(inputSchema); + } catch (Exception e) { + throw new RuntimeException(e); + } + if (outputSchema == null && outputDict == null) { + outputSchema = inputSchema; + } + + } + + @Override + public Schema getInputSchema() { + return inputSchema; + } + + @Override + public List> mapSequence(List> sequence) { + List> out = new ArrayList<>(); + for (List l : sequence) { + out.add(map(l)); + } + return out; + } + + @Override + public Object map(Object input) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public Object mapSequence(Object sequence) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + + @Override + public List map(List writables) { + PythonVariables pyInputs = getPyInputsFromWritables(writables); + Preconditions.checkNotNull(pyInputs, "Inputs must not be null!"); + try { + if (returnAllVariables) { + return getWritablesFromPyOutputs(pythonJob.execAndReturnAllVariables(pyInputs)); + } + + if (outputDict != null) { + pythonJob.exec(pyInputs, outputs); + PythonVariables out = PythonUtils.expandInnerDict(outputs, outputDict); + return getWritablesFromPyOutputs(out); + } else { + pythonJob.exec(pyInputs, outputs); + + return getWritablesFromPyOutputs(outputs); + } + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public String[] outputColumnNames() { + return outputs.getVariables(); + } + + @Override + public String outputColumnName() { + return outputColumnNames()[0]; + } + + @Override + public String[] columnNames() { + return outputs.getVariables(); + } + + @Override + public String columnName() { + return columnNames()[0]; + } + + public Schema transform(Schema inputSchema) { + return outputSchema; + } + + + private PythonVariables getPyInputsFromWritables(List writables) { + PythonVariables ret = new PythonVariables(); + + for (String name : inputs.getVariables()) { + int colIdx = inputSchema.getIndexOfColumn(name); + Writable w = writables.get(colIdx); + PythonType pyType = inputs.getType(name); + switch (pyType.getName()) { + case INT: + if (w instanceof LongWritable) { + ret.addInt(name, ((LongWritable) w).get()); + } else { + ret.addInt(name, ((IntWritable) w).get()); + } + break; + case FLOAT: + if (w instanceof DoubleWritable) { + ret.addFloat(name, ((DoubleWritable) w).get()); + } else { + ret.addFloat(name, ((FloatWritable) w).get()); + } + break; + case STR: + ret.addStr(name, w.toString()); + break; + case NDARRAY: + ret.addNDArray(name, ((NDArrayWritable) w).get()); + break; + case BOOL: + ret.addBool(name, ((BooleanWritable) w).get()); + break; + default: + throw new RuntimeException("Unsupported input type:" + pyType); + } + + } + return ret; + } + + private List getWritablesFromPyOutputs(PythonVariables pyOuts) { + List out = new ArrayList<>(); + String[] varNames; + varNames = pyOuts.getVariables(); + Schema.Builder schemaBuilder = new Schema.Builder(); + for (int i = 0; i < varNames.length; i++) { + String name = varNames[i]; + PythonType pyType = pyOuts.getType(name); + switch (pyType.getName()) { + case INT: + schemaBuilder.addColumnLong(name); + break; + case FLOAT: + schemaBuilder.addColumnDouble(name); + break; + case STR: + case DICT: + case LIST: + schemaBuilder.addColumnString(name); + break; + case NDARRAY: + INDArray arr = pyOuts.getNDArrayValue(name); + schemaBuilder.addColumnNDArray(name, arr.shape()); + break; + case BOOL: + schemaBuilder.addColumnBoolean(name); + break; + default: + throw new IllegalStateException("Unable to support type " + pyType.getName()); + } + } + this.outputSchema = schemaBuilder.build(); + + + for (int i = 0; i < varNames.length; i++) { + String name = varNames[i]; + PythonType pyType = pyOuts.getType(name); + + switch (pyType.getName()) { + case INT: + out.add(new LongWritable(pyOuts.getIntValue(name))); + break; + case FLOAT: + out.add(new DoubleWritable(pyOuts.getFloatValue(name))); + break; + case STR: + out.add(new Text(pyOuts.getStrValue(name))); + break; + case NDARRAY: + INDArray arr = pyOuts.getNDArrayValue(name); + out.add(new NDArrayWritable(arr)); + break; + case DICT: + Map dictValue = pyOuts.getDictValue(name); + Map noNullValues = new java.util.HashMap<>(); + for (Map.Entry entry : dictValue.entrySet()) { + if (entry.getValue() != org.json.JSONObject.NULL) { + noNullValues.put(entry.getKey(), entry.getValue()); + } + } + + try { + out.add(new Text(ObjectMapperHolder.getJsonMapper().writeValueAsString(noNullValues))); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unable to serialize dictionary " + name + " to json!"); + } + break; + case LIST: + Object[] listValue = pyOuts.getListValue(name).toArray(); + try { + out.add(new Text(ObjectMapperHolder.getJsonMapper().writeValueAsString(listValue))); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unable to serialize list vlaue " + name + " to json!"); + } + break; + case BOOL: + out.add(new BooleanWritable(pyOuts.getBooleanValue(name))); + break; + default: + throw new IllegalStateException("Unable to support type " + pyType.getName()); + } + } + return out; + } + + +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonType.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonType.java new file mode 100644 index 000000000..d0a3f488f --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonType.java @@ -0,0 +1,238 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.Pointer; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.datavec.python.Python.importModule; + + +/** + * + * @param Corresponding Java type for the Python type + */ +public abstract class PythonType { + + public abstract T toJava(PythonObject pythonObject) throws PythonException; + private final TypeName typeName; + + public enum TypeName{ + STR, + INT, + FLOAT, + BOOL, + LIST, + DICT, + NDARRAY, + BYTES + } + private PythonType(TypeName typeName){ + this.typeName = typeName; + } + public TypeName getName(){return typeName;} + public String toString(){ + return getName().name(); + } + public static PythonType valueOf(String typeName) throws PythonException{ + try{ + typeName.valueOf(typeName); + } catch (IllegalArgumentException iae){ + throw new PythonException("Invalid python type: " + typeName, iae); + } + try{ + return (PythonType)PythonType.class.getField(typeName).get(null); // shouldn't fail + } catch (Exception e){ + throw new RuntimeException(e); + } + + } + public static PythonType valueOf(TypeName typeName){ + try{ + return valueOf(typeName.name()); // shouldn't fail + }catch (PythonException pe){ + throw new RuntimeException(pe); + } + } + + /** + * Since multiple java types can map to the same python type, + * this method "normalizes" all supported incoming objects to T + * + * @param object object to be converted to type T + * @return + */ + public T convert(Object object) throws PythonException { + return (T) object; + } + + public static final PythonType STR = new PythonType(TypeName.STR) { + @Override + public String toJava(PythonObject pythonObject) throws PythonException { + if (!Python.isinstance(pythonObject, Python.strType())) { + throw new PythonException("Expected variable to be str, but was " + Python.type(pythonObject)); + } + return pythonObject.toString(); + } + + @Override + public String convert(Object object) { + return object.toString(); + } + }; + + public static final PythonType INT = new PythonType(TypeName.INT) { + @Override + public Long toJava(PythonObject pythonObject) throws PythonException { + if (!Python.isinstance(pythonObject, Python.intType())) { + throw new PythonException("Expected variable to be int, but was " + Python.type(pythonObject)); + } + return pythonObject.toLong(); + } + + @Override + public Long convert(Object object) throws PythonException { + if (object instanceof Number) { + return ((Number) object).longValue(); + } + throw new PythonException("Unable to cast " + object + " to Long."); + } + }; + + public static final PythonType FLOAT = new PythonType(TypeName.FLOAT) { + @Override + public Double toJava(PythonObject pythonObject) throws PythonException { + if (!Python.isinstance(pythonObject, Python.floatType())) { + throw new PythonException("Expected variable to be float, but was " + Python.type(pythonObject)); + } + return pythonObject.toDouble(); + } + + @Override + public Double convert(Object object) throws PythonException { + if (object instanceof Number) { + return ((Number) object).doubleValue(); + } + throw new PythonException("Unable to cast " + object + " to Double."); + } + }; + + public static final PythonType BOOL = new PythonType(TypeName.BOOL) { + @Override + public Boolean toJava(PythonObject pythonObject) throws PythonException { + if (!Python.isinstance(pythonObject, Python.boolType())) { + throw new PythonException("Expected variable to be bool, but was " + Python.type(pythonObject)); + } + return pythonObject.toBoolean(); + } + + @Override + public Boolean convert(Object object) throws PythonException { + if (object instanceof Number) { + return ((Number) object).intValue() != 0; + } else if (object instanceof Boolean) { + return (Boolean) object; + } + throw new PythonException("Unable to cast " + object + " to Boolean."); + } + }; + + public static final PythonType LIST = new PythonType(TypeName.LIST) { + @Override + public List toJava(PythonObject pythonObject) throws PythonException { + if (!Python.isinstance(pythonObject, Python.listType())) { + throw new PythonException("Expected variable to be list, but was " + Python.type(pythonObject)); + } + return pythonObject.toList(); + } + + @Override + public List convert(Object object) throws PythonException { + if (object instanceof java.util.List) { + return (List) object; + } else if (object instanceof org.json.JSONArray) { + org.json.JSONArray jsonArray = (org.json.JSONArray) object; + return jsonArray.toList(); + + } else if (object instanceof Object[]) { + return Arrays.asList((Object[]) object); + } + throw new PythonException("Unable to cast " + object + " to List."); + } + }; + + public static final PythonType DICT = new PythonType(TypeName.DICT) { + @Override + public Map toJava(PythonObject pythonObject) throws PythonException { + if (!Python.isinstance(pythonObject, Python.dictType())) { + throw new PythonException("Expected variable to be dict, but was " + Python.type(pythonObject)); + } + return pythonObject.toMap(); + } + + @Override + public Map convert(Object object) throws PythonException { + if (object instanceof Map) { + return (Map) object; + } + throw new PythonException("Unable to cast " + object + " to Map."); + } + }; + + public static final PythonType NDARRAY = new PythonType(TypeName.NDARRAY) { + @Override + public INDArray toJava(PythonObject pythonObject) throws PythonException { + PythonObject np = importModule("numpy"); + if (!Python.isinstance(pythonObject, np.attr("ndarray"), np.attr("generic"))) { + throw new PythonException("Expected variable to be numpy.ndarray, but was " + Python.type(pythonObject)); + } + return pythonObject.toNumpy().getNd4jArray(); + } + + @Override + public INDArray convert(Object object) throws PythonException { + if (object instanceof INDArray) { + return (INDArray) object; + } else if (object instanceof NumpyArray) { + return ((NumpyArray) object).getNd4jArray(); + } + throw new PythonException("Unable to cast " + object + " to INDArray."); + } + }; + + public static final PythonType BYTES = new PythonType(TypeName.BYTES) { + @Override + public BytePointer toJava(PythonObject pythonObject) throws PythonException { + return pythonObject.toBytePointer(); + } + + @Override + public BytePointer convert(Object object) throws PythonException { + if (object instanceof BytePointer) { + return (BytePointer) object; + } else if (object instanceof Pointer) { + return new BytePointer((Pointer) object); + } + throw new PythonException("Unable to cast " + object + " to BytePointer."); + } + }; +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonUtils.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonUtils.java new file mode 100644 index 000000000..d3e991b35 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonUtils.java @@ -0,0 +1,297 @@ +package org.datavec.python; + +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.metadata.BooleanMetaData; +import org.datavec.api.transform.schema.Schema; +import org.json.JSONArray; +import org.json.JSONObject; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.api.buffer.DataType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * List of utilities for executing python transforms. + * + * @author Adam Gibson + */ +public class PythonUtils { + + /** + * Create a {@link Schema} + * from {@link PythonVariables}. + * Types are mapped to types of the same name. + * + * @param input the input {@link PythonVariables} + * @return the output {@link Schema} + */ + public static Schema fromPythonVariables(PythonVariables input) { + Schema.Builder schemaBuilder = new Schema.Builder(); + Preconditions.checkState(input.getVariables() != null && input.getVariables().length > 0, "Input must have variables. Found none."); + for (String varName: input.getVariables()) { + + switch (input.getType(varName).getName()) { + case INT: + schemaBuilder.addColumnInteger(varName); + break; + case STR: + schemaBuilder.addColumnString(varName); + break; + case FLOAT: + schemaBuilder.addColumnFloat(varName); + break; + case NDARRAY: + schemaBuilder.addColumnNDArray(varName, null); + break; + case BOOL: + schemaBuilder.addColumn(new BooleanMetaData(varName)); + } + } + + return schemaBuilder.build(); + } + + /** + * Create a {@link Schema} from an input + * {@link PythonVariables} + * Types are mapped to types of the same name + * + * @param input the input schema + * @return the output python variables. + */ + public static PythonVariables fromSchema(Schema input) { + PythonVariables ret = new PythonVariables(); + for (int i = 0; i < input.numColumns(); i++) { + String currColumnName = input.getName(i); + ColumnType columnType = input.getType(i); + switch (columnType) { + case NDArray: + ret.add(currColumnName, PythonType.NDARRAY); + break; + case Boolean: + ret.add(currColumnName, PythonType.BOOL); + break; + case Categorical: + case String: + ret.add(currColumnName, PythonType.STR); + break; + case Double: + case Float: + ret.add(currColumnName, PythonType.FLOAT); + break; + case Integer: + case Long: + ret.add(currColumnName, PythonType.INT); + break; + case Bytes: + ret.add(currColumnName, PythonType.BYTES); + break; + case Time: + throw new UnsupportedOperationException("Unable to process dates with python yet."); + } + } + + return ret; + } + + /** + * Convert a {@link Schema} + * to {@link PythonVariables} + * + * @param schema the input schema + * @return the output {@link PythonVariables} where each + * name in the map is associated with a column name in the schema. + * A proper type is also chosen based on the schema + * @throws Exception + */ + public static PythonVariables schemaToPythonVariables(Schema schema) throws Exception { + PythonVariables pyVars = new PythonVariables(); + int numCols = schema.numColumns(); + for (int i = 0; i < numCols; i++) { + String colName = schema.getName(i); + ColumnType colType = schema.getType(i); + switch (colType) { + case Long: + case Integer: + pyVars.addInt(colName); + break; + case Double: + case Float: + pyVars.addFloat(colName); + break; + case String: + pyVars.addStr(colName); + break; + case NDArray: + pyVars.addNDArray(colName); + break; + case Boolean: + pyVars.addBool(colName); + break; + default: + throw new Exception("Unsupported python input type: " + colType.toString()); + } + } + + return pyVars; + } + + + public static NumpyArray mapToNumpyArray(Map map) { + String dtypeName = (String) map.get("dtype"); + DataType dtype; + if (dtypeName.equals("float64")) { + dtype = DataType.DOUBLE; + } else if (dtypeName.equals("float32")) { + dtype = DataType.FLOAT; + } else if (dtypeName.equals("int16")) { + dtype = DataType.SHORT; + } else if (dtypeName.equals("int32")) { + dtype = DataType.INT; + } else if (dtypeName.equals("int64")) { + dtype = DataType.LONG; + } else { + throw new RuntimeException("Unsupported array type " + dtypeName + "."); + } + List shapeList = (List) map.get("shape"); + long[] shape = new long[shapeList.size()]; + for (int i = 0; i < shape.length; i++) { + shape[i] = (Long) shapeList.get(i); + } + + List strideList = (List) map.get("shape"); + long[] stride = new long[strideList.size()]; + for (int i = 0; i < stride.length; i++) { + stride[i] = (Long) strideList.get(i); + } + long address = (Long) map.get("address"); + NumpyArray numpyArray = new NumpyArray(address, shape, stride, dtype, true); + return numpyArray; + } + + public static PythonVariables expandInnerDict(PythonVariables pyvars, String key) { + Map dict = pyvars.getDictValue(key); + String[] keys = (String[]) dict.keySet().toArray(new String[dict.keySet().size()]); + PythonVariables pyvars2 = new PythonVariables(); + for (String subkey : keys) { + Object value = dict.get(subkey); + if (value instanceof Map) { + Map map = (Map) value; + if (map.containsKey("_is_numpy_array")) { + pyvars2.addNDArray(subkey, mapToNumpyArray(map)); + + } else { + pyvars2.addDict(subkey, (Map) value); + } + + } else if (value instanceof List) { + pyvars2.addList(subkey, ((List) value).toArray()); + } else if (value instanceof String) { + System.out.println((String) value); + pyvars2.addStr(subkey, (String) value); + } else if (value instanceof Integer || value instanceof Long) { + Number number = (Number) value; + pyvars2.addInt(subkey, number.intValue()); + } else if (value instanceof Float || value instanceof Double) { + Number number = (Number) value; + pyvars2.addFloat(subkey, number.doubleValue()); + } else if (value instanceof NumpyArray) { + pyvars2.addNDArray(subkey, (NumpyArray) value); + } else if (value == null) { + pyvars2.addStr(subkey, "None"); // FixMe + } else { + throw new RuntimeException("Unsupported type!" + value); + } + } + return pyvars2; + } + + public static long[] jsonArrayToLongArray(JSONArray jsonArray) { + long[] longs = new long[jsonArray.length()]; + for (int i = 0; i < longs.length; i++) { + + longs[i] = jsonArray.getLong(i); + } + return longs; + } + + public static Map toMap(JSONObject jsonobj) { + Map map = new HashMap<>(); + String[] keys = (String[]) jsonobj.keySet().toArray(new String[jsonobj.keySet().size()]); + for (String key : keys) { + Object value = jsonobj.get(key); + if (value instanceof JSONArray) { + value = toList((JSONArray) value); + } else if (value instanceof JSONObject) { + JSONObject jsonobj2 = (JSONObject) value; + if (jsonobj2.has("_is_numpy_array")) { + value = jsonToNumpyArray(jsonobj2); + } else { + value = toMap(jsonobj2); + } + + } + + map.put(key, value); + } + return map; + } + + + public static List toList(JSONArray array) { + List list = new ArrayList<>(); + for (int i = 0; i < array.length(); i++) { + Object value = array.get(i); + if (value instanceof JSONArray) { + value = toList((JSONArray) value); + } else if (value instanceof JSONObject) { + JSONObject jsonobj2 = (JSONObject) value; + if (jsonobj2.has("_is_numpy_array")) { + value = jsonToNumpyArray(jsonobj2); + } else { + value = toMap(jsonobj2); + } + } + list.add(value); + } + return list; + } + + + private static NumpyArray jsonToNumpyArray(JSONObject map) { + String dtypeName = (String) map.get("dtype"); + DataType dtype; + if (dtypeName.equals("float64")) { + dtype = DataType.DOUBLE; + } else if (dtypeName.equals("float32")) { + dtype = DataType.FLOAT; + } else if (dtypeName.equals("int16")) { + dtype = DataType.SHORT; + } else if (dtypeName.equals("int32")) { + dtype = DataType.INT; + } else if (dtypeName.equals("int64")) { + dtype = DataType.LONG; + } else { + throw new RuntimeException("Unsupported array type " + dtypeName + "."); + } + List shapeList = map.getJSONArray("shape").toList(); + long[] shape = new long[shapeList.size()]; + for (int i = 0; i < shape.length; i++) { + shape[i] = ((Number) shapeList.get(i)).longValue(); + } + + List strideList = map.getJSONArray("shape").toList(); + long[] stride = new long[strideList.size()]; + for (int i = 0; i < stride.length; i++) { + stride[i] = ((Number) strideList.get(i)).longValue(); + } + long address = ((Number) map.get("address")).longValue(); + NumpyArray numpyArray = new NumpyArray(address, shape, stride, dtype, true); + return numpyArray; + } + + +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonVariables.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonVariables.java new file mode 100644 index 000000000..ade9bdfa0 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/PythonVariables.java @@ -0,0 +1,533 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import lombok.Data; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.Pointer; +import org.json.JSONObject; +import org.json.JSONArray; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.nativeblas.NativeOpsHolder; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.*; + + + +/** + * Holds python variable names, types and values. + * Also handles mapping from java types to python types. + * + * @author Fariz Rahman + */ + +@lombok.Data +public class PythonVariables implements java.io.Serializable { + + + private java.util.Map strVariables = new java.util.LinkedHashMap<>(); + private java.util.Map intVariables = new java.util.LinkedHashMap<>(); + private java.util.Map floatVariables = new java.util.LinkedHashMap<>(); + private java.util.Map boolVariables = new java.util.LinkedHashMap<>(); + private java.util.Map ndVars = new java.util.LinkedHashMap<>(); + private java.util.Map listVariables = new java.util.LinkedHashMap<>(); + private java.util.Map bytesVariables = new java.util.LinkedHashMap<>(); + private java.util.Map> dictVariables = new java.util.LinkedHashMap<>(); + private java.util.Map vars = new java.util.LinkedHashMap<>(); + private java.util.Map maps = new java.util.LinkedHashMap<>(); + + + /** + * Returns a copy of the variable + * schema in this array without the values + * + * @return an empty variables clone + * with no values + */ + public PythonVariables copySchema() { + PythonVariables ret = new PythonVariables(); + for (String varName : getVariables()) { + PythonType type = getType(varName); + ret.add(varName, type); + } + return ret; + } + + /** + * + */ + public PythonVariables() { + maps.put(PythonType.TypeName.BOOL, boolVariables); + maps.put(PythonType.TypeName.STR, strVariables); + maps.put(PythonType.TypeName.INT, intVariables); + maps.put(PythonType.TypeName.FLOAT, floatVariables); + maps.put(PythonType.TypeName.NDARRAY, ndVars); + maps.put(PythonType.TypeName.LIST, listVariables); + maps.put(PythonType.TypeName.DICT, dictVariables); + maps.put(PythonType.TypeName.BYTES, bytesVariables); + + } + + + /** + * @return true if there are no variables. + */ + public boolean isEmpty() { + return getVariables().length < 1; + } + + + /** + * @param name Name of the variable + * @param type Type of the variable + */ + public void add(String name, PythonType type) { + switch (type.getName()) { + case BOOL: + addBool(name); + break; + case STR: + addStr(name); + break; + case INT: + addInt(name); + break; + case FLOAT: + addFloat(name); + break; + case NDARRAY: + addNDArray(name); + break; + case LIST: + addList(name); + break; + case DICT: + addDict(name); + break; + case BYTES: + addBytes(name); + break; + } + } + + /** + * @param name name of the variable + * @param type type of the variable + * @param value value of the variable (must be instance of expected type) + */ + public void add(String name, PythonType type, Object value) throws PythonException { + add(name, type); + setValue(name, value); + } + + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + */ + public void addDict(String name) { + vars.put(name, PythonType.TypeName.DICT); + dictVariables.put(name, null); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + */ + public void addBool(String name) { + vars.put(name, PythonType.TypeName.BOOL); + boolVariables.put(name, null); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + */ + public void addStr(String name) { + vars.put(name, PythonType.TypeName.STR); + strVariables.put(name, null); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + */ + public void addInt(String name) { + vars.put(name, PythonType.TypeName.INT); + intVariables.put(name, null); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + */ + public void addFloat(String name) { + vars.put(name, PythonType.TypeName.FLOAT); + floatVariables.put(name, null); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + */ + public void addNDArray(String name) { + vars.put(name, PythonType.TypeName.NDARRAY); + ndVars.put(name, null); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + */ + public void addList(String name) { + vars.put(name, PythonType.TypeName.LIST); + listVariables.put(name, null); + } + + /** + * Add a boolean variable to + * the set of variables + * + * @param name the field to add + * @param value the value to add + */ + public void addBool(String name, boolean value) { + vars.put(name, PythonType.TypeName.BOOL); + boolVariables.put(name, value); + } + + /** + * Add a string variable to + * the set of variables + * + * @param name the field to add + * @param value the value to add + */ + public void addStr(String name, String value) { + vars.put(name, PythonType.TypeName.STR); + strVariables.put(name, value); + } + + /** + * Add an int variable to + * the set of variables + * + * @param name the field to add + * @param value the value to add + */ + public void addInt(String name, int value) { + vars.put(name, PythonType.TypeName.INT); + intVariables.put(name, (long) value); + } + + /** + * Add a long variable to + * the set of variables + * + * @param name the field to add + * @param value the value to add + */ + public void addInt(String name, long value) { + vars.put(name, PythonType.TypeName.INT); + intVariables.put(name, value); + } + + /** + * Add a double variable to + * the set of variables + * + * @param name the field to add + * @param value the value to add + */ + public void addFloat(String name, double value) { + vars.put(name, PythonType.TypeName.FLOAT); + floatVariables.put(name, value); + } + + /** + * Add a float variable to + * the set of variables + * + * @param name the field to add + * @param value the value to add + */ + public void addFloat(String name, float value) { + vars.put(name, PythonType.TypeName.FLOAT); + floatVariables.put(name, (double) value); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + * @param value the value to add + */ + public void addNDArray(String name, NumpyArray value) { + vars.put(name, PythonType.TypeName.NDARRAY); + ndVars.put(name, value.getNd4jArray()); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + * @param value the value to add + */ + public void addNDArray(String name, INDArray value) { + vars.put(name, PythonType.TypeName.NDARRAY); + ndVars.put(name, value); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + * @param value the value to add + */ + public void addList(String name, Object[] value) { + vars.put(name, PythonType.TypeName.LIST); + listVariables.put(name, Arrays.asList(value)); + } + + /** + * Add a null variable to + * the set of variables + * to describe the type but no value + * + * @param name the field to add + * @param value the value to add + */ + public void addDict(String name, java.util.Map value) { + vars.put(name, PythonType.TypeName.DICT); + dictVariables.put(name, value); + } + + + public void addBytes(String name){ + vars.put(name, PythonType.TypeName.BYTES); + bytesVariables.put(name, null); + } + + public void addBytes(String name, BytePointer value){ + vars.put(name, PythonType.TypeName.BYTES); + bytesVariables.put(name, value); + } + +// public void addBytes(String name, ByteBuffer value){ +// Pointer ptr = NativeOpsHolder.getInstance().getDeviceNativeOps().pointerForAddress((value.address()); +// BytePointer bp = new BytePointer(ptr); +// addBytes(name, bp); +// } + /** + * @param name name of the variable + * @param value new value for the variable + */ + public void setValue(String name, Object value) throws PythonException { + PythonType.TypeName type = vars.get(name); + maps.get(type).put(name, PythonType.valueOf(type).convert(value)); + } + + /** + * Do a general object lookup. + * The look up will happen relative to the {@link PythonType} + * of variable is described in the + * + * @param name the name of the variable to get + * @return teh value for the variable with the given name + */ + public Object getValue(String name) { + PythonType.TypeName type = vars.get(name); + java.util.Map map = maps.get(type); + return map.get(name); + } + + + /** + * Returns a boolean variable with the given name. + * + * @param name the variable name to get the value for + * @return the retrieved boolean value + */ + public boolean getBooleanValue(String name) { + return boolVariables.get(name); + } + + /** + * @param name the variable name + * @return the dictionary value + */ + public java.util.Map getDictValue(String name) { + return dictVariables.get(name); + } + + /** + * /** + * + * @param name the variable name + * @return the string value + */ + public String getStrValue(String name) { + return strVariables.get(name); + } + + /** + * @param name the variable name + * @return the long value + */ + public Long getIntValue(String name) { + return intVariables.get(name); + } + + /** + * @param name the variable name + * @return the float value + */ + public Double getFloatValue(String name) { + return floatVariables.get(name); + } + + /** + * @param name the variable name + * @return the numpy array value + */ + public INDArray getNDArrayValue(String name) { + return ndVars.get(name); + } + + /** + * @param name the variable name + * @return the list value as an object array + */ + public List getListValue(String name) { + return listVariables.get(name); + } + + /** + * @param name the variable name + * @return the bytes value as a BytePointer + */ + public BytePointer getBytesValue(String name){return bytesVariables.get(name);} + /** + * Returns the type for the given variable name + * + * @param name the name of the variable to get the type for + * @return the type for the given variable + */ + public PythonType getType(String name){ + try{ + return PythonType.valueOf(vars.get(name)); // will never fail + }catch (Exception e) + { + throw new RuntimeException(e); + } + } + + /** + * Get all the variables present as a string array + * + * @return the variable names for this variable sset + */ + public String[] getVariables() { + String[] strArr = new String[vars.size()]; + return vars.keySet().toArray(strArr); + } + + + /** + * This variables set as its json representation (an array of json objects) + * + * @return the json array output + */ + public org.json.JSONArray toJSON() { + org.json.JSONArray arr = new org.json.JSONArray(); + for (String varName : getVariables()) { + org.json.JSONObject var = new org.json.JSONObject(); + var.put("name", varName); + String varType = getType(varName).toString(); + var.put("type", varType); + arr.put(var); + } + return arr; + } + + /** + * Create a schema from a map. + * This is an empty PythonVariables + * that just contains names and types with no values + * + * @param inputTypes the input types to convert + * @return the schema from the given map + */ + public static PythonVariables schemaFromMap(java.util.Map inputTypes) throws Exception{ + PythonVariables ret = new PythonVariables(); + for (java.util.Map.Entry entry : inputTypes.entrySet()) { + ret.add(entry.getKey(), PythonType.valueOf(entry.getValue())); + } + + return ret; + } + + /** + * Get the python variable state relative to the + * input json array + * + * @param jsonArray the input json array + * @return the python variables based on the input json array + */ + public static PythonVariables fromJSON(org.json.JSONArray jsonArray) { + PythonVariables pyvars = new PythonVariables(); + for (int i = 0; i < jsonArray.length(); i++) { + org.json.JSONObject input = (org.json.JSONObject) jsonArray.get(i); + String varName = (String) input.get("name"); + String varType = (String) input.get("type"); + pyvars.maps.get(PythonType.TypeName.valueOf(varType)).put(varName, null); + } + + return pyvars; + } + + +} \ No newline at end of file diff --git a/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/keras/Model.java b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/keras/Model.java new file mode 100644 index 000000000..d8a9b0651 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/java/org/datavec/python/keras/Model.java @@ -0,0 +1,144 @@ +package org.datavec.python.keras; + +import org.datavec.python.Python; +import org.datavec.python.PythonException; +import org.datavec.python.PythonObject; +import org.datavec.python.PythonProcess; +import org.nd4j.linalg.api.ndarray.INDArray; + +public class Model { + + private PythonObject pyModel; + + + private static PythonObject installAndImportTF() throws PythonException{ + if (!PythonProcess.isPackageInstalled("tensorflow")){ + PythonProcess.pipInstall("tensorflow"); + } + return Python.importModule("tensorflow"); + } + private static PythonObject getKerasModule() throws PythonException{ + PythonObject tf = installAndImportTF(); + PythonObject keras = tf.attr("keras"); + tf.del(); + return keras; + } + + private static PythonObject loadModel(String s) throws PythonException{ + PythonObject models = getKerasModule().attr("models"); + PythonObject loadModelF = models.attr("load_model"); + PythonObject model = loadModelF.call(s); + models.del(); + loadModelF.del(); + return model; + } + + public Model(String path) throws PythonException{ + pyModel = loadModel(path); + } + + public INDArray[] predict(INDArray... inputs) throws PythonException{ + PythonObject predictF = pyModel.attr("predict"); + PythonObject inputList = new PythonObject(inputs); + PythonObject pyOut = predictF.call(inputList); + INDArray[] out; + if (Python.isinstance(pyOut, Python.listType())){ + out = new INDArray[Python.len(pyOut).toInt()]; + for(int i = 0; i < out.length; i++){ + out[i] = pyOut.get(i).toNumpy().getNd4jArray(); + } + } + else{ + out = new INDArray[]{ + pyOut.toNumpy().getNd4jArray()}; + } + + predictF.del(); + inputList.del(); + pyOut.del(); + return out; + } + + public int numInputs(){ + PythonObject inputs = pyModel.attr("inputs"); + PythonObject pyNumInputs = Python.len(inputs); + int ret = pyNumInputs.toInt(); + inputs.del(); + pyNumInputs.del(); + return ret; + } + public int numOutputs(){ + PythonObject outputs = pyModel.attr("outputs"); + PythonObject pyNumOutputs = Python.len(outputs); + int ret = pyNumOutputs.toInt(); + outputs.del(); + pyNumOutputs.del(); + return ret; + } + + public long[][] inputShapes(){ + long[][] ret = new long[numInputs()][]; + for (int i = 0; i < ret.length; i++){ + ret[i] = inputShapeAt(i); + } + return ret; + } + + public long[][] outputShapes(){ + long[][] ret = new long[numOutputs()][]; + for (int i = 0; i < ret.length; i++){ + ret[i] = outputShapeAt(i); + } + return ret; + } + + public long[] inputShapeAt(int input){ + PythonObject inputs = pyModel.attr("inputs"); + PythonObject tensor = inputs.get(input); + PythonObject tensorShape = tensor.attr("shape"); + PythonObject shapeList = Python.list(tensorShape); + PythonObject pyNdim = Python.len(shapeList); + int ndim = pyNdim.toInt(); + long[] shape = new long[ndim]; + for(int i = 0; i < shape.length; i++){ + PythonObject pyDim = shapeList.get(i); + if (pyDim == null || !Python.isinstance(pyDim, Python.intType())){ + shape[i] = -1; + } + else{ + shape[i] = pyDim.toLong(); + } + } + pyNdim.del(); + shapeList.del(); + tensorShape.del(); + tensor.del(); + inputs.del(); + return shape; + } + + public long[] outputShapeAt(int output){ + PythonObject inputs = pyModel.attr("outputs"); + PythonObject tensor = inputs.get(output); + PythonObject tensorShape = tensor.attr("shape"); + PythonObject shapeList = Python.list(tensorShape); + PythonObject pyNdim = Python.len(shapeList); + int ndim = pyNdim.toInt(); + long[] shape = new long[ndim]; + for(int i = 0; i < shape.length; i++){ + PythonObject pyDim = shapeList.get(i); + if (pyDim == null || !Python.isinstance(pyDim, Python.intType())){ + shape[i] = -1; + } + else{ + shape[i] = pyDim.toLong(); + } + } + pyNdim.del(); + shapeList.del(); + tensorShape.del(); + tensor.del(); + inputs.del(); + return shape; + } +} diff --git a/cavis-datavec/cavis-datavec-python/src/main/resources/pythonexec/pythonexec.py b/cavis-datavec/cavis-datavec-python/src/main/resources/pythonexec/pythonexec.py new file mode 100644 index 000000000..1509610c7 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/main/resources/pythonexec/pythonexec.py @@ -0,0 +1,20 @@ +import sys +import traceback +import json +import inspect + +__python_exception__ = "" +try: + pass + sys.stdout.flush() + sys.stderr.flush() +except Exception as ex: + __python_exception__ = ex + try: + exc_info = sys.exc_info() + finally: + print(ex) + traceback.print_exception(*exc_info) + sys.stdout.flush() + sys.stderr.flush() + diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..50e46e8f7 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.python; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseND4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.python"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/PythonNumpyTest.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/PythonNumpyTest.java new file mode 100644 index 000000000..1c0721c32 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/PythonNumpyTest.java @@ -0,0 +1,74 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +//@RunWith(Parameterized.class) +public class PythonNumpyTest { + + //@Parameterized.Parameters(name = "{index}: Testing with DataType={0}") + public static DataType[] data() { + return new DataType[] { + DataType.BOOL, + DataType.FLOAT16, + DataType.BFLOAT16, + DataType.FLOAT, + DataType.DOUBLE, + DataType.INT8, + DataType.INT16, + DataType.INT32, + DataType.INT64, + DataType.UINT8, + DataType.UINT16, + DataType.UINT32, + DataType.UINT64 + }; + } + + private DataType dataType; + + public PythonNumpyTest(DataType dataType) { + this.dataType = dataType; + } + + @Test + public void numpyAndNd4jConversions() throws Exception { + INDArray input = Nd4j.ones(dataType, 2, 2, 2); + + PythonVariables inputs = new PythonVariables(); + inputs.addNDArray("x", input); + + PythonVariables outputs = new PythonVariables(); + outputs.addNDArray("y"); + + PythonJob pythonJob = new PythonJob(String.format("job_%s", dataType.name()) + dataType.name(), "y = x", false); + + pythonJob.exec(inputs, outputs); + + INDArray output = outputs.getNDArrayValue("y"); + + // As numpy doesn't support BFLOAT16 we'll convert it to FLOAT + assertEquals(dataType == DataType.BFLOAT16 ? input.castTo(DataType.FLOAT) : input, + output); + } +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/ScalarAndArrayTest.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/ScalarAndArrayTest.java new file mode 100644 index 000000000..f6f39d68c --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/ScalarAndArrayTest.java @@ -0,0 +1,48 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import org.junit.jupiter.api.Test; + + +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +//@RunWith(Parameterized.class) +public class ScalarAndArrayTest { + + //@Parameterized.Parameters(name = "{index}: Testing with INDArray={0}") + public static INDArray[] data() { + return new INDArray[]{ + Nd4j.scalar(10), + Nd4j.ones(10, 10, 10, 10) + }; + } + + private INDArray indArray; + + public ScalarAndArrayTest(INDArray indArray) { + this.indArray = indArray; + } + + @Test + public void testINDArray() throws PythonException { + assertEquals(indArray, new PythonObject(indArray).toNumpy().getNd4jArray()); + } +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonContextManager.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonContextManager.java new file mode 100644 index 000000000..c185a04ec --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonContextManager.java @@ -0,0 +1,87 @@ + +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import javax.annotation.concurrent.NotThreadSafe; + +@NotThreadSafe +public class TestPythonContextManager { + + @Test + public void testInt() throws Exception{ + Python.setContext("context1"); + Python.exec("a = 1"); + Python.setContext("context2"); + Python.exec("a = 2"); + Python.setContext("context3"); + Python.exec("a = 3"); + + + Python.setContext("context1"); + Assertions.assertEquals(1, PythonExecutioner.getVariable("a").toInt()); + + Python.setContext("context2"); + Assertions.assertEquals(2, PythonExecutioner.getVariable("a").toInt()); + + Python.setContext("context3"); + Assertions.assertEquals(3, PythonExecutioner.getVariable("a").toInt()); + + PythonContextManager.deleteNonMainContexts(); + } + + @Test + public void testNDArray() throws Exception{ + Python.setContext("context1"); + Python.exec("import numpy as np"); + Python.exec("a = np.zeros((3,2)) + 1"); + + Python.setContext("context2"); + Python.exec("import numpy as np"); + Python.exec("a = np.zeros((3,2)) + 2"); + + Python.setContext("context3"); + Python.exec("import numpy as np"); + Python.exec("a = np.zeros((3,2)) + 3"); + + Python.setContext("context1"); + Python.exec("a += 1"); + + Python.setContext("context2"); + Python.exec("a += 2"); + + Python.setContext("context3"); + Python.exec("a += 3"); + + INDArray arr = Nd4j.create(DataType.DOUBLE, 3, 2); + Python.setContext("context1"); + Assertions.assertEquals(arr.add(2), PythonExecutioner.getVariable("a").toNumpy().getNd4jArray()); + + Python.setContext("context2"); + Assertions.assertEquals(arr.add(4), PythonExecutioner.getVariable("a").toNumpy().getNd4jArray()); + + Python.setContext("context3"); + Assertions.assertEquals(arr.add(6), PythonExecutioner.getVariable("a").toNumpy().getNd4jArray()); + } + +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonDict.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonDict.java new file mode 100644 index 000000000..e28695961 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonDict.java @@ -0,0 +1,59 @@ + +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@javax.annotation.concurrent.NotThreadSafe +public class TestPythonDict { + + @Test + public void testPythonDictFromMap() throws Exception{ + Map map = new HashMap<>(); + map.put("a", 1); + map.put("b", "a"); + map.put("1", Arrays.asList(1, 2, 3, "4", Arrays.asList("x", 2.3))); + Map innerMap = new HashMap<>(); + innerMap.put("k", 32); + map.put("inner", innerMap); + map.put("ndarray", Nd4j.linspace(1, 4, 4)); + innerMap.put("ndarray", Nd4j.linspace(5, 8, 4)); + PythonObject dict = new PythonObject(map); + assertEquals(map.size(), Python.len(dict).toInt()); + assertEquals("{'a': 1, '1': [1, 2, 3, '4', ['" + + "x', 2.3]], 'b': 'a', 'inner': {'k': 32," + + " 'ndarray': array([5., 6., 7., 8.], dty" + + "pe=float32)}, 'ndarray': array([1., 2., " + + "3., 4.], dtype=float32)}", + dict.toString()); + Map map2 = dict.toMap(); + PythonObject dict2 = new PythonObject(map2); + assertEquals(dict.toString(), dict2.toString()); + + + } + +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonExecutioner.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonExecutioner.java new file mode 100644 index 000000000..290236792 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonExecutioner.java @@ -0,0 +1,414 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import org.bytedeco.javacpp.BytePointer; +import org.junit.jupiter.api.Assertions; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.BaseDataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.nativeblas.OpaqueDataBuffer; + +import java.lang.reflect.Method; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + + +@javax.annotation.concurrent.NotThreadSafe +public class TestPythonExecutioner { + + + @org.junit.jupiter.api.Test + public void testPythonSysVersion() throws PythonException { + Python.exec("import sys; print(sys.version)"); + } + + @Test + public void testStr() throws Exception { + + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + pyInputs.addStr("x", "Hello"); + pyInputs.addStr("y", "World"); + + pyOutputs.addStr("z"); + + String code = "z = x + ' ' + y"; + + Python.exec(code, pyInputs, pyOutputs); + + String z = pyOutputs.getStrValue("z"); + + System.out.println(z); + + assertEquals("Hello World", z); + } + + @Test + public void testInt() throws Exception { + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + pyInputs.addInt("x", 10); + pyInputs.addInt("y", 20); + + String code = "z = x + y"; + + pyOutputs.addInt("z"); + + + Python.exec(code, pyInputs, pyOutputs); + + long z = pyOutputs.getIntValue("z"); + + Assertions.assertEquals(30, z); + + } + + @Test + public void testList() throws Exception { + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + Object[] x = new Object[]{1L, 2L, 3L, "a", "b", "c"}; + Object[] y = new Object[]{4L, 5L, 6L, "d", "e", "f"}; + + pyInputs.addList("x", x); + pyInputs.addList("y", y); + + String code = "z = x + y"; + + pyOutputs.addList("z"); + + + Python.exec(code, pyInputs, pyOutputs); + + Object[] z = pyOutputs.getListValue("z").toArray(); + + Assertions.assertEquals(z.length, x.length + y.length); + + for (int i = 0; i < x.length; i++) { + if (x[i] instanceof Number) { + Number xNum = (Number) x[i]; + Number zNum = (Number) z[i]; + Assertions.assertEquals(xNum.intValue(), zNum.intValue()); + } else { + Assertions.assertEquals(x[i], z[i]); + } + + } + for (int i = 0; i < y.length; i++) { + if (y[i] instanceof Number) { + Number yNum = (Number) y[i]; + Number zNum = (Number) z[x.length + i]; + Assertions.assertEquals(yNum.intValue(), zNum.intValue()); + } else { + Assertions.assertEquals(y[i], z[x.length + i]); + + } + + } + + } + + @Test + public void testNDArrayFloat() throws Exception { + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + pyInputs.addNDArray("x", Nd4j.zeros(DataType.FLOAT, 2, 3)); + pyInputs.addNDArray("y", Nd4j.ones(DataType.FLOAT, 2, 3)); + pyOutputs.addNDArray("z"); + + String code = "z = x + y"; + + Python.exec(code, pyInputs, pyOutputs); + INDArray z = pyOutputs.getNDArrayValue("z"); + + Assertions.assertEquals(6.0, z.sum().getDouble(0), 1e-5); + + + } + + @Test + //@Ignore + public void testTensorflowCustomAnaconda() throws PythonException { + Python.exec("import tensorflow as tf"); + } + + @Test + public void testNDArrayDouble() throws Exception { + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + pyInputs.addNDArray("x", Nd4j.zeros(DataType.DOUBLE, 2, 3)); + pyInputs.addNDArray("y", Nd4j.ones(DataType.DOUBLE, 2, 3)); + pyOutputs.addNDArray("z"); + + String code = "z = x + y"; + + Python.exec(code, pyInputs, pyOutputs); + INDArray z = pyOutputs.getNDArrayValue("z"); + + Assertions.assertEquals(6.0, z.sum().getDouble(0), 1e-5); + } + + @Test + public void testNDArrayShort() throws Exception { + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + pyInputs.addNDArray("x", Nd4j.zeros(DataType.SHORT, 2, 3)); + pyInputs.addNDArray("y", Nd4j.ones(DataType.SHORT, 2, 3)); + pyOutputs.addNDArray("z"); + + String code = "z = x + y"; + + Python.exec(code, pyInputs, pyOutputs); + INDArray z = pyOutputs.getNDArrayValue("z"); + + Assertions.assertEquals(6.0, z.sum().getDouble(0), 1e-5); + } + + + @Test + public void testNDArrayInt() throws Exception { + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + pyInputs.addNDArray("x", Nd4j.zeros(DataType.INT, 2, 3)); + pyInputs.addNDArray("y", Nd4j.ones(DataType.INT, 2, 3)); + pyOutputs.addNDArray("z"); + + String code = "z = x + y"; + + Python.exec(code, pyInputs, pyOutputs); + INDArray z = pyOutputs.getNDArrayValue("z"); + + Assertions.assertEquals(6.0, z.sum().getDouble(0), 1e-5); + + } + + @Test + public void testNDArrayLong() throws Exception { + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + pyInputs.addNDArray("x", Nd4j.zeros(DataType.LONG, 2, 3)); + pyInputs.addNDArray("y", Nd4j.ones(DataType.LONG, 2, 3)); + pyOutputs.addNDArray("z"); + + String code = "z = x + y"; + + Python.exec(code, pyInputs, pyOutputs); + INDArray z = pyOutputs.getNDArrayValue("z"); + + Assertions.assertEquals(6.0, z.sum().getDouble(0), 1e-5); + + } + + + @Test + public void testNDArrayNoCopy() throws Exception{ + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + INDArray arr = Nd4j.rand(3, 2); + ((BaseDataBuffer)arr.data()).syncToPrimary(); + pyInputs.addNDArray("x", arr); + pyOutputs.addNDArray("x"); + INDArray expected = arr.mul(2.3); + String code = "x *= 2.3"; + Python.exec(code, pyInputs, pyOutputs); + Assertions.assertEquals(pyInputs.getNDArrayValue("x"), pyOutputs.getNDArrayValue("x")); + Assertions.assertEquals(expected, pyOutputs.getNDArrayValue("x")); + Assertions.assertEquals(arr.data().address(), pyOutputs.getNDArrayValue("x").data().address()); + } + + @Test + public void testNDArrayInplace() throws Exception{ + PythonVariables pyInputs = new PythonVariables(); + INDArray arr = Nd4j.rand(3, 2); + ((BaseDataBuffer)arr.data()).syncToPrimary(); + pyInputs.addNDArray("x", arr); + INDArray expected = arr.mul(2.3); + String code = "x *= 2.3"; + Python.exec(code, pyInputs, null); + Assertions.assertEquals(expected, arr); + } + + @Test + public void testByteBufferInput() throws Exception{ + //ByteBuffer buff = ByteBuffer.allocateDirect(3); + INDArray buff = Nd4j.zeros(new int[]{3}, DataType.BYTE); + buff.putScalar(0, 97); // a + buff.putScalar(1, 98); // b + buff.putScalar(2, 99); // c + ((BaseDataBuffer)buff.data()).syncToPrimary(); + PythonVariables pyInputs = new PythonVariables(); + pyInputs.addBytes("buff", new BytePointer(buff.data().pointer())); + + PythonVariables pyOutputs= new PythonVariables(); + pyOutputs.addStr("out"); + + String code = "out = bytes(buff).decode()"; + Python.exec(code, pyInputs, pyOutputs); + Assertions.assertEquals("abc", pyOutputs.getStrValue("out")); + + } + + + @Test + public void testByteBufferOutputNoCopy() throws Exception{ + INDArray buff = Nd4j.zeros(new int[]{3}, DataType.BYTE); + buff.putScalar(0, 97); // a + buff.putScalar(1, 98); // b + buff.putScalar(2, 99); // c + ((BaseDataBuffer)buff.data()).syncToPrimary(); + + + PythonVariables pyInputs = new PythonVariables(); + pyInputs.addBytes("buff", new BytePointer(buff.data().pointer())); + + PythonVariables pyOutputs = new PythonVariables(); + pyOutputs.addBytes("buff"); // same name as input, because inplace update + + String code = "buff[0]=99\nbuff[1]=98\nbuff[2]=97"; + Python.exec(code, pyInputs, pyOutputs); + Assertions.assertEquals("cba", pyOutputs.getBytesValue("buff").getString()); + Assertions.assertEquals(buff.data().address(), pyOutputs.getBytesValue("buff").address()); + } + + @Test + public void testByteBufferInplace() throws Exception{ + INDArray buff = Nd4j.zeros(new int[]{3}, DataType.BYTE); + buff.putScalar(0, 97); // a + buff.putScalar(1, 98); // b + buff.putScalar(2, 99); // c + ((BaseDataBuffer)buff.data()).syncToPrimary(); + + PythonVariables pyInputs = new PythonVariables(); + pyInputs.addBytes("buff", new BytePointer(buff.data().pointer())); + String code = "buff[0]+=2\nbuff[2]-=2"; + Python.exec(code, pyInputs, null); + Assertions.assertEquals("cba", pyInputs.getBytesValue("buff").getString()); + INDArray expected = buff.dup(); + expected.putScalar(0, 99); + expected.putScalar(2, 97); + Assertions.assertEquals(buff, expected); + + } + + @Test + public void testByteBufferOutputWithCopy() throws Exception{ + INDArray buff = Nd4j.zeros(new int[]{3}, DataType.BYTE); + buff.putScalar(0, 97); // a + buff.putScalar(1, 98); // b + buff.putScalar(2, 99); // c + ((BaseDataBuffer)buff.data()).syncToPrimary(); + + + PythonVariables pyInputs = new PythonVariables(); + pyInputs.addBytes("buff", new BytePointer(buff.data().pointer())); + + PythonVariables pyOutputs = new PythonVariables(); + pyOutputs.addBytes("out"); + + String code = "buff[0]=99\nbuff[1]=98\nbuff[2]=97\nout=bytes(buff)"; + Python.exec(code, pyInputs, pyOutputs); + Assertions.assertEquals("cba", pyOutputs.getBytesValue("out").getString()); + } + + @Test + public void testDoubleDeviceAllocation() throws Exception{ + if(!"CUDA".equalsIgnoreCase(Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"))){ + return; + } + // Test to make sure that multiple device buffers are not allocated + // for the same host buffer + INDArray arr = Nd4j.rand(3, 2); + ((BaseDataBuffer)arr.data()).syncToPrimary(); + long deviceAddress1 = getDeviceAddress(arr); + PythonVariables pyInputs = new PythonVariables(); + pyInputs.addNDArray("arr", arr); + PythonVariables pyOutputs = new PythonVariables(); + pyOutputs.addNDArray("arr"); + String code = "arr += 2"; + Python.exec(code, pyInputs, pyOutputs); + INDArray arr2 = pyOutputs.getNDArrayValue("arr"); + long deviceAddress2 = getDeviceAddress(arr2); + Assertions.assertEquals(deviceAddress1, deviceAddress2); + + + } + + @Test + public void testBadCode() throws Exception{ + Python.setContext("badcode"); + PythonVariables pyInputs = new PythonVariables(); + PythonVariables pyOutputs = new PythonVariables(); + + pyInputs.addNDArray("x", Nd4j.zeros(DataType.LONG, 2, 3)); + pyInputs.addNDArray("y", Nd4j.ones(DataType.LONG, 2, 3)); + pyOutputs.addNDArray("z"); + + String code = "z = x + a"; + + try{ + Python.exec(code, pyInputs, pyOutputs); + fail("No exception thrown"); + } catch (PythonException pe ){ + Assertions.assertEquals("NameError: name 'a' is not defined", pe.getMessage()); + } + + Python.setMainContext(); + } + + @Test + public void testIsNone(){ + PythonObject d = Python.dict(); + PythonObject none = d.attr("get").call("x"); + Assertions.assertTrue(none.isNone()); + d.set(new PythonObject("x"), new PythonObject("y")); + PythonObject notNone = d.attr("get").call("x"); + Assertions.assertFalse(notNone.isNone()); + Assertions.assertEquals("y", notNone.toString()); + } + + private static long getDeviceAddress(INDArray array){ + if(!"CUDA".equalsIgnoreCase(Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"))){ + throw new IllegalStateException("Cannot ge device pointer for non-CUDA device"); + } + + //Use reflection here as OpaqueDataBuffer is only available on BaseCudaDataBuffer and BaseCpuDataBuffer - not DataBuffer/BaseDataBuffer + // due to it being defined in nd4j-native-api, not nd4j-api + try { + Class c = Class.forName("org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer"); + Method m = c.getMethod("getOpaqueDataBuffer"); + OpaqueDataBuffer db = (OpaqueDataBuffer) m.invoke(array.data()); + long address = db.specialBuffer().address(); + return address; + } catch (Throwable t){ + throw new RuntimeException("Error getting OpaqueDataBuffer", t); + } + } + +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonJob.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonJob.java new file mode 100644 index 000000000..37e04a6cc --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonJob.java @@ -0,0 +1,326 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + + +@javax.annotation.concurrent.NotThreadSafe +public class TestPythonJob { + + @Test + public void testPythonJobBasic() throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code = "c = a + b"; + PythonJob job = new PythonJob("job1", code, false); + + PythonVariables inputs = new PythonVariables(); + inputs.addInt("a", 2); + inputs.addInt("b", 3); + + PythonVariables outputs = new PythonVariables(); + outputs.addInt("c"); + + job.exec(inputs, outputs); + + assertEquals(5L, (long)outputs.getIntValue("c")); + + inputs = new PythonVariables(); + inputs.addFloat("a", 3.0); + inputs.addFloat("b", 4.0); + + outputs = new PythonVariables(); + outputs.addFloat("c"); + + + job.exec(inputs, outputs); + + assertEquals(7.0, outputs.getFloatValue("c"), 1e-5); + + + inputs = new PythonVariables(); + inputs.addNDArray("a", Nd4j.zeros(3, 2).add(4)); + inputs.addNDArray("b", Nd4j.zeros(3, 2).add(5)); + + outputs = new PythonVariables(); + outputs.addNDArray("c"); + + + job.exec(inputs, outputs); + + assertEquals(Nd4j.zeros(3, 2).add(9), outputs.getNDArrayValue("c")); + } + + @Test + public void testPythonJobReturnAllVariables()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code = "c = a + b"; + PythonJob job = new PythonJob("job1", code, false); + + PythonVariables inputs = new PythonVariables(); + inputs.addInt("a", 2); + inputs.addInt("b", 3); + + + PythonVariables outputs = job.execAndReturnAllVariables(inputs); + + assertEquals(5L, (long)outputs.getIntValue("c")); + + inputs = new PythonVariables(); + inputs.addFloat("a", 3.0); + inputs.addFloat("b", 4.0); + + outputs = job.execAndReturnAllVariables(inputs); + + assertEquals(7.0, outputs.getFloatValue("c"), 1e-5); + + + inputs = new PythonVariables(); + inputs.addNDArray("a", Nd4j.zeros(3, 2).add(4)); + inputs.addNDArray("b", Nd4j.zeros(3, 2).add(5)); + + outputs = job.execAndReturnAllVariables(inputs); + + assertEquals(Nd4j.zeros(3, 2).add(9), outputs.getNDArrayValue("c")); + } + + @Test + public void testMultiplePythonJobsParallel()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code1 = "c = a + b"; + PythonJob job1 = new PythonJob("job1", code1, false); + + String code2 = "c = a - b"; + PythonJob job2 = new PythonJob("job2", code2, false); + + PythonVariables inputs = new PythonVariables(); + inputs.addInt("a", 2); + inputs.addInt("b", 3); + + PythonVariables outputs = new PythonVariables(); + outputs.addInt("c"); + + job1.exec(inputs, outputs); + + assertEquals(5L, (long)outputs.getIntValue("c")); + + job2.exec(inputs, outputs); + + assertEquals(-1L, (long)outputs.getIntValue("c")); + + inputs = new PythonVariables(); + inputs.addFloat("a", 3.0); + inputs.addFloat("b", 4.0); + + outputs = new PythonVariables(); + outputs.addFloat("c"); + + + job1.exec(inputs, outputs); + + assertEquals(7.0, outputs.getFloatValue("c"), 1e-5); + + job2.exec(inputs, outputs); + + assertEquals(-1L, outputs.getFloatValue("c"), 1e-5); + + + inputs = new PythonVariables(); + inputs.addNDArray("a", Nd4j.zeros(3, 2).add(4)); + inputs.addNDArray("b", Nd4j.zeros(3, 2).add(5)); + + outputs = new PythonVariables(); + outputs.addNDArray("c"); + + + job1.exec(inputs, outputs); + + assertEquals(Nd4j.zeros(3, 2).add(9), outputs.getNDArrayValue("c")); + + job2.exec(inputs, outputs); + + assertEquals(Nd4j.zeros(3, 2).sub(1), outputs.getNDArrayValue("c")); + } + @Test + public void testPythonJobSetupRun()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code = "five=None\n" + + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " c = a + b + five\n"+ + " return {'c':c}\n\n"; + PythonJob job = new PythonJob("job1", code, true); + + PythonVariables inputs = new PythonVariables(); + inputs.addInt("a", 2); + inputs.addInt("b", 3); + + PythonVariables outputs = new PythonVariables(); + outputs.addInt("c"); + + job.exec(inputs, outputs); + + assertEquals(10L, (long)outputs.getIntValue("c")); + + inputs = new PythonVariables(); + inputs.addFloat("a", 3.0); + inputs.addFloat("b", 4.0); + + outputs = new PythonVariables(); + outputs.addFloat("c"); + + + job.exec(inputs, outputs); + + assertEquals(12.0, outputs.getFloatValue("c"), 1e-5); + + + inputs = new PythonVariables(); + inputs.addNDArray("a", Nd4j.zeros(3, 2).add(4)); + inputs.addNDArray("b", Nd4j.zeros(3, 2).add(5)); + + outputs = new PythonVariables(); + outputs.addNDArray("c"); + + + job.exec(inputs, outputs); + + assertEquals(Nd4j.zeros(3, 2).add(14), outputs.getNDArrayValue("c")); + } + @Test + public void testPythonJobSetupRunAndReturnAllVariables()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code = "five=None\n" + + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " c = a + b + five\n"+ + " return {'c':c}\n\n"; + PythonJob job = new PythonJob("job1", code, true); + + PythonVariables inputs = new PythonVariables(); + inputs.addInt("a", 2); + inputs.addInt("b", 3); + + + PythonVariables outputs = job.execAndReturnAllVariables(inputs); + + assertEquals(10L, (long)outputs.getIntValue("c")); + + inputs = new PythonVariables(); + inputs.addFloat("a", 3.0); + inputs.addFloat("b", 4.0); + + outputs = job.execAndReturnAllVariables(inputs); + + assertEquals(12.0, outputs.getFloatValue("c"), 1e-5); + + + inputs = new PythonVariables(); + inputs.addNDArray("a", Nd4j.zeros(3, 2).add(4)); + inputs.addNDArray("b", Nd4j.zeros(3, 2).add(5)); + + outputs = job.execAndReturnAllVariables(inputs); + + assertEquals(Nd4j.zeros(3, 2).add(14), outputs.getNDArrayValue("c")); + } + + @Test + public void testMultiplePythonJobsSetupRunParallel()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code1 = "five=None\n" + + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " c = a + b + five\n"+ + " return {'c':c}\n\n"; + PythonJob job1 = new PythonJob("job1", code1, true); + + String code2 = "five=None\n" + + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " c = a + b - five\n"+ + " return {'c':c}\n\n"; + PythonJob job2 = new PythonJob("job2", code2, true); + + PythonVariables inputs = new PythonVariables(); + inputs.addInt("a", 2); + inputs.addInt("b", 3); + + PythonVariables outputs = new PythonVariables(); + outputs.addInt("c"); + + job1.exec(inputs, outputs); + + assertEquals(10L, (long)outputs.getIntValue("c")); + + job2.exec(inputs, outputs); + + assertEquals(0L, (long)outputs.getIntValue("c")); + + inputs = new PythonVariables(); + inputs.addFloat("a", 3.0); + inputs.addFloat("b", 4.0); + + outputs = new PythonVariables(); + outputs.addFloat("c"); + + + job1.exec(inputs, outputs); + + assertEquals(12.0, outputs.getFloatValue("c"), 1e-5); + + job2.exec(inputs, outputs); + + assertEquals(2L, outputs.getFloatValue("c"), 1e-5); + + + inputs = new PythonVariables(); + inputs.addNDArray("a", Nd4j.zeros(3, 2).add(4)); + inputs.addNDArray("b", Nd4j.zeros(3, 2).add(5)); + + outputs = new PythonVariables(); + outputs.addNDArray("c"); + + + job1.exec(inputs, outputs); + + assertEquals(Nd4j.zeros(3, 2).add(14), outputs.getNDArrayValue("c")); + + job2.exec(inputs, outputs); + + assertEquals(Nd4j.zeros(3, 2).add(4), outputs.getNDArrayValue("c")); + } + +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonList.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonList.java new file mode 100644 index 000000000..259431cba --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonList.java @@ -0,0 +1,107 @@ + +/******************************************************************************* + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@javax.annotation.concurrent.NotThreadSafe +public class TestPythonList { + + @Test + public void testPythonListFromIntArray() { + PythonObject pyList = new PythonObject(new Integer[]{1, 2, 3, 4, 5}); + pyList.attr("append").call(6); + pyList.attr("append").call(7); + pyList.attr("append").call(8); + assertEquals(8, Python.len(pyList).toInt()); + for (int i = 0; i < 8; i++) { + assertEquals(i + 1, pyList.get(i).toInt()); + } + + } + + @Test + public void testPythonListFromLongArray() { + PythonObject pyList = new PythonObject(new Long[]{1L, 2L, 3L, 4L, 5L}); + pyList.attr("append").call(6); + pyList.attr("append").call(7); + pyList.attr("append").call(8); + assertEquals(8, Python.len(pyList).toInt()); + for (int i = 0; i < 8; i++) { + assertEquals(i + 1, pyList.get(i).toInt()); + } + + } + + @Test + public void testPythonListFromDoubleArray() { + PythonObject pyList = new PythonObject(new Double[]{1., 2., 3., 4., 5.}); + pyList.attr("append").call(6); + pyList.attr("append").call(7); + pyList.attr("append").call(8); + assertEquals(8, Python.len(pyList).toInt()); + for (int i = 0; i < 8; i++) { + assertEquals(i + 1, pyList.get(i).toInt()); + assertEquals((double) i + 1, pyList.get(i).toDouble(), 1e-5); + } + + } + + @Test + public void testPythonListFromStringArray() { + PythonObject pyList = new PythonObject(new String[]{"abcd", "efg"}); + pyList.attr("append").call("hijk"); + pyList.attr("append").call("lmnop"); + assertEquals("abcdefghijklmnop", new PythonObject("").attr("join").call(pyList).toString()); + } + + @Test + public void testPythonListFromMixedArray()throws Exception { + Map map = new HashMap<>(); + map.put(1, "a"); + map.put("a", Arrays.asList("a", "b", "c")); + map.put("arr", Nd4j.linspace(1, 4, 4)); + Object[] objs = new Object[]{ + 1, 2, "a", 3f, 4L, 5.0, Arrays.asList(10, + 20, "b", 30f, 40L, 50.0, map + + ), map + }; + PythonObject pyList = new PythonObject(objs); + System.out.println(pyList.toString()); + String expectedStr = "[1, 2, 'a', 3.0, 4, 5.0, [10" + + ", 20, 'b', 30.0, 40, 50.0, {'arr': array([1.," + + " 2., 3., 4.], dtype=float32), 1: 'a', 'a': [" + + "'a', 'b', 'c']}], {'arr': array([1., 2., 3.," + + " 4.], dtype=float32), 1: 'a', 'a': ['a', 'b', 'c']}]"; + assertEquals(expectedStr, pyList.toString()); + List objs2 = pyList.toList(); + PythonObject pyList2 = new PythonObject(objs2); + assertEquals(pyList.toString(), pyList2.toString()); + } + +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonVariables.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonVariables.java new file mode 100644 index 000000000..b709e1608 --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestPythonVariables.java @@ -0,0 +1,98 @@ +/* + * + * * ****************************************************************************** + * * * Copyright (c) 2015-2019 Skymind Inc. + * * * Copyright (c) 2019 Konduit AI. + * * * + * * * This program and the accompanying materials are made available under the + * * * terms of the Apache License, Version 2.0 which is available at + * * * https://www.apache.org/licenses/LICENSE-2.0. + * * * + * * * Unless required by applicable law or agreed to in writing, software + * * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * * License for the specific language governing permissions and limitations + * * * under the License. + * * * + * * * SPDX-License-Identifier: Apache-2.0 + * * ***************************************************************************** + * + * + */ + +package org.datavec.python; + +import org.bytedeco.javacpp.BytePointer; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.BaseDataBuffer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestPythonVariables { + + @Test + public void testDataAssociations() throws PythonException{ + PythonVariables pythonVariables = new PythonVariables(); + PythonType[] types = { + PythonType.INT, + PythonType.FLOAT, + PythonType.STR, + PythonType.BOOL, + PythonType.DICT, + PythonType.LIST, + PythonType.LIST, + PythonType.NDARRAY, + PythonType.BYTES + }; + + INDArray arr = Nd4j.scalar(1.0); + ((BaseDataBuffer)arr.data()).syncToPrimary(); + BytePointer bp = new BytePointer(arr.data().pointer()); + Object[] values = { + 1L,1.0,"1",true, Collections.singletonMap("1",1), + new Object[]{1}, Arrays.asList(1), arr, bp + }; + + Object[] expectedValues = { + 1L,1.0,"1",true, Collections.singletonMap("1",1), + Arrays.asList(1), Arrays.asList(1), arr, bp + }; + + for(int i = 0; i < types.length; i++) { + testInsertGet(pythonVariables,types[i].getName().name() + i,values[i],types[i],expectedValues[i]); + } + + assertEquals(types.length,pythonVariables.getVariables().length); + + } + + private void testInsertGet(PythonVariables pythonVariables,String key,Object value,PythonType type,Object expectedValue) throws PythonException{ + pythonVariables.add(key, type); + assertNull(pythonVariables.getValue(key)); + pythonVariables.setValue(key,value); + assertNotNull(pythonVariables.getValue(key)); + Object actualValue = pythonVariables.getValue(key); + if (expectedValue instanceof Object[]){ + assertTrue(actualValue instanceof List); + Object[] actualArr = ((List)actualValue).toArray(); + Object[] expectedArr = (Object[])expectedValue; + assertArrayEquals(expectedArr, actualArr); + } + else{ + assertEquals(expectedValue,pythonVariables.getValue(key)); + } + + } + + +} diff --git a/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestSerde.java b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestSerde.java new file mode 100644 index 000000000..16ab6629c --- /dev/null +++ b/cavis-datavec/cavis-datavec-python/src/test/java/org/datavec/python/TestSerde.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.python; + +import org.datavec.api.transform.Transform; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.serde.JsonSerializer; +import org.datavec.api.transform.serde.YamlSerializer; +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +public class TestSerde { + + public static YamlSerializer y = new YamlSerializer(); + public static JsonSerializer j = new JsonSerializer(); + + @Test @Timeout(60) + public void testBasicSerde(){ + Schema schema = new Schema.Builder() + .addColumnInteger("col1") + .addColumnFloat("col2") + .addColumnString("col3") + .addColumnDouble("col4") + .build(); + + Transform t = PythonTransform.builder().code( + "col1+=3\ncol2+=2\ncol3+='a'\ncol4+=2.0" + ).inputSchema(schema).outputSchema(schema).build(); + + String yaml = y.serialize(t); + String json = j.serialize(t); + + Transform t2 = y.deserializeTransform(yaml); + Transform t3 = j.deserializeTransform(json); + assertEquals(t, t2); + assertEquals(t, t3); + } + +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/op-ir.proto b/cavis-datavec/cavis-datavec-spark/build.gradle similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/op-ir.proto rename to cavis-datavec/cavis-datavec-spark/build.gradle diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/build.gradle b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/build.gradle new file mode 100644 index 000000000..b7bd6a553 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/build.gradle @@ -0,0 +1,72 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +plugins { + id 'java-library' + id 'maven-publish' + id 'signing' +} + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +ext { + scalaVersion = rootProject.ext.scalaVersion +} + +dependencies { + + implementation "org.scala-lang:scala-library" + compileOnly ("org.apache.spark:spark-core_${scalaVersion}") { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + testCompileOnly ("org.apache.spark:spark-core_${scalaVersion}") { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + testRuntimeOnly ("org.apache.spark:spark-core_${scalaVersion}") { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + compileOnly "org.apache.spark:spark-sql_${scalaVersion}" + testCompileOnly "org.apache.spark:spark-sql_${scalaVersion}" + testRuntimeOnly "org.apache.spark:spark-sql_${scalaVersion}" + implementation "commons-collections:commons-collections" + implementation 'commons-io:commons-io' + implementation "org.apache.commons:commons-math3" + implementation 'org.slf4j:slf4j-api' + testImplementation "com.sun.jna:jna:3.0.9" + testImplementation "com.tdunning:t-digest:3.2" + testImplementation "joda-time:joda-time:2.10.3" + + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" + implementation "com.fasterxml.jackson.datatype:jackson-datatype-joda" + implementation "org.bytedeco:javacpp" + + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataHadoop + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testImplementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage + testImplementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataCodec + testImplementation projects.cavisDatavec.cavisDatavecLocal + testImplementation projects.cavisDatavec.cavisDatavecPython + +} \ No newline at end of file diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/SequenceEmptyRecordFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/SequenceEmptyRecordFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/SequenceEmptyRecordFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/SequenceEmptyRecordFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/EmptyRecordFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/EmptyRecordFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/EmptyRecordFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/EmptyRecordFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/LineRecordReaderFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/LineRecordReaderFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/LineRecordReaderFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/LineRecordReaderFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/RecordReaderFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/RecordReaderFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/RecordReaderFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/RecordReaderFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/SequenceRecordReaderFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/SequenceRecordReaderFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/SequenceRecordReaderFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/SequenceRecordReaderFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/data/FilesAsBytesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/data/FilesAsBytesFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/data/FilesAsBytesFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/data/FilesAsBytesFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/data/RecordReaderBytesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/data/RecordReaderBytesFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/data/RecordReaderBytesFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/data/RecordReaderBytesFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/data/SequenceRecordReaderBytesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/data/SequenceRecordReaderBytesFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/data/SequenceRecordReaderBytesFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/data/SequenceRecordReaderBytesFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/BytesPairWritable.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/BytesPairWritable.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/BytesPairWritable.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/BytesPairWritable.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/MapToBytesPairWritableFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/MapToBytesPairWritableFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/MapToBytesPairWritableFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/MapToBytesPairWritableFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PairSequenceRecordReaderBytesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PairSequenceRecordReaderBytesFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PairSequenceRecordReaderBytesFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PairSequenceRecordReaderBytesFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverter.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverter.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverter.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverter.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverterFilename.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverterFilename.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverterFilename.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverterFilename.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverterNumber.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverterNumber.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverterNumber.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyConverterNumber.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/functions/pairdata/PathToKeyFunction.java diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/SparkStorageUtils.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/SparkStorageUtils.java new file mode 100644 index 000000000..323012432 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/SparkStorageUtils.java @@ -0,0 +1,363 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.storage; + +import org.apache.commons.io.FilenameUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; +import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.datavec.api.writable.Writable; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; +import org.datavec.hadoop.records.reader.mapfile.record.SequenceRecordWritable; +import org.datavec.spark.storage.functions.RecordLoadPairFunction; +import org.datavec.spark.storage.functions.RecordSavePrepPairFunction; +import org.datavec.spark.storage.functions.SequenceRecordLoadPairFunction; +import org.datavec.spark.storage.functions.SequenceRecordSavePrepPairFunction; + +import java.util.List; + +public class SparkStorageUtils { + + /** + * Configuration key for the map file interval. + * This is defined in MapFile.Writer.INDEX_INTERVAL but unfortunately that field is private, hence cannot be + * referenced here. + */ + public static final String MAP_FILE_INDEX_INTERVAL_KEY = "io.map.index.interval"; + + /** + * By default, a map file's index stores only a fraction of the keys. This is good, in that it reduces memory + * requirements (all keys are loaded into memory); however, it has a cost in terms of time taken for look up. + * Instead of using the default interval of 128, Will use a default interval of 1: given that the keys are LongWritable + * objects, the marginal increase in space is more than outweighed by the increased performance for use cases such as + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader} and {@link org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader} + */ + public static final int DEFAULT_MAP_FILE_INTERVAL = 1; + + private SparkStorageUtils() {} + + /** + * Save a {@code JavaRDD>} to a Hadoop {@link org.apache.hadoop.io.SequenceFile}. Each record is given + * a unique (but noncontiguous) {@link LongWritable} key, and values are stored as {@link RecordWritable} instances. + *

+ * Use {@link #restoreSequenceFile(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the sequence file + * @param rdd RDD to save + * @see #saveSequenceFileSequences(String, JavaRDD) + * @see #saveMapFile(String, JavaRDD) + */ + public static void saveSequenceFile(String path, JavaRDD> rdd) { + saveSequenceFile(path, rdd, null); + } + + /** + * Save a {@code JavaRDD>} to a Hadoop {@link org.apache.hadoop.io.SequenceFile}. Each record is given + * a unique (but noncontiguous) {@link LongWritable} key, and values are stored as {@link RecordWritable} instances. + *

+ * Use {@link #restoreSequenceFile(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the sequence file + * @param rdd RDD to save + * @param maxOutputFiles Nullable. If non-null: first coalesce the RDD to the specified size (number of partitions) + * to limit the maximum number of output sequence files + * @see #saveSequenceFileSequences(String, JavaRDD) + * @see #saveMapFile(String, JavaRDD) + */ + public static void saveSequenceFile(String path, JavaRDD> rdd, Integer maxOutputFiles) { + path = FilenameUtils.normalize(path, true); + if (maxOutputFiles != null) { + rdd = rdd.coalesce(maxOutputFiles); + } + JavaPairRDD, Long> dataIndexPairs = rdd.zipWithUniqueId(); //Note: Long values are unique + NOT contiguous; more efficient than zipWithIndex + JavaPairRDD keyedByIndex = + dataIndexPairs.mapToPair(new RecordSavePrepPairFunction()); + + keyedByIndex.saveAsNewAPIHadoopFile(path, LongWritable.class, RecordWritable.class, + SequenceFileOutputFormat.class); + } + + /** + * Restore a {@code JavaRDD>} previously saved with {@link #saveSequenceFile(String, JavaRDD)} + * + * @param path Path of the sequence file + * @param sc Spark context + * @return The restored RDD + */ + public static JavaRDD> restoreSequenceFile(String path, JavaSparkContext sc) { + return restoreMapFile(path, sc).values(); + } + + /** + * Save a {@code JavaRDD>>} to a Hadoop {@link org.apache.hadoop.io.SequenceFile}. Each record + * is given a unique (but noncontiguous) {@link LongWritable} key, and values are stored as {@link SequenceRecordWritable} instances. + *

+ * Use {@link #restoreSequenceFileSequences(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the sequence file + * @param rdd RDD to save + * @see #saveSequenceFile(String, JavaRDD) + * @see #saveMapFileSequences(String, JavaRDD) + */ + public static void saveSequenceFileSequences(String path, JavaRDD>> rdd) { + saveSequenceFileSequences(path, rdd, null); + } + + /** + * Save a {@code JavaRDD>>} to a Hadoop {@link org.apache.hadoop.io.SequenceFile}. Each record + * is given a unique (but noncontiguous) {@link LongWritable} key, and values are stored as {@link SequenceRecordWritable} instances. + *

+ * Use {@link #restoreSequenceFileSequences(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the sequence file + * @param rdd RDD to save + * @param maxOutputFiles Nullable. If non-null: first coalesce the RDD to the specified size (number of partitions) + * to limit the maximum number of output sequence files + * @see #saveSequenceFile(String, JavaRDD) + * @see #saveMapFileSequences(String, JavaRDD) + */ + public static void saveSequenceFileSequences(String path, JavaRDD>> rdd, + Integer maxOutputFiles) { + path = FilenameUtils.normalize(path, true); + if (maxOutputFiles != null) { + rdd = rdd.coalesce(maxOutputFiles); + } + JavaPairRDD>, Long> dataIndexPairs = rdd.zipWithUniqueId(); //Note: Long values are unique + NOT contiguous; more efficient than zipWithIndex + JavaPairRDD keyedByIndex = + dataIndexPairs.mapToPair(new SequenceRecordSavePrepPairFunction()); + + keyedByIndex.saveAsNewAPIHadoopFile(path, LongWritable.class, SequenceRecordWritable.class, + SequenceFileOutputFormat.class); + } + + /** + * Restore a {@code JavaRDD>} previously saved with {@link #saveSequenceFileSequences(String, JavaRDD)} + * + * @param path Path of the sequence file + * @param sc Spark context + * @return The restored RDD + */ + public static JavaRDD>> restoreSequenceFileSequences(String path, JavaSparkContext sc) { + return restoreMapFileSequences(path, sc).values(); + } + + + /** + * Save a {@code JavaRDD>} to a Hadoop {@link org.apache.hadoop.io.MapFile}. Each record is + * given a unique and contiguous {@link LongWritable} key, and values are stored as + * {@link RecordWritable} instances.
+ * Note 1: If contiguous keys are not required, using a sequence file instead is preferable from a performance + * point of view. Contiguous keys are often only required for non-Spark use cases, such as with + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader}
+ * Note 2: This use a MapFile interval of {@link #DEFAULT_MAP_FILE_INTERVAL}, which is usually suitable for + * use cases such as {@link org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader}. Use + * {@link #saveMapFile(String, JavaRDD, int, Integer)} or {@link #saveMapFile(String, JavaRDD, Configuration, Integer)} + * to customize this.
+ *

+ * Use {@link #restoreMapFile(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the MapFile + * @param rdd RDD to save + * @see #saveMapFileSequences(String, JavaRDD) + * @see #saveSequenceFile(String, JavaRDD) + */ + public static void saveMapFile(String path, JavaRDD> rdd) { + saveMapFile(path, rdd, DEFAULT_MAP_FILE_INTERVAL, null); + } + + /** + * Save a {@code JavaRDD>} to a Hadoop {@link org.apache.hadoop.io.MapFile}. Each record is + * given a unique and contiguous {@link LongWritable} key, and values are stored as + * {@link RecordWritable} instances.
+ * Note: If contiguous keys are not required, using a sequence file instead is preferable from a performance + * point of view. Contiguous keys are often only required for non-Spark use cases, such as with + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader} + *

+ * Use {@link #restoreMapFileSequences(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the MapFile + * @param rdd RDD to save + * @param interval The map file index interval to use. Smaller values may result in the faster look up, at the + * expense of more memory/disk use. However, usually the increase is relatively minor, due to + * keys being stored as LongWritable objects + * @param maxOutputFiles Nullable. If non-null: first coalesce the RDD to the specified size (number of partitions) + * to limit the maximum number of output map files + * @see #saveMapFileSequences(String, JavaRDD) + * @see #saveSequenceFile(String, JavaRDD) + */ + public static void saveMapFile(String path, JavaRDD> rdd, int interval, + Integer maxOutputFiles) { + Configuration c = new Configuration(); + c.set(MAP_FILE_INDEX_INTERVAL_KEY, String.valueOf(interval)); + saveMapFile(path, rdd, c, maxOutputFiles); + } + + /** + * Save a {@code JavaRDD>} to a Hadoop {@link org.apache.hadoop.io.MapFile}. Each record is + * given a unique and contiguous {@link LongWritable} key, and values are stored as + * {@link RecordWritable} instances.
+ * Note: If contiguous keys are not required, using a sequence file instead is preferable from a performance + * point of view. Contiguous keys are often only required for non-Spark use cases, such as with + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader} + *

+ * Use {@link #restoreMapFileSequences(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the MapFile + * @param rdd RDD to save + * @param c Configuration object, used to customise options for the map file + * @param maxOutputFiles Nullable. If non-null: first coalesce the RDD to the specified size (number of partitions) + * to limit the maximum number of output map files + * @see #saveMapFileSequences(String, JavaRDD) + * @see #saveSequenceFile(String, JavaRDD) + */ + public static void saveMapFile(String path, JavaRDD> rdd, Configuration c, + Integer maxOutputFiles) { + path = FilenameUtils.normalize(path, true); + if (maxOutputFiles != null) { + rdd = rdd.coalesce(maxOutputFiles); + } + JavaPairRDD, Long> dataIndexPairs = rdd.zipWithIndex(); //Note: Long values are unique + contiguous, but requires a count + JavaPairRDD keyedByIndex = + dataIndexPairs.mapToPair(new RecordSavePrepPairFunction()); + + keyedByIndex.saveAsNewAPIHadoopFile(path, LongWritable.class, RecordWritable.class, MapFileOutputFormat.class, + c); + } + + /** + * Restore a {@code JavaPairRDD>} previously saved with {@link #saveMapFile(String, JavaRDD)}}
+ * Note that if the keys are not required, simply use {@code restoreMapFile(...).values()} + * + * @param path Path of the MapFile + * @param sc Spark context + * @return The restored RDD, with their unique indices as the key + */ + public static JavaPairRDD> restoreMapFile(String path, JavaSparkContext sc) { + Configuration c = new Configuration(); + c.set(FileInputFormat.INPUT_DIR, FilenameUtils.normalize(path, true)); + JavaPairRDD pairRDD = + sc.newAPIHadoopRDD(c, SequenceFileInputFormat.class, LongWritable.class, RecordWritable.class); + + return pairRDD.mapToPair(new RecordLoadPairFunction()); + } + + /** + * Save a {@code JavaRDD>>} to a Hadoop {@link org.apache.hadoop.io.MapFile}. Each record is + * given a unique and contiguous {@link LongWritable} key, and values are stored as + * {@link SequenceRecordWritable} instances.
+ * Note 1: If contiguous keys are not required, using a sequence file instead is preferable from a performance + * point of view. Contiguous keys are often only required for non-Spark use cases, such as with + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader}
+ * Note 2: This use a MapFile interval of {@link #DEFAULT_MAP_FILE_INTERVAL}, which is usually suitable for + * use cases such as {@link org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader}. Use + * {@link #saveMapFileSequences(String, JavaRDD, int, Integer)} or {@link #saveMapFileSequences(String, JavaRDD, Configuration, Integer)} + * to customize this.
+ *

+ * Use {@link #restoreMapFileSequences(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the MapFile + * @param rdd RDD to save + * @see #saveMapFileSequences(String, JavaRDD) + * @see #saveSequenceFile(String, JavaRDD) + */ + public static void saveMapFileSequences(String path, JavaRDD>> rdd) { + saveMapFileSequences(path, rdd, DEFAULT_MAP_FILE_INTERVAL, null); + } + + /** + * Save a {@code JavaRDD>>} to a Hadoop {@link org.apache.hadoop.io.MapFile}. Each record is + * given a unique and contiguous {@link LongWritable} key, and values are stored as + * {@link SequenceRecordWritable} instances.
+ * Note: If contiguous keys are not required, using a sequence file instead is preferable from a performance + * point of view. Contiguous keys are often only required for non-Spark use cases, such as with + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader}
+ *

+ * Use {@link #restoreMapFileSequences(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the MapFile + * @param rdd RDD to save + * @param interval The map file index interval to use. Smaller values may result in the faster look up, at the + * expense of more memory/disk use. However, usually the increase is relatively minor, due to + * keys being stored as LongWritable objects + * @see #saveMapFileSequences(String, JavaRDD) + * @see #saveSequenceFile(String, JavaRDD) + */ + public static void saveMapFileSequences(String path, JavaRDD>> rdd, int interval, + Integer maxOutputFiles) { + Configuration c = new Configuration(); + c.set(MAP_FILE_INDEX_INTERVAL_KEY, String.valueOf(interval)); + saveMapFileSequences(path, rdd, c, maxOutputFiles); + } + + /** + * Save a {@code JavaRDD>>} to a Hadoop {@link org.apache.hadoop.io.MapFile}. Each record is + * given a unique and contiguous {@link LongWritable} key, and values are stored as + * {@link SequenceRecordWritable} instances.
+ * Note: If contiguous keys are not required, using a sequence file instead is preferable from a performance + * point of view. Contiguous keys are often only required for non-Spark use cases, such as with + * {@link org.datavec.hadoop.records.reader.mapfile.MapFileSequenceRecordReader}
+ *

+ * Use {@link #restoreMapFileSequences(String, JavaSparkContext)} to restore values saved with this method. + * + * @param path Path to save the MapFile + * @param rdd RDD to save + * @param c Configuration object, used to customise options for the map file + * @see #saveMapFileSequences(String, JavaRDD) + * @see #saveSequenceFile(String, JavaRDD) + */ + public static void saveMapFileSequences(String path, JavaRDD>> rdd, Configuration c, + Integer maxOutputFiles) { + path = FilenameUtils.normalize(path, true); + if (maxOutputFiles != null) { + rdd = rdd.coalesce(maxOutputFiles); + } + JavaPairRDD>, Long> dataIndexPairs = rdd.zipWithIndex(); + JavaPairRDD keyedByIndex = + dataIndexPairs.mapToPair(new SequenceRecordSavePrepPairFunction()); + + keyedByIndex.saveAsNewAPIHadoopFile(path, LongWritable.class, SequenceRecordWritable.class, + MapFileOutputFormat.class, c); + } + + /** + * Restore a {@code JavaPairRDD>>} previously saved with {@link #saveMapFile(String, JavaRDD)}}
+ * Note that if the keys are not required, simply use {@code restoreMapFileSequences(...).values()} + * + * @param path Path of the MapFile + * @param sc Spark context + * @return The restored RDD, with their unique indices as the key + */ + public static JavaPairRDD>> restoreMapFileSequences(String path, JavaSparkContext sc) { + Configuration c = new Configuration(); + c.set(FileInputFormat.INPUT_DIR, FilenameUtils.normalize(path, true)); + JavaPairRDD pairRDD = sc.newAPIHadoopRDD(c, SequenceFileInputFormat.class, + LongWritable.class, SequenceRecordWritable.class); + + return pairRDD.mapToPair(new SequenceRecordLoadPairFunction()); + } + +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/RecordLoadPairFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/RecordLoadPairFunction.java new file mode 100644 index 000000000..192c0e7d0 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/RecordLoadPairFunction.java @@ -0,0 +1,37 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.storage.functions; + +import org.apache.hadoop.io.LongWritable; +import org.apache.spark.api.java.function.PairFunction; +import org.datavec.api.writable.Writable; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; +import scala.Tuple2; + +import java.util.List; + +public class RecordLoadPairFunction + implements PairFunction, Long, List> { + @Override + public Tuple2> call(Tuple2 t2) throws Exception { + return new Tuple2<>(t2._1().get(), t2._2().getRecord()); + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/RecordSavePrepPairFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/RecordSavePrepPairFunction.java new file mode 100644 index 000000000..048f6b191 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/RecordSavePrepPairFunction.java @@ -0,0 +1,37 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.storage.functions; + +import org.apache.hadoop.io.LongWritable; +import org.apache.spark.api.java.function.PairFunction; +import org.datavec.api.writable.Writable; +import org.datavec.hadoop.records.reader.mapfile.record.RecordWritable; +import scala.Tuple2; + +import java.util.List; + +public class RecordSavePrepPairFunction + implements PairFunction, Long>, LongWritable, RecordWritable> { + @Override + public Tuple2 call(Tuple2, Long> t2) throws Exception { + return new Tuple2<>(new LongWritable(t2._2()), new RecordWritable(t2._1())); + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/SequenceRecordLoadPairFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/SequenceRecordLoadPairFunction.java new file mode 100644 index 000000000..a8296cd6e --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/SequenceRecordLoadPairFunction.java @@ -0,0 +1,37 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.storage.functions; + +import org.apache.hadoop.io.LongWritable; +import org.apache.spark.api.java.function.PairFunction; +import org.datavec.api.writable.Writable; +import org.datavec.hadoop.records.reader.mapfile.record.SequenceRecordWritable; +import scala.Tuple2; + +import java.util.List; + +public class SequenceRecordLoadPairFunction + implements PairFunction, Long, List>> { + @Override + public Tuple2>> call(Tuple2 t2) throws Exception { + return new Tuple2<>(t2._1().get(), t2._2().getSequenceRecord()); + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/SequenceRecordSavePrepPairFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/SequenceRecordSavePrepPairFunction.java new file mode 100644 index 000000000..072beb5de --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/storage/functions/SequenceRecordSavePrepPairFunction.java @@ -0,0 +1,37 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.storage.functions; + +import org.apache.hadoop.io.LongWritable; +import org.apache.spark.api.java.function.PairFunction; +import org.datavec.api.writable.Writable; +import org.datavec.hadoop.records.reader.mapfile.record.SequenceRecordWritable; +import scala.Tuple2; + +import java.util.List; + +public class SequenceRecordSavePrepPairFunction + implements PairFunction>, Long>, LongWritable, SequenceRecordWritable> { + @Override + public Tuple2 call(Tuple2>, Long> t2) throws Exception { + return new Tuple2<>(new LongWritable(t2._2()), new SequenceRecordWritable(t2._1())); + } +} diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/AnalyzeSpark.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/AnalyzeSpark.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/AnalyzeSpark.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/AnalyzeSpark.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/DataFrames.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/DataFrames.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/DataFrames.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/DataFrames.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/Normalization.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/Normalization.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/Normalization.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/Normalization.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/SparkTransformExecutor.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/SparkTransformExecutor.java similarity index 99% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/SparkTransformExecutor.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/SparkTransformExecutor.java index 47f27e71d..9595c5a7b 100644 --- a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/SparkTransformExecutor.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/SparkTransformExecutor.java @@ -57,6 +57,9 @@ import scala.Tuple2; import java.util.Comparator; import java.util.List; +/** + * Executes a transform process {@link TransformProcess} on Apache Spark + */ public class SparkTransformExecutor { private static final Logger log = LoggerFactory.getLogger(SparkTransformExecutor.class); diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/CategoricalToPairFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/CategoricalToPairFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/CategoricalToPairFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/CategoricalToPairFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/SelectColumnFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/SelectColumnFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/SelectColumnFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/SelectColumnFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/SequenceFlatMapFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/SequenceFlatMapFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/SequenceFlatMapFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/SequenceFlatMapFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/SequenceLengthFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/SequenceLengthFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/SequenceLengthFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/SequenceLengthFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/StringLengthFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/StringLengthFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/StringLengthFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/StringLengthFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/WritableToDoubleFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/WritableToDoubleFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/WritableToDoubleFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/WritableToDoubleFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/WritableToStringFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/WritableToStringFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/WritableToStringFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/WritableToStringFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/aggregate/AnalysisAddFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/aggregate/AnalysisAddFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/aggregate/AnalysisAddFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/aggregate/AnalysisAddFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/aggregate/AnalysisCombineFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/aggregate/AnalysisCombineFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/aggregate/AnalysisCombineFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/aggregate/AnalysisCombineFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/histogram/HistogramAddFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/histogram/HistogramAddFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/histogram/HistogramAddFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/histogram/HistogramAddFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/histogram/HistogramCombineFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/histogram/HistogramCombineFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/histogram/HistogramCombineFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/histogram/HistogramCombineFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/seqlength/IntToDoubleFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/seqlength/IntToDoubleFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/seqlength/IntToDoubleFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/seqlength/IntToDoubleFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisAddFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisAddFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisAddFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisAddFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisCounter.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisCounter.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisCounter.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisCounter.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisMergeFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisMergeFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisMergeFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/seqlength/SequenceLengthAnalysisMergeFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/string/StringAnalysisMergeFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/string/StringAnalysisMergeFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/string/StringAnalysisMergeFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/string/StringAnalysisMergeFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/unique/UniqueAddFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/unique/UniqueAddFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/unique/UniqueAddFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/unique/UniqueAddFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/unique/UniqueMergeFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/unique/UniqueMergeFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/analysis/unique/UniqueMergeFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/analysis/unique/UniqueMergeFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/filter/FilterWritablesBySchemaFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/filter/FilterWritablesBySchemaFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/filter/FilterWritablesBySchemaFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/filter/FilterWritablesBySchemaFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/filter/SparkFilterFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/filter/SparkFilterFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/filter/SparkFilterFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/filter/SparkFilterFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/ExecuteJoinFromCoGroupFlatMapFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/ExecuteJoinFromCoGroupFlatMapFunction.java similarity index 99% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/ExecuteJoinFromCoGroupFlatMapFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/ExecuteJoinFromCoGroupFlatMapFunction.java index c70cea258..0edd54fb8 100644 --- a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/ExecuteJoinFromCoGroupFlatMapFunction.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/ExecuteJoinFromCoGroupFlatMapFunction.java @@ -20,7 +20,7 @@ package org.datavec.spark.transform.join; -import org.nd4j.shade.guava.collect.Iterables; +import com.google.common.collect.Iterables; import org.apache.spark.api.java.function.FlatMapFunction; import org.datavec.api.transform.join.Join; import org.datavec.api.writable.Writable; diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/ExtractKeysFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/ExtractKeysFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/ExtractKeysFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/ExtractKeysFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/FilterAndFlattenJoinedValues.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/FilterAndFlattenJoinedValues.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/FilterAndFlattenJoinedValues.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/FilterAndFlattenJoinedValues.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/JoinedValue.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/JoinedValue.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/join/JoinedValue.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/join/JoinedValue.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/ColumnAsKeyPairFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/ColumnAsKeyPairFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/ColumnAsKeyPairFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/ColumnAsKeyPairFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/ColumnToKeyPairTransform.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/ColumnToKeyPairTransform.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/ColumnToKeyPairTransform.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/ColumnToKeyPairTransform.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/NDArrayToWritablesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/NDArrayToWritablesFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/NDArrayToWritablesFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/NDArrayToWritablesFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/SequenceMergeFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/SequenceMergeFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/SequenceMergeFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/SequenceMergeFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/SequenceWritablesToStringFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/SequenceWritablesToStringFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/SequenceWritablesToStringFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/SequenceWritablesToStringFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/StringToWritablesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/StringToWritablesFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/StringToWritablesFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/StringToWritablesFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/SumLongsFunction2.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/SumLongsFunction2.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/SumLongsFunction2.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/SumLongsFunction2.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/WritablesToNDArrayFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/WritablesToNDArrayFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/WritablesToNDArrayFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/WritablesToNDArrayFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/WritablesToStringFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/WritablesToStringFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/WritablesToStringFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/WritablesToStringFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/comparator/Tuple2Comparator.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/comparator/Tuple2Comparator.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/misc/comparator/Tuple2Comparator.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/misc/comparator/Tuple2Comparator.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/rank/UnzipForCalculateSortedRankFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/rank/UnzipForCalculateSortedRankFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/rank/UnzipForCalculateSortedRankFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/rank/UnzipForCalculateSortedRankFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/reduce/MapToPairForReducerFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/reduce/MapToPairForReducerFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/reduce/MapToPairForReducerFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/reduce/MapToPairForReducerFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/reduce/ReducerFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/reduce/ReducerFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/reduce/ReducerFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/reduce/ReducerFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/ConvertToSequenceLengthOne.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/ConvertToSequenceLengthOne.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/ConvertToSequenceLengthOne.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/ConvertToSequenceLengthOne.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkGroupToSequenceFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkGroupToSequenceFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkGroupToSequenceFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkGroupToSequenceFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkMapToPairByColumnFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkMapToPairByColumnFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkMapToPairByColumnFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkMapToPairByColumnFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkMapToPairByMultipleColumnsFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkMapToPairByMultipleColumnsFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkMapToPairByMultipleColumnsFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkMapToPairByMultipleColumnsFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkSequenceFilterFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkSequenceFilterFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkSequenceFilterFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkSequenceFilterFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkSequenceTransformFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkSequenceTransformFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sequence/SparkSequenceTransformFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sequence/SparkSequenceTransformFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/SequenceToRows.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/SequenceToRows.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/SequenceToRows.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/SequenceToRows.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/ToRecord.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/ToRecord.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/ToRecord.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/ToRecord.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/ToRow.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/ToRow.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/ToRow.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/ToRow.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceCreateCombiner.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceCreateCombiner.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceCreateCombiner.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceCreateCombiner.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceMergeCombiner.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceMergeCombiner.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceMergeCombiner.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceMergeCombiner.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceMergeValue.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceMergeValue.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceMergeValue.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/sparkfunction/sequence/DataFrameToSequenceMergeValue.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/transform/SequenceSplitFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/transform/SequenceSplitFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/transform/SequenceSplitFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/transform/SequenceSplitFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/transform/SparkTransformFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/transform/SparkTransformFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/transform/SparkTransformFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/transform/SparkTransformFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/transform/SparkTransformProcessFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/transform/SparkTransformProcessFunction.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/transform/SparkTransformProcessFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/transform/SparkTransformProcessFunction.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/utils/SparkExport.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/utils/SparkExport.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/utils/SparkExport.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/utils/SparkExport.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/utils/SparkUtils.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/utils/SparkUtils.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/utils/SparkUtils.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/utils/SparkUtils.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/transform/utils/adapter/BiFunctionAdapter.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/utils/adapter/BiFunctionAdapter.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/transform/utils/adapter/BiFunctionAdapter.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/transform/utils/adapter/BiFunctionAdapter.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/util/BroadcastHadoopConfigHolder.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/util/BroadcastHadoopConfigHolder.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/util/BroadcastHadoopConfigHolder.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/util/BroadcastHadoopConfigHolder.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/util/DataVecSparkUtil.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/util/DataVecSparkUtil.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/util/DataVecSparkUtil.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/util/DataVecSparkUtil.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/util/DefaultHadoopConfig.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/util/DefaultHadoopConfig.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/util/DefaultHadoopConfig.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/util/DefaultHadoopConfig.java diff --git a/datavec/datavec-spark/src/main/java/org/datavec/spark/util/SerializableHadoopConfig.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/util/SerializableHadoopConfig.java similarity index 100% rename from datavec/datavec-spark/src/main/java/org/datavec/spark/util/SerializableHadoopConfig.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/main/java/org/datavec/spark/util/SerializableHadoopConfig.java diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/AssertTestsExtendBaseClass.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/AssertTestsExtendBaseClass.java similarity index 100% rename from datavec/datavec-spark/src/test/java/org/datavec/spark/AssertTestsExtendBaseClass.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/AssertTestsExtendBaseClass.java diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/BaseSparkSessionTest.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/BaseSparkSessionTest.java new file mode 100644 index 000000000..fe111d1e6 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/BaseSparkSessionTest.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; + +import java.io.File; +import java.io.Serializable; + +@Slf4j +public abstract class BaseSparkSessionTest implements Serializable { + private SparkSession spark; + + public SparkSession getSession() { + SparkConf sparkConf = new SparkConf() + .setMaster("spark://10.5.5.200:7077") + .setAppName(BaseSparkSessionTest.class.getSimpleName()) + .set("spark.driver.bindAddress", "10.5.5.145") + .set("spark.network.timeout", "240000") + .set("spark.driver.host", "10.5.5.145") + .set("spark.deploy.mode", "client") + .set("spark.executor.memory", "4g") + .set("spark.cores.max", "8") + .set("spark.worker.cleanup.enabled", "true") + .set("spark.driver.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.executor.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.hadoop.fs.defaultFS", "hdfs://10.5.5.200:9000"); + + this.spark = SparkSession.builder() + .config(sparkConf) + .getOrCreate(); + + return this.spark; + } + + @BeforeAll + public void beforeAll() { + + } + + @AfterAll + public synchronized void afterAll() { + getSession().close(); + + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/BaseSparkTest.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/BaseSparkTest.java new file mode 100644 index 000000000..83435ab47 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/BaseSparkTest.java @@ -0,0 +1,128 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import java.io.File; +import java.io.Serializable; + +@Slf4j +public abstract class BaseSparkTest implements Serializable { + protected static JavaSparkContext sc; + + @BeforeEach + public void before() { + sc = getContext(); + } + + @AfterEach + public synchronized void after() { + sc.close(); + //Wait until it's stopped, to avoid race conditions during tests + for (int i = 0; i < 100; i++) { + if (!sc.sc().stopped().get()) { + try { + Thread.sleep(100L); + } catch (InterruptedException e) { + log.error("",e); + } + } else { + break; + } + } + if (!sc.sc().stopped().get()) { + throw new RuntimeException("Spark context is not stopped after 10s"); + } + + + sc = null; + } + + public synchronized JavaSparkContext getContext() { + if (sc != null) + return sc; + + /* + SparkConf sparkConf = new SparkConf().setMaster("local[*]").set("spark.driver.host", "localhost") + .set("spark.driverEnv.SPARK_LOCAL_IP", "127.0.0.1") + .set("spark.executorEnv.SPARK_LOCAL_IP", "127.0.0.1").setAppName("sparktest"); + + */ + SparkConf sparkConf = new SparkConf() + .setMaster("spark://10.5.5.200:7077") + .setAppName("Brian4") + .set("spark.driver.bindAddress", "10.5.5.145") + .set("spark.network.timeout", "240000") + .set("spark.driver.host", "10.5.5.145") + .set("spark.deploy.mode", "client") + .set("spark.executor.memory", "4g") + //.set("spark.cores.max", "8") + .set("spark.worker.cleanup.enabled", "true") + .set("spark.driver.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + .set("spark.executor.extraJavaOptions", "-Dlog4j.configurationFile=log4j2.xml") + //.set("spark.driver.extraClassPath", "brian-spark-dl4j-tests-1.0.1-SNAPSHOT-bin.jar;brian-spark-dl4j-tests-1.0.1-SNAPSHOT-tests.jar") + //.set("spark.executor.extraClassPath", "brian-spark-dl4j-tests-1.0.1-SNAPSHOT-bin.jar;brian-spark-dl4j-tests-1.0.1-SNAPSHOT-tests.jar") + .set("spark.hadoop.fs.defaultFS", "hdfs://10.5.5.200:9000"); + //.set("spark.jars.packages", "com.fasterxml.jackson.datatype:jackson-datatype-joda:2.10.4"); + //.set("spark.driver.cores", "2") + //.set("spark.driver.memory", "8g") + //.set("spark.driver.host", "10.5.5.145") + //.setExecutorEnv("spark.executor.cores", "2") + //.setExecutorEnv("spark.executor.memory", "2g") + //.set("spark.submit.deployMode", "client") + if (useKryo()) { + sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + } + + + + + SparkSession spark = SparkSession.builder() + .config(sparkConf) + .getOrCreate(); + + sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + /* + Whatever is in classpath (driver), is added to the Spark Executors + */ + final String clpath = System.getProperty("java.class.path"); + final String separator = System.getProperty("path.separator"); + final String[] a = clpath.split(separator); + for(String s : a) { + File f = new File(s); + if(f.exists() && f.isFile() && s.endsWith(".jar")) { + sc.addJar(s); + } + } + return sc; + } + + public boolean useKryo(){ + return false; + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/ClasspathLoadedTests.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/ClasspathLoadedTests.java new file mode 100644 index 000000000..3948f7e6b --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/ClasspathLoadedTests.java @@ -0,0 +1,36 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.datavec.spark; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; + +@Slf4j +public class ClasspathLoadedTests extends BaseSparkSessionTest { + + @Test + public void verifyClasspath() { + ClassLoader cl = ClassLoader.getSystemClassLoader(); + log.info( "java.class.path {}", System.getProperty("java.class.path")); + //cl.asInstanceOf[java.net.URLClassLoader].getURLs.foreach(println) + } +} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/TestKryoSerialization.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/TestKryoSerialization.java similarity index 93% rename from datavec/datavec-spark/src/test/java/org/datavec/spark/TestKryoSerialization.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/TestKryoSerialization.java index 512c6054e..a684fb61d 100644 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/TestKryoSerialization.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/TestKryoSerialization.java @@ -25,10 +25,8 @@ import org.apache.spark.serializer.SerializerInstance; import org.datavec.api.records.reader.RecordReader; import org.datavec.api.records.reader.impl.csv.CSVRecordReader; import org.datavec.api.split.FileSplit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import java.nio.ByteBuffer; @@ -36,10 +34,7 @@ import java.nio.ByteBuffer; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) + public class TestKryoSerialization extends BaseSparkTest { @Override diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestLineRecordReaderFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestLineRecordReaderFunction.java new file mode 100644 index 000000000..d7a906597 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestLineRecordReaderFunction.java @@ -0,0 +1,73 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.functions; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.Writable; +import org.datavec.spark.BaseSparkTest; +import org.junit.jupiter.api.Test; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestLineRecordReaderFunction extends BaseSparkTest { + + @Test + public void testLineRecordReader() throws Exception { + + File dataFile = new ClassPathResource("iris.dat").getFile(); + List lines = FileUtils.readLines(dataFile); + + JavaSparkContext sc = getContext(); + JavaRDD linesRdd = sc.parallelize(lines); + + CSVRecordReader rr = new CSVRecordReader(0, ','); + + JavaRDD> out = linesRdd.map(new LineRecordReaderFunction(rr)); + List> outList = out.collect(); + + + CSVRecordReader rr2 = new CSVRecordReader(0, ','); + rr2.initialize(new FileSplit(dataFile)); + Set> expectedSet = new HashSet<>(); + int totalCount = 0; + while (rr2.hasNext()) { + expectedSet.add(rr2.next()); + totalCount++; + } + + assertEquals(totalCount, outList.size()); + + for (List line : outList) { + assertTrue(expectedSet.contains(line)); + } + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestNDArrayToWritablesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestNDArrayToWritablesFunction.java new file mode 100644 index 000000000..4990cfe03 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestNDArrayToWritablesFunction.java @@ -0,0 +1,56 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.functions; + +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; +import org.datavec.spark.transform.misc.NDArrayToWritablesFunction; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestNDArrayToWritablesFunction { + + @Test + public void testNDArrayToWritablesScalars() throws Exception { + INDArray arr = Nd4j.arange(5); + List expected = new ArrayList<>(); + for (int i = 0; i < 5; i++) + expected.add(new DoubleWritable(i)); + List actual = new NDArrayToWritablesFunction().call(arr); + assertEquals(expected, actual); + } + + @Test + public void testNDArrayToWritablesArray() throws Exception { + INDArray arr = Nd4j.arange(5); + List expected = Arrays.asList((Writable) new NDArrayWritable(arr)); + List actual = new NDArrayToWritablesFunction(true).call(arr); + assertEquals(expected, actual); + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestPairSequenceRecordReaderBytesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestPairSequenceRecordReaderBytesFunction.java new file mode 100644 index 000000000..b96041e3f --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestPairSequenceRecordReaderBytesFunction.java @@ -0,0 +1,146 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.functions; + +import com.sun.jna.Platform; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.Writable; +import org.datavec.codec.reader.CodecRecordReader; +import org.datavec.spark.BaseSparkTest; +import org.datavec.spark.functions.pairdata.BytesPairWritable; +import org.datavec.spark.functions.pairdata.PairSequenceRecordReaderBytesFunction; +import org.datavec.spark.functions.pairdata.PathToKeyConverter; +import org.datavec.spark.functions.pairdata.PathToKeyConverterFilename; +import org.datavec.spark.util.DataVecSparkUtil; + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.io.ClassPathResource; +import scala.Tuple2; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestPairSequenceRecordReaderBytesFunction extends BaseSparkTest { + + @TempDir + public File testDir; + + @Test + public void test() throws Exception { + //Goal: combine separate files together into a hadoop sequence file, for later parsing by a SequenceRecordReader + //For example: use to combine input and labels data from separate files for training a RNN + if(Platform.isWindows()) { + return; + } + JavaSparkContext sc = getContext(); + + File f = testDir; + new ClassPathResource("datavec-spark/video/").copyDirectory(f); + String path = f.getAbsolutePath() + "/*"; + + PathToKeyConverter pathConverter = new PathToKeyConverterFilename(); + JavaPairRDD toWrite = + DataVecSparkUtil.combineFilesForSequenceFile(sc, path, path, pathConverter); + + Path p = Files.createTempDirectory("dl4j_rrbytesPairOut"); + p.toFile().deleteOnExit(); + String outPath = p.toString() + "/out"; + new File(outPath).deleteOnExit(); + toWrite.saveAsNewAPIHadoopFile(outPath, Text.class, BytesPairWritable.class, SequenceFileOutputFormat.class); + + //Load back into memory: + JavaPairRDD fromSeq = sc.sequenceFile(outPath, Text.class, BytesPairWritable.class); + + SequenceRecordReader srr1 = getReader(); + SequenceRecordReader srr2 = getReader(); + PairSequenceRecordReaderBytesFunction psrbf = new PairSequenceRecordReaderBytesFunction(srr1, srr2); + + JavaRDD>, List>>> writables = fromSeq.map(psrbf); + List>, List>>> fromSequenceFile = writables.collect(); + + //Load manually (single copy) and compare: + InputSplit is = new FileSplit(f, new String[] {"mp4"}, true); + SequenceRecordReader srr = getReader(); + srr.initialize(is); + + List>> list = new ArrayList<>(4); + while (srr.hasNext()) { + list.add(srr.sequenceRecord()); + } + + assertEquals(4, list.size()); + assertEquals(4, fromSequenceFile.size()); + + boolean[] found = new boolean[4]; + for (int i = 0; i < 4; i++) { + int foundIndex = -1; + Tuple2>, List>> tuple2 = fromSequenceFile.get(i); + List> seq1 = tuple2._1(); + List> seq2 = tuple2._2(); + assertEquals(seq1, seq2); + + for (int j = 0; j < 4; j++) { + if (seq1.equals(list.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + int count = 0; + for (boolean b : found) + if (b) + count++; + assertEquals(4, count); //Expect all 4 and exactly 4 pairwise matches between spark and local versions + + } + + private static SequenceRecordReader getReader() { + SequenceRecordReader seqRR = new CodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "0"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "25"); + conf.set(CodecRecordReader.ROWS, "64"); + conf.set(CodecRecordReader.COLUMNS, "64"); + seqRR.setConf(conf); + return seqRR; + } +} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestRecordReaderBytesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestRecordReaderBytesFunction.java similarity index 94% rename from datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestRecordReaderBytesFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestRecordReaderBytesFunction.java index 68c9aca65..aa6e5f76d 100644 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestRecordReaderBytesFunction.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestRecordReaderBytesFunction.java @@ -37,12 +37,10 @@ import org.datavec.spark.BaseSparkTest; import org.datavec.spark.functions.data.FilesAsBytesFunction; import org.datavec.spark.functions.data.RecordReaderBytesFunction; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import java.nio.file.Files; @@ -53,23 +51,21 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) + public class TestRecordReaderBytesFunction extends BaseSparkTest { - + @TempDir + public File testDir; @Test - public void testRecordReaderBytesFunction(@TempDir Path testDir) throws Exception { + public void testRecordReaderBytesFunction() throws Exception { if(Platform.isWindows()) { return; } JavaSparkContext sc = getContext(); //Local file path - File f = testDir.toFile(); + File f = testDir; new ClassPathResource("datavec-spark/imagetest/").copyDirectory(f); List labelsList = Arrays.asList("0", "1"); //Need this for Spark: can't infer without init call String path = f.getAbsolutePath() + "/*"; diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestRecordReaderFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestRecordReaderFunction.java similarity index 93% rename from datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestRecordReaderFunction.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestRecordReaderFunction.java index dc436ea8e..a1695ee63 100644 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestRecordReaderFunction.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestRecordReaderFunction.java @@ -32,33 +32,30 @@ import org.datavec.api.writable.Writable; import org.datavec.image.recordreader.ImageRecordReader; import org.datavec.spark.BaseSparkTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) + public class TestRecordReaderFunction extends BaseSparkTest { + @TempDir + public File testDir; + @Test - public void testRecordReaderFunction(@TempDir Path testDir) throws Exception { + public void testRecordReaderFunction() throws Exception { if(Platform.isWindows()) { return; } - File f = testDir.toFile(); + File f = testDir; new ClassPathResource("datavec-spark/imagetest/").copyDirectory(f); List labelsList = Arrays.asList("0", "1"); //Need this for Spark: can't infer without init call diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderBytesFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderBytesFunction.java new file mode 100644 index 000000000..2f9bc4410 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderBytesFunction.java @@ -0,0 +1,133 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.functions; + +import com.sun.jna.Platform; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.input.PortableDataStream; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.Writable; +import org.datavec.codec.reader.CodecRecordReader; +import org.datavec.spark.BaseSparkTest; +import org.datavec.spark.functions.data.FilesAsBytesFunction; +import org.datavec.spark.functions.data.SequenceRecordReaderBytesFunction; + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestSequenceRecordReaderBytesFunction extends BaseSparkTest { + + @TempDir + public File testDir; + + @Test + public void testRecordReaderBytesFunction() throws Exception { + if(Platform.isWindows()) { + return; + } + //Local file path + File f = testDir; + new ClassPathResource("datavec-spark/video/").copyDirectory(f); + String path = f.getAbsolutePath() + "/*"; + + //Load binary data from local file system, convert to a sequence file: + //Load and convert + JavaPairRDD origData = sc.binaryFiles(path); + JavaPairRDD filesAsBytes = origData.mapToPair(new FilesAsBytesFunction()); + //Write the sequence file: + Path p = Files.createTempDirectory("dl4j_rrbytesTest"); + p.toFile().deleteOnExit(); + String outPath = p.toString() + "/out"; + filesAsBytes.saveAsNewAPIHadoopFile(outPath, Text.class, BytesWritable.class, SequenceFileOutputFormat.class); + + //Load data from sequence file, parse via SequenceRecordReader: + JavaPairRDD fromSeqFile = sc.sequenceFile(outPath, Text.class, BytesWritable.class); + SequenceRecordReader seqRR = new CodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "0"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "25"); + conf.set(CodecRecordReader.ROWS, "64"); + conf.set(CodecRecordReader.COLUMNS, "64"); + Configuration confCopy = new Configuration(conf); + seqRR.setConf(conf); + JavaRDD>> dataVecData = fromSeqFile.map(new SequenceRecordReaderBytesFunction(seqRR)); + + + + //Next: do the same thing locally, and compare the results + InputSplit is = new FileSplit(f, new String[] {"mp4"}, true); + SequenceRecordReader srr = new CodecRecordReader(); + srr.initialize(is); + srr.setConf(confCopy); + + List>> list = new ArrayList<>(4); + while (srr.hasNext()) { + list.add(srr.sequenceRecord()); + } + assertEquals(4, list.size()); + + List>> fromSequenceFile = dataVecData.collect(); + + assertEquals(4, list.size()); + assertEquals(4, fromSequenceFile.size()); + + boolean[] found = new boolean[4]; + for (int i = 0; i < 4; i++) { + int foundIndex = -1; + List> collection = fromSequenceFile.get(i); + for (int j = 0; j < 4; j++) { + if (collection.equals(list.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + int count = 0; + for (boolean b : found) + if (b) + count++; + assertEquals(4, count); //Expect all 4 and exactly 4 pairwise matches between spark and local versions + } + +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderFunction.java new file mode 100644 index 000000000..208bc42d1 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderFunction.java @@ -0,0 +1,204 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.functions; + +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.input.PortableDataStream; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.ArrayWritable; +import org.datavec.api.writable.Writable; +import org.datavec.codec.reader.CodecRecordReader; +import org.datavec.spark.BaseSparkTest; + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestSequenceRecordReaderFunction extends BaseSparkTest { + + @TempDir + public File testDir; + + @Test + public void testSequenceRecordReaderFunctionCSV() throws Exception { + JavaSparkContext sc = getContext(); + + File f = testDir; + new ClassPathResource("datavec-spark/csvsequence/").copyDirectory(f); + + String path = f.getAbsolutePath() + "/*"; + + JavaPairRDD origData = sc.binaryFiles(path); + assertEquals(3, origData.count()); //3 CSV files + + SequenceRecordReaderFunction srrf = new SequenceRecordReaderFunction(new CSVSequenceRecordReader(1, ",")); //CSV, skip 1 line + JavaRDD>> rdd = origData.map(srrf); + List>> listSpark = rdd.collect(); + + assertEquals(3, listSpark.size()); + for (int i = 0; i < 3; i++) { + List> thisSequence = listSpark.get(i); + assertEquals(4, thisSequence.size()); //Expect exactly 4 time steps in sequence + for (List c : thisSequence) { + assertEquals(3, c.size()); //3 values per time step + } + } + + //Load normally, and check that we get the same results (order not withstanding) + InputSplit is = new FileSplit(f, new String[] {"txt"}, true); + // System.out.println("Locations:"); + // System.out.println(Arrays.toString(is.locations())); + + SequenceRecordReader srr = new CSVSequenceRecordReader(1, ","); + srr.initialize(is); + + List>> list = new ArrayList<>(3); + while (srr.hasNext()) { + list.add(srr.sequenceRecord()); + } + assertEquals(3, list.size()); + + // System.out.println("Spark list:"); + // for(List> c : listSpark ) System.out.println(c); + // System.out.println("Local list:"); + // for(List> c : list ) System.out.println(c); + + //Check that each of the values from Spark equals exactly one of the values doing it normally + boolean[] found = new boolean[3]; + for (int i = 0; i < 3; i++) { + int foundIndex = -1; + List> collection = listSpark.get(i); + for (int j = 0; j < 3; j++) { + if (collection.equals(list.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + int count = 0; + for (boolean b : found) + if (b) + count++; + assertEquals(3, count); //Expect all 3 and exactly 3 pairwise matches between spark and local versions + } + + + + @Test + public void testSequenceRecordReaderFunctionVideo() throws Exception { + JavaSparkContext sc = getContext(); + + File f = testDir; + new ClassPathResource("datavec-spark/video/").copyDirectory(f); + + String path = f.getAbsolutePath() + "/*"; + + JavaPairRDD origData = sc.binaryFiles(path); + // System.out.println(origData.collectAsMap().keySet()); + assertEquals(4, origData.count()); //4 video files + + //Load 64x64, 25 frames - originally, 130x130, 150 frames + SequenceRecordReader sparkSeqReader = new CodecRecordReader(); + Configuration conf = new Configuration(); + conf.set(CodecRecordReader.RAVEL, "true"); + conf.set(CodecRecordReader.START_FRAME, "0"); + conf.set(CodecRecordReader.TOTAL_FRAMES, "25"); + conf.set(CodecRecordReader.ROWS, "64"); + conf.set(CodecRecordReader.COLUMNS, "64"); + Configuration confCopy = new Configuration(conf); + sparkSeqReader.setConf(conf); + + SequenceRecordReaderFunction srrf = new SequenceRecordReaderFunction(sparkSeqReader); + JavaRDD>> rdd = origData.map(srrf); + List>> listSpark = rdd.collect(); + + assertEquals(4, listSpark.size()); + for (int i = 0; i < 4; i++) { + List> thisSequence = listSpark.get(i); + assertEquals(25, thisSequence.size()); //Expect exactly 25 time steps (frames) in sequence + for (List c : thisSequence) { + assertEquals(1, c.size()); //64*64 videos, RGB + assertEquals(64 * 64 * 3, ((ArrayWritable) c.iterator().next()).length()); + } + } + + //Load normally, and check that we get the same results (order not withstanding) + InputSplit is = new FileSplit(f, new String[] {"mp4"}, true); + // System.out.println("Locations:"); + // System.out.println(Arrays.toString(is.locations())); + + SequenceRecordReader srr = new CodecRecordReader(); + srr.initialize(is); + srr.setConf(confCopy); + + + List>> list = new ArrayList<>(4); + while (srr.hasNext()) { + list.add(srr.sequenceRecord()); + } + assertEquals(4, list.size()); + + // System.out.println("Spark list:"); + // for(List> c : listSpark ) System.out.println(c); + // System.out.println("Local list:"); + // for(List> c : list ) System.out.println(c); + + //Check that each of the values from Spark equals exactly one of the values doing it locally + boolean[] found = new boolean[4]; + for (int i = 0; i < 4; i++) { + int foundIndex = -1; + List> collection = listSpark.get(i); + for (int j = 0; j < 4; j++) { + if (collection.equals(list.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + int count = 0; + for (boolean b : found) + if (b) + count++; + assertEquals(4, count); //Expect all 4 and exactly 4 pairwise matches between spark and local versions + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestWritablesToNDArrayFunction.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestWritablesToNDArrayFunction.java new file mode 100644 index 000000000..62021a252 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestWritablesToNDArrayFunction.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.functions; + +import org.datavec.api.writable.*; +import org.datavec.spark.transform.misc.WritablesToNDArrayFunction; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestWritablesToNDArrayFunction { + + @Test + public void testWritablesToNDArrayAllScalars() throws Exception { + List l = new ArrayList<>(); + for (int i = 0; i < 5; i++) + l.add(new IntWritable(i)); + INDArray expected = Nd4j.arange(5).castTo(DataType.FLOAT).reshape(1,5); + assertEquals(expected, new WritablesToNDArrayFunction().call(l)); + } + + @Test + public void testWritablesToNDArrayMixed() throws Exception { + List l = new ArrayList<>(); + l.add(new IntWritable(0)); + l.add(new IntWritable(1)); + INDArray arr = Nd4j.arange(2, 5); + l.add(new NDArrayWritable(arr)); + l.add(new IntWritable(5)); + arr = Nd4j.arange(6, 9); + l.add(new NDArrayWritable(arr)); + l.add(new IntWritable(9)); + + INDArray expected = Nd4j.arange(10).castTo(DataType.FLOAT).reshape(1,10); + assertEquals(expected, new WritablesToNDArrayFunction().call(l)); + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestWritablesToStringFunctions.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestWritablesToStringFunctions.java new file mode 100644 index 000000000..070bda4ed --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/functions/TestWritablesToStringFunctions.java @@ -0,0 +1,97 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.functions; + +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.PairFunction; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.spark.BaseSparkTest; +import org.datavec.spark.transform.misc.SequenceWritablesToStringFunction; +import org.datavec.spark.transform.misc.WritablesToStringFunction; +import org.junit.jupiter.api.Test; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestWritablesToStringFunctions extends BaseSparkTest { + + @Test + public void testCGroup() { + List> leftMap = new ArrayList<>(); + List> rightMap = new ArrayList<>(); + + leftMap.add(new Tuple2<>("cat","adam")); + leftMap.add(new Tuple2<>("dog","adam")); + + rightMap.add(new Tuple2<>("fish","alex")); + rightMap.add(new Tuple2<>("cat","alice")); + rightMap.add(new Tuple2<>("dog","steve")); + + List pets = Arrays.asList("cat","dog"); + + + + JavaSparkContext sc = getContext(); + JavaPairRDD left = sc.parallelize(leftMap).mapToPair(new PairFunction, String, String>() { + @Override + public Tuple2 call(Tuple2 stringStringTuple2) throws Exception { + return stringStringTuple2; + } + }); + + JavaPairRDD right = sc.parallelize(rightMap).mapToPair(new PairFunction, String, String>() { + @Override + public Tuple2 call(Tuple2 stringStringTuple2) throws Exception { + return stringStringTuple2; + } + }); + + System.out.println(left.cogroup(right).collect()); + } + + @Test + public void testWritablesToString() throws Exception { + + List l = Arrays.asList(new DoubleWritable(1.5), new Text("someValue")); + String expected = l.get(0).toString() + "," + l.get(1).toString(); + + assertEquals(expected, new WritablesToStringFunction(",").call(l)); + } + + @Test + public void testSequenceWritablesToString() throws Exception { + + List> l = Arrays.asList(Arrays.asList(new DoubleWritable(1.5), new Text("someValue")), + Arrays.asList(new DoubleWritable(2.5), new Text("otherValue"))); + + String expected = l.get(0).get(0).toString() + "," + l.get(0).get(1).toString() + "\n" + + l.get(1).get(0).toString() + "," + l.get(1).get(1).toString(); + + assertEquals(expected, new SequenceWritablesToStringFunction(",").call(l)); + } +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/storage/TestSparkStorageUtils.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/storage/TestSparkStorageUtils.java new file mode 100644 index 000000000..a0a10e876 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/storage/TestSparkStorageUtils.java @@ -0,0 +1,151 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.storage; + +import com.sun.jna.Platform; +import com.google.common.io.Files; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.datavec.api.writable.*; +import org.datavec.spark.BaseSparkTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestSparkStorageUtils extends BaseSparkTest { + + @Test + public void testSaveRestoreMapFile() { + if(Platform.isWindows()) { + return; + } + List> l = new ArrayList<>(); + l.add(Arrays.asList(new Text("zero"), new IntWritable(0), + new DoubleWritable(0), new NDArrayWritable(Nd4j.valueArrayOf(10, 0.0)))); + l.add(Arrays.asList(new Text("one"), new IntWritable(11), + new DoubleWritable(11.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 11.0)))); + l.add(Arrays.asList(new Text("two"), new IntWritable(22), + new DoubleWritable(22.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 22.0)))); + + JavaRDD> rdd = sc.parallelize(l); + + File f = Files.createTempDir(); + f.delete(); + f.deleteOnExit(); + String path = "file:///" + f.getAbsolutePath(); + + SparkStorageUtils.saveMapFile(path, rdd); + JavaPairRDD> restored = SparkStorageUtils.restoreMapFile(path, sc); + + Map> m = restored.collectAsMap(); + + assertEquals(3, m.size()); + for (int i = 0; i < 3; i++) { + assertEquals(l.get(i), m.get((long) i)); + } + + + //Also test sequence file: + f = Files.createTempDir(); + f.delete(); + f.deleteOnExit(); + path = "file:///" + f.getAbsolutePath(); + + SparkStorageUtils.saveSequenceFile(path, rdd); + List> restored2 = SparkStorageUtils.restoreSequenceFile(path, sc).collect(); + + //Sequence file loading + collect iteration order is not guaranteed (depends on number of partitions, etc) + assertEquals(3, restored2.size()); + assertTrue(l.containsAll(restored2) && restored2.containsAll(l)); + } + + @Test + public void testSaveRestoreMapFileSequences() { + if(Platform.isWindows()) { + return; + } + List>> l = new ArrayList<>(); + l.add(Arrays.asList( + Arrays.asList(new Text("zero"), new IntWritable(0), + new DoubleWritable(0), new NDArrayWritable(Nd4j.valueArrayOf(10, 0.0))), + Arrays.asList(new Text("one"), new IntWritable(1), + new DoubleWritable(1.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 1.0))), + Arrays.asList(new Text("two"), new IntWritable(2), + new DoubleWritable(2.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 2.0))))); + + l.add(Arrays.asList( + Arrays.asList(new Text("Bzero"), new IntWritable(10), + new DoubleWritable(10), new NDArrayWritable(Nd4j.valueArrayOf(10, 10.0))), + Arrays.asList(new Text("Bone"), new IntWritable(11), + new DoubleWritable(11.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 11.0))), + Arrays.asList(new Text("Btwo"), new IntWritable(12), + new DoubleWritable(12.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 12.0))))); + + l.add(Arrays.asList( + Arrays.asList(new Text("Czero"), new IntWritable(20), + new DoubleWritable(20), new NDArrayWritable(Nd4j.valueArrayOf(10, 20.0))), + Arrays.asList(new Text("Cone"), new IntWritable(21), + new DoubleWritable(21.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 21.0))), + Arrays.asList(new Text("Ctwo"), new IntWritable(22), + new DoubleWritable(22.0), new NDArrayWritable(Nd4j.valueArrayOf(10, 22.0))))); + + JavaRDD>> rdd = sc.parallelize(l); + + File f = Files.createTempDir(); + f.delete(); + f.deleteOnExit(); + String path = "file:///" + f.getAbsolutePath(); + + SparkStorageUtils.saveMapFileSequences(path, rdd); + JavaPairRDD>> restored = SparkStorageUtils.restoreMapFileSequences(path, sc); + + Map>> m = restored.collectAsMap(); + + assertEquals(3, m.size()); + for (int i = 0; i < 3; i++) { + assertEquals(l.get(i), m.get((long) i)); + } + + //Also test sequence file: + f = Files.createTempDir(); + f.delete(); + f.deleteOnExit(); + path = "file:///" + f.getAbsolutePath(); + + SparkStorageUtils.saveSequenceFileSequences(path, rdd); + List>> restored2 = SparkStorageUtils.restoreSequenceFileSequences(path, sc).collect(); + + //Sequence file loading + collect iteration order is not guaranteed (depends on number of partitions, etc) + assertEquals(3, restored2.size()); + assertTrue(l.containsAll(restored2) && restored2.containsAll(l)); + } + + + +} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/DataFramesTests.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/DataFramesTests.java similarity index 81% rename from datavec/datavec-spark/src/test/java/org/datavec/spark/transform/DataFramesTests.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/DataFramesTests.java index 62b045348..62237f0b4 100644 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/DataFramesTests.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/DataFramesTests.java @@ -30,21 +30,14 @@ import org.datavec.api.util.ndarray.RecordConverter; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; import org.datavec.spark.BaseSparkTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) + public class DataFramesTests extends BaseSparkTest { @Test @@ -117,15 +110,15 @@ public class DataFramesTests extends BaseSparkTest { public void testNormalize() { List> data = new ArrayList<>(); - data.add(Arrays.asList(new DoubleWritable(1), new DoubleWritable(10))); - data.add(Arrays.asList(new DoubleWritable(2), new DoubleWritable(20))); - data.add(Arrays.asList(new DoubleWritable(3), new DoubleWritable(30))); + data.add(Arrays.asList(new DoubleWritable(1), new DoubleWritable(10))); + data.add(Arrays.asList(new DoubleWritable(2), new DoubleWritable(20))); + data.add(Arrays.asList(new DoubleWritable(3), new DoubleWritable(30))); List> expMinMax = new ArrayList<>(); - expMinMax.add(Arrays.asList(new DoubleWritable(0.0), new DoubleWritable(0.0))); - expMinMax.add(Arrays.asList(new DoubleWritable(0.5), new DoubleWritable(0.5))); - expMinMax.add(Arrays.asList(new DoubleWritable(1.0), new DoubleWritable(1.0))); + expMinMax.add(Arrays.asList(new DoubleWritable(0.0), new DoubleWritable(0.0))); + expMinMax.add(Arrays.asList(new DoubleWritable(0.5), new DoubleWritable(0.5))); + expMinMax.add(Arrays.asList(new DoubleWritable(1.0), new DoubleWritable(1.0))); double m1 = (1 + 2 + 3) / 3.0; double s1 = new StandardDeviation().evaluate(new double[] {1, 2, 3}); @@ -134,11 +127,11 @@ public class DataFramesTests extends BaseSparkTest { List> expStandardize = new ArrayList<>(); expStandardize.add( - Arrays.asList(new DoubleWritable((1 - m1) / s1), new DoubleWritable((10 - m2) / s2))); + Arrays.asList(new DoubleWritable((1 - m1) / s1), new DoubleWritable((10 - m2) / s2))); expStandardize.add( - Arrays.asList(new DoubleWritable((2 - m1) / s1), new DoubleWritable((20 - m2) / s2))); + Arrays.asList(new DoubleWritable((2 - m1) / s1), new DoubleWritable((20 - m2) / s2))); expStandardize.add( - Arrays.asList(new DoubleWritable((3 - m1) / s1), new DoubleWritable((30 - m2) / s2))); + Arrays.asList(new DoubleWritable((3 - m1) / s1), new DoubleWritable((30 - m2) / s2))); JavaRDD> rdd = sc.parallelize(data); @@ -185,13 +178,13 @@ public class DataFramesTests extends BaseSparkTest { List>> sequences = new ArrayList<>(); List> seq1 = new ArrayList<>(); - seq1.add(Arrays.asList(new DoubleWritable(1), new DoubleWritable(10), new DoubleWritable(100))); - seq1.add(Arrays.asList(new DoubleWritable(2), new DoubleWritable(20), new DoubleWritable(200))); - seq1.add(Arrays.asList(new DoubleWritable(3), new DoubleWritable(30), new DoubleWritable(300))); + seq1.add(Arrays.asList(new DoubleWritable(1), new DoubleWritable(10), new DoubleWritable(100))); + seq1.add(Arrays.asList(new DoubleWritable(2), new DoubleWritable(20), new DoubleWritable(200))); + seq1.add(Arrays.asList(new DoubleWritable(3), new DoubleWritable(30), new DoubleWritable(300))); List> seq2 = new ArrayList<>(); - seq2.add(Arrays.asList(new DoubleWritable(4), new DoubleWritable(40), new DoubleWritable(400))); - seq2.add(Arrays.asList(new DoubleWritable(5), new DoubleWritable(50), new DoubleWritable(500))); + seq2.add(Arrays.asList(new DoubleWritable(4), new DoubleWritable(40), new DoubleWritable(400))); + seq2.add(Arrays.asList(new DoubleWritable(5), new DoubleWritable(50), new DoubleWritable(500))); sequences.add(seq1); sequences.add(seq2); @@ -206,21 +199,21 @@ public class DataFramesTests extends BaseSparkTest { //Min/max normalization: List> expSeq1MinMax = new ArrayList<>(); - expSeq1MinMax.add(Arrays.asList(new DoubleWritable((1 - 1.0) / (5.0 - 1.0)), + expSeq1MinMax.add(Arrays.asList(new DoubleWritable((1 - 1.0) / (5.0 - 1.0)), new DoubleWritable((10 - 10.0) / (50.0 - 10.0)), new DoubleWritable((100 - 100.0) / (500.0 - 100.0)))); - expSeq1MinMax.add(Arrays.asList(new DoubleWritable((2 - 1.0) / (5.0 - 1.0)), + expSeq1MinMax.add(Arrays.asList(new DoubleWritable((2 - 1.0) / (5.0 - 1.0)), new DoubleWritable((20 - 10.0) / (50.0 - 10.0)), new DoubleWritable((200 - 100.0) / (500.0 - 100.0)))); - expSeq1MinMax.add(Arrays.asList(new DoubleWritable((3 - 1.0) / (5.0 - 1.0)), + expSeq1MinMax.add(Arrays.asList(new DoubleWritable((3 - 1.0) / (5.0 - 1.0)), new DoubleWritable((30 - 10.0) / (50.0 - 10.0)), new DoubleWritable((300 - 100.0) / (500.0 - 100.0)))); List> expSeq2MinMax = new ArrayList<>(); - expSeq2MinMax.add(Arrays.asList(new DoubleWritable((4 - 1.0) / (5.0 - 1.0)), + expSeq2MinMax.add(Arrays.asList(new DoubleWritable((4 - 1.0) / (5.0 - 1.0)), new DoubleWritable((40 - 10.0) / (50.0 - 10.0)), new DoubleWritable((400 - 100.0) / (500.0 - 100.0)))); - expSeq2MinMax.add(Arrays.asList(new DoubleWritable((5 - 1.0) / (5.0 - 1.0)), + expSeq2MinMax.add(Arrays.asList(new DoubleWritable((5 - 1.0) / (5.0 - 1.0)), new DoubleWritable((50 - 10.0) / (50.0 - 10.0)), new DoubleWritable((500 - 100.0) / (500.0 - 100.0)))); @@ -253,17 +246,17 @@ public class DataFramesTests extends BaseSparkTest { double s3 = new StandardDeviation().evaluate(new double[] {100, 200, 300, 400, 500}); List> expSeq1Std = new ArrayList<>(); - expSeq1Std.add(Arrays.asList(new DoubleWritable((1 - m1) / s1), new DoubleWritable((10 - m2) / s2), + expSeq1Std.add(Arrays.asList(new DoubleWritable((1 - m1) / s1), new DoubleWritable((10 - m2) / s2), new DoubleWritable((100 - m3) / s3))); - expSeq1Std.add(Arrays.asList(new DoubleWritable((2 - m1) / s1), new DoubleWritable((20 - m2) / s2), + expSeq1Std.add(Arrays.asList(new DoubleWritable((2 - m1) / s1), new DoubleWritable((20 - m2) / s2), new DoubleWritable((200 - m3) / s3))); - expSeq1Std.add(Arrays.asList(new DoubleWritable((3 - m1) / s1), new DoubleWritable((30 - m2) / s2), + expSeq1Std.add(Arrays.asList(new DoubleWritable((3 - m1) / s1), new DoubleWritable((30 - m2) / s2), new DoubleWritable((300 - m3) / s3))); List> expSeq2Std = new ArrayList<>(); - expSeq2Std.add(Arrays.asList(new DoubleWritable((4 - m1) / s1), new DoubleWritable((40 - m2) / s2), + expSeq2Std.add(Arrays.asList(new DoubleWritable((4 - m1) / s1), new DoubleWritable((40 - m2) / s2), new DoubleWritable((400 - m3) / s3))); - expSeq2Std.add(Arrays.asList(new DoubleWritable((5 - m1) / s1), new DoubleWritable((50 - m2) / s2), + expSeq2Std.add(Arrays.asList(new DoubleWritable((5 - m1) / s1), new DoubleWritable((50 - m2) / s2), new DoubleWritable((500 - m3) / s3))); diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/ExecutionTest.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/ExecutionTest.java new file mode 100644 index 000000000..c863af460 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/ExecutionTest.java @@ -0,0 +1,372 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.transform; + +import org.apache.spark.api.java.JavaRDD; +import org.datavec.api.transform.MathOp; +import org.datavec.api.transform.ReduceOp; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.reduce.Reducer; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.schema.SequenceSchema; +import org.datavec.api.transform.transform.categorical.FirstDigitTransform; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.spark.BaseSparkTest; +import org.datavec.python.PythonTransform; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ExecutionTest extends BaseSparkTest { + + @Test + public void testExecutionSimple() { + Schema schema = new Schema.Builder().addColumnInteger("col0") + .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build(); + + TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1") + .doubleMathOp("col2", MathOp.Add, 10.0).build(); + + List> inputData = new ArrayList<>(); + inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); + inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); + inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); + + JavaRDD> rdd = sc.parallelize(inputData); + + List> out = new ArrayList<>(SparkTransformExecutor.execute(rdd, tp).collect()); + + Collections.sort(out, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt()); + } + }); + + List> expected = new ArrayList<>(); + expected.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1))); + expected.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1))); + expected.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1))); + + assertEquals(expected, out); + } + + @Test + public void testExecutionSequence() { + + Schema schema = new SequenceSchema.Builder().addColumnInteger("col0") + .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build(); + + TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1") + .doubleMathOp("col2", MathOp.Add, 10.0).build(); + + List>> inputSequences = new ArrayList<>(); + List> seq1 = new ArrayList<>(); + seq1.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); + seq1.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); + seq1.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); + List> seq2 = new ArrayList<>(); + seq2.add(Arrays.asList(new IntWritable(3), new Text("state0"), new DoubleWritable(3.1))); + seq2.add(Arrays.asList(new IntWritable(4), new Text("state1"), new DoubleWritable(4.1))); + + inputSequences.add(seq1); + inputSequences.add(seq2); + + JavaRDD>> rdd = sc.parallelize(inputSequences); + + List>> out = + new ArrayList<>(SparkTransformExecutor.executeSequenceToSequence(rdd, tp).collect()); + + Collections.sort(out, new Comparator>>() { + @Override + public int compare(List> o1, List> o2) { + return -Integer.compare(o1.size(), o2.size()); + } + }); + + List>> expectedSequence = new ArrayList<>(); + List> seq1e = new ArrayList<>(); + seq1e.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1))); + seq1e.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1))); + seq1e.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1))); + List> seq2e = new ArrayList<>(); + seq2e.add(Arrays.asList(new IntWritable(3), new IntWritable(0), new DoubleWritable(13.1))); + seq2e.add(Arrays.asList(new IntWritable(4), new IntWritable(1), new DoubleWritable(14.1))); + + expectedSequence.add(seq1e); + expectedSequence.add(seq2e); + + assertEquals(expectedSequence, out); + } + + + @Test + public void testReductionGlobal() { + + List> in = Arrays.asList( + Arrays.asList(new Text("first"), new DoubleWritable(3.0)), + Arrays.asList(new Text("second"), new DoubleWritable(5.0)) + ); + + JavaRDD> inData = sc.parallelize(in); + + Schema s = new Schema.Builder() + .addColumnString("textCol") + .addColumnDouble("doubleCol") + .build(); + + TransformProcess tp = new TransformProcess.Builder(s) + .reduce(new Reducer.Builder(ReduceOp.TakeFirst) + .takeFirstColumns("textCol") + .meanColumns("doubleCol").build()) + .build(); + + JavaRDD> outRdd = SparkTransformExecutor.execute(inData, tp); + + List> out = outRdd.collect(); + + List> expOut = Collections.singletonList(Arrays.asList(new Text("first"), new DoubleWritable(4.0))); + + assertEquals(expOut, out); + } + + @Test + public void testReductionByKey(){ + + List> in = Arrays.asList( + Arrays.asList(new IntWritable(0), new Text("first"), new DoubleWritable(3.0)), + Arrays.asList(new IntWritable(0), new Text("second"), new DoubleWritable(5.0)), + Arrays.asList(new IntWritable(1), new Text("f"), new DoubleWritable(30.0)), + Arrays.asList(new IntWritable(1), new Text("s"), new DoubleWritable(50.0)) + ); + + JavaRDD> inData = sc.parallelize(in); + + Schema s = new Schema.Builder() + .addColumnInteger("intCol") + .addColumnString("textCol") + .addColumnDouble("doubleCol") + .build(); + + TransformProcess tp = new TransformProcess.Builder(s) + .reduce(new Reducer.Builder(ReduceOp.TakeFirst) + .keyColumns("intCol") + .takeFirstColumns("textCol") + .meanColumns("doubleCol").build()) + .build(); + + JavaRDD> outRdd = SparkTransformExecutor.execute(inData, tp); + + List> out = outRdd.collect(); + + List> expOut = Arrays.asList( + Arrays.asList(new IntWritable(0), new Text("first"), new DoubleWritable(4.0)), + Arrays.asList(new IntWritable(1), new Text("f"), new DoubleWritable(40.0))); + + out = new ArrayList<>(out); + Collections.sort( + out, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt()); + } + } + ); + + assertEquals(expOut, out); + } + + + @Test + public void testUniqueMultiCol(){ + + Schema schema = new Schema.Builder() + .addColumnInteger("col0") + .addColumnCategorical("col1", "state0", "state1", "state2") + .addColumnDouble("col2").build(); + + List> inputData = new ArrayList<>(); + inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); + inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); + inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); + inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); + inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); + inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); + inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); + inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); + inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); + + JavaRDD> rdd = sc.parallelize(inputData); + + Map> l = AnalyzeSpark.getUnique(Arrays.asList("col0", "col1"), schema, rdd); + + assertEquals(2, l.size()); + List c0 = l.get("col0"); + assertEquals(3, c0.size()); + assertTrue(c0.contains(new IntWritable(0)) && c0.contains(new IntWritable(1)) && c0.contains(new IntWritable(2))); + + List c1 = l.get("col1"); + assertEquals(3, c1.size()); + assertTrue(c1.contains(new Text("state0")) && c1.contains(new Text("state1")) && c1.contains(new Text("state2"))); + } + + @Test @Timeout(60) + //@Ignore("AB 2019/05/21 - Fine locally, timeouts on CI - Issue #7657 and #7771") + public void testPythonExecution() throws Exception { + Schema schema = new Schema.Builder().addColumnInteger("col0") + .addColumnString("col1").addColumnDouble("col2").build(); + + Schema finalSchema = new Schema.Builder().addColumnInteger("col0") + .addColumnInteger("col1").addColumnDouble("col2").build(); + String pythonCode = "col1 = ['state0', 'state1', 'state2'].index(col1)\ncol2 += 10.0"; + TransformProcess tp = new TransformProcess.Builder(schema).transform( + PythonTransform.builder().code( + "first = np.sin(first)\nsecond = np.cos(second)") + .outputSchema(finalSchema).build() + ).build(); + List> inputData = new ArrayList<>(); + inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); + inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); + inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); + + JavaRDD> rdd = sc.parallelize(inputData); + + List> out = new ArrayList<>(SparkTransformExecutor.execute(rdd, tp).collect()); + + Collections.sort(out, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt()); + } + }); + + List> expected = new ArrayList<>(); + expected.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1))); + expected.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1))); + expected.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1))); + + assertEquals(expected, out); + } + + @Test @Timeout(60) + //@Ignore("AB 2019/05/21 - Fine locally, timeouts on CI - Issue #7657 and #7771") + public void testPythonExecutionWithNDArrays() throws Exception { + long[] shape = new long[]{3, 2}; + Schema schema = new Schema.Builder().addColumnInteger("id").addColumnNDArray("col1", shape) + .addColumnNDArray("col2", shape).build(); + + Schema finalSchema = new Schema.Builder().addColumnInteger("id").addColumnNDArray("col1", shape) + .addColumnNDArray("col2", shape).addColumnNDArray("col3", shape).build(); + + String pythonCode = "col3 = col1 + col2"; + TransformProcess tp = new TransformProcess.Builder(schema).transform( + PythonTransform.builder().code( + "first = np.sin(first)\nsecond = np.cos(second)") + .outputSchema(schema).build() + ).build(); + + INDArray zeros = Nd4j.zeros(shape); + INDArray ones = Nd4j.ones(shape); + INDArray twos = ones.add(ones); + + List> inputData = new ArrayList<>(); + inputData.add(Arrays.asList(new IntWritable(0), new NDArrayWritable(zeros), new NDArrayWritable(zeros))); + inputData.add(Arrays.asList(new IntWritable(1), new NDArrayWritable(zeros), new NDArrayWritable(ones))); + inputData.add(Arrays.asList(new IntWritable(2), new NDArrayWritable(ones), new NDArrayWritable(ones))); + + JavaRDD> rdd = sc.parallelize(inputData); + + List> out = new ArrayList<>(SparkTransformExecutor.execute(rdd, tp).collect()); + + Collections.sort(out, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt()); + } + }); + + List> expected = new ArrayList<>(); + expected.add(Arrays.asList(new IntWritable(0), new NDArrayWritable(zeros), new NDArrayWritable(zeros), new NDArrayWritable(zeros))); + expected.add(Arrays.asList(new IntWritable(1), new NDArrayWritable(zeros), new NDArrayWritable(ones), new NDArrayWritable(ones))); + expected.add(Arrays.asList(new IntWritable(2), new NDArrayWritable(ones), new NDArrayWritable(ones), new NDArrayWritable(twos))); + } + + @Test + public void testFirstDigitTransformBenfordsLaw(){ + Schema s = new Schema.Builder() + .addColumnString("data") + .addColumnDouble("double") + .addColumnString("stringNumber") + .build(); + + List> in = Arrays.asList( + Arrays.asList(new Text("a"), new DoubleWritable(3.14159), new Text("8e-4")), + Arrays.asList(new Text("a2"), new DoubleWritable(3.14159), new Text("7e-4")), + Arrays.asList(new Text("b"), new DoubleWritable(2.71828), new Text("7e2")), + Arrays.asList(new Text("c"), new DoubleWritable(1.61803), new Text("6e8")), + Arrays.asList(new Text("c"), new DoubleWritable(1.61803), new Text("2.0")), + Arrays.asList(new Text("c"), new DoubleWritable(1.61803), new Text("2.1")), + Arrays.asList(new Text("c"), new DoubleWritable(1.61803), new Text("2.2")), + Arrays.asList(new Text("c"), new DoubleWritable(-2), new Text("non numerical"))); + + //Test Benfords law use case: + TransformProcess tp = new TransformProcess.Builder(s) + .firstDigitTransform("double", "fdDouble", FirstDigitTransform.Mode.EXCEPTION_ON_INVALID) + .firstDigitTransform("stringNumber", "stringNumber", FirstDigitTransform.Mode.INCLUDE_OTHER_CATEGORY) + .removeAllColumnsExceptFor("stringNumber") + .categoricalToOneHot("stringNumber") + .reduce(new Reducer.Builder(ReduceOp.Sum).build()) + .build(); + + JavaRDD> rdd = sc.parallelize(in); + + + List> out = SparkTransformExecutor.execute(rdd, tp).collect(); + assertEquals(1, out.size()); + + List l = out.get(0); + List exp = Arrays.asList( + new IntWritable(0), //0 + new IntWritable(0), //1 + new IntWritable(3), //2 + new IntWritable(0), //3 + new IntWritable(0), //4 + new IntWritable(0), //5 + new IntWritable(1), //6 + new IntWritable(2), //7 + new IntWritable(1), //8 + new IntWritable(0), //9 + new IntWritable(1)); //Other + assertEquals(exp, l); + } + +} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/NormalizationTests.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/NormalizationTests.java similarity index 96% rename from datavec/datavec-spark/src/test/java/org/datavec/spark/transform/NormalizationTests.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/NormalizationTests.java index 32a34eb57..7fd6d9d7f 100644 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/NormalizationTests.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/NormalizationTests.java @@ -28,10 +28,7 @@ import org.datavec.api.util.ndarray.RecordConverter; import org.datavec.api.writable.DoubleWritable; import org.datavec.api.writable.Writable; import org.datavec.spark.BaseSparkTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -43,13 +40,9 @@ import org.nd4j.linalg.factory.Nd4j; import java.util.ArrayList; import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +import static org.junit.jupiter.api.Assertions.assertEquals; + public class NormalizationTests extends BaseSparkTest { diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/analysis/TestAnalysis.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/analysis/TestAnalysis.java similarity index 98% rename from datavec/datavec-spark/src/test/java/org/datavec/spark/transform/analysis/TestAnalysis.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/analysis/TestAnalysis.java index 90713a7af..4fc4f3323 100644 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/analysis/TestAnalysis.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/analysis/TestAnalysis.java @@ -38,9 +38,7 @@ import org.datavec.local.transforms.AnalyzeLocal; import org.datavec.spark.BaseSparkTest; import org.datavec.spark.transform.AnalyzeSpark; import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.io.ClassPathResource; @@ -50,10 +48,7 @@ import java.nio.file.Files; import java.util.*; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) + public class TestAnalysis extends BaseSparkTest { @Test diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/join/TestJoin.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/join/TestJoin.java new file mode 100644 index 000000000..29da7a0a4 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/join/TestJoin.java @@ -0,0 +1,236 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.transform.join; + +import org.apache.spark.api.java.JavaRDD; +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.join.Join; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.*; +import org.datavec.spark.BaseSparkTest; +import org.datavec.spark.transform.SparkTransformExecutor; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestJoin extends BaseSparkTest { + + @Test + public void testJoinOneToMany_ManyToOne() { + + Schema customerInfoSchema = + new Schema.Builder().addColumnLong("customerID").addColumnString("customerName").build(); + + Schema purchasesSchema = new Schema.Builder().addColumnLong("purchaseID").addColumnLong("customerID") + .addColumnDouble("amount").build(); + + List> infoList = new ArrayList<>(); + infoList.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"))); + infoList.add(Arrays.asList(new LongWritable(98765), new Text("Customer98765"))); + infoList.add(Arrays.asList(new LongWritable(50000), new Text("Customer50000"))); + + List> purchaseList = new ArrayList<>(); + purchaseList.add(Arrays.asList(new LongWritable(1000000), new LongWritable(12345), + new DoubleWritable(10.00))); + purchaseList.add(Arrays.asList(new LongWritable(1000001), new LongWritable(12345), + new DoubleWritable(20.00))); + purchaseList.add(Arrays.asList(new LongWritable(1000002), new LongWritable(98765), + new DoubleWritable(30.00))); + + Join join = new Join.Builder(Join.JoinType.RightOuter).setJoinColumns("customerID") + .setSchemas(customerInfoSchema, purchasesSchema).build(); + + List> expected = new ArrayList<>(); + expected.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"), + new LongWritable(1000000), new DoubleWritable(10.00))); + expected.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"), + new LongWritable(1000001), new DoubleWritable(20.00))); + expected.add(Arrays.asList(new LongWritable(98765), new Text("Customer98765"), + new LongWritable(1000002), new DoubleWritable(30.00))); + + + + JavaRDD> info = sc.parallelize(infoList); + JavaRDD> purchases = sc.parallelize(purchaseList); + + JavaRDD> joined = SparkTransformExecutor.executeJoin(join, info, purchases); + List> joinedList = new ArrayList<>(joined.collect()); + //Sort by order ID (column 3, index 2) + Collections.sort(joinedList, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Long.compare(o1.get(2).toLong(), o2.get(2).toLong()); + } + }); + assertEquals(expected, joinedList); + + assertEquals(3, joinedList.size()); + + List expectedColNames = Arrays.asList("customerID", "customerName", "purchaseID", "amount"); + assertEquals(expectedColNames, join.getOutputSchema().getColumnNames()); + + List expectedColTypes = + Arrays.asList(ColumnType.Long, ColumnType.String, ColumnType.Long, ColumnType.Double); + assertEquals(expectedColTypes, join.getOutputSchema().getColumnTypes()); + + + //Test Many to one: same thing, but swap the order... + Join join2 = new Join.Builder(Join.JoinType.LeftOuter).setJoinColumns("customerID") + .setSchemas(purchasesSchema, customerInfoSchema).build(); + + List> expectedManyToOne = new ArrayList<>(); + expectedManyToOne.add(Arrays.asList(new LongWritable(1000000), new LongWritable(12345), + new DoubleWritable(10.00), new Text("Customer12345"))); + expectedManyToOne.add(Arrays.asList(new LongWritable(1000001), new LongWritable(12345), + new DoubleWritable(20.00), new Text("Customer12345"))); + expectedManyToOne.add(Arrays.asList(new LongWritable(1000002), new LongWritable(98765), + new DoubleWritable(30.00), new Text("Customer98765"))); + + JavaRDD> joined2 = SparkTransformExecutor.executeJoin(join2, purchases, info); + List> joinedList2 = new ArrayList<>(joined2.collect()); + //Sort by order ID (column 0) + Collections.sort(joinedList2, new Comparator>() { + @Override + public int compare(List o1, List o2) { + return Long.compare(o1.get(0).toLong(), o2.get(0).toLong()); + } + }); + assertEquals(3, joinedList2.size()); + + assertEquals(expectedManyToOne, joinedList2); + + List expectedColNames2 = Arrays.asList("purchaseID", "customerID", "amount", "customerName"); + assertEquals(expectedColNames2, join2.getOutputSchema().getColumnNames()); + + List expectedColTypes2 = + Arrays.asList(ColumnType.Long, ColumnType.Long, ColumnType.Double, ColumnType.String); + assertEquals(expectedColTypes2, join2.getOutputSchema().getColumnTypes()); + } + + + @Test + public void testJoinManyToMany() { + Schema schema1 = new Schema.Builder().addColumnLong("id") + .addColumnCategorical("category", Arrays.asList("cat0", "cat1", "cat2")).build(); + + Schema schema2 = new Schema.Builder().addColumnLong("otherId") + .addColumnCategorical("otherCategory", Arrays.asList("cat0", "cat1", "cat2")).build(); + + List> first = new ArrayList<>(); + first.add(Arrays.asList(new LongWritable(0), new Text("cat0"))); + first.add(Arrays.asList(new LongWritable(1), new Text("cat0"))); + first.add(Arrays.asList(new LongWritable(2), new Text("cat1"))); + + List> second = new ArrayList<>(); + second.add(Arrays.asList(new LongWritable(100), new Text("cat0"))); + second.add(Arrays.asList(new LongWritable(101), new Text("cat0"))); + second.add(Arrays.asList(new LongWritable(102), new Text("cat2"))); + + + + List> expOuterJoin = new ArrayList<>(); + expOuterJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); + expOuterJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); + expOuterJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); + expOuterJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); + expOuterJoin.add(Arrays.asList(new LongWritable(2), new Text("cat1"), new NullWritable())); + expOuterJoin.add(Arrays.asList(new NullWritable(), new Text("cat2"), new LongWritable(102))); + + List> expLeftJoin = new ArrayList<>(); + expLeftJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); + expLeftJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); + expLeftJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); + expLeftJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); + expLeftJoin.add(Arrays.asList(new LongWritable(2), new Text("cat1"), new NullWritable())); + + + List> expRightJoin = new ArrayList<>(); + expRightJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); + expRightJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); + expRightJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); + expRightJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); + expRightJoin.add(Arrays.asList(new NullWritable(), new Text("cat2"), new LongWritable(102))); + + List> expInnerJoin = new ArrayList<>(); + expInnerJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); + expInnerJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); + expInnerJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); + expInnerJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); + + JavaRDD> firstRDD = sc.parallelize(first); + JavaRDD> secondRDD = sc.parallelize(second); + + int count = 0; + for (Join.JoinType jt : Join.JoinType.values()) { + Join join = new Join.Builder(jt).setJoinColumnsLeft("category").setJoinColumnsRight("otherCategory") + .setSchemas(schema1, schema2).build(); + List> out = + new ArrayList<>(SparkTransformExecutor.executeJoin(join, firstRDD, secondRDD).collect()); + + //Sort output by column 0, then column 1, then column 2 for comparison to expected... + Collections.sort(out, new Comparator>() { + @Override + public int compare(List o1, List o2) { + Writable w1 = o1.get(0); + Writable w2 = o2.get(0); + if (w1 instanceof NullWritable) + return 1; + else if (w2 instanceof NullWritable) + return -1; + int c = Long.compare(w1.toLong(), w2.toLong()); + if (c != 0) + return c; + c = o1.get(1).toString().compareTo(o2.get(1).toString()); + if (c != 0) + return c; + w1 = o1.get(2); + w2 = o2.get(2); + if (w1 instanceof NullWritable) + return 1; + else if (w2 instanceof NullWritable) + return -1; + return Long.compare(w1.toLong(), w2.toLong()); + } + }); + + switch (jt) { + case Inner: + assertEquals(expInnerJoin, out); + break; + case LeftOuter: + assertEquals(expLeftJoin, out); + break; + case RightOuter: + assertEquals(expRightJoin, out); + break; + case FullOuter: + assertEquals(expOuterJoin, out); + break; + } + count++; + } + + assertEquals(4, count); + } + +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/rank/TestCalculateSortedRank.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/rank/TestCalculateSortedRank.java new file mode 100644 index 000000000..6ff564418 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/rank/TestCalculateSortedRank.java @@ -0,0 +1,78 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.transform.rank; + +import org.apache.spark.api.java.JavaRDD; +import org.datavec.api.transform.ColumnType; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.api.writable.comparator.DoubleWritableComparator; +import org.datavec.spark.BaseSparkTest; +import org.datavec.spark.transform.SparkTransformExecutor; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestCalculateSortedRank extends BaseSparkTest { + + @Test + public void testCalculateSortedRank() { + + List> data = new ArrayList<>(); + data.add(Arrays.asList((Writable) new Text("0"), new DoubleWritable(0.0))); + data.add(Arrays.asList((Writable) new Text("3"), new DoubleWritable(0.3))); + data.add(Arrays.asList((Writable) new Text("2"), new DoubleWritable(0.2))); + data.add(Arrays.asList((Writable) new Text("1"), new DoubleWritable(0.1))); + + JavaRDD> rdd = sc.parallelize(data); + + Schema schema = new Schema.Builder().addColumnsString("TextCol").addColumnDouble("DoubleCol").build(); + + TransformProcess tp = new TransformProcess.Builder(schema) + .calculateSortedRank("rank", "DoubleCol", new DoubleWritableComparator()).build(); + + Schema outSchema = tp.getFinalSchema(); + assertEquals(3, outSchema.numColumns()); + assertEquals(Arrays.asList("TextCol", "DoubleCol", "rank"), outSchema.getColumnNames()); + assertEquals(Arrays.asList(ColumnType.String, ColumnType.Double, ColumnType.Long), outSchema.getColumnTypes()); + + JavaRDD> out = SparkTransformExecutor.execute(rdd, tp); + + List> collected = out.collect(); + assertEquals(4, collected.size()); + for (int i = 0; i < 4; i++) + assertEquals(3, collected.get(i).size()); + + for (List example : collected) { + int exampleNum = example.get(0).toInt(); + int rank = example.get(2).toInt(); + assertEquals(exampleNum, rank); + } + } + +} diff --git a/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/sequence/TestConvertToSequence.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/sequence/TestConvertToSequence.java new file mode 100644 index 000000000..7faca7235 --- /dev/null +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/transform/sequence/TestConvertToSequence.java @@ -0,0 +1,117 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.datavec.spark.transform.sequence; + +import org.apache.spark.api.java.JavaRDD; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.sequence.comparator.NumericalColumnComparator; +import org.datavec.api.writable.LongWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.spark.BaseSparkTest; +import org.datavec.spark.transform.SparkTransformExecutor; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestConvertToSequence extends BaseSparkTest { + + @Test + public void testConvertToSequenceCompoundKey() { + + Schema s = new Schema.Builder().addColumnsString("key1", "key2").addColumnLong("time").build(); + + List> allExamples = + Arrays.asList(Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(10)), + Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(10)), + Arrays.asList(new Text("k1a"), new Text("k2a"), + new LongWritable(-10)), + Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(5)), + Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(0))); + + TransformProcess tp = new TransformProcess.Builder(s) + .convertToSequence(Arrays.asList("key1", "key2"), new NumericalColumnComparator("time")) + .build(); + + JavaRDD> rdd = sc.parallelize(allExamples); + + List>> out = SparkTransformExecutor.executeToSequence(rdd, tp).collect(); + + assertEquals(2, out.size()); + List> seq0; + List> seq1; + if (out.get(0).size() == 3) { + seq0 = out.get(0); + seq1 = out.get(1); + } else { + seq0 = out.get(1); + seq1 = out.get(0); + } + + List> expSeq0 = Arrays.asList( + Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(-10)), + Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(0)), + Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(10))); + + List> expSeq1 = Arrays.asList( + Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(5)), + Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(10))); + + assertEquals(expSeq0, seq0); + assertEquals(expSeq1, seq1); + } + + @Test + public void testConvertToSequenceLength1(){ + + Schema s = new Schema.Builder() + .addColumnsString("string") + .addColumnLong("long") + .build(); + + List> allExamples = Arrays.asList( + Arrays.asList(new Text("a"), new LongWritable(0)), + Arrays.asList(new Text("b"), new LongWritable(1)), + Arrays.asList(new Text("c"), new LongWritable(2))); + + TransformProcess tp = new TransformProcess.Builder(s) + .convertToSequence() + .build(); + + JavaRDD> rdd = sc.parallelize(allExamples); + + JavaRDD>> out = SparkTransformExecutor.executeToSequence(rdd, tp); + + List>> out2 = out.collect(); + + assertEquals(3, out2.size()); + + for( int i=0; i<3; i++ ){ + assertTrue(out2.contains(Collections.singletonList(allExamples.get(i)))); + } + } +} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/util/TestSparkUtil.java b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/util/TestSparkUtil.java similarity index 86% rename from datavec/datavec-spark/src/test/java/org/datavec/spark/util/TestSparkUtil.java rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/util/TestSparkUtil.java index 92a4e968a..a2dd04ce0 100644 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/util/TestSparkUtil.java +++ b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/java/org/datavec/spark/util/TestSparkUtil.java @@ -28,9 +28,7 @@ import org.datavec.api.writable.Text; import org.datavec.api.writable.Writable; import org.datavec.spark.BaseSparkTest; import org.datavec.spark.transform.utils.SparkUtils; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import java.io.FileInputStream; @@ -40,9 +38,6 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.SPARK) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) public class TestSparkUtil extends BaseSparkTest { @Test @@ -51,8 +46,8 @@ public class TestSparkUtil extends BaseSparkTest { return; } List> l = new ArrayList<>(); - l.add(Arrays.asList(new Text("abc"), new DoubleWritable(2.0), new IntWritable(-1))); - l.add(Arrays.asList(new Text("def"), new DoubleWritable(4.0), new IntWritable(-2))); + l.add(Arrays.asList(new Text("abc"), new DoubleWritable(2.0), new IntWritable(-1))); + l.add(Arrays.asList(new Text("def"), new DoubleWritable(4.0), new IntWritable(-2))); File f = File.createTempFile("testSparkUtil", "txt"); f.deleteOnExit(); diff --git a/datavec/datavec-spark/src/test/resources/log4j.properties b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/resources/log4j.properties similarity index 100% rename from datavec/datavec-spark/src/test/resources/log4j.properties rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/resources/log4j.properties diff --git a/datavec/datavec-spark/src/test/resources/logback.xml b/cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/resources/logback.xml similarity index 100% rename from datavec/datavec-spark/src/test/resources/logback.xml rename to cavis-datavec/cavis-datavec-spark/cavis-datavec-spark-core/src/test/resources/logback.xml diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/pom.xml b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/pom.xml new file mode 100644 index 000000000..ee4269a55 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/pom.xml @@ -0,0 +1,57 @@ + + + + + datavec-spark-inference-parent + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + datavec-spark-inference-client + jar + + datavec-spark-inference-client + + + + + net.brutex.ai + datavec-spark-inference-server_2.11 + 1.0.0-SNAPSHOT + test + + + com.mashape.unirest + unirest-java + ${unirest.version} + + + net.brutex.ai + datavec-spark-inference-model + ${project.version} + + + + net.brutex.ai + nd4j-common-tests + ${project.version} + test + + + diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/main/java/org/datavec/spark/inference/client/DataVecTransformClient.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/main/java/org/datavec/spark/inference/client/DataVecTransformClient.java new file mode 100644 index 000000000..b55755c63 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/main/java/org/datavec/spark/inference/client/DataVecTransformClient.java @@ -0,0 +1,291 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.client; + + +import com.mashape.unirest.http.ObjectMapper; +import com.mashape.unirest.http.Unirest; +import com.mashape.unirest.http.exceptions.UnirestException; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.datavec.api.transform.TransformProcess; +import org.datavec.image.transform.ImageTransformProcess; +import org.datavec.spark.inference.model.model.*; +import org.datavec.spark.inference.model.service.DataVecTransformService; +import com.fasterxml.jackson.core.JsonProcessingException; + +import java.io.IOException; + +/** + * Created by agibsonccc on 6/12/17. + */ +@AllArgsConstructor +@Slf4j +public class DataVecTransformClient implements DataVecTransformService { + private String url; + + static { + // Only one time + Unirest.setObjectMapper(new ObjectMapper() { + private com.fasterxml.jackson.databind.ObjectMapper jacksonObjectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public T readValue(String value, Class valueType) { + try { + return jacksonObjectMapper.readValue(value, valueType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String writeValue(Object value) { + try { + return jacksonObjectMapper.writeValueAsString(value); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + }); + } + + /** + * @param transformProcess + */ + @Override + public void setCSVTransformProcess(TransformProcess transformProcess) { + try { + String s = transformProcess.toJson(); + Unirest.post(url + "/transformprocess").header("accept", "application/json") + .header("Content-Type", "application/json").body(s).asJson(); + + } catch (UnirestException e) { + log.error("Error in setCSVTransformProcess()", e); + } + } + + @Override + public void setImageTransformProcess(ImageTransformProcess imageTransformProcess) { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + /** + * @return + */ + @Override + public TransformProcess getCSVTransformProcess() { + try { + String s = Unirest.get(url + "/transformprocess").header("accept", "application/json") + .header("Content-Type", "application/json").asString().getBody(); + return TransformProcess.fromJson(s); + } catch (UnirestException e) { + log.error("Error in getCSVTransformProcess()",e); + } + + return null; + } + + @Override + public ImageTransformProcess getImageTransformProcess() { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + /** + * @param transform + * @return + */ + @Override + public SingleCSVRecord transformIncremental(SingleCSVRecord transform) { + try { + SingleCSVRecord singleCsvRecord = Unirest.post(url + "/transformincremental") + .header("accept", "application/json") + .header("Content-Type", "application/json") + .body(transform).asObject(SingleCSVRecord.class).getBody(); + return singleCsvRecord; + } catch (UnirestException e) { + log.error("Error in transformIncremental(SingleCSVRecord)",e); + } + return null; + } + + + /** + * @param batchCSVRecord + * @return + */ + @Override + public SequenceBatchCSVRecord transform(SequenceBatchCSVRecord batchCSVRecord) { + try { + SequenceBatchCSVRecord batchCSVRecord1 = Unirest.post(url + "/transform").header("accept", "application/json") + .header("Content-Type", "application/json") + .header(SEQUENCE_OR_NOT_HEADER,"TRUE") + .body(batchCSVRecord) + .asObject(SequenceBatchCSVRecord.class) + .getBody(); + return batchCSVRecord1; + } catch (UnirestException e) { + log.error("",e); + } + + return null; + } + /** + * @param batchCSVRecord + * @return + */ + @Override + public BatchCSVRecord transform(BatchCSVRecord batchCSVRecord) { + try { + BatchCSVRecord batchCSVRecord1 = Unirest.post(url + "/transform").header("accept", "application/json") + .header("Content-Type", "application/json") + .header(SEQUENCE_OR_NOT_HEADER,"FALSE") + .body(batchCSVRecord) + .asObject(BatchCSVRecord.class) + .getBody(); + return batchCSVRecord1; + } catch (UnirestException e) { + log.error("Error in transform(BatchCSVRecord)", e); + } + + return null; + } + + /** + * @param batchCSVRecord + * @return + */ + @Override + public Base64NDArrayBody transformArray(BatchCSVRecord batchCSVRecord) { + try { + Base64NDArrayBody batchArray1 = Unirest.post(url + "/transformarray").header("accept", "application/json") + .header("Content-Type", "application/json").body(batchCSVRecord) + .asObject(Base64NDArrayBody.class).getBody(); + return batchArray1; + } catch (UnirestException e) { + log.error("Error in transformArray(BatchCSVRecord)",e); + } + + return null; + } + + /** + * @param singleCsvRecord + * @return + */ + @Override + public Base64NDArrayBody transformArrayIncremental(SingleCSVRecord singleCsvRecord) { + try { + Base64NDArrayBody array = Unirest.post(url + "/transformincrementalarray") + .header("accept", "application/json").header("Content-Type", "application/json") + .body(singleCsvRecord).asObject(Base64NDArrayBody.class).getBody(); + return array; + } catch (UnirestException e) { + log.error("Error in transformArrayIncremental(SingleCSVRecord)",e); + } + + return null; + } + + @Override + public Base64NDArrayBody transformIncrementalArray(SingleImageRecord singleImageRecord) throws IOException { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public Base64NDArrayBody transformArray(BatchImageRecord batchImageRecord) throws IOException { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + /** + * @param singleCsvRecord + * @return + */ + @Override + public Base64NDArrayBody transformSequenceArrayIncremental(BatchCSVRecord singleCsvRecord) { + try { + Base64NDArrayBody array = Unirest.post(url + "/transformincrementalarray") + .header("accept", "application/json") + .header("Content-Type", "application/json") + .header(SEQUENCE_OR_NOT_HEADER,"true") + .body(singleCsvRecord).asObject(Base64NDArrayBody.class).getBody(); + return array; + } catch (UnirestException e) { + log.error("Error in transformSequenceArrayIncremental",e); + } + + return null; + } + + /** + * @param batchCSVRecord + * @return + */ + @Override + public Base64NDArrayBody transformSequenceArray(SequenceBatchCSVRecord batchCSVRecord) { + try { + Base64NDArrayBody batchArray1 = Unirest.post(url + "/transformarray").header("accept", "application/json") + .header("Content-Type", "application/json") + .header(SEQUENCE_OR_NOT_HEADER,"true") + .body(batchCSVRecord) + .asObject(Base64NDArrayBody.class).getBody(); + return batchArray1; + } catch (UnirestException e) { + log.error("Error in transformSequenceArray",e); + } + + return null; + } + + /** + * @param batchCSVRecord + * @return + */ + @Override + public SequenceBatchCSVRecord transformSequence(SequenceBatchCSVRecord batchCSVRecord) { + try { + SequenceBatchCSVRecord batchCSVRecord1 = Unirest.post(url + "/transform") + .header("accept", "application/json") + .header("Content-Type", "application/json") + .header(SEQUENCE_OR_NOT_HEADER,"true") + .body(batchCSVRecord) + .asObject(SequenceBatchCSVRecord.class).getBody(); + return batchCSVRecord1; + } catch (UnirestException e) { + log.error("Error in transformSequence"); + } + + return null; + } + + /** + * @param transform + * @return + */ + @Override + public SequenceBatchCSVRecord transformSequenceIncremental(BatchCSVRecord transform) { + try { + SequenceBatchCSVRecord singleCsvRecord = Unirest.post(url + "/transformincremental") + .header("accept", "application/json") + .header("Content-Type", "application/json") + .header(SEQUENCE_OR_NOT_HEADER,"true") + .body(transform).asObject(SequenceBatchCSVRecord.class).getBody(); + return singleCsvRecord; + } catch (UnirestException e) { + log.error("Error in transformSequenceIncremental"); + } + return null; + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/java/org/datavec/transform/client/AssertTestsExtendBaseClass.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/java/org/datavec/transform/client/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..a919a8fd9 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/java/org/datavec/transform/client/AssertTestsExtendBaseClass.java @@ -0,0 +1,49 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.transform.client; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.transform.client"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/java/org/datavec/transform/client/DataVecTransformClientTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/java/org/datavec/transform/client/DataVecTransformClientTest.java new file mode 100644 index 000000000..a4215b381 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/java/org/datavec/transform/client/DataVecTransformClientTest.java @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.transform.client; + +import org.apache.commons.io.FileUtils; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.spark.inference.server.CSVSparkTransformServer; +import org.datavec.spark.inference.client.DataVecTransformClient; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchCSVRecord; +import org.datavec.spark.inference.model.model.SequenceBatchCSVRecord; +import org.datavec.spark.inference.model.model.SingleCSVRecord; +import org.junit.AfterClass; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.serde.base64.Nd4jBase64; + +import java.io.File; +import java.io.IOException; +import java.net.ServerSocket; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeNotNull; + +/** + * Created by agibsonccc on 6/12/17. + */ +public class DataVecTransformClientTest { + private static CSVSparkTransformServer server; + private static int port = getAvailablePort(); + private static DataVecTransformClient client; + private static Schema schema = new Schema.Builder().addColumnDouble("1.0").addColumnDouble("2.0").build(); + private static TransformProcess transformProcess = + new TransformProcess.Builder(schema).convertToDouble("1.0").convertToDouble("2.0").build(); + private static File fileSave = new File(UUID.randomUUID().toString() + ".json"); + + @BeforeAll + public static void beforeClass() throws Exception { + FileUtils.write(fileSave, transformProcess.toJson()); + fileSave.deleteOnExit(); + server = new CSVSparkTransformServer(); + server.runMain(new String[] {"-dp", String.valueOf(port)}); + + client = new DataVecTransformClient("http://localhost:" + port); + client.setCSVTransformProcess(transformProcess); + } + + @AfterClass + public static void afterClass() throws Exception { + server.stop(); + } + + + @Test + public void testSequenceClient() { + SequenceBatchCSVRecord sequenceBatchCSVRecord = new SequenceBatchCSVRecord(); + SingleCSVRecord singleCsvRecord = new SingleCSVRecord(new String[] {"0", "0"}); + + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(Arrays.asList(singleCsvRecord, singleCsvRecord)); + List batchCSVRecordList = new ArrayList<>(); + for(int i = 0; i < 5; i++) { + batchCSVRecordList.add(batchCSVRecord); + } + + sequenceBatchCSVRecord.add(batchCSVRecordList); + + SequenceBatchCSVRecord sequenceBatchCSVRecord1 = client.transformSequence(sequenceBatchCSVRecord); + assumeNotNull(sequenceBatchCSVRecord1); + + Base64NDArrayBody array = client.transformSequenceArray(sequenceBatchCSVRecord); + assumeNotNull(array); + + Base64NDArrayBody incrementalBody = client.transformSequenceArrayIncremental(batchCSVRecord); + assumeNotNull(incrementalBody); + + Base64NDArrayBody incrementalSequenceBody = client.transformSequenceArrayIncremental(batchCSVRecord); + assumeNotNull(incrementalSequenceBody); + } + + @Test + public void testRecord() throws Exception { + SingleCSVRecord singleCsvRecord = new SingleCSVRecord(new String[] {"0", "0"}); + SingleCSVRecord transformed = client.transformIncremental(singleCsvRecord); + assertEquals(singleCsvRecord.getValues().size(), transformed.getValues().size()); + Base64NDArrayBody body = client.transformArrayIncremental(singleCsvRecord); + INDArray arr = Nd4jBase64.fromBase64(body.getNdarray()); + assumeNotNull(arr); + } + + @Test + public void testBatchRecord() throws Exception { + SingleCSVRecord singleCsvRecord = new SingleCSVRecord(new String[] {"0", "0"}); + + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(Arrays.asList(singleCsvRecord, singleCsvRecord)); + BatchCSVRecord batchCSVRecord1 = client.transform(batchCSVRecord); + assertEquals(batchCSVRecord.getRecords().size(), batchCSVRecord1.getRecords().size()); + + Base64NDArrayBody body = client.transformArray(batchCSVRecord); + INDArray arr = Nd4jBase64.fromBase64(body.getNdarray()); + assumeNotNull(arr); + } + + + + public static int getAvailablePort() { + try { + ServerSocket socket = new ServerSocket(0); + try { + return socket.getLocalPort(); + } finally { + socket.close(); + } + } catch (IOException e) { + throw new IllegalStateException("Cannot find available port: " + e.getMessage(), e); + } + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/resources/application.conf b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/resources/application.conf new file mode 100644 index 000000000..dbac92d83 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-client/src/test/resources/application.conf @@ -0,0 +1,6 @@ +play.modules.enabled += com.lightbend.lagom.discovery.zookeeper.ZooKeeperServiceLocatorModule +play.modules.enabled += io.skymind.skil.service.PredictionModule +play.crypto.secret = as8dufasdfuasdfjkasdkfalksjfk +play.server.pidfile.path=/tmp/RUNNING_PID + +play.server.http.port = 9600 diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/pom.xml b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/pom.xml new file mode 100644 index 000000000..29b2784f1 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/pom.xml @@ -0,0 +1,56 @@ + + + + + datavec-spark-inference-parent + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + datavec-spark-inference-model + jar + + datavec-spark-inference-model + + + + + net.brutex.ai + datavec-api + ${project.version} + + + net.brutex.ai + datavec-data-image + ${project.version} + + + net.brutex.ai + datavec-local + ${project.version} + + + + net.brutex.ai + nd4j-common-tests + ${project.version} + test + + + diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/CSVSparkTransform.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/CSVSparkTransform.java new file mode 100644 index 000000000..6b3c80800 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/CSVSparkTransform.java @@ -0,0 +1,288 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.FieldVector; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.util.ndarray.RecordConverter; +import org.datavec.api.writable.Writable; +import org.datavec.arrow.ArrowConverter; +import org.datavec.arrow.recordreader.ArrowWritableRecordBatch; +import org.datavec.arrow.recordreader.ArrowWritableRecordTimeSeriesBatch; +import org.datavec.local.transforms.LocalTransformExecutor; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchCSVRecord; +import org.datavec.spark.inference.model.model.SequenceBatchCSVRecord; +import org.datavec.spark.inference.model.model.SingleCSVRecord; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.serde.base64.Nd4jBase64; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.datavec.arrow.ArrowConverter.*; +import static org.datavec.local.transforms.LocalTransformExecutor.execute; +import static org.datavec.local.transforms.LocalTransformExecutor.executeToSequence; + +/** + * CSVSpark Transform runs + * the actual {@link TransformProcess} + * + * @author Adan Gibson + */ +@AllArgsConstructor +@Slf4j +public class CSVSparkTransform { + @Getter + private TransformProcess transformProcess; + private static BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE); + + /** + * Convert a raw record via + * the {@link TransformProcess} + * to a base 64ed ndarray + * @param batch the record to convert + * @return teh base 64ed ndarray + * @throws IOException + */ + public Base64NDArrayBody toArray(BatchCSVRecord batch) throws IOException { + List> converted = execute(toArrowWritables(toArrowColumnsString( + bufferAllocator,transformProcess.getInitialSchema(), + batch.getRecordsAsString()), + transformProcess.getInitialSchema()),transformProcess); + + ArrowWritableRecordBatch arrowRecordBatch = (ArrowWritableRecordBatch) converted; + INDArray convert = ArrowConverter.toArray(arrowRecordBatch); + return new Base64NDArrayBody(Nd4jBase64.base64String(convert)); + } + + /** + * Convert a raw record via + * the {@link TransformProcess} + * to a base 64ed ndarray + * @param record the record to convert + * @return the base 64ed ndarray + * @throws IOException + */ + public Base64NDArrayBody toArray(SingleCSVRecord record) throws IOException { + List record2 = toArrowWritablesSingle( + toArrowColumnsStringSingle(bufferAllocator, + transformProcess.getInitialSchema(),record.getValues()), + transformProcess.getInitialSchema()); + List finalRecord = execute(Arrays.asList(record2),transformProcess).get(0); + INDArray convert = RecordConverter.toArray(DataType.DOUBLE, finalRecord); + return new Base64NDArrayBody(Nd4jBase64.base64String(convert)); + } + + /** + * Runs the transform process + * @param batch the record to transform + * @return the transformed record + */ + public BatchCSVRecord transform(BatchCSVRecord batch) { + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(); + List> converted = execute(toArrowWritables(toArrowColumnsString( + bufferAllocator,transformProcess.getInitialSchema(), + batch.getRecordsAsString()), + transformProcess.getInitialSchema()),transformProcess); + int numCols = converted.get(0).size(); + for (int row = 0; row < converted.size(); row++) { + String[] values = new String[numCols]; + for (int i = 0; i < values.length; i++) + values[i] = converted.get(row).get(i).toString(); + batchCSVRecord.add(new SingleCSVRecord(values)); + } + + return batchCSVRecord; + + } + + /** + * Runs the transform process + * @param record the record to transform + * @return the transformed record + */ + public SingleCSVRecord transform(SingleCSVRecord record) { + List record2 = toArrowWritablesSingle( + toArrowColumnsStringSingle(bufferAllocator, + transformProcess.getInitialSchema(),record.getValues()), + transformProcess.getInitialSchema()); + List finalRecord = execute(Arrays.asList(record2),transformProcess).get(0); + String[] values = new String[finalRecord.size()]; + for (int i = 0; i < values.length; i++) + values[i] = finalRecord.get(i).toString(); + return new SingleCSVRecord(values); + + } + + /** + * + * @param transform + * @return + */ + public SequenceBatchCSVRecord transformSequenceIncremental(BatchCSVRecord transform) { + /** + * Sequence schema? + */ + List>> converted = executeToSequence( + toArrowWritables(toArrowColumnsStringTimeSeries( + bufferAllocator, transformProcess.getInitialSchema(), + Arrays.asList(transform.getRecordsAsString())), + transformProcess.getInitialSchema()), transformProcess); + + SequenceBatchCSVRecord batchCSVRecord = new SequenceBatchCSVRecord(); + for (int i = 0; i < converted.size(); i++) { + BatchCSVRecord batchCSVRecord1 = BatchCSVRecord.fromWritables(converted.get(i)); + batchCSVRecord.add(Arrays.asList(batchCSVRecord1)); + } + + return batchCSVRecord; + } + + /** + * + * @param batchCSVRecordSequence + * @return + */ + public SequenceBatchCSVRecord transformSequence(SequenceBatchCSVRecord batchCSVRecordSequence) { + List>> recordsAsString = batchCSVRecordSequence.getRecordsAsString(); + boolean allSameLength = true; + Integer length = null; + for(List> record : recordsAsString) { + if(length == null) { + length = record.size(); + } + else if(record.size() != length) { + allSameLength = false; + } + } + + if(allSameLength) { + List fieldVectors = toArrowColumnsStringTimeSeries(bufferAllocator, transformProcess.getInitialSchema(), recordsAsString); + ArrowWritableRecordTimeSeriesBatch arrowWritableRecordTimeSeriesBatch = new ArrowWritableRecordTimeSeriesBatch(fieldVectors, + transformProcess.getInitialSchema(), + recordsAsString.get(0).get(0).size()); + val transformed = LocalTransformExecutor.executeSequenceToSequence(arrowWritableRecordTimeSeriesBatch,transformProcess); + return SequenceBatchCSVRecord.fromWritables(transformed); + } + + else { + val transformed = LocalTransformExecutor.executeSequenceToSequence(LocalTransformExecutor.convertStringInputTimeSeries(batchCSVRecordSequence.getRecordsAsString(),transformProcess.getInitialSchema()),transformProcess); + return SequenceBatchCSVRecord.fromWritables(transformed); + + } + } + + /** + * TODO: optimize + * @param batchCSVRecordSequence + * @return + */ + public Base64NDArrayBody transformSequenceArray(SequenceBatchCSVRecord batchCSVRecordSequence) { + List>> strings = batchCSVRecordSequence.getRecordsAsString(); + boolean allSameLength = true; + Integer length = null; + for(List> record : strings) { + if(length == null) { + length = record.size(); + } + else if(record.size() != length) { + allSameLength = false; + } + } + + if(allSameLength) { + List fieldVectors = toArrowColumnsStringTimeSeries(bufferAllocator, transformProcess.getInitialSchema(), strings); + ArrowWritableRecordTimeSeriesBatch arrowWritableRecordTimeSeriesBatch = new ArrowWritableRecordTimeSeriesBatch(fieldVectors,transformProcess.getInitialSchema(),strings.get(0).get(0).size()); + val transformed = LocalTransformExecutor.executeSequenceToSequence(arrowWritableRecordTimeSeriesBatch,transformProcess); + INDArray arr = RecordConverter.toTensor(transformed).reshape(strings.size(),strings.get(0).get(0).size(),strings.get(0).size()); + try { + return new Base64NDArrayBody(Nd4jBase64.base64String(arr)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + else { + val transformed = LocalTransformExecutor.executeSequenceToSequence(LocalTransformExecutor.convertStringInputTimeSeries(batchCSVRecordSequence.getRecordsAsString(),transformProcess.getInitialSchema()),transformProcess); + INDArray arr = RecordConverter.toTensor(transformed).reshape(strings.size(),strings.get(0).get(0).size(),strings.get(0).size()); + try { + return new Base64NDArrayBody(Nd4jBase64.base64String(arr)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + } + + /** + * + * @param singleCsvRecord + * @return + */ + public Base64NDArrayBody transformSequenceArrayIncremental(BatchCSVRecord singleCsvRecord) { + List>> converted = executeToSequence(toArrowWritables(toArrowColumnsString( + bufferAllocator,transformProcess.getInitialSchema(), + singleCsvRecord.getRecordsAsString()), + transformProcess.getInitialSchema()),transformProcess); + ArrowWritableRecordTimeSeriesBatch arrowWritableRecordBatch = (ArrowWritableRecordTimeSeriesBatch) converted; + INDArray arr = RecordConverter.toTensor(arrowWritableRecordBatch); + try { + return new Base64NDArrayBody(Nd4jBase64.base64String(arr)); + } catch (IOException e) { + log.error("",e); + } + + return null; + } + + public SequenceBatchCSVRecord transform(SequenceBatchCSVRecord batchCSVRecord) { + List>> strings = batchCSVRecord.getRecordsAsString(); + boolean allSameLength = true; + Integer length = null; + for(List> record : strings) { + if(length == null) { + length = record.size(); + } + else if(record.size() != length) { + allSameLength = false; + } + } + + if(allSameLength) { + List fieldVectors = toArrowColumnsStringTimeSeries(bufferAllocator, transformProcess.getInitialSchema(), strings); + ArrowWritableRecordTimeSeriesBatch arrowWritableRecordTimeSeriesBatch = new ArrowWritableRecordTimeSeriesBatch(fieldVectors,transformProcess.getInitialSchema(),strings.get(0).get(0).size()); + val transformed = LocalTransformExecutor.executeSequenceToSequence(arrowWritableRecordTimeSeriesBatch,transformProcess); + return SequenceBatchCSVRecord.fromWritables(transformed); + } + + else { + val transformed = LocalTransformExecutor.executeSequenceToSequence(LocalTransformExecutor.convertStringInputTimeSeries(batchCSVRecord.getRecordsAsString(),transformProcess.getInitialSchema()),transformProcess); + return SequenceBatchCSVRecord.fromWritables(transformed); + + } + + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/ImageSparkTransform.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/ImageSparkTransform.java new file mode 100644 index 000000000..1e30d1ead --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/ImageSparkTransform.java @@ -0,0 +1,63 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.datavec.image.data.ImageWritable; +import org.datavec.image.transform.ImageTransformProcess; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchImageRecord; +import org.datavec.spark.inference.model.model.SingleImageRecord; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.serde.base64.Nd4jBase64; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Created by kepricon on 17. 5. 24. + */ +@AllArgsConstructor +public class ImageSparkTransform { + @Getter + private ImageTransformProcess imageTransformProcess; + + public Base64NDArrayBody toArray(SingleImageRecord record) throws IOException { + ImageWritable record2 = imageTransformProcess.transformFileUriToInput(record.getUri()); + INDArray finalRecord = imageTransformProcess.executeArray(record2); + + return new Base64NDArrayBody(Nd4jBase64.base64String(finalRecord)); + } + + public Base64NDArrayBody toArray(BatchImageRecord batch) throws IOException { + List records = new ArrayList<>(); + + for (SingleImageRecord imgRecord : batch.getRecords()) { + ImageWritable record2 = imageTransformProcess.transformFileUriToInput(imgRecord.getUri()); + INDArray finalRecord = imageTransformProcess.executeArray(record2); + records.add(finalRecord); + } + + INDArray array = Nd4j.concat(0, records.toArray(new INDArray[records.size()])); + + return new Base64NDArrayBody(Nd4jBase64.base64String(array)); + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/Base64NDArrayBody.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/Base64NDArrayBody.java new file mode 100644 index 000000000..b2a6b9dc3 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/Base64NDArrayBody.java @@ -0,0 +1,31 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Created by agibsonccc on 12/24/16. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class Base64NDArrayBody { + private String ndarray; +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/BatchCSVRecord.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/BatchCSVRecord.java new file mode 100644 index 000000000..bba2621c6 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/BatchCSVRecord.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.datavec.api.writable.Writable; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Created by agibsonccc on 1/21/17. + */ +@Data +@AllArgsConstructor +@Builder +@NoArgsConstructor +public class BatchCSVRecord implements Serializable { + private List records; + + + /** + * Get the records as a list of strings + * (basically the underlying values for + * {@link SingleCSVRecord}) + * @return + */ + public List> getRecordsAsString() { + if(records == null) + records = new ArrayList<>(); + List> ret = new ArrayList<>(); + for(SingleCSVRecord csvRecord : records) { + ret.add(csvRecord.getValues()); + } + return ret; + } + + + /** + * Create a batch csv record + * from a list of writables. + * @param batch + * @return + */ + public static BatchCSVRecord fromWritables(List> batch) { + List records = new ArrayList<>(batch.size()); + for(List list : batch) { + List add = new ArrayList<>(list.size()); + for(Writable writable : list) { + add.add(writable.toString()); + } + records.add(new SingleCSVRecord(add)); + } + + return BatchCSVRecord.builder().records(records).build(); + } + + + /** + * Add a record + * @param record + */ + public void add(SingleCSVRecord record) { + if (records == null) + records = new ArrayList<>(); + records.add(record); + } + + + /** + * Return a batch record based on a dataset + * @param dataSet the dataset to get the batch record for + * @return the batch record + */ + public static BatchCSVRecord fromDataSet(DataSet dataSet) { + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(); + for (int i = 0; i < dataSet.numExamples(); i++) { + batchCSVRecord.add(SingleCSVRecord.fromRow(dataSet.get(i))); + } + + return batchCSVRecord; + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/BatchImageRecord.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/BatchImageRecord.java new file mode 100644 index 000000000..5d7b52c4f --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/BatchImageRecord.java @@ -0,0 +1,49 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +/** + * Created by kepricon on 17. 5. 24. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class BatchImageRecord { + private List records; + + /** + * Add a record + * @param record + */ + public void add(SingleImageRecord record) { + if (records == null) + records = new ArrayList<>(); + records.add(record); + } + + public void add(URI uri) { + this.add(new SingleImageRecord(uri)); + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SequenceBatchCSVRecord.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SequenceBatchCSVRecord.java new file mode 100644 index 000000000..bbbb64f8d --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SequenceBatchCSVRecord.java @@ -0,0 +1,105 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.datavec.api.writable.Writable; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.MultiDataSet; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Created by agibsonccc on 1/21/17. + */ +@Data +@AllArgsConstructor +@Builder +@NoArgsConstructor +public class SequenceBatchCSVRecord implements Serializable { + private List> records; + + /** + * Add a record + * @param record + */ + public void add(List record) { + if (records == null) + records = new ArrayList<>(); + records.add(record); + } + + /** + * Get the records as a list of strings directly + * (this basically "unpacks" the objects) + * @return + */ + public List>> getRecordsAsString() { + if(records == null) + Collections.emptyList(); + List>> ret = new ArrayList<>(records.size()); + for(List record : records) { + List> add = new ArrayList<>(); + for(BatchCSVRecord batchCSVRecord : record) { + for (SingleCSVRecord singleCSVRecord : batchCSVRecord.getRecords()) { + add.add(singleCSVRecord.getValues()); + } + } + + ret.add(add); + } + + return ret; + } + + /** + * Convert a writables time series to a sequence batch + * @param input + * @return + */ + public static SequenceBatchCSVRecord fromWritables(List>> input) { + SequenceBatchCSVRecord ret = new SequenceBatchCSVRecord(); + for(int i = 0; i < input.size(); i++) { + ret.add(Arrays.asList(BatchCSVRecord.fromWritables(input.get(i)))); + } + + return ret; + } + + + /** + * Return a batch record based on a dataset + * @param dataSet the dataset to get the batch record for + * @return the batch record + */ + public static SequenceBatchCSVRecord fromDataSet(MultiDataSet dataSet) { + SequenceBatchCSVRecord batchCSVRecord = new SequenceBatchCSVRecord(); + for (int i = 0; i < dataSet.numFeatureArrays(); i++) { + batchCSVRecord.add(Arrays.asList(BatchCSVRecord.fromDataSet(new DataSet(dataSet.getFeatures(i),dataSet.getLabels(i))))); + } + + return batchCSVRecord; + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SingleCSVRecord.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SingleCSVRecord.java new file mode 100644 index 000000000..62ab812b4 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SingleCSVRecord.java @@ -0,0 +1,94 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; + +/** + * Created by agibsonccc on 12/24/16. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SingleCSVRecord implements Serializable { + private List values; + + /** + * Create from an array of values uses list internally) + * @param values + */ + public SingleCSVRecord(String...values) { + this.values = Arrays.asList(values); + } + + /** + * Instantiate a csv record from a vector + * given either an input dataset and a + * one hot matrix, the index will be appended to + * the end of the record, or for regression + * it will append all values in the labels + * @param row the input vectors + * @return the record from this {@link DataSet} + */ + public static SingleCSVRecord fromRow(DataSet row) { + if (!row.getFeatures().isVector() && !row.getFeatures().isScalar()) + throw new IllegalArgumentException("Passed in dataset must represent a scalar or vector"); + if (!row.getLabels().isVector() && !row.getLabels().isScalar()) + throw new IllegalArgumentException("Passed in dataset labels must be a scalar or vector"); + //classification + SingleCSVRecord record; + int idx = 0; + if (row.getLabels().sumNumber().doubleValue() == 1.0) { + String[] values = new String[row.getFeatures().columns() + 1]; + for (int i = 0; i < row.getFeatures().length(); i++) { + values[idx++] = String.valueOf(row.getFeatures().getDouble(i)); + } + int maxIdx = 0; + for (int i = 0; i < row.getLabels().length(); i++) { + if (row.getLabels().getDouble(maxIdx) < row.getLabels().getDouble(i)) { + maxIdx = i; + } + } + + values[idx++] = String.valueOf(maxIdx); + record = new SingleCSVRecord(values); + } + //regression (any number of values) + else { + String[] values = new String[row.getFeatures().columns() + row.getLabels().columns()]; + for (int i = 0; i < row.getFeatures().length(); i++) { + values[idx++] = String.valueOf(row.getFeatures().getDouble(i)); + } + for (int i = 0; i < row.getLabels().length(); i++) { + values[idx++] = String.valueOf(row.getLabels().getDouble(i)); + } + + + record = new SingleCSVRecord(values); + + } + return record; + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SingleImageRecord.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SingleImageRecord.java new file mode 100644 index 000000000..e864bcd9d --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/model/SingleImageRecord.java @@ -0,0 +1,33 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.net.URI; + +/** + * Created by kepricon on 17. 5. 24. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SingleImageRecord { + private URI uri; +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/service/DataVecTransformService.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/service/DataVecTransformService.java new file mode 100644 index 000000000..caefc649f --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/main/java/org/datavec/spark/inference/model/service/DataVecTransformService.java @@ -0,0 +1,130 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.model.service; + +import org.datavec.api.transform.TransformProcess; +import org.datavec.image.transform.ImageTransformProcess; +import org.datavec.spark.inference.model.model.*; + +import java.io.IOException; + +/** + * Created by agibsonccc on 6/12/17. + */ +public interface DataVecTransformService { + + String SEQUENCE_OR_NOT_HEADER = "Sequence"; + + + /** + * + * @param transformProcess + */ + void setCSVTransformProcess(TransformProcess transformProcess); + + /** + * + * @param imageTransformProcess + */ + void setImageTransformProcess(ImageTransformProcess imageTransformProcess); + + /** + * + * @return + */ + TransformProcess getCSVTransformProcess(); + + /** + * + * @return + */ + ImageTransformProcess getImageTransformProcess(); + + /** + * + * @param singleCsvRecord + * @return + */ + SingleCSVRecord transformIncremental(SingleCSVRecord singleCsvRecord); + + SequenceBatchCSVRecord transform(SequenceBatchCSVRecord batchCSVRecord); + + /** + * + * @param batchCSVRecord + * @return + */ + BatchCSVRecord transform(BatchCSVRecord batchCSVRecord); + + /** + * + * @param batchCSVRecord + * @return + */ + Base64NDArrayBody transformArray(BatchCSVRecord batchCSVRecord); + + /** + * + * @param singleCsvRecord + * @return + */ + Base64NDArrayBody transformArrayIncremental(SingleCSVRecord singleCsvRecord); + + /** + * + * @param singleImageRecord + * @return + * @throws IOException + */ + Base64NDArrayBody transformIncrementalArray(SingleImageRecord singleImageRecord) throws IOException; + + /** + * + * @param batchImageRecord + * @return + * @throws IOException + */ + Base64NDArrayBody transformArray(BatchImageRecord batchImageRecord) throws IOException; + + /** + * + * @param singleCsvRecord + * @return + */ + Base64NDArrayBody transformSequenceArrayIncremental(BatchCSVRecord singleCsvRecord); + + /** + * + * @param batchCSVRecord + * @return + */ + Base64NDArrayBody transformSequenceArray(SequenceBatchCSVRecord batchCSVRecord); + + /** + * + * @param batchCSVRecord + * @return + */ + SequenceBatchCSVRecord transformSequence(SequenceBatchCSVRecord batchCSVRecord); + + /** + * + * @param transform + * @return + */ + SequenceBatchCSVRecord transformSequenceIncremental(BatchCSVRecord transform); +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/AssertTestsExtendBaseClass.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..56e24fef3 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.spark.transform; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.spark.transform"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/BatchCSVRecordTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/BatchCSVRecordTest.java new file mode 100644 index 000000000..2636025ff --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/BatchCSVRecordTest.java @@ -0,0 +1,39 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + +import org.datavec.spark.inference.model.model.BatchCSVRecord; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Created by agibsonccc on 2/12/17. + */ +public class BatchCSVRecordTest { + + @Test + public void testBatchRecordCreationFromDataSet() { + DataSet dataSet = new DataSet(Nd4j.create(2, 2), Nd4j.create(new double[][] {{1, 1}, {1, 1}})); + + BatchCSVRecord batchCSVRecord = BatchCSVRecord.fromDataSet(dataSet); + assertEquals(2, batchCSVRecord.getRecords().size()); + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/CSVSparkTransformTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/CSVSparkTransformTest.java new file mode 100644 index 000000000..fb1c0bd8b --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/CSVSparkTransformTest.java @@ -0,0 +1,211 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.transform.integer.BaseIntegerTransform; +import org.datavec.api.transform.transform.nlp.TextToCharacterIndexTransform; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.Text; +import org.datavec.api.writable.Writable; +import org.datavec.spark.inference.model.CSVSparkTransform; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchCSVRecord; +import org.datavec.spark.inference.model.model.SequenceBatchCSVRecord; +import org.datavec.spark.inference.model.model.SingleCSVRecord; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.serde.base64.Nd4jBase64; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Created by agibsonccc on 12/24/16. + */ +public class CSVSparkTransformTest { + @Test + public void testTransformer() throws Exception { + List input = new ArrayList<>(); + input.add(new DoubleWritable(1.0)); + input.add(new DoubleWritable(2.0)); + + Schema schema = new Schema.Builder().addColumnDouble("1.0").addColumnDouble("2.0").build(); + List output = new ArrayList<>(); + output.add(new Text("1.0")); + output.add(new Text("2.0")); + + TransformProcess transformProcess = + new TransformProcess.Builder(schema).convertToString("1.0").convertToString("2.0").build(); + CSVSparkTransform csvSparkTransform = new CSVSparkTransform(transformProcess); + String[] values = new String[] {"1.0", "2.0"}; + SingleCSVRecord record = csvSparkTransform.transform(new SingleCSVRecord(values)); + Base64NDArrayBody body = csvSparkTransform.toArray(new SingleCSVRecord(values)); + INDArray fromBase64 = Nd4jBase64.fromBase64(body.getNdarray()); + assertTrue(fromBase64.isVector()); +// System.out.println("Base 64ed array " + fromBase64); + } + + @Test + public void testTransformerBatch() throws Exception { + List input = new ArrayList<>(); + input.add(new DoubleWritable(1.0)); + input.add(new DoubleWritable(2.0)); + + Schema schema = new Schema.Builder().addColumnDouble("1.0").addColumnDouble("2.0").build(); + List output = new ArrayList<>(); + output.add(new Text("1.0")); + output.add(new Text("2.0")); + + TransformProcess transformProcess = + new TransformProcess.Builder(schema).convertToString("1.0").convertToString("2.0").build(); + CSVSparkTransform csvSparkTransform = new CSVSparkTransform(transformProcess); + String[] values = new String[] {"1.0", "2.0"}; + SingleCSVRecord record = csvSparkTransform.transform(new SingleCSVRecord(values)); + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(); + for (int i = 0; i < 3; i++) + batchCSVRecord.add(record); + //data type is string, unable to convert + BatchCSVRecord batchCSVRecord1 = csvSparkTransform.transform(batchCSVRecord); + /* Base64NDArrayBody body = csvSparkTransform.toArray(batchCSVRecord1); + INDArray fromBase64 = Nd4jBase64.fromBase64(body.getNdarray()); + assertTrue(fromBase64.isMatrix()); + System.out.println("Base 64ed array " + fromBase64); */ + } + + + + @Test + public void testSingleBatchSequence() throws Exception { + List input = new ArrayList<>(); + input.add(new DoubleWritable(1.0)); + input.add(new DoubleWritable(2.0)); + + Schema schema = new Schema.Builder().addColumnDouble("1.0").addColumnDouble("2.0").build(); + List output = new ArrayList<>(); + output.add(new Text("1.0")); + output.add(new Text("2.0")); + + TransformProcess transformProcess = + new TransformProcess.Builder(schema).convertToString("1.0").convertToString("2.0").build(); + CSVSparkTransform csvSparkTransform = new CSVSparkTransform(transformProcess); + String[] values = new String[] {"1.0", "2.0"}; + SingleCSVRecord record = csvSparkTransform.transform(new SingleCSVRecord(values)); + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(); + for (int i = 0; i < 3; i++) + batchCSVRecord.add(record); + BatchCSVRecord batchCSVRecord1 = csvSparkTransform.transform(batchCSVRecord); + SequenceBatchCSVRecord sequenceBatchCSVRecord = new SequenceBatchCSVRecord(); + sequenceBatchCSVRecord.add(Arrays.asList(batchCSVRecord)); + Base64NDArrayBody sequenceArray = csvSparkTransform.transformSequenceArray(sequenceBatchCSVRecord); + INDArray outputBody = Nd4jBase64.fromBase64(sequenceArray.getNdarray()); + + + //ensure accumulation + sequenceBatchCSVRecord.add(Arrays.asList(batchCSVRecord)); + sequenceArray = csvSparkTransform.transformSequenceArray(sequenceBatchCSVRecord); + assertArrayEquals(new long[]{2,2,3},Nd4jBase64.fromBase64(sequenceArray.getNdarray()).shape()); + + SequenceBatchCSVRecord transformed = csvSparkTransform.transformSequence(sequenceBatchCSVRecord); + assertNotNull(transformed.getRecords()); +// System.out.println(transformed); + + + } + + @Test + public void testSpecificSequence() throws Exception { + final Schema schema = new Schema.Builder() + .addColumnsString("action") + .build(); + + final TransformProcess transformProcess = new TransformProcess.Builder(schema) + .removeAllColumnsExceptFor("action") + .transform(new ConverToLowercase("action")) + .convertToSequence() + .transform(new TextToCharacterIndexTransform("action", "action_sequence", + defaultCharIndex(), false)) + .integerToOneHot("action_sequence",0,29) + .build(); + + final String[] data1 = new String[] { "test1" }; + final String[] data2 = new String[] { "test2" }; + final BatchCSVRecord batchCsvRecord = new BatchCSVRecord( + Arrays.asList( + new SingleCSVRecord(data1), + new SingleCSVRecord(data2))); + + final CSVSparkTransform transform = new CSVSparkTransform(transformProcess); +// System.out.println(transform.transformSequenceIncremental(batchCsvRecord)); + transform.transformSequenceIncremental(batchCsvRecord); + assertEquals(3,Nd4jBase64.fromBase64(transform.transformSequenceArrayIncremental(batchCsvRecord).getNdarray()).rank()); + + } + + private static Map defaultCharIndex() { + Map ret = new TreeMap<>(); + + ret.put('a',0); + ret.put('b',1); + ret.put('c',2); + ret.put('d',3); + ret.put('e',4); + ret.put('f',5); + ret.put('g',6); + ret.put('h',7); + ret.put('i',8); + ret.put('j',9); + ret.put('k',10); + ret.put('l',11); + ret.put('m',12); + ret.put('n',13); + ret.put('o',14); + ret.put('p',15); + ret.put('q',16); + ret.put('r',17); + ret.put('s',18); + ret.put('t',19); + ret.put('u',20); + ret.put('v',21); + ret.put('w',22); + ret.put('x',23); + ret.put('y',24); + ret.put('z',25); + ret.put('/',26); + ret.put(' ',27); + ret.put('(',28); + ret.put(')',29); + + return ret; + } + + public static class ConverToLowercase extends BaseIntegerTransform { + public ConverToLowercase(String column) { + super(column); + } + + public Text map(Writable writable) { + return new Text(writable.toString().toLowerCase()); + } + + public Object map(Object input) { + return new Text(input.toString().toLowerCase()); + } + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/ImageSparkTransformTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/ImageSparkTransformTest.java new file mode 100644 index 000000000..f58bfac0f --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/ImageSparkTransformTest.java @@ -0,0 +1,85 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + +import org.datavec.image.transform.ImageTransformProcess; +import org.datavec.spark.inference.model.ImageSparkTransform; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchImageRecord; +import org.datavec.spark.inference.model.model.SingleImageRecord; + +import org.junit.jupiter.api.Test; + +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.serde.base64.Nd4jBase64; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Created by kepricon on 17. 5. 24. + */ +public class ImageSparkTransformTest { + + @TempDir + public File testDir; + + @Test + public void testSingleImageSparkTransform() throws Exception { + int seed = 12345; + + File f1 = new ClassPathResource("datavec-spark-inference/testimages/class1/A.jpg").getFile(); + + SingleImageRecord imgRecord = new SingleImageRecord(f1.toURI()); + + ImageTransformProcess imgTransformProcess = new ImageTransformProcess.Builder().seed(seed) + .scaleImageTransform(10).cropImageTransform(5).build(); + + ImageSparkTransform imgSparkTransform = new ImageSparkTransform(imgTransformProcess); + Base64NDArrayBody body = imgSparkTransform.toArray(imgRecord); + + INDArray fromBase64 = Nd4jBase64.fromBase64(body.getNdarray()); +// System.out.println("Base 64ed array " + fromBase64); + assertEquals(1, fromBase64.size(0)); + } + + @Test + public void testBatchImageSparkTransform() throws Exception { + int seed = 12345; + + File f0 = new ClassPathResource("datavec-spark-inference/testimages/class1/A.jpg").getFile(); + File f1 = new ClassPathResource("datavec-spark-inference/testimages/class1/B.png").getFile(); + File f2 = new ClassPathResource("datavec-spark-inference/testimages/class1/C.jpg").getFile(); + + BatchImageRecord batch = new BatchImageRecord(); + batch.add(f0.toURI()); + batch.add(f1.toURI()); + batch.add(f2.toURI()); + + ImageTransformProcess imgTransformProcess = new ImageTransformProcess.Builder().seed(seed) + .scaleImageTransform(10).cropImageTransform(5).build(); + + ImageSparkTransform imgSparkTransform = new ImageSparkTransform(imgTransformProcess); + Base64NDArrayBody body = imgSparkTransform.toArray(batch); + + INDArray fromBase64 = Nd4jBase64.fromBase64(body.getNdarray()); +// System.out.println("Base 64ed array " + fromBase64); + assertEquals(3, fromBase64.size(0)); + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/SingleCSVRecordTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/SingleCSVRecordTest.java new file mode 100644 index 000000000..0f5db7f9c --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/SingleCSVRecordTest.java @@ -0,0 +1,61 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + +import org.datavec.spark.inference.model.model.SingleCSVRecord; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Created by agibsonccc on 2/12/17. + */ +public class SingleCSVRecordTest { + + @Test + public void testVectorAssertion() { + assertThrows(IllegalArgumentException.class, () -> { + DataSet dataSet = new DataSet(Nd4j.create(2, 2), Nd4j.create(1, 1)); + SingleCSVRecord singleCsvRecord = SingleCSVRecord.fromRow(dataSet); + fail(singleCsvRecord.toString() + " should have thrown an exception"); + }); + } + + @Test + public void testVectorOneHotLabel() { + DataSet dataSet = new DataSet(Nd4j.create(2, 2), Nd4j.create(new double[][] {{0, 1}, {1, 0}})); + + //assert + SingleCSVRecord singleCsvRecord = SingleCSVRecord.fromRow(dataSet.get(0)); + assertEquals(3, singleCsvRecord.getValues().size()); + + } + + @Test + public void testVectorRegression() { + DataSet dataSet = new DataSet(Nd4j.create(2, 2), Nd4j.create(new double[][] {{1, 1}, {1, 1}})); + + //assert + SingleCSVRecord singleCsvRecord = SingleCSVRecord.fromRow(dataSet.get(0)); + assertEquals(4, singleCsvRecord.getValues().size()); + + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/SingleImageRecordTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/SingleImageRecordTest.java new file mode 100644 index 000000000..2e1a10c2c --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-model/src/test/java/org/datavec/spark/transform/SingleImageRecordTest.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + +import org.datavec.spark.inference.model.model.SingleImageRecord; + +import org.junit.jupiter.api.Test; + +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; + +/** + * Created by kepricon on 17. 5. 24. + */ +public class SingleImageRecordTest { + + @TempDir + public File testDir; + + @Test + public void testImageRecord() throws Exception { + File f = testDir; + new ClassPathResource("datavec-spark-inference/testimages/").copyDirectory(f); + File f0 = new File(f, "class0/0.jpg"); + File f1 = new File(f, "/class1/A.jpg"); + + SingleImageRecord imgRecord = new SingleImageRecord(f0.toURI()); + + // need jackson test? + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/pom.xml b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/pom.xml new file mode 100644 index 000000000..66885d6a9 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/pom.xml @@ -0,0 +1,173 @@ + + + + + datavec-spark-inference-parent + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + datavec-spark-inference-server_2.11 + jar + 1.0.0-SNAPSHOT + datavec-spark-inference-server + + + + 2.12 + ${scala.binary.version} + + + + + net.brutex.ai + datavec-spark-inference-model + ${project.version} + + + + net.brutex.ai + datavec-spark_2.11 + ${project.version} + + + + net.brutex.ai + datavec-data-image + ${project.version} + + + + joda-time + joda-time + ${jodatime.version} + + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + + org.hibernate + hibernate-validator + ${hibernate.version} + + + + org.scala-lang + scala-library + 2.12.14 + + + + org.scala-lang + scala-reflect + 2.12.14 + + + + com.typesafe.play + play-java_2.11 + ${playframework.version} + + + com.google.code.findbugs + jsr305 + + + net.jodah + typetools + + + + + + net.jodah + typetools + ${jodah.typetools.version} + + + + com.typesafe.play + play-json_2.11 + ${playframework.version} + + + + com.typesafe.play + play-server_2.11 + ${playframework.version} + + + + com.typesafe.play + play_2.11 + ${playframework.version} + + + + com.typesafe.play + play-netty-server_2.11 + ${playframework.version} + + + + com.typesafe.akka + akka-cluster_2.11 + 2.5.23 + + + + com.mashape.unirest + unirest-java + ${unirest.version} + test + + + + com.beust + jcommander + ${jcommander.version} + + + + org.apache.spark + spark-core_2.11 + ${spark.version} + + + + net.brutex.ai + nd4j-common-tests + ${project.version} + test + + + + + + test-nd4j-native + + + test-nd4j-cuda-10.2 + + + diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/CSVSparkTransformServer.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/CSVSparkTransformServer.java new file mode 100644 index 000000000..dd1d8f9f8 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/CSVSparkTransformServer.java @@ -0,0 +1,359 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.server; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.ParameterException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.datavec.api.transform.TransformProcess; +import org.datavec.image.transform.ImageTransformProcess; +import org.datavec.spark.inference.model.CSVSparkTransform; +import org.datavec.spark.inference.model.model.*; +import play.BuiltInComponents; +import play.Mode; +import play.routing.Router; +import play.routing.RoutingDsl; +import play.server.Server; + +import java.io.File; +import java.io.IOException; +import java.util.Base64; +import java.util.Random; + +import static play.mvc.Results.*; + +/** + * A rest server for using an + * {@link TransformProcess} based on simple + * csv values and a schema via REST. + *

+ * The input values are an {@link SingleCSVRecord} + * which (based on the input schema) will automatically + * have their values transformed. + * + * @author Adam Gibson + */ +@Slf4j +@Data +public class CSVSparkTransformServer extends SparkTransformServer { + private CSVSparkTransform transform; + + public void runMain(String[] args) throws Exception { + JCommander jcmdr = new JCommander(this); + + try { + jcmdr.parse(args); + } catch (ParameterException e) { + //User provides invalid input -> print the usage info + jcmdr.usage(); + if (jsonPath == null) + System.err.println("Json path parameter is missing."); + try { + Thread.sleep(500); + } catch (Exception e2) { + } + System.exit(1); + } + + if (jsonPath != null) { + String json = FileUtils.readFileToString(new File(jsonPath)); + TransformProcess transformProcess = TransformProcess.fromJson(json); + transform = new CSVSparkTransform(transformProcess); + } else { + log.warn("Server started with no json for transform process. Please ensure you specify a transform process via sending a post request with raw json" + + "to /transformprocess"); + } + + //Set play secret key, if required + //http://www.playframework.com/documentation/latest/ApplicationSecret + String crypto = System.getProperty("play.crypto.secret"); + if (crypto == null || "changeme".equals(crypto) || "".equals(crypto) ) { + byte[] newCrypto = new byte[1024]; + + new Random().nextBytes(newCrypto); + + String base64 = Base64.getEncoder().encodeToString(newCrypto); + System.setProperty("play.crypto.secret", base64); + } + + + server = Server.forRouter(Mode.PROD, port, this::createRouter); + } + + protected Router createRouter(BuiltInComponents b){ + RoutingDsl routingDsl = RoutingDsl.fromComponents(b); + + routingDsl.GET("/transformprocess").routingTo(req -> { + try { + if (transform == null) + return badRequest(); + return ok(transform.getTransformProcess().toJson()).as(contentType); + } catch (Exception e) { + log.error("Error in GET /transformprocess",e); + return internalServerError(e.getMessage()); + } + }); + + routingDsl.POST("/transformprocess").routingTo(req -> { + try { + TransformProcess transformProcess = TransformProcess.fromJson(getJsonText(req)); + setCSVTransformProcess(transformProcess); + log.info("Transform process initialized"); + return ok(objectMapper.writeValueAsString(transformProcess)).as(contentType); + } catch (Exception e) { + log.error("Error in POST /transformprocess",e); + return internalServerError(e.getMessage()); + } + }); + + routingDsl.POST("/transformincremental").routingTo(req -> { + if (isSequence(req)) { + try { + BatchCSVRecord record = objectMapper.readValue(getJsonText(req), BatchCSVRecord.class); + if (record == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(transformSequenceIncremental(record))).as(contentType); + } catch (Exception e) { + log.error("Error in /transformincremental", e); + return internalServerError(e.getMessage()); + } + } else { + try { + SingleCSVRecord record = objectMapper.readValue(getJsonText(req), SingleCSVRecord.class); + if (record == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(transformIncremental(record))).as(contentType); + } catch (Exception e) { + log.error("Error in /transformincremental", e); + return internalServerError(e.getMessage()); + } + } + }); + + routingDsl.POST("/transform").routingTo(req -> { + if (isSequence(req)) { + try { + SequenceBatchCSVRecord batch = transformSequence(objectMapper.readValue(getJsonText(req), SequenceBatchCSVRecord.class)); + if (batch == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(batch)).as(contentType); + } catch (Exception e) { + log.error("Error in /transform", e); + return internalServerError(e.getMessage()); + } + } else { + try { + BatchCSVRecord input = objectMapper.readValue(getJsonText(req), BatchCSVRecord.class); + BatchCSVRecord batch = transform(input); + if (batch == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(batch)).as(contentType); + } catch (Exception e) { + log.error("Error in /transform", e); + return internalServerError(e.getMessage()); + } + } + }); + + routingDsl.POST("/transformincrementalarray").routingTo(req -> { + if (isSequence(req)) { + try { + BatchCSVRecord record = objectMapper.readValue(getJsonText(req), BatchCSVRecord.class); + if (record == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(transformSequenceArrayIncremental(record))).as(contentType); + } catch (Exception e) { + log.error("Error in /transformincrementalarray", e); + return internalServerError(e.getMessage()); + } + } else { + try { + SingleCSVRecord record = objectMapper.readValue(getJsonText(req), SingleCSVRecord.class); + if (record == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(transformArrayIncremental(record))).as(contentType); + } catch (Exception e) { + log.error("Error in /transformincrementalarray", e); + return internalServerError(e.getMessage()); + } + } + }); + + routingDsl.POST("/transformarray").routingTo(req -> { + if (isSequence(req)) { + try { + SequenceBatchCSVRecord batchCSVRecord = objectMapper.readValue(getJsonText(req), SequenceBatchCSVRecord.class); + if (batchCSVRecord == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(transformSequenceArray(batchCSVRecord))).as(contentType); + } catch (Exception e) { + log.error("Error in /transformarray", e); + return internalServerError(e.getMessage()); + } + } else { + try { + BatchCSVRecord batchCSVRecord = objectMapper.readValue(getJsonText(req), BatchCSVRecord.class); + if (batchCSVRecord == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(transformArray(batchCSVRecord))).as(contentType); + } catch (Exception e) { + log.error("Error in /transformarray", e); + return internalServerError(e.getMessage()); + } + } + }); + + return routingDsl.build(); + } + + public static void main(String[] args) throws Exception { + new CSVSparkTransformServer().runMain(args); + } + + /** + * @param transformProcess + */ + @Override + public void setCSVTransformProcess(TransformProcess transformProcess) { + this.transform = new CSVSparkTransform(transformProcess); + } + + @Override + public void setImageTransformProcess(ImageTransformProcess imageTransformProcess) { + log.error("Unsupported operation: setImageTransformProcess not supported for class", getClass()); + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + /** + * @return + */ + @Override + public TransformProcess getCSVTransformProcess() { + return transform.getTransformProcess(); + } + + @Override + public ImageTransformProcess getImageTransformProcess() { + log.error("Unsupported operation: getImageTransformProcess not supported for class", getClass()); + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + + /** + * + */ + /** + * @param transform + * @return + */ + @Override + public SequenceBatchCSVRecord transformSequenceIncremental(BatchCSVRecord transform) { + return this.transform.transformSequenceIncremental(transform); + } + + /** + * @param batchCSVRecord + * @return + */ + @Override + public SequenceBatchCSVRecord transformSequence(SequenceBatchCSVRecord batchCSVRecord) { + return transform.transformSequence(batchCSVRecord); + } + + /** + * @param batchCSVRecord + * @return + */ + @Override + public Base64NDArrayBody transformSequenceArray(SequenceBatchCSVRecord batchCSVRecord) { + return this.transform.transformSequenceArray(batchCSVRecord); + } + + /** + * @param singleCsvRecord + * @return + */ + @Override + public Base64NDArrayBody transformSequenceArrayIncremental(BatchCSVRecord singleCsvRecord) { + return this.transform.transformSequenceArrayIncremental(singleCsvRecord); + } + + /** + * @param transform + * @return + */ + @Override + public SingleCSVRecord transformIncremental(SingleCSVRecord transform) { + return this.transform.transform(transform); + } + + @Override + public SequenceBatchCSVRecord transform(SequenceBatchCSVRecord batchCSVRecord) { + return this.transform.transform(batchCSVRecord); + } + + /** + * @param batchCSVRecord + * @return + */ + @Override + public BatchCSVRecord transform(BatchCSVRecord batchCSVRecord) { + return transform.transform(batchCSVRecord); + } + + /** + * @param batchCSVRecord + * @return + */ + @Override + public Base64NDArrayBody transformArray(BatchCSVRecord batchCSVRecord) { + try { + return this.transform.toArray(batchCSVRecord); + } catch (IOException e) { + log.error("Error in transformArray",e); + throw new IllegalStateException("Transform array shouldn't throw exception"); + } + } + + /** + * @param singleCsvRecord + * @return + */ + @Override + public Base64NDArrayBody transformArrayIncremental(SingleCSVRecord singleCsvRecord) { + try { + return this.transform.toArray(singleCsvRecord); + } catch (IOException e) { + log.error("Error in transformArrayIncremental",e); + throw new IllegalStateException("Transform array shouldn't throw exception"); + } + } + + @Override + public Base64NDArrayBody transformIncrementalArray(SingleImageRecord singleImageRecord) throws IOException { + log.error("Unsupported operation: transformIncrementalArray(SingleImageRecord) not supported for class", getClass()); + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public Base64NDArrayBody transformArray(BatchImageRecord batchImageRecord) throws IOException { + log.error("Unsupported operation: transformArray(BatchImageRecord) not supported for class", getClass()); + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/ImageSparkTransformServer.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/ImageSparkTransformServer.java new file mode 100644 index 000000000..5b7b29cd2 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/ImageSparkTransformServer.java @@ -0,0 +1,260 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.server; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.ParameterException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.datavec.api.transform.TransformProcess; +import org.datavec.image.transform.ImageTransformProcess; +import org.datavec.spark.inference.model.ImageSparkTransform; +import org.datavec.spark.inference.model.model.*; +import play.BuiltInComponents; +import play.Mode; +import play.libs.Files; +import play.mvc.Http; +import play.routing.Router; +import play.routing.RoutingDsl; +import play.server.Server; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static play.mvc.Results.*; + +/** + * Created by kepricon on 17. 6. 19. + */ +@Slf4j +@Data +public class ImageSparkTransformServer extends SparkTransformServer { + private ImageSparkTransform transform; + + public void runMain(String[] args) throws Exception { + JCommander jcmdr = new JCommander(this); + + try { + jcmdr.parse(args); + } catch (ParameterException e) { + //User provides invalid input -> print the usage info + jcmdr.usage(); + if (jsonPath == null) + System.err.println("Json path parameter is missing."); + try { + Thread.sleep(500); + } catch (Exception e2) { + } + System.exit(1); + } + + if (jsonPath != null) { + String json = FileUtils.readFileToString(new File(jsonPath)); + ImageTransformProcess transformProcess = ImageTransformProcess.fromJson(json); + transform = new ImageSparkTransform(transformProcess); + } else { + log.warn("Server started with no json for transform process. Please ensure you specify a transform process via sending a post request with raw json" + + "to /transformprocess"); + } + + server = Server.forRouter(Mode.PROD, port, this::createRouter); + } + + protected Router createRouter(BuiltInComponents builtInComponents){ + RoutingDsl routingDsl = RoutingDsl.fromComponents(builtInComponents); + + routingDsl.GET("/transformprocess").routingTo(req -> { + try { + if (transform == null) + return badRequest(); + log.info("Transform process initialized"); + return ok(objectMapper.writeValueAsString(transform.getImageTransformProcess())).as(contentType); + } catch (Exception e) { + log.error("",e); + return internalServerError(); + } + }); + + routingDsl.POST("/transformprocess").routingTo(req -> { + try { + ImageTransformProcess transformProcess = ImageTransformProcess.fromJson(getJsonText(req)); + setImageTransformProcess(transformProcess); + log.info("Transform process initialized"); + return ok(objectMapper.writeValueAsString(transformProcess)).as(contentType); + } catch (Exception e) { + log.error("",e); + return internalServerError(); + } + }); + + routingDsl.POST("/transformincrementalarray").routingTo(req -> { + try { + SingleImageRecord record = objectMapper.readValue(getJsonText(req), SingleImageRecord.class); + if (record == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(transformIncrementalArray(record))).as(contentType); + } catch (Exception e) { + log.error("",e); + return internalServerError(); + } + }); + + routingDsl.POST("/transformincrementalimage").routingTo(req -> { + try { + Http.MultipartFormData body = req.body().asMultipartFormData(); + List> files = body.getFiles(); + if (files.isEmpty() || files.get(0).getRef() == null ) { + return badRequest(); + } + + File file = files.get(0).getRef().path().toFile(); + SingleImageRecord record = new SingleImageRecord(file.toURI()); + + return ok(objectMapper.writeValueAsString(transformIncrementalArray(record))).as(contentType); + } catch (Exception e) { + log.error("",e); + return internalServerError(); + } + }); + + routingDsl.POST("/transformarray").routingTo(req -> { + try { + BatchImageRecord batch = objectMapper.readValue(getJsonText(req), BatchImageRecord.class); + if (batch == null) + return badRequest(); + return ok(objectMapper.writeValueAsString(transformArray(batch))).as(contentType); + } catch (Exception e) { + log.error("",e); + return internalServerError(); + } + }); + + routingDsl.POST("/transformimage").routingTo(req -> { + try { + Http.MultipartFormData body = req.body().asMultipartFormData(); + List> files = body.getFiles(); + if (files.size() == 0) { + return badRequest(); + } + + List records = new ArrayList<>(); + + for (Http.MultipartFormData.FilePart filePart : files) { + Files.TemporaryFile file = filePart.getRef(); + if (file != null) { + SingleImageRecord record = new SingleImageRecord(file.path().toUri()); + records.add(record); + } + } + + BatchImageRecord batch = new BatchImageRecord(records); + + return ok(objectMapper.writeValueAsString(transformArray(batch))).as(contentType); + } catch (Exception e) { + log.error("",e); + return internalServerError(); + } + }); + + return routingDsl.build(); + } + + @Override + public Base64NDArrayBody transformSequenceArrayIncremental(BatchCSVRecord singleCsvRecord) { + throw new UnsupportedOperationException(); + } + + @Override + public Base64NDArrayBody transformSequenceArray(SequenceBatchCSVRecord batchCSVRecord) { + throw new UnsupportedOperationException(); + + } + + @Override + public SequenceBatchCSVRecord transformSequence(SequenceBatchCSVRecord batchCSVRecord) { + throw new UnsupportedOperationException(); + + } + + @Override + public SequenceBatchCSVRecord transformSequenceIncremental(BatchCSVRecord transform) { + throw new UnsupportedOperationException(); + + } + + @Override + public void setCSVTransformProcess(TransformProcess transformProcess) { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public void setImageTransformProcess(ImageTransformProcess imageTransformProcess) { + this.transform = new ImageSparkTransform(imageTransformProcess); + } + + @Override + public TransformProcess getCSVTransformProcess() { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public ImageTransformProcess getImageTransformProcess() { + return transform.getImageTransformProcess(); + } + + @Override + public SingleCSVRecord transformIncremental(SingleCSVRecord singleCsvRecord) { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public SequenceBatchCSVRecord transform(SequenceBatchCSVRecord batchCSVRecord) { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public BatchCSVRecord transform(BatchCSVRecord batchCSVRecord) { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public Base64NDArrayBody transformArray(BatchCSVRecord batchCSVRecord) { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public Base64NDArrayBody transformArrayIncremental(SingleCSVRecord singleCsvRecord) { + throw new UnsupportedOperationException("Invalid operation for " + this.getClass()); + } + + @Override + public Base64NDArrayBody transformIncrementalArray(SingleImageRecord record) throws IOException { + return transform.toArray(record); + } + + @Override + public Base64NDArrayBody transformArray(BatchImageRecord batch) throws IOException { + return transform.toArray(batch); + } + + public static void main(String[] args) throws Exception { + new ImageSparkTransformServer().runMain(args); + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/SparkTransformServer.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/SparkTransformServer.java new file mode 100644 index 000000000..1f366141b --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/SparkTransformServer.java @@ -0,0 +1,66 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.server; + +import com.beust.jcommander.Parameter; +import com.fasterxml.jackson.databind.JsonNode; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchCSVRecord; +import org.datavec.spark.inference.model.service.DataVecTransformService; +import com.fasterxml.jackson.databind.ObjectMapper; +import play.mvc.Http; +import play.server.Server; + +/** + * Created by kepricon on 17. 6. 20. + */ +public abstract class SparkTransformServer implements DataVecTransformService { + @Parameter(names = {"-j", "--jsonPath"}, arity = 1) + protected String jsonPath = null; + @Parameter(names = {"-dp", "--dataVecPort"}, arity = 1) + protected int port = 9000; + @Parameter(names = {"-dt", "--dataType"}, arity = 1) + private TransformDataType transformDataType = null; + protected Server server; + protected static ObjectMapper objectMapper = new ObjectMapper(); + protected static String contentType = "application/json"; + + public abstract void runMain(String[] args) throws Exception; + + /** + * Stop the server + */ + public void stop() { + if (server != null) + server.stop(); + } + + protected boolean isSequence(Http.Request request) { + return request.hasHeader(SEQUENCE_OR_NOT_HEADER) + && request.header(SEQUENCE_OR_NOT_HEADER).get().equalsIgnoreCase("true"); + } + + protected String getJsonText(Http.Request request) { + JsonNode tryJson = request.body().asJson(); + if (tryJson != null) + return tryJson.toString(); + else + return request.body().asText(); + } + + public abstract Base64NDArrayBody transformSequenceArrayIncremental(BatchCSVRecord singleCsvRecord); +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/SparkTransformServerChooser.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/SparkTransformServerChooser.java new file mode 100644 index 000000000..10013329d --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/SparkTransformServerChooser.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.server; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.io.InvalidClassException; +import java.util.Arrays; +import java.util.List; + +/** + * Created by kepricon on 17. 6. 20. + */ +@Data +@Slf4j +public class SparkTransformServerChooser { + private SparkTransformServer sparkTransformServer = null; + private TransformDataType transformDataType = null; + + public void runMain(String[] args) throws Exception { + + int pos = getMatchingPosition(args, "-dt", "--dataType"); + if (pos == -1) { + log.error("no valid options"); + log.error("-dt, --dataType Options: [CSV, IMAGE]"); + throw new Exception("no valid options"); + } else { + transformDataType = TransformDataType.valueOf(args[pos + 1]); + } + + switch (transformDataType) { + case CSV: + sparkTransformServer = new CSVSparkTransformServer(); + break; + case IMAGE: + sparkTransformServer = new ImageSparkTransformServer(); + break; + default: + throw new InvalidClassException("no matching SparkTransform class"); + } + + sparkTransformServer.runMain(args); + } + + private int getMatchingPosition(String[] args, String... options) { + List optionList = Arrays.asList(options); + + for (int i = 0; i < args.length; i++) { + if (optionList.contains(args[i])) { + return i; + } + } + return -1; + } + + + public static void main(String[] args) throws Exception { + new SparkTransformServerChooser().runMain(args); + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/TransformDataType.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/TransformDataType.java new file mode 100644 index 000000000..d2c1932dc --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/java/org/datavec/spark/inference/server/TransformDataType.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.inference.server; + +/** + * Created by kepricon on 17. 6. 20. + */ +public enum TransformDataType { + CSV, IMAGE, +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/resources/application.conf b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/resources/application.conf new file mode 100644 index 000000000..28a4aa208 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/main/resources/application.conf @@ -0,0 +1,350 @@ +# This is the main configuration file for the application. +# https://www.playframework.com/documentation/latest/ConfigFile +# ~~~~~ +# Play uses HOCON as its configuration file format. HOCON has a number +# of advantages over other config formats, but there are two things that +# can be used when modifying settings. +# +# You can include other configuration files in this main application.conf file: +#include "extra-config.conf" +# +# You can declare variables and substitute for them: +#mykey = ${some.value} +# +# And if an environment variable exists when there is no other subsitution, then +# HOCON will fall back to substituting environment variable: +#mykey = ${JAVA_HOME} + +## Akka +# https://www.playframework.com/documentation/latest/ScalaAkka#Configuration +# https://www.playframework.com/documentation/latest/JavaAkka#Configuration +# ~~~~~ +# Play uses Akka internally and exposes Akka Streams and actors in Websockets and +# other streaming HTTP responses. +akka { + # "akka.log-config-on-start" is extraordinarly useful because it log the complete + # configuration at INFO level, including defaults and overrides, so it s worth + # putting at the very top. + # + # Put the following in your conf/logback.xml file: + # + # + # + # And then uncomment this line to debug the configuration. + # + #log-config-on-start = true +} + +## Modules +# https://www.playframework.com/documentation/latest/Modules +# ~~~~~ +# Control which modules are loaded when Play starts. Note that modules are +# the replacement for "GlobalSettings", which are deprecated in 2.5.x. +# Please see https://www.playframework.com/documentation/latest/GlobalSettings +# for more information. +# +# You can also extend Play functionality by using one of the publically available +# Play modules: https://playframework.com/documentation/latest/ModuleDirectory +play.modules { + # By default, Play will load any class called Module that is defined + # in the root package (the "app" directory), or you can define them + # explicitly below. + # If there are any built-in modules that you want to disable, you can list them here. + #enabled += my.application.Module + + # If there are any built-in modules that you want to disable, you can list them here. + #disabled += "" +} + +## Internationalisation +# https://www.playframework.com/documentation/latest/JavaI18N +# https://www.playframework.com/documentation/latest/ScalaI18N +# ~~~~~ +# Play comes with its own i18n settings, which allow the user's preferred language +# to map through to internal messages, or allow the language to be stored in a cookie. +play.i18n { + # The application languages + langs = [ "en" ] + + # Whether the language cookie should be secure or not + #langCookieSecure = true + + # Whether the HTTP only attribute of the cookie should be set to true + #langCookieHttpOnly = true +} + +## Play HTTP settings +# ~~~~~ +play.http { + ## Router + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # Define the Router object to use for this application. + # This router will be looked up first when the application is starting up, + # so make sure this is the entry point. + # Furthermore, it's assumed your route file is named properly. + # So for an application router like `my.application.Router`, + # you may need to define a router file `conf/my.application.routes`. + # Default to Routes in the root package (aka "apps" folder) (and conf/routes) + #router = my.application.Router + + ## Action Creator + # https://www.playframework.com/documentation/latest/JavaActionCreator + # ~~~~~ + #actionCreator = null + + ## ErrorHandler + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # If null, will attempt to load a class called ErrorHandler in the root package, + #errorHandler = null + + ## Filters + # https://www.playframework.com/documentation/latest/ScalaHttpFilters + # https://www.playframework.com/documentation/latest/JavaHttpFilters + # ~~~~~ + # Filters run code on every request. They can be used to perform + # common logic for all your actions, e.g. adding common headers. + # Defaults to "Filters" in the root package (aka "apps" folder) + # Alternatively you can explicitly register a class here. + #filters += my.application.Filters + + ## Session & Flash + # https://www.playframework.com/documentation/latest/JavaSessionFlash + # https://www.playframework.com/documentation/latest/ScalaSessionFlash + # ~~~~~ + session { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + + # Sets the max-age field of the cookie to 5 minutes. + # NOTE: this only sets when the browser will discard the cookie. Play will consider any + # cookie value with a valid signature to be a valid session forever. To implement a server side session timeout, + # you need to put a timestamp in the session and check it at regular intervals to possibly expire it. + #maxAge = 300 + + # Sets the domain on the session cookie. + #domain = "example.com" + } + + flash { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + } +} + +## Netty Provider +# https://www.playframework.com/documentation/latest/SettingsNetty +# ~~~~~ +play.server.netty { + # Whether the Netty wire should be logged + #log.wire = true + + # If you run Play on Linux, you can use Netty's native socket transport + # for higher performance with less garbage. + #transport = "native" +} + +## WS (HTTP Client) +# https://www.playframework.com/documentation/latest/ScalaWS#Configuring-WS +# ~~~~~ +# The HTTP client primarily used for REST APIs. The default client can be +# configured directly, but you can also create different client instances +# with customized settings. You must enable this by adding to build.sbt: +# +# libraryDependencies += ws // or javaWs if using java +# +play.ws { + # Sets HTTP requests not to follow 302 requests + #followRedirects = false + + # Sets the maximum number of open HTTP connections for the client. + #ahc.maxConnectionsTotal = 50 + + ## WS SSL + # https://www.playframework.com/documentation/latest/WsSSL + # ~~~~~ + ssl { + # Configuring HTTPS with Play WS does not require programming. You can + # set up both trustManager and keyManager for mutual authentication, and + # turn on JSSE debugging in development with a reload. + #debug.handshake = true + #trustManager = { + # stores = [ + # { type = "JKS", path = "exampletrust.jks" } + # ] + #} + } +} + +## Cache +# https://www.playframework.com/documentation/latest/JavaCache +# https://www.playframework.com/documentation/latest/ScalaCache +# ~~~~~ +# Play comes with an integrated cache API that can reduce the operational +# overhead of repeated requests. You must enable this by adding to build.sbt: +# +# libraryDependencies += cache +# +play.cache { + # If you want to bind several caches, you can bind the individually + #bindCaches = ["db-cache", "user-cache", "session-cache"] +} + +## Filters +# https://www.playframework.com/documentation/latest/Filters +# ~~~~~ +# There are a number of built-in filters that can be enabled and configured +# to give Play greater security. You must enable this by adding to build.sbt: +# +# libraryDependencies += filters +# +play.filters { + ## CORS filter configuration + # https://www.playframework.com/documentation/latest/CorsFilter + # ~~~~~ + # CORS is a protocol that allows web applications to make requests from the browser + # across different domains. + # NOTE: You MUST apply the CORS configuration before the CSRF filter, as CSRF has + # dependencies on CORS settings. + cors { + # Filter paths by a whitelist of path prefixes + #pathPrefixes = ["/some/path", ...] + + # The allowed origins. If null, all origins are allowed. + #allowedOrigins = ["http://www.example.com"] + + # The allowed HTTP methods. If null, all methods are allowed + #allowedHttpMethods = ["GET", "POST"] + } + + ## CSRF Filter + # https://www.playframework.com/documentation/latest/ScalaCsrf#Applying-a-global-CSRF-filter + # https://www.playframework.com/documentation/latest/JavaCsrf#Applying-a-global-CSRF-filter + # ~~~~~ + # Play supports multiple methods for verifying that a request is not a CSRF request. + # The primary mechanism is a CSRF token. This token gets placed either in the query string + # or body of every form submitted, and also gets placed in the users session. + # Play then verifies that both tokens are present and match. + csrf { + # Sets the cookie to be sent only over HTTPS + #cookie.secure = true + + # Defaults to CSRFErrorHandler in the root package. + #errorHandler = MyCSRFErrorHandler + } + + ## Security headers filter configuration + # https://www.playframework.com/documentation/latest/SecurityHeaders + # ~~~~~ + # Defines security headers that prevent XSS attacks. + # If enabled, then all options are set to the below configuration by default: + headers { + # The X-Frame-Options header. If null, the header is not set. + #frameOptions = "DENY" + + # The X-XSS-Protection header. If null, the header is not set. + #xssProtection = "1; mode=block" + + # The X-Content-Type-Options header. If null, the header is not set. + #contentTypeOptions = "nosniff" + + # The X-Permitted-Cross-Domain-Policies header. If null, the header is not set. + #permittedCrossDomainPolicies = "master-only" + + # The Content-Security-Policy header. If null, the header is not set. + #contentSecurityPolicy = "default-src 'self'" + } + + ## Allowed hosts filter configuration + # https://www.playframework.com/documentation/latest/AllowedHostsFilter + # ~~~~~ + # Play provides a filter that lets you configure which hosts can access your application. + # This is useful to prevent cache poisoning attacks. + hosts { + # Allow requests to example.com, its subdomains, and localhost:9000. + #allowed = [".example.com", "localhost:9000"] + } +} + +## Evolutions +# https://www.playframework.com/documentation/latest/Evolutions +# ~~~~~ +# Evolutions allows database scripts to be automatically run on startup in dev mode +# for database migrations. You must enable this by adding to build.sbt: +# +# libraryDependencies += evolutions +# +play.evolutions { + # You can disable evolutions for a specific datasource if necessary + #db.default.enabled = false +} + +## Database Connection Pool +# https://www.playframework.com/documentation/latest/SettingsJDBC +# ~~~~~ +# Play doesn't require a JDBC database to run, but you can easily enable one. +# +# libraryDependencies += jdbc +# +play.db { + # The combination of these two settings results in "db.default" as the + # default JDBC pool: + #config = "db" + #default = "default" + + # Play uses HikariCP as the default connection pool. You can override + # settings by changing the prototype: + prototype { + # Sets a fixed JDBC connection pool size of 50 + #hikaricp.minimumIdle = 50 + #hikaricp.maximumPoolSize = 50 + } +} + +## JDBC Datasource +# https://www.playframework.com/documentation/latest/JavaDatabase +# https://www.playframework.com/documentation/latest/ScalaDatabase +# ~~~~~ +# Once JDBC datasource is set up, you can work with several different +# database options: +# +# Slick (Scala preferred option): https://www.playframework.com/documentation/latest/PlaySlick +# JPA (Java preferred option): https://playframework.com/documentation/latest/JavaJPA +# EBean: https://playframework.com/documentation/latest/JavaEbean +# Anorm: https://www.playframework.com/documentation/latest/ScalaAnorm +# +db { + # You can declare as many datasources as you want. + # By convention, the default datasource is named `default` + + # https://www.playframework.com/documentation/latest/Developing-with-the-H2-Database + default.driver = org.h2.Driver + default.url = "jdbc:h2:mem:play" + #default.username = sa + #default.password = "" + + # You can expose this datasource via JNDI if needed (Useful for JPA) + default.jndiName=DefaultDS + + # You can turn on SQL logging for any datasource + # https://www.playframework.com/documentation/latest/Highlights25#Logging-SQL-statements + #default.logSql=true +} + +jpa.default=defaultPersistenceUnit + + +#Increase default maximum post length - used for remote listener functionality +#Can get response 413 with larger networks without setting this +# parsers.text.maxLength is deprecated, use play.http.parser.maxMemoryBuffer instead +#parsers.text.maxLength=10M +play.http.parser.maxMemoryBuffer=10M diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/AssertTestsExtendBaseClass.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..56e24fef3 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/AssertTestsExtendBaseClass.java @@ -0,0 +1,50 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.datavec.spark.transform; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.tests.AbstractAssertTestsClass; +import org.nd4j.common.tests.BaseND4JTest; + +import java.util.*; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4jTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alex Black + */ + +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) + return new HashSet<>(); + } + + @Override + protected String getPackageName() { + return "org.datavec.spark.transform"; + } + + @Override + protected Class getBaseClass() { + return BaseND4JTest.class; + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/CSVSparkTransformServerNoJsonTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/CSVSparkTransformServerNoJsonTest.java new file mode 100644 index 000000000..e7f70087e --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/CSVSparkTransformServerNoJsonTest.java @@ -0,0 +1,126 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + +import com.mashape.unirest.http.JsonNode; +import com.mashape.unirest.http.ObjectMapper; +import com.mashape.unirest.http.Unirest; +import org.apache.commons.io.FileUtils; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.spark.inference.server.CSVSparkTransformServer; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchCSVRecord; +import org.datavec.spark.inference.model.model.SingleCSVRecord; +import org.junit.AfterClass; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeNotNull; + +/** + * Created by agibsonccc on 1/22/17. + */ +public class CSVSparkTransformServerNoJsonTest { + + private static CSVSparkTransformServer server; + private static Schema schema = new Schema.Builder().addColumnDouble("1.0").addColumnDouble("2.0").build(); + private static TransformProcess transformProcess = + new TransformProcess.Builder(schema).convertToDouble("1.0").convertToDouble("2.0").build(); + private static File fileSave = new File(UUID.randomUUID().toString() + ".json"); + + @BeforeAll + public static void before() throws Exception { + server = new CSVSparkTransformServer(); + FileUtils.write(fileSave, transformProcess.toJson()); + + // Only one time + Unirest.setObjectMapper(new ObjectMapper() { + private com.fasterxml.jackson.databind.ObjectMapper jacksonObjectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public T readValue(String value, Class valueType) { + try { + return jacksonObjectMapper.readValue(value, valueType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String writeValue(Object value) { + try { + return jacksonObjectMapper.writeValueAsString(value); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + + server.runMain(new String[] {"-dp", "9050"}); + } + + @AfterClass + public static void after() throws Exception { + fileSave.delete(); + server.stop(); + + } + + + + @Test + public void testServer() throws Exception { + assertTrue(server.getTransform() == null); + JsonNode jsonStatus = Unirest.post("http://localhost:9050/transformprocess") + .header("accept", "application/json").header("Content-Type", "application/json") + .body(transformProcess.toJson()).asJson().getBody(); + assumeNotNull(server.getTransform()); + + String[] values = new String[] {"1.0", "2.0"}; + SingleCSVRecord record = new SingleCSVRecord(values); + JsonNode jsonNode = + Unirest.post("http://localhost:9050/transformincremental").header("accept", "application/json") + .header("Content-Type", "application/json").body(record).asJson().getBody(); + SingleCSVRecord singleCsvRecord = Unirest.post("http://localhost:9050/transformincremental") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asObject(SingleCSVRecord.class).getBody(); + + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(); + for (int i = 0; i < 3; i++) + batchCSVRecord.add(singleCsvRecord); + /* BatchCSVRecord batchCSVRecord1 = Unirest.post("http://localhost:9050/transform") + .header("accept", "application/json").header("Content-Type", "application/json") + .body(batchCSVRecord).asObject(BatchCSVRecord.class).getBody(); + + Base64NDArrayBody array = Unirest.post("http://localhost:9050/transformincrementalarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asObject(Base64NDArrayBody.class).getBody(); +*/ + Base64NDArrayBody batchArray1 = Unirest.post("http://localhost:9050/transformarray") + .header("accept", "application/json").header("Content-Type", "application/json") + .body(batchCSVRecord).asObject(Base64NDArrayBody.class).getBody(); + + + + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/CSVSparkTransformServerTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/CSVSparkTransformServerTest.java new file mode 100644 index 000000000..5db7254f2 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/CSVSparkTransformServerTest.java @@ -0,0 +1,120 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + + +import com.mashape.unirest.http.JsonNode; +import com.mashape.unirest.http.ObjectMapper; +import com.mashape.unirest.http.Unirest; +import org.apache.commons.io.FileUtils; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.spark.inference.server.CSVSparkTransformServer; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchCSVRecord; +import org.datavec.spark.inference.model.model.SingleCSVRecord; +import org.junit.AfterClass; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +/** + * Created by agibsonccc on 1/22/17. + */ +public class CSVSparkTransformServerTest { + + private static CSVSparkTransformServer server; + private static Schema schema = new Schema.Builder().addColumnDouble("1.0").addColumnDouble("2.0").build(); + private static TransformProcess transformProcess = + new TransformProcess.Builder(schema).convertToDouble("1.0").convertToDouble("2.0").build(); + private static File fileSave = new File(UUID.randomUUID().toString() + ".json"); + + @BeforeAll + public static void before() throws Exception { + server = new CSVSparkTransformServer(); + FileUtils.write(fileSave, transformProcess.toJson()); + // Only one time + + Unirest.setObjectMapper(new ObjectMapper() { + private com.fasterxml.jackson.databind.ObjectMapper jacksonObjectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public T readValue(String value, Class valueType) { + try { + return jacksonObjectMapper.readValue(value, valueType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String writeValue(Object value) { + try { + return jacksonObjectMapper.writeValueAsString(value); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + + server.runMain(new String[] {"--jsonPath", fileSave.getAbsolutePath(), "-dp", "9050"}); + } + + @AfterClass + public static void after() throws Exception { + fileSave.deleteOnExit(); + server.stop(); + + } + + + + @Test + public void testServer() throws Exception { + String[] values = new String[] {"1.0", "2.0"}; + SingleCSVRecord record = new SingleCSVRecord(values); + JsonNode jsonNode = + Unirest.post("http://localhost:9050/transformincremental").header("accept", "application/json") + .header("Content-Type", "application/json").body(record).asJson().getBody(); + SingleCSVRecord singleCsvRecord = Unirest.post("http://localhost:9050/transformincremental") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asObject(SingleCSVRecord.class).getBody(); + + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(); + for (int i = 0; i < 3; i++) + batchCSVRecord.add(singleCsvRecord); + BatchCSVRecord batchCSVRecord1 = Unirest.post("http://localhost:9050/transform") + .header("accept", "application/json").header("Content-Type", "application/json") + .body(batchCSVRecord).asObject(BatchCSVRecord.class).getBody(); + + Base64NDArrayBody array = Unirest.post("http://localhost:9050/transformincrementalarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asObject(Base64NDArrayBody.class).getBody(); + + Base64NDArrayBody batchArray1 = Unirest.post("http://localhost:9050/transformarray") + .header("accept", "application/json").header("Content-Type", "application/json") + .body(batchCSVRecord).asObject(Base64NDArrayBody.class).getBody(); + + + + + + } + +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/ImageSparkTransformServerTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/ImageSparkTransformServerTest.java new file mode 100644 index 000000000..6c3e962eb --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/ImageSparkTransformServerTest.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + + +import com.mashape.unirest.http.JsonNode; +import com.mashape.unirest.http.ObjectMapper; +import com.mashape.unirest.http.Unirest; +import org.apache.commons.io.FileUtils; +import org.datavec.image.transform.ImageTransformProcess; +import org.datavec.spark.inference.server.ImageSparkTransformServer; +import org.datavec.spark.inference.model.model.Base64NDArrayBody; +import org.datavec.spark.inference.model.model.BatchImageRecord; +import org.datavec.spark.inference.model.model.SingleImageRecord; +import org.junit.AfterClass; +import org.junit.jupiter.api.BeforeAll; + +import org.junit.jupiter.api.Test; + +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.serde.base64.Nd4jBase64; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Created by kepricon on 17. 6. 19. + */ +public class ImageSparkTransformServerTest { + + @TempDir + public File testDir; + + private static ImageSparkTransformServer server; + private static File fileSave = new File(UUID.randomUUID().toString() + ".json"); + + @BeforeAll + public static void before() throws Exception { + server = new ImageSparkTransformServer(); + + ImageTransformProcess imgTransformProcess = new ImageTransformProcess.Builder().seed(12345) + .scaleImageTransform(10).cropImageTransform(5).build(); + + FileUtils.write(fileSave, imgTransformProcess.toJson()); + + Unirest.setObjectMapper(new ObjectMapper() { + private com.fasterxml.jackson.databind.ObjectMapper jacksonObjectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public T readValue(String value, Class valueType) { + try { + return jacksonObjectMapper.readValue(value, valueType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String writeValue(Object value) { + try { + return jacksonObjectMapper.writeValueAsString(value); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + + server.runMain(new String[] {"--jsonPath", fileSave.getAbsolutePath(), "-dp", "9060"}); + } + + @AfterClass + public static void after() throws Exception { + fileSave.deleteOnExit(); + server.stop(); + + } + + @Test + public void testImageServer() throws Exception { + SingleImageRecord record = + new SingleImageRecord(new ClassPathResource("datavec-spark-inference/testimages/class0/0.jpg").getFile().toURI()); + JsonNode jsonNode = Unirest.post("http://localhost:9060/transformincrementalarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asJson().getBody(); + Base64NDArrayBody array = Unirest.post("http://localhost:9060/transformincrementalarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asObject(Base64NDArrayBody.class).getBody(); + + BatchImageRecord batch = new BatchImageRecord(); + batch.add(new ClassPathResource("datavec-spark-inference/testimages/class0/0.jpg").getFile().toURI()); + batch.add(new ClassPathResource("datavec-spark-inference/testimages/class0/1.png").getFile().toURI()); + batch.add(new ClassPathResource("datavec-spark-inference/testimages/class0/2.jpg").getFile().toURI()); + + JsonNode jsonNodeBatch = + Unirest.post("http://localhost:9060/transformarray").header("accept", "application/json") + .header("Content-Type", "application/json").body(batch).asJson().getBody(); + Base64NDArrayBody batchArray = Unirest.post("http://localhost:9060/transformarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(batch) + .asObject(Base64NDArrayBody.class).getBody(); + + INDArray result = getNDArray(jsonNode); + assertEquals(1, result.size(0)); + + INDArray batchResult = getNDArray(jsonNodeBatch); + assertEquals(3, batchResult.size(0)); + +// System.out.println(array); + } + + @Test + public void testImageServerMultipart() throws Exception { + JsonNode jsonNode = Unirest.post("http://localhost:9060/transformimage") + .header("accept", "application/json") + .field("file1", new ClassPathResource("datavec-spark-inference/testimages/class0/0.jpg").getFile()) + .field("file2", new ClassPathResource("datavec-spark-inference/testimages/class0/1.png").getFile()) + .field("file3", new ClassPathResource("datavec-spark-inference/testimages/class0/2.jpg").getFile()) + .asJson().getBody(); + + + INDArray batchResult = getNDArray(jsonNode); + assertEquals(3, batchResult.size(0)); + +// System.out.println(batchResult); + } + + @Test + public void testImageServerSingleMultipart() throws Exception { + File f = testDir; + File imgFile = new ClassPathResource("datavec-spark-inference/testimages/class0/0.jpg").getTempFileFromArchive(f); + + JsonNode jsonNode = Unirest.post("http://localhost:9060/transformimage") + .header("accept", "application/json") + .field("file1", imgFile) + .asJson().getBody(); + + + INDArray result = getNDArray(jsonNode); + assertEquals(1, result.size(0)); + +// System.out.println(result); + } + + public INDArray getNDArray(JsonNode node) throws IOException { + return Nd4jBase64.fromBase64(node.getObject().getString("ndarray")); + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/SparkTransformServerTest.java b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/SparkTransformServerTest.java new file mode 100644 index 000000000..37773c2d2 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/java/org/datavec/spark/transform/SparkTransformServerTest.java @@ -0,0 +1,167 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.datavec.spark.transform; + + +import com.mashape.unirest.http.JsonNode; +import com.mashape.unirest.http.ObjectMapper; +import com.mashape.unirest.http.Unirest; +import org.apache.commons.io.FileUtils; +import org.datavec.api.transform.TransformProcess; +import org.datavec.api.transform.schema.Schema; +import org.datavec.image.transform.ImageTransformProcess; +import org.datavec.spark.inference.server.SparkTransformServerChooser; +import org.datavec.spark.inference.server.TransformDataType; +import org.datavec.spark.inference.model.model.*; +import org.junit.AfterClass; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.serde.base64.Nd4jBase64; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Created by kepricon on 17. 6. 20. + */ +public class SparkTransformServerTest { + private static SparkTransformServerChooser serverChooser; + private static Schema schema = new Schema.Builder().addColumnDouble("1.0").addColumnDouble("2.0").build(); + private static TransformProcess transformProcess = + new TransformProcess.Builder(schema).convertToDouble("1.0").convertToDouble( "2.0").build(); + + private static File imageTransformFile = new File(UUID.randomUUID().toString() + ".json"); + private static File csvTransformFile = new File(UUID.randomUUID().toString() + ".json"); + + @BeforeAll + public static void before() throws Exception { + serverChooser = new SparkTransformServerChooser(); + + ImageTransformProcess imgTransformProcess = new ImageTransformProcess.Builder().seed(12345) + .scaleImageTransform(10).cropImageTransform(5).build(); + + FileUtils.write(imageTransformFile, imgTransformProcess.toJson()); + + FileUtils.write(csvTransformFile, transformProcess.toJson()); + + Unirest.setObjectMapper(new ObjectMapper() { + private com.fasterxml.jackson.databind.ObjectMapper jacksonObjectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public T readValue(String value, Class valueType) { + try { + return jacksonObjectMapper.readValue(value, valueType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String writeValue(Object value) { + try { + return jacksonObjectMapper.writeValueAsString(value); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + + + } + + @AfterClass + public static void after() throws Exception { + imageTransformFile.deleteOnExit(); + csvTransformFile.deleteOnExit(); + } + + @Test + public void testImageServer() throws Exception { + serverChooser.runMain(new String[] {"--jsonPath", imageTransformFile.getAbsolutePath(), "-dp", "9060", "-dt", + TransformDataType.IMAGE.toString()}); + + SingleImageRecord record = + new SingleImageRecord(new ClassPathResource("datavec-spark-inference/testimages/class0/0.jpg").getFile().toURI()); + JsonNode jsonNode = Unirest.post("http://localhost:9060/transformincrementalarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asJson().getBody(); + Base64NDArrayBody array = Unirest.post("http://localhost:9060/transformincrementalarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asObject(Base64NDArrayBody.class).getBody(); + + BatchImageRecord batch = new BatchImageRecord(); + batch.add(new ClassPathResource("datavec-spark-inference/testimages/class0/0.jpg").getFile().toURI()); + batch.add(new ClassPathResource("datavec-spark-inference/testimages/class0/1.png").getFile().toURI()); + batch.add(new ClassPathResource("datavec-spark-inference/testimages/class0/2.jpg").getFile().toURI()); + + JsonNode jsonNodeBatch = + Unirest.post("http://localhost:9060/transformarray").header("accept", "application/json") + .header("Content-Type", "application/json").body(batch).asJson().getBody(); + Base64NDArrayBody batchArray = Unirest.post("http://localhost:9060/transformarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(batch) + .asObject(Base64NDArrayBody.class).getBody(); + + INDArray result = getNDArray(jsonNode); + assertEquals(1, result.size(0)); + + INDArray batchResult = getNDArray(jsonNodeBatch); + assertEquals(3, batchResult.size(0)); + + serverChooser.getSparkTransformServer().stop(); + } + + @Test + public void testCSVServer() throws Exception { + serverChooser.runMain(new String[] {"--jsonPath", csvTransformFile.getAbsolutePath(), "-dp", "9050", "-dt", + TransformDataType.CSV.toString()}); + + String[] values = new String[] {"1.0", "2.0"}; + SingleCSVRecord record = new SingleCSVRecord(values); + JsonNode jsonNode = + Unirest.post("http://localhost:9050/transformincremental").header("accept", "application/json") + .header("Content-Type", "application/json").body(record).asJson().getBody(); + SingleCSVRecord singleCsvRecord = Unirest.post("http://localhost:9050/transformincremental") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asObject(SingleCSVRecord.class).getBody(); + + BatchCSVRecord batchCSVRecord = new BatchCSVRecord(); + for (int i = 0; i < 3; i++) + batchCSVRecord.add(singleCsvRecord); + BatchCSVRecord batchCSVRecord1 = Unirest.post("http://localhost:9050/transform") + .header("accept", "application/json").header("Content-Type", "application/json") + .body(batchCSVRecord).asObject(BatchCSVRecord.class).getBody(); + + Base64NDArrayBody array = Unirest.post("http://localhost:9050/transformincrementalarray") + .header("accept", "application/json").header("Content-Type", "application/json").body(record) + .asObject(Base64NDArrayBody.class).getBody(); + + Base64NDArrayBody batchArray1 = Unirest.post("http://localhost:9050/transformarray") + .header("accept", "application/json").header("Content-Type", "application/json") + .body(batchCSVRecord).asObject(Base64NDArrayBody.class).getBody(); + + + serverChooser.getSparkTransformServer().stop(); + } + + public INDArray getNDArray(JsonNode node) throws IOException { + return Nd4jBase64.fromBase64(node.getObject().getString("ndarray")); + } +} diff --git a/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/resources/application.conf b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/resources/application.conf new file mode 100644 index 000000000..dbac92d83 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/datavec-spark-inference-server/src/test/resources/application.conf @@ -0,0 +1,6 @@ +play.modules.enabled += com.lightbend.lagom.discovery.zookeeper.ZooKeeperServiceLocatorModule +play.modules.enabled += io.skymind.skil.service.PredictionModule +play.crypto.secret = as8dufasdfuasdfjkasdkfalksjfk +play.server.pidfile.path=/tmp/RUNNING_PID + +play.server.http.port = 9600 diff --git a/cavis-datavec/datavec-spark-inference-parent/pom.xml b/cavis-datavec/datavec-spark-inference-parent/pom.xml new file mode 100644 index 000000000..750095167 --- /dev/null +++ b/cavis-datavec/datavec-spark-inference-parent/pom.xml @@ -0,0 +1,44 @@ + + + + + datavec-parent + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + datavec-spark-inference-parent + pom + + datavec-spark-inference-parent + + datavec-spark-inference-server + datavec-spark-inference-client + datavec-spark-inference-model + + + + + test-nd4j-native + + + test-nd4j-cuda-10.2 + + + diff --git a/cavis-dnn/build.gradle b/cavis-dnn/build.gradle new file mode 100644 index 000000000..4b2df0eb4 --- /dev/null +++ b/cavis-dnn/build.gradle @@ -0,0 +1,6 @@ +subprojects { + apply plugin: "java-library" + apply plugin: "maven-publish" + apply plugin: "signing" + +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-api/build.gradle b/cavis-dnn/cavis-dnn-api/build.gradle new file mode 100644 index 000000000..c0fc57e93 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/build.gradle @@ -0,0 +1,77 @@ +buildscript { + + dependencies { + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.16' + } +} + +plugins { + id 'java-library' + id 'com.google.protobuf' version '0.8.16' + id 'idea' + id 'maven-publish' +} +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +group 'net.brutex' +version '1.0.0-SNAPSHOT' + +dependencies { + testRuntimeOnly 'net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT' + + if(withCuda()) { + implementation "org.bytedeco:cuda" + } + + implementation 'com.google.protobuf:protobuf-java' + implementation 'org.bytedeco:javacpp' + implementation 'com.google.flatbuffers:flatbuffers-java' + implementation 'com.google.guava:guava' + implementation 'org.apache.commons:commons-lang3' + implementation 'commons-io:commons-io' + implementation 'org.slf4j:slf4j-api' + implementation 'com.fasterxml.jackson.core:jackson-annotations' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + implementation 'org.apache.commons:commons-collections4' + implementation 'org.apache.commons:commons-compress' + implementation 'com.jakewharton.byteunits:byteunits' + implementation 'org.apache.commons:commons-math3' + implementation 'net.ericaro:neoitertools' + implementation 'commons-net:commons-net' + implementation 'com.github.oshi:oshi-core' + + api project(':cavis-dnn:cavis-dnn-common') +} + +sourceSets { + main { + proto { + //srcDirs += 'src/main/protobuf/tf/ + srcDirs += 'src/main/protobuf/tf' + srcDirs += 'src/main/protobuf/nd4j' + srcDirs += 'src/main/protobuf/onnx' + + } + java { + srcDir 'src/main/java' + } + } +} + +protobuf { + // Configure the protoc executable + protoc { + // Download from repositories + artifact = 'com.google.protobuf:protoc:3.0.0' + } +} + +idea { + module { + // proto files and generated Java files are automatically added as + // source dirs. + // If you have additional sources, add them here: + //sourceDirs += file("/path/sto/other/source"); + } +} \ No newline at end of file diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/TFGraphRunnerService.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/TFGraphRunnerService.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/TFGraphRunnerService.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/TFGraphRunnerService.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/adapters/InferenceAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/adapters/InferenceAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/adapters/InferenceAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/adapters/InferenceAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/adapters/InputAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/adapters/InputAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/adapters/InputAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/adapters/InputAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/adapters/OutputAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/adapters/OutputAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/adapters/OutputAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/adapters/OutputAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/BasicGraphExecutioner.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/BasicGraphExecutioner.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/BasicGraphExecutioner.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/BasicGraphExecutioner.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/GraphExecutioner.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/GraphExecutioner.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/GraphExecutioner.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/GraphExecutioner.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/Node.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/Node.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/Node.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/Node.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/conf/ExecutionMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/conf/ExecutionMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/conf/ExecutionMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/conf/ExecutionMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/conf/ExecutorConfiguration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/conf/ExecutorConfiguration.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/conf/ExecutorConfiguration.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/conf/ExecutorConfiguration.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/conf/OutputMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/conf/OutputMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/conf/OutputMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/conf/OutputMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/input/Operands.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/input/Operands.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/input/Operands.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/input/Operands.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/input/OperandsAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/input/OperandsAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/execution/input/OperandsAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/execution/input/OperandsAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java index e7e134963..3ab872d91 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/functions/DifferentialFunction.java @@ -38,7 +38,7 @@ import org.nd4j.linalg.api.ops.Op; import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.shape.LongShapeDescriptor; import org.nd4j.linalg.exception.ND4JIllegalStateException; -import org.nd4j.shade.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.tensorflow.framework.AttrValue; import org.tensorflow.framework.GraphDef; import org.tensorflow.framework.NodeDef; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/At.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/At.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/At.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/At.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/BaseEvaluationListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/BaseEvaluationListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/BaseEvaluationListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/BaseEvaluationListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/BaseListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/BaseListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/BaseListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/BaseListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/Listener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/Listener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/Listener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/Listener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/ListenerEvaluations.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/ListenerEvaluations.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/ListenerEvaluations.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/ListenerEvaluations.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/ListenerResponse.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/ListenerResponse.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/ListenerResponse.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/ListenerResponse.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/ListenerVariables.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/ListenerVariables.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/ListenerVariables.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/ListenerVariables.java index 4b4b7ae30..a66afbbd2 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/ListenerVariables.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/ListenerVariables.java @@ -20,7 +20,7 @@ package org.nd4j.autodiff.listeners; -import org.nd4j.shade.guava.collect.Sets; +import com.google.common.collect.Sets; import java.util.Arrays; import java.util.HashSet; @@ -32,10 +32,6 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.Setter; import org.nd4j.autodiff.samediff.SDVariable; -import org.nd4j.autodiff.samediff.SameDiff; -import org.nd4j.autodiff.samediff.internal.SameDiffOp; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.api.MultiDataSet; @RequiredArgsConstructor @Getter diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/Loss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/Loss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/Loss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/Loss.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/Operation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/Operation.java new file mode 100644 index 000000000..10bd4ed24 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/Operation.java @@ -0,0 +1,54 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.autodiff.listeners; + +import java.util.Map; + +import org.nd4j.autodiff.samediff.SameDiff; + +public enum Operation { + /** + * The training operation: {@link SameDiff#fit()} methods training step (everything except validation). + */ + TRAINING, + /** + * The training validation operation: the validation step during {@link SameDiff#fit()} methods. + */ + TRAINING_VALIDATION, + /** + * Inference operations: {@link SameDiff#output()}, {@link SameDiff#batchOutput()} and {@link SameDiff#exec(Map, String...)} ()} methods, + * as well as {@link SameDiff#execBackwards(Map, Operation, String...)} methods. + */ + INFERENCE, + /** + * Evaluation operations: {@link SameDiff#evaluate()} methods. + */ + EVALUATION; + + public boolean isTrainingPhase() { + return this == TRAINING || this == TRAINING_VALIDATION; + } + + public boolean isValidation() { + return this == TRAINING_VALIDATION; + } + +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/checkpoint/Checkpoint.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/checkpoint/Checkpoint.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/checkpoint/Checkpoint.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/checkpoint/Checkpoint.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/checkpoint/CheckpointListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/checkpoint/CheckpointListener.java new file mode 100644 index 000000000..7bc8f044e --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/checkpoint/CheckpointListener.java @@ -0,0 +1,604 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.autodiff.listeners.checkpoint; + + +import com.google.common.io.Files; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.nd4j.autodiff.listeners.At; +import org.nd4j.autodiff.listeners.BaseListener; +import org.nd4j.autodiff.listeners.ListenerResponse; +import org.nd4j.autodiff.listeners.Loss; +import org.nd4j.autodiff.listeners.records.LossCurve; +import org.nd4j.autodiff.listeners.Operation; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class CheckpointListener extends BaseListener implements Serializable { + + private enum KeepMode {ALL, LAST, LAST_AND_EVERY}; + + private File rootDir; + private String fileNamePrefix; + private KeepMode keepMode; + private int keepLast; + private int keepEvery; + private boolean logSaving; + private boolean deleteExisting; + private boolean saveUpdaterState; + + private Integer saveEveryNEpochs; + private Integer saveEveryNIterations; + private boolean saveEveryNIterSinceLast; + private Long saveEveryAmount; + private TimeUnit saveEveryUnit; + private Long saveEveryMs; + private boolean saveEverySinceLast; + + private int lastCheckpointNum = -1; + private File checkpointRecordFile; + + private Checkpoint lastCheckpoint; + private long startTime = -1; + private int startIter = -1; + private Long lastSaveEveryMsNoSinceLast; + + private CheckpointListener(Builder builder){ + this.rootDir = builder.rootDir; + this.fileNamePrefix = builder.fileNamePrefix; + this.keepMode = builder.keepMode; + this.keepLast = builder.keepLast; + this.keepEvery = builder.keepEvery; + this.logSaving = builder.logSaving; + this.deleteExisting = builder.deleteExisting; + this.saveUpdaterState = builder.saveUpdaterState; + + this.saveEveryNEpochs = builder.saveEveryNEpochs; + this.saveEveryNIterations = builder.saveEveryNIterations; + this.saveEveryNIterSinceLast = builder.saveEveryNIterSinceLast; + this.saveEveryAmount = builder.saveEveryAmount; + this.saveEveryUnit = builder.saveEveryUnit; + this.saveEverySinceLast = builder.saveEverySinceLast; + + if(saveEveryAmount != null){ + saveEveryMs = TimeUnit.MILLISECONDS.convert(saveEveryAmount, saveEveryUnit); + } + + if(!rootDir.exists()){ + rootDir.mkdir(); + } + + this.checkpointRecordFile = new File(rootDir, "checkpointInfo.txt"); + if(this.checkpointRecordFile.exists() && this.checkpointRecordFile.length() > 0){ + + if(deleteExisting){ + //Delete any files matching: + //"checkpoint_" + checkpointNum + "_" + modelType + ".zip"; + this.checkpointRecordFile.delete(); + File[] files = rootDir.listFiles(); + if(files != null && files.length > 0){ + for(File f : files){ + String name = f.getName(); + if(name.startsWith("checkpoint_") && (name.endsWith("MultiLayerNetwork.zip") || name.endsWith("ComputationGraph.zip"))){ + f.delete(); + } + } + } + } else { + throw new IllegalStateException("Detected existing checkpoint files at directory " + rootDir.getAbsolutePath() + + ". Use deleteExisting(true) to delete existing checkpoint files when present."); + } + } + } + + @Override + public ListenerResponse epochEnd(SameDiff sameDiff, At at, LossCurve lossCurve, long epochTimeMillis) { + if(saveEveryNEpochs != null && (at.epoch()+1) % saveEveryNEpochs == 0){ + //Save: + saveCheckpoint(sameDiff, at); + } + //General saving conditions: don't need to check here - will check in iterationDone + return ListenerResponse.CONTINUE; + } + + @Override + public boolean isActive(Operation operation) { + return operation == Operation.TRAINING; + } + + @Override + public void iterationDone(SameDiff sd, At at, MultiDataSet dataSet, Loss loss) { + if (startTime < 0) { + startTime = System.currentTimeMillis(); + startIter = at.iteration(); + return; + } + + //Check iterations saving condition: + if(saveEveryNIterations != null){ + if(saveEveryNIterSinceLast){ + //Consider last saved model when deciding whether to save + long lastSaveIter = (lastCheckpoint != null ? lastCheckpoint.getIteration() : startIter); + if(at.iteration() - lastSaveIter >= saveEveryNIterations){ + saveCheckpoint(sd, at); + return; + } + } else { + //Same every N iterations, regardless of saving time + if((at.iteration()+1) % saveEveryNIterations == 0){ + saveCheckpoint(sd, at); + return; + } + } + } + + //Check time saving condition: + long time = System.currentTimeMillis(); + if(saveEveryUnit != null){ + if(saveEverySinceLast){ + //Consider last saved when deciding whether to save + long lastSaveTime = (lastCheckpoint != null ? lastCheckpoint.getTimestamp() : startTime); + if((time - lastSaveTime) >= saveEveryMs){ + saveCheckpoint(sd, at); + return; + } + } else { + //Save periodically, regardless of when last model was saved + long lastSave = (lastSaveEveryMsNoSinceLast != null ? lastSaveEveryMsNoSinceLast : startTime); + if((time - lastSave) > saveEveryMs){ + saveCheckpoint(sd, at); + lastSaveEveryMsNoSinceLast = time; + return; + } + } + } + } + + private void saveCheckpoint(SameDiff sd, At at) { + try{ + saveCheckpointHelper(sd, at); + } catch (Exception e){ + throw new RuntimeException("Error saving checkpoint", e); + } + } + + private void saveCheckpointHelper(SameDiff model, At at) throws Exception { + if(!checkpointRecordFile.exists()){ + checkpointRecordFile.createNewFile(); + writeCheckpointInfo(Checkpoint.getFileHeader() + "\n", checkpointRecordFile); + } + + Checkpoint c = new Checkpoint(++lastCheckpointNum, System.currentTimeMillis(), at.iteration(), at.epoch(),null); + String filename = getFileName(lastCheckpointNum, at, c.getTimestamp()); + c.setFilename(filename); + + File saveFile = new File(rootDir, c.getFilename()); + model.save(saveFile, this.saveUpdaterState); + + String s = c.toFileString(); + writeCheckpointInfo(s + "\n", checkpointRecordFile); + + if(logSaving){ + log.info("Model checkpoint saved: epoch {}, iteration {}, path: {}", c.getEpoch(), c.getIteration(), + new File(rootDir, c.getFilename()).getPath() ); + } + this.lastCheckpoint = c; + + + //Finally: determine if we should delete some old models... + if(keepMode == null || keepMode == KeepMode.ALL){ + return; + } else if(keepMode == KeepMode.LAST){ + List checkpoints = availableCheckpoints(); + Iterator iter = checkpoints.iterator(); + while(checkpoints.size() > keepLast){ + Checkpoint toRemove = iter.next(); + File f = getFileForCheckpoint(toRemove); + f.delete(); + iter.remove(); + } + } else { + //Keep mode: last N and every M + for(Checkpoint cp : availableCheckpoints()){ + if(cp.getCheckpointNum() > 0 && (cp.getCheckpointNum()+1) % keepEvery == 0){ + //One of the "every M to keep" models + continue; + } else if(cp.getCheckpointNum() > lastCheckpointNum - keepLast ){ //Example: latest is 5, keep last 2 -> keep checkpoints 4 and 5 + //One of last N to keep + continue; + } + //Otherwise: delete file + File f = getFileForCheckpoint(cp); + f.delete(); + } + } + } + + //Filename format: "_checkpoint-#_epoch-#_iter-#_YYYY-MM-dd_HH-MM-ss.bin" + private String getFileName(int checkpointNum, At at, long time){ + StringBuilder sb = new StringBuilder(); + if(fileNamePrefix != null){ + sb.append(fileNamePrefix); + if(!fileNamePrefix.endsWith("_")){ + sb.append("_"); + } + } + sb.append("checkpoint-") + .append(checkpointNum) + .append("_epoch-").append(at.epoch()) + .append("_iter-").append(at.iteration()); + + SimpleDateFormat sdf = new SimpleDateFormat("YYYY-MM-dd_HH-mm-ss"); + String date = sdf.format(new Date(time)); + sb.append("_").append(date) + .append(".bin"); + + return sb.toString(); + } + + private static String writeCheckpointInfo(String str, File f){ + try { + if(!f.exists()){ + f.createNewFile(); + } + Files.append(str, f, StandardCharsets.UTF_8); + } catch (IOException e){ + throw new RuntimeException(e); + } + return str; + } + + /** + * List all available checkpoints. A checkpoint is 'available' if the file can be loaded. Any checkpoint files that + * have been automatically deleted (given the configuration) will not be returned here. + * + * @return List of checkpoint files that can be loaded + */ + public List availableCheckpoints(){ + if(!checkpointRecordFile.exists()){ + return Collections.emptyList(); + } + + return availableCheckpoints(rootDir); + } + + /** + * List all available checkpoints. A checkpoint is 'available' if the file can be loaded. Any checkpoint files that + * have been automatically deleted (given the configuration) will not be returned here. + * Note that the checkpointInfo.txt file must exist, as this stores checkpoint information + * + * @return List of checkpoint files that can be loaded from the specified directory + */ + public static List availableCheckpoints(File directory){ + File checkpointRecordFile = new File(directory, "checkpointInfo.txt"); + Preconditions.checkState(checkpointRecordFile.exists(), "Could not find checkpoint record file at expected path %s", checkpointRecordFile.getAbsolutePath()); + + List lines; + try(InputStream is = new BufferedInputStream(new FileInputStream(checkpointRecordFile))){ + lines = IOUtils.readLines(is); + } catch (IOException e){ + throw new RuntimeException("Error loading checkpoint data from file: " + checkpointRecordFile.getAbsolutePath(), e); + } + + List out = new ArrayList<>(lines.size()-1); //Assume first line is header + for( int i=1; i all = availableCheckpoints(rootDir); + if(all.isEmpty()){ + return null; + } + return all.get(all.size()-1); + } + + /** + * Get the model file for the given checkpoint. Checkpoint model file must exist + * + * @param checkpoint Checkpoint to get the model file for + * @return Model file for the checkpoint + */ + public File getFileForCheckpoint(Checkpoint checkpoint){ + return getFileForCheckpoint(checkpoint.getCheckpointNum()); + } + + /** + * Get the model file for the given checkpoint number. Checkpoint model file must exist + * + * @param checkpointNum Checkpoint number to get the model file for + * @return Model file for the checkpoint + */ + public File getFileForCheckpoint(int checkpointNum) { + return getFileForCheckpoint(rootDir, checkpointNum); + } + + public static File getFileForCheckpoint(File rootDir, int checkpointNum){ + //Scan the root directory, for a file matching the checkpoint filename pattern: + //Filename format: "_checkpoint-#_epoch-#_iter-#_YYYY-MM-dd_HH-MM-ss.bin" + + if(checkpointNum < 0){ + throw new IllegalArgumentException("Invalid checkpoint number: " + checkpointNum); + } + + String contains = "_checkpoint-" + checkpointNum + "_epoch-"; + + File[] allFiles = rootDir.listFiles(); + if(allFiles != null){ + for(File f : allFiles){ + if(f.getAbsolutePath().contains(contains)){ + return f; + } + } + } + + throw new IllegalStateException("Model file for checkpoint " + checkpointNum + " does not exist"); + } + + /** + * Load a given checkpoint number + * + * @param loadUpdaterState If true: load the updater state. See {@link SameDiff#load(File, boolean)} for more details + * + */ + public SameDiff loadCheckpoint(int checkpointNum, boolean loadUpdaterState){ + return loadCheckpoint(rootDir, checkpointNum, loadUpdaterState); + } + + /** + * Load a SameDiff instance for the given checkpoint that resides in the specified root directory + * + * @param rootDir Directory that the checkpoint resides in + * @param checkpointNum Checkpoint model number to load + * @param loadUpdaterState If true: load the updater state. See {@link SameDiff#load(File, boolean)} for more details + * @return The loaded model + */ + public static SameDiff loadCheckpoint(File rootDir, int checkpointNum, boolean loadUpdaterState) { + File f = getFileForCheckpoint(rootDir, checkpointNum); + return SameDiff.load(f, loadUpdaterState); + } + + /** + * Load the last (most recent) checkpoint from the specified root directory + * @param rootDir Root directory to load checpoint from + * @return ComputationGraph for last checkpoint + */ + public static SameDiff loadLastCheckpoint(File rootDir, boolean loadUpdaterState){ + Checkpoint last = lastCheckpoint(rootDir); + return loadCheckpoint(rootDir, last.getCheckpointNum(), loadUpdaterState); + } + + public static Builder builder(@NonNull File rootDir){ + return new Builder(rootDir); + } + + public static class Builder { + + private File rootDir; + private String fileNamePrefix = "SameDiff"; + private KeepMode keepMode; + private int keepLast; + private int keepEvery; + private boolean saveUpdaterState = true; + private boolean logSaving = true; + private boolean deleteExisting = false; + + private Integer saveEveryNEpochs; + private Integer saveEveryNIterations; + private boolean saveEveryNIterSinceLast; + private Long saveEveryAmount; + private TimeUnit saveEveryUnit; + private boolean saveEverySinceLast; + + /** + * @param rootDir Root directory to save models to + */ + public Builder(@NonNull String rootDir){ + this(new File(rootDir)); + } + + /** + * @param rootDir Root directory to save models to + */ + public Builder(@NonNull File rootDir){ + this.rootDir = rootDir; + } + + public Builder fileNamePrefix(String fileNamePrefix){ + this.fileNamePrefix = fileNamePrefix; + return this; + } + + /** + * Save a model at the end of every epoch + */ + public Builder saveEveryEpoch(){ + return saveEveryNEpochs(1); + } + + /** + * Save a model at the end of every N epochs + */ + public Builder saveEveryNEpochs(int n){ + this.saveEveryNEpochs = n; + return this; + } + + /** + * Save a model every N iterations + */ + public Builder saveEveryNIterations(int n){ + return saveEveryNIterations(n, false); + } + + /** + * Save a model every N iterations (if sinceLast == false), or if N iterations have passed since + * the last model vas saved (if sinceLast == true) + */ + public Builder saveEveryNIterations(int n, boolean sinceLast){ + this.saveEveryNIterations = n; + this.saveEveryNIterSinceLast = sinceLast; + return this; + } + + /** + * Save a model periodically + * + * @param amount Quantity of the specified time unit + * @param timeUnit Time unit + */ + public Builder saveEvery(long amount, TimeUnit timeUnit){ + return saveEvery(amount, timeUnit, false); + } + + /** + * Save a model periodically (if sinceLast == false), or if the specified amount of time has elapsed since + * the last model was saved (if sinceLast == true) + * + * @param amount Quantity of the specified time unit + * @param timeUnit Time unit + */ + public Builder saveEvery(long amount, TimeUnit timeUnit, boolean sinceLast){ + this.saveEveryAmount = amount; + this.saveEveryUnit = timeUnit; + this.saveEverySinceLast = sinceLast; + return this; + } + + /** + * Keep all model checkpoints - i.e., don't delete any. Note that this is the default. + */ + public Builder keepAll(){ + this.keepMode = KeepMode.ALL; + return this; + } + + /** + * Keep only the last N most recent model checkpoint files. Older checkpoints will automatically be deleted. + * @param n Number of most recent checkpoints to keep + */ + public Builder keepLast(int n){ + if(n <= 0){ + throw new IllegalArgumentException("Number of model files to keep should be > 0 (got: " + n + ")"); + } + this.keepMode = KeepMode.LAST; + this.keepLast = n; + return this; + } + + /** + * Keep the last N most recent model checkpoint files, and every M checkpoint files.
+ * For example: suppose you save every 100 iterations, for 2050 iteration, and use keepLastAndEvery(3,5). + * This means after 2050 iterations you would have saved 20 checkpoints - some of which will be deleted. + * Those remaining in this example: iterations 500, 1000, 1500, 1800, 1900, 2000. + * @param nLast Most recent checkpoints to keep + * @param everyN Every N checkpoints to keep (regardless of age) + */ + public Builder keepLastAndEvery(int nLast, int everyN){ + if(nLast <= 0){ + throw new IllegalArgumentException("Most recent number of model files to keep should be > 0 (got: " + + nLast + ")"); + } + if(everyN <= 0){ + throw new IllegalArgumentException("Every n model files to keep should be > 0 (got: " + + everyN + ")"); + } + + this.keepMode = KeepMode.LAST_AND_EVERY; + this.keepLast = nLast; + this.keepEvery = everyN; + return this; + } + + /** + * If true (the default) log a message every time a model is saved + * + * @param logSaving Whether checkpoint saves should be logged or not + */ + public Builder logSaving(boolean logSaving){ + this.logSaving = logSaving; + return this; + } + + /** + * Whether the updater state (history/state for Adam, Nesterov momentum, etc) should be saved with each checkpoint.
+ * Updater state is saved by default. + * If you expect to continue training on any of the checkpoints, this should be set to true. However, it will increase + * the file size. + * @param saveUpdaterState If true: updater state will be saved with checkpoints. False: not saved. + */ + public Builder saveUpdaterState(boolean saveUpdaterState){ + this.saveUpdaterState = saveUpdaterState; + return this; + } + + /** + * If the checkpoint listener is set to save to a non-empty directory, should the CheckpointListener-related + * content be deleted?
+ * This is disabled by default (and instead, an exception will be thrown if existing data is found)
+ * WARNING: Be careful when enabling this, as it deletes all saved checkpoint models in the specified directory! + */ + public Builder deleteExisting(boolean deleteExisting){ + this.deleteExisting = deleteExisting; + return this; + } + + public CheckpointListener build(){ + if(saveEveryNEpochs == null && saveEveryAmount == null && saveEveryNIterations == null){ + throw new IllegalStateException("Cannot construct listener: no models will be saved (must use at least" + + " one of: save every N epochs, every N iterations, or every T time periods)"); + } + + return new CheckpointListener(this); + } + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/debugging/ArraySavingListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/debugging/ArraySavingListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/debugging/ArraySavingListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/debugging/ArraySavingListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/debugging/ExecDebuggingListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/debugging/ExecDebuggingListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/debugging/ExecDebuggingListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/debugging/ExecDebuggingListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/debugging/OpBenchmarkListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/debugging/OpBenchmarkListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/debugging/OpBenchmarkListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/debugging/OpBenchmarkListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/impl/HistoryListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/impl/HistoryListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/impl/HistoryListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/impl/HistoryListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/impl/ScoreListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/impl/ScoreListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/impl/ScoreListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/impl/ScoreListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/impl/UIListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/impl/UIListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/impl/UIListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/impl/UIListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/ProfilingListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/ProfilingListener.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/ProfilingListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/ProfilingListener.java index 791049958..c2d20756f 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/ProfilingListener.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/ProfilingListener.java @@ -35,10 +35,10 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.dataset.api.MultiDataSet; import org.nd4j.common.primitives.AtomicBoolean; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import java.io.*; import java.lang.management.ManagementFactory; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/Config.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/Config.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/Config.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/Config.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/OpStats.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/OpStats.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/OpStats.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/OpStats.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/ProfileAnalyzer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/ProfileAnalyzer.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/ProfileAnalyzer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/ProfileAnalyzer.java index 08c5f7727..68520efe7 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/ProfileAnalyzer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/listeners/profiler/comparison/ProfileAnalyzer.java @@ -22,7 +22,6 @@ package org.nd4j.autodiff.listeners.profiler.comparison; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; import org.nd4j.autodiff.functions.DifferentialFunction; import org.nd4j.autodiff.listeners.profiler.ProfilingListener; import org.nd4j.autodiff.listeners.profiler.data.Phase; @@ -34,10 +33,10 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; import org.nd4j.list.NDArrayList; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.*; -import java.nio.charset.Charset; +import java.io.File; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.*; @@ -141,20 +140,13 @@ public class ProfileAnalyzer { public static TraceEvent[] getTraceEvents(File file, ProfileFormat profileFormat, boolean aggregateTFSubOps) { ObjectMapper json = ProfilingListener.jsonMapper(); - String content = null; - try(BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(file))) { - try { - content = IOUtils.toString(bufferedInputStream, Charset.defaultCharset()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } catch (FileNotFoundException e) { - e.printStackTrace(); + String content; + try { + content = FileUtils.readFileToString(file, StandardCharsets.UTF_8); } catch (IOException e) { - e.printStackTrace(); + throw new RuntimeException(e); } - if (!content.matches(".*]\\s*")) { if (content.endsWith(",")) { //Has comma, missing ] @@ -198,7 +190,7 @@ public class ProfileAnalyzer { } - if(aggregateTFSubOps) { + if(aggregateTFSubOps){ //For CUDA ops, TF will log sub-ops like: //fire2/e1x1/Conv2D:Conv2D#id=74,device=/job:localhost/replica:0/task:0/device:GPU:0,async=false#@@cudnn::maxwell::gemm::computeOffsetsKernel(cudnn::maxwell::gemm::ComputeOffsetsParams) //fire2/e1x1/Conv2D:Conv2D#id=74,device=/job:localhost/replica:0/task:0/device:GPU:0,async=false#@@maxwell_scudnn_128x64_relu_interior_nn @@ -226,7 +218,7 @@ public class ProfileAnalyzer { } last = te; - if(te.getArgs() == null || te.getArgs().isEmpty()) { + if(te.getArgs() == null || te.getArgs().isEmpty()){ out.add(te); continue; } @@ -268,7 +260,7 @@ public class ProfileAnalyzer { } //Strip everything after ":" in "fire2/e1x1/Conv2D:Conv2D#id=74,device=/job:localhost/..." - for( int i = 0; i < out.size(); i++) { + for( int i=0; i> { private static final String SCOPE_PANIC_MSG = "If required, arrays in workspaces can be detached using INDArray.detach() before being passed to the SameDiff instance.\n" + diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/SameDiffOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/SameDiffOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/SameDiffOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/SameDiffOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/SessionMemMgr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/SessionMemMgr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/SessionMemMgr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/SessionMemMgr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/TrainingSession.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/TrainingSession.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/TrainingSession.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/TrainingSession.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/Variable.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/AbstractMemoryMgr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/AbstractMemoryMgr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/AbstractMemoryMgr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/AbstractMemoryMgr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCacheMemoryMgr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCacheMemoryMgr.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCacheMemoryMgr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCacheMemoryMgr.java index 754520dfb..b07f4e094 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCacheMemoryMgr.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCacheMemoryMgr.java @@ -26,7 +26,6 @@ import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.shape.LongShapeDescriptor; -import org.nd4j.linalg.api.shape.options.ArrayOptionsHelper; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.util.ArrayUtil; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCloseMemoryMgr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCloseMemoryMgr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCloseMemoryMgr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/ArrayCloseMemoryMgr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/CloseValidationMemoryMgr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/CloseValidationMemoryMgr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/CloseValidationMemoryMgr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/CloseValidationMemoryMgr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/NoOpMemoryMgr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/NoOpMemoryMgr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/NoOpMemoryMgr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/internal/memory/NoOpMemoryMgr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBaseOps.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBaseOps.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBaseOps.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBaseOps.java index 2838f66f3..a80836439 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBaseOps.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBaseOps.java @@ -1,26 +1,31 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; import java.lang.String; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicInteger; + import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.common.base.Preconditions; @@ -301,7 +306,7 @@ public class SDBaseOps { * @param transposeB Whether to transpose B arrays or not */ public SDVariable[] batchMmul(SDVariable[] inputsA, SDVariable[] inputsB, boolean transposeA, - boolean transposeB) { + boolean transposeB) { SDValidation.validateNumerical("batchMmul", "inputsA", inputsA); Preconditions.checkArgument(inputsA.length >= 1, "inputsA has incorrect size/length. Expected: inputsA.length >= 1, got %s", inputsA.length); SDValidation.validateNumerical("batchMmul", "inputsB", inputsB); @@ -325,7 +330,7 @@ public class SDBaseOps { * @param transposeB Whether to transpose B arrays or not */ public SDVariable[] batchMmul(String[] names, SDVariable[] inputsA, SDVariable[] inputsB, - boolean transposeA, boolean transposeB) { + boolean transposeA, boolean transposeB) { SDValidation.validateNumerical("batchMmul", "inputsA", inputsA); Preconditions.checkArgument(inputsA.length >= 1, "inputsA has incorrect size/length. Expected: inputsA.length >= 1, got %s", inputsA.length); SDValidation.validateNumerical("batchMmul", "inputsB", inputsB); @@ -476,7 +481,7 @@ public class SDBaseOps { * @return output Output variable (NUMERIC type) */ public SDVariable cumprod(String name, SDVariable in, boolean exclusive, boolean reverse, - int... axis) { + int... axis) { SDValidation.validateNumerical("cumprod", "in", in); Preconditions.checkArgument(axis.length >= 1, "axis has incorrect size/length. Expected: axis.length >= 1, got %s", axis.length); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.custom.CumProd(sd,in, exclusive, reverse, axis).outputVariable(); @@ -557,7 +562,7 @@ public class SDBaseOps { * @return output (NUMERIC type) */ public SDVariable cumsum(String name, SDVariable in, boolean exclusive, boolean reverse, - int... axis) { + int... axis) { SDValidation.validateNumerical("cumsum", "in", in); Preconditions.checkArgument(axis.length >= 1, "axis has incorrect size/length. Expected: axis.length >= 1, got %s", axis.length); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.custom.CumSum(sd,in, exclusive, reverse, axis).outputVariable(); @@ -674,7 +679,7 @@ public class SDBaseOps { * @param numPartitions Number of partitions, >= 1 */ public SDVariable[] dynamicPartition(String[] names, SDVariable x, SDVariable partitions, - int numPartitions) { + int numPartitions) { SDValidation.validateNumerical("dynamicPartition", "x", x); SDValidation.validateInteger("dynamicPartition", "partitions", partitions); SDVariable[] out = new org.nd4j.linalg.api.ops.impl.transforms.custom.DynamicPartition(sd,x, partitions, numPartitions).outputVariables(); @@ -1183,7 +1188,7 @@ public class SDBaseOps { * @return output INDArray with linearly spaced elements (NUMERIC type) */ public SDVariable linspace(String name, DataType dataType, double start, double stop, - long number) { + long number) { SDVariable out = new org.nd4j.linalg.api.ops.impl.shape.Linspace(sd,dataType, start, stop, number).outputVariable(); return sd.updateVariableNameAndReference(out, name); } @@ -1199,7 +1204,7 @@ public class SDBaseOps { * @return output INDArray with linearly spaced elements (NUMERIC type) */ public SDVariable linspace(SDVariable start, SDVariable stop, SDVariable number, - DataType dataType) { + DataType dataType) { SDValidation.validateNumerical("linspace", "start", start); SDValidation.validateNumerical("linspace", "stop", stop); SDValidation.validateInteger("linspace", "number", number); @@ -1218,7 +1223,7 @@ public class SDBaseOps { * @return output INDArray with linearly spaced elements (NUMERIC type) */ public SDVariable linspace(String name, SDVariable start, SDVariable stop, SDVariable number, - DataType dataType) { + DataType dataType) { SDValidation.validateNumerical("linspace", "start", start); SDValidation.validateNumerical("linspace", "stop", stop); SDValidation.validateInteger("linspace", "number", number); @@ -1439,7 +1444,7 @@ public class SDBaseOps { * @return output Number of elements that the condition is satisfied for (NUMERIC type) */ public SDVariable matchConditionCount(SDVariable in, Condition condition, boolean keepDim, - int... dimensions) { + int... dimensions) { SDValidation.validateNumerical("matchConditionCount", "in", in); Preconditions.checkArgument(dimensions.length >= 0, "dimensions has incorrect size/length. Expected: dimensions.length >= 0, got %s", dimensions.length); return new org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition(sd,in, condition, keepDim, dimensions).outputVariable(); @@ -1463,7 +1468,7 @@ public class SDBaseOps { * @return output Number of elements that the condition is satisfied for (NUMERIC type) */ public SDVariable matchConditionCount(String name, SDVariable in, Condition condition, - boolean keepDim, int... dimensions) { + boolean keepDim, int... dimensions) { SDValidation.validateNumerical("matchConditionCount", "in", in); Preconditions.checkArgument(dimensions.length >= 0, "dimensions has incorrect size/length. Expected: dimensions.length >= 0, got %s", dimensions.length); SDVariable out = new org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition(sd,in, condition, keepDim, dimensions).outputVariable(); @@ -1508,7 +1513,7 @@ public class SDBaseOps { * @return output Number of elements that the condition is satisfied for (NUMERIC type) */ public SDVariable matchConditionCount(String name, SDVariable in, Condition condition, - int... dimensions) { + int... dimensions) { SDValidation.validateNumerical("matchConditionCount", "in", in); Preconditions.checkArgument(dimensions.length >= 0, "dimensions has incorrect size/length. Expected: dimensions.length >= 0, got %s", dimensions.length); SDVariable out = new org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition(sd,in, condition, false, dimensions).outputVariable(); @@ -1889,7 +1894,7 @@ public class SDBaseOps { * @return output (NUMERIC type) */ public SDVariable mmul(SDVariable x, SDVariable y, boolean transposeX, boolean transposeY, - boolean transposeZ) { + boolean transposeZ) { SDValidation.validateNumerical("mmul", "x", x); SDValidation.validateNumerical("mmul", "y", y); return new org.nd4j.linalg.api.ops.impl.reduce.Mmul(sd,x, y, transposeX, transposeY, transposeZ).outputVariable(); @@ -1908,7 +1913,7 @@ public class SDBaseOps { * @return output (NUMERIC type) */ public SDVariable mmul(String name, SDVariable x, SDVariable y, boolean transposeX, - boolean transposeY, boolean transposeZ) { + boolean transposeY, boolean transposeZ) { SDValidation.validateNumerical("mmul", "x", x); SDValidation.validateNumerical("mmul", "y", y); SDVariable out = new org.nd4j.linalg.api.ops.impl.reduce.Mmul(sd,x, y, transposeX, transposeY, transposeZ).outputVariable(); @@ -2298,14 +2303,14 @@ public class SDBaseOps { * * @param indices Indices - value 0 to depth-1 (NUMERIC type) * @param depth Number of classes - * @param axis - * @param on - * @param off + * @param axis + * @param on + * @param off * @param dataType Output data type * @return output Output variable (NUMERIC type) */ public SDVariable oneHot(SDVariable indices, int depth, int axis, double on, double off, - DataType dataType) { + DataType dataType) { SDValidation.validateNumerical("oneHot", "indices", indices); return new org.nd4j.linalg.api.ops.impl.shape.OneHot(sd,indices, depth, axis, on, off, dataType).outputVariable(); } @@ -2318,14 +2323,14 @@ public class SDBaseOps { * @param name name May be null. Name for the output variable * @param indices Indices - value 0 to depth-1 (NUMERIC type) * @param depth Number of classes - * @param axis - * @param on - * @param off + * @param axis + * @param on + * @param off * @param dataType Output data type * @return output Output variable (NUMERIC type) */ public SDVariable oneHot(String name, SDVariable indices, int depth, int axis, double on, - double off, DataType dataType) { + double off, DataType dataType) { SDValidation.validateNumerical("oneHot", "indices", indices); SDVariable out = new org.nd4j.linalg.api.ops.impl.shape.OneHot(sd,indices, depth, axis, on, off, dataType).outputVariable(); return sd.updateVariableNameAndReference(out, name); @@ -2338,9 +2343,9 @@ public class SDBaseOps { * * @param indices Indices - value 0 to depth-1 (NUMERIC type) * @param depth Number of classes - * @param axis - * @param on - * @param off + * @param axis + * @param on + * @param off * @return output Output variable (NUMERIC type) */ public SDVariable oneHot(SDVariable indices, int depth, int axis, double on, double off) { @@ -2356,13 +2361,13 @@ public class SDBaseOps { * @param name name May be null. Name for the output variable * @param indices Indices - value 0 to depth-1 (NUMERIC type) * @param depth Number of classes - * @param axis - * @param on - * @param off + * @param axis + * @param on + * @param off * @return output Output variable (NUMERIC type) */ public SDVariable oneHot(String name, SDVariable indices, int depth, int axis, double on, - double off) { + double off) { SDValidation.validateNumerical("oneHot", "indices", indices); SDVariable out = new org.nd4j.linalg.api.ops.impl.shape.OneHot(sd,indices, depth, axis, on, off, DataType.FLOAT).outputVariable(); return sd.updateVariableNameAndReference(out, name); @@ -2430,7 +2435,7 @@ public class SDBaseOps { * As per onesLike(String, SDVariable) but the output datatype may be specified
* * @param input (NUMERIC type) - * @param dataType + * @param dataType * @return output (NUMERIC type) */ public SDVariable onesLike(SDVariable input, DataType dataType) { @@ -2443,7 +2448,7 @@ public class SDBaseOps { * * @param name name May be null. Name for the output variable * @param input (NUMERIC type) - * @param dataType + * @param dataType * @return output (NUMERIC type) */ public SDVariable onesLike(String name, SDVariable input, DataType dataType) { @@ -2606,7 +2611,7 @@ public class SDBaseOps { * @param from Initial/smallest value * @param to Largest value (exclusive) * @param step Step size - * @param dataType + * @param dataType * @return output INDArray with the specified values (NUMERIC type) */ public SDVariable range(double from, double to, double step, DataType dataType) { @@ -2622,7 +2627,7 @@ public class SDBaseOps { * @param from Initial/smallest value * @param to Largest value (exclusive) * @param step Step size - * @param dataType + * @param dataType * @return output INDArray with the specified values (NUMERIC type) */ public SDVariable range(String name, double from, double to, double step, DataType dataType) { @@ -2638,7 +2643,7 @@ public class SDBaseOps { * @param from Initial/smallest value (NUMERIC type) * @param to Largest value (exclusive) (NUMERIC type) * @param step Step size (NUMERIC type) - * @param dataType + * @param dataType * @return output INDArray with the specified values (NUMERIC type) */ public SDVariable range(SDVariable from, SDVariable to, SDVariable step, DataType dataType) { @@ -2657,11 +2662,11 @@ public class SDBaseOps { * @param from Initial/smallest value (NUMERIC type) * @param to Largest value (exclusive) (NUMERIC type) * @param step Step size (NUMERIC type) - * @param dataType + * @param dataType * @return output INDArray with the specified values (NUMERIC type) */ public SDVariable range(String name, SDVariable from, SDVariable to, SDVariable step, - DataType dataType) { + DataType dataType) { SDValidation.validateNumerical("range", "from", from); SDValidation.validateNumerical("range", "to", to); SDValidation.validateNumerical("range", "step", step); @@ -2721,7 +2726,7 @@ public class SDBaseOps { * @return output New array with values replaced where condition is satisfied (NUMERIC type) */ public SDVariable replaceWhere(String name, SDVariable update, SDVariable from, - Condition condition) { + Condition condition) { SDValidation.validateNumerical("replaceWhere", "update", update); SDValidation.validateNumerical("replaceWhere", "from", from); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.comparison.CompareAndReplace(sd,update, from, condition).outputVariable(); @@ -2755,7 +2760,7 @@ public class SDBaseOps { * @return output New array with values replaced where condition is satisfied (NUMERIC type) */ public SDVariable replaceWhere(String name, SDVariable update, double value, - Condition condition) { + Condition condition) { SDValidation.validateNumerical("replaceWhere", "update", update); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.comparison.CompareAndSet(sd,update, value, condition).outputVariable(); return sd.updateVariableNameAndReference(out, name); @@ -2793,6 +2798,47 @@ public class SDBaseOps { return sd.updateVariableNameAndReference(out, name); } + + /** + * Split the input in to a list of sub tensors + * @param input the input to split + * @param numSizeSplits the number of splits + * @param splitDim the dimension to split along + * @return the set of output variables + */ + public SDVariable[] split(SDVariable input,int numSizeSplits,int splitDim) { + SDValidation.validateNumerical("split",input); + SDVariable[] out = new org.nd4j.linalg.api.ops.impl.shape.Split(sd,input,numSizeSplits,splitDim).outputVariables(); + return out; + } + + /** + * Split the input in to a list of sub tensors + * @param name the potential name of the input + * @param input the input to split + * @param numSizeSplits the number of splits + * @param splitDim the dimension to split along + * @return the set of output variables + */ + public SDVariable[] split(String name,SDVariable input,int numSizeSplits,int splitDim) { + SDValidation.validateNumerical("split",input); + SDVariable[] out = new org.nd4j.linalg.api.ops.impl.shape.Split(sd,input,numSizeSplits,splitDim).outputVariables(); + SDVariable[] ret = new SDVariable[out.length]; + AtomicInteger index = new AtomicInteger(0); + Arrays.stream(out).forEach(output -> { + if(index.get() < 1) { + ret[index.get()] = sd.updateVariableNameAndReference(output,name); + index.incrementAndGet(); + } + else { + ret[index.get()] = sd.updateVariableNameAndReference(output,name + ":" + index.get()); + index.incrementAndGet(); + } + }); + + return ret; + } + /** * Reshape the input variable to the specified (fixed) shape. The output variable will have the same values as the
* input, but with the specified shape.
@@ -2883,7 +2929,7 @@ public class SDBaseOps { * @return output Reversed sequences (NUMERIC type) */ public SDVariable reverseSequence(SDVariable x, SDVariable seq_lengths, int seqDim, - int batchDim) { + int batchDim) { SDValidation.validateNumerical("reverseSequence", "x", x); SDValidation.validateInteger("reverseSequence", "seq_lengths", seq_lengths); return new org.nd4j.linalg.api.ops.impl.transforms.custom.ReverseSequence(sd,x, seq_lengths, seqDim, batchDim).outputVariable(); @@ -2900,7 +2946,7 @@ public class SDBaseOps { * @return output Reversed sequences (NUMERIC type) */ public SDVariable reverseSequence(String name, SDVariable x, SDVariable seq_lengths, int seqDim, - int batchDim) { + int batchDim) { SDValidation.validateNumerical("reverseSequence", "x", x); SDValidation.validateInteger("reverseSequence", "seq_lengths", seq_lengths); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.custom.ReverseSequence(sd,x, seq_lengths, seqDim, batchDim).outputVariable(); @@ -3076,7 +3122,7 @@ public class SDBaseOps { * @return output The updated variable (NUMERIC type) */ public SDVariable scatterAdd(String name, SDVariable ref, SDVariable indices, - SDVariable updates) { + SDVariable updates) { SDValidation.validateNumerical("scatterAdd", "ref", ref); SDValidation.validateNumerical("scatterAdd", "indices", indices); SDValidation.validateNumerical("scatterAdd", "updates", updates); @@ -3119,7 +3165,7 @@ public class SDBaseOps { * @return output The updated variable (NUMERIC type) */ public SDVariable scatterDiv(String name, SDVariable ref, SDVariable indices, - SDVariable updates) { + SDVariable updates) { SDValidation.validateNumerical("scatterDiv", "ref", ref); SDValidation.validateNumerical("scatterDiv", "indices", indices); SDValidation.validateNumerical("scatterDiv", "updates", updates); @@ -3162,7 +3208,7 @@ public class SDBaseOps { * @return output The updated variable (NUMERIC type) */ public SDVariable scatterMax(String name, SDVariable ref, SDVariable indices, - SDVariable updates) { + SDVariable updates) { SDValidation.validateNumerical("scatterMax", "ref", ref); SDValidation.validateNumerical("scatterMax", "indices", indices); SDValidation.validateNumerical("scatterMax", "updates", updates); @@ -3205,7 +3251,7 @@ public class SDBaseOps { * @return output The updated variable (NUMERIC type) */ public SDVariable scatterMin(String name, SDVariable ref, SDVariable indices, - SDVariable updates) { + SDVariable updates) { SDValidation.validateNumerical("scatterMin", "ref", ref); SDValidation.validateNumerical("scatterMin", "indices", indices); SDValidation.validateNumerical("scatterMin", "updates", updates); @@ -3248,7 +3294,7 @@ public class SDBaseOps { * @return output The updated variable (NUMERIC type) */ public SDVariable scatterMul(String name, SDVariable ref, SDVariable indices, - SDVariable updates) { + SDVariable updates) { SDValidation.validateNumerical("scatterMul", "ref", ref); SDValidation.validateNumerical("scatterMul", "indices", indices); SDValidation.validateNumerical("scatterMul", "updates", updates); @@ -3291,7 +3337,7 @@ public class SDBaseOps { * @return output The updated variable (NUMERIC type) */ public SDVariable scatterSub(String name, SDVariable ref, SDVariable indices, - SDVariable updates) { + SDVariable updates) { SDValidation.validateNumerical("scatterSub", "ref", ref); SDValidation.validateNumerical("scatterSub", "indices", indices); SDValidation.validateNumerical("scatterSub", "updates", updates); @@ -3334,7 +3380,7 @@ public class SDBaseOps { * @return output The updated variable (NUMERIC type) */ public SDVariable scatterUpdate(String name, SDVariable ref, SDVariable indices, - SDVariable updates) { + SDVariable updates) { SDValidation.validateNumerical("scatterUpdate", "ref", ref); SDValidation.validateNumerical("scatterUpdate", "indices", indices); SDValidation.validateNumerical("scatterUpdate", "updates", updates); @@ -3548,7 +3594,7 @@ public class SDBaseOps { * * @param lengths Lengths of the sequences (NUMERIC type) * @param maxLen Maximum sequence length - * @param dataType + * @param dataType * @return output Output variable (NUMERIC type) */ public SDVariable sequenceMask(SDVariable lengths, int maxLen, DataType dataType) { @@ -3563,7 +3609,7 @@ public class SDBaseOps { * @param name name May be null. Name for the output variable * @param lengths Lengths of the sequences (NUMERIC type) * @param maxLen Maximum sequence length - * @param dataType + * @param dataType * @return output Output variable (NUMERIC type) */ public SDVariable sequenceMask(String name, SDVariable lengths, int maxLen, DataType dataType) { @@ -3578,7 +3624,7 @@ public class SDBaseOps { * * @param lengths Lengths of the sequences (NUMERIC type) * @param maxLen Maximum sequence length (INT type) - * @param dataType + * @param dataType * @return output Output variable (NUMERIC type) */ public SDVariable sequenceMask(SDVariable lengths, SDVariable maxLen, DataType dataType) { @@ -3594,11 +3640,11 @@ public class SDBaseOps { * @param name name May be null. Name for the output variable * @param lengths Lengths of the sequences (NUMERIC type) * @param maxLen Maximum sequence length (INT type) - * @param dataType + * @param dataType * @return output Output variable (NUMERIC type) */ public SDVariable sequenceMask(String name, SDVariable lengths, SDVariable maxLen, - DataType dataType) { + DataType dataType) { SDValidation.validateNumerical("sequenceMask", "lengths", lengths); SDValidation.validateInteger("sequenceMask", "maxLen", maxLen); SDVariable out = new org.nd4j.linalg.api.ops.impl.shape.SequenceMask(sd,lengths, maxLen, dataType).outputVariable(); @@ -3609,7 +3655,7 @@ public class SDBaseOps { * see sequenceMask(String, SDVariable, SDVariable, DataType)
* * @param lengths (NUMERIC type) - * @param dataType + * @param dataType * @return output (NUMERIC type) */ public SDVariable sequenceMask(SDVariable lengths, DataType dataType) { @@ -3622,7 +3668,7 @@ public class SDBaseOps { * * @param name name May be null. Name for the output variable * @param lengths (NUMERIC type) - * @param dataType + * @param dataType * @return output (NUMERIC type) */ public SDVariable sequenceMask(String name, SDVariable lengths, DataType dataType) { @@ -3799,32 +3845,6 @@ public class SDBaseOps { return sd.updateVariableNameAndReference(out, name); } - /** - * Split a value in to a list of ndarrays.
- * - * @param input Input to split (NUMERIC type) - * @param numSplit Number of splits - * @param splitDim The dimension to split on - */ - public SDVariable[] split(SDVariable input, int numSplit, int splitDim) { - SDValidation.validateNumerical("split", "input", input); - return new org.nd4j.linalg.api.ops.impl.shape.Split(sd,input, numSplit, splitDim).outputVariables(); - } - - /** - * Split a value in to a list of ndarrays.
- * - * @param names names May be null. Arrays of names for the output variables. - * @param input Input to split (NUMERIC type) - * @param numSplit Number of splits - * @param splitDim The dimension to split on - */ - public SDVariable[] split(String[] names, SDVariable input, int numSplit, int splitDim) { - SDValidation.validateNumerical("split", "input", input); - SDVariable[] out = new org.nd4j.linalg.api.ops.impl.shape.Split(sd,input, numSplit, splitDim).outputVariables(); - return sd.updateVariableNamesAndReferences(out, names); - } - /** * Squared L2 norm: see norm2(String, SDVariable, boolean, int...)
* @@ -3836,7 +3856,7 @@ public class SDBaseOps { * keepDims = false: [a,c]
* * @param x (NUMERIC type) - * @param keepDims + * @param keepDims * @param dimensions (Size: AtLeast(min=0)) * @return output (NUMERIC type) */ @@ -3858,7 +3878,7 @@ public class SDBaseOps { * * @param name name May be null. Name for the output variable * @param x (NUMERIC type) - * @param keepDims + * @param keepDims * @param dimensions (Size: AtLeast(min=0)) * @return output (NUMERIC type) */ @@ -3994,7 +4014,7 @@ public class SDBaseOps { * @return output reduced array of rank (input rank - num dimensions) (NUMERIC type) */ public SDVariable standardDeviation(SDVariable x, boolean biasCorrected, boolean keepDims, - int... dimensions) { + int... dimensions) { SDValidation.validateNumerical("standardDeviation", "x", x); Preconditions.checkArgument(dimensions.length >= 0, "dimensions has incorrect size/length. Expected: dimensions.length >= 0, got %s", dimensions.length); return new org.nd4j.linalg.api.ops.impl.summarystats.StandardDeviation(sd,x, biasCorrected, keepDims, dimensions).outputVariable(); @@ -4018,7 +4038,7 @@ public class SDBaseOps { * @return output reduced array of rank (input rank - num dimensions) (NUMERIC type) */ public SDVariable standardDeviation(String name, SDVariable x, boolean biasCorrected, - boolean keepDims, int... dimensions) { + boolean keepDims, int... dimensions) { SDValidation.validateNumerical("standardDeviation", "x", x); Preconditions.checkArgument(dimensions.length >= 0, "dimensions has incorrect size/length. Expected: dimensions.length >= 0, got %s", dimensions.length); SDVariable out = new org.nd4j.linalg.api.ops.impl.summarystats.StandardDeviation(sd,x, biasCorrected, keepDims, dimensions).outputVariable(); @@ -4063,7 +4083,7 @@ public class SDBaseOps { * @return output reduced array of rank (input rank - num dimensions) (NUMERIC type) */ public SDVariable standardDeviation(String name, SDVariable x, boolean biasCorrected, - int... dimensions) { + int... dimensions) { SDValidation.validateNumerical("standardDeviation", "x", x); Preconditions.checkArgument(dimensions.length >= 0, "dimensions has incorrect size/length. Expected: dimensions.length >= 0, got %s", dimensions.length); SDVariable out = new org.nd4j.linalg.api.ops.impl.summarystats.StandardDeviation(sd,x, biasCorrected, false, dimensions).outputVariable(); @@ -4092,7 +4112,7 @@ public class SDBaseOps { * @return output A subset of the input array (NUMERIC type) */ public SDVariable stridedSlice(SDVariable in, long[] begin, long[] end, long[] strides, - int beginMask, int endMask, int ellipsisMask, int newAxisMask, int shrinkAxisMask) { + int beginMask, int endMask, int ellipsisMask, int newAxisMask, int shrinkAxisMask) { SDValidation.validateNumerical("stridedSlice", "in", in); Preconditions.checkArgument(begin.length >= 1, "begin has incorrect size/length. Expected: begin.length >= 1, got %s", begin.length); Preconditions.checkArgument(end.length >= 1, "end has incorrect size/length. Expected: end.length >= 1, got %s", end.length); @@ -4123,8 +4143,8 @@ public class SDBaseOps { * @return output A subset of the input array (NUMERIC type) */ public SDVariable stridedSlice(String name, SDVariable in, long[] begin, long[] end, - long[] strides, int beginMask, int endMask, int ellipsisMask, int newAxisMask, - int shrinkAxisMask) { + long[] strides, int beginMask, int endMask, int ellipsisMask, int newAxisMask, + int shrinkAxisMask) { SDValidation.validateNumerical("stridedSlice", "in", in); Preconditions.checkArgument(begin.length >= 1, "begin has incorrect size/length. Expected: begin.length >= 1, got %s", begin.length); Preconditions.checkArgument(end.length >= 1, "end has incorrect size/length. Expected: end.length >= 1, got %s", end.length); @@ -4175,7 +4195,7 @@ public class SDBaseOps { * @return output A subset of the input array (NUMERIC type) */ public SDVariable stridedSlice(String name, SDVariable in, long[] begin, long[] end, - long... strides) { + long... strides) { SDValidation.validateNumerical("stridedSlice", "in", in); Preconditions.checkArgument(begin.length >= 1, "begin has incorrect size/length. Expected: begin.length >= 1, got %s", begin.length); Preconditions.checkArgument(end.length >= 1, "end has incorrect size/length. Expected: end.length >= 1, got %s", end.length); @@ -4309,7 +4329,7 @@ public class SDBaseOps { * @return output Output variable (NUMERIC type) */ public SDVariable tensorMmul(SDVariable x, SDVariable y, int[] dimensionsX, int[] dimensionsY, - boolean transposeX, boolean transposeY, boolean transposeZ) { + boolean transposeX, boolean transposeY, boolean transposeZ) { SDValidation.validateNumerical("tensorMmul", "x", x); SDValidation.validateNumerical("tensorMmul", "y", y); Preconditions.checkArgument(dimensionsX.length >= 1, "dimensionsX has incorrect size/length. Expected: dimensionsX.length >= 1, got %s", dimensionsX.length); @@ -4331,7 +4351,7 @@ public class SDBaseOps { * @return output Output variable (NUMERIC type) */ public SDVariable tensorMmul(String name, SDVariable x, SDVariable y, int[] dimensionsX, - int[] dimensionsY, boolean transposeX, boolean transposeY, boolean transposeZ) { + int[] dimensionsY, boolean transposeX, boolean transposeY, boolean transposeZ) { SDValidation.validateNumerical("tensorMmul", "x", x); SDValidation.validateNumerical("tensorMmul", "y", y); Preconditions.checkArgument(dimensionsX.length >= 1, "dimensionsX has incorrect size/length. Expected: dimensionsX.length >= 1, got %s", dimensionsX.length); @@ -4368,7 +4388,7 @@ public class SDBaseOps { * @return output Output variable (NUMERIC type) */ public SDVariable tensorMmul(String name, SDVariable x, SDVariable y, int[] dimensionsX, - int... dimensionsY) { + int... dimensionsY) { SDValidation.validateNumerical("tensorMmul", "x", x); SDValidation.validateNumerical("tensorMmul", "y", y); Preconditions.checkArgument(dimensionsX.length >= 1, "dimensionsX has incorrect size/length. Expected: dimensionsX.length >= 1, got %s", dimensionsX.length); @@ -4501,7 +4521,7 @@ public class SDBaseOps { * @return output Unsorted segment output (NUMERIC type) */ public SDVariable unsortedSegmentMax(String name, SDVariable data, SDVariable segmentIds, - int numSegments) { + int numSegments) { SDValidation.validateNumerical("unsortedSegmentMax", "data", data); SDValidation.validateNumerical("unsortedSegmentMax", "segmentIds", segmentIds); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentMax(sd,data, segmentIds, numSegments).outputVariable(); @@ -4540,7 +4560,7 @@ public class SDBaseOps { * @return output Unsorted segment output (NUMERIC type) */ public SDVariable unsortedSegmentMean(String name, SDVariable data, SDVariable segmentIds, - int numSegments) { + int numSegments) { SDValidation.validateNumerical("unsortedSegmentMean", "data", data); SDValidation.validateNumerical("unsortedSegmentMean", "segmentIds", segmentIds); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentMean(sd,data, segmentIds, numSegments).outputVariable(); @@ -4579,7 +4599,7 @@ public class SDBaseOps { * @return output Unsorted segment output (NUMERIC type) */ public SDVariable unsortedSegmentMin(String name, SDVariable data, SDVariable segmentIds, - int numSegments) { + int numSegments) { SDValidation.validateNumerical("unsortedSegmentMin", "data", data); SDValidation.validateNumerical("unsortedSegmentMin", "segmentIds", segmentIds); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentMin(sd,data, segmentIds, numSegments).outputVariable(); @@ -4618,7 +4638,7 @@ public class SDBaseOps { * @return output Unsorted segment output (NUMERIC type) */ public SDVariable unsortedSegmentProd(String name, SDVariable data, SDVariable segmentIds, - int numSegments) { + int numSegments) { SDValidation.validateNumerical("unsortedSegmentProd", "data", data); SDValidation.validateNumerical("unsortedSegmentProd", "segmentIds", segmentIds); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentProd(sd,data, segmentIds, numSegments).outputVariable(); @@ -4655,7 +4675,7 @@ public class SDBaseOps { * @return output Unsorted segment output (NUMERIC type) */ public SDVariable unsortedSegmentSqrtN(String name, SDVariable data, SDVariable segmentIds, - int numSegments) { + int numSegments) { SDValidation.validateNumerical("unsortedSegmentSqrtN", "data", data); SDValidation.validateNumerical("unsortedSegmentSqrtN", "segmentIds", segmentIds); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentSqrtN(sd,data, segmentIds, numSegments).outputVariable(); @@ -4694,7 +4714,7 @@ public class SDBaseOps { * @return output Unsorted segment output (NUMERIC type) */ public SDVariable unsortedSegmentSum(String name, SDVariable data, SDVariable segmentIds, - int numSegments) { + int numSegments) { SDValidation.validateNumerical("unsortedSegmentSum", "data", data); SDValidation.validateNumerical("unsortedSegmentSum", "segmentIds", segmentIds); SDVariable out = new org.nd4j.linalg.api.ops.impl.transforms.segment.UnsortedSegmentSum(sd,data, segmentIds, numSegments).outputVariable(); @@ -4750,7 +4770,7 @@ public class SDBaseOps { * @return output reduced array of rank (input rank - num dimensions) (NUMERIC type) */ public SDVariable variance(SDVariable x, boolean biasCorrected, boolean keepDims, - int... dimensions) { + int... dimensions) { SDValidation.validateNumerical("variance", "x", x); Preconditions.checkArgument(dimensions.length >= 0, "dimensions has incorrect size/length. Expected: dimensions.length >= 0, got %s", dimensions.length); return new org.nd4j.linalg.api.ops.impl.summarystats.Variance(sd,x, biasCorrected, keepDims, dimensions).outputVariable(); @@ -4774,7 +4794,7 @@ public class SDBaseOps { * @return output reduced array of rank (input rank - num dimensions) (NUMERIC type) */ public SDVariable variance(String name, SDVariable x, boolean biasCorrected, boolean keepDims, - int... dimensions) { + int... dimensions) { SDValidation.validateNumerical("variance", "x", x); Preconditions.checkArgument(dimensions.length >= 0, "dimensions has incorrect size/length. Expected: dimensions.length >= 0, got %s", dimensions.length); SDVariable out = new org.nd4j.linalg.api.ops.impl.summarystats.Variance(sd,x, biasCorrected, keepDims, dimensions).outputVariable(); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBitwise.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBitwise.java similarity index 93% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBitwise.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBitwise.java index 299330359..00102c498 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBitwise.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDBitwise.java @@ -1,20 +1,22 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; @@ -248,7 +250,7 @@ public class SDBitwise extends SDOps { /** * Bitwise left cyclical shift operation. Supports broadcasting.
- * Unlike #leftShift(INDArray, INDArray) the bits will "wrap around":
+ * Unlike {@link #leftShift(INDArray, INDArray)} the bits will "wrap around":
* {@code leftShiftCyclic(01110000, 2) -> 11000001}
* * @param x Input to be bit shifted (INT type) @@ -263,7 +265,7 @@ public class SDBitwise extends SDOps { /** * Bitwise left cyclical shift operation. Supports broadcasting.
- * Unlike #leftShift(INDArray, INDArray) the bits will "wrap around":
+ * Unlike {@link #leftShift(INDArray, INDArray)} the bits will "wrap around":
* {@code leftShiftCyclic(01110000, 2) -> 11000001}
* * @param name name May be null. Name for the output variable @@ -346,7 +348,7 @@ public class SDBitwise extends SDOps { /** * Bitwise right cyclical shift operation. Supports broadcasting.
- * Unlike rightShift(INDArray, INDArray) the bits will "wrap around":
+ * Unlike {@link #rightShift(INDArray, INDArray)} the bits will "wrap around":
* {@code rightShiftCyclic(00001110, 2) -> 10000011}
* * @param x Input to be bit shifted (INT type) @@ -361,7 +363,7 @@ public class SDBitwise extends SDOps { /** * Bitwise right cyclical shift operation. Supports broadcasting.
- * Unlike rightShift(INDArray, INDArray) the bits will "wrap around":
+ * Unlike {@link #rightShift(INDArray, INDArray)} the bits will "wrap around":
* {@code rightShiftCyclic(00001110, 2) -> 10000011}
* * @param name name May be null. Name for the output variable diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDCNN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDCNN.java similarity index 91% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDCNN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDCNN.java index 066f9b150..b89e4ec3c 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDCNN.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDCNN.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; -import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; - import java.lang.String; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; @@ -42,7 +42,8 @@ public class SDCNN extends SDOps { /** * 2D Convolution layer operation - average pooling 2d
* - * @param input the input to average pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object * @return output Result after applying average pooling on the input (NUMERIC type) */ @@ -55,7 +56,8 @@ public class SDCNN extends SDOps { * 2D Convolution layer operation - average pooling 2d
* * @param name name May be null. Name for the output variable - * @param input the input to average pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object * @return output Result after applying average pooling on the input (NUMERIC type) */ @@ -68,7 +70,9 @@ public class SDCNN extends SDOps { /** * 3D convolution layer operation - average pooling 3d
* - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param Pooling3DConfig Configuration Object * @return output after applying average pooling on the input (NUMERIC type) */ @@ -81,7 +85,9 @@ public class SDCNN extends SDOps { * 3D convolution layer operation - average pooling 3d
* * @param name name May be null. Name for the output variable - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param Pooling3DConfig Configuration Object * @return output after applying average pooling on the input (NUMERIC type) */ @@ -296,7 +302,9 @@ public class SDCNN extends SDOps { /** * Convolution 3D operation with optional bias
* - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param weights Weights for conv3d. Rank 5 with shape [kernelDepth, kernelHeight, kernelWidth, inputChannels, outputChannels]. (NUMERIC type) * @param bias Optional 1D bias array with shape [outputChannels]. May be null. (NUMERIC type) * @param Conv3DConfig Configuration Object @@ -314,7 +322,9 @@ public class SDCNN extends SDOps { * Convolution 3D operation with optional bias
* * @param name name May be null. Name for the output variable - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param weights Weights for conv3d. Rank 5 with shape [kernelDepth, kernelHeight, kernelWidth, inputChannels, outputChannels]. (NUMERIC type) * @param bias Optional 1D bias array with shape [outputChannels]. May be null. (NUMERIC type) * @param Conv3DConfig Configuration Object @@ -332,7 +342,9 @@ public class SDCNN extends SDOps { /** * Convolution 3D operation with optional bias
* - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param weights Weights for conv3d. Rank 5 with shape [kernelDepth, kernelHeight, kernelWidth, inputChannels, outputChannels]. (NUMERIC type) * @param Conv3DConfig Configuration Object * @return output Conv3d output variable (NUMERIC type) @@ -347,7 +359,9 @@ public class SDCNN extends SDOps { * Convolution 3D operation with optional bias
* * @param name name May be null. Name for the output variable - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param weights Weights for conv3d. Rank 5 with shape [kernelDepth, kernelHeight, kernelWidth, inputChannels, outputChannels]. (NUMERIC type) * @param Conv3DConfig Configuration Object * @return output Conv3d output variable (NUMERIC type) @@ -363,7 +377,8 @@ public class SDCNN extends SDOps { /** * 2D deconvolution operation with optional bias
* - * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param weights Weights for the 2d deconvolution operation. 4 dimensions with format [inputChannels, outputChannels, kernelHeight, kernelWidth] (NUMERIC type) * @param bias Optional 1D bias array with shape [outputChannels]. May be null. (NUMERIC type) * @param DeConv2DConfig Configuration Object @@ -381,7 +396,8 @@ public class SDCNN extends SDOps { * 2D deconvolution operation with optional bias
* * @param name name May be null. Name for the output variable - * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param weights Weights for the 2d deconvolution operation. 4 dimensions with format [inputChannels, outputChannels, kernelHeight, kernelWidth] (NUMERIC type) * @param bias Optional 1D bias array with shape [outputChannels]. May be null. (NUMERIC type) * @param DeConv2DConfig Configuration Object @@ -399,7 +415,8 @@ public class SDCNN extends SDOps { /** * 2D deconvolution operation with optional bias
* - * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param weights Weights for the 2d deconvolution operation. 4 dimensions with format [inputChannels, outputChannels, kernelHeight, kernelWidth] (NUMERIC type) * @param DeConv2DConfig Configuration Object * @return output result of deconv2d op (NUMERIC type) @@ -415,7 +432,8 @@ public class SDCNN extends SDOps { * 2D deconvolution operation with optional bias
* * @param name name May be null. Name for the output variable - * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param weights Weights for the 2d deconvolution operation. 4 dimensions with format [inputChannels, outputChannels, kernelHeight, kernelWidth] (NUMERIC type) * @param DeConv2DConfig Configuration Object * @return output result of deconv2d op (NUMERIC type) @@ -501,7 +519,8 @@ public class SDCNN extends SDOps { * Example: if input has shape [mb, 8, 2, 2] and block size is 2, then output size is [mb, 8/(2*2), 2*2, 2*2]
* = [mb, 2, 4, 4]
* - * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param blockSize Block size, in the height/width dimension * @param dataFormat Data format: "NCHW" or "NHWC" * @return output Output variable (NUMERIC type) @@ -518,7 +537,8 @@ public class SDCNN extends SDOps { * = [mb, 2, 4, 4]
* * @param name name May be null. Name for the output variable - * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param blockSize Block size, in the height/width dimension * @param dataFormat Data format: "NCHW" or "NHWC" * @return output Output variable (NUMERIC type) @@ -736,7 +756,8 @@ public class SDCNN extends SDOps { /** * 2D Convolution layer operation - Max pooling on the input and outputs both max values and indices
* - * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object */ public SDVariable[] maxPoolWithArgmax(SDVariable input, Pooling2DConfig Pooling2DConfig) { @@ -748,7 +769,8 @@ public class SDCNN extends SDOps { * 2D Convolution layer operation - Max pooling on the input and outputs both max values and indices
* * @param names names May be null. Arrays of names for the output variables. - * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object */ public SDVariable[] maxPoolWithArgmax(String[] names, SDVariable input, @@ -761,7 +783,8 @@ public class SDCNN extends SDOps { /** * 2D Convolution layer operation - max pooling 2d
* - * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object * @return output Result after applying max pooling on the input (NUMERIC type) */ @@ -774,7 +797,8 @@ public class SDCNN extends SDOps { * 2D Convolution layer operation - max pooling 2d
* * @param name name May be null. Name for the output variable - * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object * @return output Result after applying max pooling on the input (NUMERIC type) */ @@ -787,7 +811,9 @@ public class SDCNN extends SDOps { /** * 3D convolution layer operation - max pooling 3d operation.
* - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param Pooling3DConfig Configuration Object * @return output Result after applying max pooling on the input (NUMERIC type) */ @@ -800,7 +826,9 @@ public class SDCNN extends SDOps { * 3D convolution layer operation - max pooling 3d operation.
* * @param name name May be null. Name for the output variable - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param Pooling3DConfig Configuration Object * @return output Result after applying max pooling on the input (NUMERIC type) */ @@ -813,7 +841,8 @@ public class SDCNN extends SDOps { /** * Separable 2D convolution operation with optional bias
* - * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param depthWeights Separable conv2d depth weights. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, depthMultiplier] (NUMERIC type) * @param pointWeights Point weights, rank 4 with format [1, 1, inputChannels*depthMultiplier, outputChannels]. May be null (NUMERIC type) * @param bias Optional bias, rank 1 with shape [outputChannels]. May be null. (NUMERIC type) @@ -833,7 +862,8 @@ public class SDCNN extends SDOps { * Separable 2D convolution operation with optional bias
* * @param name name May be null. Name for the output variable - * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param depthWeights Separable conv2d depth weights. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, depthMultiplier] (NUMERIC type) * @param pointWeights Point weights, rank 4 with format [1, 1, inputChannels*depthMultiplier, outputChannels]. May be null (NUMERIC type) * @param bias Optional bias, rank 1 with shape [outputChannels]. May be null. (NUMERIC type) @@ -853,7 +883,8 @@ public class SDCNN extends SDOps { /** * Separable 2D convolution operation with optional bias
* - * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param depthWeights Separable conv2d depth weights. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, depthMultiplier] (NUMERIC type) * @param pointWeights Point weights, rank 4 with format [1, 1, inputChannels*depthMultiplier, outputChannels]. May be null (NUMERIC type) * @param Conv2DConfig Configuration Object @@ -871,7 +902,8 @@ public class SDCNN extends SDOps { * Separable 2D convolution operation with optional bias
* * @param name name May be null. Name for the output variable - * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param depthWeights Separable conv2d depth weights. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, depthMultiplier] (NUMERIC type) * @param pointWeights Point weights, rank 4 with format [1, 1, inputChannels*depthMultiplier, outputChannels]. May be null (NUMERIC type) * @param Conv2DConfig Configuration Object @@ -932,7 +964,8 @@ public class SDCNN extends SDOps { * Example: if input has shape [mb, 2, 4, 4] and block size is 2, then output size is [mb, 8/(2*2), 2*2, 2*2]
* = [mb, 2, 4, 4]
* - * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param blockSize Block size, in the height/width dimension * @param dataFormat Data format: "NCHW" or "NHWC" * @return output Output variable (NUMERIC type) @@ -949,7 +982,8 @@ public class SDCNN extends SDOps { * = [mb, 2, 4, 4]
* * @param name name May be null. Name for the output variable - * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param blockSize Block size, in the height/width dimension * @param dataFormat Data format: "NCHW" or "NHWC" * @return output Output variable (NUMERIC type) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDImage.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDImage.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDImage.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDImage.java index 907a8e48e..6317e0941 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDImage.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDImage.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; -import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; - import java.lang.String; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLinalg.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLinalg.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLinalg.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLinalg.java index ae97b2c4a..2ddba4fcb 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLinalg.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLinalg.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; -import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; - import java.lang.String; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLoss.java similarity index 89% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLoss.java index 533d22783..c6fef378e 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLoss.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDLoss.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; -import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; - import java.lang.String; import org.nd4j.autodiff.loss.LossReduce; import org.nd4j.autodiff.samediff.SDVariable; @@ -36,7 +36,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output loss variable (NUMERIC type) */ public SDVariable absoluteDifference(SDVariable label, SDVariable predictions, SDVariable weights, @@ -56,7 +56,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output loss variable (NUMERIC type) */ public SDVariable absoluteDifference(String name, SDVariable label, SDVariable predictions, @@ -116,7 +116,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is use (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param dimension Dimension to perform the cosine distance over * @return output Cosine distance loss (NUMERIC type) */ @@ -141,7 +141,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is use (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param dimension Dimension to perform the cosine distance over * @return output Cosine distance loss (NUMERIC type) */ @@ -202,49 +202,6 @@ public class SDLoss extends SDOps { return sd.updateVariableNameAndReference(out, name); } - /** - * CTC Loss: Connectionist Temporal Classification Loss. See:
- * https://dl.acm.org/citation.cfm?id=1143891
- * - * @param targetLabels Label array (NUMERIC type) - * @param logitInput Inputs (NUMERIC type) - * @param targetLabelLengths Length of the target label (NUMERIC type) - * @param logitInputLengths Length of the input (NUMERIC type) - * @return output Ctc loss (NUMERIC type) - */ - public SDVariable ctcLoss(SDVariable targetLabels, SDVariable logitInput, - SDVariable targetLabelLengths, SDVariable logitInputLengths) { - SDValidation.validateNumerical("ctcLoss", "targetLabels", targetLabels); - SDValidation.validateNumerical("ctcLoss", "logitInput", logitInput); - SDValidation.validateNumerical("ctcLoss", "targetLabelLengths", targetLabelLengths); - SDValidation.validateNumerical("ctcLoss", "logitInputLengths", logitInputLengths); - SDVariable out = new org.nd4j.linalg.api.ops.impl.loss.CtcLoss(sd,targetLabels, logitInput, targetLabelLengths, logitInputLengths).outputVariable(); - out.markAsLoss(); - return out; - } - - /** - * CTC Loss: Connectionist Temporal Classification Loss. See:
- * https://dl.acm.org/citation.cfm?id=1143891
- * - * @param name name May be null. Name for the output variable - * @param targetLabels Label array (NUMERIC type) - * @param logitInput Inputs (NUMERIC type) - * @param targetLabelLengths Length of the target label (NUMERIC type) - * @param logitInputLengths Length of the input (NUMERIC type) - * @return output Ctc loss (NUMERIC type) - */ - public SDVariable ctcLoss(String name, SDVariable targetLabels, SDVariable logitInput, - SDVariable targetLabelLengths, SDVariable logitInputLengths) { - SDValidation.validateNumerical("ctcLoss", "targetLabels", targetLabels); - SDValidation.validateNumerical("ctcLoss", "logitInput", logitInput); - SDValidation.validateNumerical("ctcLoss", "targetLabelLengths", targetLabelLengths); - SDValidation.validateNumerical("ctcLoss", "logitInputLengths", logitInputLengths); - SDVariable out = new org.nd4j.linalg.api.ops.impl.loss.CtcLoss(sd,targetLabels, logitInput, targetLabelLengths, logitInputLengths).outputVariable(); - out.markAsLoss(); - return sd.updateVariableNameAndReference(out, name); - } - /** * Hinge loss: a loss function used for training classifiers.
* Implements {@code L = max(0, 1 - t * predictions)} where t is the label values after internally converting to {-1,1}
@@ -253,7 +210,7 @@ public class SDLoss extends SDOps { * @param label Label array. Each value should be 0.0 or 1.0 (internally -1 to 1 is used) (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable (NUMERIC type) */ public SDVariable hingeLoss(SDVariable label, SDVariable predictions, SDVariable weights, @@ -275,7 +232,7 @@ public class SDLoss extends SDOps { * @param label Label array. Each value should be 0.0 or 1.0 (internally -1 to 1 is used) (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable (NUMERIC type) */ public SDVariable hingeLoss(String name, SDVariable label, SDVariable predictions, @@ -340,7 +297,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param delta Loss function delta value * @return output Huber loss (NUMERIC type) */ @@ -367,7 +324,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param delta Loss function delta value * @return output Huber loss (NUMERIC type) */ @@ -466,7 +423,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param epsilon epsilon * @return output Log loss (NUMERIC type) */ @@ -488,7 +445,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param epsilon epsilon * @return output Log loss (NUMERIC type) */ @@ -542,7 +499,7 @@ public class SDLoss extends SDOps { * @param label Label array. Each value should be 0.0 or 1.0 (NUMERIC type) * @param predictions Predictions array (has to be log(x) of actual predictions) (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param full Boolean flag. true for logPoissonFull, false for logPoisson * @return output Loss variable (NUMERIC type) */ @@ -564,7 +521,7 @@ public class SDLoss extends SDOps { * @param label Label array. Each value should be 0.0 or 1.0 (NUMERIC type) * @param predictions Predictions array (has to be log(x) of actual predictions) (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param full Boolean flag. true for logPoissonFull, false for logPoisson * @return output Loss variable (NUMERIC type) */ @@ -628,7 +585,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used. Must be either null, scalar, or have shape [batchSize] (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable, scalar output (NUMERIC type) */ public SDVariable meanPairwiseSquaredError(SDVariable label, SDVariable predictions, @@ -651,7 +608,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used. Must be either null, scalar, or have shape [batchSize] (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable, scalar output (NUMERIC type) */ public SDVariable meanPairwiseSquaredError(String name, SDVariable label, SDVariable predictions, @@ -709,13 +666,13 @@ public class SDLoss extends SDOps { /** * Mean squared error loss function. Implements {@code (label[i] - prediction[i])^2} - i.e., squared error on a per-element basis.
- * When averaged (using LossReduce#MEAN_BY_WEIGHT or LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT (the default))
+ * When averaged (using {@link LossReduce#MEAN_BY_WEIGHT} or {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} (the default))
* this is the mean squared error loss function.
* * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable (NUMERIC type) */ public SDVariable meanSquaredError(SDVariable label, SDVariable predictions, SDVariable weights, @@ -730,14 +687,14 @@ public class SDLoss extends SDOps { /** * Mean squared error loss function. Implements {@code (label[i] - prediction[i])^2} - i.e., squared error on a per-element basis.
- * When averaged (using LossReduce#MEAN_BY_WEIGHT or LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT (the default))
+ * When averaged (using {@link LossReduce#MEAN_BY_WEIGHT} or {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} (the default))
* this is the mean squared error loss function.
* * @param name name May be null. Name for the output variable * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable (NUMERIC type) */ public SDVariable meanSquaredError(String name, SDVariable label, SDVariable predictions, @@ -752,7 +709,7 @@ public class SDLoss extends SDOps { /** * Mean squared error loss function. Implements {@code (label[i] - prediction[i])^2} - i.e., squared error on a per-element basis.
- * When averaged (using LossReduce#MEAN_BY_WEIGHT or LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT (the default))
+ * When averaged (using {@link LossReduce#MEAN_BY_WEIGHT} or {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} (the default))
* this is the mean squared error loss function.
* * @param label Label array (NUMERIC type) @@ -771,7 +728,7 @@ public class SDLoss extends SDOps { /** * Mean squared error loss function. Implements {@code (label[i] - prediction[i])^2} - i.e., squared error on a per-element basis.
- * When averaged (using LossReduce#MEAN_BY_WEIGHT or LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT (the default))
+ * When averaged (using {@link LossReduce#MEAN_BY_WEIGHT} or {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} (the default))
* this is the mean squared error loss function.
* * @param name name May be null. Name for the output variable @@ -807,7 +764,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictionLogits Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param labelSmoothing Label smoothing value. Default value: 0 * @return output Loss variable (NUMERIC type) */ @@ -839,7 +796,7 @@ public class SDLoss extends SDOps { * @param label Label array (NUMERIC type) * @param predictionLogits Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param labelSmoothing Label smoothing value. Default value: 0 * @return output Loss variable (NUMERIC type) */ @@ -915,7 +872,7 @@ public class SDLoss extends SDOps { /** * Applies the softmax activation function to the input, then implement multi-class cross entropy:
* {@code -sum_classes label[i] * log(p[c])} where {@code p = softmax(logits)}
- * If LossReduce#NONE is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
+ * If {@link LossReduce#NONE} is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
* otherwise, the output is a scalar.
*


* When label smoothing is > 0, the following label smoothing is used:
@@ -927,7 +884,7 @@ public class SDLoss extends SDOps { * @param oneHotLabels Label array. Should be one-hot per example and same shape as predictions (for example, [mb, nOut]) (NUMERIC type) * @param logitPredictions Predictions array (pre-softmax) (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param labelSmoothing Label smoothing value. Default value: 0 * @return output Loss variable (NUMERIC type) */ @@ -944,7 +901,7 @@ public class SDLoss extends SDOps { /** * Applies the softmax activation function to the input, then implement multi-class cross entropy:
* {@code -sum_classes label[i] * log(p[c])} where {@code p = softmax(logits)}
- * If LossReduce#NONE is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
+ * If {@link LossReduce#NONE} is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
* otherwise, the output is a scalar.
*


* When label smoothing is > 0, the following label smoothing is used:
@@ -957,7 +914,7 @@ public class SDLoss extends SDOps { * @param oneHotLabels Label array. Should be one-hot per example and same shape as predictions (for example, [mb, nOut]) (NUMERIC type) * @param logitPredictions Predictions array (pre-softmax) (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param labelSmoothing Label smoothing value. Default value: 0 * @return output Loss variable (NUMERIC type) */ @@ -975,7 +932,7 @@ public class SDLoss extends SDOps { /** * Applies the softmax activation function to the input, then implement multi-class cross entropy:
* {@code -sum_classes label[i] * log(p[c])} where {@code p = softmax(logits)}
- * If LossReduce#NONE is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
+ * If {@link LossReduce#NONE} is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
* otherwise, the output is a scalar.
*


* When label smoothing is > 0, the following label smoothing is used:
@@ -1002,7 +959,7 @@ public class SDLoss extends SDOps { /** * Applies the softmax activation function to the input, then implement multi-class cross entropy:
* {@code -sum_classes label[i] * log(p[c])} where {@code p = softmax(logits)}
- * If LossReduce#NONE is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
+ * If {@link LossReduce#NONE} is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
* otherwise, the output is a scalar.
*


* When label smoothing is > 0, the following label smoothing is used:
diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDMath.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDMath.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDMath.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDMath.java index 15a26059f..5c3579396 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDMath.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDMath.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; -import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; - import java.lang.String; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDNN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDNN.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDNN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDNN.java index 016fc721d..846291e47 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDNN.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDNN.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; -import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; - import java.lang.String; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDOps.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDOps.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDOps.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDOps.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRNN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRNN.java similarity index 87% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRNN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRNN.java index a72420888..4322910b4 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRNN.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRNN.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; -import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; - import java.lang.String; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; @@ -144,22 +144,22 @@ public class SDRNN extends SDOps { /** * Long Short-Term Memory layer - Hochreiter 1997.
- * SUPPORTS following data formats:
- * for unidirectional:
- * TNS: shapes [timeLength, numExamples, inOutSize]
- * NST: shapes [numExamples, inOutSize, timeLength]
+ * SUPPORTS following data formats:\n
+ * for unidirectional: \n" +
+ * TNS: shapes [timeLength, numExamples, inOutSize]\n
+ * NST: shapes [numExamples, inOutSize, timeLength]\n
* NTS: shapes [numExamples, timeLength, inOutSize]
- * for bidirectional:
- * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)
- * SUPPORTS following direction modes:
+ * for bidirectional:\n
+ * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)\n
+ * SUPPORTS following direction modes:\n
* FWD: forward
* BWD: backward
- * BIDIR_SUM: bidirectional sum
- * BIDIR_CONCAT: bidirectional concat
- * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)
+ * BIDIR_SUM: bidirectional sum\n
+ * BIDIR_CONCAT: bidirectional concat\n" +
+ * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)"
* You may use different gate configurations:
- * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum
- * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")
+ * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum\n
+ * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")\n
* Also this layer supports MKLDNN (DNNL) and cuDNN acceleration
* * @param x Input, with shape dependent on the data format (in config). (NUMERIC type) @@ -180,22 +180,22 @@ public class SDRNN extends SDOps { /** * Long Short-Term Memory layer - Hochreiter 1997.
- * SUPPORTS following data formats:
- * for unidirectional:
- * TNS: shapes [timeLength, numExamples, inOutSize]
- * NST: shapes [numExamples, inOutSize, timeLength]
+ * SUPPORTS following data formats:\n
+ * for unidirectional: \n" +
+ * TNS: shapes [timeLength, numExamples, inOutSize]\n
+ * NST: shapes [numExamples, inOutSize, timeLength]\n
* NTS: shapes [numExamples, timeLength, inOutSize]
- * for bidirectional:
- * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)
- * SUPPORTS following direction modes:
+ * for bidirectional:\n
+ * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)\n
+ * SUPPORTS following direction modes:\n
* FWD: forward
* BWD: backward
- * BIDIR_SUM: bidirectional sum
- * BIDIR_CONCAT: bidirectional concat
- * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)
+ * BIDIR_SUM: bidirectional sum\n
+ * BIDIR_CONCAT: bidirectional concat\n" +
+ * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)"
* You may use different gate configurations:
- * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum
- * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")
+ * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum\n
+ * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")\n
* Also this layer supports MKLDNN (DNNL) and cuDNN acceleration
* * @param names names May be null. Arrays of names for the output variables. @@ -218,22 +218,22 @@ public class SDRNN extends SDOps { /** * Long Short-Term Memory layer - Hochreiter 1997.
- * SUPPORTS following data formats:
- * for unidirectional:
- * TNS: shapes [timeLength, numExamples, inOutSize]
- * NST: shapes [numExamples, inOutSize, timeLength]
+ * SUPPORTS following data formats:\n
+ * for unidirectional: \n" +
+ * TNS: shapes [timeLength, numExamples, inOutSize]\n
+ * NST: shapes [numExamples, inOutSize, timeLength]\n
* NTS: shapes [numExamples, timeLength, inOutSize]
- * for bidirectional:
- * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)
- * SUPPORTS following direction modes:
+ * for bidirectional:\n
+ * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)\n
+ * SUPPORTS following direction modes:\n
* FWD: forward
* BWD: backward
- * BIDIR_SUM: bidirectional sum
- * BIDIR_CONCAT: bidirectional concat
- * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)
+ * BIDIR_SUM: bidirectional sum\n
+ * BIDIR_CONCAT: bidirectional concat\n" +
+ * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)"
* You may use different gate configurations:
- * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum
- * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")
+ * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum\n
+ * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")\n
* Also this layer supports MKLDNN (DNNL) and cuDNN acceleration
* * @param x Input, with shape dependent on the data format (in config). (NUMERIC type) @@ -248,22 +248,22 @@ public class SDRNN extends SDOps { /** * Long Short-Term Memory layer - Hochreiter 1997.
- * SUPPORTS following data formats:
- * for unidirectional:
- * TNS: shapes [timeLength, numExamples, inOutSize]
- * NST: shapes [numExamples, inOutSize, timeLength]
+ * SUPPORTS following data formats:\n
+ * for unidirectional: \n" +
+ * TNS: shapes [timeLength, numExamples, inOutSize]\n
+ * NST: shapes [numExamples, inOutSize, timeLength]\n
* NTS: shapes [numExamples, timeLength, inOutSize]
- * for bidirectional:
- * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)
- * SUPPORTS following direction modes:
+ * for bidirectional:\n
+ * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)\n
+ * SUPPORTS following direction modes:\n
* FWD: forward
* BWD: backward
- * BIDIR_SUM: bidirectional sum
- * BIDIR_CONCAT: bidirectional concat
- * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)
+ * BIDIR_SUM: bidirectional sum\n
+ * BIDIR_CONCAT: bidirectional concat\n" +
+ * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)"
* You may use different gate configurations:
- * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum
- * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")
+ * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum\n
+ * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")\n
* Also this layer supports MKLDNN (DNNL) and cuDNN acceleration
* * @param names names May be null. Arrays of names for the output variables. diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRandom.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRandom.java similarity index 94% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRandom.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRandom.java index 3bd31f3b6..d57afe876 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRandom.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDRandom.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.autodiff.samediff.ops; -import static org.nd4j.autodiff.samediff.ops.SDValidation.isSameType; - import java.lang.String; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDValidation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDValidation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDValidation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/ops/SDValidation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/serde/FlatBuffersMapper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/serde/FlatBuffersMapper.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/serde/FlatBuffersMapper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/serde/FlatBuffersMapper.java index 479feb01c..00fad8994 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/serde/FlatBuffersMapper.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/serde/FlatBuffersMapper.java @@ -22,7 +22,7 @@ package org.nd4j.autodiff.samediff.serde; import org.nd4j.autodiff.samediff.internal.SameDiffOp; import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.shade.guava.primitives.Ints; +import com.google.common.primitives.Ints; import com.google.flatbuffers.FlatBufferBuilder; import java.nio.ByteOrder; import java.util.*; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/serde/LegacyOpMapper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/serde/LegacyOpMapper.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/serde/LegacyOpMapper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/serde/LegacyOpMapper.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/GraphTransformUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/GraphTransformUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/GraphTransformUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/GraphTransformUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/OpPredicate.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/OpPredicate.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/OpPredicate.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/OpPredicate.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraph.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraph.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraph.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraph.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraphPredicate.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraphPredicate.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraphPredicate.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraphPredicate.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraphProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraphProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraphProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/samediff/transform/SubGraphProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/util/SameDiffUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/util/SameDiffUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/util/SameDiffUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/util/SameDiffUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/ActivationGradientCheckListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/ActivationGradientCheckListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/ActivationGradientCheckListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/ActivationGradientCheckListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/GradCheckUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/GradCheckUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/GradCheckUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/GradCheckUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/OpTestCase.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/OpTestCase.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/OpTestCase.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/OpTestCase.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/OpValidation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/OpValidation.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/OpValidation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/OpValidation.java index 627b9bdfe..8746ef281 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/OpValidation.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/OpValidation.java @@ -25,8 +25,8 @@ import org.nd4j.linalg.api.ops.custom.*; import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMax; import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMin; import org.nd4j.linalg.api.ops.impl.reduce.HashCode; -import org.nd4j.shade.guava.collect.ImmutableSet; -import org.nd4j.shade.guava.reflect.ClassPath; +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.ClassPath; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.nd4j.autodiff.functions.DifferentialFunction; @@ -563,7 +563,7 @@ public class OpValidation { ImmutableSet info; try { //Dependency note: this ClassPath class was added in Guava 14 - info = org.nd4j.shade.guava.reflect.ClassPath.from(DifferentialFunctionClassHolder.class.getClassLoader()) + info = com.google.common.reflect.ClassPath.from(DifferentialFunctionClassHolder.class.getClassLoader()) .getTopLevelClassesRecursive("org.nd4j.linalg.api.ops"); } catch (IOException e) { //Should never happen diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/TestCase.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/TestCase.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/TestCase.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/TestCase.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/functions/EqualityFn.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/functions/EqualityFn.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/functions/EqualityFn.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/functions/EqualityFn.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/functions/RelErrorFn.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/functions/RelErrorFn.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/functions/RelErrorFn.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/functions/RelErrorFn.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/listeners/NonInplaceValidationListener.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/listeners/NonInplaceValidationListener.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/listeners/NonInplaceValidationListener.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/autodiff/validation/listeners/NonInplaceValidationListener.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/context/Nd4jContext.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/context/Nd4jContext.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/context/Nd4jContext.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/context/Nd4jContext.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/CellAct.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/CellAct.java new file mode 100644 index 000000000..8e6bca5b8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/CellAct.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +public enum CellAct { + TANH, + + RELU, + + SIGMOID, + + AFFINE, + + LEAKY_RELU, + + THRESHHOLD_RELU, + + SCALED_TAHN, + + HARD_SIGMOID, + + ELU, + + SOFTSIGN, + + SOFTPLUS +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/DataFormat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/DataFormat.java new file mode 100644 index 000000000..547b142aa --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/DataFormat.java @@ -0,0 +1,27 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +public enum DataFormat { + NCHW, + + NHWC +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/GateAct.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/GateAct.java new file mode 100644 index 000000000..1e38b989e --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/GateAct.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +public enum GateAct { + TANH, + + RELU, + + SIGMOID, + + AFFINE, + + LEAKY_RELU, + + THRESHHOLD_RELU, + + SCALED_TAHN, + + HARD_SIGMOID, + + ELU, + + SOFTSIGN, + + SOFTPLUS +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/ImageResizeMethod.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/ImageResizeMethod.java new file mode 100644 index 000000000..ad8c82573 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/ImageResizeMethod.java @@ -0,0 +1,32 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +public enum ImageResizeMethod { + ResizeBilinear, // as java require + ResizeNearest, + ResizeBicubic, + ResizeArea, + ResizeGaussian, + ResizeLanczos3, + ResizeLanczos5, + ResizeMitchellcubic; +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/LSTMDataFormat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/LSTMDataFormat.java new file mode 100644 index 000000000..e72d1c50f --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/LSTMDataFormat.java @@ -0,0 +1,31 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +public enum LSTMDataFormat { + TNS, + + NST, + + NTS, + + T2NS +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/LSTMDirectionMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/LSTMDirectionMode.java new file mode 100644 index 000000000..488bd567a --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/LSTMDirectionMode.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +public enum LSTMDirectionMode { + FWD, + + BWD, + + BIDIR_SUM, + + BIDIR_CONCAT, + + BIDIR_EXTRA_DIM +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/OutAct.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/OutAct.java new file mode 100644 index 000000000..162227778 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/OutAct.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +public enum OutAct { + TANH, + + RELU, + + SIGMOID, + + AFFINE, + + LEAKY_RELU, + + THRESHHOLD_RELU, + + SCALED_TAHN, + + HARD_SIGMOID, + + ELU, + + SOFTSIGN, + + SOFTPLUS +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/PadMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/PadMode.java new file mode 100644 index 000000000..dd17c9580 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/PadMode.java @@ -0,0 +1,31 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +/** + * Padding format */ +public enum PadMode { + CONSTANT, + + REFLECT, + + SYMMETRIC +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/PartitionMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/PartitionMode.java new file mode 100644 index 000000000..42f0df0b8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/PartitionMode.java @@ -0,0 +1,29 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +/** + * partition_mode == 0 - i.e. 'mod' , 1 - 'div' */ +public enum PartitionMode { + MOD, + + DIV +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/RnnDataFormat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/RnnDataFormat.java new file mode 100644 index 000000000..5d5b5af4c --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/RnnDataFormat.java @@ -0,0 +1,29 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.enums; + +public enum RnnDataFormat { + TNS, + + NST, + + NTS +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/enums/WeightsFormat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/WeightsFormat.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/enums/WeightsFormat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/enums/WeightsFormat.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/BaseEvaluation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/BaseEvaluation.java new file mode 100644 index 000000000..ec62b7bb3 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/BaseEvaluation.java @@ -0,0 +1,316 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.evaluation; + +import lombok.EqualsAndHashCode; +import lombok.NonNull; +import org.nd4j.common.base.Preconditions; +import org.nd4j.evaluation.classification.*; +import org.nd4j.evaluation.regression.RegressionEvaluation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastTo; +import org.nd4j.linalg.api.shape.Shape; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.primitives.Triple; +import org.nd4j.common.util.ArrayUtil; +import org.nd4j.serde.json.JsonMappers; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; + +@EqualsAndHashCode +public abstract class BaseEvaluation implements IEvaluation { + + /** + * @param yaml YAML representation + * @param clazz Class + * @param Type to return + * @return Evaluation instance + */ + public static T fromYaml(String yaml, Class clazz) { + try { + return JsonMappers.getYamlMapper().readValue(yaml, clazz); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * @param json Jason representation of the evaluation instance + * @param clazz Class + * @param Type to return + * @return Evaluation instance + */ + public static T fromJson(String json, Class clazz) { + try { + return JsonMappers.getMapper().readValue(json, clazz); + } catch (InvalidTypeIdException e) { + if (e.getMessage().contains("Could not resolve type id")) { + try { + return (T) attempFromLegacyFromJson(json, e); + } catch (Throwable t) { + throw new RuntimeException("Cannot deserialize from JSON - JSON is invalid?", t); + } + } + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Attempt to load DL4J IEvaluation JSON from 1.0.0-beta2 or earlier. + * Given IEvaluation classes were moved to ND4J with no major changes, a simple "find and replace" for the class + * names is used. + * + * @param json JSON to attempt to deserialize + * @param originalException Original exception to be re-thrown if it isn't legacy JSON + */ + protected static T attempFromLegacyFromJson(String json, InvalidTypeIdException originalException) throws InvalidTypeIdException { + if (json.contains("org.deeplearning4j.eval.Evaluation")) { + String newJson = json.replaceAll("org.deeplearning4j.eval.Evaluation", "org.nd4j.evaluation.classification.Evaluation"); + return (T) fromJson(newJson, Evaluation.class); + } + + if (json.contains("org.deeplearning4j.eval.EvaluationBinary")) { + String newJson = json.replaceAll("org.deeplearning4j.eval.EvaluationBinary", "org.nd4j.evaluation.classification.EvaluationBinary") + .replaceAll("org.deeplearning4j.eval.ROC", "org.nd4j.evaluation.classification.ROC") + .replaceAll("org.deeplearning4j.eval.curves.", "org.nd4j.evaluation.curves."); + return (T) fromJson(newJson, EvaluationBinary.class); + } + + if (json.contains("org.deeplearning4j.eval.EvaluationCalibration")) { + String newJson = json.replaceAll("org.deeplearning4j.eval.EvaluationCalibration", "org.nd4j.evaluation.classification.EvaluationCalibration") + .replaceAll("org.deeplearning4j.eval.curves.", "org.nd4j.evaluation.curves."); + return (T) fromJson(newJson, EvaluationCalibration.class); + } + + if (json.contains("org.deeplearning4j.eval.ROCBinary")) { + String newJson = json.replaceAll("org.deeplearning4j.eval.ROCBinary", "org.nd4j.evaluation.classification.ROCBinary") + .replaceAll("org.deeplearning4j.eval.ROC", "org.nd4j.evaluation.classification.ROC") //Nested ROC instances internally + .replaceAll("org.deeplearning4j.eval.curves.", "org.nd4j.evaluation.curves."); + + return (T) fromJson(newJson, ROCBinary.class); + } + + if (json.contains("org.deeplearning4j.eval.ROCMultiClass")) { + String newJson = json.replaceAll("org.deeplearning4j.eval.ROCMultiClass", "org.nd4j.evaluation.classification.ROCMultiClass") + .replaceAll("org.deeplearning4j.eval.ROC", "org.nd4j.evaluation.classification.ROC") //Nested ROC instances internally + .replaceAll("org.deeplearning4j.eval.curves.", "org.nd4j.evaluation.curves."); + return (T) fromJson(newJson, ROCMultiClass.class); + } + + if (json.contains("org.deeplearning4j.eval.ROC")) { //Has to be checked after ROCBinary/ROCMultiClass due to it being a prefix + String newJson = json.replaceAll("org.deeplearning4j.eval.ROC", "org.nd4j.evaluation.classification.ROC") + .replaceAll("org.deeplearning4j.eval.curves.", "org.nd4j.evaluation.curves."); + return (T) fromJson(newJson, ROC.class); + } + + if (json.contains("org.deeplearning4j.eval.RegressionEvaluation")) { + String newJson = json.replaceAll("org.deeplearning4j.eval.RegressionEvaluation", "org.nd4j.evaluation.regression.RegressionEvaluation"); + return (T) fromJson(newJson, RegressionEvaluation.class); + } + + throw originalException; + } + + public static Triple reshapeAndExtractNotMasked(INDArray labels, INDArray predictions, INDArray mask, int axis) { + + if (labels.rank() == 2) { + Preconditions.checkState(axis == 1, "Only axis=1 is supported 2d data - got axis=%s for labels array shape %ndShape", axis, labels); + if (mask == null) { + //no-op + return new Triple<>(labels, predictions, null); + } else { + //2 possible cases: per-output masking, and per example masking + if (mask.rank() == 1 || mask.isColumnVector()) { + int notMaskedCount = mask.neq(0.0).castTo(DataType.INT).sumNumber().intValue(); + if (notMaskedCount == 0) { + //All steps masked - nothing left to evaluate + return null; + } + if (notMaskedCount == mask.length()) { + //No masked steps - returned as-is + return new Triple<>(labels, predictions, null); + } + int[] arr = mask.toIntVector(); + int[] idxs = new int[notMaskedCount]; + int pos = 0; + for (int i = 0; i < arr.length; i++) { + if (arr[i] != 0) { + idxs[pos++] = i; + } + } + INDArray retLabel = Nd4j.pullRows(labels, 1, idxs, 'c'); + INDArray retPredictions = Nd4j.pullRows(predictions, 1, idxs, 'c'); + return new Triple<>(retLabel, retPredictions, null); + } else { + Preconditions.checkState(labels.equalShapes(mask), "If a mask array is present for 2d data, it must either be a vector (column vector)" + + " or have shape equal to the labels (for per-output masking, when supported). Got labels shape %ndShape, mask shape %ndShape", + labels, mask); + //Assume evaluation instances with per-output masking will handle that themselves (or throw exception if not supported) + return new Triple<>(labels, predictions, mask); + } + } + } else if (labels.rank() == 3 || labels.rank() == 4 || labels.rank() == 5) { + if(mask == null){ + return reshapeSameShapeTo2d(axis, labels, predictions, mask); + } else { + if(labels.rank() == 3) { + if (mask.rank() == 2) { + //Per time step masking + Pair p = EvaluationUtils.extractNonMaskedTimeSteps(labels, predictions, mask); + if (p == null) { + return null; + } + return new Triple<>(p.getFirst(), p.getSecond(), null); + } else { + //Per output mask + Preconditions.checkState(labels.equalShapes(mask), "If a mask array is present for 3d data, it must either be 2d (shape [minibatch, sequenceLength])" + + " or have shape equal to the labels (for per-output masking, when supported). Got labels shape %ndShape, mask shape %ndShape", + labels, mask); + //Assume evaluation instances with per-output masking will handle that themselves (or throw exception if not supported) + //Just reshape to 2d + + return reshapeSameShapeTo2d(axis, labels, predictions, mask); + } + } else { + if(labels.equalShapes(mask)){ + //Per output masking case + return reshapeSameShapeTo2d(axis, labels, predictions, mask); + } else if(mask.rank() == 1){ + //Treat 1D mask as per-example masking + Preconditions.checkState(mask.length() == labels.size(0), "For rank 4 labels with shape %ndShape and 1d" + + " mask of shape %ndShape, the mask array length must equal labels dimension 0 size", labels, mask); + long[] reshape = ArrayUtil.nTimes(labels.rank(), 1L); + reshape[0] = mask.size(0); + INDArray mReshape = mask.reshape(reshape); + INDArray bMask = Nd4j.createUninitialized(mask.dataType(), labels.shape()); + BroadcastTo b = new BroadcastTo(mReshape, labels.shape(), bMask); + Nd4j.exec(b); + return reshapeSameShapeTo2d(axis, labels, predictions, bMask); + } else if(mask.rank() == labels.rank() && Shape.areShapesBroadcastable(mask.shape(), labels.shape())){ + //Same rank, but different shape -> broadcast + INDArray bMask = Nd4j.createUninitialized(mask.dataType(), labels.shape()); + BroadcastTo b = new BroadcastTo(mask, labels.shape(), bMask); + Nd4j.exec(b); + return reshapeSameShapeTo2d(axis, labels, predictions, bMask); + } + throw new UnsupportedOperationException("Evaluation case not supported: labels shape " + Arrays.toString(labels.shape()) + + " with mask shape " + Arrays.toString(mask.shape())); + } + } + } else { + throw new IllegalStateException("Unknown array type passed to evaluation: labels array rank " + labels.rank() + + " with shape " + labels.shapeInfoToString() + ". Labels and predictions must always be rank 2 or higher, with leading dimension being minibatch dimension"); + } + } + + private static Triple reshapeSameShapeTo2d(int axis, INDArray labels, INDArray predictions, INDArray mask){ + int[] permuteDims = new int[labels.rank()]; + int j=0; + for( int i=0; i(lOut, pOut, mOut); + } + + @Override + public void eval(INDArray labels, INDArray networkPredictions) { + eval(labels, networkPredictions, null, null); + } + + @Override + public void eval(@NonNull INDArray labels, @NonNull final INDArray predictions, final List recordMetaData) { + eval(labels, predictions, null, recordMetaData); + } + + @Override + public void eval(INDArray labels, INDArray networkPredictions, INDArray maskArray) { + eval(labels, networkPredictions, maskArray, null); + } + + @Override + public void evalTimeSeries(INDArray labels, INDArray predicted) { + evalTimeSeries(labels, predicted, null); + } + + @Override + public void evalTimeSeries(INDArray labels, INDArray predictions, INDArray labelsMask) { + Pair pair = EvaluationUtils.extractNonMaskedTimeSteps(labels, predictions, labelsMask); + if (pair == null) { + //No non-masked steps + return; + } + INDArray labels2d = pair.getFirst(); + INDArray predicted2d = pair.getSecond(); + + eval(labels2d, predicted2d); + } + + /** + * @return JSON representation of the evaluation instance + */ + @Override + public String toJson() { + try { + return JsonMappers.getMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() { + return stats(); + } + + /** + * @return YAML representation of the evaluation instance + */ + @Override + public String toYaml() { + try { + return JsonMappers.getYamlMapper().writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/EvaluationAveraging.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/EvaluationAveraging.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/EvaluationAveraging.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/EvaluationAveraging.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/EvaluationUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/EvaluationUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/EvaluationUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/EvaluationUtils.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/IEvaluation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/IEvaluation.java new file mode 100644 index 000000000..4731250ca --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/IEvaluation.java @@ -0,0 +1,109 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.evaluation; + +import org.nd4j.linalg.api.ndarray.INDArray; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; +import java.util.List; + +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY) +public interface IEvaluation extends Serializable { + + + /** + * + * @param labels + * @param networkPredictions + */ + void eval(INDArray labels, INDArray networkPredictions); + + /** + * + * @param labels + * @param networkPredictions + * @param recordMetaData + */ + void eval(INDArray labels, INDArray networkPredictions, List recordMetaData); + + void eval(INDArray labels, INDArray networkPredictions, INDArray maskArray, List recordMetaData); + + /** + * + * @param labels + * @param networkPredictions + * @param maskArray + */ + void eval(INDArray labels, INDArray networkPredictions, INDArray maskArray); + + + /** + * @deprecated Use {@link #eval(INDArray, INDArray)} + */ + @Deprecated + void evalTimeSeries(INDArray labels, INDArray predicted); + + /** + * @deprecated Use {@link #eval(INDArray, INDArray, INDArray)} + */ + @Deprecated + void evalTimeSeries(INDArray labels, INDArray predicted, INDArray labelsMaskArray); + + /** + * + * @param other + */ + void merge(T other); + + /** + * + */ + void reset(); + + /** + * + * @return + */ + String stats(); + + /** + * + * @return + */ + String toJson(); + + /** + * + * @return + */ + String toYaml(); + + /** + * Get the value of a given metric for this evaluation. + */ + double getValue(IMetric metric); + + /** + * Get a new instance of this evaluation, with the same configuration but no data. + */ + T newInstance(); +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/IMetric.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/IMetric.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/IMetric.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/IMetric.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ConfusionMatrix.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ConfusionMatrix.java new file mode 100644 index 000000000..8e865da20 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ConfusionMatrix.java @@ -0,0 +1,264 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.evaluation.classification; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; +import lombok.Getter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class ConfusionMatrix> implements Serializable { + @Getter + private volatile Map> matrix; + private List classes; + + /** + * Creates an empty confusion Matrix + */ + public ConfusionMatrix(List classes) { + this.matrix = new ConcurrentHashMap<>(); + this.classes = classes; + } + + public ConfusionMatrix() { + this(new ArrayList()); + } + + /** + * Creates a new ConfusionMatrix initialized with the contents of another ConfusionMatrix. + */ + public ConfusionMatrix(ConfusionMatrix other) { + this(other.getClasses()); + this.add(other); + } + + /** + * Increments the entry specified by actual and predicted by one. + */ + public synchronized void add(T actual, T predicted) { + add(actual, predicted, 1); + } + + /** + * Increments the entry specified by actual and predicted by count. + */ + public synchronized void add(T actual, T predicted, int count) { + if (matrix.containsKey(actual)) { + matrix.get(actual).add(predicted, count); + } else { + Multiset counts = HashMultiset.create(); + counts.add(predicted, count); + matrix.put(actual, counts); + } + } + + /** + * Adds the entries from another confusion matrix to this one. + */ + public synchronized void add(ConfusionMatrix other) { + for (T actual : other.matrix.keySet()) { + Multiset counts = other.matrix.get(actual); + for (T predicted : counts.elementSet()) { + int count = counts.count(predicted); + this.add(actual, predicted, count); + } + } + } + + /** + * Gives the applyTransformToDestination of all classes in the confusion matrix. + */ + public List getClasses() { + if (classes == null) + classes = new ArrayList<>(); + return classes; + } + + /** + * Gives the count of the number of times the "predicted" class was predicted for the "actual" + * class. + */ + public synchronized int getCount(T actual, T predicted) { + if (!matrix.containsKey(actual)) { + return 0; + } else { + return matrix.get(actual).count(predicted); + } + } + + /** + * Computes the total number of times the class was predicted by the classifier. + */ + public synchronized int getPredictedTotal(T predicted) { + int total = 0; + for (T actual : classes) { + total += getCount(actual, predicted); + } + return total; + } + + /** + * Computes the total number of times the class actually appeared in the data. + */ + public synchronized int getActualTotal(T actual) { + if (!matrix.containsKey(actual)) { + return 0; + } else { + int total = 0; + for (T elem : matrix.get(actual).elementSet()) { + total += matrix.get(actual).count(elem); + } + return total; + } + } + + @Override + public String toString() { + return matrix.toString(); + } + + /** + * Outputs the ConfusionMatrix as comma-separated values for easy import into spreadsheets + */ + public String toCSV() { + StringBuilder builder = new StringBuilder(); + + // Header Row + builder.append(",,Predicted Class,\n"); + + // Predicted Classes Header Row + builder.append(",,"); + for (T predicted : classes) { + builder.append(String.format("%s,", predicted)); + } + builder.append("Total\n"); + + // Data Rows + String firstColumnLabel = "Actual Class,"; + for (T actual : classes) { + builder.append(firstColumnLabel); + firstColumnLabel = ","; + builder.append(String.format("%s,", actual)); + + for (T predicted : classes) { + builder.append(getCount(actual, predicted)); + builder.append(","); + } + // Actual Class Totals Column + builder.append(getActualTotal(actual)); + builder.append("\n"); + } + + // Predicted Class Totals Row + builder.append(",Total,"); + for (T predicted : classes) { + builder.append(getPredictedTotal(predicted)); + builder.append(","); + } + builder.append("\n"); + + return builder.toString(); + } + + /** + * Outputs Confusion Matrix in an HTML table. Cascading Style Sheets (CSS) can control the table's + * appearance by defining the empty-space, actual-count-header, predicted-class-header, and + * count-element classes. For example + * + * @return html string + */ + public String toHTML() { + StringBuilder builder = new StringBuilder(); + + int numClasses = classes.size(); + // Header Row + builder.append("\n"); + builder.append("%n", + numClasses + 1)); + + // Predicted Classes Header Row + builder.append(""); + // builder.append(""); + for (T predicted : classes) { + builder.append(""); + } + builder.append(""); + builder.append("\n"); + + // Data Rows + String firstColumnLabel = String.format( + "", numClasses + 1); + for (T actual : classes) { + builder.append(firstColumnLabel); + firstColumnLabel = ""; + builder.append(String.format("", actual)); + + for (T predicted : classes) { + builder.append(""); + } + + // Actual Class Totals Column + builder.append(""); + builder.append("\n"); + } + + // Predicted Class Totals Row + builder.append(""); + for (T predicted : classes) { + builder.append(""); + } + builder.append("\n"); + builder.append("\n"); + builder.append("
"); + builder.append(String.format("Predicted Class
"); + builder.append(predicted); + builder.append("Total
Actual Class
%s"); + builder.append(getCount(actual, predicted)); + builder.append(""); + builder.append(getActualTotal(actual)); + builder.append("
Total"); + builder.append(getPredictedTotal(predicted)); + builder.append("
\n"); + + return builder.toString(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ConfusionMatrix)) + return false; + ConfusionMatrix c = (ConfusionMatrix) o; + return matrix.equals(c.matrix) && classes.equals(c.classes); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + (matrix == null ? 0 : matrix.hashCode()); + result = 31 * result + (classes == null ? 0 : classes.hashCode()); + return result; + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/Evaluation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/Evaluation.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/Evaluation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/Evaluation.java index 6634724d0..df2151210 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/Evaluation.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/Evaluation.java @@ -41,9 +41,9 @@ import org.nd4j.common.primitives.Pair; import org.nd4j.common.primitives.Triple; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.Serializable; import java.text.DecimalFormat; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/EvaluationBinary.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationBinary.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/EvaluationBinary.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationBinary.java index 0e6d97186..715e23d37 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/EvaluationBinary.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationBinary.java @@ -37,8 +37,8 @@ import org.nd4j.linalg.indexing.conditions.Conditions; import org.nd4j.common.primitives.Triple; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.Serializable; import java.util.ArrayList; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationCalibration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationCalibration.java new file mode 100644 index 000000000..e154fad61 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/EvaluationCalibration.java @@ -0,0 +1,482 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.evaluation.classification; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.val; +import org.nd4j.common.base.Preconditions; +import org.nd4j.evaluation.BaseEvaluation; +import org.nd4j.evaluation.IMetric; +import org.nd4j.evaluation.curves.Histogram; +import org.nd4j.evaluation.curves.ReliabilityDiagram; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition; +import org.nd4j.linalg.api.ops.impl.transforms.any.IsMax; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.conditions.Conditions; +import org.nd4j.linalg.lossfunctions.LossUtil; +import org.nd4j.linalg.ops.transforms.Transforms; +import org.nd4j.common.primitives.Triple; +import org.nd4j.serde.jackson.shaded.NDArrayDeSerializer; +import org.nd4j.serde.jackson.shaded.NDArraySerializer; +import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; +import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.io.Serializable; +import java.util.List; + +@Getter +@EqualsAndHashCode +public class EvaluationCalibration extends BaseEvaluation { + + public static final int DEFAULT_RELIABILITY_DIAG_NUM_BINS = 10; + public static final int DEFAULT_HISTOGRAM_NUM_BINS = 50; + + private final int reliabilityDiagNumBins; + private final int histogramNumBins; + private final boolean excludeEmptyBins; + + @EqualsAndHashCode.Exclude //Exclude axis: otherwise 2 Evaluation instances could contain identical stats and fail equality + protected int axis = 1; + + @JsonSerialize(using = NDArraySerializer.class) + @JsonDeserialize(using = NDArrayDeSerializer.class) + private INDArray rDiagBinPosCount; + @JsonSerialize(using = NDArraySerializer.class) + @JsonDeserialize(using = NDArrayDeSerializer.class) + private INDArray rDiagBinTotalCount; + @JsonSerialize(using = NDArraySerializer.class) + @JsonDeserialize(using = NDArrayDeSerializer.class) + private INDArray rDiagBinSumPredictions; + + @JsonSerialize(using = NDArrayTextSerializer.class) + @JsonDeserialize(using = NDArrayTextDeSerializer.class) + private INDArray labelCountsEachClass; + @JsonSerialize(using = NDArrayTextSerializer.class) + @JsonDeserialize(using = NDArrayTextDeSerializer.class) + private INDArray predictionCountsEachClass; + + @JsonSerialize(using = NDArrayTextSerializer.class) + @JsonDeserialize(using = NDArrayTextDeSerializer.class) + private INDArray residualPlotOverall; + @JsonSerialize(using = NDArraySerializer.class) + @JsonDeserialize(using = NDArrayDeSerializer.class) + private INDArray residualPlotByLabelClass; + + @JsonSerialize(using = NDArrayTextSerializer.class) + @JsonDeserialize(using = NDArrayTextDeSerializer.class) + private INDArray probHistogramOverall; //Simple histogram over all probabilities + @JsonSerialize(using = NDArraySerializer.class) + @JsonDeserialize(using = NDArrayDeSerializer.class) + private INDArray probHistogramByLabelClass; //Histogram - for each label class separately + + protected EvaluationCalibration(int axis, int reliabilityDiagNumBins, int histogramNumBins, boolean excludeEmptyBins) { + this.axis = axis; + this.reliabilityDiagNumBins = reliabilityDiagNumBins; + this.histogramNumBins = histogramNumBins; + this.excludeEmptyBins = excludeEmptyBins; + } + + /** + * Create an EvaluationCalibration instance with the default number of bins + */ + public EvaluationCalibration() { + this(DEFAULT_RELIABILITY_DIAG_NUM_BINS, DEFAULT_HISTOGRAM_NUM_BINS, true); + } + + /** + * Create an EvaluationCalibration instance with the specified number of bins + * + * @param reliabilityDiagNumBins Number of bins for the reliability diagram (usually 10) + * @param histogramNumBins Number of bins for the histograms + */ + public EvaluationCalibration(int reliabilityDiagNumBins, int histogramNumBins) { + this(reliabilityDiagNumBins, histogramNumBins, true); + } + + /** + * Create an EvaluationCalibration instance with the specified number of bins + * + * @param reliabilityDiagNumBins Number of bins for the reliability diagram (usually 10) + * @param histogramNumBins Number of bins for the histograms + * @param excludeEmptyBins For the reliability diagram, whether empty bins should be excluded + */ + public EvaluationCalibration(@JsonProperty("reliabilityDiagNumBins") int reliabilityDiagNumBins, + @JsonProperty("histogramNumBins") int histogramNumBins, + @JsonProperty("excludeEmptyBins") boolean excludeEmptyBins) { + this.reliabilityDiagNumBins = reliabilityDiagNumBins; + this.histogramNumBins = histogramNumBins; + this.excludeEmptyBins = excludeEmptyBins; + } + + /** + * Set the axis for evaluation - this is the dimension along which the probability (and label classes) are present.
+ * For DL4J, this can be left as the default setting (axis = 1).
+ * Axis should be set as follows:
+ * For 2D (OutputLayer), shape [minibatch, numClasses] - axis = 1
+ * For 3D, RNNs/CNN1D (DL4J RnnOutputLayer), NCW format, shape [minibatch, numClasses, sequenceLength] - axis = 1
+ * For 3D, RNNs/CNN1D (DL4J RnnOutputLayer), NWC format, shape [minibatch, sequenceLength, numClasses] - axis = 2
+ * For 4D, CNN2D (DL4J CnnLossLayer), NCHW format, shape [minibatch, channels, height, width] - axis = 1
+ * For 4D, CNN2D, NHWC format, shape [minibatch, height, width, channels] - axis = 3
+ * + * @param axis Axis to use for evaluation + */ + public void setAxis(int axis){ + this.axis = axis; + } + + /** + * Get the axis - see {@link #setAxis(int)} for details + */ + public int getAxis(){ + return axis; + } + + @Override + public void eval(INDArray labels, INDArray predictions, INDArray mask) { + + Triple triple = BaseEvaluation.reshapeAndExtractNotMasked(labels, predictions, mask, axis); + if(triple == null){ + //All values masked out; no-op + return; + } + + INDArray labels2d = triple.getFirst(); + INDArray predictions2d = triple.getSecond(); + INDArray maskArray = triple.getThird(); + Preconditions.checkState(maskArray == null, "Per-output masking for EvaluationCalibration is not supported"); + + //Stats for the reliability diagram: one reliability diagram for each class + // For each bin, we need: (a) the number of positive cases AND total cases, (b) the average probability + + val nClasses = labels2d.size(1); + + if (rDiagBinPosCount == null) { + DataType dt = DataType.DOUBLE; + //Initialize + rDiagBinPosCount = Nd4j.create(DataType.LONG, reliabilityDiagNumBins, nClasses); + rDiagBinTotalCount = Nd4j.create(DataType.LONG, reliabilityDiagNumBins, nClasses); + rDiagBinSumPredictions = Nd4j.create(dt, reliabilityDiagNumBins, nClasses); + + labelCountsEachClass = Nd4j.create(DataType.LONG, 1, nClasses); + predictionCountsEachClass = Nd4j.create(DataType.LONG, 1, nClasses); + + residualPlotOverall = Nd4j.create(dt, 1, histogramNumBins); + residualPlotByLabelClass = Nd4j.create(dt, histogramNumBins, nClasses); + + probHistogramOverall = Nd4j.create(dt, 1, histogramNumBins); + probHistogramByLabelClass = Nd4j.create(dt, histogramNumBins, nClasses); + } + + + //First: loop over classes, determine positive count and total count - for each bin + double histogramBinSize = 1.0 / histogramNumBins; + double reliabilityBinSize = 1.0 / reliabilityDiagNumBins; + + INDArray p = predictions2d; + INDArray l = labels2d; + + if (maskArray != null) { + //2 options: per-output masking, or + if (maskArray.isColumnVectorOrScalar()) { + //Per-example masking + l = l.mulColumnVector(maskArray); + } else { + l = l.mul(maskArray); + } + } + + for (int j = 0; j < reliabilityDiagNumBins; j++) { + INDArray geqBinLower = p.gte(j * reliabilityBinSize).castTo(predictions2d.dataType()); + INDArray ltBinUpper; + if (j == reliabilityDiagNumBins - 1) { + //Handle edge case + ltBinUpper = p.lte(1.0).castTo(predictions2d.dataType()); + } else { + ltBinUpper = p.lt((j + 1) * reliabilityBinSize).castTo(predictions2d.dataType()); + } + + //Calculate bit-mask over each entry - whether that entry is in the current bin or not + INDArray currBinBitMask = geqBinLower.muli(ltBinUpper); + if (maskArray != null) { + if (maskArray.isColumnVectorOrScalar()) { + currBinBitMask.muliColumnVector(maskArray); + } else { + currBinBitMask.muli(maskArray); + } + } + + INDArray isPosLabelForBin = l.mul(currBinBitMask); + INDArray maskedProbs = predictions2d.mul(currBinBitMask); + + INDArray numPredictionsCurrBin = currBinBitMask.sum(0); + + rDiagBinSumPredictions.getRow(j).addi(maskedProbs.sum(0).castTo(rDiagBinSumPredictions.dataType())); + rDiagBinPosCount.getRow(j).addi(isPosLabelForBin.sum(0).castTo(rDiagBinPosCount.dataType())); + rDiagBinTotalCount.getRow(j).addi(numPredictionsCurrBin.castTo(rDiagBinTotalCount.dataType())); + } + + + //Second, we want histograms of: + //(a) Distribution of label classes: label counts for each class + //(b) Distribution of prediction classes: prediction counts for each class + //(c) residual plots, for each class - (i) all instances, (ii) positive instances only, (iii) negative only + //(d) Histograms of probabilities, for each class + + labelCountsEachClass.addi(labels2d.sum(0).castTo(labelCountsEachClass.dataType())); + //For prediction counts: do an IsMax op, but we need to take masking into account... + INDArray isPredictedClass = Nd4j.getExecutioner().exec(new IsMax(p, p.ulike(), 1))[0]; + if (maskArray != null) { + LossUtil.applyMask(isPredictedClass, maskArray); + } + predictionCountsEachClass.addi(isPredictedClass.sum(0).castTo(predictionCountsEachClass.dataType())); + + + + //Residual plots: want histogram of |labels - predicted prob| + + //ND4J's histogram op: dynamically calculates the bin positions, which is not what I want here... + INDArray labelsSubPredicted = labels2d.sub(predictions2d); + INDArray maskedProbs = predictions2d.dup(); + Transforms.abs(labelsSubPredicted, false); + + //if masking: replace entries with < 0 to effectively remove them + if (maskArray != null) { + //Assume per-example masking + INDArray newMask = maskArray.mul(-10); + labelsSubPredicted.addiColumnVector(newMask); + maskedProbs.addiColumnVector(newMask); + } + + for (int j = 0; j < histogramNumBins; j++) { + INDArray geqBinLower = labelsSubPredicted.gte(j * histogramBinSize).castTo(predictions2d.dataType()); + INDArray ltBinUpper; + INDArray geqBinLowerProbs = maskedProbs.gte(j * histogramBinSize).castTo(predictions2d.dataType()); + INDArray ltBinUpperProbs; + if (j == histogramNumBins - 1) { + //Handle edge case + ltBinUpper = labelsSubPredicted.lte(1.0).castTo(predictions2d.dataType()); + ltBinUpperProbs = maskedProbs.lte(1.0).castTo(predictions2d.dataType()); + } else { + ltBinUpper = labelsSubPredicted.lt((j + 1) * histogramBinSize).castTo(predictions2d.dataType()); + ltBinUpperProbs = maskedProbs.lt((j + 1) * histogramBinSize).castTo(predictions2d.dataType()); + } + + INDArray currBinBitMask = geqBinLower.muli(ltBinUpper); + INDArray currBinBitMaskProbs = geqBinLowerProbs.muli(ltBinUpperProbs); + + int newTotalCount = residualPlotOverall.getInt(0, j) + currBinBitMask.sumNumber().intValue(); + residualPlotOverall.putScalar(0, j, newTotalCount); + + //Counts for positive class only: values are in the current bin AND it's a positive label + INDArray isPosLabelForBin = l.mul(currBinBitMask); + + residualPlotByLabelClass.getRow(j).addi(isPosLabelForBin.sum(0).castTo(residualPlotByLabelClass.dataType())); + + int probNewTotalCount = probHistogramOverall.getInt(0, j) + currBinBitMaskProbs.sumNumber().intValue(); + probHistogramOverall.putScalar(0, j, probNewTotalCount); + + INDArray isPosLabelForBinProbs = l.mul(currBinBitMaskProbs); + INDArray temp = isPosLabelForBinProbs.sum(0); + probHistogramByLabelClass.getRow(j).addi(temp.castTo(probHistogramByLabelClass.dataType())); + } + } + + @Override + public void eval(INDArray labels, INDArray networkPredictions) { + eval(labels, networkPredictions, (INDArray) null); + } + + @Override + public void eval(INDArray labels, INDArray networkPredictions, INDArray maskArray, List recordMetaData) { + eval(labels, networkPredictions, maskArray); + } + + @Override + public void merge(EvaluationCalibration other) { + if (reliabilityDiagNumBins != other.reliabilityDiagNumBins) { + throw new UnsupportedOperationException( + "Cannot merge EvaluationCalibration instances with different numbers of bins"); + } + + if (other.rDiagBinPosCount == null) { + return; + } + + if (rDiagBinPosCount == null) { + this.rDiagBinPosCount = other.rDiagBinPosCount; + this.rDiagBinTotalCount = other.rDiagBinTotalCount; + this.rDiagBinSumPredictions = other.rDiagBinSumPredictions; + } + + this.rDiagBinPosCount.addi(other.rDiagBinPosCount); + this.rDiagBinTotalCount.addi(other.rDiagBinTotalCount); + this.rDiagBinSumPredictions.addi(other.rDiagBinSumPredictions); + } + + @Override + public void reset() { + rDiagBinPosCount = null; + rDiagBinTotalCount = null; + rDiagBinSumPredictions = null; + } + + @Override + public String stats() { + return "EvaluationCalibration(nBins=" + reliabilityDiagNumBins + ")"; + } + + public int numClasses() { + if (rDiagBinTotalCount == null) { + return -1; + } + + return (int) rDiagBinTotalCount.size(1); + } + + /** + * Get the reliability diagram for the specified class + * + * @param classIdx Index of the class to get the reliability diagram for + */ + public ReliabilityDiagram getReliabilityDiagram(int classIdx) { + Preconditions.checkState(rDiagBinPosCount != null, "Unable to get reliability diagram: no evaluation has been performed (no data)"); + INDArray totalCountBins = rDiagBinTotalCount.getColumn(classIdx); + INDArray countPositiveBins = rDiagBinPosCount.getColumn(classIdx); + + double[] meanPredictionBins = rDiagBinSumPredictions.getColumn(classIdx).castTo(DataType.DOUBLE) + .div(totalCountBins.castTo(DataType.DOUBLE)).data().asDouble(); + + double[] fracPositives = countPositiveBins.castTo(DataType.DOUBLE).div(totalCountBins.castTo(DataType.DOUBLE)).data().asDouble(); + + if (excludeEmptyBins) { + val condition = new MatchCondition(totalCountBins, Conditions.equals(0)); + int numZeroBins = Nd4j.getExecutioner().exec(condition).getInt(0); + if (numZeroBins != 0) { + double[] mpb = meanPredictionBins; + double[] fp = fracPositives; + + meanPredictionBins = new double[(int) (totalCountBins.length() - numZeroBins)]; + fracPositives = new double[meanPredictionBins.length]; + int j = 0; + for (int i = 0; i < mpb.length; i++) { + if (totalCountBins.getDouble(i) != 0) { + meanPredictionBins[j] = mpb[i]; + fracPositives[j] = fp[i]; + j++; + } + } + } + } + String title = "Reliability Diagram: Class " + classIdx; + return new ReliabilityDiagram(title, meanPredictionBins, fracPositives); + } + + /** + * @return The number of observed labels for each class. For N classes, be returned array is of length N, with + * out[i] being the number of labels of class i + */ + public int[] getLabelCountsEachClass() { + return labelCountsEachClass == null ? null : labelCountsEachClass.data().asInt(); + } + + /** + * @return The number of network predictions for each class. For N classes, be returned array is of length N, with + * out[i] being the number of predicted values (max probability) for class i + */ + public int[] getPredictionCountsEachClass() { + return predictionCountsEachClass == null ? null : predictionCountsEachClass.data().asInt(); + } + + /** + * Get the residual plot for all classes combined. The residual plot is defined as a histogram of
+ * |label_i - prob(class_i | input)| for all classes i and examples.
+ * In general, small residuals indicate a superior classifier to large residuals. + * + * @return Residual plot (histogram) - all predictions/classes + */ + public Histogram getResidualPlotAllClasses() { + String title = "Residual Plot - All Predictions and Classes"; + int[] counts = residualPlotOverall.data().asInt(); + return new Histogram(title, 0.0, 1.0, counts); + } + + /** + * Get the residual plot, only for examples of the specified class.. The residual plot is defined as a histogram of
+ * |label_i - prob(class_i | input)| for all and examples; for this particular method, only predictions where + * i == labelClassIdx are included.
+ * In general, small residuals indicate a superior classifier to large residuals. + * + * @param labelClassIdx Index of the class to get the residual plot for + * @return Residual plot (histogram) - all predictions/classes + */ + public Histogram getResidualPlot(int labelClassIdx) { + Preconditions.checkState(rDiagBinPosCount != null, "Unable to get residual plot: no evaluation has been performed (no data)"); + String title = "Residual Plot - Predictions for Label Class " + labelClassIdx; + int[] counts = residualPlotByLabelClass.getColumn(labelClassIdx).dup().data().asInt(); + return new Histogram(title, 0.0, 1.0, counts); + } + + /** + * Return a probability histogram for all predictions/classes. + * + * @return Probability histogram + */ + public Histogram getProbabilityHistogramAllClasses() { + String title = "Network Probabilities Histogram - All Predictions and Classes"; + int[] counts = probHistogramOverall.data().asInt(); + return new Histogram(title, 0.0, 1.0, counts); + } + + /** + * Return a probability histogram of the specified label class index. That is, for label class index i, + * a histogram of P(class_i | input) is returned, only for those examples that are labelled as class i. + * + * @param labelClassIdx Index of the label class to get the histogram for + * @return Probability histogram + */ + public Histogram getProbabilityHistogram(int labelClassIdx) { + Preconditions.checkState(rDiagBinPosCount != null, "Unable to get probability histogram: no evaluation has been performed (no data)"); + String title = "Network Probabilities Histogram - P(class " + labelClassIdx + ") - Data Labelled Class " + + labelClassIdx + " Only"; + int[] counts = probHistogramByLabelClass.getColumn(labelClassIdx).dup().data().asInt(); + return new Histogram(title, 0.0, 1.0, counts); + } + + public static EvaluationCalibration fromJson(String json){ + return fromJson(json, EvaluationCalibration.class); + } + + @Override + public double getValue(IMetric metric){ + throw new IllegalStateException("Can't get value for non-calibration Metric " + metric); + } + + @Override + public EvaluationCalibration newInstance() { + return new EvaluationCalibration(axis, reliabilityDiagNumBins, histogramNumBins, excludeEmptyBins); + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROC.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROC.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROC.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROC.java index ef1774210..f669f377d 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROC.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROC.java @@ -42,9 +42,9 @@ import org.nd4j.linalg.indexing.conditions.Condition; import org.nd4j.linalg.indexing.conditions.Conditions; import org.nd4j.common.primitives.Pair; import org.nd4j.common.primitives.Triple; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.Serializable; import java.util.Arrays; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROCBinary.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROCBinary.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROCBinary.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROCBinary.java index 49682e2df..17ddeb982 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROCBinary.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROCBinary.java @@ -32,7 +32,7 @@ import org.nd4j.evaluation.serde.ROCArraySerializer; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Triple; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.Serializable; import java.util.ArrayList; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROCMultiClass.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROCMultiClass.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROCMultiClass.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROCMultiClass.java index c943418ef..78dffe507 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/classification/ROCMultiClass.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/classification/ROCMultiClass.java @@ -31,7 +31,7 @@ import org.nd4j.evaluation.curves.RocCurve; import org.nd4j.evaluation.serde.ROCArraySerializer; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Triple; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.Serializable; import java.util.Arrays; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/curves/BaseCurve.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/BaseCurve.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/curves/BaseCurve.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/BaseCurve.java index 3ad33cac2..d9d8d2ae9 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/curves/BaseCurve.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/BaseCurve.java @@ -21,8 +21,8 @@ package org.nd4j.evaluation.curves; import org.nd4j.serde.json.JsonMappers; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/curves/BaseHistogram.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/BaseHistogram.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/curves/BaseHistogram.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/BaseHistogram.java index 36ac34f00..118410183 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/curves/BaseHistogram.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/BaseHistogram.java @@ -21,8 +21,8 @@ package org.nd4j.evaluation.curves; import org.nd4j.serde.json.JsonMappers; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/Histogram.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/Histogram.java new file mode 100644 index 000000000..afaf32b32 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/Histogram.java @@ -0,0 +1,93 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.evaluation.curves; + +import lombok.Data; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Data +public class Histogram extends BaseHistogram { + private final String title; + private final double lower; + private final double upper; + private final int[] binCounts; + + public Histogram(@JsonProperty("title") String title, @JsonProperty("lower") double lower, + @JsonProperty("upper") double upper, @JsonProperty("binCounts") int[] binCounts) { + this.title = title; + this.lower = lower; + this.upper = upper; + this.binCounts = binCounts; + } + + @Override + public int numPoints() { + return binCounts.length; + } + + @Override + public double[] getBinLowerBounds() { + double step = 1.0 / binCounts.length; + double[] out = new double[binCounts.length]; + for (int i = 0; i < out.length; i++) { + out[i] = i * step; + } + return out; + } + + @Override + public double[] getBinUpperBounds() { + double step = 1.0 / binCounts.length; + double[] out = new double[binCounts.length]; + for (int i = 0; i < out.length - 1; i++) { + out[i] = (i + 1) * step; + } + out[out.length - 1] = 1.0; + return out; + } + + @Override + public double[] getBinMidValues() { + double step = 1.0 / binCounts.length; + double[] out = new double[binCounts.length]; + for (int i = 0; i < out.length; i++) { + out[i] = (i + 0.5) * step; + } + return out; + } + + /** + * @param json JSON representation + * @return Instance of the histogram + */ + public static Histogram fromJson(String json) { + return BaseHistogram.fromJson(json, Histogram.class); + } + + /** + * + * @param yaml YAML representation + * @return Instance of the histogram + */ + public static Histogram fromYaml(String yaml) { + return BaseHistogram.fromYaml(yaml, Histogram.class); + } +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/PrecisionRecallCurve.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/PrecisionRecallCurve.java new file mode 100644 index 000000000..7838eb8c7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/PrecisionRecallCurve.java @@ -0,0 +1,251 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.evaluation.curves; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.nd4j.common.base.Preconditions; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Arrays; + +@Data +@EqualsAndHashCode(exclude = {"area"}, callSuper = false) +public class PrecisionRecallCurve extends BaseCurve { + + private double[] threshold; + private double[] precision; + private double[] recall; + private int[] tpCount; + private int[] fpCount; + private int[] fnCount; + private int totalCount; + + private Double area; + + public PrecisionRecallCurve(@JsonProperty("threshold") double[] threshold, + @JsonProperty("precision") double[] precision, @JsonProperty("recall") double[] recall, + @JsonProperty("tpCount") int[] tpCount, @JsonProperty("fpCount") int[] fpCount, + @JsonProperty("fnCount") int[] fnCount, @JsonProperty("totalCount") int totalCount) { + this.threshold = threshold; + this.precision = precision; + this.recall = recall; + this.tpCount = tpCount; + this.fpCount = fpCount; + this.fnCount = fnCount; + this.totalCount = totalCount; + } + + @Override + public int numPoints() { + return threshold.length; + } + + @Override + public double[] getX() { + return recall; + } + + @Override + public double[] getY() { + return precision; + } + + @Override + public String getTitle() { + return "Precision-Recall Curve (Area=" + format(calculateAUPRC(), DEFAULT_FORMAT_PREC) + ")"; + } + + /** + * @param i Point number, 0 to numPoints()-1 inclusive + * @return Threshold of a given point + */ + public double getThreshold(int i) { + Preconditions.checkArgument(i >= 0 && i < threshold.length, "Invalid index: " + i); + return threshold[i]; + } + + /** + * @param i Point number, 0 to numPoints()-1 inclusive + * @return Precision of a given point + */ + public double getPrecision(int i) { + Preconditions.checkArgument(i >= 0 && i < precision.length, "Invalid index: " + i); + return precision[i]; + } + + /** + * @param i Point number, 0 to numPoints()-1 inclusive + * @return Recall of a given point + */ + public double getRecall(int i) { + Preconditions.checkArgument(i >= 0 && i < recall.length, "Invalid index: " + i); + return recall[i]; + } + + /** + * @return The area under the precision recall curve + */ + public double calculateAUPRC() { + if (area != null) { + return area; + } + + area = calculateArea(); + return area; + } + + /** + * Get the point (index, threshold, precision, recall) at the given threshold.
+ * Note that if the threshold is not found exactly, the next highest threshold exceeding the requested threshold + * is returned + * + * @param threshold Threshold to get the point for + * @return point (index, threshold, precision, recall) at the given threshold + */ + public Point getPointAtThreshold(double threshold) { + + //Return (closest) point number, precision, recall, whether it's interpolated or not + + //Binary search to find closest threshold + + int idx = Arrays.binarySearch(this.threshold, threshold); + if (idx < 0) { + //Not found (usual case). binarySearch javadoc: + /* + index of the search key, if it is contained in the array; + otherwise, (-(insertion point) - 1). The + insertion point is defined as the point at which the + key would be inserted into the array: the index of the first + element greater than the key, or a.length if all + elements in the array are less than the specified key. + */ + idx = -idx - 1; + } + + //At this point: idx = exact, on the next highest + double thr = this.threshold[idx]; + double pr = precision[idx]; + double rec = recall[idx]; + + return new Point(idx, thr, pr, rec); + } + + /** + * Get the point (index, threshold, precision, recall) at the given precision.
+ * Specifically, return the points at the lowest threshold that has precision equal to or greater than the + * requested precision. + * + * @param precision Precision to get the point for + * @return point (index, threshold, precision, recall) at (or closest exceeding) the given precision + */ + public Point getPointAtPrecision(double precision) { + //Find the LOWEST threshold that gives the specified precision + + for (int i = 0; i < this.precision.length; i++) { + if (this.precision[i] >= precision) { + return new Point(i, threshold[i], this.precision[i], recall[i]); + } + } + + //Not found, return last point. Should never happen though... + int i = threshold.length - 1; + return new Point(i, threshold[i], this.precision[i], this.recall[i]); + } + + /** + * Get the point (index, threshold, precision, recall) at the given recall.
+ * Specifically, return the points at the highest threshold that has recall equal to or greater than the + * requested recall. + * + * @param recall Recall to get the point for + * @return point (index, threshold, precision, recall) at (or closest exceeding) the given recall + */ + public Point getPointAtRecall(double recall) { + Point foundPoint = null; + //Find the HIGHEST threshold that gives the specified recall + for (int i = this.recall.length - 1; i >= 0; i--) { + if (this.recall[i] >= recall) { + if (foundPoint == null ||(this.recall[i] == foundPoint.getRecall() && this.precision[i] >= foundPoint.getPrecision())) { + foundPoint = new Point(i, threshold[i], precision[i], this.recall[i]); + } + } + } + if (foundPoint == null){ + //Not found - return first point. Should never happen... + foundPoint = new Point(0, threshold[0], precision[0], this.recall[0]); + } + return foundPoint; + } + + /** + * Get the binary confusion matrix for the given threshold. As per {@link #getPointAtThreshold(double)}, + * if the threshold is not found exactly, the next highest threshold exceeding the requested threshold + * is returned + * + * @param threshold Threshold at which to get the confusion matrix + * @return Binary confusion matrix + */ + public Confusion getConfusionMatrixAtThreshold(double threshold) { + Point p = getPointAtThreshold(threshold); + int idx = p.idx; + int tn = totalCount - (tpCount[idx] + fpCount[idx] + fnCount[idx]); + return new Confusion(p, tpCount[idx], fpCount[idx], fnCount[idx], tn); + } + + /** + * Get the binary confusion matrix for the given position. As per {@link #getPointAtThreshold(double)}. + * + * @param point Position at which to get the binary confusion matrix + * @return Binary confusion matrix + */ + public Confusion getConfusionMatrixAtPoint(int point) { + return getConfusionMatrixAtThreshold(threshold[point]); + } + + public static PrecisionRecallCurve fromJson(String json) { + return fromJson(json, PrecisionRecallCurve.class); + } + + public static PrecisionRecallCurve fromYaml(String yaml) { + return fromYaml(yaml, PrecisionRecallCurve.class); + } + + @AllArgsConstructor + @Data + public static class Point { + private final int idx; + private final double threshold; + private final double precision; + private final double recall; + } + + @AllArgsConstructor + @Data + public static class Confusion { + private final Point point; + private final int tpCount; + private final int fpCount; + private final int fnCount; + private final int tnCount; + } +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/ReliabilityDiagram.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/ReliabilityDiagram.java new file mode 100644 index 000000000..ed38dd539 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/ReliabilityDiagram.java @@ -0,0 +1,65 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.evaluation.curves; + +import lombok.Getter; +import lombok.NonNull; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Getter +public class ReliabilityDiagram extends BaseCurve { + + private final String title; + private final double[] meanPredictedValueX; + private final double[] fractionPositivesY; + + + public ReliabilityDiagram(@JsonProperty("title") String title, + @NonNull @JsonProperty("meanPredictedValueX") double[] meanPredictedValueX, + @NonNull @JsonProperty("fractionPositivesY") double[] fractionPositivesY) { + this.title = title; + this.meanPredictedValueX = meanPredictedValueX; + this.fractionPositivesY = fractionPositivesY; + } + + @Override + public int numPoints() { + return meanPredictedValueX.length; + } + + @Override + public double[] getX() { + return getMeanPredictedValueX(); + } + + @Override + public double[] getY() { + return getFractionPositivesY(); + } + + @Override + public String getTitle() { + if (title == null) { + return "Reliability Diagram"; + } + return title; + } +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/RocCurve.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/RocCurve.java new file mode 100644 index 000000000..0a1f5e6b1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/curves/RocCurve.java @@ -0,0 +1,113 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.evaluation.curves; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.nd4j.common.base.Preconditions; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Data +@EqualsAndHashCode(exclude = {"auc"}, callSuper = false) +public class RocCurve extends BaseCurve { + + private double[] threshold; + private double[] fpr; + private double[] tpr; + + private Double auc; + + public RocCurve(@JsonProperty("threshold") double[] threshold, @JsonProperty("fpr") double[] fpr, + @JsonProperty("tpr") double[] tpr) { + this.threshold = threshold; + this.fpr = fpr; + this.tpr = tpr; + } + + + @Override + public int numPoints() { + return threshold.length; + } + + @Override + public double[] getX() { + return fpr; + } + + @Override + public double[] getY() { + return tpr; + } + + @Override + public String getTitle() { + return "ROC (Area=" + format(calculateAUC(), DEFAULT_FORMAT_PREC) + ")"; + } + + /** + * @param i Point number, 0 to numPoints()-1 inclusive + * @return Threshold of a given point + */ + public double getThreshold(int i) { + Preconditions.checkArgument(i >= 0 && i < threshold.length, "Invalid index: " + i); + return threshold[i]; + } + + /** + * @param i Point number, 0 to numPoints()-1 inclusive + * @return True positive rate of a given point + */ + public double getTruePositiveRate(int i) { + Preconditions.checkArgument(i >= 0 && i < tpr.length, "Invalid index: " + i); + return tpr[i]; + } + + /** + * @param i Point number, 0 to numPoints()-1 inclusive + * @return False positive rate of a given point + */ + public double getFalsePositiveRate(int i) { + Preconditions.checkArgument(i >= 0 && i < fpr.length, "Invalid index: " + i); + return fpr[i]; + } + + /** + * Calculate and return the area under ROC curve + */ + public double calculateAUC() { + if (auc != null) { + return auc; + } + + auc = calculateArea(); + return auc; + } + + public static RocCurve fromJson(String json) { + return fromJson(json, RocCurve.class); + } + + public static RocCurve fromYaml(String yaml) { + return fromYaml(yaml, RocCurve.class); + } + +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/CustomEvaluation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/CustomEvaluation.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/CustomEvaluation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/CustomEvaluation.java index 5978cba09..4d0f697f4 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/CustomEvaluation.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/CustomEvaluation.java @@ -20,7 +20,7 @@ package org.nd4j.evaluation.custom; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.collect.Lists; import java.io.Serializable; import java.util.ArrayList; import java.util.List; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/EvaluationLambda.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/EvaluationLambda.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/EvaluationLambda.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/EvaluationLambda.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/MergeLambda.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/MergeLambda.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/MergeLambda.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/MergeLambda.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/ResultLambda.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/ResultLambda.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/custom/ResultLambda.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/custom/ResultLambda.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/meta/Prediction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/meta/Prediction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/meta/Prediction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/meta/Prediction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/regression/RegressionEvaluation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/regression/RegressionEvaluation.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/regression/RegressionEvaluation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/regression/RegressionEvaluation.java index 60fd53523..d34bd5c4e 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/regression/RegressionEvaluation.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/regression/RegressionEvaluation.java @@ -33,8 +33,8 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Triple; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.Serializable; import java.util.ArrayList; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixDeserializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixDeserializer.java similarity index 87% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixDeserializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixDeserializer.java index bb9366ea3..e01f0d51a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixDeserializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixDeserializer.java @@ -21,13 +21,13 @@ package org.nd4j.evaluation.serde; import org.nd4j.evaluation.classification.ConfusionMatrix; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.node.ArrayNode; -import org.nd4j.shade.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.util.ArrayList; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixSerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixSerializer.java similarity index 88% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixSerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixSerializer.java index 134b6213e..4234fed16 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixSerializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ConfusionMatrixSerializer.java @@ -20,12 +20,12 @@ package org.nd4j.evaluation.serde; -import org.nd4j.shade.guava.collect.Multiset; +import com.google.common.collect.Multiset; import org.nd4j.evaluation.classification.ConfusionMatrix; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; import java.util.LinkedHashMap; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ROCArraySerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ROCArraySerializer.java similarity index 88% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ROCArraySerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ROCArraySerializer.java index 829a1aa30..6da654cae 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ROCArraySerializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ROCArraySerializer.java @@ -21,10 +21,10 @@ package org.nd4j.evaluation.serde; import org.nd4j.evaluation.classification.ROC; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ROCSerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ROCSerializer.java similarity index 93% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ROCSerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ROCSerializer.java index ecdaa773d..a39c23a77 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/evaluation/serde/ROCSerializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/evaluation/serde/ROCSerializer.java @@ -21,10 +21,10 @@ package org.nd4j.evaluation.serde; import org.nd4j.evaluation.classification.ROC; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; -import org.nd4j.shade.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; import java.io.IOException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ByteOrder.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ByteOrder.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ByteOrder.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ByteOrder.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/DType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/DType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/DType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/DType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/Direction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/Direction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/Direction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/Direction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ExecutionMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ExecutionMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ExecutionMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ExecutionMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatArray.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatArray.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatArray.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatArray.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatArrayList.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatArrayList.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatArrayList.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatArrayList.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatConfiguration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatConfiguration.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatConfiguration.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatConfiguration.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatDropRequest.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatDropRequest.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatDropRequest.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatDropRequest.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatGraph.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatGraph.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatGraph.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatGraph.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatInferenceRequest.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatInferenceRequest.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatInferenceRequest.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatInferenceRequest.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatNode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatNode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatNode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatNode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatProperties.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatProperties.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatProperties.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatProperties.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatResponse.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatResponse.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatResponse.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatResponse.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatResult.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatResult.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatResult.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatResult.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatTiming.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatTiming.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatTiming.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatTiming.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatVariable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatVariable.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FlatVariable.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FlatVariable.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FrameIteration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FrameIteration.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/FrameIteration.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/FrameIteration.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/InputType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/InputType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/InputType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/InputType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/IntPair.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/IntPair.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/IntPair.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/IntPair.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/IntTriple.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/IntTriple.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/IntTriple.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/IntTriple.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/LongPair.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/LongPair.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/LongPair.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/LongPair.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/LongTriple.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/LongTriple.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/LongTriple.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/LongTriple.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/OpClass.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/OpClass.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/OpClass.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/OpClass.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/OpType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/OpType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/OpType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/OpType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/OutputMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/OutputMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/OutputMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/OutputMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ProfilingMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ProfilingMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ProfilingMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ProfilingMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIAddName.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIAddName.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIAddName.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIAddName.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIEvent.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIEvent.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIEvent.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIEvent.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIEventSubtype.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIEventSubtype.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIEventSubtype.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIEventSubtype.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIEventType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIEventType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIEventType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIEventType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIGraphStructure.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIGraphStructure.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIGraphStructure.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIGraphStructure.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIHardwareState.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIHardwareState.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIHardwareState.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIHardwareState.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIHistogram.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIHistogram.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIHistogram.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIHistogram.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIHistogramType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIHistogramType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIHistogramType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIHistogramType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIInfoType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIInfoType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIInfoType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIInfoType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIStaticInfoRecord.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIStaticInfoRecord.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIStaticInfoRecord.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIStaticInfoRecord.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UISummaryStatistics.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UISummaryStatistics.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UISummaryStatistics.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UISummaryStatistics.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UISystemInfo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UISystemInfo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UISystemInfo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UISystemInfo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIVariable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIVariable.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UIVariable.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UIVariable.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UpdaterState.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UpdaterState.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/UpdaterState.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/UpdaterState.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/VarType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/VarType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/VarType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/VarType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ui/LogFileWriter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ui/LogFileWriter.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ui/LogFileWriter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ui/LogFileWriter.java index eb5e37c41..b4e29c98e 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/ui/LogFileWriter.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/graph/ui/LogFileWriter.java @@ -46,7 +46,6 @@ import org.nd4j.common.primitives.Pair; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; @@ -176,15 +175,13 @@ public class LogFileWriter { //Read header ByteBuffer bb = ByteBuffer.allocate(lengthHeader); f.getChannel().read(bb); - Buffer buffer = (Buffer) bb; - buffer.flip(); //Flip for reading + bb.flip(); //Flip for reading UIStaticInfoRecord r = UIStaticInfoRecord.getRootAsUIStaticInfoRecord(bb); //Read content bb = ByteBuffer.allocate(lengthContent); f.getChannel().read(bb); - Buffer buffer1 = (Buffer) bb; - buffer1.flip(); //Flip for reading + bb.flip(); //Flip for reading byte infoType = r.infoType(); Table t; @@ -251,15 +248,13 @@ public class LogFileWriter { //Read header ByteBuffer bb = ByteBuffer.allocate(lengthHeader); f.getChannel().read(bb); - Buffer buffer2 = (Buffer) bb; - buffer2.flip();//Flip for reading + bb.flip(); //Flip for reading UIEvent e = UIEvent.getRootAsUIEvent(bb); //Read Content bb = ByteBuffer.allocate(lengthContent); f.getChannel().read(bb); - Buffer buffer3 = (Buffer) bb; - buffer3.flip(); //Flip for reading + bb.flip(); //Flip for reading byte infoType = e.eventType(); Table t; @@ -642,8 +637,7 @@ public class LogFileWriter { int l2 = bb2 == null ? 0 : bb2.remaining(); header.putInt(l1); header.putInt(l2); - Buffer buffer = (Buffer) header; - buffer.flip(); + header.flip(); //System.out.println("Lengths - header, content: " + l1 + ", " + l2); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/NoOpNameFoundException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/NoOpNameFoundException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/NoOpNameFoundException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/NoOpNameFoundException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/VariableUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/VariableUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/VariableUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/VariableUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/converters/DifferentialFunctionClassHolder.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/converters/DifferentialFunctionClassHolder.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/converters/DifferentialFunctionClassHolder.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/converters/DifferentialFunctionClassHolder.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/converters/ImportClassMapping.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/converters/ImportClassMapping.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/converters/ImportClassMapping.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/converters/ImportClassMapping.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptorParser.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptorParser.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptorParser.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptorParser.java index 62f8ffc41..e0fa4698a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptorParser.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/OnnxDescriptorParser.java @@ -21,7 +21,7 @@ package org.nd4j.imports.descriptors.onnx; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.InputStream; import java.util.HashMap; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/OpDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/OpDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/OpDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/OpDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/TensorDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/TensorDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/onnx/TensorDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/onnx/TensorDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/AttributeAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/AttributeAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/AttributeAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/AttributeAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/PropertyMapping.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/PropertyMapping.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/PropertyMapping.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/PropertyMapping.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/BooleanAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/BooleanAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/BooleanAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/BooleanAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/ConditionalFieldValueIntIndexArrayAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/ConditionalFieldValueIntIndexArrayAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/ConditionalFieldValueIntIndexArrayAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/ConditionalFieldValueIntIndexArrayAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/ConditionalFieldValueNDArrayShapeAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/ConditionalFieldValueNDArrayShapeAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/ConditionalFieldValueNDArrayShapeAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/ConditionalFieldValueNDArrayShapeAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/DataTypeAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/DataTypeAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/DataTypeAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/DataTypeAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/IntArrayIntIndexAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/IntArrayIntIndexAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/IntArrayIntIndexAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/IntArrayIntIndexAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/NDArrayShapeAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/NDArrayShapeAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/NDArrayShapeAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/NDArrayShapeAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/SizeThresholdIntArrayIntIndexAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/SizeThresholdIntArrayIntIndexAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/SizeThresholdIntArrayIntIndexAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/SizeThresholdIntArrayIntIndexAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/StringEqualsAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/StringEqualsAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/StringEqualsAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/StringEqualsAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/StringNotEqualsAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/StringNotEqualsAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/StringNotEqualsAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/properties/adapters/StringNotEqualsAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/tensorflow/TensorflowDescriptorParser.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/tensorflow/TensorflowDescriptorParser.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/tensorflow/TensorflowDescriptorParser.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/tensorflow/TensorflowDescriptorParser.java index 47232cd14..f137ec83f 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/descriptors/tensorflow/TensorflowDescriptorParser.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/descriptors/tensorflow/TensorflowDescriptorParser.java @@ -20,7 +20,7 @@ package org.nd4j.imports.descriptors.tensorflow; -import org.nd4j.shade.protobuf.TextFormat; +import com.google.protobuf.TextFormat; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.common.io.ClassPathResource; import org.tensorflow.framework.OpDef; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/OpImportFilter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/OpImportFilter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/OpImportFilter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/OpImportFilter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/OpImportOverride.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/OpImportOverride.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/OpImportOverride.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/OpImportOverride.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/tf/TFGraphMapper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/tf/TFGraphMapper.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/tf/TFGraphMapper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/tf/TFGraphMapper.java index de2323801..7e1e236fb 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/tf/TFGraphMapper.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/tf/TFGraphMapper.java @@ -41,10 +41,10 @@ import org.nd4j.imports.tensorflow.TFImportOverride; import org.nd4j.imports.tensorflow.TFOpImportFilter; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.impl.controlflow.compat.Merge; -import org.nd4j.shade.guava.primitives.Floats; -import org.nd4j.shade.guava.primitives.Ints; -import org.nd4j.shade.protobuf.Message; -import org.nd4j.shade.protobuf.TextFormat; +import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; +import com.google.protobuf.Message; +import com.google.protobuf.TextFormat; import org.tensorflow.framework.*; import org.apache.commons.collections4.set.ListOrderedSet; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/tf/tensors/TFTensorMapper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/tf/tensors/TFTensorMapper.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/tf/tensors/TFTensorMapper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/tf/tensors/TFTensorMapper.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/tf/tensors/TFTensorMappers.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/tf/tensors/TFTensorMappers.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/graphmapper/tf/tensors/TFTensorMappers.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/graphmapper/tf/tensors/TFTensorMappers.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/tensorflow/TFImportOverride.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/tensorflow/TFImportOverride.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/tensorflow/TFImportOverride.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/tensorflow/TFImportOverride.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/tensorflow/TFImportStatus.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/tensorflow/TFImportStatus.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/tensorflow/TFImportStatus.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/tensorflow/TFImportStatus.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/tensorflow/TFOpImportFilter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/tensorflow/TFOpImportFilter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/tensorflow/TFOpImportFilter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/tensorflow/TFOpImportFilter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/tensorflow/TensorFlowImportValidator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/tensorflow/TensorFlowImportValidator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/imports/tensorflow/TensorFlowImportValidator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/imports/tensorflow/TensorFlowImportValidator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/Activation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/Activation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/Activation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/Activation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/BaseActivationFunction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/BaseActivationFunction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/BaseActivationFunction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/BaseActivationFunction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/IActivation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/IActivation.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/IActivation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/IActivation.java index 17516a88a..702119a7a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/IActivation.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/IActivation.java @@ -23,8 +23,8 @@ package org.nd4j.linalg.activations; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; import org.nd4j.serde.json.LegacyIActivationDeserializerHelper; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationCube.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationCube.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationCube.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationCube.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationELU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationELU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationELU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationELU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationGELU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationGELU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationGELU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationGELU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardSigmoid.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardSigmoid.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardSigmoid.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardSigmoid.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardTanH.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardTanH.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardTanH.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationHardTanH.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationIdentity.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationIdentity.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationIdentity.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationIdentity.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationLReLU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationLReLU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationLReLU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationLReLU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationMish.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationMish.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationMish.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationMish.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationPReLU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationPReLU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationPReLU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationPReLU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java index e03837717..56b0c9b95 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRReLU.java @@ -30,7 +30,7 @@ import org.nd4j.linalg.api.ops.impl.scalar.RectifiedLinear; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.indexing.BooleanIndexing; import org.nd4j.linalg.indexing.conditions.Conditions; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @EqualsAndHashCode(callSuper = false) @JsonIgnoreProperties({"alpha"}) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRationalTanh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRationalTanh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRationalTanh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRationalTanh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU6.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU6.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU6.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationReLU6.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRectifiedTanh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRectifiedTanh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRectifiedTanh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationRectifiedTanh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSELU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSELU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSELU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSELU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSigmoid.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSigmoid.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSigmoid.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSigmoid.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftPlus.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftPlus.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftPlus.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftPlus.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftSign.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftSign.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftSign.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftSign.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftmax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftmax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftmax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSoftmax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSwish.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSwish.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSwish.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationSwish.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationTanH.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationTanH.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationTanH.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationTanH.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationThresholdedReLU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationThresholdedReLU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationThresholdedReLU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/activations/impl/ActivationThresholdedReLU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Blas.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Blas.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Blas.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Blas.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/BlasBufferUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/BlasBufferUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/BlasBufferUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/BlasBufferUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/BlasException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/BlasException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/BlasException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/BlasException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Lapack.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Lapack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Lapack.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Lapack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Level1.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Level1.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Level1.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Level1.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Level2.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Level2.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Level2.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Level2.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Level3.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Level3.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/Level3.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/Level3.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLapack.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLapack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLapack.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLapack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel1.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel1.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel1.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel1.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel2.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel2.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel2.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel2.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel3.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel3.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel3.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/impl/BaseLevel3.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/params/GemmParams.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/params/GemmParams.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/params/GemmParams.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/params/GemmParams.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/params/GemvParameters.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/params/GemvParameters.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/params/GemvParameters.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/params/GemvParameters.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/params/MMulTranspose.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/params/MMulTranspose.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/blas/params/MMulTranspose.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/blas/params/MMulTranspose.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/BaseDataBuffer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/BaseDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/BaseDataBuffer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/BaseDataBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataBuffer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/DataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataBuffer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/DataBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/DataType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/DataType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataTypeEx.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/DataTypeEx.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataTypeEx.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/DataTypeEx.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/allocation/MemoryStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/allocation/MemoryStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/allocation/MemoryStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/allocation/MemoryStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/factory/DataBufferFactory.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/factory/DataBufferFactory.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/factory/DataBufferFactory.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/factory/DataBufferFactory.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/util/AllocUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/util/AllocUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/util/AllocUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/util/AllocUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/util/DataTypeUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/util/DataTypeUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/util/DataTypeUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/buffer/util/DataTypeUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/AffinityManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/AffinityManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/AffinityManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/AffinityManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/ArrayType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/ArrayType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/ArrayType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/ArrayType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/BasicAffinityManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/BasicAffinityManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/BasicAffinityManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/BasicAffinityManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/BasicDistributedINDArray.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/BasicDistributedINDArray.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/BasicDistributedINDArray.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/BasicDistributedINDArray.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/DistributedINDArray.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/DistributedINDArray.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/concurrency/DistributedINDArray.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/concurrency/DistributedINDArray.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/environment/Nd4jEnvironment.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/environment/Nd4jEnvironment.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/environment/Nd4jEnvironment.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/environment/Nd4jEnvironment.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/FirstAxisIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/FirstAxisIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/FirstAxisIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/FirstAxisIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/FlatIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/FlatIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/FlatIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/FlatIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/INDArrayIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/INDArrayIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/INDArrayIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/INDArrayIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/LinearIndexLookup.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/LinearIndexLookup.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/LinearIndexLookup.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/LinearIndexLookup.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/NdIndexIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/NdIndexIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/iter/NdIndexIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/iter/NdIndexIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/AllocationsTracker.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/AllocationsTracker.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/AllocationsTracker.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/AllocationsTracker.java index 8429e6d61..0f450224d 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/AllocationsTracker.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/AllocationsTracker.java @@ -22,7 +22,6 @@ package org.nd4j.linalg.api.memory; import lombok.extern.slf4j.Slf4j; import lombok.val; -import lombok.var; import org.nd4j.linalg.api.memory.enums.AllocationKind; import java.util.Map; @@ -42,7 +41,7 @@ public class AllocationsTracker { } protected DeviceAllocationsTracker trackerForDevice(Integer deviceId) { - var tracker = devices.get(deviceId); + DeviceAllocationsTracker tracker = devices.get(deviceId); if (tracker == null) { synchronized (this) { tracker = devices.get(deviceId); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/BasicMemoryManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/BasicMemoryManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/BasicMemoryManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/BasicMemoryManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/Deallocatable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/Deallocatable.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/Deallocatable.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/Deallocatable.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/Deallocator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/Deallocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/Deallocator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/Deallocator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/DeviceAllocationsTracker.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/DeviceAllocationsTracker.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/DeviceAllocationsTracker.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/DeviceAllocationsTracker.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/MemcpyDirection.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/MemcpyDirection.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/MemcpyDirection.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/MemcpyDirection.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/MemoryManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/MemoryManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/MemoryManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/MemoryManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/MemoryWorkspace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/MemoryWorkspace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/MemoryWorkspace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/MemoryWorkspace.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/MemoryWorkspaceManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/MemoryWorkspaceManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/MemoryWorkspaceManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/MemoryWorkspaceManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/abstracts/DummyWorkspace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/abstracts/DummyWorkspace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/abstracts/DummyWorkspace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/abstracts/DummyWorkspace.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/abstracts/Nd4jWorkspace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/abstracts/Nd4jWorkspace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/abstracts/Nd4jWorkspace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/abstracts/Nd4jWorkspace.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/conf/WorkspaceConfiguration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/conf/WorkspaceConfiguration.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/conf/WorkspaceConfiguration.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/conf/WorkspaceConfiguration.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatableReference.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatableReference.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatableReference.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatableReference.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatorService.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatorService.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatorService.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/deallocation/DeallocatorService.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/AllocationKind.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/AllocationKind.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/AllocationKind.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/AllocationKind.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/AllocationPolicy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/AllocationPolicy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/AllocationPolicy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/AllocationPolicy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/DebugMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/DebugMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/DebugMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/DebugMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/LearningPolicy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/LearningPolicy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/LearningPolicy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/LearningPolicy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/LocationPolicy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/LocationPolicy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/LocationPolicy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/LocationPolicy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/MemoryKind.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/MemoryKind.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/MemoryKind.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/MemoryKind.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/MirroringPolicy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/MirroringPolicy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/MirroringPolicy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/MirroringPolicy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/ResetPolicy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/ResetPolicy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/ResetPolicy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/ResetPolicy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/SpillPolicy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/SpillPolicy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/enums/SpillPolicy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/enums/SpillPolicy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/pointers/ImmortalFloatPointer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/pointers/ImmortalFloatPointer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/pointers/ImmortalFloatPointer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/pointers/ImmortalFloatPointer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/pointers/PagedPointer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/pointers/PagedPointer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/pointers/PagedPointer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/pointers/PagedPointer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/pointers/PointersPair.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/pointers/PointersPair.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/pointers/PointersPair.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/pointers/PointersPair.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/provider/BasicWorkspaceManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/provider/BasicWorkspaceManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/provider/BasicWorkspaceManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/provider/BasicWorkspaceManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/stash/BasicStash.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/stash/BasicStash.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/stash/BasicStash.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/stash/BasicStash.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/stash/BasicStashManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/stash/BasicStashManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/stash/BasicStashManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/stash/BasicStashManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/stash/Stash.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/stash/Stash.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/stash/Stash.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/stash/Stash.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/stash/StashManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/stash/StashManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/memory/stash/StashManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/memory/stash/StashManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java index c3594696d..c948afee7 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java @@ -21,8 +21,8 @@ package org.nd4j.linalg.api.ndarray; -import org.nd4j.shade.guava.primitives.Ints; -import org.nd4j.shade.guava.primitives.Longs; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; import com.google.flatbuffers.FlatBufferBuilder; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -1015,6 +1015,7 @@ public abstract class BaseNDArray implements INDArray, Iterable { } } + Pair tadInfo = Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(this, dimension); DataBuffer shapeInfo = tadInfo.getFirst(); val jShapeInfo = shapeInfo.asLong(); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArrayProxy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArrayProxy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArrayProxy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArrayProxy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseShapeInfoProvider.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseShapeInfoProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseShapeInfoProvider.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseShapeInfoProvider.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/INDArray.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/INDArray.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/INDArray.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/INDArray.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/INDArrayStatistics.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/INDArrayStatistics.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/INDArrayStatistics.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/INDArrayStatistics.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/JvmShapeInfo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/JvmShapeInfo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/JvmShapeInfo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/JvmShapeInfo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/ShapeInfoProvider.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/ShapeInfoProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/ShapeInfoProvider.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ndarray/ShapeInfoProvider.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseBroadcastBoolOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseBroadcastBoolOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseBroadcastBoolOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseBroadcastBoolOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseBroadcastOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseBroadcastOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseBroadcastOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseBroadcastOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseIndexAccumulation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseIndexAccumulation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseIndexAccumulation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseIndexAccumulation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseOpContext.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseOpContext.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseOpContext.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseOpContext.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceBoolOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceBoolOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceBoolOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceBoolOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceFloatOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceFloatOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceFloatOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceFloatOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceLongOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceLongOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceLongOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceLongOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceOp.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceOp.java index d566b5e7f..cdfa8f993 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceOp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceOp.java @@ -170,11 +170,6 @@ public abstract class BaseReduceOp extends BaseOp implements ReduceOp { } } - @Override - public boolean isKeepDims() { - return keepDims; - } - public abstract List calculateOutputShape(); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceSameOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceSameOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceSameOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseReduceSameOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseScalarBoolOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseScalarBoolOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseScalarBoolOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseScalarBoolOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseScalarOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseScalarOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseScalarOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseScalarOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformAnyOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformAnyOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformAnyOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformAnyOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformBoolOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformBoolOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformBoolOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformBoolOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformFloatOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformFloatOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformFloatOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformFloatOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformSameOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformSameOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformSameOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformSameOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformStrictOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformStrictOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformStrictOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BaseTransformStrictOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BroadcastOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BroadcastOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/BroadcastOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/BroadcastOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/CustomOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/CustomOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/CustomOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/CustomOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/CustomOpDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/CustomOpDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/CustomOpDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/CustomOpDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java index dd82ebc2a..794b00eb0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java @@ -21,9 +21,9 @@ package org.nd4j.linalg.api.ops; import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.shade.guava.collect.Lists; -import org.nd4j.shade.guava.primitives.Doubles; -import org.nd4j.shade.guava.primitives.Longs; +import com.google.common.collect.Lists; +import com.google.common.primitives.Doubles; +import com.google.common.primitives.Longs; import lombok.*; import lombok.extern.slf4j.Slf4j; import onnx.Onnx; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ExecutionMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ExecutionMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ExecutionMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ExecutionMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/GridOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/GridOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/GridOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/GridOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/IndexAccumulation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/IndexAccumulation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/IndexAccumulation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/IndexAccumulation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/LossFunction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/LossFunction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/LossFunction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/LossFunction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/MetaOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/MetaOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/MetaOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/MetaOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/NoOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/NoOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/NoOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/NoOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/Op.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/Op.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/Op.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/Op.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/OpContext.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/OpContext.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/OpContext.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/OpContext.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/RandomOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/RandomOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/RandomOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/RandomOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceBoolOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceBoolOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceBoolOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceBoolOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceFloatOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceFloatOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceFloatOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceFloatOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceLongOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceLongOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceLongOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceLongOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceSameOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceSameOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ReduceSameOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ReduceSameOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ScalarOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ScalarOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/ScalarOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/ScalarOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformBoolOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformBoolOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformBoolOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformBoolOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformFloatOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformFloatOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformFloatOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformFloatOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformSameOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformSameOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformSameOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformSameOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformStrictOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformStrictOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/TransformStrictOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/TransformStrictOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Aggregate.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Aggregate.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Aggregate.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Aggregate.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/BaseAggregate.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/BaseAggregate.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/BaseAggregate.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/BaseAggregate.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Batch.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Batch.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Batch.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Batch.java index 7b2d02935..32801a893 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Batch.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/Batch.java @@ -20,7 +20,7 @@ package org.nd4j.linalg.api.ops.aggregates; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.collect.Lists; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/impl/AggregateAxpy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/impl/AggregateAxpy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/impl/AggregateAxpy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/impl/AggregateAxpy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/impl/AggregateGEMM.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/impl/AggregateGEMM.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/impl/AggregateGEMM.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/aggregates/impl/AggregateGEMM.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compat/CompatSparseToDense.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compat/CompatSparseToDense.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compat/CompatSparseToDense.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compat/CompatSparseToDense.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compat/CompatStringSplit.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compat/CompatStringSplit.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compat/CompatStringSplit.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compat/CompatStringSplit.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/DecodeBitmap.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compression/DecodeBitmap.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/DecodeBitmap.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compression/DecodeBitmap.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/DecodeThreshold.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compression/DecodeThreshold.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/DecodeThreshold.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compression/DecodeThreshold.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeBitmap.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeBitmap.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeBitmap.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeBitmap.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeThreshold.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeThreshold.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeThreshold.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/compression/EncodeThreshold.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustContrast.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustContrast.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustContrast.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustContrast.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustHue.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustHue.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustHue.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustHue.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustSaturation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustSaturation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustSaturation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/AdjustSaturation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesEdgeForces.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesEdgeForces.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesEdgeForces.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesEdgeForces.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesHutGains.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesHutGains.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesHutGains.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesHutGains.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesHutSymmetrize.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesHutSymmetrize.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesHutSymmetrize.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BarnesHutSymmetrize.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BetaInc.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BetaInc.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BetaInc.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BetaInc.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BitCast.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BitCast.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/BitCast.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/BitCast.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/CompareAndBitpack.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/CompareAndBitpack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/CompareAndBitpack.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/CompareAndBitpack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Digamma.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Digamma.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Digamma.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Digamma.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/DivideNoNan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/DivideNoNan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/DivideNoNan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/DivideNoNan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/DrawBoundingBoxes.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/DrawBoundingBoxes.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/DrawBoundingBoxes.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/DrawBoundingBoxes.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/FakeQuantWithMinMaxVarsPerChannel.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/FakeQuantWithMinMaxVarsPerChannel.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/FakeQuantWithMinMaxVarsPerChannel.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/FakeQuantWithMinMaxVarsPerChannel.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Flatten.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Flatten.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Flatten.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Flatten.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/FusedBatchNorm.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/FusedBatchNorm.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/FusedBatchNorm.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/FusedBatchNorm.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/HsvToRgb.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/HsvToRgb.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/HsvToRgb.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/HsvToRgb.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Igamma.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Igamma.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Igamma.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Igamma.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Igammac.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Igammac.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Igammac.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Igammac.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/KnnMinDistance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/KnnMinDistance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/KnnMinDistance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/KnnMinDistance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lgamma.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lgamma.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lgamma.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lgamma.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/LinearSolve.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/LinearSolve.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/LinearSolve.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/LinearSolve.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Logdet.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Logdet.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Logdet.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Logdet.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lstsq.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lstsq.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lstsq.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lstsq.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lu.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lu.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lu.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Lu.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/MatrixBandPart.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/MatrixBandPart.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/MatrixBandPart.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/MatrixBandPart.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Polygamma.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Polygamma.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Polygamma.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Polygamma.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RandomCrop.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RandomCrop.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RandomCrop.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RandomCrop.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToGrayscale.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToGrayscale.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToGrayscale.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToGrayscale.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToHsv.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToHsv.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToHsv.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToHsv.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToYiq.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToYiq.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToYiq.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToYiq.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToYuv.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToYuv.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToYuv.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/RgbToYuv.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Roll.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Roll.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Roll.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Roll.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/SpTreeCell.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/SpTreeCell.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/SpTreeCell.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/SpTreeCell.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/ToggleBits.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/ToggleBits.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/ToggleBits.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/ToggleBits.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Tri.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Tri.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Tri.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Tri.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/TriangularSolve.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/TriangularSolve.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/TriangularSolve.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/TriangularSolve.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Triu.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Triu.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/Triu.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/Triu.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/TriuBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/TriuBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/TriuBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/TriuBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/YiqToRgb.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/YiqToRgb.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/YiqToRgb.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/YiqToRgb.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/YuvToRgb.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/YuvToRgb.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/custom/YuvToRgb.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/custom/YuvToRgb.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java index c0981548c..deb2e7cb0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/DefaultOpExecutioner.java @@ -646,7 +646,7 @@ public abstract class DefaultOpExecutioner implements OpExecutioner { @Override public TADManager getTADManager() { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("A backend implementation needs to provide a TADManager"); } /** diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/GridExecutioner.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/GridExecutioner.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/GridExecutioner.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/GridExecutioner.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpExecutioner.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpExecutioner.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpExecutioner.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpExecutioner.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpExecutionerUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpExecutionerUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpExecutionerUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpExecutionerUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpStatus.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpStatus.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpStatus.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/executioner/OpStatus.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/grid/GridDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/grid/GridDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/grid/GridDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/grid/GridDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/grid/GridPointers.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/grid/GridPointers.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/grid/GridPointers.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/grid/GridPointers.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/grid/OpDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/grid/OpDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/grid/OpDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/grid/OpDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BiasAdd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BiasAdd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BiasAdd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BiasAdd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BiasAddGrad.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BiasAddGrad.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BiasAddGrad.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BiasAddGrad.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAddOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAddOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAddOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastAddOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastCopyOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastCopyOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastCopyOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastCopyOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastDivOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastDivOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastDivOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastDivOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastGradientArgs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastGradientArgs.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastGradientArgs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastGradientArgs.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMulOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMulOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMulOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastMulOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastRDivOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastRDivOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastRDivOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastRDivOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastRSubOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastRSubOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastRSubOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastRSubOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastSubOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastSubOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastSubOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastSubOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastTo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastTo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastTo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/BroadcastTo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastEqualTo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastEqualTo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastEqualTo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastEqualTo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastGreaterThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastGreaterThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastGreaterThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastGreaterThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastGreaterThanOrEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastGreaterThanOrEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastGreaterThanOrEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastGreaterThanOrEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastLessThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastLessThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastLessThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastLessThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastLessThanOrEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastLessThanOrEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastLessThanOrEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastLessThanOrEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastNotEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastNotEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastNotEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/broadcast/bool/BroadcastNotEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/Select.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/Select.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/Select.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/Select.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/Where.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/Where.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/Where.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/Where.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/WhereNumpy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/WhereNumpy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/WhereNumpy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/WhereNumpy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/BaseCompatOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/BaseCompatOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/BaseCompatOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/BaseCompatOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Enter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Exit.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Exit.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Exit.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Exit.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/LoopCond.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/LoopCond.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/LoopCond.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/LoopCond.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Merge.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Merge.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Merge.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Merge.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/NextIteration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/NextIteration.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/NextIteration.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/NextIteration.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/StopGradient.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/StopGradient.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/StopGradient.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/StopGradient.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Switch.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Switch.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Switch.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/Switch.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/While.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/While.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/While.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/controlflow/compat/While.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/grid/BaseGridOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/grid/BaseGridOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/grid/BaseGridOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/grid/BaseGridOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/grid/FreeGridOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/grid/FreeGridOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/grid/FreeGridOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/grid/FreeGridOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/CropAndResize.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/CropAndResize.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/CropAndResize.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/CropAndResize.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ExtractImagePatches.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ExtractImagePatches.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ExtractImagePatches.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ExtractImagePatches.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ImageResize.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ImageResize.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ImageResize.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ImageResize.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppression.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppression.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppression.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppression.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppressionV3.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppressionV3.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppressionV3.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppressionV3.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppressionWithOverlaps.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppressionWithOverlaps.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppressionWithOverlaps.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/NonMaxSuppressionWithOverlaps.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeArea.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeArea.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeArea.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeArea.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeBicubic.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeBicubic.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeBicubic.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeBicubic.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeBilinear.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeBilinear.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeBilinear.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeBilinear.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeNearestNeighbor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeNearestNeighbor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeNearestNeighbor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/image/ResizeNearestNeighbor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/FirstIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/FirstIndex.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/FirstIndex.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/FirstIndex.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/LastIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/LastIndex.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/LastIndex.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/LastIndex.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgAmin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/indexaccum/custom/ArgMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/ExternalErrorsFunction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/ExternalErrorsFunction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/ExternalErrorsFunction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/ExternalErrorsFunction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/AvgPooling2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/AvgPooling2D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/AvgPooling2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/AvgPooling2D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/AvgPooling3D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/AvgPooling3D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/AvgPooling3D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/AvgPooling3D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/BatchNorm.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/BatchNorm.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/BatchNorm.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/BatchNorm.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/BatchNormDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/BatchNormDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/BatchNormDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/BatchNormDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Col2Im.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Col2Im.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Col2Im.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Col2Im.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv1D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv1D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv1D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv1D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv1DDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv1DDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv1DDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv1DDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv2D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv2D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv2DDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv2DDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv2DDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv2DDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv3D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv3D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv3D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv3D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv3DDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv3DDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv3DDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Conv3DDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2DDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2DDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2DDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2DDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2DTF.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2DTF.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2DTF.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv2DTF.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3DDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3DDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3DDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3DDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3DTF.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3DTF.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3DTF.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DeConv3DTF.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthToSpace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthToSpace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthToSpace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthToSpace.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthwiseConv2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthwiseConv2D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthwiseConv2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthwiseConv2D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthwiseConv2DBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthwiseConv2DBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthwiseConv2DBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/DepthwiseConv2DBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Im2col.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Im2col.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Im2col.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Im2col.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Im2colBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Im2colBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Im2colBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Im2colBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/LocalResponseNormalization.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/LocalResponseNormalization.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/LocalResponseNormalization.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/LocalResponseNormalization.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/LocalResponseNormalizationDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/LocalResponseNormalizationDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/LocalResponseNormalizationDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/LocalResponseNormalizationDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPoolWithArgmax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPoolWithArgmax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPoolWithArgmax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPoolWithArgmax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPooling2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPooling2D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPooling2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPooling2D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPooling3D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPooling3D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPooling3D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/MaxPooling3D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling2D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling2D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling2DDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling2DDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling2DDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling2DDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling3D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling3D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling3D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling3D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling3DDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling3DDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling3DDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Pooling3DDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SConv2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SConv2D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SConv2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SConv2D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SConv2DDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SConv2DDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SConv2DDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SConv2DDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SpaceToDepth.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SpaceToDepth.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SpaceToDepth.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/SpaceToDepth.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling2d.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling2d.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling2d.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling2d.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling2dDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling2dDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling2dDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling2dDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling3d.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling3d.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling3d.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling3d.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling3dBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling3dBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling3dBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/Upsampling3dBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/BaseConvolutionConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/BaseConvolutionConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/BaseConvolutionConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/BaseConvolutionConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv1DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv1DConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv1DConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv1DConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv2DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv2DConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv2DConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv2DConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv3DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv3DConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv3DConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Conv3DConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv2DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv2DConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv2DConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv2DConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv3DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv3DConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv3DConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/DeConv3DConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/LocalResponseNormalizationConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/LocalResponseNormalizationConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/LocalResponseNormalizationConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/LocalResponseNormalizationConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/PaddingMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/PaddingMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/PaddingMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/PaddingMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling2DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling2DConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling2DConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling2DConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling3DConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling3DConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling3DConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/convolution/config/Pooling3DConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRUBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRUBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRUBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRUBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRUCell.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRUCell.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRUCell.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/GRUCell.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMBlock.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMBlock.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMBlock.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMBlock.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMBlockCell.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMBlockCell.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMBlockCell.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMBlockCell.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMCell.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMCell.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMCell.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMCell.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayer.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayer.java index 895a7bf75..0915a11e5 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayer.java @@ -30,7 +30,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.impl.layers.recurrent.config.LSTMLayerConfig; import org.nd4j.linalg.api.ops.impl.layers.recurrent.weights.LSTMLayerWeights; -import org.nd4j.shade.guava.primitives.Booleans; +import com.google.common.primitives.Booleans; import java.util.ArrayList; import java.util.Arrays; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayerBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayerBp.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayerBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayerBp.java index 294649e3d..ded99a2cd 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayerBp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/LSTMLayerBp.java @@ -29,7 +29,7 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.impl.layers.recurrent.config.LSTMLayerConfig; import org.nd4j.linalg.api.ops.impl.layers.recurrent.weights.LSTMLayerWeights; -import org.nd4j.shade.guava.primitives.Booleans; +import com.google.common.primitives.Booleans; import java.util.ArrayList; import java.util.List; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/SRU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/SRU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/SRU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/SRU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/SRUCell.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/SRUCell.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/SRUCell.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/SRUCell.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/GRUCellConfiguration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/GRUCellConfiguration.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/GRUCellConfiguration.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/GRUCellConfiguration.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMActivations.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMActivations.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMActivations.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMActivations.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMCellConfiguration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMCellConfiguration.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMCellConfiguration.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMCellConfiguration.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMConfiguration.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMConfiguration.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMConfiguration.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMConfiguration.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMDataFormat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMDataFormat.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMDataFormat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMDataFormat.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMDirectionMode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMDirectionMode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMDirectionMode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMDirectionMode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMLayerConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMLayerConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMLayerConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/LSTMLayerConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/RnnDataFormat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/RnnDataFormat.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/RnnDataFormat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/config/RnnDataFormat.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/GRUCellOutputs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/GRUCellOutputs.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/GRUCellOutputs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/GRUCellOutputs.java index 62e3c88ba..9524a82ef 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/GRUCellOutputs.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/GRUCellOutputs.java @@ -25,7 +25,6 @@ import java.util.List; import lombok.Getter; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.GRUCell; @Getter public class GRUCellOutputs { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMCellOutputs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMCellOutputs.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMCellOutputs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMCellOutputs.java index 5d5f31a20..6949a984e 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMCellOutputs.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMCellOutputs.java @@ -25,7 +25,6 @@ import java.util.List; import lombok.Getter; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.LSTMBlockCell; @Getter public class LSTMCellOutputs { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMLayerOutputs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMLayerOutputs.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMLayerOutputs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/LSTMLayerOutputs.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRUCellOutputs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRUCellOutputs.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRUCellOutputs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRUCellOutputs.java index 2f123c8c3..a6612cc65 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRUCellOutputs.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRUCellOutputs.java @@ -25,7 +25,6 @@ import java.util.List; import lombok.Getter; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.GRUCell; @Getter public class SRUCellOutputs { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRULayerOutputs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRULayerOutputs.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRULayerOutputs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRULayerOutputs.java index 45f833580..5052c16d2 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRULayerOutputs.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/outputs/SRULayerOutputs.java @@ -27,7 +27,6 @@ import lombok.Getter; import org.nd4j.autodiff.samediff.SDIndex; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.GRUCell; @Getter public class SRULayerOutputs { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/GRUWeights.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/GRUWeights.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/GRUWeights.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/GRUWeights.java index 3fe54634c..f439ce175 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/GRUWeights.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/GRUWeights.java @@ -25,7 +25,6 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.GRUCell; @EqualsAndHashCode(callSuper = true) @Data diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMLayerWeights.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMLayerWeights.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMLayerWeights.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMLayerWeights.java index e1762622b..abec3b398 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMLayerWeights.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMLayerWeights.java @@ -26,7 +26,6 @@ import lombok.EqualsAndHashCode; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.LSTMLayer; import org.nd4j.common.util.ArrayUtil; @EqualsAndHashCode(callSuper = true) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMWeights.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMWeights.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMWeights.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMWeights.java index 9de8c2343..439c98660 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMWeights.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/LSTMWeights.java @@ -25,8 +25,6 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.LSTMBlockCell; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.LSTMLayer; @EqualsAndHashCode(callSuper = true) @Data diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/RNNWeights.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/RNNWeights.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/RNNWeights.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/RNNWeights.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/SRUWeights.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/SRUWeights.java similarity index 93% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/SRUWeights.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/SRUWeights.java index 2e5abc507..e777f01fd 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/SRUWeights.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/layers/recurrent/weights/SRUWeights.java @@ -25,8 +25,6 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.SRU; -import org.nd4j.linalg.api.ops.impl.layers.recurrent.SRUCell; @EqualsAndHashCode(callSuper = true) @Data diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/AbsoluteDifferenceLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/AbsoluteDifferenceLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/AbsoluteDifferenceLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/AbsoluteDifferenceLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/BaseLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/BaseLoss.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/BaseLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/BaseLoss.java index f76fea691..aee494750 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/BaseLoss.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/BaseLoss.java @@ -50,7 +50,7 @@ public abstract class BaseLoss extends DynamicCustomOp { addArgs(); } - protected static INDArray getWeights(INDArray weights, INDArray predictions) { + protected static INDArray getWeights(INDArray weights, INDArray predictions){ return (weights != null) ? weights : Nd4j.scalar(predictions.dataType(), 1.0); } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/CosineDistanceLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/CosineDistanceLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/CosineDistanceLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/CosineDistanceLoss.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/CtcLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/CtcLoss.java new file mode 100644 index 000000000..0b21c8aa0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/CtcLoss.java @@ -0,0 +1,60 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.linalg.api.ops.impl.loss; + +import org.nd4j.autodiff.loss.LossReduce; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.loss.bp.AbsoluteDifferenceLossBp; + +import java.util.List; + +public class CtcLoss extends BaseLoss { + + + public CtcLoss(SameDiff sameDiff, LossReduce lossReduce, SDVariable predictions, SDVariable weights, SDVariable labels){ + super(sameDiff, lossReduce, predictions, weights, labels); + } + + public CtcLoss(SameDiff sameDiff, SDVariable label, SDVariable predictions, SDVariable weights, + LossReduce lossReduce) { + this(sameDiff, lossReduce, predictions, weights, label); + } + + public CtcLoss(INDArray labels, INDArray predictions, INDArray weights, LossReduce lossReduce){ + super(lossReduce, predictions, weights, labels); + } + + public CtcLoss(){ } + + @Override + public String opName() { + return "ctc_loss"; + } + + @Override + public List doDiff(List grad){ + //No external gradient + //Args are: predictions, weights, label + return new AbsoluteDifferenceLossBp(sameDiff, lossReduce, arg(0), arg(1), arg(2)).outputs(); + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HingeLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HingeLoss.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HingeLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HingeLoss.java index 3829deaf8..d0fcf2e83 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HingeLoss.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HingeLoss.java @@ -40,7 +40,7 @@ public class HingeLoss extends BaseLoss { this(sameDiff, lossReduce, predictions, weights, labels); } - public HingeLoss(INDArray labels, INDArray predictions, INDArray weights, LossReduce lossReduce) { + public HingeLoss(INDArray labels, INDArray predictions, INDArray weights, LossReduce lossReduce){ super(lossReduce, predictions, weights, labels); } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HuberLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HuberLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HuberLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/HuberLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/L2Loss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/L2Loss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/L2Loss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/L2Loss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/LogLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/LogLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/LogLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/LogLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/LogPoissonLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/LogPoissonLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/LogPoissonLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/LogPoissonLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/MeanPairwiseSquaredErrorLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/MeanPairwiseSquaredErrorLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/MeanPairwiseSquaredErrorLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/MeanPairwiseSquaredErrorLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/MeanSquaredErrorLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/MeanSquaredErrorLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/MeanSquaredErrorLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/MeanSquaredErrorLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SigmoidCrossEntropyLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SigmoidCrossEntropyLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SigmoidCrossEntropyLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SigmoidCrossEntropyLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SoftmaxCrossEntropyLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SoftmaxCrossEntropyLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SoftmaxCrossEntropyLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SoftmaxCrossEntropyLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SoftmaxCrossEntropyWithLogitsLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SoftmaxCrossEntropyWithLogitsLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SoftmaxCrossEntropyWithLogitsLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SoftmaxCrossEntropyWithLogitsLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SparseSoftmaxCrossEntropyLossWithLogits.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SparseSoftmaxCrossEntropyLossWithLogits.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SparseSoftmaxCrossEntropyLossWithLogits.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/SparseSoftmaxCrossEntropyLossWithLogits.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/WeightedCrossEntropyLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/WeightedCrossEntropyLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/WeightedCrossEntropyLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/WeightedCrossEntropyLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/AbsoluteDifferenceLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/AbsoluteDifferenceLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/AbsoluteDifferenceLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/AbsoluteDifferenceLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/BaseLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/BaseLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/BaseLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/BaseLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CosineDistanceLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CosineDistanceLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CosineDistanceLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CosineDistanceLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CtcLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CtcLossBp.java similarity index 79% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CtcLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CtcLossBp.java index 3084f3ce8..bc5b8461d 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CtcLossBp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/CtcLossBp.java @@ -20,17 +20,17 @@ package org.nd4j.linalg.api.ops.impl.loss.bp; +import org.nd4j.autodiff.loss.LossReduce; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; -import org.nd4j.linalg.api.ops.DynamicCustomOp; import java.util.List; -public class CtcLossBp extends DynamicCustomOp { +public class CtcLossBp extends BaseLossBp { - public CtcLossBp(SameDiff sameDiff, SDVariable targetLabels,SDVariable logitInputs,SDVariable targetLabelLengths,SDVariable logitInputLengths){ - super(sameDiff,new SDVariable[]{targetLabels,logitInputs,targetLabelLengths,logitInputLengths}); + public CtcLossBp(SameDiff sameDiff, LossReduce lossReduce, SDVariable predictions, SDVariable weights, SDVariable labels){ + super(sameDiff, lossReduce, predictions, weights, labels); } public CtcLossBp(){ } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/HingeLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/HingeLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/HingeLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/HingeLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/HuberLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/HuberLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/HuberLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/HuberLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/LogLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/LogLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/LogLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/LogLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/LogPoissonLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/LogPoissonLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/LogPoissonLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/LogPoissonLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/MeanPairwiseSquaredErrorLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/MeanPairwiseSquaredErrorLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/MeanPairwiseSquaredErrorLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/MeanPairwiseSquaredErrorLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/MeanSquaredErrorLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/MeanSquaredErrorLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/MeanSquaredErrorLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/MeanSquaredErrorLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SigmoidCrossEntropyLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SigmoidCrossEntropyLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SigmoidCrossEntropyLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SigmoidCrossEntropyLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SoftmaxCrossEntropyLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SoftmaxCrossEntropyLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SoftmaxCrossEntropyLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SoftmaxCrossEntropyLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SoftmaxCrossEntropyWithLogitsLossBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SoftmaxCrossEntropyWithLogitsLossBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SoftmaxCrossEntropyWithLogitsLossBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SoftmaxCrossEntropyWithLogitsLossBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SparseSoftmaxCrossEntropyLossWithLogitsBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SparseSoftmaxCrossEntropyLossWithLogitsBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SparseSoftmaxCrossEntropyLossWithLogitsBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/loss/bp/SparseSoftmaxCrossEntropyLossWithLogitsBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/BaseMetaOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/BaseMetaOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/BaseMetaOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/BaseMetaOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/InvertedPredicateMetaOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/InvertedPredicateMetaOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/InvertedPredicateMetaOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/InvertedPredicateMetaOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/PostulateMetaOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/PostulateMetaOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/PostulateMetaOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/PostulateMetaOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/PredicateMetaOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/PredicateMetaOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/PredicateMetaOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/PredicateMetaOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/ReduceMetaOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/ReduceMetaOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/ReduceMetaOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/meta/ReduceMetaOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/nlp/CbowRound.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/nlp/CbowRound.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/nlp/CbowRound.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/nlp/CbowRound.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/nlp/SkipGramRound.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/nlp/SkipGramRound.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/nlp/SkipGramRound.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/nlp/SkipGramRound.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/HashCode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/HashCode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/HashCode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/HashCode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Mmul.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Mmul.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Mmul.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Mmul.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/MmulBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/MmulBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/MmulBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/MmulBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Moments.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Moments.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Moments.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/Moments.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/NormalizeMoments.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/NormalizeMoments.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/NormalizeMoments.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/NormalizeMoments.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/SufficientStatistics.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/SufficientStatistics.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/SufficientStatistics.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/SufficientStatistics.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/TensorMmul.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/TensorMmul.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/TensorMmul.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/TensorMmul.java index 1fcfc96c6..c80c7cfb6 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/TensorMmul.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/TensorMmul.java @@ -22,8 +22,8 @@ package org.nd4j.linalg.api.ops.impl.reduce; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.shade.guava.primitives.Ints; -import org.nd4j.shade.guava.primitives.Longs; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; import lombok.NoArgsConstructor; import lombok.val; import onnx.Onnx; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/ZeroFraction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/ZeroFraction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/ZeroFraction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/ZeroFraction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/All.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/All.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/All.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/All.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/Any.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/Any.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/Any.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/Any.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/IsInf.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/IsInf.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/IsInf.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/IsInf.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/IsNaN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/IsNaN.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/IsNaN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bool/IsNaN.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/BaseReductionBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/BaseReductionBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/BaseReductionBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/BaseReductionBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/CumProdBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/CumProdBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/CumProdBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/CumProdBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/CumSumBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/CumSumBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/CumSumBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/CumSumBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/DotBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/DotBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/DotBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/DotBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MaxBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MaxBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MaxBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MaxBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MeanBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MeanBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MeanBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MeanBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MinBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MinBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MinBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/MinBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/Norm1Bp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/Norm1Bp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/Norm1Bp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/Norm1Bp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/Norm2Bp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/Norm2Bp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/Norm2Bp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/Norm2Bp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/NormMaxBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/NormMaxBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/NormMaxBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/NormMaxBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/PowBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/PowBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/PowBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/PowBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/ProdBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/ProdBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/ProdBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/ProdBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/SquaredNormBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/SquaredNormBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/SquaredNormBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/SquaredNormBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/StandardDeviationBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/StandardDeviationBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/StandardDeviationBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/StandardDeviationBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/SumBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/SumBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/SumBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/SumBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/VarianceBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/VarianceBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/VarianceBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/bp/VarianceBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/BatchMmul.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/BatchMmul.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/BatchMmul.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/BatchMmul.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/LogSumExp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/LogSumExp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/LogSumExp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/custom/LogSumExp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/AMean.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/AMean.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/AMean.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/AMean.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Entropy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Entropy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Entropy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Entropy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/LogEntropy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/LogEntropy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/LogEntropy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/LogEntropy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Mean.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Mean.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Mean.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Mean.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Norm1.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Norm1.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Norm1.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Norm1.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Norm2.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Norm2.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Norm2.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/Norm2.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/NormMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/NormMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/NormMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/NormMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/ShannonEntropy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/ShannonEntropy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/ShannonEntropy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/ShannonEntropy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/SquaredNorm.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/SquaredNorm.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/SquaredNorm.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/floating/SquaredNorm.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/CountNonZero.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/CountNonZero.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/CountNonZero.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/CountNonZero.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/CountZero.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/CountZero.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/CountZero.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/CountZero.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/MatchCondition.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/MatchCondition.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/MatchCondition.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/longer/MatchCondition.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/AMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/AMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/AMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/AMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/AMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/AMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/AMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/AMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/ASum.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/ASum.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/ASum.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/ASum.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Max.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Max.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Max.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Max.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Min.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Min.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Min.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Min.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Prod.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Prod.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Prod.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Prod.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Sum.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Sum.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Sum.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce/same/Sum.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/BaseReduce3Op.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/BaseReduce3Op.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/BaseReduce3Op.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/BaseReduce3Op.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/CosineDistance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/CosineDistance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/CosineDistance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/CosineDistance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/CosineSimilarity.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/CosineSimilarity.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/CosineSimilarity.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/CosineSimilarity.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/Dot.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/Dot.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/Dot.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/Dot.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/EqualsWithEps.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/EqualsWithEps.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/EqualsWithEps.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/EqualsWithEps.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/EuclideanDistance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/EuclideanDistance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/EuclideanDistance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/EuclideanDistance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/HammingDistance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/HammingDistance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/HammingDistance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/HammingDistance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/JaccardDistance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/JaccardDistance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/JaccardDistance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/JaccardDistance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/ManhattanDistance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/ManhattanDistance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/ManhattanDistance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/reduce3/ManhattanDistance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/LeakyReLU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/LeakyReLU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/LeakyReLU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/LeakyReLU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/LogX.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/LogX.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/LogX.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/LogX.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/PRelu.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/PRelu.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/PRelu.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/PRelu.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Pow.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Pow.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Pow.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Pow.java index 5709321c1..662748a5c 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Pow.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Pow.java @@ -22,7 +22,6 @@ package org.nd4j.linalg.api.ops.impl.scalar; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; -import org.nd4j.imports.NoOpNameFoundException; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.BaseScalarOp; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/PowDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/PowDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/PowDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/PowDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/RectifiedLinear.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/RectifiedLinear.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/RectifiedLinear.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/RectifiedLinear.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/RectifiedLinearDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/RectifiedLinearDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/RectifiedLinearDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/RectifiedLinearDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Relu6.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Relu6.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Relu6.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Relu6.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ReplaceNans.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ReplaceNans.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ReplaceNans.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ReplaceNans.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarAdd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarAdd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarAdd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarAdd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarDivision.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarDivision.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarDivision.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarDivision.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarFMod.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarFMod.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarFMod.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarFMod.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMultiplication.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMultiplication.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMultiplication.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarMultiplication.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarRemainder.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarRemainder.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarRemainder.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarRemainder.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarReverseDivision.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarReverseDivision.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarReverseDivision.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarReverseDivision.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarReverseSubtraction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarReverseSubtraction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarReverseSubtraction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarReverseSubtraction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarSet.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarSet.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarSet.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarSet.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarSubtraction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarSubtraction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarSubtraction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/ScalarSubtraction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Step.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Step.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Step.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/Step.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarAnd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarAnd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarAnd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarAnd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarEps.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarEps.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarEps.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarEps.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarEquals.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarEquals.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarEquals.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarEquals.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarGreaterThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarGreaterThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarGreaterThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarGreaterThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarGreaterThanOrEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarGreaterThanOrEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarGreaterThanOrEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarGreaterThanOrEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarLessThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarLessThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarLessThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarLessThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarLessThanOrEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarLessThanOrEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarLessThanOrEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarLessThanOrEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarNot.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarNot.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarNot.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarNot.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarNotEquals.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarNotEquals.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarNotEquals.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarNotEquals.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarOr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarOr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarOr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarOr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarSetValue.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarSetValue.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarSetValue.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarSetValue.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarXor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarXor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarXor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scalar/comparison/ScalarXor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterAdd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterAdd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterAdd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterAdd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterDiv.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterDiv.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterDiv.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterDiv.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMul.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMul.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMul.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterMul.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdAdd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdAdd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdAdd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdAdd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdSub.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdSub.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdSub.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdSub.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdUpdate.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdUpdate.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdUpdate.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterNdUpdate.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterSub.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterSub.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterSub.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterSub.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterUpdate.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterUpdate.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterUpdate.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/scatter/ScatterUpdate.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ApplyGradientDescent.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ApplyGradientDescent.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ApplyGradientDescent.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ApplyGradientDescent.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/BroadcastDynamicShape.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/BroadcastDynamicShape.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/BroadcastDynamicShape.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/BroadcastDynamicShape.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Concat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Concat.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Concat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Concat.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ConfusionMatrix.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ConfusionMatrix.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ConfusionMatrix.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ConfusionMatrix.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Create.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Create.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Create.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Create.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Cross.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Cross.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Cross.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Cross.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Diag.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Diag.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Diag.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Diag.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/DiagPart.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/DiagPart.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/DiagPart.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/DiagPart.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ExpandDims.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ExpandDims.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ExpandDims.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ExpandDims.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Eye.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Eye.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Eye.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Eye.java index d52e802d2..8b2b7e6a0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Eye.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Eye.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.shape.LongShapeDescriptor; -import org.nd4j.shade.guava.base.Preconditions; +import com.google.common.base.Preconditions; import java.util.Collections; import java.util.List; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Flatten2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Flatten2D.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Flatten2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Flatten2D.java index 42e655468..e0e012e86 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Flatten2D.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Flatten2D.java @@ -21,7 +21,6 @@ package org.nd4j.linalg.api.ops.impl.shape; import lombok.NoArgsConstructor; -import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; import onnx.Onnx; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Gather.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Gather.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Gather.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Gather.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/GatherNd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/GatherNd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/GatherNd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/GatherNd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Linspace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Linspace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Linspace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Linspace.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeAvg.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeAvg.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeAvg.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeAvg.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeMaxIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeMaxIndex.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeMaxIndex.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeMaxIndex.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeSum.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeSum.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeSum.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MergeSum.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MeshGrid.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MeshGrid.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MeshGrid.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/MeshGrid.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/OneHot.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/OneHot.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/OneHot.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/OneHot.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/OnesLike.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/OnesLike.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/OnesLike.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/OnesLike.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ParallelStack.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ParallelStack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ParallelStack.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ParallelStack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Permute.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Permute.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Permute.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Permute.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Rank.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Rank.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Rank.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Rank.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ReductionShape.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ReductionShape.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ReductionShape.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ReductionShape.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Repeat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Repeat.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Repeat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Repeat.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Reshape.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Reshape.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Reshape.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Reshape.java index e0cff5ab3..977241e23 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Reshape.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Reshape.java @@ -89,7 +89,9 @@ public class Reshape extends DynamicCustomOp { return; } else if(nodeDef.getInputCount() == 1){ val shape = nodeDef.getAttrOrThrow("Tshape"); - if (!shape.hasShape()) { + + if (!shape.isInitialized()) { + //FIXME was: if (!shape.hasShape()) { val shapeRet = new long[2]; shapeRet[0] = 1; shapeRet[1] = shape.getValueCase().getNumber(); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SequenceMask.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SequenceMask.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SequenceMask.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SequenceMask.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Shape.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Shape.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Shape.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Shape.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ShapeN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ShapeN.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ShapeN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ShapeN.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Size.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Size.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Size.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Size.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SizeAt.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SizeAt.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SizeAt.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SizeAt.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Slice.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Slice.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Slice.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Slice.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Split.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Split.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Split.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Split.java index 8325c074d..f6856c0e8 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Split.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Split.java @@ -58,11 +58,6 @@ public class Split extends DynamicCustomOp { super(null, new INDArray[]{in}, wrapOrNull(out), null, (List)null); } - public Split(INDArray input, int numSplit, int splitDim) { - super(null,input,null,Collections.emptyList(),new int[0]); - addIArgument(numSplit,splitDim); - } - @Override public String opName() { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SplitV.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SplitV.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SplitV.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/SplitV.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Squeeze.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Squeeze.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Squeeze.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Squeeze.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Stack.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Stack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Stack.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Stack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/StridedSlice.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/StridedSlice.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/StridedSlice.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/StridedSlice.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Tile.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Tile.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Tile.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Tile.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Transpose.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Transpose.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Transpose.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Transpose.java index 21a4fc2d7..c3abb1059 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Transpose.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Transpose.java @@ -20,7 +20,7 @@ package org.nd4j.linalg.api.ops.impl.shape; -import org.nd4j.shade.guava.primitives.Ints; +import com.google.common.primitives.Ints; import lombok.val; import onnx.Onnx; import org.nd4j.autodiff.samediff.SDVariable; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Unstack.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Unstack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Unstack.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/Unstack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ZerosLike.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ZerosLike.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ZerosLike.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/ZerosLike.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/ConcatBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/ConcatBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/ConcatBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/ConcatBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/MergeAvgBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/MergeAvgBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/MergeAvgBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/MergeAvgBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/MergeMaxBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/MergeMaxBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/MergeMaxBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/MergeMaxBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/SliceBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/SliceBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/SliceBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/SliceBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/StridedSliceBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/StridedSliceBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/StridedSliceBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/StridedSliceBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/TileBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/TileBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/TileBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/bp/TileBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/BaseTensorOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/BaseTensorOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/BaseTensorOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/BaseTensorOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/EmbeddingLookup.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/EmbeddingLookup.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/EmbeddingLookup.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/EmbeddingLookup.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArray.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArray.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArray.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArray.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayConcat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayConcat.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayConcat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayConcat.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayGather.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayGather.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayGather.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayGather.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayRead.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayRead.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayRead.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayRead.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayScatter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayScatter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayScatter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayScatter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArraySize.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArraySize.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArraySize.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArraySize.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArraySplit.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArraySplit.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArraySplit.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArraySplit.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayWrite.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayWrite.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayWrite.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArrayWrite.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/summarystats/StandardDeviation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/summarystats/StandardDeviation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/summarystats/StandardDeviation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/summarystats/StandardDeviation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/summarystats/Variance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/summarystats/Variance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/summarystats/Variance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/summarystats/Variance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Angle.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Angle.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Angle.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Angle.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Assert.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Assert.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Assert.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Assert.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/BaseDynamicTransformOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/BaseDynamicTransformOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/BaseDynamicTransformOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/BaseDynamicTransformOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/BinCount.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/BinCount.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/BinCount.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/BinCount.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/CheckNumerics.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/CheckNumerics.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/CheckNumerics.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/CheckNumerics.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Cholesky.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Cholesky.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Cholesky.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Cholesky.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Histogram.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Histogram.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Histogram.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Histogram.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/HistogramFixedWidth.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/HistogramFixedWidth.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/HistogramFixedWidth.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/HistogramFixedWidth.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/IdentityN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/IdentityN.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/IdentityN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/IdentityN.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/MaxOut.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/MaxOut.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/MaxOut.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/MaxOut.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/NthElement.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/NthElement.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/NthElement.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/NthElement.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Pad.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Pad.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Pad.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/Pad.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/ReluLayer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/ReluLayer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/ReluLayer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/ReluLayer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/any/Assign.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/any/Assign.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/any/Assign.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/any/Assign.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/any/IsMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/any/IsMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/any/IsMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/any/IsMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/BooleanNot.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/BooleanNot.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/BooleanNot.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/BooleanNot.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsFinite.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsFinite.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsFinite.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsFinite.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsInf.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsInf.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsInf.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsInf.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsNaN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsNaN.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsNaN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/IsNaN.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/MatchConditionTransform.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/MatchConditionTransform.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/MatchConditionTransform.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/bool/MatchConditionTransform.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByAvgNorm.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByAvgNorm.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByAvgNorm.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByAvgNorm.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByNorm.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByNorm.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByNorm.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByNorm.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByNormBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByNormBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByNormBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByNormBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByValue.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByValue.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByValue.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/clip/ClipByValue.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/CompareAndReplace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/CompareAndReplace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/CompareAndReplace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/CompareAndReplace.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/CompareAndSet.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/CompareAndSet.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/CompareAndSet.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/CompareAndSet.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/Eps.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/Eps.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/Eps.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/comparison/Eps.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ATan2.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ATan2.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ATan2.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ATan2.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Assign.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Assign.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Assign.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Assign.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BatchToSpace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BatchToSpace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BatchToSpace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BatchToSpace.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BatchToSpaceND.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BatchToSpaceND.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BatchToSpaceND.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BatchToSpaceND.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitsHammingDistance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitsHammingDistance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitsHammingDistance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitsHammingDistance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseAnd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseAnd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseAnd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseAnd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseOr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseOr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseOr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseOr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseXor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseXor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseXor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/BitwiseXor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CReLU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CReLU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CReLU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CReLU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CReluBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CReluBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CReluBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CReluBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Choose.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Choose.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Choose.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Choose.java index 315356ccc..69665508a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Choose.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Choose.java @@ -20,8 +20,8 @@ package org.nd4j.linalg.api.ops.impl.transforms.custom; -import org.nd4j.shade.guava.primitives.Doubles; -import org.nd4j.shade.guava.primitives.Ints; +import com.google.common.primitives.Doubles; +import com.google.common.primitives.Ints; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.common.base.Preconditions; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CumProd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CumProd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CumProd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CumProd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CumSum.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CumSum.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CumSum.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CumSum.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CyclicRShiftBits.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CyclicRShiftBits.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CyclicRShiftBits.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CyclicRShiftBits.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CyclicShiftBits.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CyclicShiftBits.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CyclicShiftBits.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/CyclicShiftBits.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Dilation2D.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Dilation2D.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Dilation2D.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Dilation2D.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DotProductAttention.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DotProductAttention.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DotProductAttention.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DotProductAttention.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DotProductAttentionBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DotProductAttentionBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DotProductAttentionBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DotProductAttentionBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DynamicPartition.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DynamicPartition.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DynamicPartition.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DynamicPartition.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DynamicStitch.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DynamicStitch.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DynamicStitch.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/DynamicStitch.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/EqualTo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/EqualTo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/EqualTo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/EqualTo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/FakeQuantWithMinMaxArgs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/FakeQuantWithMinMaxArgs.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/FakeQuantWithMinMaxArgs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/FakeQuantWithMinMaxArgs.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/FakeQuantWithMinMaxVars.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/FakeQuantWithMinMaxVars.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/FakeQuantWithMinMaxVars.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/FakeQuantWithMinMaxVars.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Fill.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Fill.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Fill.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Fill.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/GreaterThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/GreaterThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/GreaterThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/GreaterThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/GreaterThanOrEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/GreaterThanOrEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/GreaterThanOrEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/GreaterThanOrEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/InTopK.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/InTopK.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/InTopK.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/InTopK.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/InvertPermutation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/InvertPermutation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/InvertPermutation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/InvertPermutation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsNonDecreasing.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsNonDecreasing.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsNonDecreasing.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsNonDecreasing.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsNumericTensor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsNumericTensor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsNumericTensor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsNumericTensor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsStrictlyIncreasing.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsStrictlyIncreasing.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsStrictlyIncreasing.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/IsStrictlyIncreasing.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNorm.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNorm.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNorm.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNorm.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNormBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNormBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNormBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LayerNormBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LessThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LessThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LessThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LessThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LessThanOrEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LessThanOrEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LessThanOrEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LessThanOrEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ListDiff.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ListDiff.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ListDiff.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ListDiff.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogMatrixDeterminant.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogMatrixDeterminant.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogMatrixDeterminant.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogMatrixDeterminant.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogSoftMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogSoftMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogSoftMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogSoftMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalAnd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalAnd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalAnd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalAnd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalNot.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalNot.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalNot.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalNot.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalOr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalOr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalOr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalOr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalXor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalXor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalXor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/LogicalXor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDeterminant.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDeterminant.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDeterminant.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDeterminant.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDiag.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDiag.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDiag.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDiag.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDiagPart.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDiagPart.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDiagPart.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixDiagPart.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixInverse.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixInverse.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixInverse.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixInverse.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixSetDiag.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixSetDiag.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixSetDiag.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MatrixSetDiag.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Max.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Max.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Max.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Max.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MaximumBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MaximumBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MaximumBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MaximumBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Min.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Min.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Min.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Min.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MirrorPad.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MirrorPad.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MirrorPad.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MirrorPad.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MultiHeadDotProductAttention.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MultiHeadDotProductAttention.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MultiHeadDotProductAttention.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MultiHeadDotProductAttention.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MultiHeadDotProductAttentionBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MultiHeadDotProductAttentionBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MultiHeadDotProductAttentionBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/MultiHeadDotProductAttentionBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/NotEqualTo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/NotEqualTo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/NotEqualTo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/NotEqualTo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ParallelConcat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ParallelConcat.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ParallelConcat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ParallelConcat.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Pow.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Pow.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Pow.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Pow.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Qr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Qr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Qr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Qr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/RShiftBits.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/RShiftBits.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/RShiftBits.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/RShiftBits.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Reverse.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Reverse.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Reverse.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Reverse.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseSequence.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseSequence.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseSequence.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseSequence.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseV2.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseV2.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseV2.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ReverseV2.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ShiftBits.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ShiftBits.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ShiftBits.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ShiftBits.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SoftMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SoftMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SoftMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SoftMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SpaceToBatch.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SpaceToBatch.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SpaceToBatch.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SpaceToBatch.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SpaceToBatchND.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SpaceToBatchND.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SpaceToBatchND.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/SpaceToBatchND.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Standardize.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Standardize.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Standardize.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Standardize.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/StandardizeBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/StandardizeBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/StandardizeBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/StandardizeBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Svd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Svd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Svd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Svd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ThresholdRelu.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ThresholdRelu.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ThresholdRelu.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ThresholdRelu.java index fc5d4422f..42b3d7543 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ThresholdRelu.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/ThresholdRelu.java @@ -30,7 +30,6 @@ import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.scalar.RectifiedLinear; import org.nd4j.linalg.api.ops.impl.transforms.gradient.ThresholdReluBp; public class ThresholdRelu extends DynamicCustomOp { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/TopK.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/TopK.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/TopK.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/TopK.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Trace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Trace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Trace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Trace.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Unique.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Unique.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Unique.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Unique.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/UniqueWithCounts.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/UniqueWithCounts.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/UniqueWithCounts.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/UniqueWithCounts.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/XwPlusB.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/XwPlusB.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/XwPlusB.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/XwPlusB.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Zeta.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Zeta.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Zeta.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/Zeta.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMean.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMean.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMean.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMean.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentProd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentProd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentProd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentProd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentSum.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentSum.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentSum.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/custom/segment/SegmentSum.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/dtype/Cast.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/dtype/Cast.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/dtype/Cast.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/dtype/Cast.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/floating/RSqrt.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/floating/RSqrt.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/floating/RSqrt.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/floating/RSqrt.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/floating/Sqrt.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/floating/Sqrt.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/floating/Sqrt.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/floating/Sqrt.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/CubeBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/CubeBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/CubeBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/CubeBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/CubeDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/CubeDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/CubeDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/CubeDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/DynamicPartitionBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/DynamicPartitionBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/DynamicPartitionBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/DynamicPartitionBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/EluBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/EluBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/EluBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/EluBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/GradientBackwardsMarker.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/GradientBackwardsMarker.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/GradientBackwardsMarker.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/GradientBackwardsMarker.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardSigmoidBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardSigmoidBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardSigmoidBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardSigmoidBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardSigmoidDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardSigmoidDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardSigmoidDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardSigmoidDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardTanhBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardTanhBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardTanhBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardTanhBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardTanhDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardTanhDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardTanhDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/HardTanhDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LeakyReLUBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LeakyReLUBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LeakyReLUBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LeakyReLUBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LeakyReLUDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LeakyReLUDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LeakyReLUDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LeakyReLUDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LogSoftMaxDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LogSoftMaxDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LogSoftMaxDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/LogSoftMaxDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/PReluBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/PReluBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/PReluBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/PReluBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RationalTanhBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RationalTanhBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RationalTanhBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RationalTanhBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RationalTanhDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RationalTanhDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RationalTanhDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RationalTanhDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RectifiedTanhBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RectifiedTanhBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RectifiedTanhBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RectifiedTanhBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RectifiedTanhDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RectifiedTanhDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RectifiedTanhDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/RectifiedTanhDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/Relu6Derivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/Relu6Derivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/Relu6Derivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/Relu6Derivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SELUDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SELUDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SELUDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SELUDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SeluBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SeluBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SeluBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SeluBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SigmoidDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SigmoidDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SigmoidDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SigmoidDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftPlusBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftPlusBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftPlusBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftPlusBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftSignBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftSignBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftSignBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftSignBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftSignDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftSignDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftSignDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftSignDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftmaxBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftmaxBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftmaxBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/SoftmaxBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/TanhDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/TanhDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/TanhDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/TanhDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/ThresholdReluBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/ThresholdReluBp.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/ThresholdReluBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/ThresholdReluBp.java index 723910602..537d6466e 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/ThresholdReluBp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/gradient/ThresholdReluBp.java @@ -30,8 +30,6 @@ import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.DynamicCustomOp; -import org.nd4j.linalg.api.ops.impl.scalar.RectifiedLinear; -import org.nd4j.linalg.api.ops.impl.transforms.custom.ThresholdRelu; public class ThresholdReluBp extends DynamicCustomOp { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/BinaryMinimalRelativeError.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/BinaryMinimalRelativeError.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/BinaryMinimalRelativeError.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/BinaryMinimalRelativeError.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/BinaryRelativeError.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/BinaryRelativeError.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/BinaryRelativeError.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/BinaryRelativeError.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/RelativeError.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/RelativeError.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/RelativeError.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/RelativeError.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/Set.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/Set.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/Set.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/Set.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/AddOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/AddOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/AddOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/AddOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/Axpy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/Axpy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/Axpy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/Axpy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/CopyOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/CopyOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/CopyOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/CopyOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/DivOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/DivOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/DivOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/DivOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FModOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FModOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FModOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FModOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FloorDivOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FloorDivOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FloorDivOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FloorDivOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FloorModOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FloorModOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FloorModOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/FloorModOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/MergeAddOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/MergeAddOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/MergeAddOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/MergeAddOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/ModOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/ModOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/ModOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/ModOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/MulOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/MulOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/MulOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/MulOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/PowPairwise.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/PowPairwise.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/PowPairwise.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/PowPairwise.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RDivOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RDivOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RDivOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RDivOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RSubOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RSubOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RSubOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RSubOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RealDivOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RealDivOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RealDivOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RealDivOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RemainderOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RemainderOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RemainderOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/RemainderOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/SquaredDifferenceOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/SquaredDifferenceOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/SquaredDifferenceOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/SquaredDifferenceOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/SubOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/SubOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/SubOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/SubOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/TruncateDivOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/TruncateDivOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/TruncateDivOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/TruncateDivOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/AddBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/AddBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/AddBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/AddBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/BaseArithmeticBackpropOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/BaseArithmeticBackpropOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/BaseArithmeticBackpropOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/BaseArithmeticBackpropOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/DivBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/DivBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/DivBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/DivBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/FloorDivBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/FloorDivBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/FloorDivBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/FloorDivBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/FloorModBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/FloorModBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/FloorModBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/FloorModBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/MergeAddBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/MergeAddBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/MergeAddBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/MergeAddBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/ModBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/ModBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/ModBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/ModBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/MulBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/MulBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/MulBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/MulBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/RDivBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/RDivBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/RDivBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/RDivBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/RSubBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/RSubBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/RSubBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/RSubBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/SquaredDifferenceBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/SquaredDifferenceBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/SquaredDifferenceBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/SquaredDifferenceBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/SubBpOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/SubBpOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/SubBpOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/arithmetic/bp/SubBpOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/And.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/And.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/And.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/And.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Not.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Not.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Not.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Not.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Or.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Or.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Or.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Or.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Xor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Xor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Xor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/pairwise/bool/Xor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/AMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/AMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/AMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/AMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/AMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/AMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/AMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/AMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Abs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Abs.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Abs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Abs.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Ceil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Ceil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Ceil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Ceil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Cube.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Cube.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Cube.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Cube.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Floor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Floor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Floor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Floor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Identity.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Identity.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Identity.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Identity.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Max.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Max.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Max.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Max.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Min.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Min.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Min.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Min.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Negative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Negative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Negative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Negative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/OneMinus.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/OneMinus.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/OneMinus.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/OneMinus.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Reciprocal.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Reciprocal.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Reciprocal.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Reciprocal.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Round.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Round.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Round.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Round.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Sign.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Sign.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Sign.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Sign.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Square.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Square.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Square.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/Square.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/TimesOneMinus.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/TimesOneMinus.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/TimesOneMinus.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/same/TimesOneMinus.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMax.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMax.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMean.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMean.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMean.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMean.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentMin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentProd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentProd.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentProd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentProd.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSqrtN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSqrtN.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSqrtN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSqrtN.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSum.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSum.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSum.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSum.java index 7cc62d163..b6de7fee7 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSum.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/UnsortedSegmentSum.java @@ -28,7 +28,6 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.impl.transforms.segment.bp.UnsortedSegmentSumBp; -import org.nd4j.linalg.factory.Nd4j; import java.util.Collections; import java.util.List; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMaxBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMaxBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMaxBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMaxBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMeanBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMeanBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMeanBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMeanBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMinBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMinBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMinBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentMinBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentProdBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentProdBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentProdBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentProdBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentSumBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentSumBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentSumBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/SegmentSumBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMaxBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMaxBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMaxBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMaxBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMeanBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMeanBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMeanBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMeanBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMinBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMinBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMinBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentMinBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentProdBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentProdBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentProdBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentProdBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentSqrtNBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentSqrtNBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentSqrtNBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentSqrtNBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentSumBp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentSumBp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentSumBp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/segment/bp/UnsortedSegmentSumBp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ACos.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ACos.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ACos.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ACos.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ACosh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ACosh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ACosh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ACosh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ASin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ASin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ASin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ASin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ASinh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ASinh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ASinh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ASinh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ATan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ATan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ATan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ATan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ATanh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ATanh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ATanh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ATanh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Cos.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Cos.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Cos.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Cos.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Cosh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Cosh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Cosh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Cosh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ELU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ELU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ELU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/ELU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Erf.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Erf.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Erf.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Erf.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Erfc.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Erfc.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Erfc.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Erfc.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Exp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Exp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Exp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Exp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Expm1.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Expm1.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Expm1.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Expm1.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/GELU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/GELU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/GELU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/GELU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/GELUDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/GELUDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/GELUDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/GELUDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/HardSigmoid.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/HardSigmoid.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/HardSigmoid.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/HardSigmoid.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/HardTanh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/HardTanh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/HardTanh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/HardTanh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Log.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Log.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Log.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Log.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Log1p.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Log1p.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Log1p.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Log1p.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/LogSigmoid.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/LogSigmoid.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/LogSigmoid.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/LogSigmoid.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Mish.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Mish.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Mish.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Mish.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/MishDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/MishDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/MishDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/MishDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/PreciseGELU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/PreciseGELU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/PreciseGELU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/PreciseGELU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/PreciseGELUDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/PreciseGELUDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/PreciseGELUDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/PreciseGELUDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/RationalTanh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/RationalTanh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/RationalTanh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/RationalTanh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/RectifiedTanh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/RectifiedTanh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/RectifiedTanh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/RectifiedTanh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Rint.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Rint.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Rint.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Rint.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SELU.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SELU.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SELU.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SELU.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SetRange.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SetRange.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SetRange.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SetRange.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sigmoid.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sigmoid.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sigmoid.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sigmoid.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SigmoidDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SigmoidDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SigmoidDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SigmoidDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sin.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sin.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sin.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sin.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sinh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sinh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sinh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Sinh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SoftPlus.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SoftPlus.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SoftPlus.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SoftPlus.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SoftSign.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SoftSign.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SoftSign.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SoftSign.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Stabilize.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Stabilize.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Stabilize.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Stabilize.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Swish.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Swish.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Swish.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Swish.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SwishDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SwishDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SwishDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/SwishDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Tan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Tan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Tan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Tan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/TanDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/TanDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/TanDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/TanDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Tanh.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Tanh.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Tanh.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/Tanh.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/TanhDerivative.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/TanhDerivative.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/TanhDerivative.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/transforms/strict/TanhDerivative.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaBeliefUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaBeliefUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaBeliefUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaBeliefUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaDeltaUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaDeltaUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaDeltaUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaDeltaUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaGradUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaGradUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaGradUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaGradUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaMaxUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaMaxUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaMaxUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdaMaxUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdamUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdamUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdamUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AdamUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AmsGradUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AmsGradUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AmsGradUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/AmsGradUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/NadamUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/NadamUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/NadamUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/NadamUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/NesterovsUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/NesterovsUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/NesterovsUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/NesterovsUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/RmsPropUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/RmsPropUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/RmsPropUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/RmsPropUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/SgdUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/SgdUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/SgdUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/impl/updaters/SgdUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/performance/PerformanceTracker.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/performance/PerformanceTracker.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/performance/PerformanceTracker.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/performance/PerformanceTracker.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/performance/primitives/AveragingTransactionsHolder.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/performance/primitives/AveragingTransactionsHolder.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/performance/primitives/AveragingTransactionsHolder.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/performance/primitives/AveragingTransactionsHolder.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/persistence/RestoreV2.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/persistence/RestoreV2.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/persistence/RestoreV2.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/persistence/RestoreV2.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/persistence/SaveV2.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/persistence/SaveV2.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/persistence/SaveV2.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/persistence/SaveV2.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/BaseRandomOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/BaseRandomOp.java similarity index 88% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/BaseRandomOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/BaseRandomOp.java index 45bd84324..6d82da9cd 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/BaseRandomOp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/BaseRandomOp.java @@ -27,7 +27,6 @@ import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.BaseOp; -import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.RandomOp; import org.nd4j.linalg.api.shape.LongShapeDescriptor; @@ -46,10 +45,6 @@ public abstract class BaseRandomOp extends BaseOp implements RandomOp { Preconditions.checkNotNull(i_v, "Input variable can't be null with this constructor"); this.sameDiff = sameDiff; this.xVertexId = i_v.name(); - if(i_v.getShape() != null) - this.shape = i_v.getShape(); - else if(i_v.getArr().shape() != null) - this.shape = i_v.getArr().shape(); sameDiff.addArgsFor(new String[]{xVertexId},this); } @@ -76,7 +71,14 @@ public abstract class BaseRandomOp extends BaseOp implements RandomOp { return calculateOutputShape(null); } - + @Override + public List calculateOutputShape(OpContext opContext) { + if(shape != null){ + return Collections.singletonList(LongShapeDescriptor.fromShape(shape, dataType)); + } else { + return Collections.singletonList(LongShapeDescriptor.fromShape(shape, Shape.pickPairwiseDataType(args()[0].dataType(), Nd4j.dataType()))); + } + } @Override public List calculateOutputDataTypes(List inputDataTypes) { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/compat/RandomStandardNormal.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/compat/RandomStandardNormal.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/compat/RandomStandardNormal.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/compat/RandomStandardNormal.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/DistributionUniform.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/DistributionUniform.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/DistributionUniform.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/DistributionUniform.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomBernoulli.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomBernoulli.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomBernoulli.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomBernoulli.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomExponential.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomExponential.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomExponential.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomExponential.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomGamma.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomGamma.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomGamma.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomGamma.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomNormal.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomNormal.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomNormal.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomNormal.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomPoisson.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomPoisson.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomPoisson.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomPoisson.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomShuffle.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomShuffle.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomShuffle.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/custom/RandomShuffle.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/AlphaDropOut.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/AlphaDropOut.java similarity index 83% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/AlphaDropOut.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/AlphaDropOut.java index 68c7cfa24..0189376c3 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/AlphaDropOut.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/AlphaDropOut.java @@ -24,11 +24,8 @@ import lombok.NonNull; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.imports.NoOpNameFoundException; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.random.BaseRandomOp; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; -import java.util.Arrays; import java.util.List; public class AlphaDropOut extends BaseRandomOp { @@ -75,17 +72,6 @@ public class AlphaDropOut extends BaseRandomOp { throw new NoOpNameFoundException("No tensorflow op opName found for " + opName()); } - @Override - public List calculateOutputShape(OpContext oc) { - return calculateOutputShape(); - } - - @Override - public List calculateOutputShape() { - LongShapeDescriptor longShapeDescriptor = LongShapeDescriptor.fromShape(shape,dataType); - return Arrays.asList(longShapeDescriptor); - } - @Override public List doDiff(List f1) { return null; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BernoulliDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BernoulliDistribution.java similarity index 89% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BernoulliDistribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BernoulliDistribution.java index 67552c12e..b50de8980 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BernoulliDistribution.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BernoulliDistribution.java @@ -27,13 +27,10 @@ import org.nd4j.common.base.Preconditions; import org.nd4j.imports.NoOpNameFoundException; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.random.BaseRandomOp; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -111,16 +108,7 @@ public class BernoulliDistribution extends BaseRandomOp { throw new NoOpNameFoundException("No tensorflow op opName found for " + opName()); } - @Override - public List calculateOutputShape(OpContext oc) { - return calculateOutputShape(); - } - @Override - public List calculateOutputShape() { - LongShapeDescriptor longShapeDescriptor = LongShapeDescriptor.fromShape(shape,dataType); - return Arrays.asList(longShapeDescriptor); - } @Override public List doDiff(List f1) { return Collections.emptyList(); //No SDVariable args diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistribution.java new file mode 100644 index 000000000..e29c00c56 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistribution.java @@ -0,0 +1,151 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.linalg.api.ops.random.impl; + +import lombok.NonNull; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.base.Preconditions; +import org.nd4j.imports.NoOpNameFoundException; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.BaseRandomOp; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collections; +import java.util.List; + +public class BinomialDistribution extends BaseRandomOp { + private int trials; + private double probability; + + public BinomialDistribution(SameDiff sd, int trials, double probability, long[] shape){ + super(sd, shape); + this.trials = trials; + this.probability = probability; + this.extraArgs = new Object[] {(double) this.trials, this.probability}; + } + + public BinomialDistribution(SameDiff sd, int trials, double probability, DataType dataType, long[] shape){ + this(sd, trials, probability, shape); + super.dataType = dataType; + } + + public BinomialDistribution(int trials, double probability, DataType dt, long[] shape){ + this(Nd4j.createUninitialized(dt, shape), trials, probability); + } + + public BinomialDistribution() { + super(); + } + + /** + * This op fills Z with binomial distribution over given trials with single given probability for all trials + * @param z + * @param trials + * @param probability + */ + public BinomialDistribution(@NonNull INDArray z, int trials, double probability) { + super(z, z, z); + this.trials = trials; + this.probability = probability; + this.extraArgs = new Object[] {(double) this.trials, this.probability}; + } + + /** + * This op fills Z with binomial distribution over given trials with probability for each trial given as probabilities INDArray + * @param z + * @param trials + * @param probabilities array with probability value for each trial + */ + public BinomialDistribution(@NonNull INDArray z, int trials, @NonNull INDArray probabilities) { + super(z, probabilities, z); + if (trials > probabilities.length()) + throw new IllegalStateException("Number of trials is > then amount of probabilities provided"); + + if (probabilities.elementWiseStride() < 1) + throw new IllegalStateException("Probabilities array shouldn't have negative elementWiseStride"); + + Preconditions.checkArgument(probabilities.dataType() == z.dataType(), "Probabilities and Z operand should have same data type"); + + this.trials = trials; + this.probability = 0.0; + this.extraArgs = new Object[] {(double) this.trials, this.probability}; + } + + /** + * This op fills Z with binomial distribution over given trials with probability for each trial given as probabilities INDArray + * + * @param z + * @param probabilities + */ + public BinomialDistribution(@NonNull INDArray z, @NonNull INDArray probabilities) { + this(z, (int) probabilities.length(), probabilities); + } + + @Override + public int opNum() { + return 8; + } + + @Override + public String opName() { + return "distribution_binomial"; + } + + @Override + public String onnxName() { + throw new NoOpNameFoundException("No onnx op opName found for " + opName()); + } + + @Override + public String tensorflowName() { + throw new NoOpNameFoundException("No tensorflow op opName found for " + opName()); + } + + + + @Override + public List doDiff(List f1) { + return Collections.emptyList(); + } + + @Override + public void setZ(INDArray z){ + //We want all 3 args set to z for this op + this.x = z; + this.y = z; + this.z = z; + } + + @Override + public List calculateOutputDataTypes(List inputDataTypes){ + Preconditions.checkState(inputDataTypes == null || inputDataTypes.isEmpty(), "Expected no input datatypes (no args) for %s, got %s", getClass(), inputDataTypes); + //Input data type specifies the shape; output data type should be any float + //TODO MAKE CONFIGUREABLE - https://github.com/eclipse/deeplearning4j/issues/6854 + return Collections.singletonList(DataType.DOUBLE); + } + + @Override + public boolean isTripleArgRngOp() { + return true; + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistributionEx.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistributionEx.java similarity index 86% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistributionEx.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistributionEx.java index 6694d1ca6..ecc65c132 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistributionEx.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/BinomialDistributionEx.java @@ -25,11 +25,8 @@ import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.common.base.Preconditions; import org.nd4j.imports.NoOpNameFoundException; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.random.BaseRandomOp; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; -import java.util.Arrays; import java.util.List; public class BinomialDistributionEx extends BaseRandomOp { @@ -108,17 +105,6 @@ public class BinomialDistributionEx extends BaseRandomOp { @Override public List doDiff(List f1) { - throw new UnsupportedOperationException("BinomialDistributionEx does not have a derivative."); - } - - @Override - public List calculateOutputShape(OpContext oc) { - return calculateOutputShape(); - } - - @Override - public List calculateOutputShape() { - LongShapeDescriptor longShapeDescriptor = LongShapeDescriptor.fromShape(shape,dataType); - return Arrays.asList(longShapeDescriptor); + return null; } } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Choice.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Choice.java similarity index 79% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Choice.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Choice.java index a601207e5..c53354a58 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Choice.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Choice.java @@ -25,11 +25,8 @@ import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.common.base.Preconditions; import org.nd4j.imports.NoOpNameFoundException; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.random.BaseRandomOp; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; -import java.util.Arrays; import java.util.List; public class Choice extends BaseRandomOp { @@ -42,7 +39,7 @@ public class Choice extends BaseRandomOp { super(source, probabilities, z); Preconditions.checkArgument(source.dataType() == probabilities.dataType() && z.dataType() == source.dataType(), "Data types of all arguments should match"); Preconditions.checkState(source.length() == probabilities.length(), "From & probabilities length mismatch: %s vs. %s", - source.length(), probabilities.length()); + source.length(), probabilities.length()); if (probabilities.elementWiseStride() < 1 || source.elementWiseStride() < 1) throw new IllegalStateException("Source and probabilities should have element-wise stride >= 1"); this.extraArgs = new Object[] {0.0}; @@ -69,19 +66,8 @@ public class Choice extends BaseRandomOp { throw new NoOpNameFoundException("No tensorflow op opName found for " + opName()); } - @Override - public List calculateOutputShape(OpContext oc) { - return calculateOutputShape(); - } - - @Override - public List calculateOutputShape() { - LongShapeDescriptor longShapeDescriptor = LongShapeDescriptor.fromShape(shape,dataType); - return Arrays.asList(longShapeDescriptor); - } - @Override public List doDiff(List f1) { - throw new UnsupportedOperationException("Choice does not have a derivative"); + return null; } } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOut.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOut.java similarity index 85% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOut.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOut.java index 21ec7fc98..271886a46 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOut.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOut.java @@ -25,11 +25,7 @@ import lombok.NonNull; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.random.BaseRandomOp; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; - -import java.util.Arrays; import java.util.List; @NoArgsConstructor @@ -40,8 +36,8 @@ public class DropOut extends BaseRandomOp { public DropOut(SameDiff sameDiff, SDVariable input, double p) { super(sameDiff, input); this.p = p; - this.extraArgs = new Object[] {p}; - + //https://github.com/deeplearning4j/deeplearning4j/issues/5650 + throw new UnsupportedOperationException("Dropout SameDiff support disabled pending backprop support"); } public DropOut(@NonNull INDArray x, double p) { @@ -69,12 +65,6 @@ public class DropOut extends BaseRandomOp { return Type.RANDOM ; } - @Override - public List calculateOutputShape(OpContext oc) { - INDArray input = oc.getInputArray(0); - return Arrays.asList(input.shapeDescriptor()); - } - @Override public List doDiff(List f1) { throw new UnsupportedOperationException("Not supported"); //We should only use *inverted* dropout with samediff diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOutInverted.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOutInverted.java similarity index 80% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOutInverted.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOutInverted.java index e1b3cfc16..759d7f520 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOutInverted.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/DropOutInverted.java @@ -25,14 +25,11 @@ import onnx.Onnx; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.random.BaseRandomOp; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; import org.tensorflow.framework.AttrValue; import org.tensorflow.framework.GraphDef; import org.tensorflow.framework.NodeDef; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -46,7 +43,6 @@ public class DropOutInverted extends BaseRandomOp { public DropOutInverted(SameDiff sameDiff, SDVariable input, double p) { super(sameDiff, input); this.p = p; - this.extraArgs = new Object[]{p}; } public DropOutInverted(@NonNull INDArray x, double p) { @@ -86,18 +82,6 @@ public class DropOutInverted extends BaseRandomOp { @Override public List doDiff(List f1) { - throw new UnsupportedOperationException("DropOutInverted does not have a derivative."); + return null; } - - @Override - public List calculateOutputShape(OpContext oc) { - return calculateOutputShape(); - } - - @Override - public List calculateOutputShape() { - LongShapeDescriptor longShapeDescriptor = LongShapeDescriptor.fromShape(shape,dataType); - return Arrays.asList(longShapeDescriptor); - } - } diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/GaussianDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/GaussianDistribution.java new file mode 100644 index 000000000..5795b3457 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/GaussianDistribution.java @@ -0,0 +1,151 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.linalg.api.ops.random.impl; + +import lombok.NonNull; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.base.Preconditions; +import org.nd4j.imports.NoOpNameFoundException; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.BaseRandomOp; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collections; +import java.util.List; + +public class GaussianDistribution extends BaseRandomOp { + private double mean; + private double stddev; + + public GaussianDistribution(SameDiff sd, double mean, double stddev, long[] shape){ + super(sd, shape); + this.mean = mean; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + public GaussianDistribution(SameDiff sd, double mean, double stddev, DataType dataType, long[] shape){ + super(sd, shape); + this.mean = mean; + this.stddev = stddev; + this.dataType = dataType; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + public GaussianDistribution() { + super(); + } + + public GaussianDistribution(double mean, double stddev, DataType datatype, long... shape){ + this(Nd4j.createUninitialized(datatype, shape), mean, stddev); + } + + /** + * This op fills Z with random values within stddev..mean..stddev boundaries + * @param z + * @param mean + * @param stddev + */ + public GaussianDistribution(@NonNull INDArray z, double mean, double stddev) { + super(z, z, z); + this.mean = mean; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + + public GaussianDistribution(@NonNull INDArray z, @NonNull INDArray means, double stddev) { + super(z, means, z); + if (z.length() != means.length()) + throw new IllegalStateException("Result length should be equal to provided Means length"); + + if (means.elementWiseStride() < 1) + throw new IllegalStateException("Means array can't have negative EWS"); + + this.mean = 0.0; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + /** + * This op fills Z with random values within -1.0..0..1.0 + * @param z + */ + public GaussianDistribution(@NonNull INDArray z) { + this(z, 0.0, 1.0); + } + + /** + * This op fills Z with random values within stddev..0..stddev + * @param z + */ + public GaussianDistribution(@NonNull INDArray z, double stddev) { + this(z, 0.0, stddev); + } + + @Override + public int opNum() { + return 6; + } + + + @Override + public String opName() { + return "distribution_gaussian"; + } + + @Override + public String onnxName() { + throw new NoOpNameFoundException("No onnx op opName found for " + opName()); + } + + @Override + public String tensorflowName() { + throw new NoOpNameFoundException("No tensorflow op opName found for " + opName()); + } + + @Override + public void setZ(INDArray z){ + //We want all 3 args set to z for this op + this.x = z; + this.y = z; + this.z = z; + } + + + @Override + public List doDiff(List f1) { + return Collections.emptyList(); + } + + @Override + public List calculateOutputDataTypes(List inputDataTypes){ + Preconditions.checkState(inputDataTypes == null || inputDataTypes.isEmpty(), "Expected no input datatypes (no args) for %s, got %s", getClass(), inputDataTypes); + return Collections.singletonList(dataType); + } + + @Override + public boolean isTripleArgRngOp() { + return true; + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Linspace.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Linspace.java similarity index 92% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Linspace.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Linspace.java index 71d0ab2c4..8bc772cf0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Linspace.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Linspace.java @@ -26,12 +26,10 @@ import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.imports.NoOpNameFoundException; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.random.BaseRandomOp; import org.nd4j.linalg.api.shape.LongShapeDescriptor; import org.nd4j.linalg.factory.Nd4j; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -119,6 +117,11 @@ public class Linspace extends BaseRandomOp { this.y = null; } + @Override + public List calculateOutputShape() { + return Collections.singletonList(LongShapeDescriptor.fromShape(new long[]{length}, DataType.FLOAT)); //TODO Don't hardcode float! + } + @Override public String onnxName() { throw new NoOpNameFoundException("No onnx op opName found for " + opName()); @@ -130,17 +133,6 @@ public class Linspace extends BaseRandomOp { } - @Override - public List calculateOutputShape(OpContext oc) { - return calculateOutputShape(); - } - - @Override - public List calculateOutputShape() { - LongShapeDescriptor longShapeDescriptor = LongShapeDescriptor.fromShape(shape,dataType); - return Arrays.asList(longShapeDescriptor); - } - @Override public List doDiff(List f1) { //No inputs diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/LogNormalDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/LogNormalDistribution.java new file mode 100644 index 000000000..f28ec024b --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/LogNormalDistribution.java @@ -0,0 +1,145 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.linalg.api.ops.random.impl; + +import lombok.NonNull; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.base.Preconditions; +import org.nd4j.imports.NoOpNameFoundException; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.BaseRandomOp; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collections; +import java.util.List; + +public class LogNormalDistribution extends BaseRandomOp { + private double mean; + private double stddev; + + public LogNormalDistribution() { + super(); + } + + public LogNormalDistribution(SameDiff sd, double mean, double stdev, long... shape){ + super(sd, shape); + this.mean = mean; + this.stddev = stdev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + public LogNormalDistribution(SameDiff sd, double mean, double stdev, DataType dataType, long... shape){ + this(sd, mean, stdev,shape); + this.dataType = dataType; + } + + public LogNormalDistribution(double mean, double stddev, DataType datatype, long... shape){ + this(Nd4j.createUninitialized(datatype, shape), mean, stddev); + } + + /** + * This op fills Z with random values within stddev..mean..stddev boundaries + * @param z + * @param mean + * @param stddev + */ + public LogNormalDistribution(@NonNull INDArray z, double mean, double stddev) { + super(z, z, z); + this.mean = mean; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + + public LogNormalDistribution(@NonNull INDArray z, @NonNull INDArray means, double stddev) { + super(z,means,z); + if (z.length() != means.length()) + throw new IllegalStateException("Result length should be equal to provided Means length"); + + if (means.elementWiseStride() < 1) + throw new IllegalStateException("Means array can't have negative EWS"); + this.mean = 0.0; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + /** + * This op fills Z with random values within -1.0..0..1.0 + * @param z + */ + public LogNormalDistribution(@NonNull INDArray z) { + this(z, 0.0, 1.0); + } + + /** + * This op fills Z with random values within stddev..0..stddev + * @param z + */ + public LogNormalDistribution(@NonNull INDArray z, double stddev) { + this(z, 0.0, stddev); + } + + @Override + public String onnxName() { + throw new NoOpNameFoundException("No onnx op opName found for " + opName()); + } + + @Override + public String tensorflowName() { + throw new NoOpNameFoundException("No tensorflow op opName found for " + opName()); + } + + @Override + public int opNum() { + return 10; + } + + @Override + public String opName() { + return "distribution_lognormal"; + } + + @Override + public void setZ(INDArray z){ + //We want all 3 args set to z for this op + this.x = z; + this.y = z; + this.z = z; + } + + @Override + public List doDiff(List f1) { + return Collections.emptyList(); + } + + @Override + public List calculateOutputDataTypes(List inputDataTypes){ + Preconditions.checkState(inputDataTypes == null || inputDataTypes.isEmpty(), "Expected no input datatypes (no args) for %s, got %s", getClass(), inputDataTypes); + return Collections.singletonList(dataType); + } + + @Override + public boolean isTripleArgRngOp() { + return true; + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/ProbablisticMerge.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/ProbablisticMerge.java similarity index 82% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/ProbablisticMerge.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/ProbablisticMerge.java index a8e8e0699..0f3aed89a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/ProbablisticMerge.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/ProbablisticMerge.java @@ -24,11 +24,8 @@ import lombok.NonNull; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.imports.NoOpNameFoundException; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.api.ops.random.BaseRandomOp; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; -import java.util.Arrays; import java.util.List; public class ProbablisticMerge extends BaseRandomOp { @@ -69,17 +66,6 @@ public class ProbablisticMerge extends BaseRandomOp { throw new NoOpNameFoundException("No tensorflow op opName found for " + opName()); } - @Override - public List calculateOutputShape(OpContext oc) { - return calculateOutputShape(); - } - - @Override - public List calculateOutputShape() { - LongShapeDescriptor longShapeDescriptor = LongShapeDescriptor.fromShape(shape,dataType); - return Arrays.asList(longShapeDescriptor); - } - @Override public List doDiff(List f1) { return null; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Range.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Range.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Range.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/Range.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/TruncatedNormalDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/TruncatedNormalDistribution.java new file mode 100644 index 000000000..e5a9c6627 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/TruncatedNormalDistribution.java @@ -0,0 +1,150 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.linalg.api.ops.random.impl; + +import lombok.NonNull; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.base.Preconditions; +import org.nd4j.imports.NoOpNameFoundException; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.BaseRandomOp; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collections; +import java.util.List; + +public class TruncatedNormalDistribution extends BaseRandomOp { + private double mean; + private double stddev; + + public TruncatedNormalDistribution() { + super(); + } + + public TruncatedNormalDistribution(SameDiff sd, double mean, double stddev, long[] shape){ + super(sd, shape); + this.mean = mean; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + public TruncatedNormalDistribution(SameDiff sd, double mean, double stddev, DataType dataType, long[] shape) { + super(sd, shape); + this.mean = mean; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + public TruncatedNormalDistribution(double mean, double stddev, DataType datatype, long... shape){ + this(Nd4j.createUninitialized(datatype, shape), mean, stddev); + } + + /** + * This op fills Z with random values within stddev..mean..stddev boundaries + * @param z + * @param mean + * @param stddev + */ + public TruncatedNormalDistribution(@NonNull INDArray z, double mean, double stddev) { + super(z,z,z); + this.mean = mean; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + + public TruncatedNormalDistribution(@NonNull INDArray z, @NonNull INDArray means, double stddev) { + super(z, means, z); + if (z.length() != means.length()) + throw new IllegalStateException("Result length should be equal to provided Means length"); + + if (means.elementWiseStride() < 1) + throw new IllegalStateException("Means array can't have negative EWS"); + + this.mean = 0.0; + this.stddev = stddev; + this.extraArgs = new Object[] {this.mean, this.stddev}; + } + + /** + * This op fills Z with random values within -1.0..0..1.0 + * @param z + */ + public TruncatedNormalDistribution(@NonNull INDArray z) { + this(z, 0.0, 1.0); + } + + /** + * This op fills Z with random values within stddev..0..stddev + * @param z + */ + public TruncatedNormalDistribution(@NonNull INDArray z, double stddev) { + this(z, 0.0, stddev); + } + + @Override + public int opNum() { + return 11; + } + + @Override + public String opName() { + return "distribution_truncated"; + } + + @Override + public String onnxName() { + throw new NoOpNameFoundException("No onnx op opName found for " + opName()); + } + + @Override + public String tensorflowName() { + throw new NoOpNameFoundException("No tensorflow op opName found for " + opName()); + } + + @Override + public void setZ(INDArray z){ + //We want all 3 args set to z for this op + this.x = z; + this.y = z; + this.z = z; + } + + @Override + public List doDiff(List f1) { + return Collections.emptyList(); + } + + @Override + public List calculateOutputDataTypes(List inputDataTypes){ + Preconditions.checkState(inputDataTypes == null || inputDataTypes.isEmpty(), "Expected no input datatypes (no args) for %s, got %s", getClass(), inputDataTypes); + //Input data type specifies the shape; output data type should be any float + //TODO MAKE CONFIGUREABLE - https://github.com/eclipse/deeplearning4j/issues/6854 + return Collections.singletonList(DataType.DOUBLE); + } + + @Override + public boolean isTripleArgRngOp() { + return true; + } +} diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/UniformDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/UniformDistribution.java new file mode 100644 index 000000000..4781cb9b8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/random/impl/UniformDistribution.java @@ -0,0 +1,116 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.linalg.api.ops.random.impl; + +import lombok.NonNull; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.base.Preconditions; +import org.nd4j.imports.NoOpNameFoundException; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.BaseRandomOp; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collections; +import java.util.List; + +public class UniformDistribution extends BaseRandomOp { + private double from; + private double to; + + public UniformDistribution() { + super(); + } + + public UniformDistribution(SameDiff sd, double from, double to, long[] shape){ + super(sd, shape); + this.from = from; + this.to = to; + this.extraArgs = new Object[] {this.from, this.to}; + } + + public UniformDistribution(SameDiff sd, double from, double to, DataType dataType, long[] shape) { + this(sd, from, to, shape); + this.dataType = dataType; + } + + public UniformDistribution(double min, double max, DataType datatype, long... shape){ + this(Nd4j.createUninitialized(datatype, shape), min, max); + } + + /** + * This op fills Z with random values within from...to boundaries + * @param z + * @param from + * @param to + */ + public UniformDistribution(@NonNull INDArray z, double from, double to) { + super(null, null, z); + this.from = from; + this.to = to; + this.extraArgs = new Object[] {this.from, this.to}; + } + + /** + * This op fills Z with random values within 0...1 + * @param z + */ + public UniformDistribution(@NonNull INDArray z) { + this(z, 0.0, 1.0); + } + + /** + * This op fills Z with random values within 0...to + * @param z + */ + public UniformDistribution(@NonNull INDArray z, double to) { + this(z, 0.0, to); + } + + @Override + public int opNum() { + return 0; + } + + @Override + public String opName() { + return "distribution_uniform"; + } + + @Override + public String onnxName() { + throw new NoOpNameFoundException("No onnx op opName found for " + opName()); + } + + @Override + public List doDiff(List f1) { + return Collections.emptyList(); + } + + @Override + public List calculateOutputDataTypes(List inputDataTypes){ + Preconditions.checkState(inputDataTypes == null || inputDataTypes.isEmpty(), "Expected no input datatypes (no args) for %s, got %s", getClass(), inputDataTypes); + //Input data type specifies the shape; output data type should be any float + //TODO MAKE CONFIGUREABLE - https://github.com/eclipse/deeplearning4j/issues/6854 + return Collections.singletonList(dataType); + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/util/PrintAffinity.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/util/PrintAffinity.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/util/PrintAffinity.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/util/PrintAffinity.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/util/PrintVariable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/util/PrintVariable.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/util/PrintVariable.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/ops/util/PrintVariable.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/DefaultRandom.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/DefaultRandom.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/DefaultRandom.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/DefaultRandom.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/Random.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/Random.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/Random.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/Random.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/BaseDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/BaseDistribution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/BaseDistribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/BaseDistribution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/Distribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/Distribution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/Distribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/Distribution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/factory/DefaultDistributionFactory.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/factory/DefaultDistributionFactory.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/factory/DefaultDistributionFactory.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/factory/DefaultDistributionFactory.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/factory/DistributionFactory.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/factory/DistributionFactory.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/factory/DistributionFactory.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/factory/DistributionFactory.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/BinomialDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/BinomialDistribution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/BinomialDistribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/BinomialDistribution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/ConstantDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/ConstantDistribution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/ConstantDistribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/ConstantDistribution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/LogNormalDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/LogNormalDistribution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/LogNormalDistribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/LogNormalDistribution.java diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/NormalDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/NormalDistribution.java new file mode 100644 index 000000000..a7ccc5caf --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/NormalDistribution.java @@ -0,0 +1,355 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.linalg.api.rng.distribution.impl; + +import org.apache.commons.math3.exception.NotStrictlyPositiveException; +import org.apache.commons.math3.exception.NumberIsTooLargeException; +import org.apache.commons.math3.exception.OutOfRangeException; +import org.apache.commons.math3.exception.util.LocalizedFormats; +import org.apache.commons.math3.special.Erf; +import org.apache.commons.math3.util.FastMath; +import org.nd4j.linalg.api.iter.NdIndexIterator; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.impl.GaussianDistribution; +import org.nd4j.linalg.api.rng.Random; +import org.nd4j.linalg.api.rng.distribution.BaseDistribution; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Iterator; + +public class NormalDistribution extends BaseDistribution { + /** + * Default inverse cumulative probability accuracy. + * + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** + * Serializable version identifier. + */ + private static final long serialVersionUID = 8589540077390120676L; + /** + * √(2 π) + */ + private static final double SQRT2PI = FastMath.sqrt(2 * FastMath.PI); + /** + * √(2) + */ + private static final double SQRT2 = FastMath.sqrt(2.0); + /** + * Standard deviation of this distribution. + */ + private final double standardDeviation; + /** + * Mean of this distribution. + */ + private double mean; + private INDArray means; + /** + * Inverse cumulative probability accuracy. + */ + private double solverAbsoluteAccuracy; + + public NormalDistribution(Random rng, double standardDeviation, INDArray means) { + super(rng); + this.standardDeviation = standardDeviation; + this.means = means; + } + + public NormalDistribution(double standardDeviation, INDArray means) { + this.standardDeviation = standardDeviation; + this.means = means; + } + + /** + * Create a normal distribution with mean equal to zero and standard + * deviation equal to one. + */ + public NormalDistribution() { + this(0, 1); + } + + /** + * Create a normal distribution using the given mean and standard deviation. + * + * @param mean Mean for this distribution. + * @param sd Standard deviation for this distribution. + * @throws org.apache.commons.math3.exception.NotStrictlyPositiveException if {@code sd <= 0}. + */ + public NormalDistribution(double mean, double sd) throws NotStrictlyPositiveException { + this(mean, sd, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + public NormalDistribution(Random rng, double mean, double sd) throws NotStrictlyPositiveException { + this(rng, mean, sd, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Create a normal distribution using the given mean, standard deviation and + * inverse cumulative distribution accuracy. + * + * @param mean Mean for this distribution. + * @param sd Standard deviation for this distribution. + * @param inverseCumAccuracy Inverse cumulative probability accuracy. + * @throws NotStrictlyPositiveException if {@code sd <= 0}. + * @since 2.1 + */ + public NormalDistribution(double mean, double sd, double inverseCumAccuracy) throws NotStrictlyPositiveException { + this(Nd4j.getRandom(), mean, sd, inverseCumAccuracy); + } + + /** + * Creates a normal distribution. + * + * @param rng Random number generator. + * @param mean Mean for this distribution. + * @param sd Standard deviation for this distribution. + * @param inverseCumAccuracy Inverse cumulative probability accuracy. + * @throws NotStrictlyPositiveException if {@code sd <= 0}. + * @since 3.1 + */ + public NormalDistribution(Random rng, double mean, double sd, double inverseCumAccuracy) + throws NotStrictlyPositiveException { + super(rng); + + if (sd <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.STANDARD_DEVIATION, sd); + } + + this.mean = mean; + standardDeviation = sd; + solverAbsoluteAccuracy = inverseCumAccuracy; + } + + public NormalDistribution(INDArray mean, double std) { + this.means = mean; + this.standardDeviation = std; + this.random = Nd4j.getRandom(); + } + + /** + * Access the mean. + * + * @return the mean for this distribution. + */ + public double getMean() { + return mean; + } + + /** + * Access the standard deviation. + * + * @return the standard deviation for this distribution. + */ + public double getStandardDeviation() { + return standardDeviation; + } + + /** + * {@inheritDoc} + */ + public double density(double x) { + if (means != null) + throw new IllegalStateException("Unable to sample from more than one mean"); + final double x0 = x - mean; + final double x1 = x0 / standardDeviation; + return FastMath.exp(-0.5 * x1 * x1) / (standardDeviation * SQRT2PI); + } + + /** + * {@inheritDoc} + *

+ * If {@code x} is more than 40 standard deviations from the mean, 0 or 1 + * is returned, as in these cases the actual value is within + * {@code Double.MIN_VALUE} of 0 or 1. + */ + public double cumulativeProbability(double x) { + if (means != null) + throw new IllegalStateException("Unable to sample from more than one mean"); + final double dev = x - mean; + if (FastMath.abs(dev) > 40 * standardDeviation) { + return dev < 0 ? 0.0d : 1.0d; + } + return 0.5 * (1 + Erf.erf(dev / (standardDeviation * SQRT2))); + } + + /** + * {@inheritDoc} + * + * @since 3.2 + */ + @Override + public double inverseCumulativeProbability(final double p) throws OutOfRangeException { + if (p < 0.0 || p > 1.0) { + throw new OutOfRangeException(p, 0, 1); + } + if (means != null) + throw new IllegalStateException("Unable to sample from more than one mean"); + + return mean + standardDeviation * SQRT2 * Erf.erfInv(2 * p - 1); + } + + /** + * {@inheritDoc} + * + * @deprecated See {@link org.apache.commons.math3.distribution.RealDistribution#cumulativeProbability(double, double)} + */ + @Override + @Deprecated + public double cumulativeProbability(double x0, double x1) throws NumberIsTooLargeException { + return probability(x0, x1); + } + + /** + * {@inheritDoc} + */ + @Override + public double probability(double x0, double x1) throws NumberIsTooLargeException { + if (x0 > x1) { + throw new NumberIsTooLargeException(LocalizedFormats.LOWER_ENDPOINT_ABOVE_UPPER_ENDPOINT, x0, x1, true); + } + final double denom = standardDeviation * SQRT2; + final double v0 = (x0 - mean) / denom; + final double v1 = (x1 - mean) / denom; + return 0.5 * Erf.erf(v0, v1); + } + + /** + * {@inheritDoc} + */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + *

+ * For mean parameter {@code mu}, the mean is {@code mu}. + */ + public double getNumericalMean() { + return getMean(); + } + + /** + * {@inheritDoc} + *

+ * For standard deviation parameter {@code s}, the variance is {@code s^2}. + */ + public double getNumericalVariance() { + final double s = getStandardDeviation(); + return s * s; + } + + /** + * {@inheritDoc} + *

+ * The lower bound of the support is always negative infinity + * no matter the parameters. + * + * @return lower bound of the support (always + * {@code Double.NEGATIVE_INFINITY}) + */ + public double getSupportLowerBound() { + return Double.NEGATIVE_INFINITY; + } + + /** + * {@inheritDoc} + *

+ * The upper bound of the support is always positive infinity + * no matter the parameters. + * + * @return upper bound of the support (always + * {@code Double.POSITIVE_INFINITY}) + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** + * {@inheritDoc} + */ + public boolean isSupportLowerBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + *

+ * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public double sample() { + if (means != null) + throw new IllegalStateException("Unable to sample from more than one mean"); + return standardDeviation * random.nextGaussian() + mean; + } + + @Override + public INDArray sample(int[] shape) { + final INDArray ret = Nd4j.createUninitialized(shape, Nd4j.order()); + return sample(ret); + } + + @Override + public INDArray sample(INDArray ret) { + if (random.getStatePointer() != null) { + if (means != null) { + return Nd4j.getExecutioner().exec(new GaussianDistribution( + ret, means, standardDeviation), random); + } else { + return Nd4j.getExecutioner().exec(new GaussianDistribution( + ret, mean, standardDeviation), random); + } + } else { + Iterator idxIter = new NdIndexIterator(ret.shape()); //For consistent values irrespective of c vs. fortran ordering + long len = ret.length(); + if (means != null) { + for (int i = 0; i < len; i++) { + long[] idx = idxIter.next(); + ret.putScalar(idx, standardDeviation * random.nextGaussian() + means.getDouble(idx)); + } + } else { + for (int i = 0; i < len; i++) { + ret.putScalar(idxIter.next(), standardDeviation * random.nextGaussian() + mean); + } + } + return ret; + } + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/OrthogonalDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/OrthogonalDistribution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/OrthogonalDistribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/OrthogonalDistribution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/SaddlePointExpansion.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/SaddlePointExpansion.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/SaddlePointExpansion.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/SaddlePointExpansion.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/TruncatedNormalDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/TruncatedNormalDistribution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/TruncatedNormalDistribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/TruncatedNormalDistribution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/UniformDistribution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/UniformDistribution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/UniformDistribution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/rng/distribution/impl/UniformDistribution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/LongShapeDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/LongShapeDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/LongShapeDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/LongShapeDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/Shape.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/Shape.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/Shape.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/Shape.java index 4898f3370..cacc677cb 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/Shape.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/Shape.java @@ -21,8 +21,8 @@ package org.nd4j.linalg.api.shape; -import org.nd4j.shade.guava.primitives.Ints; -import org.nd4j.shade.guava.primitives.Longs; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; import lombok.NonNull; import lombok.val; import org.nd4j.common.base.Preconditions; @@ -3539,7 +3539,7 @@ public class Shape { return shape.length; } - public static int rankFromShape(long[] shape) { + public static int rankFromShape(long[] shape){ if(shape == null){ throw new ND4JIllegalStateException("Cannot get rank from null shape array"); } @@ -3551,7 +3551,7 @@ public class Shape { } public static void assertBroadcastable(@NonNull int[] x, @NonNull int[] y){ - if(!areShapesBroadcastable(x, y)) { + if(!areShapesBroadcastable(x, y)){ throw new ND4JIllegalStateException("Arrays are different shape and are not broadcastable." + " Array 1 shape = " + Arrays.toString(x) + ", array 2 shape = " + Arrays.toString(y)); } @@ -3570,7 +3570,7 @@ public class Shape { } public static boolean areShapesBroadcastable(@NonNull int[] x, @NonNull int[] y){ - //Ported from: https://github.com/eclipse/deeplearning4j/libnd4j/blob/master/include/helpers/impl/ShapeUtils.cpp + //Ported from: https://github.com/deeplearning4j/libnd4j/blob/master/include/helpers/impl/ShapeUtils.cpp int minRank = Math.min(x.length, y.length); for( int i=-1; i>= -minRank; i--){ @@ -3583,7 +3583,7 @@ public class Shape { } public static boolean areShapesBroadcastable(@NonNull long[] left, @NonNull long[] right){ - //Ported from: https://github.com/eclipse/deeplearning4j/libnd4j/blob/master/include/helpers/impl/ShapeUtils.cpp + //Ported from: https://github.com/deeplearning4j/libnd4j/blob/master/include/helpers/impl/ShapeUtils.cpp int minRank = Math.min(left.length, right.length); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/ShapeDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/ShapeDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/ShapeDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/ShapeDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/TadPack.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/TadPack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/TadPack.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/TadPack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/loop/coordinatefunction/CoordinateFunction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/loop/coordinatefunction/CoordinateFunction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/loop/coordinatefunction/CoordinateFunction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/loop/coordinatefunction/CoordinateFunction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/options/ArrayOptionsHelper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/options/ArrayOptionsHelper.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/options/ArrayOptionsHelper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/options/ArrayOptionsHelper.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/options/ArrayType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/options/ArrayType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/shape/options/ArrayType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/api/shape/options/ArrayType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/ArrayDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/ArrayDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/ArrayDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/ArrayDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/BasicConstantHandler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/BasicConstantHandler.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/BasicConstantHandler.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/BasicConstantHandler.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/ConstantHandler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/ConstantHandler.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/ConstantHandler.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/ConstantHandler.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/TADManager.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/TADManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/TADManager.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/TADManager.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/TadDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/TadDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/cache/TadDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/cache/TadDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/checkutil/CheckUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/checkutil/CheckUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/checkutil/CheckUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/checkutil/CheckUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/checkutil/NDArrayCreationUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/checkutil/NDArrayCreationUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/checkutil/NDArrayCreationUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/checkutil/NDArrayCreationUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/AbstractStorage.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/AbstractStorage.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/AbstractStorage.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/AbstractStorage.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/BasicNDArrayCompressor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/BasicNDArrayCompressor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/BasicNDArrayCompressor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/BasicNDArrayCompressor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressedDataBuffer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressedDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressedDataBuffer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressedDataBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressionAlgorithm.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressionAlgorithm.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressionAlgorithm.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressionAlgorithm.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressionDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressionDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressionDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressionDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressionType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressionType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressionType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressionType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressionUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressionUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressionUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/CompressionUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/NDArrayCompressor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/NDArrayCompressor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/NDArrayCompressor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/NDArrayCompressor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/ThresholdCompression.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/ThresholdCompression.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/ThresholdCompression.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/compression/ThresholdCompression.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/BaseConvolution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/BaseConvolution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/BaseConvolution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/BaseConvolution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/Convolution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/Convolution.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/Convolution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/Convolution.java index 8be873b8e..4d91be1ba 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/Convolution.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/Convolution.java @@ -28,12 +28,8 @@ import org.nd4j.linalg.api.ops.impl.layers.convolution.Im2col; import org.nd4j.linalg.api.ops.impl.layers.convolution.Pooling2D; import org.nd4j.linalg.api.ops.impl.layers.convolution.config.Conv2DConfig; import org.nd4j.linalg.api.ops.impl.layers.convolution.config.Pooling2DConfig; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; import org.nd4j.linalg.factory.Nd4j; -import java.util.Arrays; -import java.util.List; - public class Convolution { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/ConvolutionInstance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/ConvolutionInstance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/ConvolutionInstance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/ConvolutionInstance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/DefaultConvolutionInstance.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/DefaultConvolutionInstance.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/DefaultConvolutionInstance.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/DefaultConvolutionInstance.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/OldConvolution.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/OldConvolution.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/convolution/OldConvolution.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/convolution/OldConvolution.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/AsyncDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/AsyncDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/AsyncDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/AsyncDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/AsyncMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/AsyncMultiDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/AsyncMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/AsyncMultiDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java index 95fa779a9..0e400c756 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/BalanceMinibatches.java @@ -20,8 +20,8 @@ package org.nd4j.linalg.dataset; -import org.nd4j.shade.guava.collect.Lists; -import org.nd4j.shade.guava.collect.Maps; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/DataSet.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/DataSet.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/DataSet.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/DataSet.java index 57467ace9..f66afa29f 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/DataSet.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/DataSet.java @@ -20,7 +20,7 @@ package org.nd4j.linalg.dataset; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.ndarray.INDArray; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/ExistingMiniBatchDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/ExistingMiniBatchDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/ExistingMiniBatchDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/ExistingMiniBatchDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/MiniBatchFileDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/MiniBatchFileDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/MiniBatchFileDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/MiniBatchFileDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/MultiDataSet.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/MultiDataSet.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/MultiDataSet.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/MultiDataSet.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/SplitTestAndTrain.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/SplitTestAndTrain.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/SplitTestAndTrain.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/SplitTestAndTrain.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/ViewIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/ViewIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/ViewIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/ViewIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/adapter/MultiDataSetIteratorAdapter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/adapter/MultiDataSetIteratorAdapter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/adapter/MultiDataSetIteratorAdapter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/adapter/MultiDataSetIteratorAdapter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/adapter/SingletonDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/adapter/SingletonDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/adapter/SingletonDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/adapter/SingletonDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/adapter/SingletonMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/adapter/SingletonMultiDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/adapter/SingletonMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/adapter/SingletonMultiDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/DataSet.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/DataSet.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/DataSet.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/DataSet.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/DataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/DataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/DataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/DataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/DataSetUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/DataSetUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/DataSetUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/DataSetUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/MultiDataSet.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/MultiDataSet.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/MultiDataSet.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/MultiDataSet.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/MultiDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/MultiDataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/MultiDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/MultiDataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BaseDatasetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BaseDatasetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BaseDatasetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BaseDatasetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BlockDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BlockDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BlockDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BlockDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BlockMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BlockMultiDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BlockMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/BlockMultiDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/CachingDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/CachingDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/CachingDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/CachingDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/DataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/DataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/DataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/DataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/DataSetIteratorFactory.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/DataSetIteratorFactory.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/DataSetIteratorFactory.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/DataSetIteratorFactory.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/KFoldIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/KFoldIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/KFoldIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/KFoldIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultiDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultiDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultiDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultiDataSetIteratorFactory.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultiDataSetIteratorFactory.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultiDataSetIteratorFactory.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultiDataSetIteratorFactory.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultipleEpochsIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultipleEpochsIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultipleEpochsIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/MultipleEpochsIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/ParallelDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/ParallelDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/ParallelDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/ParallelDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/ParallelMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/ParallelMultiDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/ParallelMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/ParallelMultiDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/SamplingDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/SamplingDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/SamplingDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/SamplingDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/StandardScaler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/StandardScaler.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/StandardScaler.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/StandardScaler.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/TestDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/TestDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/TestDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/TestDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/TestMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/TestMultiDataSetIterator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/TestMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/TestMultiDataSetIterator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/DataSetCache.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/DataSetCache.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/DataSetCache.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/DataSetCache.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InFileAndMemoryDataSetCache.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InFileAndMemoryDataSetCache.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InFileAndMemoryDataSetCache.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InFileAndMemoryDataSetCache.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InFileDataSetCache.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InFileDataSetCache.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InFileDataSetCache.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InFileDataSetCache.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InMemoryDataSetCache.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InMemoryDataSetCache.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InMemoryDataSetCache.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/cache/InMemoryDataSetCache.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/enums/InequalityHandling.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/enums/InequalityHandling.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/enums/InequalityHandling.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/enums/InequalityHandling.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/fetcher/BaseDataFetcher.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/fetcher/BaseDataFetcher.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/fetcher/BaseDataFetcher.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/fetcher/BaseDataFetcher.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/fetcher/DataSetFetcher.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/fetcher/DataSetFetcher.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/fetcher/DataSetFetcher.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/iterator/fetcher/DataSetFetcher.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractDataSetNormalizer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractDataSetNormalizer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractDataSetNormalizer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractDataSetNormalizer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractMultiDataSetNormalizer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractMultiDataSetNormalizer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractMultiDataSetNormalizer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractMultiDataSetNormalizer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractNormalizer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractNormalizer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractNormalizer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/AbstractNormalizer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CompositeDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CompositeDataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CompositeDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CompositeDataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CompositeMultiDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CompositeMultiDataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CompositeMultiDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CompositeMultiDataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CropAndResizeDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CropAndResizeDataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CropAndResizeDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/CropAndResizeDataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/DataNormalization.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/DataNormalization.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/DataNormalization.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/DataNormalization.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImageFlatteningDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImageFlatteningDataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImageFlatteningDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImageFlatteningDataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImageMultiPreProcessingScaler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImageMultiPreProcessingScaler.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImageMultiPreProcessingScaler.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImageMultiPreProcessingScaler.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImagePreProcessingScaler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImagePreProcessingScaler.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImagePreProcessingScaler.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/ImagePreProcessingScaler.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/LabelLastTimeStepPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/LabelLastTimeStepPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/LabelLastTimeStepPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/LabelLastTimeStepPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MinMaxStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MinMaxStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MinMaxStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MinMaxStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiDataNormalization.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiDataNormalization.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiDataNormalization.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiDataNormalization.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerHybrid.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerHybrid.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerHybrid.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerHybrid.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerMinMaxScaler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerMinMaxScaler.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerMinMaxScaler.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerMinMaxScaler.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerStandardize.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerStandardize.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerStandardize.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/MultiNormalizerStandardize.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/Normalizer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/Normalizer.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/Normalizer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/Normalizer.java index ca299af83..c5ccb63a4 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/Normalizer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/Normalizer.java @@ -20,8 +20,6 @@ package org.nd4j.linalg.dataset.api.preprocessor; -import org.nd4j.linalg.dataset.api.DataSet; -import org.nd4j.linalg.dataset.api.MultiDataSet; import org.nd4j.linalg.dataset.api.preprocessor.serializer.NormalizerSerializerStrategy; import org.nd4j.linalg.dataset.api.preprocessor.serializer.NormalizerType; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerMinMaxScaler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerMinMaxScaler.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerMinMaxScaler.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerMinMaxScaler.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerStandardize.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerStandardize.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerStandardize.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerStandardize.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/NormalizerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/PermuteDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/PermuteDataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/PermuteDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/PermuteDataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/RGBtoGrayscaleDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/RGBtoGrayscaleDataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/RGBtoGrayscaleDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/RGBtoGrayscaleDataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/StandardizeStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/StandardizeStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/StandardizeStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/StandardizeStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/VGG16ImagePreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/VGG16ImagePreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/VGG16ImagePreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/VGG16ImagePreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/BaseUnderSamplingPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/BaseUnderSamplingPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/BaseUnderSamplingPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/BaseUnderSamplingPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/UnderSamplingByMaskingMultiDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/UnderSamplingByMaskingMultiDataSetPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/UnderSamplingByMaskingMultiDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/UnderSamplingByMaskingMultiDataSetPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/UnderSamplingByMaskingPreProcessor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/UnderSamplingByMaskingPreProcessor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/UnderSamplingByMaskingPreProcessor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/classimbalance/UnderSamplingByMaskingPreProcessor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/CustomSerializerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/CustomSerializerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/CustomSerializerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/CustomSerializerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/ImagePreProcessingSerializerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/ImagePreProcessingSerializerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/ImagePreProcessingSerializerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/ImagePreProcessingSerializerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MinMaxSerializerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MinMaxSerializerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MinMaxSerializerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MinMaxSerializerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiHybridSerializerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiHybridSerializerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiHybridSerializerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiHybridSerializerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiMinMaxSerializerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiMinMaxSerializerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiMinMaxSerializerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiMinMaxSerializerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiStandardizeSerializerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiStandardizeSerializerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiStandardizeSerializerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/MultiStandardizeSerializerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerSerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerSerializer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerSerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerSerializer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerSerializerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerSerializerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerSerializerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerSerializerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/NormalizerType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/StandardizeSerializerStrategy.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/StandardizeSerializerStrategy.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/StandardizeSerializerStrategy.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/serializer/StandardizeSerializerStrategy.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/DistributionStats.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/DistributionStats.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/DistributionStats.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/DistributionStats.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/MinMaxStats.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/MinMaxStats.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/MinMaxStats.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/MinMaxStats.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/NormalizerStats.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/NormalizerStats.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/NormalizerStats.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/api/preprocessor/stats/NormalizerStats.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/callbacks/DataSetCallback.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/callbacks/DataSetCallback.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/callbacks/DataSetCallback.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/callbacks/DataSetCallback.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/callbacks/DefaultCallback.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/callbacks/DefaultCallback.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dataset/callbacks/DefaultCallback.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dataset/callbacks/DefaultCallback.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dimensionalityreduction/PCA.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dimensionalityreduction/PCA.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dimensionalityreduction/PCA.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dimensionalityreduction/PCA.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dimensionalityreduction/RandomProjection.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dimensionalityreduction/RandomProjection.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/dimensionalityreduction/RandomProjection.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/dimensionalityreduction/RandomProjection.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/eigen/Eigen.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/eigen/Eigen.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/eigen/Eigen.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/eigen/Eigen.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/EnvironmentalAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/EnvironmentalAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/EnvironmentalAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/EnvironmentalAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/DebugAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/DebugAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/DebugAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/DebugAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/FallbackAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/FallbackAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/FallbackAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/FallbackAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/NDArrayUnpackAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/NDArrayUnpackAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/NDArrayUnpackAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/NDArrayUnpackAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/OmpNumThreadsAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/OmpNumThreadsAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/OmpNumThreadsAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/OmpNumThreadsAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/VerboseAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/VerboseAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/VerboseAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/VerboseAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesBypassAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesBypassAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesBypassAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesBypassAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesDebugAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesDebugAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesDebugAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesDebugAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesSpillAction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesSpillAction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesSpillAction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/env/impl/WorkspacesSpillAction.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JArraySizeException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JArraySizeException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JArraySizeException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JArraySizeException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalAccessException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalAccessException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalAccessException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalAccessException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalArgumentException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalArgumentException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalArgumentException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalArgumentException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalStateException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalStateException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalStateException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JIllegalStateException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JOpProfilerException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JOpProfilerException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JOpProfilerException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JOpProfilerException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JUnknownDataTypeException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JUnknownDataTypeException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4JUnknownDataTypeException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4JUnknownDataTypeException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4UnresolvedOutputVariables.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4UnresolvedOutputVariables.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/ND4UnresolvedOutputVariables.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/ND4UnresolvedOutputVariables.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/Nd4jNoSuchWorkspaceException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/Nd4jNoSuchWorkspaceException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/exception/Nd4jNoSuchWorkspaceException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/exception/Nd4jNoSuchWorkspaceException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/executors/ExecutorServiceProvider.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/executors/ExecutorServiceProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/executors/ExecutorServiceProvider.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/executors/ExecutorServiceProvider.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/BaseBlasWrapper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/BaseBlasWrapper.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/BaseBlasWrapper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/BaseBlasWrapper.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/BaseNDArrayFactory.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/BaseNDArrayFactory.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/BaseNDArrayFactory.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/BaseNDArrayFactory.java index 3eddc856a..667432f6c 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/BaseNDArrayFactory.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/BaseNDArrayFactory.java @@ -1289,7 +1289,7 @@ public abstract class BaseNDArrayFactory implements NDArrayFactory { public INDArray scalar(Number value) { MemoryWorkspace ws = Nd4j.getMemoryManager().getCurrentWorkspace(); - if (value instanceof Double || value instanceof AtomicDouble) /* note that org.nd4j.linalg.primitives.AtomicDouble extends org.nd4j.shade.guava.util.concurrent.AtomicDouble */ + if (value instanceof Double || value instanceof AtomicDouble) /* note that org.nd4j.linalg.primitives.AtomicDouble extends com.google.common.util.concurrent.AtomicDouble */ return scalar(value.doubleValue()); else if (value instanceof Float) return scalar(value.floatValue()); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/BlasWrapper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/BlasWrapper.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/BlasWrapper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/BlasWrapper.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Broadcast.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/Broadcast.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Broadcast.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/Broadcast.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/DataTypeValidation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/DataTypeValidation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/DataTypeValidation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/DataTypeValidation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Environment.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/Environment.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Environment.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/Environment.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/NDArrayFactory.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/NDArrayFactory.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/NDArrayFactory.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/NDArrayFactory.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/NDValidation.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/NDValidation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/NDValidation.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/NDValidation.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java index 0a20a9798..46f538dfc 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java @@ -21,21 +21,26 @@ package org.nd4j.linalg.factory; import lombok.extern.slf4j.Slf4j; +import org.bytedeco.javacpp.*; +import org.bytedeco.javacpp.indexer.*; +import org.nd4j.linalg.api.buffer.BaseDataBuffer; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.BaseShapeInfoProvider; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ndarray.ShapeInfoProvider; import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMax; import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMin; import org.nd4j.common.config.ND4JClassLoading; import org.nd4j.linalg.factory.ops.*; -import org.nd4j.shade.guava.primitives.Ints; -import org.nd4j.shade.guava.primitives.Longs; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; import lombok.NonNull; import lombok.val; -import lombok.var; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; import org.apache.commons.lang3.ArrayUtils; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.indexer.*; import org.nd4j.autodiff.samediff.serde.FlatBuffersMapper; import org.nd4j.common.base.Preconditions; import org.nd4j.common.config.ND4JEnvironmentVars; @@ -43,14 +48,12 @@ import org.nd4j.common.config.ND4JSystemProperties; import org.nd4j.context.Nd4jContext; import org.nd4j.graph.FlatArray; import org.nd4j.linalg.api.blas.params.MMulTranspose; -import org.nd4j.linalg.api.buffer.*; import org.nd4j.linalg.api.buffer.factory.DataBufferFactory; import org.nd4j.linalg.api.buffer.util.DataTypeUtil; import org.nd4j.linalg.api.concurrency.AffinityManager; import org.nd4j.linalg.api.concurrency.BasicAffinityManager; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.memory.MemoryWorkspaceManager; -import org.nd4j.linalg.api.ndarray.*; import org.nd4j.linalg.api.ops.CustomOp; import org.nd4j.linalg.api.ops.DynamicCustomOp; import org.nd4j.linalg.api.ops.Op; @@ -2308,7 +2311,7 @@ public class Nd4j { data2.add(readSplit(data)); } float[][] fArr = new float[data2.size()][0]; - for(int i = 0; i < data2.size(); i++) { + for(int i=0; i - * - * @param input Input to split (NUMERIC type) - * @param numSplit Number of splits - * @param splitDim The dimension to split on - */ - public INDArray[] split(INDArray input, int numSplit, int splitDim) { - NDValidation.validateNumerical("split", "input", input); - return Nd4j.exec(new org.nd4j.linalg.api.ops.impl.shape.Split(input, numSplit, splitDim)); - } - /** * Squared L2 norm: see norm2(String, SDVariable, boolean, int...)
* diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDBitwise.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDBitwise.java similarity index 87% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDBitwise.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDBitwise.java index 9b85a44d7..7bab6e455 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDBitwise.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDBitwise.java @@ -1,20 +1,22 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; @@ -132,7 +134,7 @@ public class NDBitwise { /** * Bitwise left cyclical shift operation. Supports broadcasting.
- * Unlike #leftShift(INDArray, INDArray) the bits will "wrap around":
+ * Unlike {@link #leftShift(INDArray, INDArray)} the bits will "wrap around":
* {@code leftShiftCyclic(01110000, 2) -> 11000001}
* * @param x Input to be bit shifted (INT type) @@ -178,7 +180,7 @@ public class NDBitwise { /** * Bitwise right cyclical shift operation. Supports broadcasting.
- * Unlike rightShift(INDArray, INDArray) the bits will "wrap around":
+ * Unlike {@link #rightShift(INDArray, INDArray)} the bits will "wrap around":
* {@code rightShiftCyclic(00001110, 2) -> 10000011}
* * @param x Input to be bit shifted (INT type) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDCNN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDCNN.java similarity index 89% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDCNN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDCNN.java index 45b30816d..38bccd9a6 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDCNN.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDCNN.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; -import static org.nd4j.linalg.factory.NDValidation.isSameType; - import org.nd4j.common.base.Preconditions; import org.nd4j.enums.DataFormat; import org.nd4j.linalg.api.ndarray.INDArray; @@ -41,7 +41,8 @@ public class NDCNN { /** * 2D Convolution layer operation - average pooling 2d
* - * @param input the input to average pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object * @return output Result after applying average pooling on the input (NUMERIC type) */ @@ -53,7 +54,9 @@ public class NDCNN { /** * 3D convolution layer operation - average pooling 3d
* - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param Pooling3DConfig Configuration Object * @return output after applying average pooling on the input (NUMERIC type) */ @@ -158,7 +161,9 @@ public class NDCNN { /** * Convolution 3D operation with optional bias
* - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param weights Weights for conv3d. Rank 5 with shape [kernelDepth, kernelHeight, kernelWidth, inputChannels, outputChannels]. (NUMERIC type) * @param bias Optional 1D bias array with shape [outputChannels]. May be null. (NUMERIC type) * @param Conv3DConfig Configuration Object @@ -175,7 +180,9 @@ public class NDCNN { /** * Convolution 3D operation with optional bias
* - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param weights Weights for conv3d. Rank 5 with shape [kernelDepth, kernelHeight, kernelWidth, inputChannels, outputChannels]. (NUMERIC type) * @param Conv3DConfig Configuration Object * @return output Conv3d output variable (NUMERIC type) @@ -189,7 +196,8 @@ public class NDCNN { /** * 2D deconvolution operation with optional bias
* - * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param weights Weights for the 2d deconvolution operation. 4 dimensions with format [inputChannels, outputChannels, kernelHeight, kernelWidth] (NUMERIC type) * @param bias Optional 1D bias array with shape [outputChannels]. May be null. (NUMERIC type) * @param DeConv2DConfig Configuration Object @@ -206,7 +214,8 @@ public class NDCNN { /** * 2D deconvolution operation with optional bias
* - * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to deconvolution 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param weights Weights for the 2d deconvolution operation. 4 dimensions with format [inputChannels, outputChannels, kernelHeight, kernelWidth] (NUMERIC type) * @param DeConv2DConfig Configuration Object * @return output result of deconv2d op (NUMERIC type) @@ -254,7 +263,8 @@ public class NDCNN { * Example: if input has shape [mb, 8, 2, 2] and block size is 2, then output size is [mb, 8/(2*2), 2*2, 2*2]
* = [mb, 2, 4, 4]
* - * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param blockSize Block size, in the height/width dimension * @param dataFormat Data format: "NCHW" or "NHWC" * @return output Output variable (NUMERIC type) @@ -363,7 +373,8 @@ public class NDCNN { /** * 2D Convolution layer operation - Max pooling on the input and outputs both max values and indices
* - * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object */ public INDArray[] maxPoolWithArgmax(INDArray input, Pooling2DConfig Pooling2DConfig) { @@ -374,7 +385,8 @@ public class NDCNN { /** * 2D Convolution layer operation - max pooling 2d
* - * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param input the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param Pooling2DConfig Configuration Object * @return output Result after applying max pooling on the input (NUMERIC type) */ @@ -386,7 +398,9 @@ public class NDCNN { /** * 3D convolution layer operation - max pooling 3d operation.
* - * @param input the input to average pooling 3d operation - 5d activations in NCDHW format (shape [minibatch, channels, depth, height, width]) or NDHWC format (shape [minibatch, depth, height, width, channels]) (NUMERIC type) + * @param input the input to average pooling 3d operation - 5d activations in NCDHW format + * (shape [minibatch, channels, depth, height, width]) or NDHWC format + * (shape [minibatch, depth, height, width, channels]) (NUMERIC type) * @param Pooling3DConfig Configuration Object * @return output Result after applying max pooling on the input (NUMERIC type) */ @@ -398,7 +412,8 @@ public class NDCNN { /** * Separable 2D convolution operation with optional bias
* - * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param depthWeights Separable conv2d depth weights. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, depthMultiplier] (NUMERIC type) * @param pointWeights Point weights, rank 4 with format [1, 1, inputChannels*depthMultiplier, outputChannels]. May be null (NUMERIC type) * @param bias Optional bias, rank 1 with shape [outputChannels]. May be null. (NUMERIC type) @@ -417,7 +432,8 @@ public class NDCNN { /** * Separable 2D convolution operation with optional bias
* - * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param layerInput the input to max pooling 2d operation - 4d CNN (image) activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param depthWeights Separable conv2d depth weights. 4 dimensions with format [kernelHeight, kernelWidth, inputChannels, depthMultiplier] (NUMERIC type) * @param pointWeights Point weights, rank 4 with format [1, 1, inputChannels*depthMultiplier, outputChannels]. May be null (NUMERIC type) * @param Conv2DConfig Configuration Object @@ -455,7 +471,8 @@ public class NDCNN { * Example: if input has shape [mb, 2, 4, 4] and block size is 2, then output size is [mb, 8/(2*2), 2*2, 2*2]
* = [mb, 2, 4, 4]
* - * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) + * @param x the input to depth to space pooling 2d operation - 4d activations in NCHW format + * (shape [minibatch, channels, height, width]) or NHWC format (shape [minibatch, height, width, channels]) (NUMERIC type) * @param blockSize Block size, in the height/width dimension * @param dataFormat Data format: "NCHW" or "NHWC" * @return output Output variable (NUMERIC type) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDImage.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDImage.java similarity index 93% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDImage.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDImage.java index e2443b300..5e6d2c947 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDImage.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDImage.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; -import static org.nd4j.linalg.factory.NDValidation.isSameType; - import org.nd4j.common.base.Preconditions; import org.nd4j.enums.ImageResizeMethod; import org.nd4j.linalg.api.ndarray.INDArray; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDLinalg.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDLinalg.java similarity index 92% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDLinalg.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDLinalg.java index 0112515dd..3bf70c4b0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDLinalg.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDLinalg.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; -import static org.nd4j.linalg.factory.NDValidation.isSameType; - import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.NDValidation; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDLoss.java similarity index 87% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDLoss.java index fd362144d..ec9f2b8b2 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDLoss.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDLoss.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; -import static org.nd4j.linalg.factory.NDValidation.isSameType; - import org.nd4j.autodiff.loss.LossReduce; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.NDValidation; @@ -35,7 +35,7 @@ public class NDLoss { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output loss variable (NUMERIC type) */ public INDArray absoluteDifference(INDArray label, INDArray predictions, INDArray weights, @@ -71,7 +71,7 @@ public class NDLoss { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is use (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param dimension Dimension to perform the cosine distance over * @return output Cosine distance loss (NUMERIC type) */ @@ -104,25 +104,6 @@ public class NDLoss { return Nd4j.exec(new org.nd4j.linalg.api.ops.impl.loss.CosineDistanceLoss(label, predictions, weights, org.nd4j.autodiff.loss.LossReduce.MEAN_BY_NONZERO_WEIGHT_COUNT, dimension))[0]; } - /** - * CTC Loss: Connectionist Temporal Classification Loss. See:
- * https://dl.acm.org/citation.cfm?id=1143891
- * - * @param targetLabels Label array (NUMERIC type) - * @param logitInput Inputs (NUMERIC type) - * @param targetLabelLengths Length of the target label (NUMERIC type) - * @param logitInputLengths Length of the input (NUMERIC type) - * @return output Ctc loss (NUMERIC type) - */ - public INDArray ctcLoss(INDArray targetLabels, INDArray logitInput, INDArray targetLabelLengths, - INDArray logitInputLengths) { - NDValidation.validateNumerical("ctcLoss", "targetLabels", targetLabels); - NDValidation.validateNumerical("ctcLoss", "logitInput", logitInput); - NDValidation.validateNumerical("ctcLoss", "targetLabelLengths", targetLabelLengths); - NDValidation.validateNumerical("ctcLoss", "logitInputLengths", logitInputLengths); - return Nd4j.exec(new org.nd4j.linalg.api.ops.impl.loss.CtcLoss(targetLabels, logitInput, targetLabelLengths, logitInputLengths))[0]; - } - /** * Hinge loss: a loss function used for training classifiers.
* Implements {@code L = max(0, 1 - t * predictions)} where t is the label values after internally converting to {-1,1}
@@ -131,7 +112,7 @@ public class NDLoss { * @param label Label array. Each value should be 0.0 or 1.0 (internally -1 to 1 is used) (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable (NUMERIC type) */ public INDArray hingeLoss(INDArray label, INDArray predictions, INDArray weights, @@ -171,7 +152,7 @@ public class NDLoss { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param delta Loss function delta value * @return output Huber loss (NUMERIC type) */ @@ -223,7 +204,7 @@ public class NDLoss { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param epsilon epsilon * @return output Log loss (NUMERIC type) */ @@ -256,7 +237,7 @@ public class NDLoss { * @param label Label array. Each value should be 0.0 or 1.0 (NUMERIC type) * @param predictions Predictions array (has to be log(x) of actual predictions) (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param full Boolean flag. true for logPoissonFull, false for logPoisson * @return output Loss variable (NUMERIC type) */ @@ -294,7 +275,7 @@ public class NDLoss { * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used. Must be either null, scalar, or have shape [batchSize] (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable, scalar output (NUMERIC type) */ public INDArray meanPairwiseSquaredError(INDArray label, INDArray predictions, INDArray weights, @@ -325,13 +306,13 @@ public class NDLoss { /** * Mean squared error loss function. Implements {@code (label[i] - prediction[i])^2} - i.e., squared error on a per-element basis.
- * When averaged (using LossReduce#MEAN_BY_WEIGHT or LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT (the default))
+ * When averaged (using {@link LossReduce#MEAN_BY_WEIGHT} or {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} (the default))
* this is the mean squared error loss function.
* * @param label Label array (NUMERIC type) * @param predictions Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @return output Loss variable (NUMERIC type) */ public INDArray meanSquaredError(INDArray label, INDArray predictions, INDArray weights, @@ -344,7 +325,7 @@ public class NDLoss { /** * Mean squared error loss function. Implements {@code (label[i] - prediction[i])^2} - i.e., squared error on a per-element basis.
- * When averaged (using LossReduce#MEAN_BY_WEIGHT or LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT (the default))
+ * When averaged (using {@link LossReduce#MEAN_BY_WEIGHT} or {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} (the default))
* this is the mean squared error loss function.
* * @param label Label array (NUMERIC type) @@ -376,7 +357,7 @@ public class NDLoss { * @param label Label array (NUMERIC type) * @param predictionLogits Predictions array (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param labelSmoothing Label smoothing value. Default value: 0 * @return output Loss variable (NUMERIC type) */ @@ -417,7 +398,7 @@ public class NDLoss { /** * Applies the softmax activation function to the input, then implement multi-class cross entropy:
* {@code -sum_classes label[i] * log(p[c])} where {@code p = softmax(logits)}
- * If LossReduce#NONE is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
+ * If {@link LossReduce#NONE} is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
* otherwise, the output is a scalar.
*


* When label smoothing is > 0, the following label smoothing is used:
@@ -429,7 +410,7 @@ public class NDLoss { * @param oneHotLabels Label array. Should be one-hot per example and same shape as predictions (for example, [mb, nOut]) (NUMERIC type) * @param logitPredictions Predictions array (pre-softmax) (NUMERIC type) * @param weights Weights array. May be null. If null, a weight of 1.0 is used (NUMERIC type) - * @param lossReduce Reduction type for the loss. See LossReduce for more details. Default: LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT + * @param lossReduce Reduction type for the loss. See {@link LossReduce} for more details. Default: {@link LossReduce#MEAN_BY_NONZERO_WEIGHT_COUNT} * @param labelSmoothing Label smoothing value. Default value: 0 * @return output Loss variable (NUMERIC type) */ @@ -444,7 +425,7 @@ public class NDLoss { /** * Applies the softmax activation function to the input, then implement multi-class cross entropy:
* {@code -sum_classes label[i] * log(p[c])} where {@code p = softmax(logits)}
- * If LossReduce#NONE is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
+ * If {@link LossReduce#NONE} is used, returned shape is [numExamples] out for [numExamples, numClasses] predicitons/labels;
* otherwise, the output is a scalar.
*


* When label smoothing is > 0, the following label smoothing is used:
diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDMath.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDMath.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDMath.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDMath.java index cf03080f0..2387e9177 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDMath.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDMath.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; -import static org.nd4j.linalg.factory.NDValidation.isSameType; - import org.nd4j.common.base.Preconditions; import org.nd4j.enums.PartitionMode; import org.nd4j.linalg.api.buffer.DataType; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDNN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDNN.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDNN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDNN.java index 55a3bb778..1ae68e99d 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDNN.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDNN.java @@ -1,29 +1,28 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; -import static org.nd4j.linalg.factory.NDValidation.isSameType; - import org.nd4j.common.base.Preconditions; import org.nd4j.enums.PadMode; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.Op; import org.nd4j.linalg.factory.NDValidation; import org.nd4j.linalg.factory.Nd4j; @@ -132,7 +131,7 @@ public class NDNN { */ public INDArray dropout(INDArray input, double inputRetainProbability) { NDValidation.validateNumerical("dropout", "input", input); - return Nd4j.exec((Op) new org.nd4j.linalg.api.ops.random.impl.DropOut(input, inputRetainProbability)); + return Nd4j.exec(new org.nd4j.linalg.api.ops.random.impl.DropOut(input, inputRetainProbability)); } /** diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDRNN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDRNN.java similarity index 82% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDRNN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDRNN.java index f9a594421..a60e0ef9e 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDRNN.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDRNN.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; -import static org.nd4j.linalg.factory.NDValidation.isSameType; - import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.impl.layers.recurrent.config.LSTMConfiguration; import org.nd4j.linalg.api.ops.impl.layers.recurrent.config.LSTMLayerConfig; @@ -85,22 +85,22 @@ public class NDRNN { /** * Long Short-Term Memory layer - Hochreiter 1997.
- * SUPPORTS following data formats:
- * for unidirectional:
- * TNS: shapes [timeLength, numExamples, inOutSize]
- * NST: shapes [numExamples, inOutSize, timeLength]
+ * SUPPORTS following data formats:\n
+ * for unidirectional: \n" +
+ * TNS: shapes [timeLength, numExamples, inOutSize]\n
+ * NST: shapes [numExamples, inOutSize, timeLength]\n
* NTS: shapes [numExamples, timeLength, inOutSize]
- * for bidirectional:
- * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)
- * SUPPORTS following direction modes:
+ * for bidirectional:\n
+ * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)\n
+ * SUPPORTS following direction modes:\n
* FWD: forward
* BWD: backward
- * BIDIR_SUM: bidirectional sum
- * BIDIR_CONCAT: bidirectional concat
- * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)
+ * BIDIR_SUM: bidirectional sum\n
+ * BIDIR_CONCAT: bidirectional concat\n" +
+ * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)"
* You may use different gate configurations:
- * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum
- * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")
+ * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum\n
+ * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")\n
* Also this layer supports MKLDNN (DNNL) and cuDNN acceleration
* * @param x Input, with shape dependent on the data format (in config). (NUMERIC type) @@ -121,22 +121,22 @@ public class NDRNN { /** * Long Short-Term Memory layer - Hochreiter 1997.
- * SUPPORTS following data formats:
- * for unidirectional:
- * TNS: shapes [timeLength, numExamples, inOutSize]
- * NST: shapes [numExamples, inOutSize, timeLength]
+ * SUPPORTS following data formats:\n
+ * for unidirectional: \n" +
+ * TNS: shapes [timeLength, numExamples, inOutSize]\n
+ * NST: shapes [numExamples, inOutSize, timeLength]\n
* NTS: shapes [numExamples, timeLength, inOutSize]
- * for bidirectional:
- * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)
- * SUPPORTS following direction modes:
+ * for bidirectional:\n
+ * T2NS: shapes [timeLength, 2, numExamples, inOutSize] (for ONNX)\n
+ * SUPPORTS following direction modes:\n
* FWD: forward
* BWD: backward
- * BIDIR_SUM: bidirectional sum
- * BIDIR_CONCAT: bidirectional concat
- * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)
+ * BIDIR_SUM: bidirectional sum\n
+ * BIDIR_CONCAT: bidirectional concat\n" +
+ * BIDIR_EXTRA_DIM: bidirectional extra output dim (in conjunction with format dataFormat - T2NS)"
* You may use different gate configurations:
- * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum
- * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")
+ * specify gate/cell/out aplha/beta and numbers of activations for gate/cell/out described in activations enum\n
+ * ("RELU","SIGMOID","AFFINE","LEAKY_RELU","THRESHHOLD_RELU","SCALED_TAHN","HARD_SIGMOID","ELU","SOFTSIGN","SOFTPLUS")\n
* Also this layer supports MKLDNN (DNNL) and cuDNN acceleration
* * @param x Input, with shape dependent on the data format (in config). (NUMERIC type) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDRandom.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDRandom.java similarity index 88% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDRandom.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDRandom.java index f40a67891..76f57c52a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/NDRandom.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/factory/ops/NDRandom.java @@ -1,25 +1,25 @@ -/******************************************************************************* - * Copyright (c) 2019-2020 Konduit K.K. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -//================== GENERATED CODE - DO NOT MODIFY THIS FILE ================== +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ package org.nd4j.linalg.factory.ops; -import static org.nd4j.linalg.factory.NDValidation.isSameType; - import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/Heartbeat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/Heartbeat.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/Heartbeat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/Heartbeat.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Environment.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Environment.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Environment.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Environment.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Event.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Event.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Event.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Event.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Task.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Task.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Task.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/reports/Task.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/utils/EnvironmentUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/utils/EnvironmentUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/utils/EnvironmentUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/utils/EnvironmentUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/utils/TaskUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/utils/TaskUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/heartbeat/utils/TaskUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/heartbeat/utils/TaskUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/BooleanIndexing.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/BooleanIndexing.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/BooleanIndexing.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/BooleanIndexing.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/INDArrayIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/INDArrayIndex.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/INDArrayIndex.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/INDArrayIndex.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/IndexInfo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/IndexInfo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/IndexInfo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/IndexInfo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/Indices.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/Indices.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/Indices.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/Indices.java index fae60a3fe..fb505698a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/Indices.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/Indices.java @@ -21,8 +21,8 @@ package org.nd4j.linalg.indexing; import org.nd4j.linalg.exception.ND4JArraySizeException; -import org.nd4j.shade.guava.primitives.Ints; -import org.nd4j.shade.guava.primitives.Longs; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.shape.Shape; import org.nd4j.linalg.exception.ND4JIllegalStateException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/IntervalIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/IntervalIndex.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/IntervalIndex.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/IntervalIndex.java index 7a0df157e..9939e5465 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/IntervalIndex.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/IntervalIndex.java @@ -20,7 +20,7 @@ package org.nd4j.linalg.indexing; -import org.nd4j.shade.guava.primitives.Longs; +import com.google.common.primitives.Longs; import lombok.Getter; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.ndarray.INDArray; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/NDArrayIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/NDArrayIndex.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/NDArrayIndex.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/NDArrayIndex.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/NDArrayIndexAll.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/NDArrayIndexAll.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/NDArrayIndexAll.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/NDArrayIndexAll.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/NewAxis.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/NewAxis.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/NewAxis.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/NewAxis.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/PointIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/PointIndex.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/PointIndex.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/PointIndex.java index 0d33aa2f1..acd69d529 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/PointIndex.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/PointIndex.java @@ -20,7 +20,6 @@ package org.nd4j.linalg.indexing; -import org.nd4j.shade.guava.primitives.Longs; import lombok.EqualsAndHashCode; import org.nd4j.linalg.api.ndarray.INDArray; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/SpecifiedIndex.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/SpecifiedIndex.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/SpecifiedIndex.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/SpecifiedIndex.java index 40a396288..f2109e8d2 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/SpecifiedIndex.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/SpecifiedIndex.java @@ -20,7 +20,6 @@ package org.nd4j.linalg.indexing; -import org.nd4j.shade.guava.primitives.Longs; import lombok.Data; import net.ericaro.neoitertools.Generator; import net.ericaro.neoitertools.Itertools; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueGreaterOrEqualsThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueGreaterOrEqualsThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueGreaterOrEqualsThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueGreaterOrEqualsThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueGreaterThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueGreaterThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueGreaterThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueGreaterThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueLessOrEqualsThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueLessOrEqualsThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueLessOrEqualsThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueLessOrEqualsThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueLessThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueLessThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueLessThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/AbsValueLessThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/And.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/And.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/And.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/And.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/BaseCondition.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/BaseCondition.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/BaseCondition.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/BaseCondition.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/Condition.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/Condition.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/Condition.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/Condition.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/ConditionBuilder.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/ConditionBuilder.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/ConditionBuilder.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/ConditionBuilder.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/ConditionEquals.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/ConditionEquals.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/ConditionEquals.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/ConditionEquals.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/Conditions.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/Conditions.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/Conditions.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/Conditions.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/EpsilonEquals.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/EpsilonEquals.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/EpsilonEquals.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/EpsilonEquals.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/EpsilonNotEquals.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/EpsilonNotEquals.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/EpsilonNotEquals.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/EpsilonNotEquals.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/EqualsCondition.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/EqualsCondition.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/EqualsCondition.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/EqualsCondition.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/GreaterThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/GreaterThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/GreaterThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/GreaterThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/GreaterThanOrEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/GreaterThanOrEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/GreaterThanOrEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/GreaterThanOrEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsFinite.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsFinite.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsFinite.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsFinite.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsInfinite.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsInfinite.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsInfinite.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsInfinite.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsNaN.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsNaN.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsNaN.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/IsNaN.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/LessThan.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/LessThan.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/LessThan.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/LessThan.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/LessThanOrEqual.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/LessThanOrEqual.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/LessThanOrEqual.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/LessThanOrEqual.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/Not.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/Not.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/Not.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/Not.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/NotEqualsCondition.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/NotEqualsCondition.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/NotEqualsCondition.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/NotEqualsCondition.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/NotFinite.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/NotFinite.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/NotFinite.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/NotFinite.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/Or.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/Or.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/conditions/Or.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/conditions/Or.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/functions/Identity.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/functions/Identity.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/functions/Identity.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/functions/Identity.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/functions/StableNumber.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/functions/StableNumber.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/functions/StableNumber.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/functions/StableNumber.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/functions/Value.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/functions/Value.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/functions/Value.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/functions/Value.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/functions/Zero.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/functions/Zero.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/indexing/functions/Zero.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/indexing/functions/Zero.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/inverse/InvertMatrix.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/inverse/InvertMatrix.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/inverse/InvertMatrix.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/inverse/InvertMatrix.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AMSGradUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AMSGradUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AMSGradUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AMSGradUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdaBeliefUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdaBeliefUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdaBeliefUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdaBeliefUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdaDeltaUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdaDeltaUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdaDeltaUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdaDeltaUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdaGradUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdaGradUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdaGradUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdaGradUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdaMaxUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdaMaxUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdaMaxUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdaMaxUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdamUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdamUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/AdamUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/AdamUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/GradientUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/GradientUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/GradientUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/GradientUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/NadamUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/NadamUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/NadamUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/NadamUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/NesterovsUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/NesterovsUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/NesterovsUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/NesterovsUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/NoOpUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/NoOpUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/NoOpUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/NoOpUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/RmsPropUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/RmsPropUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/RmsPropUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/RmsPropUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/SgdUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/SgdUpdater.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/SgdUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/SgdUpdater.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AMSGrad.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AMSGrad.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AMSGrad.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AMSGrad.java index af96e84db..b76ef26ef 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AMSGrad.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AMSGrad.java @@ -26,7 +26,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.AMSGradUpdater; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaBelief.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaBelief.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaBelief.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaBelief.java index aa5d3f00d..9d3da0a5c 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaBelief.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaBelief.java @@ -26,7 +26,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.AdaBeliefUpdater; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaDelta.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaDelta.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaDelta.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaDelta.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaGrad.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaGrad.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaGrad.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaGrad.java index 67481b04f..d58e9b16a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaGrad.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaGrad.java @@ -26,7 +26,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.AdaGradUpdater; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java index a92c47fde..72785aba4 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/AdaMax.java @@ -25,7 +25,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.AdaMaxUpdater; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Adam.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Adam.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Adam.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Adam.java index 5eaefbb6d..10d7d83d0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Adam.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Adam.java @@ -26,7 +26,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.AdamUpdater; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/IUpdater.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/IUpdater.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/IUpdater.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/IUpdater.java index 55ff3226b..2cd538fd8 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/IUpdater.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/IUpdater.java @@ -23,9 +23,9 @@ package org.nd4j.linalg.learning.config; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Nadam.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Nadam.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Nadam.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Nadam.java index 0e09643e4..b92fde302 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Nadam.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Nadam.java @@ -26,7 +26,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.learning.NadamUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Nesterovs.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Nesterovs.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Nesterovs.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Nesterovs.java index 0512de280..3309b26cc 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Nesterovs.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Nesterovs.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.learning.NesterovsUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/NoOp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/NoOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/NoOp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/NoOp.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/RmsProp.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/RmsProp.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/RmsProp.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/RmsProp.java index 1d54876d5..017336fd9 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/RmsProp.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/RmsProp.java @@ -26,7 +26,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.learning.RmsPropUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Sgd.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Sgd.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Sgd.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Sgd.java index d41cf2b02..8ee0a4386 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/config/Sgd.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/config/Sgd.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.GradientUpdater; import org.nd4j.linalg.learning.SgdUpdater; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/legacy/AdaGrad.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/legacy/AdaGrad.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/legacy/AdaGrad.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/legacy/AdaGrad.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/L1Regularization.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/L1Regularization.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/L1Regularization.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/L1Regularization.java index 04146221f..c140cf1af 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/L1Regularization.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/L1Regularization.java @@ -28,7 +28,7 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.linalg.schedule.FixedSchedule; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class L1Regularization implements Regularization { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/L2Regularization.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/L2Regularization.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/L2Regularization.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/L2Regularization.java index d95f5140d..decc54fb6 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/L2Regularization.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/L2Regularization.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.Axpy; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.schedule.FixedSchedule; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class L2Regularization implements Regularization { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/Regularization.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/Regularization.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/Regularization.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/Regularization.java index 83ff01002..1b6ad799d 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/Regularization.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/Regularization.java @@ -21,7 +21,7 @@ package org.nd4j.linalg.learning.regularization; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/WeightDecay.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/WeightDecay.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/WeightDecay.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/WeightDecay.java index 29d8766a3..92bb08141 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/learning/regularization/WeightDecay.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/learning/regularization/WeightDecay.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.Axpy; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.schedule.FixedSchedule; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class WeightDecay implements Regularization { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/ILossFunction.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/ILossFunction.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/ILossFunction.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/ILossFunction.java index 992dd52ae..78f7ba762 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/ILossFunction.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/ILossFunction.java @@ -25,7 +25,7 @@ import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; import org.nd4j.serde.json.LegacyILossFunctionDeserializerHelper; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/LossFunctions.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/LossFunctions.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/LossFunctions.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/LossFunctions.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/LossUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/LossUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/LossUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/LossUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/SameDiffLoss.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/SameDiffLoss.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/SameDiffLoss.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/SameDiffLoss.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossBinaryXENT.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossBinaryXENT.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossBinaryXENT.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossBinaryXENT.java index db8bd50aa..70ad03d19 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossBinaryXENT.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossBinaryXENT.java @@ -38,10 +38,10 @@ import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Pair; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossCosineProximity.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossCosineProximity.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossCosineProximity.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossCosineProximity.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossFMeasure.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossFMeasure.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossFMeasure.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossFMeasure.java index bec17a86c..6ec868cdb 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossFMeasure.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossFMeasure.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.lossfunctions.ILossFunction; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Getter @EqualsAndHashCode diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossHinge.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossHinge.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossHinge.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossHinge.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossKLD.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossKLD.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossKLD.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossKLD.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL1.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL1.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL1.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL1.java index 8c868faa3..6260f4b82 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL1.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL1.java @@ -33,9 +33,9 @@ import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Pair; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL2.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL2.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL2.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL2.java index 86c97dfec..790cdd759 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL2.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossL2.java @@ -30,10 +30,10 @@ import org.nd4j.linalg.lossfunctions.LossUtil; import org.nd4j.common.primitives.Pair; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAE.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAE.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAE.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAE.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAPE.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAPE.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAPE.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAPE.java index 0debfa933..7b1b3335a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAPE.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMAPE.java @@ -34,9 +34,9 @@ import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Pair; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMCXENT.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMCXENT.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMCXENT.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMCXENT.java index 003702ec8..dde8f53e2 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMCXENT.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMCXENT.java @@ -36,10 +36,10 @@ import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Pair; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSE.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSE.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSE.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSE.java index 960830ae6..192a4939b 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSE.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSE.java @@ -23,7 +23,7 @@ package org.nd4j.linalg.lossfunctions.impl; import lombok.EqualsAndHashCode; import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode(callSuper = true) public class LossMSE extends LossL2 { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSLE.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSLE.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSLE.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSLE.java index e7e4f83c8..905677a3a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSLE.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMSLE.java @@ -31,9 +31,9 @@ import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Pair; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMixtureDensity.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMixtureDensity.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMixtureDensity.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMixtureDensity.java index 734112248..14894362b 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMixtureDensity.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMixtureDensity.java @@ -33,8 +33,8 @@ import org.nd4j.linalg.lossfunctions.ILossFunction; import org.nd4j.linalg.lossfunctions.LossUtil; import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMultiLabel.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMultiLabel.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMultiLabel.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMultiLabel.java index 27683254f..baf24ec96 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMultiLabel.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossMultiLabel.java @@ -29,7 +29,7 @@ import org.nd4j.linalg.lossfunctions.ILossFunction; import org.nd4j.linalg.lossfunctions.LossUtil; import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossNegativeLogLikelihood.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossNegativeLogLikelihood.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossNegativeLogLikelihood.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossNegativeLogLikelihood.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossPoisson.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossPoisson.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossPoisson.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossPoisson.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSparseMCXENT.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSparseMCXENT.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSparseMCXENT.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSparseMCXENT.java index c2803c7c4..fd8c612bc 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSparseMCXENT.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSparseMCXENT.java @@ -30,10 +30,8 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.impl.shape.OneHot; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode(callSuper = true) @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSquaredHinge.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSquaredHinge.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSquaredHinge.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossSquaredHinge.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossWasserstein.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossWasserstein.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossWasserstein.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/impl/LossWasserstein.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorDeserializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorDeserializer.java similarity index 88% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorDeserializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorDeserializer.java index 0acb4dca5..4b21a53db 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorDeserializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorDeserializer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.lossfunctions.serde; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorSerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorSerializer.java similarity index 89% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorSerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorSerializer.java index 92404bc5d..16e05b258 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorSerializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/lossfunctions/serde/RowVectorSerializer.java @@ -21,9 +21,9 @@ package org.nd4j.linalg.lossfunctions.serde; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/ops/transforms/Transforms.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/ops/transforms/Transforms.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/ops/transforms/Transforms.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/ops/transforms/Transforms.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/OpProfiler.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/OpProfiler.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/OpProfiler.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/OpProfiler.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/ProfilerConfig.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/ProfilerConfig.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/ProfilerConfig.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/ProfilerConfig.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/StackAggregator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/StackAggregator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/StackAggregator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/StackAggregator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/StringAggregator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/StringAggregator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/StringAggregator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/StringAggregator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/StringCounter.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/StringCounter.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/StringCounter.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/StringCounter.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/ComparableAtomicLong.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/ComparableAtomicLong.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/ComparableAtomicLong.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/ComparableAtomicLong.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackComparator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackComparator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackComparator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackComparator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackDescriptor.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackDescriptor.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackNode.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackNode.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackNode.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackNode.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackTree.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackTree.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackTree.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/StackTree.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/TimeSet.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/TimeSet.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/TimeSet.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/profiler/data/primitives/TimeSet.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/CycleSchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/CycleSchedule.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/CycleSchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/CycleSchedule.java index f2a348ed8..23aac6ab2 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/CycleSchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/CycleSchedule.java @@ -21,7 +21,7 @@ package org.nd4j.linalg.schedule; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class CycleSchedule implements ISchedule { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/ExponentialSchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/ExponentialSchedule.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/ExponentialSchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/ExponentialSchedule.java index 52661e167..f929ad98c 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/ExponentialSchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/ExponentialSchedule.java @@ -21,7 +21,7 @@ package org.nd4j.linalg.schedule; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class ExponentialSchedule implements ISchedule { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/FixedSchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/FixedSchedule.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/FixedSchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/FixedSchedule.java index 8744257ef..5cb1e21ec 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/FixedSchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/FixedSchedule.java @@ -21,7 +21,7 @@ package org.nd4j.linalg.schedule; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class FixedSchedule implements ISchedule{ diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/ISchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/ISchedule.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/ISchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/ISchedule.java index 74a64d6a0..2954c9ead 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/ISchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/ISchedule.java @@ -20,7 +20,7 @@ package org.nd4j.linalg.schedule; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/InverseSchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/InverseSchedule.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/InverseSchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/InverseSchedule.java index 4e04284ca..77244d3c6 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/InverseSchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/InverseSchedule.java @@ -22,7 +22,7 @@ package org.nd4j.linalg.schedule; import lombok.Data; import lombok.EqualsAndHashCode; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/MapSchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/MapSchedule.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/MapSchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/MapSchedule.java index 8fa76928e..578996bb9 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/MapSchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/MapSchedule.java @@ -23,8 +23,8 @@ package org.nd4j.linalg.schedule; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NonNull; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; import java.util.HashMap; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/PolySchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/PolySchedule.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/PolySchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/PolySchedule.java index 8341ea586..dad4ff237 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/PolySchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/PolySchedule.java @@ -21,7 +21,7 @@ package org.nd4j.linalg.schedule; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class PolySchedule implements ISchedule { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/RampSchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/RampSchedule.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/RampSchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/RampSchedule.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/ScheduleType.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/ScheduleType.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/ScheduleType.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/ScheduleType.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/SigmoidSchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/SigmoidSchedule.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/SigmoidSchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/SigmoidSchedule.java index ff7cdb5ff..7cbdede4c 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/SigmoidSchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/SigmoidSchedule.java @@ -21,7 +21,7 @@ package org.nd4j.linalg.schedule; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class SigmoidSchedule implements ISchedule { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/StepSchedule.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/StepSchedule.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/StepSchedule.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/StepSchedule.java index ab59d6957..0c709a5ae 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/schedule/StepSchedule.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/schedule/StepSchedule.java @@ -21,7 +21,7 @@ package org.nd4j.linalg.schedule; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class StepSchedule implements ISchedule { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/string/NDArrayStrings.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/string/NDArrayStrings.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/string/NDArrayStrings.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/string/NDArrayStrings.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/AtomicThrowable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/AtomicThrowable.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/AtomicThrowable.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/AtomicThrowable.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/ConvConfigUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/ConvConfigUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/ConvConfigUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/ConvConfigUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/DataSetUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/DataSetUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/DataSetUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/DataSetUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/DeviceLocal.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/DeviceLocal.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/DeviceLocal.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/DeviceLocal.java index 35a32691a..a3f051c25 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/DeviceLocal.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/DeviceLocal.java @@ -23,7 +23,6 @@ package org.nd4j.linalg.util; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import edu.umd.cs.findbugs.annotations.Nullable; import java.util.ArrayList; import java.util.List; @@ -57,7 +56,6 @@ public abstract class DeviceLocal { * * @return */ - @Nullable public T get() { return get(Nd4j.getAffinityManager().getDeviceForCurrentThread()); } @@ -68,7 +66,6 @@ public abstract class DeviceLocal { * @param deviceId * @return */ - @Nullable public T get(int deviceId) { try { locksMap.get(deviceId).readLock().lock(); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/DeviceLocalNDArray.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/DeviceLocalNDArray.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/DeviceLocalNDArray.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/DeviceLocalNDArray.java index f51bf1cf9..376103866 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/DeviceLocalNDArray.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/DeviceLocalNDArray.java @@ -20,7 +20,6 @@ package org.nd4j.linalg.util; -import edu.umd.cs.findbugs.annotations.Nullable; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -57,7 +56,6 @@ public class DeviceLocalNDArray extends DeviceLocal { * * @return */ - @Nullable @Override public synchronized INDArray get() { val deviceId = Nd4j.getAffinityManager().getDeviceForCurrentThread(); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/FeatureUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/FeatureUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/FeatureUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/FeatureUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/HashUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/HashUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/HashUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/HashUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/LinAlgExceptions.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/LinAlgExceptions.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/LinAlgExceptions.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/LinAlgExceptions.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/LongUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/LongUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/LongUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/LongUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/ND4JTestUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/ND4JTestUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/ND4JTestUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/ND4JTestUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/NDArrayMath.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/NDArrayMath.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/NDArrayMath.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/NDArrayMath.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/NDArrayPreconditionsFormat.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/NDArrayPreconditionsFormat.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/NDArrayPreconditionsFormat.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/NDArrayPreconditionsFormat.java index 74d99013f..f86fc76df 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/NDArrayPreconditionsFormat.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/NDArrayPreconditionsFormat.java @@ -20,7 +20,6 @@ package org.nd4j.linalg.util; -import org.nd4j.common.base.Preconditions; import org.nd4j.common.base.PreconditionsFormat; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.indexing.NDArrayIndex; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/NDArrayUtil.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/NDArrayUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/NDArrayUtil.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/NDArrayUtil.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/Nd4jValidator.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/Nd4jValidator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/util/Nd4jValidator.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/util/Nd4jValidator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/BaseWorkspaceMgr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/BaseWorkspaceMgr.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/BaseWorkspaceMgr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/BaseWorkspaceMgr.java index 3ed50f4ad..af0c04d08 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/BaseWorkspaceMgr.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/BaseWorkspaceMgr.java @@ -249,7 +249,7 @@ public abstract class BaseWorkspaceMgr> implements WorkspaceMg } @Override - public INDArray create(@NonNull T arrayType, @NonNull DataType dataType, @NonNull long[] shape, @NonNull char order) { + public INDArray create(@NonNull T arrayType, @NonNull DataType dataType, @NonNull long[] shape, char order) { enforceExistsAndActive(arrayType); try(MemoryWorkspace ws = notifyScopeBorrowed(arrayType)){ return Nd4j.create(dataType, shape, order); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/ND4JWorkspaceException.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/ND4JWorkspaceException.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/ND4JWorkspaceException.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/ND4JWorkspaceException.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/WorkspaceMgr.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/WorkspaceMgr.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/WorkspaceMgr.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/WorkspaceMgr.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/WorkspaceUtils.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/WorkspaceUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/WorkspaceUtils.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/WorkspaceUtils.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/WorkspacesCloseable.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/WorkspacesCloseable.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/workspace/WorkspacesCloseable.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/linalg/workspace/WorkspacesCloseable.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/list/BaseNDArrayList.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/list/BaseNDArrayList.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/list/BaseNDArrayList.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/list/BaseNDArrayList.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/list/NDArrayList.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/list/NDArrayList.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/list/NDArrayList.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/list/NDArrayList.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/base64/Nd4jBase64.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/base64/Nd4jBase64.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/base64/Nd4jBase64.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/base64/Nd4jBase64.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/binary/BinarySerde.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/binary/BinarySerde.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/binary/BinarySerde.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/binary/BinarySerde.java index 2da2d9c04..ddb6278fb 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/binary/BinarySerde.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/binary/BinarySerde.java @@ -75,8 +75,7 @@ public class BinarySerde { ByteBuffer byteBuffer = buffer.hasArray() ? ByteBuffer.allocateDirect(buffer.array().length).put(buffer.array()) .order(ByteOrder.nativeOrder()) : buffer.order(ByteOrder.nativeOrder()); //bump the byte buffer to the proper position - Buffer buffer1 = (Buffer) byteBuffer; - buffer1.position(offset); + byteBuffer.position(offset); int rank = byteBuffer.getInt(); if (rank < 0) throw new IllegalStateException("Found negative integer. Corrupt serialization?"); @@ -100,8 +99,7 @@ public class BinarySerde { DataBuffer buff = Nd4j.createBuffer(slice, type, (int) Shape.length(shapeBuff)); //advance past the data int position = byteBuffer.position() + (buff.getElementSize() * (int) buff.length()); - Buffer buffer2 = (Buffer) byteBuffer; - buffer2.position(position); + byteBuffer.position(position); //create the final array //TODO: see how to avoid dup here INDArray arr = Nd4j.createArrayFromShapeBuffer(buff.dup(), shapeBuff.dup()); @@ -118,8 +116,7 @@ public class BinarySerde { INDArray arr = Nd4j.createArrayFromShapeBuffer(compressedDataBuffer.dup(), shapeBuff.dup()); //advance past the data int compressLength = (int) compressionDescriptor.getCompressedLength(); - Buffer buffer2 = (Buffer) byteBuffer; - buffer2.position(buffer2.position() + compressLength); + byteBuffer.position(byteBuffer.position() + compressLength); return Pair.of(arr, byteBuffer); } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayDeSerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayDeSerializer.java similarity index 86% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayDeSerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayDeSerializer.java index 8dfd3092d..f4fe5bcc5 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayDeSerializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayDeSerializer.java @@ -22,10 +22,10 @@ package org.nd4j.serde.jackson.shaded; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.serde.base64.Nd4jBase64; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArraySerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArraySerializer.java similarity index 90% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArraySerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArraySerializer.java index b86f27319..240fd86a1 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArraySerializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArraySerializer.java @@ -23,9 +23,9 @@ package org.nd4j.serde.jackson.shaded; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.serde.base64.Nd4jBase64; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextDeSerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextDeSerializer.java similarity index 93% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextDeSerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextDeSerializer.java index 019ea0e35..666d7c150 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextDeSerializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextDeSerializer.java @@ -23,11 +23,11 @@ package org.nd4j.serde.jackson.shaded; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.io.IOException; import java.util.Iterator; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java index 8f56783cb..558ffa58c 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java @@ -25,9 +25,9 @@ package org.nd4j.serde.jackson.shaded; import lombok.val; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/BaseLegacyDeserializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/BaseLegacyDeserializer.java similarity index 92% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/BaseLegacyDeserializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/BaseLegacyDeserializer.java index 1d82cddfc..b0a123c39 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/BaseLegacyDeserializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/BaseLegacyDeserializer.java @@ -22,11 +22,11 @@ package org.nd4j.serde.json; import lombok.extern.slf4j.Slf4j; import org.nd4j.common.config.ND4JClassLoading; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.ArrayList; diff --git a/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/JsonMappers.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/JsonMappers.java new file mode 100644 index 000000000..8190f6141 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/JsonMappers.java @@ -0,0 +1,78 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.serde.json; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.primitives.AtomicBoolean; +import org.nd4j.common.primitives.AtomicDouble; +import org.nd4j.common.primitives.serde.JsonDeserializerAtomicBoolean; +import org.nd4j.common.primitives.serde.JsonDeserializerAtomicDouble; +import org.nd4j.common.primitives.serde.JsonSerializerAtomicBoolean; +import org.nd4j.common.primitives.serde.JsonSerializerAtomicDouble; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +@Slf4j +public class JsonMappers { + + private static ObjectMapper jsonMapper = configureMapper(new ObjectMapper()); + private static ObjectMapper yamlMapper = configureMapper(new ObjectMapper(new YAMLFactory())); + + /** + * @return The default/primary ObjectMapper for deserializing JSON objects + */ + public static ObjectMapper getMapper(){ + return jsonMapper; + } + + /** + * @return The default/primary ObjectMapper for deserializing JSON objects + */ + public static ObjectMapper getYamlMapper(){ + return jsonMapper; + } + + private static ObjectMapper configureMapper(ObjectMapper ret) { + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, false); + ret.enable(SerializationFeature.INDENT_OUTPUT); + SimpleModule atomicModule = new SimpleModule(); + atomicModule.addSerializer(AtomicDouble.class, new JsonSerializerAtomicDouble()); + atomicModule.addSerializer(AtomicBoolean.class, new JsonSerializerAtomicBoolean()); + atomicModule.addDeserializer(AtomicDouble.class, new JsonDeserializerAtomicDouble()); + atomicModule.addDeserializer(AtomicBoolean.class, new JsonDeserializerAtomicBoolean()); + ret.registerModule(atomicModule); + //Serialize fields only, not using getters + ret.setVisibilityChecker(ret.getSerializationConfig().getDefaultVisibilityChecker() + .withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withSetterVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.ANY) + ); + return ret; + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializer.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializer.java index d906f112c..a763385fd 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializer.java @@ -23,7 +23,7 @@ package org.nd4j.serde.json; import lombok.NonNull; import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.activations.impl.*; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializerHelper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializerHelper.java similarity index 94% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializerHelper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializerHelper.java index 6edeb59d0..804bf62f2 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializerHelper.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyIActivationDeserializerHelper.java @@ -20,7 +20,7 @@ package org.nd4j.serde.json; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @JsonDeserialize(using = LegacyIActivationDeserializer.class) public class LegacyIActivationDeserializerHelper { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializer.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializer.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializer.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializer.java index 3bc3387a1..b0a5c1b72 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializer.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializer.java @@ -24,7 +24,7 @@ package org.nd4j.serde.json; import lombok.NonNull; import org.nd4j.linalg.lossfunctions.ILossFunction; import org.nd4j.linalg.lossfunctions.impl.*; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializerHelper.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializerHelper.java similarity index 94% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializerHelper.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializerHelper.java index ef1954766..490508865 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializerHelper.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/serde/json/LegacyILossFunctionDeserializerHelper.java @@ -20,7 +20,7 @@ package org.nd4j.serde.json; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @JsonDeserialize(using = LegacyILossFunctionDeserializer.class) public class LegacyILossFunctionDeserializerHelper { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/systeminfo/GPUInfo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/systeminfo/GPUInfo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfoProvider.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/systeminfo/GPUInfoProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfoProvider.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/systeminfo/GPUInfoProvider.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/SystemInfo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/systeminfo/SystemInfo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/SystemInfo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/systeminfo/SystemInfo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/versioncheck/VersionCheck.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/versioncheck/VersionCheck.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/versioncheck/VersionCheck.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/versioncheck/VersionCheck.java index 1ce48b856..5ca7116f0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/versioncheck/VersionCheck.java +++ b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/versioncheck/VersionCheck.java @@ -250,7 +250,7 @@ public class VersionCheck { } } catch (NoClassDefFoundError e){ //Should only happen on Android 7.0 or earlier - silently ignore - //https://github.com/eclipse/deeplearning4j/issues/6609 + //https://github.com/deeplearning4j/deeplearning4j/issues/6609 } catch (Throwable e){ //log and skip log.debug("Error finding/loading version check resources", e); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/versioncheck/VersionInfo.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/versioncheck/VersionInfo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/versioncheck/VersionInfo.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/versioncheck/VersionInfo.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/BaseWeightInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/BaseWeightInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/BaseWeightInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/BaseWeightInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/WeightInit.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/WeightInit.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/WeightInit.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/WeightInit.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/WeightInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/WeightInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/WeightInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/WeightInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/ConstantInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/ConstantInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/ConstantInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/ConstantInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/DistributionInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/DistributionInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/DistributionInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/DistributionInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/IdentityInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/IdentityInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/IdentityInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/IdentityInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/LecunUniformInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/LecunUniformInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/LecunUniformInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/LecunUniformInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/NDArraySupplierInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/NDArraySupplierInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/NDArraySupplierInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/NDArraySupplierInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/OneInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/OneInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/OneInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/OneInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/ReluInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/ReluInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/ReluInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/ReluInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/ReluUniformInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/ReluUniformInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/ReluUniformInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/ReluUniformInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/SigmoidUniformInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/SigmoidUniformInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/SigmoidUniformInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/SigmoidUniformInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/UniformInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/UniformInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/UniformInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/UniformInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanAvgInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanAvgInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanAvgInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanAvgInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanInInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanInInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanInInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanInInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanOutInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanOutInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanOutInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalFanOutInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalUniformFanInInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalUniformFanInInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalUniformFanInInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalUniformFanInInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalUniformFanOutInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalUniformFanOutInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalUniformFanOutInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingNormalUniformFanOutInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingUniformFanAvgInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingUniformFanAvgInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/VarScalingUniformFanAvgInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/VarScalingUniformFanAvgInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/XavierFanInInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/XavierFanInInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/XavierFanInInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/XavierFanInInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/XavierInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/XavierInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/XavierInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/XavierInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/XavierUniformInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/XavierUniformInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/XavierUniformInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/XavierUniformInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/ZeroInitScheme.java b/cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/ZeroInitScheme.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/weightinit/impl/ZeroInitScheme.java rename to cavis-dnn/cavis-dnn-api/src/main/java/org/nd4j/weightinit/impl/ZeroInitScheme.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/nd4j/mapper.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/nd4j/mapper.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/nd4j/mapper.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/nd4j/mapper.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/nd4j/op.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/nd4j/op.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/nd4j/op.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/nd4j/op.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/nd4j/tensor.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/nd4j/tensor.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/nd4j/tensor.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/nd4j/tensor.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/onnx-ml.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/onnx/onnx-ml.proto.xxx similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/onnx-ml.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/onnx/onnx-ml.proto.xxx diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/onnx-operators.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/onnx/onnx-operators.proto similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/onnx-operators.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/onnx/onnx-operators.proto index 48890a516..1acd5ee0b 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/onnx-operators.proto +++ b/cavis-dnn/cavis-dnn-api/src/main/protobuf/onnx/onnx-operators.proto @@ -9,7 +9,7 @@ syntax = "proto3"; package onnx; -import "onnx.proto"; + // // This file contains the proto definitions for OperatorSetProto and diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/onnx.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/onnx/onnx.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/onnx.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/onnx/onnx.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/allocation_description.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/allocation_description.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/allocation_description.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/allocation_description.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/api_def.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/api_def.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/api_def.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/api_def.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/attr_value.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/attr_value.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/attr_value.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/attr_value.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/cost_graph.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/cost_graph.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/cost_graph.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/cost_graph.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/device_attributes.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/device_attributes.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/device_attributes.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/device_attributes.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/function.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/function.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/function.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/function.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/graph.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/graph.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/graph.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/graph.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/graph_transfer_info.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/graph_transfer_info.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/graph_transfer_info.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/graph_transfer_info.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/iterator.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/iterator.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/iterator.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/iterator.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/kernel_def.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/kernel_def.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/kernel_def.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/kernel_def.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/log_memory.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/log_memory.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/log_memory.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/log_memory.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/node_def.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/node_def.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/node_def.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/node_def.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/op_def.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/op_def.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/op_def.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/op_def.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/reader_base.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/reader_base.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/reader_base.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/reader_base.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/remote_fused_graph_execute_info.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/remote_fused_graph_execute_info.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/remote_fused_graph_execute_info.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/remote_fused_graph_execute_info.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/resource_handle.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/resource_handle.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/resource_handle.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/resource_handle.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/step_stats.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/step_stats.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/step_stats.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/step_stats.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/summary.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/summary.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/summary.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/summary.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/tensor.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/tensor.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/tensor.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/tensor.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_description.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_description.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_description.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_description.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_shape.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_shape.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_shape.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_shape.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_slice.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_slice.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_slice.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/tensor_slice.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/types.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/types.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/types.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/types.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/variable.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/variable.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/variable.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/variable.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/versions.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/versions.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/framework/versions.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/framework/versions.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/bmp/testdata/lena.bmp b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/bmp/testdata/lena.bmp similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/bmp/testdata/lena.bmp rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/bmp/testdata/lena.bmp diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/core/error_codes.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/core/error_codes.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/core/error_codes.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/core/error_codes.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/BUILD b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/BUILD similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/BUILD rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/BUILD diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/snapfn.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/snapfn.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/snapfn.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/snapfn.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/lena.gif b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/lena.gif similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/lena.gif rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/lena.gif diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/optimized.gif b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/optimized.gif similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/optimized.gif rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/optimized.gif diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/scan.gif b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/scan.gif similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/scan.gif rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gif/testdata/scan.gif diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_internal.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_internal.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_internal.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_internal.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatrep.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatrep.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatrep.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatrep.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/priority_queue_util.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/priority_queue_util.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/priority_queue_util.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/priority_queue_util.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/stl_util.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/stl_util.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/stl_util.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/stl_util.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_accelerate.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_accelerate.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_accelerate.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_accelerate.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/path_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/path_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/proto_encode_helper.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/proto_encode_helper.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/proto_encode_helper.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/proto_encode_helper.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader_writer_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader_writer_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader_writer_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader_writer_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/recordio_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/recordio_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/recordio_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/recordio_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_format.txt b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_format.txt similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_format.txt rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_format.txt diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_options.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_options.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_options.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_options.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_buffers_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_buffers_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_buffers_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_buffers_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_compression_options.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_compression_options.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_compression_options.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_compression_options.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/lmdb/testdata/data.mdb b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/lmdb/testdata/data.mdb similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/lmdb/testdata/data.mdb rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/lmdb/testdata/data.mdb diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collected_metrics.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collected_metrics.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collected_metrics.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collected_metrics.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_counter.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_counter.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_counter.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_counter.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_gauge.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_gauge.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_gauge.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_gauge.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_sampler.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_sampler.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_sampler.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_sampler.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/exact_uniform_int.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/exact_uniform_int.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/exact_uniform_int.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/exact_uniform_int.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test_utils.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test_utils.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test_utils.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test_utils.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.cc similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.cc index f691746a8..987e4fe73 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.cc +++ b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.cc @@ -182,7 +182,7 @@ size_t DoubleToBuffer(double value, char* buffer) { // DBL_DIG is 15 for IEEE-754 doubles, which are used on almost all // platforms these days. Just in case some system exists where DBL_DIG // is significantly larger -- and risks overflowing our buffer -- we have - // this + // this assert. static_assert(DBL_DIG < 20, "DBL_DIG is too big"); if (std::abs(value) <= kDoublePrecisionCheckMax) { @@ -363,7 +363,7 @@ size_t FloatToBuffer(float value, char* buffer) { // FLT_DIG is 6 for IEEE-754 floats, which are used on almost all // platforms these days. Just in case some system exists where FLT_DIG // is significantly larger -- and risks overflowing our buffer -- we have - // this + // this assert. static_assert(FLT_DIG < 10, "FLT_DIG is too big"); int snprintf_result = diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.h b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.h similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.h rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.h diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io_test.cc b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io_test.cc similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io_test.cc rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io_test.cc diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/checkpointable_object_graph.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/checkpointable_object_graph.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/checkpointable_object_graph.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/checkpointable_object_graph.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/cluster.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/cluster.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/cluster.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/cluster.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/config.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/config.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/config.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/config.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/control_flow.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/control_flow.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/control_flow.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/control_flow.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/critical_section.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/critical_section.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/critical_section.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/critical_section.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/debug.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/debug.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/debug.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/debug.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/device_properties.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/device_properties.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/device_properties.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/device_properties.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/eager_service.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/eager_service.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/eager_service.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/eager_service.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/master.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/master.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/master.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/master.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/master_service.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/master_service.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/master_service.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/master_service.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/meta_graph.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/meta_graph.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/meta_graph.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/meta_graph.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/named_tensor.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/named_tensor.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/named_tensor.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/named_tensor.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/queue_runner.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/queue_runner.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/queue_runner.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/queue_runner.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/rewriter_config.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/rewriter_config.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/rewriter_config.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/rewriter_config.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/saved_model.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/saved_model.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/saved_model.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/saved_model.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/saver.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/saver.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/saver.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/saver.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensor_bundle.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensor_bundle.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensor_bundle.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensor_bundle.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensorflow_server.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensorflow_server.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensorflow_server.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensorflow_server.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/transport_options.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/transport_options.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/transport_options.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/transport_options.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker_service.proto b/cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker_service.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker_service.proto rename to cavis-dnn/cavis-dnn-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker_service.proto diff --git a/cavis-dnn/cavis-dnn-api/src/main/resources/META-INF/services/org.nd4j.linalg.env.EnvironmentalAction b/cavis-dnn/cavis-dnn-api/src/main/resources/META-INF/services/org.nd4j.linalg.env.EnvironmentalAction new file mode 100644 index 000000000..1994fc794 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/main/resources/META-INF/services/org.nd4j.linalg.env.EnvironmentalAction @@ -0,0 +1,9 @@ + +org.nd4j.linalg.env.impl.DebugAction +org.nd4j.linalg.env.impl.VerboseAction +org.nd4j.linalg.env.impl.FallbackAction +org.nd4j.linalg.env.impl.WorkspacesBypassAction +org.nd4j.linalg.env.impl.WorkspacesDebugAction +org.nd4j.linalg.env.impl.WorkspacesSpillAction +org.nd4j.linalg.env.impl.OmpNumThreadsAction +org.nd4j.linalg.env.impl.NDArrayUnpackAction \ No newline at end of file diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/functions.properties b/cavis-dnn/cavis-dnn-api/src/main/resources/functions.properties similarity index 94% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/functions.properties rename to cavis-dnn/cavis-dnn-api/src/main/resources/functions.properties index 320bd6644..d8f1eb283 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/functions.properties +++ b/cavis-dnn/cavis-dnn-api/src/main/resources/functions.properties @@ -21,4 +21,4 @@ org.nd4j.linalg.ops.transform.names=abs,acos,asin,atan,ceil,cos,exp,floor,hardtanh,identity,log,maxout,negative,pow,round,sigmoid,sign,sin,sqrt,stabilize,tanh org.nd4j.linalg.ops.transform.packages=org.nd4j.linalg.api.ops.transforms org.nd4j.linalg.ops.accum.names=max,mean,min,norm1,norm2,normmax,prod,std,sum,var -org.nd4j.linalgops.accum.packages=org.nd4j.linalg.api.ops.accum +org.nd4j.linalg.ops.accum.packages=org.nd4j.linalg.api.ops.accum diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/resources/onnx-op-def.pbtxt b/cavis-dnn/cavis-dnn-api/src/main/resources/onnx.pbtxt similarity index 100% rename from nd4j/nd4j-backends/nd4j-tests/src/test/resources/onnx-op-def.pbtxt rename to cavis-dnn/cavis-dnn-api/src/main/resources/onnx.pbtxt diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/onnxops.json b/cavis-dnn/cavis-dnn-api/src/main/resources/onnxops.json similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/onnxops.json rename to cavis-dnn/cavis-dnn-api/src/main/resources/onnxops.json diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native-preset/org.bytedecojavacpp1.5.4, b/cavis-dnn/cavis-dnn-api/src/main/resources/op-ir.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native-preset/org.bytedecojavacpp1.5.4, rename to cavis-dnn/cavis-dnn-api/src/main/resources/op-ir.proto diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/ops.proto b/cavis-dnn/cavis-dnn-api/src/main/resources/ops.proto similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/ops.proto rename to cavis-dnn/cavis-dnn-api/src/main/resources/ops.proto diff --git a/cavis-dnn/cavis-dnn-api/src/test/java/org/nd4j/linalg/factory/Nd4jTest.java b/cavis-dnn/cavis-dnn-api/src/test/java/org/nd4j/linalg/factory/Nd4jTest.java new file mode 100644 index 000000000..a72b953f5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-api/src/test/java/org/nd4j/linalg/factory/Nd4jTest.java @@ -0,0 +1,55 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.nd4j.linalg.factory; + + +//import org.bytedeco.cuda.presets.cudart; +import org.bytedeco.javacpp.Loader; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.*; + +class Nd4jTest { + + @Test + void intantiateClass() throws ClassNotFoundException, InstantiationException, IllegalAccessException, IOException, InterruptedException { + /* try { + Loader.load(cudart.class); + } catch (UnsatisfiedLinkError e) { + //} finally { + String path = Loader.cacheResource(cudart.class, "windows-x86_64/jnicudart.dll").getPath(); + new ProcessBuilder("C:\\Users\\brian\\Downloads\\Dependencies_x64_Release\\DependenciesGui.exe", path).start().waitFor(); + } + try { + Nd4j clazz = (Nd4j) Class.forName("org.nd4j.linalg.factory.Nd4j").newInstance(); + } catch (UnsatisfiedLinkError e) { + //} finally { + String path = Loader.cacheResource(cudart.class, "windows-x86_64/jnicudart.dll").getPath(); + new ProcessBuilder("C:\\Users\\brian\\Downloads\\Dependencies_x64_Release\\DependenciesGui.exe", path).start().waitFor(); + } + */ + + } + +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-common-tests/build.gradle b/cavis-dnn/cavis-dnn-common-tests/build.gradle new file mode 100644 index 000000000..18ce64851 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common-tests/build.gradle @@ -0,0 +1,28 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +dependencies { + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisNd4j.cavisNd4jCommonTests + implementation 'ch.qos.logback:logback-classic' + implementation "org.bytedeco:javacpp" + implementation "org.junit.jupiter:junit-jupiter-api" +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java b/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java new file mode 100644 index 000000000..aca151aa2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java @@ -0,0 +1,206 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j; + +import ch.qos.logback.classic.LoggerContext; +import lombok.extern.slf4j.Slf4j; +import org.bytedeco.javacpp.Pointer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.config.ND4JSystemProperties; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.api.ops.executioner.OpExecutioner; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.profiler.ProfilerConfig; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.lang.management.ManagementFactory; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +@Slf4j +public abstract class BaseDL4JTest { + + protected long startTime; + protected int threadCountBefore; + + private final int DEFAULT_THREADS = Runtime.getRuntime().availableProcessors(); + + /** + * Override this to specify the number of threads for C++ execution, via + * {@link org.nd4j.linalg.factory.Environment#setMaxMasterThreads(int)} + * @return Number of threads to use for C++ op execution + */ + public int numThreads(){ + return DEFAULT_THREADS; + } + + /** + * Override this method to set the default timeout for methods in the test class + */ + public long getTimeoutMilliseconds(){ + return 90_000; + } + + /** + * Override this to set the profiling mode for the tests defined in the child class + */ + public OpExecutioner.ProfilingMode getProfilingMode(){ + return OpExecutioner.ProfilingMode.SCOPE_PANIC; + } + + /** + * Override this to set the datatype of the tests defined in the child class + */ + public DataType getDataType(){ + return DataType.DOUBLE; + } + + public DataType getDefaultFPDataType(){ + return getDataType(); + } + + protected static Boolean integrationTest; + + /** + * @return True if integration tests maven profile is enabled, false otherwise. + */ + public static boolean isIntegrationTests(){ + if(integrationTest == null){ + String prop = System.getenv("DL4J_INTEGRATION_TESTS"); + integrationTest = Boolean.parseBoolean(prop); + } + return integrationTest; + } + + /** + * Call this as the first line of a test in order to skip that test, only when the integration tests maven profile is not enabled. + * This can be used to dynamically skip integration tests when the integration test profile is not enabled. + * Note that the integration test profile is not enabled by default - "integration-tests" profile + */ + public static void skipUnlessIntegrationTests(){ + Assumptions.assumeTrue(isIntegrationTests(), () -> "Skipping integration test - integration profile is not enabled"); + } + + @BeforeEach + public void beforeTest(){ + log.info("{}:", getClass().getName()); + //log.info("{}.{}", getClass().getSimpleName()); + //Suppress ND4J initialization - don't need this logged for every test... + System.setProperty(ND4JSystemProperties.LOG_INITIALIZATION, "false"); + System.setProperty(ND4JSystemProperties.ND4J_IGNORE_AVX, "true"); + Nd4j.getExecutioner().setProfilingMode(getProfilingMode()); + Nd4j.getExecutioner().setProfilingConfig(ProfilerConfig.builder().build()); + Nd4j.setDefaultDataTypes(getDataType(), getDefaultFPDataType()); + Nd4j.getExecutioner().setProfilingConfig(ProfilerConfig.builder().build()); + Nd4j.getExecutioner().enableDebugMode(false); + Nd4j.getExecutioner().enableVerboseMode(false); + int numThreads = numThreads(); + Preconditions.checkState(numThreads > 0, "Number of threads must be > 0"); + if(numThreads != Nd4j.getEnvironment().maxMasterThreads()) { + Nd4j.getEnvironment().setMaxMasterThreads(numThreads); + } + startTime = System.currentTimeMillis(); + threadCountBefore = ManagementFactory.getThreadMXBean().getThreadCount(); + } + + @AfterEach + public void afterTest(){ + //Attempt to keep workspaces isolated between tests + Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); + MemoryWorkspace currWS = Nd4j.getMemoryManager().getCurrentWorkspace(); + Nd4j.getMemoryManager().setCurrentWorkspace(null); + if(currWS != null){ + //Not really safe to continue testing under this situation... other tests will likely fail with obscure + // errors that are hard to track back to this + log.error("Open workspace leaked from test! Exiting - {}, isOpen = {} - {}", currWS.getId(), currWS.isScopeActive(), currWS); + System.out.println("Open workspace leaked from test! Exiting - " + currWS.getId() + ", isOpen = " + currWS.isScopeActive() + " - " + currWS); + System.out.flush(); + //Try to flush logs also: + try{ Thread.sleep(1000); } catch (InterruptedException e){ } + ILoggerFactory lf = LoggerFactory.getILoggerFactory(); + if( lf instanceof LoggerContext){ + ((LoggerContext)lf).stop(); + } + try{ Thread.sleep(1000); } catch (InterruptedException e){ } + System.exit(1); + } + + StringBuilder sb = new StringBuilder(); + long maxPhys = Pointer.maxPhysicalBytes(); + long maxBytes = Pointer.maxBytes(); + long currPhys = Pointer.physicalBytes(); + long currBytes = Pointer.totalBytes(); + + long jvmTotal = Runtime.getRuntime().totalMemory(); + long jvmMax = Runtime.getRuntime().maxMemory(); + + int threadsAfter = ManagementFactory.getThreadMXBean().getThreadCount(); + + long duration = System.currentTimeMillis() - startTime; + sb.append(getClass().getSimpleName()).append(".").append("") + .append(": ").append(duration).append(" ms") + .append(", threadCount: (").append(threadCountBefore).append("->").append(threadsAfter).append(")") + .append(", jvmTotal=").append(jvmTotal) + .append(", jvmMax=").append(jvmMax) + .append(", totalBytes=").append(currBytes).append(", maxBytes=").append(maxBytes) + .append(", currPhys=").append(currPhys).append(", maxPhys=").append(maxPhys); + + List ws = Nd4j.getWorkspaceManager().getAllWorkspacesForCurrentThread(); + if(ws != null && ws.size() > 0){ + long currSize = 0; + for(MemoryWorkspace w : ws){ + currSize += w.getCurrentSize(); + } + if(currSize > 0){ + sb.append(", threadWSSize=").append(currSize) + .append(" (").append(ws.size()).append(" WSs)"); + } + } + + + Properties p = Nd4j.getExecutioner().getEnvironmentInformation(); + Object o = p.get("cuda.devicesInformation"); + if(o instanceof List){ + List> l = (List>) o; + if(l.size() > 0) { + + sb.append(" [").append(l.size()) + .append(" GPUs: "); + + for (int i = 0; i < l.size(); i++) { + Map m = l.get(i); + if(i > 0) + sb.append(","); + sb.append("(").append(m.get("cuda.freeMemory")).append(" free, ") + .append(m.get("cuda.totalMemory")).append(" total)"); + } + sb.append("]"); + } + } + log.info(sb.toString()); + } +} diff --git a/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/Performance.java b/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/Performance.java new file mode 100644 index 000000000..3198161a9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common-tests/src/main/java/org/deeplearning4j/Performance.java @@ -0,0 +1,35 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.deeplearning4j; + +import org.junit.jupiter.api.Tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@Tag("performance") +public @interface Performance { +} diff --git a/cavis-dnn/cavis-dnn-common/build.gradle b/cavis-dnn/cavis-dnn-common/build.gradle new file mode 100644 index 000000000..e48cae638 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/build.gradle @@ -0,0 +1,19 @@ + +plugins { + id 'java-library' + id 'maven-publish' + id 'signing' +} + +dependencies { + api platform(project(':cavis-common-platform')) + + implementation 'com.google.guava:guava' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'org.slf4j:slf4j-api' + implementation 'commons-io:commons-io' + implementation 'commons-codec:commons-codec' + implementation 'org.apache.commons:commons-math3' + implementation 'org.apache.commons:commons-lang3' + implementation 'org.apache.commons:commons-compress' +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/config/DL4JClassLoading.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/config/DL4JClassLoading.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/config/DL4JClassLoading.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/config/DL4JClassLoading.java diff --git a/deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/config/DL4JEnvironmentVars.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/config/DL4JEnvironmentVars.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/config/DL4JEnvironmentVars.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/config/DL4JEnvironmentVars.java diff --git a/deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/config/DL4JSystemProperties.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/config/DL4JSystemProperties.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/config/DL4JSystemProperties.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/config/DL4JSystemProperties.java diff --git a/deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/resources/DL4JResources.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/resources/DL4JResources.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/resources/DL4JResources.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/resources/DL4JResources.java diff --git a/deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/resources/ResourceType.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/resources/ResourceType.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/resources/ResourceType.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/resources/ResourceType.java diff --git a/deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/util/DL4JFileUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/util/DL4JFileUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/main/java/org/deeplearning4j/common/util/DL4JFileUtils.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/deeplearning4j/common/util/DL4JFileUtils.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/base/Preconditions.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/base/Preconditions.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/base/Preconditions.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/base/Preconditions.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/base/PreconditionsFormat.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/base/PreconditionsFormat.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/base/PreconditionsFormat.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/base/PreconditionsFormat.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/collection/CompactHeapStringList.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/CompactHeapStringList.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/collection/CompactHeapStringList.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/CompactHeapStringList.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/IntArrayKeyMap.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/IntArrayKeyMap.java new file mode 100644 index 000000000..17de2f5a1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/IntArrayKeyMap.java @@ -0,0 +1,159 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.collection; + +import lombok.Getter; +import org.nd4j.common.base.Preconditions; +import com.google.common.primitives.Ints; + +import java.util.*; + +public class IntArrayKeyMap implements Map { + + private Map map = new LinkedHashMap<>(); + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean containsKey(Object o) { + return map.containsKey(new IntArray((int[]) o)); + } + + @Override + public boolean containsValue(Object o) { + return map.containsValue(new IntArray((int[]) o)); + } + + @Override + public V get(Object o) { + return map.get(new IntArray((int[]) o)); + } + + @Override + public V put(int[] ints, V v) { + return map.put(new IntArray(ints),v); + } + + @Override + public V remove(Object o) { + return map.remove(new IntArray((int[]) o)); + } + + @Override + public void putAll(Map map) { + for(Entry entry : map.entrySet()) { + this.map.put(new IntArray(entry.getKey()),entry.getValue()); + } + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public Set keySet() { + Set intArrays = map.keySet(); + Set ret = new LinkedHashSet<>(); + for(IntArray intArray : intArrays) + ret.add(intArray.backingArray); + return ret; + } + + @Override + public Collection values() { + return map.values(); + } + + @Override + public Set> entrySet() { + Set> intArrays = map.entrySet(); + Set> ret = new LinkedHashSet<>(); + for(Entry intArray : intArrays) { + final Entry intArray2 = intArray; + ret.add(new Entry() { + @Override + public int[] getKey() { + return intArray2.getKey().backingArray; + } + + @Override + public V getValue() { + return intArray2.getValue(); + } + + @Override + public V setValue(V v) { + return intArray2.setValue(v); + } + }); + } + return ret; + } + + + public static class IntArray implements Comparable { + @Getter + private int[] backingArray; + + public IntArray(int[] backingArray) { + Preconditions.checkNotNull(backingArray,"Backing array must not be null!"); + this.backingArray = Ints.toArray(new LinkedHashSet<>(Ints.asList(backingArray))); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IntArray intArray = (IntArray) o; + + return Arrays.equals(intArray.backingArray,backingArray); + } + + @Override + public int hashCode() { + return Arrays.hashCode(backingArray); + } + + @Override + public int compareTo(IntArray intArray) { + if(this.backingArray.length == 0 || intArray.backingArray.length == 0) { + return 1; + } + + else if(Arrays.equals(backingArray,intArray.backingArray)) + return 1; + + return Ints.compare(Ints.max(backingArray),Ints.max(intArray.backingArray)); + } + } + + +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/collection/IntArrayKeySet.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/IntArrayKeySet.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/collection/IntArrayKeySet.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/IntArrayKeySet.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalMap.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/MultiDimensionalMap.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalMap.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/MultiDimensionalMap.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalSet.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/MultiDimensionalSet.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalSet.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collection/MultiDimensionalSet.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/collections/WeakIdentityHashMap.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collections/WeakIdentityHashMap.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/collections/WeakIdentityHashMap.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/collections/WeakIdentityHashMap.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/config/ND4JClassLoading.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/config/ND4JClassLoading.java new file mode 100644 index 000000000..a16c7bac4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/config/ND4JClassLoading.java @@ -0,0 +1,77 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.config; + +import lombok.extern.slf4j.Slf4j; + +import java.util.ServiceLoader; + +@Slf4j +public final class ND4JClassLoading { + private static ClassLoader nd4jClassloader = Thread.currentThread().getContextClassLoader(); + + private ND4JClassLoading() { + } + + public static ClassLoader getNd4jClassloader() { + return ND4JClassLoading.nd4jClassloader; + } + + public static void setNd4jClassloaderFromClass(Class clazz) { + setNd4jClassloader(clazz.getClassLoader()); + } + + public static void setNd4jClassloader(ClassLoader nd4jClassloader) { + ND4JClassLoading.nd4jClassloader = nd4jClassloader; + log.debug("Global class-loader for ND4J was changed."); + } + + public static boolean classPresentOnClasspath(String className) { + return classPresentOnClasspath(className, nd4jClassloader); + } + + public static boolean classPresentOnClasspath(String className, ClassLoader classLoader) { + return loadClassByName(className, false, classLoader) != null; + } + + public static Class loadClassByName(String className) { + return loadClassByName(className, true, nd4jClassloader); + } + + @SuppressWarnings("unchecked") + public static Class loadClassByName(String className, boolean initialize, ClassLoader classLoader) { + try { + log.info(String.format("Trying to load class [%s]", className)); + return (Class) Class.forName(className, initialize, classLoader); + } catch (ClassNotFoundException classNotFoundException) { + log.error(String.format("Cannot find class [%s] of provided class-loader.", className)); + return null; + } + } + + public static ServiceLoader loadService(Class serviceClass) { + return loadService(serviceClass, nd4jClassloader); + } + + public static ServiceLoader loadService(Class serviceClass, ClassLoader classLoader) { + return ServiceLoader.load(serviceClass, classLoader); + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/config/ND4JEnvironmentVars.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/config/ND4JEnvironmentVars.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/config/ND4JEnvironmentVars.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/config/ND4JEnvironmentVars.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/config/ND4JSystemProperties.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/config/ND4JSystemProperties.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/config/ND4JSystemProperties.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/config/ND4JSystemProperties.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/BiConsumer.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/BiConsumer.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/BiConsumer.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/BiConsumer.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/BiFunction.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/BiFunction.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/BiFunction.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/BiFunction.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/BiPredicate.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/BiPredicate.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/BiPredicate.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/BiPredicate.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/Consumer.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/Consumer.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/Consumer.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/Consumer.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/Function.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/Function.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/Function.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/Function.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/FunctionalUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/FunctionalUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/FunctionalUtils.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/FunctionalUtils.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/Predicate.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/Predicate.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/Predicate.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/Predicate.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/Supplier.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/Supplier.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/Supplier.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/Supplier.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/function/UnaryOperator.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/UnaryOperator.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/function/UnaryOperator.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/function/UnaryOperator.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/holder/ObjectMapperHolder.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/holder/ObjectMapperHolder.java new file mode 100644 index 000000000..0cd5166a1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/holder/ObjectMapperHolder.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.holder; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class ObjectMapperHolder { + + private static ObjectMapper objectMapper = getMapper(); + + private ObjectMapperHolder() {} + + + /** + * Get a single object mapper for use + * with reading and writing json + * @return + */ + public static ObjectMapper getJsonMapper() { + return objectMapper; + } + + private static ObjectMapper getMapper() { + ObjectMapper om = new ObjectMapper(); + //Serialize fields only, not using getters + //Not all getters are supported - for example, UserEntity + om.setVisibilityChecker(om.getSerializationConfig() + .getDefaultVisibilityChecker() + .withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withSetterVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)); + om.setSerializationInclusion(JsonInclude.Include.NON_NULL); + return om; + } + + + +} diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/AbstractFileResolvingResource.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/AbstractFileResolvingResource.java new file mode 100644 index 000000000..eb21e75a6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/AbstractFileResolvingResource.java @@ -0,0 +1,158 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.net.URLConnection; + + +public abstract class AbstractFileResolvingResource extends AbstractResource { + public AbstractFileResolvingResource() {} + + @Override + public File getFile() throws IOException { + URL url = this.getURL(); + return url.getProtocol().startsWith("vfs") + ? VfsResourceDelegate.getResource(url).getFile() + : ResourceUtils.getFile(url, this.getDescription()); + } + + @Override + protected File getFileForLastModifiedCheck() throws IOException { + URL url = this.getURL(); + if (ResourceUtils.isJarURL(url)) { + URL actualUrl = ResourceUtils.extractJarFileURL(url); + return actualUrl.getProtocol().startsWith("vfs") + ? VfsResourceDelegate.getResource(actualUrl).getFile() + : ResourceUtils.getFile(actualUrl, "Jar URL"); + } else { + return this.getFile(); + } + } + + protected File getFile(URI uri) throws IOException { + return uri.getScheme().startsWith("vfs") + ? VfsResourceDelegate.getResource(uri).getFile() + : ResourceUtils.getFile(uri, this.getDescription()); + } + + @Override + public boolean exists() { + try { + URL ex = this.getURL(); + if (ResourceUtils.isFileURL(ex)) { + return this.getFile().exists(); + } else { + URLConnection con = ex.openConnection(); + ResourceUtils.useCachesIfNecessary(con); + HttpURLConnection httpCon = con instanceof HttpURLConnection ? (HttpURLConnection) con : null; + if (httpCon != null) { + httpCon.setRequestMethod("HEAD"); + int is = httpCon.getResponseCode(); + if (is == 200) { + return true; + } + + if (is == 404) { + return false; + } + } + + if (con.getContentLength() >= 0) { + return true; + } else if (httpCon != null) { + httpCon.disconnect(); + return false; + } else { + InputStream is1 = this.getInputStream(); + is1.close(); + return true; + } + } + } catch (IOException var5) { + return false; + } + } + + @Override + public boolean isReadable() { + try { + URL ex = this.getURL(); + if (!ResourceUtils.isFileURL(ex)) { + return true; + } else { + File file = this.getFile(); + return file.canRead() && !file.isDirectory(); + } + } catch (IOException var3) { + return false; + } + } + + @Override + public long contentLength() throws IOException { + URL url = this.getURL(); + if (ResourceUtils.isFileURL(url)) { + return this.getFile().length(); + } else { + URLConnection con = url.openConnection(); + ResourceUtils.useCachesIfNecessary(con); + if (con instanceof HttpURLConnection) { + ((HttpURLConnection) con).setRequestMethod("HEAD"); + } + + return (long) con.getContentLength(); + } + } + + @Override + public long lastModified() throws IOException { + URL url = this.getURL(); + if (!ResourceUtils.isFileURL(url) && !ResourceUtils.isJarURL(url)) { + URLConnection con = url.openConnection(); + ResourceUtils.useCachesIfNecessary(con); + if (con instanceof HttpURLConnection) { + ((HttpURLConnection) con).setRequestMethod("HEAD"); + } + + return con.getLastModified(); + } else { + return super.lastModified(); + } + } + + private static class VfsResourceDelegate { + private VfsResourceDelegate() {} + + public static Resource getResource(URL url) throws IOException { + return new VfsResource(VfsUtils.getRoot(url)); + } + + public static Resource getResource(URI uri) throws IOException { + return new VfsResource(VfsUtils.getRoot(uri)); + } + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/AbstractResource.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/AbstractResource.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/AbstractResource.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/AbstractResource.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/Assert.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/Assert.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/Assert.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/Assert.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/ClassPathResource.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ClassPathResource.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/ClassPathResource.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ClassPathResource.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/CollectionUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/CollectionUtils.java new file mode 100644 index 000000000..4212e69e9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/CollectionUtils.java @@ -0,0 +1,408 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import org.nd4j.common.util.MultiValueMap; + +import java.io.Serializable; +import java.util.*; +import java.util.Map.Entry; + + +public abstract class CollectionUtils { + public CollectionUtils() {} + + public static boolean isEmpty(Collection collection) { + return collection == null || collection.isEmpty(); + } + + public static boolean isEmpty(Map map) { + return map == null || map.isEmpty(); + } + + public static List arrayToList(Object source) { + return Arrays.asList(ObjectUtils.toObjectArray(source)); + } + + public static void mergeArrayIntoCollection(Object array, Collection collection) { + if (collection == null) { + throw new IllegalArgumentException("Collection must not be null"); + } else { + Object[] arr = ObjectUtils.toObjectArray(array); + Object[] arr$ = arr; + int len$ = arr.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Object elem = arr$[i$]; + collection.add(elem); + } + + } + } + + public static void mergePropertiesIntoMap(Properties props, Map map) { + if (map == null) { + throw new IllegalArgumentException("Map must not be null"); + } else { + String key; + Object value; + if (props != null) { + for (Enumeration en = props.propertyNames(); en.hasMoreElements(); map.put(key, value)) { + key = (String) en.nextElement(); + value = props.getProperty(key); + if (value == null) { + value = props.get(key); + } + } + } + + } + } + + public static boolean contains(Iterator iterator, Object element) { + if (iterator != null) { + while (iterator.hasNext()) { + Object candidate = iterator.next(); + if (ObjectUtils.nullSafeEquals(candidate, element)) { + return true; + } + } + } + + return false; + } + + public static boolean contains(Enumeration enumeration, Object element) { + if (enumeration != null) { + while (enumeration.hasMoreElements()) { + Object candidate = enumeration.nextElement(); + if (ObjectUtils.nullSafeEquals(candidate, element)) { + return true; + } + } + } + + return false; + } + + public static boolean containsInstance(Collection collection, Object element) { + if (collection != null) { + Iterator i$ = collection.iterator(); + + while (i$.hasNext()) { + Object candidate = i$.next(); + if (candidate == element) { + return true; + } + } + } + + return false; + } + + public static boolean containsAny(Collection source, Collection candidates) { + if (!isEmpty(source) && !isEmpty(candidates)) { + Iterator i$ = candidates.iterator(); + + Object candidate; + do { + if (!i$.hasNext()) { + return false; + } + + candidate = i$.next(); + } while (!source.contains(candidate)); + + return true; + } else { + return false; + } + } + + public static Object findFirstMatch(Collection source, Collection candidates) { + if (!isEmpty(source) && !isEmpty(candidates)) { + Iterator i$ = candidates.iterator(); + + Object candidate; + do { + if (!i$.hasNext()) { + return null; + } + + candidate = i$.next(); + } while (!source.contains(candidate)); + + return candidate; + } else { + return null; + } + } + + public static T findValueOfType(Collection collection, Class type) { + if (isEmpty((Collection) collection)) { + return null; + } else { + Object value = null; + Iterator i$ = collection.iterator(); + + while (i$.hasNext()) { + Object element = i$.next(); + if (type == null || type.isInstance(element)) { + if (value != null) { + return null; + } + + value = element; + } + } + + return (T) value; + } + } + + public static Object findValueOfType(Collection collection, Class[] types) { + if (!isEmpty((Collection) collection) && !ObjectUtils.isEmpty(types)) { + Class[] arr$ = types; + int len$ = types.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Class type = arr$[i$]; + Object value = findValueOfType(collection, type); + if (value != null) { + return value; + } + } + + return null; + } else { + return null; + } + } + + public static boolean hasUniqueObject(Collection collection) { + if (isEmpty(collection)) { + return false; + } else { + boolean hasCandidate = false; + Object candidate = null; + Iterator i$ = collection.iterator(); + + while (i$.hasNext()) { + Object elem = i$.next(); + if (!hasCandidate) { + hasCandidate = true; + candidate = elem; + } else if (candidate != elem) { + return false; + } + } + + return true; + } + } + + public static Class findCommonElementType(Collection collection) { + if (isEmpty(collection)) { + return null; + } else { + Class candidate = null; + Iterator i$ = collection.iterator(); + + while (i$.hasNext()) { + Object val = i$.next(); + if (val != null) { + if (candidate == null) { + candidate = val.getClass(); + } else if (candidate != val.getClass()) { + return null; + } + } + } + + return candidate; + } + } + + public static A[] toArray(Enumeration enumeration, A[] array) { + ArrayList elements = new ArrayList(); + + while (enumeration.hasMoreElements()) { + elements.add(enumeration.nextElement()); + } + + return (A[]) elements.toArray(array); + } + + public static Iterator toIterator(Enumeration enumeration) { + return new EnumerationIterator(enumeration); + } + + public static MultiValueMap toMultiValueMap(Map> map) { + return new MultiValueMapAdapter(map); + } + + public static MultiValueMap unmodifiableMultiValueMap(MultiValueMap map) { + Assert.notNull(map, "\'map\' must not be null"); + LinkedHashMap result = new LinkedHashMap(map.size()); + Iterator unmodifiableMap = map.entrySet().iterator(); + + while (unmodifiableMap.hasNext()) { + Entry entry = (Entry) unmodifiableMap.next(); + List values = Collections.unmodifiableList((List) entry.getValue()); + result.put(entry.getKey(), values); + } + + Map unmodifiableMap1 = Collections.unmodifiableMap(result); + return toMultiValueMap(unmodifiableMap1); + } + + private static class MultiValueMapAdapter implements MultiValueMap, Serializable { + private final Map> map; + + public MultiValueMapAdapter(Map> map) { + Assert.notNull(map, "\'map\' must not be null"); + this.map = map; + } + + public void add(K key, V value) { + List values = this.map.get(key); + if (values == null) { + values = new LinkedList<>(); + this.map.put(key, values); + } + + values.add(value); + } + + public V getFirst(K key) { + List values = this.map.get(key); + return values != null ? (V) values.get(0) : null; + } + + public void set(K key, V value) { + LinkedList values = new LinkedList(); + values.add(value); + this.map.put(key, values); + } + + public void setAll(Map values) { + Iterator i$ = values.entrySet().iterator(); + + while (i$.hasNext()) { + Entry entry = (Entry) i$.next(); + this.set((K) entry.getKey(), (V) entry.getValue()); + } + + } + + public Map toSingleValueMap() { + LinkedHashMap singleValueMap = new LinkedHashMap(this.map.size()); + Iterator i$ = this.map.entrySet().iterator(); + + while (i$.hasNext()) { + Entry entry = (Entry) i$.next(); + singleValueMap.put(entry.getKey(), ((List) entry.getValue()).get(0)); + } + + return singleValueMap; + } + + public int size() { + return this.map.size(); + } + + public boolean isEmpty() { + return this.map.isEmpty(); + } + + public boolean containsKey(Object key) { + return this.map.containsKey(key); + } + + public boolean containsValue(Object value) { + return this.map.containsValue(value); + } + + public List get(Object key) { + return this.map.get(key); + } + + public List put(K key, List value) { + return this.map.put(key, value); + } + + public List remove(Object key) { + return this.map.remove(key); + } + + public void putAll(Map> m) { + this.map.putAll(m); + } + + public void clear() { + this.map.clear(); + } + + public Set keySet() { + return this.map.keySet(); + } + + public Collection> values() { + return this.map.values(); + } + + public Set>> entrySet() { + return this.map.entrySet(); + } + + public boolean equals(Object other) { + return this == other ? true : this.map.equals(other); + } + + public int hashCode() { + return this.map.hashCode(); + } + + public String toString() { + return this.map.toString(); + } + } + + private static class EnumerationIterator implements Iterator { + private Enumeration enumeration; + + public EnumerationIterator(Enumeration enumeration) { + this.enumeration = enumeration; + } + + public boolean hasNext() { + return this.enumeration.hasMoreElements(); + } + + public E next() { + return this.enumeration.nextElement(); + } + + public void remove() throws UnsupportedOperationException { + throw new UnsupportedOperationException("Not supported"); + } + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/InputStreamSource.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/InputStreamSource.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/InputStreamSource.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/InputStreamSource.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/ObjectUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ObjectUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/ObjectUtils.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ObjectUtils.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ReflectionUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ReflectionUtils.java new file mode 100644 index 000000000..5c41dba38 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ReflectionUtils.java @@ -0,0 +1,455 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import java.lang.reflect.*; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Objects; +import java.util.regex.Pattern; + +public abstract class ReflectionUtils { + private static final Pattern CGLIB_RENAMED_METHOD_PATTERN = Pattern.compile("CGLIB\\$(.+)\\$\\d+"); + public static FieldFilter COPYABLE_FIELDS = new FieldFilter() { + public boolean matches(Field field) { + return !Modifier.isStatic(field.getModifiers()) && !Modifier.isFinal(field.getModifiers()); + } + }; + public static MethodFilter NON_BRIDGED_METHODS = new MethodFilter() { + public boolean matches(Method method) { + return !method.isBridge(); + } + }; + public static MethodFilter USER_DECLARED_METHODS = new MethodFilter() { + public boolean matches(Method method) { + return !method.isBridge() && method.getDeclaringClass() != Object.class; + } + }; + + public ReflectionUtils() {} + + public static Field findField(Class clazz, String name) { + return findField(clazz, name, null); + } + + public static Field findField(Class clazz, String name, Class type) { + Assert.notNull(clazz, "Class must not be null"); + Assert.isTrue(name != null || type != null, "Either name or opType of the field must be specified"); + + for (Class searchType = clazz; !Object.class.equals(searchType) && searchType != null; searchType = + searchType.getSuperclass()) { + Field[] fields = searchType.getDeclaredFields(); + Field[] arr$ = fields; + int len$ = fields.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Field field = arr$[i$]; + if ((name == null || name.equals(field.getName())) && (type == null || type.equals(field.getType()))) { + return field; + } + } + } + + return null; + } + + public static void setField(Field field, Object target, Object value) { + try { + field.set(target, value); + } catch (IllegalAccessException var4) { + handleReflectionException(var4); + throw new IllegalStateException("Unexpected reflection exception - " + var4.getClass().getName() + ": " + + var4.getMessage()); + } + } + + public static Object getField(Field field, Object target) { + try { + return field.get(target); + } catch (IllegalAccessException var3) { + handleReflectionException(var3); + throw new IllegalStateException("Unexpected reflection exception - " + var3.getClass().getName() + ": " + + var3.getMessage()); + } + } + + public static Method findMethod(Class clazz, String name) { + return findMethod(clazz, name, new Class[0]); + } + + public static Method findMethod(Class clazz, String name, Class... paramTypes) { + Assert.notNull(clazz, "Class must not be null"); + Assert.notNull(name, "Method name must not be null"); + + for (Class searchType = clazz; searchType != null; searchType = searchType.getSuperclass()) { + Method[] methods = searchType.isInterface() ? searchType.getMethods() : searchType.getDeclaredMethods(); + Method[] arr$ = methods; + int len$ = methods.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Method method = arr$[i$]; + if (name.equals(method.getName()) + && (paramTypes == null || Arrays.equals(paramTypes, method.getParameterTypes()))) { + return method; + } + } + } + + return null; + } + + public static Object invokeMethod(Method method, Object target) { + return invokeMethod(method, target, new Object[0]); + } + + public static Object invokeMethod(Method method, Object target, Object... args) { + try { + return method.invoke(target, args); + } catch (Exception var4) { + handleReflectionException(var4); + throw new IllegalStateException("Should never get here"); + } + } + + public static Object invokeJdbcMethod(Method method, Object target) throws SQLException { + return invokeJdbcMethod(method, target, new Object[0]); + } + + public static Object invokeJdbcMethod(Method method, Object target, Object... args) throws SQLException { + try { + return method.invoke(target, args); + } catch (IllegalAccessException var4) { + handleReflectionException(var4); + } catch (InvocationTargetException var5) { + if (var5.getTargetException() instanceof SQLException) { + throw (SQLException) var5.getTargetException(); + } + + handleInvocationTargetException(var5); + } + + throw new IllegalStateException("Should never get here"); + } + + public static void handleReflectionException(Exception ex) { + if (ex instanceof NoSuchMethodException) { + throw new IllegalStateException("Method not found: " + ex.getMessage()); + } else if (ex instanceof IllegalAccessException) { + throw new IllegalStateException("Could not access method: " + ex.getMessage()); + } else { + if (ex instanceof InvocationTargetException) { + handleInvocationTargetException((InvocationTargetException) ex); + } + + if (ex instanceof RuntimeException) { + throw (RuntimeException) ex; + } else { + throw new UndeclaredThrowableException(ex); + } + } + } + + public static void handleInvocationTargetException(InvocationTargetException ex) { + rethrowRuntimeException(ex.getTargetException()); + } + + public static void rethrowRuntimeException(Throwable ex) { + if (ex instanceof RuntimeException) { + throw (RuntimeException) ex; + } else if (ex instanceof Error) { + throw (Error) ex; + } else { + throw new UndeclaredThrowableException(ex); + } + } + + public static void rethrowException(Throwable ex) throws Exception { + if (ex instanceof Exception) { + throw (Exception) ex; + } else if (ex instanceof Error) { + throw (Error) ex; + } else { + throw new UndeclaredThrowableException(ex); + } + } + + public static boolean declaresException(Method method, Class exceptionType) { + Assert.notNull(method, "Method must not be null"); + Class[] declaredExceptions = method.getExceptionTypes(); + Class[] arr$ = declaredExceptions; + int len$ = declaredExceptions.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Class declaredException = arr$[i$]; + if (declaredException.isAssignableFrom(exceptionType)) { + return true; + } + } + + return false; + } + + public static boolean isPublicStaticFinal(Field field) { + int modifiers = field.getModifiers(); + return Modifier.isPublic(modifiers) && Modifier.isStatic(modifiers) && Modifier.isFinal(modifiers); + } + + public static boolean isEqualsMethod(Method method) { + if (method != null && method.getName().equals("equals")) { + Class[] paramTypes = method.getParameterTypes(); + return paramTypes.length == 1 && paramTypes[0] == Object.class; + } else { + return false; + } + } + + public static boolean isHashCodeMethod(Method method) { + return method != null && method.getName().equals("hashCode") && method.getParameterTypes().length == 0; + } + + public static boolean isToStringMethod(Method method) { + return method != null && method.getName().equals("toString") && method.getParameterTypes().length == 0; + } + + public static boolean isObjectMethod(Method method) { + try { + Object.class.getDeclaredMethod(method.getName(), method.getParameterTypes()); + return true; + } catch (SecurityException var2) { + return false; + } catch (NoSuchMethodException var3) { + return false; + } + } + + public static boolean isCglibRenamedMethod(Method renamedMethod) { + return CGLIB_RENAMED_METHOD_PATTERN.matcher(renamedMethod.getName()).matches(); + } + + public static void makeAccessible(Field field) { + if ((!Modifier.isPublic(field.getModifiers()) || !Modifier.isPublic(field.getDeclaringClass().getModifiers()) + || Modifier.isFinal(field.getModifiers())) && !field.isAccessible()) { + field.setAccessible(true); + } + + } + + public static void makeAccessible(Method method) { + if ((!Modifier.isPublic(method.getModifiers()) || !Modifier.isPublic(method.getDeclaringClass().getModifiers())) + && !method.isAccessible()) { + method.setAccessible(true); + } + + } + + public static void makeAccessible(Constructor ctor) { + if ((!Modifier.isPublic(ctor.getModifiers()) || !Modifier.isPublic(ctor.getDeclaringClass().getModifiers())) + && !ctor.isAccessible()) { + ctor.setAccessible(true); + } + + } + + public static void doWithMethods(Class clazz, MethodCallback mc) + throws IllegalArgumentException { + doWithMethods(clazz, mc, null); + } + + public static void doWithMethods(Class clazz, MethodCallback mc, MethodFilter mf) + throws IllegalArgumentException { + Method[] methods = clazz.getDeclaredMethods(); + Method[] arr$ = methods; + int len$ = methods.length; + + int i$; + for (i$ = 0; i$ < len$; ++i$) { + Method superIfc = arr$[i$]; + if (mf == null || mf.matches(superIfc)) { + try { + mc.doWith(superIfc); + } catch (IllegalAccessException var9) { + throw new IllegalStateException( + "Shouldn\'t be illegal to access method \'" + superIfc.getName() + "\': " + var9); + } + } + } + + if (clazz.getSuperclass() != null) { + doWithMethods(clazz.getSuperclass(), mc, mf); + } else if (clazz.isInterface()) { + Class[] var10 = clazz.getInterfaces(); + len$ = var10.length; + + for (i$ = 0; i$ < len$; ++i$) { + Class var11 = var10[i$]; + doWithMethods(var11, mc, mf); + } + } + + } + + public static Method[] getAllDeclaredMethods(Class leafClass) throws IllegalArgumentException { + final ArrayList methods = new ArrayList(32); + doWithMethods(leafClass, new MethodCallback() { + public void doWith(Method method) { + methods.add(method); + } + }); + return (Method[]) methods.toArray(new Method[methods.size()]); + } + + public static Method[] getUniqueDeclaredMethods(Class leafClass) throws IllegalArgumentException { + final ArrayList methods = new ArrayList(32); + doWithMethods(leafClass, new MethodCallback() { + public void doWith(Method method) { + boolean knownSignature = false; + Method methodBeingOverriddenWithCovariantReturnType = null; + Iterator i$ = methods.iterator(); + + while (i$.hasNext()) { + Method existingMethod = (Method) i$.next(); + if (method.getName().equals(existingMethod.getName()) + && Arrays.equals(method.getParameterTypes(), existingMethod.getParameterTypes())) { + if (existingMethod.getReturnType() != method.getReturnType() + && existingMethod.getReturnType().isAssignableFrom(method.getReturnType())) { + methodBeingOverriddenWithCovariantReturnType = existingMethod; + break; + } + + knownSignature = true; + break; + } + } + + if (methodBeingOverriddenWithCovariantReturnType != null) { + methods.remove(methodBeingOverriddenWithCovariantReturnType); + } + + if (!knownSignature && !ReflectionUtils.isCglibRenamedMethod(method)) { + methods.add(method); + } + + } + }); + return (Method[]) methods.toArray(new Method[methods.size()]); + } + + public static void doWithFields(Class clazz, FieldCallback fc) throws IllegalArgumentException { + doWithFields(clazz, fc, null); + } + + public static void doWithFields(Class clazz, FieldCallback fc, FieldFilter ff) + throws IllegalArgumentException { + Class targetClass = clazz; + + do { + Field[] fields = targetClass.getDeclaredFields(); + Field[] arr$ = fields; + int len$ = fields.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Field field = arr$[i$]; + if (ff == null || ff.matches(field)) { + try { + fc.doWith(field); + } catch (IllegalAccessException var10) { + throw new IllegalStateException( + "Shouldn\'t be illegal to access field \'" + field.getName() + "\': " + var10); + } + } + } + + targetClass = targetClass.getSuperclass(); + } while (targetClass != null && targetClass != Object.class); + + } + + public static void shallowCopyFieldState(final Object src, final Object dest) throws IllegalArgumentException { + if (src == null) { + throw new IllegalArgumentException("Source for field copy cannot be null"); + } else if (dest == null) { + throw new IllegalArgumentException("Destination for field copy cannot be null"); + } else if (!src.getClass().isAssignableFrom(dest.getClass())) { + throw new IllegalArgumentException("Destination class [" + dest.getClass().getName() + + "] must be same or subclass as source class [" + src.getClass().getName() + "]"); + } else { + doWithFields(src.getClass(), new FieldCallback() { + public void doWith(Field field) throws IllegalArgumentException, IllegalAccessException { + ReflectionUtils.makeAccessible(field); + Object srcValue = field.get(src); + field.set(dest, srcValue); + } + }, COPYABLE_FIELDS); + } + } + + /** + * Create a new instance of the specified {@link Class} by invoking + * the constructor whose argument list matches the types of the supplied + * arguments. + * + *

Provided class must have a public constructor.

+ * + * @param clazz the class to instantiate; never {@code null} + * @param args the arguments to pass to the constructor, none of which may + * be {@code null} + * @return the new instance; never {@code null} + */ + public static T newInstance(Class clazz, Object... args) { + Objects.requireNonNull(clazz, "Class must not be null"); + Objects.requireNonNull(args, "Argument array must not be null"); + if (Arrays.asList(args).contains(null)) { + throw new RuntimeException("Individual arguments must not be null"); + } + + try { + Class[] parameterTypes = Arrays.stream(args).map(Object::getClass).toArray(Class[]::new); + Constructor constructor = clazz.getDeclaredConstructor(parameterTypes); + + if (!Modifier.isPublic(constructor.getModifiers())) { + throw new IllegalArgumentException(String.format( + "Class [%s] must have public constructor in order to be instantiated.", clazz.getName())); + } + + return constructor.newInstance(args); + } catch (Throwable instantiationException) { + throw new RuntimeException(instantiationException); + } + } + + public interface FieldFilter { + boolean matches(Field var1); + } + + public interface FieldCallback { + void doWith(Field var1) throws IllegalArgumentException, IllegalAccessException; + } + + public interface MethodFilter { + boolean matches(Method var1); + } + + public interface MethodCallback { + void doWith(Method var1) throws IllegalArgumentException, IllegalAccessException; + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/Resource.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/Resource.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/Resource.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/Resource.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/ResourceUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ResourceUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/ResourceUtils.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/ResourceUtils.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/StringUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/StringUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/StringUtils.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/StringUtils.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/VfsResource.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/VfsResource.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/VfsResource.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/VfsResource.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/VfsUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/VfsUtils.java new file mode 100644 index 000000000..93b11b937 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/io/VfsUtils.java @@ -0,0 +1,232 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URI; +import java.net.URL; + + + +public abstract class VfsUtils { + private static final Logger logger = LoggerFactory.getLogger(VfsUtils.class); + private static final String VFS2_PKG = "org.jboss.virtual."; + private static final String VFS3_PKG = "org.jboss.vfs."; + private static final String VFS_NAME = "VFS"; + private static VFS_VER version; + private static Method VFS_METHOD_GET_ROOT_URL = null; + private static Method VFS_METHOD_GET_ROOT_URI = null; + private static Method VIRTUAL_FILE_METHOD_EXISTS = null; + private static Method VIRTUAL_FILE_METHOD_GET_INPUT_STREAM; + private static Method VIRTUAL_FILE_METHOD_GET_SIZE; + private static Method VIRTUAL_FILE_METHOD_GET_LAST_MODIFIED; + private static Method VIRTUAL_FILE_METHOD_TO_URL; + private static Method VIRTUAL_FILE_METHOD_TO_URI; + private static Method VIRTUAL_FILE_METHOD_GET_NAME; + private static Method VIRTUAL_FILE_METHOD_GET_PATH_NAME; + private static Method VIRTUAL_FILE_METHOD_GET_CHILD; + protected static Class VIRTUAL_FILE_VISITOR_INTERFACE; + protected static Method VIRTUAL_FILE_METHOD_VISIT; + private static Method VFS_UTILS_METHOD_IS_NESTED_FILE = null; + private static Method VFS_UTILS_METHOD_GET_COMPATIBLE_URI = null; + private static Field VISITOR_ATTRIBUTES_FIELD_RECURSE = null; + private static Method GET_PHYSICAL_FILE = null; + + public VfsUtils() {} + + protected static Object invokeVfsMethod(Method method, Object target, Object... args) throws IOException { + try { + return method.invoke(target, args); + } catch (InvocationTargetException var5) { + Throwable targetEx = var5.getTargetException(); + if (targetEx instanceof IOException) { + throw (IOException) targetEx; + } + + ReflectionUtils.handleInvocationTargetException(var5); + } catch (Exception var6) { + ReflectionUtils.handleReflectionException(var6); + } + + throw new IllegalStateException("Invalid code path reached"); + } + + static boolean exists(Object vfsResource) { + try { + return ((Boolean) invokeVfsMethod(VIRTUAL_FILE_METHOD_EXISTS, vfsResource, new Object[0])).booleanValue(); + } catch (IOException var2) { + return false; + } + } + + static boolean isReadable(Object vfsResource) { + try { + return ((Long) invokeVfsMethod(VIRTUAL_FILE_METHOD_GET_SIZE, vfsResource, new Object[0])).longValue() > 0L; + } catch (IOException var2) { + return false; + } + } + + static long getSize(Object vfsResource) throws IOException { + return ((Long) invokeVfsMethod(VIRTUAL_FILE_METHOD_GET_SIZE, vfsResource, new Object[0])).longValue(); + } + + static long getLastModified(Object vfsResource) throws IOException { + return ((Long) invokeVfsMethod(VIRTUAL_FILE_METHOD_GET_LAST_MODIFIED, vfsResource, new Object[0])).longValue(); + } + + static InputStream getInputStream(Object vfsResource) throws IOException { + return (InputStream) invokeVfsMethod(VIRTUAL_FILE_METHOD_GET_INPUT_STREAM, vfsResource, new Object[0]); + } + + static URL getURL(Object vfsResource) throws IOException { + return (URL) invokeVfsMethod(VIRTUAL_FILE_METHOD_TO_URL, vfsResource, new Object[0]); + } + + static URI getURI(Object vfsResource) throws IOException { + return (URI) invokeVfsMethod(VIRTUAL_FILE_METHOD_TO_URI, vfsResource, new Object[0]); + } + + static String getName(Object vfsResource) { + try { + return (String) invokeVfsMethod(VIRTUAL_FILE_METHOD_GET_NAME, vfsResource, new Object[0]); + } catch (IOException var2) { + throw new IllegalStateException("Cannot get resource name", var2); + } + } + + static Object getRelative(URL url) throws IOException { + return invokeVfsMethod(VFS_METHOD_GET_ROOT_URL, null, new Object[] {url}); + } + + static Object getChild(Object vfsResource, String path) throws IOException { + return invokeVfsMethod(VIRTUAL_FILE_METHOD_GET_CHILD, vfsResource, new Object[] {path}); + } + + static File getFile(Object vfsResource) throws IOException { + if (VFS_VER.V2.equals(version)) { + if (((Boolean) invokeVfsMethod(VFS_UTILS_METHOD_IS_NESTED_FILE, null, new Object[] {vfsResource})) + .booleanValue()) { + throw new IOException("File resolution not supported for nested resource: " + vfsResource); + } else { + try { + return new File((URI) invokeVfsMethod(VFS_UTILS_METHOD_GET_COMPATIBLE_URI, null, + new Object[] {vfsResource})); + } catch (Exception var2) { + throw new IOException("Failed to obtain File reference for " + vfsResource, var2); + } + } + } else { + return (File) invokeVfsMethod(GET_PHYSICAL_FILE, vfsResource, new Object[0]); + } + } + + static Object getRoot(URI url) throws IOException { + return invokeVfsMethod(VFS_METHOD_GET_ROOT_URI, null, new Object[] {url}); + } + + protected static Object getRoot(URL url) throws IOException { + return invokeVfsMethod(VFS_METHOD_GET_ROOT_URL, null, new Object[] {url}); + } + + protected static Object doGetVisitorAttribute() { + return ReflectionUtils.getField(VISITOR_ATTRIBUTES_FIELD_RECURSE, null); + } + + protected static String doGetPath(Object resource) { + return (String) ReflectionUtils.invokeMethod(VIRTUAL_FILE_METHOD_GET_PATH_NAME, resource); + } + + static { + ClassLoader loader = VfsUtils.class.getClassLoader(); + + String pkg; + Class vfsClass; + try { + vfsClass = loader.loadClass("org.jboss.vfs.VFS"); + version = VFS_VER.V3; + pkg = "org.jboss.vfs."; + if (logger.isDebugEnabled()) { + logger.debug("JBoss VFS packages for JBoss AS 6 found"); + } + } catch (ClassNotFoundException var9) { + if (logger.isDebugEnabled()) { + logger.debug("JBoss VFS packages for JBoss AS 6 not found; falling back to JBoss AS 5 packages"); + } + + try { + vfsClass = loader.loadClass("org.jboss.virtual.VFS"); + version = VFS_VER.V2; + pkg = "org.jboss.virtual."; + if (logger.isDebugEnabled()) { + logger.debug("JBoss VFS packages for JBoss AS 5 found"); + } + } catch (ClassNotFoundException var8) { + logger.error("JBoss VFS packages (for both JBoss AS 5 and 6) were not found - JBoss VFS support disabled"); + throw new IllegalStateException("Cannot detect JBoss VFS packages", var8); + } + } + + try { + String ex = VFS_VER.V3.equals(version) ? "getChild" : "getRoot"; + VFS_METHOD_GET_ROOT_URL = ReflectionUtils.findMethod(vfsClass, ex, new Class[] {URL.class}); + VFS_METHOD_GET_ROOT_URI = ReflectionUtils.findMethod(vfsClass, ex, new Class[] {URI.class}); + Class virtualFile = loader.loadClass(pkg + "VirtualFile"); + VIRTUAL_FILE_METHOD_EXISTS = ReflectionUtils.findMethod(virtualFile, "exists"); + VIRTUAL_FILE_METHOD_GET_INPUT_STREAM = ReflectionUtils.findMethod(virtualFile, "openStream"); + VIRTUAL_FILE_METHOD_GET_SIZE = ReflectionUtils.findMethod(virtualFile, "getSize"); + VIRTUAL_FILE_METHOD_GET_LAST_MODIFIED = ReflectionUtils.findMethod(virtualFile, "getLastModified"); + VIRTUAL_FILE_METHOD_TO_URI = ReflectionUtils.findMethod(virtualFile, "toURI"); + VIRTUAL_FILE_METHOD_TO_URL = ReflectionUtils.findMethod(virtualFile, "toURL"); + VIRTUAL_FILE_METHOD_GET_NAME = ReflectionUtils.findMethod(virtualFile, "getName"); + VIRTUAL_FILE_METHOD_GET_PATH_NAME = ReflectionUtils.findMethod(virtualFile, "getPathName"); + GET_PHYSICAL_FILE = ReflectionUtils.findMethod(virtualFile, "getPhysicalFile"); + ex = VFS_VER.V3.equals(version) ? "getChild" : "findChild"; + VIRTUAL_FILE_METHOD_GET_CHILD = ReflectionUtils.findMethod(virtualFile, ex, new Class[] {String.class}); + Class utilsClass = loader.loadClass(pkg + "VFSUtils"); + VFS_UTILS_METHOD_GET_COMPATIBLE_URI = + ReflectionUtils.findMethod(utilsClass, "getCompatibleURI", new Class[] {virtualFile}); + VFS_UTILS_METHOD_IS_NESTED_FILE = + ReflectionUtils.findMethod(utilsClass, "isNestedFile", new Class[] {virtualFile}); + VIRTUAL_FILE_VISITOR_INTERFACE = loader.loadClass(pkg + "VirtualFileVisitor"); + VIRTUAL_FILE_METHOD_VISIT = ReflectionUtils.findMethod(virtualFile, "visit", + new Class[] {VIRTUAL_FILE_VISITOR_INTERFACE}); + Class visitorAttributesClass = loader.loadClass(pkg + "VisitorAttributes"); + VISITOR_ATTRIBUTES_FIELD_RECURSE = ReflectionUtils.findField(visitorAttributesClass, "RECURSE"); + } catch (ClassNotFoundException var7) { + throw new IllegalStateException("Could not detect the JBoss VFS infrastructure", var7); + } + } + + private static enum VFS_VER { + V2, V3; + + private VFS_VER() {} + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/FileBatch.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/FileBatch.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/FileBatch.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/FileBatch.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/Loader.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/Loader.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/Loader.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/Loader.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSource.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/LocalFileSource.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSource.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/LocalFileSource.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSourceFactory.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/LocalFileSourceFactory.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSourceFactory.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/LocalFileSourceFactory.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/Source.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/Source.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/Source.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/Source.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/SourceFactory.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/SourceFactory.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/loader/SourceFactory.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/loader/SourceFactory.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Atomic.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Atomic.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Atomic.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Atomic.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/AtomicBoolean.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/AtomicBoolean.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/AtomicBoolean.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/AtomicBoolean.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/AtomicDouble.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/AtomicDouble.java new file mode 100644 index 000000000..fb9fb79a0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/AtomicDouble.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import org.nd4j.common.primitives.serde.JsonDeserializerAtomicDouble; +import org.nd4j.common.primitives.serde.JsonSerializerAtomicDouble; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize(using = JsonSerializerAtomicDouble.class) +@JsonDeserialize(using = JsonDeserializerAtomicDouble.class) +public class AtomicDouble extends com.google.common.util.concurrent.AtomicDouble { + + public AtomicDouble(){ + this(0.0); + } + + public AtomicDouble(@JsonProperty("value") double value){ + super(value); + } + + public AtomicDouble(float value){ + this((double)value); + } + + @Override + public boolean equals(Object o){ + //NOTE: com.google.common.util.concurrent.AtomicDouble extends Number, hence this class extends number + if(o instanceof Number){ + return get() == ((Number)o).doubleValue(); + } + return false; + } + + @Override + public int hashCode(){ + //return Double.hashCode(get()); //Java 8+ + return Double.valueOf(get()).hashCode(); + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Counter.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Counter.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Counter.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Counter.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/CounterMap.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/CounterMap.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/CounterMap.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/CounterMap.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutablePair.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/ImmutablePair.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutablePair.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/ImmutablePair.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableQuad.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/ImmutableQuad.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableQuad.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/ImmutableQuad.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableTriple.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/ImmutableTriple.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableTriple.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/ImmutableTriple.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Optional.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Optional.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Optional.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Optional.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Pair.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Pair.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Pair.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Pair.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Quad.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Quad.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Quad.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Quad.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/SynchronizedObject.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/SynchronizedObject.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/SynchronizedObject.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/SynchronizedObject.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Triple.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Triple.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/primitives/Triple.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/Triple.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicBoolean.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicBoolean.java new file mode 100644 index 000000000..6c807feea --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicBoolean.java @@ -0,0 +1,39 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives.serde; + +import org.nd4j.common.primitives.AtomicBoolean; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; + +public class JsonDeserializerAtomicBoolean extends JsonDeserializer { + @Override + public AtomicBoolean deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + boolean value = node.asBoolean(); + return new AtomicBoolean(value); + } +} diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicDouble.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicDouble.java new file mode 100644 index 000000000..d777b0072 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicDouble.java @@ -0,0 +1,39 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives.serde; + +import org.nd4j.common.primitives.AtomicDouble; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; + +public class JsonDeserializerAtomicDouble extends JsonDeserializer { + @Override + public AtomicDouble deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + double value = node.asDouble(); + return new AtomicDouble(value); + } +} diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicBoolean.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicBoolean.java new file mode 100644 index 000000000..c10f1bc95 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicBoolean.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives.serde; + +import org.nd4j.common.primitives.AtomicBoolean; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +public class JsonSerializerAtomicBoolean extends JsonSerializer { + @Override + public void serialize(AtomicBoolean atomicDouble, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException { + jsonGenerator.writeBoolean(atomicDouble.get()); + } +} diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicDouble.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicDouble.java new file mode 100644 index 000000000..1f9041ccd --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicDouble.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives.serde; + +import org.nd4j.common.primitives.AtomicDouble; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +public class JsonSerializerAtomicDouble extends JsonSerializer { + @Override + public void serialize(AtomicDouble atomicDouble, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException { + jsonGenerator.writeNumber(atomicDouble.doubleValue()); + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/resources/Downloader.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/Downloader.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/resources/Downloader.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/Downloader.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/resources/Resolver.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/Resolver.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/resources/Resolver.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/Resolver.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/resources/Resources.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/Resources.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/resources/Resources.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/Resources.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/strumpf/ResourceFile.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/strumpf/ResourceFile.java new file mode 100644 index 000000000..0141be02f --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/strumpf/ResourceFile.java @@ -0,0 +1,259 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources.strumpf; + +import org.nd4j.common.config.ND4JSystemProperties; +import com.google.common.io.Files; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.nd4j.common.base.Preconditions; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +import java.io.*; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +@AllArgsConstructor +@NoArgsConstructor +@Data +@JsonIgnoreProperties("filePath") +@Slf4j +public class ResourceFile { + /** + * Default value for resource downloading connection timeout - see {@link ND4JSystemProperties#RESOURCES_CONNECTION_TIMEOUT} + */ + public static final int DEFAULT_CONNECTION_TIMEOUT = 60000; //Timeout for connections to be established + /** + * Default value for resource downloading read timeout - see {@link ND4JSystemProperties#RESOURCES_READ_TIMEOUT} + */ + public static final int DEFAULT_READ_TIMEOUT = 60000; //Timeout for amount of time between connection established and data is available + protected static final String PATH_KEY = "full_remote_path"; + protected static final String HASH = "_hash"; + protected static final String COMPRESSED_HASH = "_compressed_hash"; + + protected static final int MAX_DOWNLOAD_ATTEMPTS = 3; + + public static final ObjectMapper MAPPER = newMapper(); + + //Note: Field naming to match Strumpf JSON format + protected int current_version; + protected Map v1; + + //Not in JSON: + protected String filePath; + + public static ResourceFile fromFile(String path) { + return fromFile(new File(path)); + } + + public static ResourceFile fromFile(File file) { + String s; + try { + s = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + ResourceFile rf = MAPPER.readValue(s, ResourceFile.class); + rf.setFilePath(file.getPath()); + return rf; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String relativePath() { + String hashKey = null; + for (String key : v1.keySet()) { + if (key.endsWith(HASH) && !key.endsWith(COMPRESSED_HASH)) { + hashKey = key; + break; + } + } + if (hashKey == null) { + throw new IllegalStateException("Could not find _hash in resource reference file: " + filePath); + } + + String relativePath = hashKey.substring(0, hashKey.length() - 5); //-5 to remove "_hash" suffix + return relativePath.replaceAll("\\\\", "/"); + } + + public boolean localFileExistsAndValid(File cacheRootDir) { + + File file = getLocalFile(cacheRootDir); + if (!file.exists()) { + return false; + } + + //File exists... but is it valid? + String sha256Property = relativePath() + HASH; + String expSha256 = v1.get(sha256Property); + + Preconditions.checkState(expSha256 != null, "Expected JSON property %s was not found in resource reference file %s", sha256Property, filePath); + + String actualSha256 = sha256(file); + if (!expSha256.equals(actualSha256)) { + return false; + } + return true; + } + + /** + * Get the local file - or where it *would* be if it has been downloaded. If it does not exist, it will not be downloaded here + * + * @return + */ + protected File getLocalFile(File cacheRootDir) { + String relativePath = relativePath(); + + //For resolving local files with different versions, we want paths like: + // ".../dir/filename.txt__v1/filename.txt" + // ".../dir/filename.txt__v2/filename.txt" + //This is to support multiple versions of files simultaneously... for example, different projects needing different + // versions, or supporting old versions of resource files etc + + int lastSlash = Math.max(relativePath.lastIndexOf('/'), relativePath.lastIndexOf('\\')); + String filename; + if (lastSlash < 0) { + filename = relativePath; + } else { + filename = relativePath.substring(lastSlash + 1); + } + + File parentDir = new File(cacheRootDir, relativePath + "__v" + current_version); + File file = new File(parentDir, filename); + return file; + } + + /** + * Get the local file - downloading and caching if required + * + * @return + */ + public File localFile(File cacheRootDir) { + if (localFileExistsAndValid(cacheRootDir)) { + return getLocalFile(cacheRootDir); + } + + //Need to download and extract... + String remotePath = v1.get(PATH_KEY); + Preconditions.checkState(remotePath != null, "No remote path was found in resource reference file %s", filePath); + File f = getLocalFile(cacheRootDir); + + File tempDir = Files.createTempDir(); + File tempFile = new File(tempDir, FilenameUtils.getName(remotePath)); + + String sha256PropertyCompressed = relativePath() + COMPRESSED_HASH; + + String sha256Compressed = v1.get(sha256PropertyCompressed); + Preconditions.checkState(sha256Compressed != null, "Expected JSON property %s was not found in resource reference file %s", sha256PropertyCompressed, filePath); + + String sha256Property = relativePath() + HASH; + String sha256Uncompressed = v1.get(sha256Property); + + String connTimeoutStr = System.getProperty(ND4JSystemProperties.RESOURCES_CONNECTION_TIMEOUT); + String readTimeoutStr = System.getProperty(ND4JSystemProperties.RESOURCES_READ_TIMEOUT); + boolean validCTimeout = connTimeoutStr != null && connTimeoutStr.matches("\\d+"); + boolean validRTimeout = readTimeoutStr != null && readTimeoutStr.matches("\\d+"); + + int connectTimeout = validCTimeout ? Integer.parseInt(connTimeoutStr) : DEFAULT_CONNECTION_TIMEOUT; + int readTimeout = validRTimeout ? Integer.parseInt(readTimeoutStr) : DEFAULT_READ_TIMEOUT; + + try { + boolean correctHash = false; + for (int tryCount = 0; tryCount < MAX_DOWNLOAD_ATTEMPTS; tryCount++) { + try { + if (tempFile.exists()) + tempFile.delete(); + log.info("Downloading remote resource {} to {}", remotePath, tempFile); + FileUtils.copyURLToFile(new URL(remotePath), tempFile, connectTimeout, readTimeout); + //Now: check if downloaded archive hash is OK + String hash = sha256(tempFile); + correctHash = sha256Compressed.equals(hash); + if (!correctHash) { + log.warn("Download of file {} failed: expected hash {} vs. actual hash {}", remotePath, sha256Compressed, hash); + continue; + } + log.info("Downloaded {} to temporary file {}", remotePath, tempFile); + break; + } catch (Throwable t) { + if (tryCount == MAX_DOWNLOAD_ATTEMPTS - 1) { + throw new RuntimeException("Error downloading test resource: " + remotePath, t); + } + log.warn("Error downloading test resource, retrying... {}", remotePath, t); + } + } + + if (!correctHash) { + throw new RuntimeException("Could not successfully download with correct hash file after " + MAX_DOWNLOAD_ATTEMPTS + + " attempts: " + remotePath); + } + + //Now, extract: + f.getParentFile().mkdirs(); + try (OutputStream os = new BufferedOutputStream(new FileOutputStream(f)); + InputStream is = new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(tempFile)))) { + IOUtils.copy(is, os); + } catch (IOException e) { + throw new RuntimeException("Error extracting resource file", e); + } + log.info("Extracted {} to {}", tempFile, f); + + //Check extracted file hash: + String extractedHash = sha256(f); + if (!extractedHash.equals(sha256Uncompressed)) { + throw new RuntimeException("Extracted file hash does not match expected hash: " + remotePath + + " -> " + f.getAbsolutePath() + " - expected has " + sha256Uncompressed + ", actual hash " + extractedHash); + } + + } finally { + tempFile.delete(); + } + + return f; + } + + public static String sha256(File f) { + try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { + return DigestUtils.sha256Hex(is); + } catch (IOException e) { + throw new RuntimeException("Error when hashing file: " + f.getPath(), e); + } + } + + + public static final ObjectMapper newMapper() { + ObjectMapper ret = new ObjectMapper(); + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); + ret.enable(SerializationFeature.INDENT_OUTPUT); + return ret; + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/resources/strumpf/StrumpfResolver.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/strumpf/StrumpfResolver.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/resources/strumpf/StrumpfResolver.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/resources/strumpf/StrumpfResolver.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/BTools.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/BTools.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/BTools.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/BTools.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/InfoLine.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/InfoLine.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/InfoLine.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/InfoLine.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/InfoValues.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/InfoValues.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/InfoValues.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/InfoValues.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/PropertyParser.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/PropertyParser.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/PropertyParser.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/PropertyParser.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/SIS.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/SIS.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/tools/SIS.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/tools/SIS.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/util/AbstractNumber.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/AbstractNumber.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/util/AbstractNumber.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/AbstractNumber.java diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/util/ArchiveUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/ArchiveUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/util/ArchiveUtils.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/ArchiveUtils.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/ArrayUtil.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/ArrayUtil.java new file mode 100644 index 000000000..8a30f0e48 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/ArrayUtil.java @@ -0,0 +1,3623 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import lombok.val; +import org.apache.commons.lang3.RandomUtils; +import org.nd4j.common.base.Preconditions; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.Array; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * @author Adam Gibson + */ +public class ArrayUtil { + + + private ArrayUtil() {} + + + /** + * Returns true if any array elements are negative. + * If the array is null, it returns false + * @param arr the array to test + * @return + */ + public static boolean containsAnyNegative(int[] arr) { + if(arr == null) + return false; + + for(int i = 0; i < arr.length; i++) { + if(arr[i] < 0) + return true; + } + return false; + } + + public static boolean containsAnyNegative(long[] arr) { + if(arr == null) + return false; + + for(int i = 0; i < arr.length; i++) { + if(arr[i] < 0) + return true; + } + return false; + } + + public static boolean contains(int[] arr, int value){ + if(arr == null) + return false; + for( int i : arr ) { + if (i == value) + return true; + } + return false; + } + + public static boolean contains(long[] arr, int value){ + if(arr == null) + return false; + for( long i : arr ) { + if (i == value) + return true; + } + return false; + } + + /** + * + * @param arrs + * @param check + * @return + */ + public static boolean anyLargerThan(int[] arrs, int check) { + for(int i = 0; i < arrs.length; i++) { + if(arrs[i] > check) + return true; + } + + return false; + } + + + /** + * + * @param arrs + * @param check + * @return + */ + public static boolean anyLessThan(int[] arrs, int check) { + for(int i = 0; i < arrs.length; i++) { + if(arrs[i] < check) + return true; + } + + return false; + } + + + /** + * Convert a int array to a string array + * @param arr the array to convert + * @return the equivalent string array + */ + public static String[] convertToString(int[] arr) { + Preconditions.checkNotNull(arr); + String[] ret = new String[arr.length]; + for(int i = 0; i < arr.length; i++) { + ret[i] = String.valueOf(arr[i]); + } + + return ret; + } + + + /** + * Proper comparison contains for list of int + * arrays + * @param list the to search + * @param target the target int array + * @return whether the given target + * array is contained in the list + */ + public static boolean listOfIntsContains(List list,int[] target) { + for(int[] arr : list) + if(Arrays.equals(target,arr)) + return true; + return false; + } + + /** + * Repeat a value n times + * @param n the number of times to repeat + * @param toReplicate the value to repeat + * @return an array of length n filled with the + * given value + */ + public static int[] nTimes(int n, int toReplicate) { + int[] ret = new int[n]; + Arrays.fill(ret, toReplicate); + return ret; + } + + public static long[] nTimes(long n, long toReplicate) { + if (n > Integer.MAX_VALUE) + throw new RuntimeException("Index overflow in nTimes"); + val ret = new long[(int) n]; + Arrays.fill(ret, toReplicate); + return ret; + } + + public static T[] nTimes(int n, T toReplicate, Class tClass){ + Preconditions.checkState(n>=0, "Invalid number of times to replicate: must be >= 0, got %s", n); + T[] out = (T[])Array.newInstance(tClass, n); + for( int i=0; i set = new HashSet<>(); + for (int i : toTest) { + if (!set.contains(i)) + set.add(i); + else + return false; + } + + return true; + } + + /** + * Credit to mikio braun from jblas + *

+ * Create a random permutation of the numbers 0, ..., size - 1. + *

+ * see Algorithm P, D.E. Knuth: The Art of Computer Programming, Vol. 2, p. 145 + */ + public static int[] randomPermutation(int size) { + Random r = new Random(); + int[] result = new int[size]; + + for (int j = 0; j < size; j++) { + result[j] = j + 1; + } + + for (int j = size - 1; j > 0; j--) { + int k = r.nextInt(j); + int temp = result[j]; + result[j] = result[k]; + result[k] = temp; + } + + return result; + } + + + public static short toBFloat16(float data) { + return (short) (Float.floatToIntBits(data) << 16); + } + + public static short toBFloat16(double data) { + return toBFloat16((float) data); + } + + public static short toHalf(float data) { + return fromFloat(data); + } + + public static short toHalf(double data) { + return fromFloat((float) data); + } + + public static short[] toHalfs(float[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = fromFloat(data[i]); + } + return ret; + } + + public static short[] toHalfs(int[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = fromFloat((float) data[i]); + } + return ret; + } + + public static short[] toHalfs(long[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = fromFloat((float) data[i]); + } + return ret; + } + + public static short[] toBfloats(float[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = toBFloat16(data[i]); + } + return ret; + } + + public static short[] toBfloats(int[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = toBFloat16((float) data[i]); + } + return ret; + } + + public static short[] toBfloats(long[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = toBFloat16((float) data[i]); + } + return ret; + } + + public static long[] toLongs(byte[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static long[] toLongs(boolean[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = data[i] ? 1 : 0; + } + return ret; + } + + public static long[] toLongs(short[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static long[] toLongs(int[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static long[] toLongs(float[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static long[] toLongs(double[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static short[] toHalfs(double[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = fromFloat((float) data[i]); + } + return ret; + } + + public static short fromFloat(float v) { + if (Float.isNaN(v)) + return (short) 0x7fff; + if (v == Float.POSITIVE_INFINITY) + return (short) 0x7c00; + if (v == Float.NEGATIVE_INFINITY) + return (short) 0xfc00; + if (v == 0.0f) + return (short) 0x0000; + if (v == -0.0f) + return (short) 0x8000; + if (v > 65504.0f) + return 0x7bff; // max value supported by half float + if (v < -65504.0f) + return (short) (0x7bff | 0x8000); + if (v > 0.0f && v < 5.96046E-8f) + return 0x0001; + if (v < 0.0f && v > -5.96046E-8f) + return (short) 0x8001; + + final int f = Float.floatToIntBits(v); + + return (short) (((f >> 16) & 0x8000) | ((((f & 0x7f800000) - 0x38000000) >> 13) & 0x7c00) + | ((f >> 13) & 0x03ff)); + } + + public static int[] toInts(float[] data) { + int[] ret = new int[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = (int) data[i]; + return ret; + } + + public static int[] toInts(double[] data) { + int[] ret = new int[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = (int) data[i]; + return ret; + } + + public static byte[] toBytes(int[] array) { + val retVal = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + retVal[i] = (byte) array[i]; + } + return retVal; + } + + public static byte[] toBytes(float[] array) { + val retVal = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + retVal[i] = (byte) array[i]; + } + return retVal; + } + + public static byte[] toBytes(double[] array) { + val retVal = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + retVal[i] = (byte) array[i]; + } + return retVal; + } + + public static byte[] toBytes(long[] array) { + val retVal = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + retVal[i] = (byte) array[i]; + } + return retVal; + } + + public static int[] toInts(long[] array) { + int[] retVal = new int[array.length]; + + for (int i = 0; i < array.length; i++) { + retVal[i] = (int) array[i]; + } + + return retVal; + } + + + public static int[] mod(int[] input,int mod) { + int[] ret = new int[input.length]; + for(int i = 0; i < ret.length; i++) { + ret[i] = input[i] % mod; + } + + return ret; + } + + + /** + * Calculate the offset for a given stride array + * @param stride the stride to use + * @param i the offset to calculate for + * @return the offset for the given + * stride + */ + public static int offsetFor(int[] stride, int i) { + int ret = 0; + for (int j = 0; j < stride.length; j++) + ret += (i * stride[j]); + return ret; + + } + + /** + * Sum of an int array + * @param add the elements + * to calculate the sum for + * @return the sum of this array + */ + public static int sum(List add) { + if (add.isEmpty()) + return 0; + int ret = 0; + for (int i = 0; i < add.size(); i++) + ret += add.get(i); + return ret; + } + + /** + * Sum of an int array + * @param add the elements + * to calculate the sum for + * @return the sum of this array + */ + public static int sum(int[] add) { + if (add.length < 1) + return 0; + int ret = 0; + for (int i = 0; i < add.length; i++) + ret += add[i]; + return ret; + } + + public static long sumLong(long... add) { + if (add.length < 1) + return 0; + int ret = 0; + for (int i = 0; i < add.length; i++) + ret += add[i]; + return ret; + } + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static int prod(List mult) { + if (mult.isEmpty()) + return 0; + int ret = 1; + for (int i = 0; i < mult.size(); i++) + ret *= mult.get(i); + return ret; + } + + + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static int prod(long... mult) { + if (mult.length < 1) + return 0; + int ret = 1; + for (int i = 0; i < mult.length; i++) + ret *= mult[i]; + return ret; + } + + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static int prod(int... mult) { + if (mult.length < 1) + return 0; + int ret = 1; + for (int i = 0; i < mult.length; i++) + ret *= mult[i]; + return ret; + } + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static long prodLong(List mult) { + if (mult.isEmpty()) + return 0; + long ret = 1; + for (int i = 0; i < mult.size(); i++) + ret *= mult.get(i).longValue(); + return ret; + } + + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static long prodLong(int... mult) { + if (mult.length < 1) + return 0; + long ret = 1; + for (int i = 0; i < mult.length; i++) + ret *= mult[i]; + return ret; + } + + public static long prodLong(long... mult) { + if (mult.length < 1) + return 0; + long ret = 1; + for (int i = 0; i < mult.length; i++) + ret *= mult[i]; + return ret; + } + + public static boolean equals(float[] data, double[] data2) { + if (data.length != data2.length) + return false; + for (int i = 0; i < data.length; i++) { + double equals = Math.abs(data2[i] - data[i]); + if (equals > 1e-6) + return false; + } + return true; + } + + + public static int[] consArray(int a, int[] as) { + int len = as.length; + int[] nas = new int[len + 1]; + nas[0] = a; + System.arraycopy(as, 0, nas, 1, len); + return nas; + } + + + /** + * Returns true if any of the elements are zero + * @param as + * @return + */ + public static boolean isZero(int[] as) { + for (int i = 0; i < as.length; i++) { + if (as[i] == 0) + return true; + } + return false; + } + + public static boolean isZero(long[] as) { + for (int i = 0; i < as.length; i++) { + if (as[i] == 0L) + return true; + } + return false; + } + + public static boolean anyMore(int[] target, int[] test) { + Preconditions.checkArgument(target.length == test.length, "Unable to compare: different sizes: length %s vs. %s", target.length, test.length); + for (int i = 0; i < target.length; i++) { + if (target[i] > test[i]) + return true; + } + return false; + } + + + public static boolean anyLess(int[] target, int[] test) { + Preconditions.checkArgument(target.length == test.length, "Unable to compare: different sizes: length %s vs. %s", target.length, test.length); + for (int i = 0; i < target.length; i++) { + if (target[i] < test[i]) + return true; + } + return false; + } + + public static boolean lessThan(int[] target, int[] test) { + Preconditions.checkArgument(target.length == test.length, "Unable to compare: different sizes: length %s vs. %s", target.length, test.length); + for (int i = 0; i < target.length; i++) { + if (target[i] < test[i]) + return true; + if (target[i] > test[i]) + return false; + } + return false; + } + + public static boolean greaterThan(int[] target, int[] test) { + Preconditions.checkArgument(target.length == test.length, "Unable to compare: different sizes: length %s vs. %s", target.length, test.length); + for (int i = 0; i < target.length; i++) { + if (target[i] > test[i]) + return true; + if (target[i] < test[i]) + return false; + } + return false; + } + + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static int calcOffset(List shape, List offsets, List strides) { + if (shape.size() != offsets.size() || shape.size() != strides.size()) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + int ret = 0; + for (int i = 0; i < offsets.size(); i++) { + //we should only do this in the general case, not on vectors + //the reason for this is we force everything including scalars + //to be 2d + if (shape.get(i) == 1 && offsets.size() > 2 && i > 0) + continue; + ret += offsets.get(i) * strides.get(i); + } + + return ret; + } + + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static int calcOffset(int[] shape, int[] offsets, int[] strides) { + if (shape.length != offsets.length || shape.length != strides.length) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + + int ret = 0; + for (int i = 0; i < offsets.length; i++) { + if (shape[i] == 1) + continue; + ret += offsets[i] * strides[i]; + } + + return ret; + } + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static long calcOffset(long[] shape, long[] offsets, long[] strides) { + if (shape.length != offsets.length || shape.length != strides.length) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + + long ret = 0; + for (int i = 0; i < offsets.length; i++) { + if (shape[i] == 1) + continue; + ret += offsets[i] * strides[i]; + } + + return ret; + } + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static long calcOffsetLong(List shape, List offsets, List strides) { + if (shape.size() != offsets.size() || shape.size() != strides.size()) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + long ret = 0; + for (int i = 0; i < offsets.size(); i++) { + //we should only do this in the general case, not on vectors + //the reason for this is we force everything including scalars + //to be 2d + if (shape.get(i) == 1 && offsets.size() > 2 && i > 0) + continue; + ret += (long) offsets.get(i) * strides.get(i); + } + + return ret; + } + + + public static long calcOffsetLong2(List shape, List offsets, List strides) { + if (shape.size() != offsets.size() || shape.size() != strides.size()) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + long ret = 0; + for (int i = 0; i < offsets.size(); i++) { + //we should only do this in the general case, not on vectors + //the reason for this is we force everything including scalars + //to be 2d + if (shape.get(i) == 1 && offsets.size() > 2 && i > 0) + continue; + ret += (long) offsets.get(i) * strides.get(i); + } + + return ret; + } + + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static long calcOffsetLong(int[] shape, int[] offsets, int[] strides) { + if (shape.length != offsets.length || shape.length != strides.length) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + + long ret = 0; + for (int i = 0; i < offsets.length; i++) { + if (shape[i] == 1) + continue; + ret += (long) offsets[i] * strides[i]; + } + + return ret; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static int dotProduct(List xs, List ys) { + int result = 0; + int n = xs.size(); + + if (ys.size() != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += xs.get(i) * ys.get(i); + } + return result; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static int dotProduct(int[] xs, int[] ys) { + int result = 0; + int n = xs.length; + + if (ys.length != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += xs[i] * ys[i]; + } + return result; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static long dotProductLong(List xs, List ys) { + long result = 0; + int n = xs.size(); + + if (ys.size() != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += (long) xs.get(i) * ys.get(i); + } + return result; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static long dotProductLong2(List xs, List ys) { + long result = 0; + int n = xs.size(); + + if (ys.size() != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += (long) xs.get(i) * ys.get(i); + } + return result; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static long dotProductLong(int[] xs, int[] ys) { + long result = 0; + int n = xs.length; + + if (ys.length != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += (long) xs[i] * ys[i]; + } + return result; + } + + + public static int[] empty() { + return new int[0]; + } + + + public static int[] of(int... arr) { + return arr; + } + + public static int[] copy(int[] copy) { + int[] ret = new int[copy.length]; + System.arraycopy(copy, 0, ret, 0, ret.length); + return ret; + } + + public static long[] copy(long[] copy) { + long[] ret = new long[copy.length]; + System.arraycopy(copy, 0, ret, 0, ret.length); + return ret; + } + + + public static double[] doubleCopyOf(float[] data) { + double[] ret = new double[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = data[i]; + return ret; + } + + public static float[] floatCopyOf(double[] data) { + if (data.length == 0) + return new float[1]; + float[] ret = new float[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = (float) data[i]; + return ret; + } + + + /** + * Returns a subset of an array from 0 to "to" (exclusive) + * + * @param data the data to getFromOrigin a subset of + * @param to the end point of the data + * @return the subset of the data specified + */ + public static double[] range(double[] data, int to) { + return range(data, to, 1); + } + + + /** + * Returns a subset of an array from 0 to "to" (exclusive) using the specified stride + * + * @param data the data to getFromOrigin a subset of + * @param to the end point of the data + * @param stride the stride to go through the array + * @return the subset of the data specified + */ + public static double[] range(double[] data, int to, int stride) { + return range(data, to, stride, 1); + } + + + /** + * Returns a subset of an array from 0 to "to" + * using the specified stride + * + * @param data the data to getFromOrigin a subset of + * @param to the end point of the data + * @param stride the stride to go through the array + * @param numElementsEachStride the number of elements to collect at each stride + * @return the subset of the data specified + */ + public static double[] range(double[] data, int to, int stride, int numElementsEachStride) { + double[] ret = new double[to / stride]; + if (ret.length < 1) + ret = new double[1]; + int count = 0; + for (int i = 0; i < data.length; i += stride) { + for (int j = 0; j < numElementsEachStride; j++) { + if (i + j >= data.length || count >= ret.length) + break; + ret[count++] = data[i + j]; + } + } + return ret; + } + + public static List toList(int... ints){ + if(ints == null){ + return null; + } + List ret = new ArrayList<>(); + for (int anInt : ints) { + ret.add(anInt); + } + return ret; + } + + public static int[] toArray(List list) { + int[] ret = new int[list.size()]; + for (int i = 0; i < list.size(); i++) + ret[i] = list.get(i); + return ret; + } + + public static long[] toArrayLong(List list) { + long[] ret = new long[list.size()]; + for (int i = 0; i < list.size(); i++) + ret[i] = list.get(i); + return ret; + } + + + public static double[] toArrayDouble(List list) { + double[] ret = new double[list.size()]; + for (int i = 0; i < list.size(); i++) + ret[i] = list.get(i); + return ret; + + } + + + /** + * Generate an int array ranging from "from" to "to". + * The total number of elements is (from-to)/increment - i.e., range(0,2,1) returns [0,1] + * If from is > to this method will count backwards + * + * @param from the from + * @param to the end point of the data + * @param increment the amount to increment by + * @return the int array with a length equal to absoluteValue(from - to) + */ + public static int[] range(int from, int to, int increment) { + int diff = Math.abs(from - to); + int[] ret = new int[diff / increment]; + if (ret.length < 1) + ret = new int[1]; + + if (from < to) { + int count = 0; + for (int i = from; i < to; i += increment) { + if (count >= ret.length) + break; + ret[count++] = i; + } + } else if (from > to) { + int count = 0; + for (int i = from - 1; i >= to; i -= increment) { + if (count >= ret.length) + break; + ret[count++] = i; + } + } + + return ret; + } + + + public static long[] range(long from, long to, long increment) { + long diff = Math.abs(from - to); + long[] ret = new long[(int) (diff / increment)]; + if (ret.length < 1) + ret = new long[1]; + + if (from < to) { + int count = 0; + for (long i = from; i < to; i += increment) { + if (count >= ret.length) + break; + ret[count++] = i; + } + } else if (from > to) { + int count = 0; + for (int i = (int) from - 1; i >= to; i -= increment) { + if (count >= ret.length) + break; + ret[count++] = i; + } + } + + return ret; + } + + /** + * Generate an int array ranging from "from" to "to". + * The total number of elements is (from-to) - i.e., range(0,2) returns [0,1] + * If from is > to this method will count backwards + * + * @param from the from + * @param to the end point of the data + * @return the int array with a length equal to absoluteValue(from - to) + */ + public static int[] range(int from, int to) { + if (from == to) + return new int[0]; + return range(from, to, 1); + } + + public static long[] range(long from, long to) { + if (from == to) + return new long[0]; + return range(from, to, 1); + } + + public static double[] toDoubles(int[] ints) { + double[] ret = new double[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (double) ints[i]; + return ret; + } + + public static double[] toDoubles(long[] ints) { + double[] ret = new double[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (double) ints[i]; + return ret; + } + + public static double[] toDoubles(float[] ints) { + double[] ret = new double[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (double) ints[i]; + return ret; + } + + public static float[] toFloats(int[][] ints) { + return toFloats(Ints.concat(ints)); + } + + public static double[] toDoubles(int[][] ints) { + return toDoubles(Ints.concat(ints)); + } + + public static short[] toShorts(long[] ints) { + val ret = new short[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (short) ints[i]; + return ret; + } + + public static short[] toShorts(int[] ints) { + val ret = new short[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (short) ints[i]; + return ret; + } + + public static short[] toShorts(float[] ints) { + val ret = new short[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (short) ints[i]; + return ret; + } + + public static short[] toShorts(double[] ints) { + val ret = new short[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (short) ints[i]; + return ret; + } + + public static float[] toFloats(int[] ints) { + float[] ret = new float[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (float) ints[i]; + return ret; + } + + public static float[] toFloats(long[] ints) { + float[] ret = new float[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (float) ints[i]; + return ret; + } + + public static float[] toFloats(double[] ints) { + float[] ret = new float[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (float) ints[i]; + return ret; + } + + public static int[] cutBelowZero(int[] data) { + val ret = new int[data.length]; + for (int i = 0; i < data.length; i++) + ret[i] = data[i] < 0 ? 0 : data[i]; + return ret; + } + + public static long[] cutBelowZero(long[] data) { + val ret = new long[data.length]; + for (int i = 0; i < data.length; i++) + ret[i] = data[i] < 0 ? 0 : data[i]; + return ret; + } + + public static short[] cutBelowZero(short[] data) { + val ret = new short[data.length]; + for (int i = 0; i < data.length; i++) + ret[i] = data[i] < 0 ? 0 : data[i]; + return ret; + } + + public static byte[] cutBelowZero(byte[] data) { + val ret = new byte[data.length]; + for (int i = 0; i < data.length; i++) + ret[i] = data[i] < 0 ? 0 : data[i]; + return ret; + } + + /** + * Return a copy of this array with the + * given index omitted + * + * @param data the data to copy + * @param index the index of the item to remove + * @param newValue the newValue to replace + * @return the new array with the omitted + * item + */ + public static int[] replace(int[] data, int index, int newValue) { + int[] copy = copy(data); + copy[index] = newValue; + return copy; + } + + /** + * Return a copy of this array with only the + * given index(es) remaining + * + * @param data the data to copy + * @param index the index of the item to remove + * @return the new array with the omitted + * item + */ + public static int[] keep(int[] data, int... index) { + if (index.length == data.length) + return data; + + int[] ret = new int[index.length]; + int count = 0; + for (int i = 0; i < data.length; i++) + if (Ints.contains(index, i)) + ret[count++] = data[i]; + + return ret; + } + + /** + * Return a copy of this array with only the + * given index(es) remaining + * + * @param data the data to copy + * @param index the index of the item to remove + * @return the new array with the omitted + * item + */ + public static long[] keep(long[] data, int... index) { + if (index.length == data.length) + return data; + + long[] ret = new long[index.length]; + int count = 0; + for (int i = 0; i < data.length; i++) + if (Ints.contains(index, i)) + ret[count++] = data[i]; + + return ret; + } + + + /** + * Return a copy of this array with the + * given index omitted + * + * PLEASE NOTE: index to be omitted must exist in source array. + * + * @param data the data to copy + * @param index the index of the item to remove + * @return the new array with the omitted + * item + */ + public static int[] removeIndex(int[] data, int... index) { + if (index.length >= data.length) { + throw new IllegalStateException("Illegal remove: indexes.length > data.length (index.length=" + + index.length + ", data.length=" + data.length + ")"); + } + int offset = 0; + /* + workaround for non-existent indexes (such as Integer.MAX_VALUE) + + + for (int i = 0; i < index.length; i ++) { + if (index[i] >= data.length || index[i] < 0) offset++; + } + */ + + int[] ret = new int[data.length - index.length + offset]; + int count = 0; + for (int i = 0; i < data.length; i++) + if (!Ints.contains(index, i)) { + ret[count++] = data[i]; + } + + return ret; + } + + public static long[] removeIndex(long[] data, int... index) { + if (index.length >= data.length) { + throw new IllegalStateException("Illegal remove: indexes.length >= data.length (index.length=" + + index.length + ", data.length=" + data.length + ")"); + } + int offset = 0; + /* + workaround for non-existent indexes (such as Integer.MAX_VALUE) + + + for (int i = 0; i < index.length; i ++) { + if (index[i] >= data.length || index[i] < 0) offset++; + } + */ + + long[] ret = new long[data.length - index.length + offset]; + int count = 0; + for (int i = 0; i < data.length; i++) + if (!Ints.contains(index, i)) { + ret[count++] = data[i]; + } + + return ret; + } + + + + /** + * Zip 2 arrays in to: + * + * @param as + * @param bs + * @return + */ + public static int[][] zip(int[] as, int[] bs) { + int[][] result = new int[as.length][2]; + for (int i = 0; i < result.length; i++) { + result[i] = new int[] {as[i], bs[i]}; + } + + return result; + } + + /** + * Get the tensor matrix multiply shape + * @param aShape the shape of the first array + * @param bShape the shape of the second array + * @param axes the axes to do the multiply + * @return the shape for tensor matrix multiply + */ + public static long[] getTensorMmulShape(long[] aShape, long[] bShape, int[][] axes) { + + int validationLength = Math.min(axes[0].length, axes[1].length); + for (int i = 0; i < validationLength; i++) { + if (aShape[axes[0][i]] != bShape[axes[1][i]]) + throw new IllegalArgumentException( + "Size of the given axes a" + " t each dimension must be the same size."); + if (axes[0][i] < 0) + axes[0][i] += aShape.length; + if (axes[1][i] < 0) + axes[1][i] += bShape.length; + + } + + List listA = new ArrayList<>(); + for (int i = 0; i < aShape.length; i++) { + if (!Ints.contains(axes[0], i)) + listA.add(i); + } + + + + List listB = new ArrayList<>(); + for (int i = 0; i < bShape.length; i++) { + if (!Ints.contains(axes[1], i)) + listB.add(i); + } + + + int n2 = 1; + int aLength = Math.min(aShape.length, axes[0].length); + for (int i = 0; i < aLength; i++) { + n2 *= aShape[axes[0][i]]; + } + + //if listA and listB are empty these donot initialize. + //so initializing with {1} which will then get overriden if not empty + long[] oldShapeA; + if (listA.size() == 0) { + oldShapeA = new long[] {1}; + } else { + oldShapeA = Longs.toArray(listA); + for (int i = 0; i < oldShapeA.length; i++) + oldShapeA[i] = aShape[(int) oldShapeA[i]]; + } + + int n3 = 1; + int bNax = Math.min(bShape.length, axes[1].length); + for (int i = 0; i < bNax; i++) { + n3 *= bShape[axes[1][i]]; + } + + + long[] oldShapeB; + if (listB.isEmpty()) { + oldShapeB = new long[] {1}; + } else { + oldShapeB = Longs.toArray(listB); + for (int i = 0; i < oldShapeB.length; i++) + oldShapeB[i] = bShape[(int) oldShapeB[i]]; + } + + + long[] aPlusB = Longs.concat(oldShapeA, oldShapeB); + return aPlusB; + } + + /** + * Permute the given input + * switching the dimensions of the input shape + * array with in the order of the specified + * dimensions + * @param shape the shape to permute + * @param dimensions the dimensions + * @return + */ + public static int[] permute(int[] shape, int[] dimensions) { + int[] ret = new int[shape.length]; + for (int i = 0; i < shape.length; i++) { + ret[i] = shape[dimensions[i]]; + } + + return ret; + } + + + public static long[] permute(long[] shape, int[] dimensions) { + val ret = new long[shape.length]; + for (int i = 0; i < shape.length; i++) { + ret[i] = shape[dimensions[i]]; + } + + return ret; + } + + + /** + * Original credit: https://github.com/alberts/array4j/blob/master/src/main/java/net/lunglet/util/ArrayUtils.java + * @param a + * @return + */ + public static int[] argsort(int[] a) { + return argsort(a, true); + } + + + /** + * + * @param a + * @param ascending + * @return + */ + public static int[] argsort(final int[] a, final boolean ascending) { + Integer[] indexes = new Integer[a.length]; + for (int i = 0; i < indexes.length; i++) { + indexes[i] = i; + } + Arrays.sort(indexes, new Comparator() { + @Override + public int compare(final Integer i1, final Integer i2) { + return (ascending ? 1 : -1) * Ints.compare(a[i1], a[i2]); + } + }); + + int[] ret = new int[indexes.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = indexes[i]; + + return ret; + } + + + + /** + * Convert all dimensions in the specified + * axes array to be positive + * based on the specified range of values + * @param range + * @param axes + * @return + */ + public static int[] convertNegativeIndices(int range, int[] axes) { + int[] axesRet = ArrayUtil.range(0, range); + int[] newAxes = ArrayUtil.copy(axes); + for (int i = 0; i < axes.length; i++) { + newAxes[i] = axes[axesRet[i]]; + } + + return newAxes; + } + + + + /** + * Generate an array from 0 to length + * and generate take a subset + * @param length the length to generate to + * @param from the begin of the interval to take + * @param to the end of the interval to take + * @return the generated array + */ + public static int[] copyOfRangeFrom(int length, int from, int to) { + return Arrays.copyOfRange(ArrayUtil.range(0, length), from, to); + + } + + //Credit: https://stackoverflow.com/questions/15533854/converting-byte-array-to-double-array + + /** + * + * @param doubleArray + * @return + */ + public static byte[] toByteArray(double[] doubleArray) { + int times = Double.SIZE / Byte.SIZE; + byte[] bytes = new byte[doubleArray.length * times]; + for (int i = 0; i < doubleArray.length; i++) { + ByteBuffer.wrap(bytes, i * times, times).putDouble(doubleArray[i]); + } + return bytes; + } + + /** + * + * @param byteArray + * @return + */ + public static double[] toDoubleArray(byte[] byteArray) { + int times = Double.SIZE / Byte.SIZE; + double[] doubles = new double[byteArray.length / times]; + for (int i = 0; i < doubles.length; i++) { + doubles[i] = ByteBuffer.wrap(byteArray, i * times, times).getDouble(); + } + return doubles; + } + + + /** + * + * @param doubleArray + * @return + */ + public static byte[] toByteArray(float[] doubleArray) { + int times = Float.SIZE / Byte.SIZE; + byte[] bytes = new byte[doubleArray.length * times]; + for (int i = 0; i < doubleArray.length; i++) { + ByteBuffer.wrap(bytes, i * times, times).putFloat(doubleArray[i]); + } + return bytes; + } + + public static long[] toLongArray(int[] intArray) { + long[] ret = new long[intArray.length]; + for (int i = 0; i < intArray.length; i++) { + ret[i] = intArray[i]; + } + return ret; + } + + public static long[] toLongArray(float[] array) { + val ret = new long[array.length]; + for (int i = 0; i < array.length; i++) { + ret[i] = (long) array[i]; + } + return ret; + } + + /** + * + * @param byteArray + * @return + */ + public static float[] toFloatArray(byte[] byteArray) { + int times = Float.SIZE / Byte.SIZE; + float[] doubles = new float[byteArray.length / times]; + for (int i = 0; i < doubles.length; i++) { + doubles[i] = ByteBuffer.wrap(byteArray, i * times, times).getFloat(); + } + return doubles; + } + + /** + * + * @param intArray + * @return + */ + public static byte[] toByteArray(int[] intArray) { + int times = Integer.SIZE / Byte.SIZE; + byte[] bytes = new byte[intArray.length * times]; + for (int i = 0; i < intArray.length; i++) { + ByteBuffer.wrap(bytes, i * times, times).putInt(intArray[i]); + } + return bytes; + } + + /** + * + * @param byteArray + * @return + */ + public static int[] toIntArray(byte[] byteArray) { + int times = Integer.SIZE / Byte.SIZE; + int[] ints = new int[byteArray.length / times]; + for (int i = 0; i < ints.length; i++) { + ints[i] = ByteBuffer.wrap(byteArray, i * times, times).getInt(); + } + return ints; + } + + + /** + * Return a copy of this array with the + * given index omitted + * + * @param data the data to copy + * @param index the index of the item to remove + * @return the new array with the omitted + * item + */ + public static int[] removeIndex(int[] data, int index) { + if (data == null) + return null; + + if (index >= data.length) + throw new IllegalArgumentException("Unable to remove index " + index + " was >= data.length"); + if (data.length < 1) + return data; + if (index < 0) + return data; + + int len = data.length; + int[] result = new int[len - 1]; + System.arraycopy(data, 0, result, 0, index); + System.arraycopy(data, index + 1, result, index, len - index - 1); + return result; + } + + public static long[] removeIndex(long[] data, int index) { + if (data == null) + return null; + + if (index >= data.length) + throw new IllegalArgumentException("Unable to remove index " + index + " was >= data.length"); + if (data.length < 1) + return data; + if (index < 0) + return data; + + int len = data.length; + long[] result = new long[len - 1]; + System.arraycopy(data, 0, result, 0, index); + System.arraycopy(data, index + 1, result, index, len - index - 1); + return result; + } + + + /** + * Create a copy of the given array + * starting at the given index with the given length. + * + * The intent here is for striding. + * + * For example in slicing, you want the major stride to be first. + * You achieve this by taking the last index + * of the matrix's stride and putting + * this as the first stride of the new ndarray + * for slicing. + * + * All of the elements except the copied elements are + * initialized as the given value + * @param valueStarting the starting value + * @param copy the array to copy + * @param idxFrom the index to start at in the from array + * @param idxAt the index to start at in the return array + * @param length the length of the array to create + * @return the given array + */ + public static int[] valueStartingAt(int valueStarting, int[] copy, int idxFrom, int idxAt, int length) { + int[] ret = new int[length]; + Arrays.fill(ret, valueStarting); + for (int i = 0; i < length; i++) { + if (i + idxFrom >= copy.length || i + idxAt >= ret.length) + break; + ret[i + idxAt] = copy[i + idxFrom]; + } + + return ret; + } + + + + /** + * Returns the array with the item in index + * removed, if the array is empty it will return the array itself + * + * @param data the data to remove data from + * @param index the index of the item to remove + * @return a copy of the array with the removed item, + * or the array itself if empty + */ + public static Integer[] removeIndex(Integer[] data, int index) { + if (data == null) + return null; + if (data.length < 1) + return data; + int len = data.length; + Integer[] result = new Integer[len - 1]; + System.arraycopy(data, 0, result, 0, index); + System.arraycopy(data, index + 1, result, index, len - index - 1); + return result; + } + + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @param startNum the start number for the strides + * @return the strides for a matrix of n dimensions + */ + public static int[] calcStridesFortran(int[] shape, int startNum) { + if (shape.length == 2 && (shape[0] == 1 || shape[1] == 1)) { + int[] ret = new int[2]; + Arrays.fill(ret, startNum); + return ret; + } + + int dimensions = shape.length; + int[] stride = new int[dimensions]; + int st = startNum; + for (int j = 0; j < stride.length; j++) { + stride[j] = st; + st *= shape[j]; + } + + return stride; + } + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @param startNum the start number for the strides + * @return the strides for a matrix of n dimensions + */ + public static long[] calcStridesFortran(long[] shape, int startNum) { + if (shape.length == 2 && (shape[0] == 1 || shape[1] == 1)) { + long[] ret = new long[2]; + Arrays.fill(ret, startNum); + return ret; + } + + int dimensions = shape.length; + long[] stride = new long[dimensions]; + int st = startNum; + for (int j = 0; j < stride.length; j++) { + stride[j] = st; + st *= shape[j]; + } + + return stride; + } + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @return the strides for a matrix of n dimensions + */ + public static int[] calcStridesFortran(int[] shape) { + return calcStridesFortran(shape, 1); + } + + public static long[] calcStridesFortran(long[] shape) { + return calcStridesFortran(shape, 1); + } + + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @param startValue the startValue for the strides + * @return the strides for a matrix of n dimensions + */ + public static int[] calcStrides(int[] shape, int startValue) { + if (shape.length == 2 && (shape[0] == 1 || shape[1] == 1)) { + int[] ret = new int[2]; + Arrays.fill(ret, startValue); + return ret; + } + + + int dimensions = shape.length; + int[] stride = new int[dimensions]; + + int st = startValue; + for (int j = dimensions - 1; j >= 0; j--) { + stride[j] = st; + st *= shape[j]; + } + + return stride; + } + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @param startValue the startValue for the strides + * @return the strides for a matrix of n dimensions + */ + public static long[] calcStrides(long[] shape, int startValue) { + if (shape.length == 2 && (shape[0] == 1 || shape[1] == 1)) { + long[] ret = new long[2]; + Arrays.fill(ret, startValue); + return ret; + } + + + int dimensions = shape.length; + long[] stride = new long[dimensions]; + + int st = startValue; + for (int j = dimensions - 1; j >= 0; j--) { + stride[j] = st; + st *= shape[j]; + } + + return stride; + } + + + /** + * Returns true if the given + * two arrays are reverse copies of each other + * @param first + * @param second + * @return + */ + public static boolean isInverse(int[] first, int[] second) { + int backWardCount = second.length - 1; + for (int i = 0; i < first.length; i++) { + if (first[i] != second[backWardCount--]) + return false; + } + return true; + } + + public static int[] plus(int[] ints, int mult) { + int[] ret = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = ints[i] + mult; + return ret; + } + + + public static int[] plus(int[] ints, int[] mult) { + if (ints.length != mult.length) + throw new IllegalArgumentException("Both arrays must have the same length"); + int[] ret = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = ints[i] + mult[i]; + return ret; + } + + public static int[] times(int[] ints, int mult) { + int[] ret = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = ints[i] * mult; + return ret; + } + + public static int[] times(int[] ints, int[] mult) { + Preconditions.checkArgument(ints.length == mult.length, "Ints and mult must be the same length"); + int[] ret = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = ints[i] * mult[i]; + return ret; + } + + + + /** + * For use with row vectors to ensure consistent strides + * with varying offsets + * + * @param arr the array to get the stride for + * @return the stride + */ + public static int nonOneStride(int[] arr) { + for (int i = 0; i < arr.length; i++) + if (arr[i] != 1) + return arr[i]; + return 1; + } + + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @return the strides for a matrix of n dimensions + */ + public static int[] calcStrides(int[] shape) { + return calcStrides(shape, 1); + } + + public static long[] calcStrides(long[] shape) { + return calcStrides(shape, 1); + } + + + /** + * Create a backwards copy of the given array + * + * @param e the array to createComplex a reverse clone of + * @return the reversed copy + */ + public static int[] reverseCopy(int[] e) { + if (e.length < 1) + return e; + + int[] copy = new int[e.length]; + for (int i = 0; i <= e.length / 2; i++) { + int temp = e[i]; + copy[i] = e[e.length - i - 1]; + copy[e.length - i - 1] = temp; + } + return copy; + } + + public static long[] reverseCopy(long[] e) { + if (e.length < 1) + return e; + + long[] copy = new long[e.length]; + for (int i = 0; i <= e.length / 2; i++) { + long temp = e[i]; + copy[i] = e[e.length - i - 1]; + copy[e.length - i - 1] = temp; + } + return copy; + } + + + public static double[] read(int length, DataInputStream dis) throws IOException { + double[] ret = new double[length]; + for (int i = 0; i < length; i++) + ret[i] = dis.readDouble(); + return ret; + } + + + public static void write(double[] data, DataOutputStream dos) throws IOException { + for (int i = 0; i < data.length; i++) + dos.writeDouble(data[i]); + } + + public static double[] readDouble(int length, DataInputStream dis) throws IOException { + double[] ret = new double[length]; + for (int i = 0; i < length; i++) + ret[i] = dis.readDouble(); + return ret; + } + + + public static float[] readFloat(int length, DataInputStream dis) throws IOException { + float[] ret = new float[length]; + for (int i = 0; i < length; i++) + ret[i] = dis.readFloat(); + return ret; + } + + + public static void write(float[] data, DataOutputStream dos) throws IOException { + for (int i = 0; i < data.length; i++) + dos.writeFloat(data[i]); + } + + + public static void assertSquare(double[]... d) { + if (d.length > 2) { + for (int i = 0; i < d.length; i++) { + assertSquare(d[i]); + } + } else { + int firstLength = d[0].length; + for (int i = 1; i < d.length; i++) { + Preconditions.checkState(d[i].length == firstLength); + } + } + } + + + /** + * Multiply the given array + * by the given scalar + * @param arr the array to multily + * @param mult the scalar to multiply by + */ + public static void multiplyBy(int[] arr, int mult) { + for (int i = 0; i < arr.length; i++) + arr[i] *= mult; + + } + + /** + * Reverse the passed in array in place + * + * @param e the array to reverse + */ + public static void reverse(int[] e) { + for (int i = 0; i <= e.length / 2; i++) { + int temp = e[i]; + e[i] = e[e.length - i - 1]; + e[e.length - i - 1] = temp; + } + } + + public static void reverse(long[] e) { + for (int i = 0; i <= e.length / 2; i++) { + long temp = e[i]; + e[i] = e[e.length - i - 1]; + e[e.length - i - 1] = temp; + } + } + + + public static List zerosMatrix(long... dimensions) { + List ret = new ArrayList<>(); + for (int i = 0; i < dimensions.length; i++) { + ret.add(new double[(int) dimensions[i]]); + } + return ret; + } + + public static List zerosMatrix(int... dimensions) { + List ret = new ArrayList<>(); + for (int i = 0; i < dimensions.length; i++) { + ret.add(new double[dimensions[i]]); + } + return ret; + } + + + public static float[] reverseCopy(float[] e) { + float[] copy = new float[e.length]; + for (int i = 0; i <= e.length / 2; i++) { + float temp = e[i]; + copy[i] = e[e.length - i - 1]; + copy[e.length - i - 1] = temp; + } + return copy; + + } + + + public static E[] reverseCopy(E[] e) { + E[] copy = (E[]) new Object[e.length]; + for (int i = 0; i <= e.length / 2; i++) { + E temp = e[i]; + copy[i] = e[e.length - i - 1]; + copy[e.length - i - 1] = temp; + } + return copy; + + } + + public static void reverse(E[] e) { + for (int i = 0; i <= e.length / 2; i++) { + E temp = e[i]; + e[i] = e[e.length - i - 1]; + e[e.length - i - 1] = temp; + } + } + + public static boolean[] flatten(boolean[][] arr) { + if(arr.length == 0 || arr[0].length == 0) + return new boolean[0]; + boolean[] ret = new boolean[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static boolean[] flatten(boolean[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new boolean[0]; + boolean[] ret = new boolean[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static float[] flatten(float[][] arr) { + if(arr.length == 0 || arr[0].length == 0) + return new float[0]; + float[] ret = new float[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + + public static float[] flatten(float[][][] arr) { + if (arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new float[0]; + float[] ret = new float[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + + return ret; + } + + public static double[] flatten(double[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new double[0]; + double[] ret = new double[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static int[] flatten(int[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new int[0]; + int[] ret = new int[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static short[] flatten(short[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new short[0]; + val ret = new short[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static byte[] flatten(byte[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new byte[0]; + val ret = new byte[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static long[] flatten(long[][][][] arr) { + val ret = new long[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static short[] flatten(short[][][][] arr) { + val ret = new short[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static byte[] flatten(byte[][][][] arr) { + val ret = new byte[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static boolean[] flatten(boolean[][][][] arr) { + val ret = new boolean[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static float[] flatten(float[][][][] arr) { + float[] ret = new float[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static double[] flatten(double[][][][] arr) { + double[] ret = new double[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static int[] flatten(int[][][][] arr) { + int[] ret = new int[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + + public static int[] flatten(int[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new int[0]; + int[] ret = new int[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static short[] flatten(short[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new short[0]; + val ret = new short[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static byte[] flatten(byte[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new byte[0]; + val ret = new byte[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static long[] flatten(long[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new long[0]; + long[] ret = new long[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static long[] flatten(long[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new long[0]; + long[] ret = new long[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + + /** + * Convert a 2darray in to a flat + * array (row wise) + * @param arr the array to flatten + * @return a flattened representation of the array + */ + public static double[] flatten(double[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new double[0]; + double[] ret = new double[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + /** + * Convert a 2darray in to a flat + * array (row wise) + * @param arr the array to flatten + * @return a flattened representation of the array + */ + public static double[] flattenF(double[][] arr) { + double[] ret = new double[arr.length * arr[0].length]; + int count = 0; + for (int j = 0; j < arr[0].length; j++) + for (int i = 0; i < arr.length; i++) + ret[count++] = arr[i][j]; + return ret; + } + + public static float[] flattenF(float[][] arr) { + float[] ret = new float[arr.length * arr[0].length]; + int count = 0; + for (int j = 0; j < arr[0].length; j++) + for (int i = 0; i < arr.length; i++) + ret[count++] = arr[i][j]; + return ret; + } + + public static int[] flattenF(int[][] arr) { + int[] ret = new int[arr.length * arr[0].length]; + int count = 0; + for (int j = 0; j < arr[0].length; j++) + for (int i = 0; i < arr.length; i++) + ret[count++] = arr[i][j]; + return ret; + } + + + public static long[] flattenF(long[][] arr) { + long[] ret = new long[arr.length * arr[0].length]; + int count = 0; + for (int j = 0; j < arr[0].length; j++) + for (int i = 0; i < arr.length; i++) + ret[count++] = arr[i][j]; + return ret; + } + + public static int[][] reshapeInt(int[] in, int rows, int cols){ + int[][] out = new int[rows][cols]; + int x = 0; + for(int i=0; i T[][] reshapeObject(T[] in, int rows, int cols){ + Object[][] out = new Object[rows][cols]; + int x = 0; + for(int i=0; i T[][][] reshapeObject(T[] in, int d0, int d1, int d2){ + Object[][][] out = new Object[d0][d1][d2]; + int x = 0; + for(int i=0; i nums) { + int length = 0; + for (int i = 0; i < nums.size(); i++) + length += nums.get(i).length; + float[] ret = new float[length]; + int count = 0; + for (float[] i : nums) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + + /** + * Combines a apply of int arrays in to one flat int array + * + * @param nums the int arrays to combineDouble + * @return one combined int array + */ + public static float[] combine(List nums) { + int length = 0; + for (int i = 0; i < nums.size(); i++) + length += nums.get(i).length; + float[] ret = new float[length]; + int count = 0; + for (float[] i : nums) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + /** + * Combines a apply of int arrays in to one flat int array + * + * @param nums the int arrays to combineDouble + * @return one combined int array + */ + public static double[] combineDouble(List nums) { + int length = 0; + for (int i = 0; i < nums.size(); i++) + length += nums.get(i).length; + double[] ret = new double[length]; + int count = 0; + for (double[] i : nums) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + /** + * Combines a apply of int arrays in to one flat int array + * + * @param ints the int arrays to combineDouble + * @return one combined int array + */ + public static double[] combine(float[]... ints) { + int length = 0; + for (int i = 0; i < ints.length; i++) + length += ints[i].length; + double[] ret = new double[length]; + int count = 0; + for (float[] i : ints) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + /** + * Combines a apply of int arrays in to one flat int array + * + * @param ints the int arrays to combineDouble + * @return one combined int array + */ + public static int[] combine(int[]... ints) { + int length = 0; + for (int i = 0; i < ints.length; i++) + length += ints[i].length; + int[] ret = new int[length]; + int count = 0; + for (int[] i : ints) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + /** + * Combines a apply of long arrays in to one flat long array + * + * @param ints the int arrays to combineDouble + * @return one combined int array + */ + public static long[] combine(long[]... ints) { + int length = 0; + for (int i = 0; i < ints.length; i++) + length += ints[i].length; + long[] ret = new long[length]; + int count = 0; + for (long[] i : ints) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + + public static E[] combine(E[]... arrs) { + int length = 0; + for (int i = 0; i < arrs.length; i++) + length += arrs[i].length; + + E[] ret = (E[]) Array.newInstance(arrs[0][0].getClass(), length); + int count = 0; + for (E[] i : arrs) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + + public static int[] toOutcomeArray(int outcome, int numOutcomes) { + int[] nums = new int[numOutcomes]; + nums[outcome] = 1; + return nums; + } + + public static double[] toDouble(int[] data) { + double[] ret = new double[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = data[i]; + return ret; + } + + public static double[] toDouble(long[] data) { + double[] ret = new double[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = data[i]; + return ret; + } + + public static float[] copy(float[] data) { + float[] result = new float[data.length]; + System.arraycopy(data, 0, result, 0, data.length); + return result; + } + + public static double[] copy(double[] data) { + double[] result = new double[data.length]; + System.arraycopy(data, 0, result, 0, data.length); + return result; + } + + + /** Convert an arbitrary-dimensional rectangular double array to flat vector.
+ * Can pass double[], double[][], double[][][], etc. + */ + public static double[] flattenDoubleArray(Object doubleArray) { + if (doubleArray instanceof double[]) + return (double[]) doubleArray; + + LinkedList stack = new LinkedList<>(); + stack.push(doubleArray); + + int[] shape = arrayShape(doubleArray); + int length = ArrayUtil.prod(shape); + double[] flat = new double[length]; + int count = 0; + + while (!stack.isEmpty()) { + Object current = stack.pop(); + if (current instanceof double[]) { + double[] arr = (double[]) current; + for (int i = 0; i < arr.length; i++) + flat[count++] = arr[i]; + } else if (current instanceof Object[]) { + Object[] o = (Object[]) current; + for (int i = o.length - 1; i >= 0; i--) + stack.push(o[i]); + } else + throw new IllegalArgumentException("Base array is not double[]"); + } + + if (count != flat.length) + throw new IllegalArgumentException("Fewer elements than expected. Array is ragged?"); + return flat; + } + + /** Convert an arbitrary-dimensional rectangular float array to flat vector.
+ * Can pass float[], float[][], float[][][], etc. + */ + public static float[] flattenFloatArray(Object floatArray) { + if (floatArray instanceof float[]) + return (float[]) floatArray; + + LinkedList stack = new LinkedList<>(); + stack.push(floatArray); + + int[] shape = arrayShape(floatArray); + int length = ArrayUtil.prod(shape); + float[] flat = new float[length]; + int count = 0; + + while (!stack.isEmpty()) { + Object current = stack.pop(); + if (current instanceof float[]) { + float[] arr = (float[]) current; + for (int i = 0; i < arr.length; i++) + flat[count++] = arr[i]; + } else if (current instanceof Object[]) { + Object[] o = (Object[]) current; + for (int i = o.length - 1; i >= 0; i--) + stack.push(o[i]); + } else + throw new IllegalArgumentException("Base array is not float[]"); + } + + if (count != flat.length) + throw new IllegalArgumentException("Fewer elements than expected. Array is ragged?"); + return flat; + } + + /** Calculate the shape of an arbitrary multi-dimensional array. Assumes:
+ * (a) array is rectangular (not ragged) and first elements (i.e., array[0][0][0]...) are non-null
+ * (b) First elements have > 0 length. So array[0].length > 0, array[0][0].length > 0, etc.
+ * Can pass any Java array opType: double[], Object[][][], float[][], etc.
+ * Length of returned array is number of dimensions; returned[i] is size of ith dimension. + */ + public static int[] arrayShape(Object array) { + return arrayShape(array, false); + } + + /** Calculate the shape of an arbitrary multi-dimensional array.
+ * Note that the method assumes the array is rectangular (not ragged) and first elements (i.e., array[0][0][0]...) are non-null
+ * Note also that if allowSize0Dims is true, any elements are length 0, all subsequent dimensions will be reported as 0. + * i.e., a double[3][0][2] would be reported as shape [3,0,0]. If allowSize0Dims is false, an exception will be thrown for this case instead. + * Can pass any Java array opType: double[], Object[][][], float[][], etc.
+ * Length of returned array is number of dimensions; returned[i] is size of ith dimension. + */ + public static int[] arrayShape(Object array, boolean allowSize0Dims) { + int nDimensions = 0; + Class c = array.getClass().getComponentType(); + while (c != null) { + nDimensions++; + c = c.getComponentType(); + } + + int[] shape = new int[nDimensions]; + Object current = array; + for (int i = 0; i < shape.length - 1; i++) { + shape[i] = ((Object[]) current).length; + if(shape[i] == 0){ + if(allowSize0Dims){ + return shape; + } + throw new IllegalStateException("Cannot calculate array shape: Array has size 0 for dimension " + i ); + } + current = ((Object[]) current)[0]; + } + + if (current instanceof Object[]) { + shape[shape.length - 1] = ((Object[]) current).length; + } else if (current instanceof double[]) { + shape[shape.length - 1] = ((double[]) current).length; + } else if (current instanceof float[]) { + shape[shape.length - 1] = ((float[]) current).length; + } else if (current instanceof long[]) { + shape[shape.length - 1] = ((long[]) current).length; + } else if (current instanceof int[]) { + shape[shape.length - 1] = ((int[]) current).length; + } else if (current instanceof byte[]) { + shape[shape.length - 1] = ((byte[]) current).length; + } else if (current instanceof char[]) { + shape[shape.length - 1] = ((char[]) current).length; + } else if (current instanceof boolean[]) { + shape[shape.length - 1] = ((boolean[]) current).length; + } else if (current instanceof short[]) { + shape[shape.length - 1] = ((short[]) current).length; + } else + throw new IllegalStateException("Unknown array type"); //Should never happen + return shape; + } + + + /** Returns the maximum value in the array */ + public static int max(int[] in) { + int max = Integer.MIN_VALUE; + for (int i = 0; i < in.length; i++) + if (in[i] > max) + max = in[i]; + return max; + } + + /** Returns the minimum value in the array */ + public static int min(int[] in) { + int min = Integer.MAX_VALUE; + for (int i = 0; i < in.length; i++) + if (in[i] < min) + min = in[i]; + return min; + } + + /** Returns the index of the maximum value in the array. + * If two entries have same maximum value, index of the first one is returned. */ + public static int argMax(int[] in) { + int maxIdx = 0; + for (int i = 1; i < in.length; i++) + if (in[i] > in[maxIdx]) + maxIdx = i; + return maxIdx; + } + + /** Returns the index of the minimum value in the array. + * If two entries have same minimum value, index of the first one is returned. */ + public static int argMin(int[] in) { + int minIdx = 0; + for (int i = 1; i < in.length; i++) + if (in[i] < in[minIdx]) + minIdx = i; + return minIdx; + } + + /** Returns the index of the maximum value in the array. + * If two entries have same maximum value, index of the first one is returned. */ + public static int argMax(long[] in) { + int maxIdx = 0; + for (int i = 1; i < in.length; i++) + if (in[i] > in[maxIdx]) + maxIdx = i; + return maxIdx; + } + + /** Returns the index of the minimum value in the array. + * If two entries have same minimum value, index of the first one is returned. */ + public static int argMin(long[] in) { + int minIdx = 0; + for (int i = 1; i < in.length; i++) + if (in[i] < in[minIdx]) + minIdx = i; + return minIdx; + } + + /** + * + * @return + */ + public static int[] buildHalfVector(Random rng, int length) { + int[] result = new int[length]; + List indexes = new ArrayList<>(); + + // we add indexes from second half only + for (int i = result.length - 1; i >= result.length / 2; i--) { + indexes.add(i); + } + + Collections.shuffle(indexes, rng); + + for (int i = 0; i < result.length; i++) { + if (i < result.length / 2) { + result[i] = indexes.get(0); + indexes.remove(0); + } else + result[i] = -1; + } + + return result; + } + + public static int[] buildInterleavedVector(Random rng, int length) { + int[] result = new int[length]; + + List indexes = new ArrayList<>(); + List odds = new ArrayList<>(); + + // we add odd indexes only to list + for (int i = 1; i < result.length; i += 2) { + indexes.add(i); + odds.add(i - 1); + } + + Collections.shuffle(indexes, rng); + + // now all even elements will be interleaved with odd elements + for (int i = 0; i < result.length; i++) { + if (i % 2 == 0 && !indexes.isEmpty()) { + int idx = indexes.get(0); + indexes.remove(0); + result[i] = idx; + } else + result[i] = -1; + } + + // for odd tad numbers, we add special random clause for last element + if (length % 2 != 0) { + int rndClause = odds.get(rng.nextInt(odds.size())); + int tmp = result[rndClause]; + result[rndClause] = result[result.length - 1]; + result[result.length - 1] = tmp; + } + + + return result; + } + + public static long[] buildInterleavedVector(Random rng, long length) { + if (length > Integer.MAX_VALUE) { + throw new RuntimeException("Integer overflow"); + } + val result = new long[(int) length]; + + List indexes = new ArrayList<>(); + List odds = new ArrayList<>(); + + // we add odd indexes only to list + for (int i = 1; i < result.length; i += 2) { + indexes.add(i); + odds.add(i - 1); + } + + Collections.shuffle(indexes, rng); + + // now all even elements will be interleaved with odd elements + for (int i = 0; i < result.length; i++) { + if (i % 2 == 0 && !indexes.isEmpty()) { + int idx = indexes.get(0); + indexes.remove(0); + result[i] = idx; + } else + result[i] = -1; + } + + // for odd tad numbers, we add special random clause for last element + if (length % 2 != 0) { + int rndClause = odds.get(rng.nextInt(odds.size())); + long tmp = result[rndClause]; + result[rndClause] = result[result.length - 1]; + result[result.length - 1] = tmp; + } + + + return result; + } + + protected static void swap(List objects, int idxA, int idxB) { + T tmpA = objects.get(idxA); + T tmpB = objects.get(idxB); + objects.set(idxA, tmpB); + objects.set(idxB, tmpA); + } + + public static void shuffleWithMap(List objects, int[] map) { + for (int i = 0; i < map.length; i++) { + if (map[i] >= 0) { + swap(objects, i, map[i]); + } + } + } + + public static int argMinOfMax(int[] first, int[] second) { + int minIdx = 0; + int maxAtMinIdx = Math.max(first[0], second[0]); + for (int i = 1; i < first.length; i++) { + int maxAtIndex = Math.max(first[i], second[i]); + if (maxAtMinIdx > maxAtIndex) { + maxAtMinIdx = maxAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static long argMinOfMax(long[] first, long[] second) { + long minIdx = 0; + long maxAtMinIdx = Math.max(first[0], second[0]); + for (int i = 1; i < first.length; i++) { + long maxAtIndex = Math.max(first[i], second[i]); + if (maxAtMinIdx > maxAtIndex) { + maxAtMinIdx = maxAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static int argMinOfMax(int[]... arrays) { + int minIdx = 0; + int maxAtMinIdx = Integer.MAX_VALUE; + + for (int i = 0; i < arrays[0].length; i++) { + int maxAtIndex = Integer.MIN_VALUE; + for (int j = 0; j < arrays.length; j++) { + maxAtIndex = Math.max(maxAtIndex, arrays[j][i]); + } + + if (maxAtMinIdx > maxAtIndex) { + maxAtMinIdx = maxAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static long argMinOfMax(long[]... arrays) { + int minIdx = 0; + long maxAtMinIdx = Long.MAX_VALUE; + + for (int i = 0; i < arrays[0].length; i++) { + long maxAtIndex = Long.MIN_VALUE; + for (int j = 0; j < arrays.length; j++) { + maxAtIndex = Math.max(maxAtIndex, arrays[j][i]); + } + + if (maxAtMinIdx > maxAtIndex) { + maxAtMinIdx = maxAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static int argMinOfSum(int[] first, int[] second) { + int minIdx = 0; + int sumAtMinIdx = first[0] + second[0]; + for (int i = 1; i < first.length; i++) { + int sumAtIndex = first[i] + second[i]; + if (sumAtMinIdx > sumAtIndex) { + sumAtMinIdx = sumAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static > Map sortMapByValue(Map map) { + List> list = new LinkedList<>(map.entrySet()); + Collections.sort(list, new Comparator>() { + @Override + public int compare(Map.Entry o1, Map.Entry o2) { + return (o1.getValue()).compareTo(o2.getValue()); + } + }); + + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : list) { + result.put(entry.getKey(), entry.getValue()); + } + return result; + } + + + public static T getRandomElement(List list) { + if (list.isEmpty()) + return null; + + return list.get(RandomUtils.nextInt(0, list.size())); + } + + /** + * Convert an int + * @param bool + * @return + */ + public static int fromBoolean(boolean bool) { + return bool ? 1 : 0; + } + + public static long[] toPrimitives(Long[] array) { + val res = new long[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static int[] toPrimitives(Integer[] array) { + val res = new int[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static short[] toPrimitives(Short[] array) { + val res = new short[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static byte[] toPrimitives(Byte[] array) { + val res = new byte[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static float[] toPrimitives(Float[] array) { + val res = new float[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static double[] toPrimitives(Double[] array) { + val res = new double[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static boolean[] toPrimitives(Boolean[] array) { + val res = new boolean[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static long[][] toPrimitives(Long[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new long[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static int[][] toPrimitives(Integer[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new int[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static short[][] toPrimitives(Short[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new short[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static byte[][] toPrimitives(Byte[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new byte[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static double[][] toPrimitives(Double[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new double[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static float[][] toPrimitives(Float[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new float[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static boolean [][] toPrimitives(Boolean[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new boolean[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static long[][][] toPrimitives(Long[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new long[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static int[][][] toPrimitives(Integer[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new int[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static short[][][] toPrimitives(Short[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new short[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static byte[][][] toPrimitives(Byte[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new byte[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static double[][][] toPrimitives(Double[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new double[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static float[][][] toPrimitives(Float[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new float[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static boolean[][][] toPrimitives(Boolean[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new boolean[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static long[][][][] toPrimitives(Long[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new long[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static int[][][][] toPrimitives(Integer[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new int[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static short[][][][] toPrimitives(Short[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new short[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static byte[][][][] toPrimitives(Byte[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new byte[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static double[][][][] toPrimitives(Double[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new double[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static float[][][][] toPrimitives(Float[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new float[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static boolean[][][][] toPrimitives(Boolean[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new boolean[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + + /** + * Assert that the specified array is not ragged (i.e., is rectangular).
+ * Can be used to check Object arrays with any number of dimensions (up to rank 4), or primitive arrays with rank 2 or higher
+ * An IllegalStateException is thrown if the array is ragged + * + * @param array Array to check + */ + public static void assertNotRagged(T[] array){ + Class c = array.getClass().getComponentType(); + int[] arrayShape = ArrayUtil.arrayShape(array, true); + int rank = arrayShape.length; + + if(rank == 1){ + //Rank 1 cannot be ragged + return; + } + + if(rank >= 2){ + for( int i=1; i= 3){ + + for( int i=0; i= 4){ + for( int i=0; i + * Example: if input is [2, 0, 1] then output is [1, 2, 0]
+ * The idea is that x.permute(input).permute(invertPermutation(input)) == x + * + * @param input 1D indices for permutation + * @return 1D inverted permutation + */ + public static int[] invertPermutation(int... input){ + int[] target = new int[input.length]; + + for(int i = 0 ; i < input.length ; i++){ + target[input[i]] = i; + } + + return target; + } + + /** + * @see #invertPermutation(int...) + * + * @param input 1D indices for permutation + * @return 1D inverted permutation + */ + public static long[] invertPermutation(long... input){ + long[] target = new long[input.length]; + + for(int i = 0 ; i < input.length ; i++){ + target[(int) input[i]] = i; + } + + return target; + } + + /** + * Is this shape an empty shape? + * Shape is considered to be an empty shape if it contains any zeros. + * Note: a length 0 shape is NOT considered empty (it's rank 0 scalar) + * @param shape Shape to check + * @return True if shape contains zeros + */ + public static boolean isEmptyShape(long[] shape){ + for( long l : shape){ + if(l == 0) + return true; + } + return false; + } + + /** + * Is this shape an empty shape? + * Shape is considered to be an empty shape if it contains any zeros. + * Note: a length 0 shape is NOT considered empty (it's rank 0 scalar) + * @param shape Shape to check + * @return True if shape contains zeros + */ + public static boolean isEmptyShape(int[] shape){ + for( int i : shape){ + if(i == 0) + return true; + } + return false; + } + + public static T[] filterNull(T... in){ + int count = 0; + for( int i=0; i implements Table { + private Table wrapped; + + public SynchronizedTable(Table wrapped) { + this.wrapped = wrapped; + } + + @Override + public synchronized boolean contains(Object rowKey, Object columnKey) { + return wrapped.contains(rowKey, columnKey); + } + + @Override + public synchronized boolean containsRow(Object rowKey) { + return wrapped.containsRow(rowKey); + } + + @Override + public synchronized boolean containsColumn(Object columnKey) { + return wrapped.containsColumn(columnKey); + } + + @Override + public synchronized boolean containsValue(Object value) { + return wrapped.containsValue(value); + } + + @Override + public synchronized V get(Object rowKey, Object columnKey) { + return wrapped.get(rowKey, columnKey); + } + + @Override + public synchronized boolean isEmpty() { + return wrapped.isEmpty(); + } + + @Override + public int size() { + return wrapped.size(); + } + + @Override + public synchronized void clear() { + wrapped.clear(); + } + + @Override + public synchronized V put(R rowKey, C columnKey, V value) { + return wrapped.put(rowKey, columnKey, value); + } + + @Override + public synchronized void putAll(Table table) { + wrapped.putAll(table); + } + + @Override + public synchronized V remove(Object rowKey, Object columnKey) { + return wrapped.remove(rowKey, columnKey); + } + + @Override + public synchronized Map row(R rowKey) { + return wrapped.row(rowKey); + } + + @Override + public synchronized Map column(C columnKey) { + return wrapped.column(columnKey); + } + + @Override + public synchronized Set> cellSet() { + return wrapped.cellSet(); + } + + @Override + public synchronized Set rowKeySet() { + return wrapped.rowKeySet(); + } + + @Override + public synchronized Set columnKeySet() { + return wrapped.columnKeySet(); + } + + @Override + public synchronized Collection values() { + return wrapped.values(); + } + + @Override + public synchronized Map> rowMap() { + return wrapped.rowMap(); + } + + @Override + public synchronized Map> columnMap() { + return wrapped.columnMap(); + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/util/ThreadUtils.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/ThreadUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/util/ThreadUtils.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/util/ThreadUtils.java diff --git a/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/validation/Nd4jCommonValidator.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/validation/Nd4jCommonValidator.java new file mode 100644 index 000000000..94da93db1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/validation/Nd4jCommonValidator.java @@ -0,0 +1,292 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.validation; + +import lombok.NonNull; +import org.apache.commons.io.FileUtils; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class Nd4jCommonValidator { + + private Nd4jCommonValidator() { + } + + /** + * Validate whether the specified file is a valid file (must exist and be non-empty) + * + * @param f File to check + * @return Result of validation + */ + public static ValidationResult isValidFile(@NonNull File f) { + ValidationResult vr = isValidFile(f, "File", false); + if (vr != null) + return vr; + return ValidationResult.builder() + .valid(true) + .formatType("File") + .path(getPath(f)) + .build(); + } + + /** + * Validate whether the specified file is a valid file + * + * @param f File to check + * @param formatType Name of the file format to include in validation results + * @param allowEmpty If true: allow empty files to pass. False: empty files will fail validation + * @return Result of validation + */ + public static ValidationResult isValidFile(@NonNull File f, String formatType, boolean allowEmpty) { + String path; + try { + path = f.getAbsolutePath(); //Very occasionally: getAbsolutePath not possible (files in JARs etc) + } catch (Throwable t) { + path = f.getPath(); + } + + if (f.exists() && !f.isFile()) { + return ValidationResult.builder() + .valid(false) + .formatType(formatType) + .path(path) + .issues(Collections.singletonList(f.isDirectory() ? "Specified path is a directory" : "Specified path is not a file")) + .build(); + } + + if (!f.exists() || !f.isFile()) { + return ValidationResult.builder() + .valid(false) + .formatType(formatType) + .path(path) + .issues(Collections.singletonList("File does not exist")) + .build(); + } + + if (!allowEmpty && f.length() <= 0) { + return ValidationResult.builder() + .valid(false) + .formatType(formatType) + .path(path) + .issues(Collections.singletonList("File is empty (length 0)")) + .build(); + } + + return null; //OK + } + + public static ValidationResult isValidJsonUTF8(@NonNull File f) { + return isValidJson(f, StandardCharsets.UTF_8); + } + + /** + * Validate whether the specified file is a valid JSON file. Note that this does not match the JSON content against a specific schema + * + * @param f File to check + * @param charset Character set for file + * @return Result of validation + */ + public static ValidationResult isValidJson(@NonNull File f, Charset charset) { + + ValidationResult vr = isValidFile(f, "JSON", false); + if (vr != null) + return vr; + + String content; + try { + content = FileUtils.readFileToString(f, charset); + } catch (IOException e) { + return ValidationResult.builder() + .valid(false) + .formatType("JSON") + .path(getPath(f)) + .issues(Collections.singletonList("Unable to read file (IOException)")) + .exception(e) + .build(); + } + + + return isValidJson(content, f); + } + + /** + * Validate whether the specified String is valid JSON. Note that this does not match the JSON content against a specific schema + * + * @param s JSON String to check + * @return Result of validation + */ + public static ValidationResult isValidJSON(String s) { + return isValidJson(s, null); + } + + + protected static ValidationResult isValidJson(String content, File f) { + try { + ObjectMapper om = new ObjectMapper(); + JavaType javaType = om.getTypeFactory().constructMapType(Map.class, String.class, Object.class); + om.readValue(content, javaType); //Don't care about result, just that it can be parsed successfully + } catch (Throwable t) { + //Jackson should tell us specifically where error occurred also + return ValidationResult.builder() + .valid(false) + .formatType("JSON") + .path(getPath(f)) + .issues(Collections.singletonList("File does not appear to be valid JSON")) + .exception(t) + .build(); + } + + + return ValidationResult.builder() + .valid(true) + .formatType("JSON") + .path(getPath(f)) + .build(); + } + + + /** + * Validate whether the specified file is a valid Zip file + * + * @param f File to check + * @param allowEmpty If true: allow empty zip files to pass validation. False: empty zip files will fail validation. + * @return Result of validation + */ + public static ValidationResult isValidZipFile(@NonNull File f, boolean allowEmpty) { + return isValidZipFile(f, allowEmpty, (List) null); + } + + /** + * Validate whether the specified file is a valid Zip file + * + * @param f File to check + * @param allowEmpty If true: allow empty zip files to pass validation. False: empty zip files will fail validation. + * @return Result of validation + */ + public static ValidationResult isValidZipFile(@NonNull File f, boolean allowEmpty, String... requiredEntries) { + return isValidZipFile(f, allowEmpty, requiredEntries == null ? null : Arrays.asList(requiredEntries)); + } + + /** + * Validate whether the specified file is a valid Zip file, and contains all of the required entries + * + * @param f File to check + * @param allowEmpty If true: allow empty zip files to pass validation. False: empty zip files will fail validation. + * @param requiredEntries If non-null, all of the specified entries must be present for the file to pass validation + * @return Result of validation + */ + public static ValidationResult isValidZipFile(@NonNull File f, boolean allowEmpty, List requiredEntries) { + ValidationResult vr = isValidFile(f, "Zip File", false); + if (vr != null) + return vr; + + ZipFile zf; + try { + zf = new ZipFile(f); + } catch (Throwable e) { + return ValidationResult.builder() + .valid(false) + .formatType("Zip File") + .path(getPath(f)) + .issues(Collections.singletonList("File does not appear to be valid zip file (not a zip file or content is corrupt)")) + .exception(e) + .build(); + } + + try { + int numEntries = zf.size(); + if (!allowEmpty && numEntries <= 0) { + return ValidationResult.builder() + .valid(false) + .formatType("Zip File") + .path(getPath(f)) + .issues(Collections.singletonList("Zip file is empty")) + .build(); + } + + if (requiredEntries != null && !requiredEntries.isEmpty()) { + List missing = null; + for (String s : requiredEntries) { + ZipEntry ze = zf.getEntry(s); + if (ze == null) { + if (missing == null) + missing = new ArrayList<>(); + missing.add(s); + } + } + + if (missing != null) { + String s = "Zip file is missing " + missing.size() + " of " + requiredEntries.size() + " required entries: " + missing; + return ValidationResult.builder() + .valid(false) + .formatType("Zip File") + .path(getPath(f)) + .issues(Collections.singletonList(s)) + .build(); + } + } + + } catch (Throwable t) { + return ValidationResult.builder() + .valid(false) + .formatType("Zip File") + .path(getPath(f)) + .issues(Collections.singletonList("Error reading zip file")) + .exception(t) + .build(); + } finally { + try { + zf.close(); + } catch (IOException e) { + } //Ignore, can't do anything about it... + } + + return ValidationResult.builder() + .valid(true) + .formatType("Zip File") + .path(getPath(f)) + .build(); + } + + + /** + * Null-safe and "no absolute path exists" safe method for getting the path of a file for validation purposes + */ + public static String getPath(File f) { + if (f == null) + return null; + try { + return f.getAbsolutePath(); //Very occasionally: getAbsolutePath not possible (files in JARs etc) + } catch (Throwable t) { + return f.getPath(); + } + } + + +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/validation/ValidationResult.java b/cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/validation/ValidationResult.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/validation/ValidationResult.java rename to cavis-dnn/cavis-dnn-common/src/main/java/org/nd4j/common/validation/ValidationResult.java diff --git a/nd4j/nd4j-common/src/test/java/org/nd4j/common/base/TestPreconditions.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/base/TestPreconditions.java similarity index 100% rename from nd4j/nd4j-common/src/test/java/org/nd4j/common/base/TestPreconditions.java rename to cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/base/TestPreconditions.java diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/function/FunctionalUtilsTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/function/FunctionalUtilsTest.java new file mode 100644 index 000000000..88a4ba98d --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/function/FunctionalUtilsTest.java @@ -0,0 +1,81 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +import org.junit.jupiter.api.Test; +import org.nd4j.common.primitives.Pair; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class FunctionalUtilsTest { + + + @Test + public void testCoGroup() { + List> leftMap = new ArrayList<>(); + List> rightMap = new ArrayList<>(); + + leftMap.add(Pair.of("cat","adam")); + leftMap.add(Pair.of("dog","adam")); + + rightMap.add(Pair.of("fish","alex")); + rightMap.add(Pair.of("cat","alice")); + rightMap.add(Pair.of("dog","steve")); + + //[(fish,([],[alex])), (dog,([adam],[steve])), (cat,([adam],[alice]))] + Map,List>> assertion = new HashMap<>(); + assertion.put("cat",Pair.of(Arrays.asList("adam"),Arrays.asList("alice"))); + assertion.put("dog",Pair.of(Arrays.asList("adam"),Arrays.asList("steve"))); + assertion.put("fish",Pair.of(Collections.emptyList(),Arrays.asList("alex"))); + + Map, List>> cogroup = FunctionalUtils.cogroup(leftMap, rightMap); + assertEquals(assertion,cogroup); + + } + + @Test + public void testGroupBy() { + List> list = new ArrayList<>(); + for(int i = 0; i < 10; i++) { + for(int j = 0; j < 5; j++) { + list.add(Pair.of(i, j)); + } + } + + Map> integerIterableMap = FunctionalUtils.groupByKey(list); + assertEquals(10,integerIterableMap.keySet().size()); + assertEquals(5,integerIterableMap.get(0).size()); + } + + @Test + public void testMapToPair() { + Map map = new HashMap<>(); + for(int i = 0; i < 5; i++) { + map.put(String.valueOf(i),String.valueOf(i)); + } + + List> pairs = FunctionalUtils.mapToPair(map); + assertEquals(map.size(),pairs.size()); + } + +} diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java new file mode 100644 index 000000000..8e7a22e6d --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java @@ -0,0 +1,51 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ClassPathResourceTest { + + + + @Test + public void testDirExtractingIntelliJ() throws Exception { + //https://github.com/deeplearning4j/deeplearning4j/issues/6483 + + ClassPathResource cpr = new ClassPathResource("somedir"); + + File f = new File(FileUtils.getTempDirectoryPath()+File.separatorChar+ UUID.randomUUID().toString()); + FileUtils.forceMkdir(f); + cpr.copyDirectory(f); + + File[] files = f.listFiles(); + assertEquals(1, files.length); + assertEquals("afile.txt", files[0].getName()); + } + +} diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/loader/TestFileBatch.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/loader/TestFileBatch.java new file mode 100644 index 000000000..bd3f9c569 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/loader/TestFileBatch.java @@ -0,0 +1,105 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.loader; + +import org.apache.commons.io.FileUtils; + +import org.junit.jupiter.api.Test; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestFileBatch { + + + @Test + public void testFileBatch() throws Exception { + File baseDir = FileUtils.getTempDirectory(); + + List fileList = new ArrayList<>(); + for( int i=0; i<10; i++ ){ + String s = "File contents - file " + i; + File f = new File(baseDir, "origFile" + i + ".txt"); + FileUtils.writeStringToFile(f, s, StandardCharsets.UTF_8); + fileList.add(f); + } + + FileBatch fb = FileBatch.forFiles(fileList); + + assertEquals(10, fb.getFileBytes().size()); + assertEquals(10, fb.getOriginalUris().size()); + for( int i=0; i<10; i++ ){ + byte[] expBytes = ("File contents - file " + i).getBytes(StandardCharsets.UTF_8); + byte[] actBytes = fb.getFileBytes().get(i); + assertArrayEquals(expBytes, actBytes); + + String expPath = fileList.get(i).toURI().toString(); + String actPath = fb.getOriginalUris().get(i); + assertEquals(expPath, actPath); + } + + //Save and load: + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + fb.writeAsZip(baos); + byte[] asBytes = baos.toByteArray(); + + FileBatch fb2; + try(ByteArrayInputStream bais = new ByteArrayInputStream(asBytes)){ + fb2 = FileBatch.readFromZip(bais); + } + + assertEquals(fb.getOriginalUris(), fb2.getOriginalUris()); + assertEquals(10, fb2.getFileBytes().size()); + for( int i=0; i<10; i++ ){ + assertArrayEquals(fb.getFileBytes().get(i), fb2.getFileBytes().get(i)); + } + + //Check that it is indeed a valid zip file: + + File f = new File(FileUtils.getTempDirectoryPath()+"/"+UUID.randomUUID().toString()); + f.delete(); + fb.writeAsZip(f); + + ZipFile zf = new ZipFile(f); + Enumeration e = zf.entries(); + int count = 0; + Set names = new HashSet<>(); + while(e.hasMoreElements()){ + ZipEntry entry = e.nextElement(); + names.add(entry.getName()); + } + + assertEquals(11, names.size()); //10 files, 1 "original file names" file + assertTrue(names.contains(FileBatch.ORIGINAL_PATHS_FILENAME)); + for( int i=0; i<10; i++ ){ + String n = "file_" + i + ".txt"; + assertTrue(names.contains(n), n); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/AtomicTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/AtomicTest.java new file mode 100644 index 000000000..660a55da2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/AtomicTest.java @@ -0,0 +1,75 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.val; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.util.SerializationUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + + + +public class AtomicTest { + + @Test + public void testEquality_1() { + val v0 = new Atomic(1327541); + val v1 = new Atomic(1327541); + val v3 = new Atomic(1327542); + + Assertions.assertEquals(v0, v1); + Assertions.assertNotEquals(v0, v3); + } + + @Test + public void testSerialization_1() throws Exception { + val v0 = new Atomic(1327541); + + try (val baos = new ByteArrayOutputStream()) { + SerializationUtils.serialize(v0, baos); + + try (val bais = new ByteArrayInputStream(baos.toByteArray())) { + Atomic v1 = SerializationUtils.deserialize(bais); + + Assertions.assertEquals(v1, v0); + } + } + } + + @Test + public void testCas_1() throws Exception { + val v0 = new Atomic(); + + v0.cas(null, "alpha"); + Assertions.assertEquals("alpha", v0.get()); + } + + @Test + public void testCas_2() throws Exception { + val v0 = new Atomic("beta"); + + v0.cas(null, "alpha"); + Assertions.assertEquals("beta", v0.get()); + } +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/CounterMapTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/CounterMapTest.java new file mode 100644 index 000000000..674e3ccdc --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/CounterMapTest.java @@ -0,0 +1,109 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import org.junit.jupiter.api.Test; + +import java.util.Iterator; + +import static org.junit.jupiter.api.Assertions.*; + +public class CounterMapTest { + + @Test + public void testIterator() { + CounterMap counterMap = new CounterMap<>(); + + counterMap.incrementCount(0, 0, 1); + counterMap.incrementCount(0, 1, 1); + counterMap.incrementCount(0, 2, 1); + counterMap.incrementCount(1, 0, 1); + counterMap.incrementCount(1, 1, 1); + counterMap.incrementCount(1, 2, 1); + + Iterator> iterator = counterMap.getIterator(); + + Pair pair = iterator.next(); + + assertEquals(0, pair.getFirst().intValue()); + assertEquals(0, pair.getSecond().intValue()); + + pair = iterator.next(); + + assertEquals(0, pair.getFirst().intValue()); + assertEquals(1, pair.getSecond().intValue()); + + pair = iterator.next(); + + assertEquals(0, pair.getFirst().intValue()); + assertEquals(2, pair.getSecond().intValue()); + + pair = iterator.next(); + + assertEquals(1, pair.getFirst().intValue()); + assertEquals(0, pair.getSecond().intValue()); + + pair = iterator.next(); + + assertEquals(1, pair.getFirst().intValue()); + assertEquals(1, pair.getSecond().intValue()); + + pair = iterator.next(); + + assertEquals(1, pair.getFirst().intValue()); + assertEquals(2, pair.getSecond().intValue()); + + + assertFalse(iterator.hasNext()); + } + + + @Test + public void testIncrementAll() { + CounterMap counterMapA = new CounterMap<>(); + + counterMapA.incrementCount(0, 0, 1); + counterMapA.incrementCount(0, 1, 1); + counterMapA.incrementCount(0, 2, 1); + counterMapA.incrementCount(1, 0, 1); + counterMapA.incrementCount(1, 1, 1); + counterMapA.incrementCount(1, 2, 1); + + CounterMap counterMapB = new CounterMap<>(); + + counterMapB.incrementCount(1, 1, 1); + counterMapB.incrementCount(2, 1, 1); + + counterMapA.incrementAll(counterMapB); + + assertEquals(2.0, counterMapA.getCount(1,1), 1e-5); + assertEquals(1.0, counterMapA.getCount(2,1), 1e-5); + assertEquals(1.0, counterMapA.getCount(0,0), 1e-5); + + + assertEquals(7, counterMapA.totalSize()); + + + counterMapA.setCount(2, 1, 17); + + assertEquals(17.0, counterMapA.getCount(2, 1), 1e-5); + } +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/CounterTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/CounterTest.java new file mode 100644 index 000000000..4e2185588 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/primitives/CounterTest.java @@ -0,0 +1,130 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +@Slf4j +public class CounterTest { + + @Test + public void testCounterIncrementAll1() { + Counter counterA = new Counter<>(); + + counterA.incrementCount("A", 1); + counterA.incrementCount("A", 1); + counterA.incrementCount("A", 1); + + + + Counter counterB = new Counter<>(); + counterB.incrementCount("B", 2); + counterB.incrementCount("B", 2); + + Assertions.assertEquals(3.0, counterA.getCount("A"), 1e-5); + Assertions.assertEquals(4.0, counterB.getCount("B"), 1e-5); + + counterA.incrementAll(counterB); + + Assertions.assertEquals(3.0, counterA.getCount("A"), 1e-5); + Assertions.assertEquals(4.0, counterA.getCount("B"), 1e-5); + + counterA.setCount("B", 234); + + Assertions.assertEquals(234.0, counterA.getCount("B"), 1e-5); + } + + + + @Test + public void testCounterTopN1() { + Counter counterA = new Counter<>(); + + counterA.incrementCount("A", 1); + counterA.incrementCount("B", 2); + counterA.incrementCount("C", 3); + counterA.incrementCount("D", 4); + counterA.incrementCount("E", 5); + + counterA.keepTopNElements(4); + + Assertions.assertEquals(4,counterA.size()); + + // we expect element A to be gone + Assertions.assertEquals(0.0, counterA.getCount("A"), 1e-5); + Assertions.assertEquals(2.0, counterA.getCount("B"), 1e-5); + Assertions.assertEquals(3.0, counterA.getCount("C"), 1e-5); + Assertions.assertEquals(4.0, counterA.getCount("D"), 1e-5); + Assertions.assertEquals(5.0, counterA.getCount("E"), 1e-5); + } + + @Test + public void testKeysSorted1() throws Exception { + Counter counterA = new Counter<>(); + + counterA.incrementCount("A", 1); + counterA.incrementCount("B", 2); + counterA.incrementCount("C", 3); + counterA.incrementCount("D", 4); + counterA.incrementCount("E", 5); + + Assertions.assertEquals("E", counterA.argMax()); + + List list = counterA.keySetSorted(); + + Assertions.assertEquals(5, list.size()); + + Assertions.assertEquals("E", list.get(0)); + Assertions.assertEquals("D", list.get(1)); + Assertions.assertEquals("C", list.get(2)); + Assertions.assertEquals("B", list.get(3)); + Assertions.assertEquals("A", list.get(4)); + } + + @Test + public void testCounterTotal() { + Counter counter = new Counter<>(); + + counter.incrementCount("A", 1); + counter.incrementCount("B", 1); + counter.incrementCount("C", 1); + + Assertions.assertEquals(3.0, counter.totalCount(), 1e-5); + + counter.setCount("B", 234); + + Assertions.assertEquals(236.0, counter.totalCount(), 1e-5); + + counter.setCount("D", 1); + + Assertions.assertEquals(237.0, counter.totalCount(), 1e-5); + + counter.removeKey("B"); + + Assertions.assertEquals(3.0, counter.totalCount(), 1e-5); + + } + +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/resources/TestArchiveUtils.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/resources/TestArchiveUtils.java new file mode 100644 index 000000000..83ab40816 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/resources/TestArchiveUtils.java @@ -0,0 +1,68 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import org.nd4j.common.util.ArchiveUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class TestArchiveUtils { + + + @Test + public void testUnzipFileTo() throws IOException { + //random txt file + File dir = FileUtils.getTempDirectory(); + String content = "test file content"; + String path = "myDir/myTestFile.txt"; + File testFile = new File(dir, path); + testFile.getParentFile().mkdir(); + FileUtils.writeStringToFile(testFile, content, StandardCharsets.UTF_8); + + //zip it as test.zip + File zipFile = new File(testFile.getParentFile(),"test.zip"); + FileOutputStream fos = new FileOutputStream(zipFile); + ZipOutputStream zipOut = new ZipOutputStream(fos); + FileInputStream fis = new FileInputStream(testFile); + ZipEntry zipEntry = new ZipEntry(testFile.getName()); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + zipOut.close(); + fis.close(); + fos.close(); + + //now unzip to a directory that doesn't previously exist + File unzipDir = new File(testFile.getParentFile(),"unzipTo"); + ArchiveUtils.unzipFileTo(zipFile.getAbsolutePath(),unzipDir.getAbsolutePath()); + } +} diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/resources/TestStrumpf.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/resources/TestStrumpf.java new file mode 100644 index 000000000..0ba67d4a8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/resources/TestStrumpf.java @@ -0,0 +1,101 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.LineIterator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.config.ND4JSystemProperties; +import org.nd4j.common.resources.strumpf.StrumpfResolver; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; + +public class TestStrumpf { + + public File testDir = FileUtils.getTempDirectory(); + + @Test + public void testResolvingReference() throws Exception { + + File f = Resources.asFile("big/raw_sentences.txt"); + Assertions.assertTrue(f.exists()); + + System.out.println(f.getAbsolutePath()); + try(Reader r = new BufferedReader(new FileReader(f))){ + LineIterator iter = IOUtils.lineIterator(r); + for( int i=0; i<5 && iter.hasNext(); i++ ){ + System.out.println("LINE " + i + ": " + iter.next()); + } + } + } + + @Test + public void testResolvingActual() throws Exception { + File f = Resources.asFile("data/irisSvmLight.txt"); + Assertions.assertTrue(f.exists()); + + //System.out.println(f.getAbsolutePath()); + int count = 0; + try(Reader r = new BufferedReader(new FileReader(f))){ + LineIterator iter = IOUtils.lineIterator(r); + while(iter.hasNext()){ + String line = iter.next(); + //System.out.println("LINE " + i + ": " + line); + count++; + } + } + + Assertions.assertEquals(12, count); //Iris normally has 150 examples; this is subset with 12 + } + + @Test + public void testResolveLocal() throws Exception { + + File dir = testDir; + + String content = "test file content"; + String path = "myDir/myTestFile.txt"; + File testFile = new File(dir, path); + testFile.getParentFile().mkdir(); + FileUtils.writeStringToFile(testFile, content, StandardCharsets.UTF_8); + + System.setProperty(ND4JSystemProperties.RESOURCES_LOCAL_DIRS, dir.getAbsolutePath()); + + try{ + StrumpfResolver r = new StrumpfResolver(); + Assertions.assertTrue(r.exists(path)); + File f = r.asFile(path); + Assertions.assertTrue(f.exists()); + Assertions.assertEquals(testFile.getAbsolutePath(), f.getAbsolutePath()); + String s = FileUtils.readFileToString(f, StandardCharsets.UTF_8); + Assertions.assertEquals(content, s); + } finally { + System.setProperty(ND4JSystemProperties.RESOURCES_LOCAL_DIRS, ""); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/BToolsTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/BToolsTest.java new file mode 100644 index 000000000..a539a3592 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/BToolsTest.java @@ -0,0 +1,138 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class BToolsTest { + // + + @Test + public void testgetMtLvESS() throws Exception { + // + assertEquals( "?", BTools.getMtLvESS( -5 ) ); + assertEquals( "", BTools.getMtLvESS( 0 ) ); + assertEquals( "...", BTools.getMtLvESS( 3 ) ); + // + } + + @Test + public void testgetMtLvISS() throws Exception { + // + assertEquals( " ", BTools.getMtLvISS() ); + // + } + + @Test + public void testgetSpaces() throws Exception { + // + assertEquals( "?", BTools.getSpaces( -3 ) ); + assertEquals( "", BTools.getSpaces( 0 ) ); + assertEquals( " ", BTools.getSpaces( 4 ) ); + // + } + + @Test + public void testgetSBln() throws Exception { + // + assertEquals( "?", BTools.getSBln() ); + assertEquals( "?", BTools.getSBln( null ) ); + assertEquals( "T", BTools.getSBln( true ) ); + assertEquals( "F", BTools.getSBln( false ) ); + assertEquals( "TFFT", BTools.getSBln( true, false, false, true ) ); + assertEquals( "FTFFT", BTools.getSBln( false, true, false, false, true ) ); + // + } + + @Test + public void testgetSDbl() throws Exception { + // + assertEquals( "NaN", BTools.getSDbl( Double.NaN, 0 ) ); + assertEquals( "-6", BTools.getSDbl( -5.5D, 0 ) ); + assertEquals( "-5.50", BTools.getSDbl( -5.5D, 2 ) ); + assertEquals( "-5.30", BTools.getSDbl( -5.3D, 2 ) ); + assertEquals( "-5", BTools.getSDbl( -5.3D, 0 ) ); + assertEquals( "0.00", BTools.getSDbl( 0D, 2 ) ); + assertEquals( "0", BTools.getSDbl( 0D, 0 ) ); + assertEquals( "0.30", BTools.getSDbl( 0.3D, 2 ) ); + assertEquals( "4.50", BTools.getSDbl( 4.5D, 2 ) ); + assertEquals( "4", BTools.getSDbl( 4.5D, 0 ) ); + assertEquals( "6", BTools.getSDbl( 5.5D, 0 ) ); + assertEquals( "12 345 678", BTools.getSDbl( 12345678D, 0 ) ); + // + assertEquals( "-456", BTools.getSDbl( -456D, 0, false ) ); + assertEquals( "-456", BTools.getSDbl( -456D, 0, true ) ); + assertEquals( "+456", BTools.getSDbl( 456D, 0, true ) ); + assertEquals( "456", BTools.getSDbl( 456D, 0, false ) ); + assertEquals( " 0", BTools.getSDbl( 0D, 0, true ) ); + assertEquals( "0", BTools.getSDbl( 0D, 0, false ) ); + // + assertEquals( " 4.50", BTools.getSDbl( 4.5D, 2, false, 6 ) ); + assertEquals( " +4.50", BTools.getSDbl( 4.5D, 2, true, 6 ) ); + assertEquals( " +456", BTools.getSDbl( 456D, 0, true, 7 ) ); + assertEquals( " 456", BTools.getSDbl( 456D, 0, false, 7 ) ); + // + } + + @Test + public void testgetSInt() throws Exception { + // + assertEquals( "23", BTools.getSInt( 23, 1 ) ); + assertEquals( "23", BTools.getSInt( 23, 2 ) ); + assertEquals( " 23", BTools.getSInt( 23, 3 ) ); + // + assertEquals( "0000056", BTools.getSInt( 56, 7, '0' ) ); + // + } + + @Test + public void testgetSIntA() throws Exception { + // + assertEquals( "?", BTools.getSIntA( null ) ); + assertEquals( "?", BTools.getSIntA( ) ); + assertEquals( "0", BTools.getSIntA( 0 ) ); + assertEquals( "5, 6, 7", BTools.getSIntA( 5, 6, 7 ) ); + int[] intA = { 2, 3, 4, 5, 6 }; + assertEquals( "2, 3, 4, 5, 6", BTools.getSIntA( intA ) ); + // + } + + @Test + public void testgetIndexCharsCount() throws Exception { + // + assertEquals( 1, BTools.getIndexCharsCount( -5 ) ); + assertEquals( 1, BTools.getIndexCharsCount( 5 ) ); + assertEquals( 3, BTools.getIndexCharsCount( 345 ) ); + // + } + + @Test + public void testgetSLcDtTm() throws Exception { + // + assertEquals( 15, BTools.getSLcDtTm().length() ); + assertEquals( "LDTm: ", BTools.getSLcDtTm().substring( 0, 6 ) ); + // + } + + +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/InfoLineTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/InfoLineTest.java new file mode 100644 index 000000000..f758941c1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/InfoLineTest.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class InfoLineTest { + // + + @Test + public void testAll() throws Exception { + // + InfoValues iv0 = new InfoValues( " A", " B" ); + InfoValues iv1 = new InfoValues( " C", " D" ); + InfoValues iv2 = new InfoValues( " E", " F", " G", " H" ); + // + iv0.vsL.add( " ab " ); + iv1.vsL.add( " cd " ); + iv2.vsL.add( " ef " ); + // + InfoLine il = new InfoLine(); + // + il.ivL.add( iv0 ); + il.ivL.add( iv1 ); + il.ivL.add( iv2 ); + // + int mtLv = 2; + // + assertEquals( ".. | A | C | E |", il.getTitleLine( mtLv, 0 ) ); + assertEquals( ".. | B | D | F |", il.getTitleLine( mtLv, 1 ) ); + assertEquals( ".. | | | G |", il.getTitleLine( mtLv, 2 ) ); + assertEquals( ".. | | | H |", il.getTitleLine( mtLv, 3 ) ); + assertEquals( ".. | ab | cd | ef |", il.getValuesLine( mtLv ) ); + // + } + + + +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/InfoValuesTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/InfoValuesTest.java new file mode 100644 index 000000000..1dc5860d7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/InfoValuesTest.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class InfoValuesTest { + // + private String[] t1_titleA = { "T0", "T1", "T2", "T3", "T4", "T5" }; + // + private String[] t2_titleA = { "", "T1", "T2" }; + // + + @Test + public void testconstructor() throws Exception { + // + InfoValues iv; + // + iv = new InfoValues( t1_titleA ); + assertEquals( "T0", iv.titleA[ 0 ] ); + assertEquals( "T1", iv.titleA[ 1 ] ); + assertEquals( "T2", iv.titleA[ 2 ] ); + assertEquals( "T3", iv.titleA[ 3 ] ); + assertEquals( "T4", iv.titleA[ 4 ] ); + assertEquals( "T5", iv.titleA[ 5 ] ); + // + iv = new InfoValues( t2_titleA ); + assertEquals( "", iv.titleA[ 0 ] ); + assertEquals( "T1", iv.titleA[ 1 ] ); + assertEquals( "T2", iv.titleA[ 2 ] ); + assertEquals( "", iv.titleA[ 3 ] ); + assertEquals( "", iv.titleA[ 4 ] ); + assertEquals( "", iv.titleA[ 5 ] ); + // + } + + @Test + public void testgetValues() throws Exception { + // + InfoValues iv; + // + iv = new InfoValues( "Test" ); + iv.vsL.add( " AB " ); + iv.vsL.add( " CD " ); + // + assertEquals( " AB | CD |", iv.getValues() ); + // + } + + +} diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/PropertyParserTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/PropertyParserTest.java new file mode 100644 index 000000000..0d6caf2db --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/PropertyParserTest.java @@ -0,0 +1,1330 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import org.junit.jupiter.api.*; + +/** + * Tests for PropertyParser + * + * @author gagatust + */ +public class PropertyParserTest { + + public PropertyParserTest() { + } + + @BeforeAll + public static void setUpClass() { + } + + @AfterAll + public static void tearDownClass() { + } + + @BeforeEach + public void setUp() { + } + + @AfterEach + public void tearDown() { + } + + /** + * Test of getProperties method, of class PropertyParser. + */ + @Test + public void testGetProperties() { + + } + + /** + * Test of setProperties method, of class PropertyParser. + */ + @Test + public void testSetProperties() { + + } + + /** + * Test of parseString method, of class PropertyParser. + */ + @Test + public void testParseString() { + System.out.println("parseString"); + String expResult; + String result; + + Properties props = new Properties(); + props.put("value1", "sTr1"); + props.put("value2", "str_2"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = "sTr1"; + result = instance.parseString("value1"); + assertEquals(expResult, result); + + expResult = "str_2"; + result = instance.parseString("value2"); + assertEquals(expResult, result); + + expResult = ""; + result = instance.parseString("empty"); + assertEquals(expResult, result); + + expResult = "abc"; + result = instance.parseString("str"); + assertEquals(expResult, result); + + expResult = "true"; + result = instance.parseString("boolean"); + assertEquals(expResult, result); + + expResult = "24.98"; + result = instance.parseString("float"); + assertEquals(expResult, result); + + expResult = "12"; + result = instance.parseString("int"); + assertEquals(expResult, result); + + expResult = "a"; + result = instance.parseString("char"); + assertEquals(expResult, result); + + try { + instance.parseString("nonexistent"); + fail("no exception"); + } catch (NullPointerException e) { + } + } + + /** + * Test of parseInt method, of class PropertyParser. + */ + @Test + public void testParseInt() { + System.out.println("parseInt"); + int expResult; + int result; + + Properties props = new Properties(); + props.put("value1", "432"); + props.put("value2", "-242"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 432; + result = instance.parseInt("value1"); + assertEquals(expResult, result); + + expResult = -242; + result = instance.parseInt("value2"); + assertEquals(expResult, result); + + try { + instance.parseInt("empty"); + fail("no exception"); + } catch (NumberFormatException e) { + } + + try { + instance.parseInt("str"); + fail("no exception"); + } catch (NumberFormatException e) { + } + + try { + instance.parseInt("boolean"); + assertEquals(expResult, result); + fail("no exception"); + } catch (NumberFormatException e) { + } + + try { + instance.parseInt("float"); + fail("no exception"); + } catch (NumberFormatException e) { + } + + expResult = 12; + result = instance.parseInt("int"); + assertEquals(expResult, result); + + try { + instance.parseInt("char"); + fail("no exception"); + } catch (NumberFormatException e) { + } + + try { + expResult = 0; + result = instance.parseInt("nonexistent"); + fail("no exception"); + assertEquals(expResult, result); + } catch (IllegalArgumentException e) { + } + } + + /** + * Test of parseBoolean method, of class PropertyParser. + */ + @Test + public void testParseBoolean() { + System.out.println("parseBoolean"); + boolean expResult; + boolean result; + + Properties props = new Properties(); + props.put("value1", "true"); + props.put("value2", "false"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = true; + result = instance.parseBoolean("value1"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("value2"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("empty"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("str"); + assertEquals(expResult, result); + + expResult = true; + result = instance.parseBoolean("boolean"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("float"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("int"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("char"); + assertEquals(expResult, result); + + try { + expResult = false; + result = instance.parseBoolean("nonexistent"); + fail("no exception"); + assertEquals(expResult, result); + } catch (IllegalArgumentException e) { + } + } + + /** + * Test of parseDouble method, of class PropertyParser. + */ + @Test + public void testParseFloat() { + System.out.println("parseFloat"); + double expResult; + double result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789f; + result = instance.parseFloat("value1"); + assertEquals(expResult, result, 0); + + expResult = -9000.001f; + result = instance.parseFloat("value2"); + assertEquals(expResult, result, 0); + + try { + instance.parseFloat("empty"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseFloat("str"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseFloat("boolean"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + expResult = 24.98f; + result = instance.parseFloat("float"); + assertEquals(expResult, result, 0); + + expResult = 12f; + result = instance.parseFloat("int"); + assertEquals(expResult, result, 0); + + try { + instance.parseFloat("char"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseFloat("nonexistent"); + fail("no exception"); + } catch (NullPointerException e) { + } + } + + /** + * Test of parseDouble method, of class PropertyParser. + */ + @Test + public void testParseDouble() { + System.out.println("parseDouble"); + double expResult; + double result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789; + result = instance.parseDouble("value1"); + assertEquals(expResult, result, 0); + + expResult = -9000.001; + result = instance.parseDouble("value2"); + assertEquals(expResult, result, 0); + + try { + instance.parseDouble("empty"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseDouble("str"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseDouble("boolean"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + expResult = 24.98; + result = instance.parseDouble("float"); + assertEquals(expResult, result, 0); + + expResult = 12; + result = instance.parseDouble("int"); + assertEquals(expResult, result, 0); + + try { + instance.parseDouble("char"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseDouble("nonexistent"); + fail("no exception"); + } catch (NullPointerException e) { + } + } + + /** + * Test of parseLong method, of class PropertyParser. + */ + @Test + public void testParseLong() { + System.out.println("parseLong"); + long expResult; + long result; + + Properties props = new Properties(); + props.put("value1", "12345678900"); + props.put("value2", "-9000001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345678900L; + result = instance.parseLong("value1"); + assertEquals(expResult, result); + + expResult = -9000001L; + result = instance.parseLong("value2"); + assertEquals(expResult, result); + + try { + instance.parseLong("empty"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseLong("str"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseLong("boolean"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseLong("float"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + expResult = 12L; + result = instance.parseLong("int"); + assertEquals(expResult, result); + + try { + instance.parseLong("char"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseLong("nonexistent"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + } + + /** + * Test of parseChar method, of class PropertyParser. + */ + @Test + public void testParseChar() { + System.out.println("parseChar"); + char expResult; + char result; + + Properties props = new Properties(); + props.put("value1", "b"); + props.put("value2", "c"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 'b'; + result = instance.parseChar("value1"); + assertEquals(expResult, result); + + expResult = 'c'; + result = instance.parseChar("value2"); + assertEquals(expResult, result); + + try { + instance.parseChar("empty"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseChar("str"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseChar("boolean"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseChar("float"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseChar("int"); + } catch (IllegalArgumentException e) { + } + + expResult = 'a'; + result = instance.parseChar("char"); + assertEquals(expResult, result); + + try { + instance.parseChar("nonexistent"); + fail("no exception"); + assertEquals(expResult, result); + } catch (NullPointerException e) { + } + } + + /** + * Test of toString method, of class PropertyParser. + */ + @Test + public void testToString_String() { + System.out.println("toString"); + String expResult; + String result; + + Properties props = new Properties(); + props.put("value1", "sTr1"); + props.put("value2", "str_2"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = "sTr1"; + result = instance.toString("value1"); + assertEquals(expResult, result); + + expResult = "str_2"; + result = instance.toString("value2"); + assertEquals(expResult, result); + + expResult = ""; + result = instance.toString("empty"); + assertEquals(expResult, result); + + expResult = "abc"; + result = instance.toString("str"); + assertEquals(expResult, result); + + expResult = "true"; + result = instance.toString("boolean"); + assertEquals(expResult, result); + + expResult = "24.98"; + result = instance.toString("float"); + assertEquals(expResult, result); + + expResult = "12"; + result = instance.toString("int"); + assertEquals(expResult, result); + + expResult = "a"; + result = instance.toString("char"); + assertEquals(expResult, result); + + expResult = ""; + result = instance.toString("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toInt method, of class PropertyParser. + */ + @Test + public void testToInt_String() { + System.out.println("toInt"); + int expResult; + int result; + + Properties props = new Properties(); + props.put("value1", "123"); + props.put("value2", "-54"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 123; + result = instance.toInt("value1"); + assertEquals(expResult, result); + + expResult = -54; + result = instance.toInt("value2"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("empty"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("str"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("boolean"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("float"); + assertEquals(expResult, result); + + expResult = 12; + result = instance.toInt("int"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("char"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toBoolean method, of class PropertyParser. + */ + @Test + public void testToBoolean_String() { + System.out.println("toBoolean"); + boolean expResult; + boolean result; + + Properties props = new Properties(); + props.put("value1", "true"); + props.put("value2", "false"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = true; + result = instance.toBoolean("value1"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("value2"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("empty"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("str"); + assertEquals(expResult, result); + + expResult = true; + result = instance.toBoolean("boolean"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("float"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("int"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("char"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toDouble method, of class PropertyParser. + */ + @Test + public void testToFloat_String() { + System.out.println("toFloat"); + float expResult; + float result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789f; + result = instance.toFloat("value1"); + assertEquals(expResult, result, 0f); + + expResult = -9000.001f; + result = instance.toFloat("value2"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("empty"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("str"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("boolean"); + assertEquals(expResult, result, 0f); + + expResult = 24.98f; + result = instance.toFloat("float"); + assertEquals(expResult, result, 0f); + + expResult = 12f; + result = instance.toFloat("int"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("char"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("nonexistent"); + assertEquals(expResult, result, 0f); + } + + /** + * Test of toDouble method, of class PropertyParser. + */ + @Test + public void testToDouble_String() { + System.out.println("toDouble"); + double expResult; + double result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789; + result = instance.toDouble("value1"); + assertEquals(expResult, result, 0); + + expResult = -9000.001; + result = instance.toDouble("value2"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("empty"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("str"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("boolean"); + assertEquals(expResult, result, 0); + + expResult = 24.98; + result = instance.toDouble("float"); + assertEquals(expResult, result, 0); + + expResult = 12; + result = instance.toDouble("int"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("char"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("nonexistent"); + assertEquals(expResult, result, 0); + } + + /** + * Test of toLong method, of class PropertyParser. + */ + @Test + public void testToLong_String() { + System.out.println("toLong"); + long expResult; + long result; + + Properties props = new Properties(); + props.put("value1", "12345678900"); + props.put("value2", "-9000001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345678900L; + result = instance.toLong("value1"); + assertEquals(expResult, result); + + expResult = -9000001L; + result = instance.toLong("value2"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("empty"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("str"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("boolean"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("float"); + assertEquals(expResult, result); + + expResult = 12L; + result = instance.toLong("int"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("char"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toChar method, of class PropertyParser. + */ + @Test + public void testToChar_String() { + System.out.println("toChar"); + char expResult; + char result; + + Properties props = new Properties(); + props.put("value1", "f"); + props.put("value2", "w"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 'f'; + result = instance.toChar("value1"); + assertEquals(expResult, result); + + expResult = 'w'; + result = instance.toChar("value2"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("empty"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("str"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("boolean"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("float"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("int"); + assertEquals(expResult, result); + + expResult = 'a'; + result = instance.toChar("char"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toString method, of class PropertyParser. + */ + @Test + public void testToString_String_String() { + System.out.println("toString"); + String expResult; + String result; + + Properties props = new Properties(); + props.put("value1", "sTr1"); + props.put("value2", "str_2"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = "sTr1"; + result = instance.toString("value1", "defStr"); + assertEquals(expResult, result); + + expResult = "str_2"; + result = instance.toString("value2", "defStr"); + assertEquals(expResult, result); + + expResult = ""; + result = instance.toString("empty", "defStr"); + assertEquals(expResult, result); + + expResult = "abc"; + result = instance.toString("str", "defStr"); + assertEquals(expResult, result); + + expResult = "true"; + result = instance.toString("boolean", "defStr"); + assertEquals(expResult, result); + + expResult = "24.98"; + result = instance.toString("float", "defStr"); + assertEquals(expResult, result); + + expResult = "12"; + result = instance.toString("int", "defStr"); + assertEquals(expResult, result); + + expResult = "a"; + result = instance.toString("char", "defStr"); + assertEquals(expResult, result); + + expResult = "defStr"; + result = instance.toString("nonexistent", "defStr"); + assertEquals(expResult, result); + } + + /** + * Test of toInt method, of class PropertyParser. + */ + @Test + public void testToInt_String_int() { + System.out.println("toInt"); + int expResult; + int result; + + Properties props = new Properties(); + props.put("value1", "123"); + props.put("value2", "-54"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 123; + result = instance.toInt("value1", 17); + assertEquals(expResult, result); + + expResult = -54; + result = instance.toInt("value2", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("empty", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("str", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("boolean", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("float", 17); + assertEquals(expResult, result); + + expResult = 12; + result = instance.toInt("int", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("char", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("nonexistent", 17); + assertEquals(expResult, result); + } + + /** + * Test of toBoolean method, of class PropertyParser. + */ + @Test + public void testToBoolean_String_boolean() { + System.out.println("toBoolean"); + + boolean expResult; + boolean result; + + Properties props = new Properties(); + props.put("value1", "true"); + props.put("value2", "false"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = true; + result = instance.toBoolean("value1", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("value2", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("empty", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("str", true); + assertEquals(expResult, result); + + expResult = true; + result = instance.toBoolean("boolean", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("float", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("int", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("char", true); + assertEquals(expResult, result); + + expResult = true; + result = instance.toBoolean("nonexistent", true); + assertEquals(expResult, result); + } + + /** + * Test of toDouble method, of class PropertyParser. + */ + @Test + public void testToFloat_String_float() { + System.out.println("toFloat"); + float expResult; + float result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789f; + result = instance.toFloat("value1", 0.123f); + assertEquals(expResult, result, 0); + + expResult = -9000.001f; + result = instance.toFloat("value2", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("empty", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("str", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("boolean", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 24.98f; + result = instance.toFloat("float", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 12; + result = instance.toFloat("int", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("char", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("nonexistent", 0.123f); + assertEquals(expResult, result, 0); + } + + /** + * Test of toDouble method, of class PropertyParser. + */ + @Test + public void testToDouble_String_double() { + System.out.println("toDouble"); + double expResult; + double result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789; + result = instance.toDouble("value1", 0.123); + assertEquals(expResult, result, 0); + + expResult = -9000.001; + result = instance.toDouble("value2", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("empty", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("str", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("boolean", 0.123); + assertEquals(expResult, result, 0); + + expResult = 24.98; + result = instance.toDouble("float", 0.123); + assertEquals(expResult, result, 0); + + expResult = 12; + result = instance.toDouble("int", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("char", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("nonexistent", 0.123); + assertEquals(expResult, result, 0); + } + + /** + * Test of toLong method, of class PropertyParser. + */ + @Test + public void testToLong_String_long() { + System.out.println("toLong"); + long expResult; + long result; + + Properties props = new Properties(); + props.put("value1", "12345678900"); + props.put("value2", "-9000001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345678900L; + result = instance.toLong("value1", 3L); + assertEquals(expResult, result); + + expResult = -9000001L; + result = instance.toLong("value2", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("empty", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("str", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("boolean", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("float", 3L); + assertEquals(expResult, result); + + expResult = 12L; + result = instance.toLong("int", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("char", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("nonexistent", 3L); + assertEquals(expResult, result); + } + + /** + * Test of toChar method, of class PropertyParser. + */ + @Test + public void testToChar_String_char() { + System.out.println("toChar"); + char expResult; + char result; + + Properties props = new Properties(); + props.put("value1", "f"); + props.put("value2", "w"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 'f'; + result = instance.toChar("value1", 't'); + assertEquals(expResult, result); + + expResult = 'w'; + result = instance.toChar("value2", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("empty", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("str", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("boolean", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("float", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("int", 't'); + assertEquals(expResult, result); + + expResult = 'a'; + result = instance.toChar("char", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("nonexistent", 't'); + assertEquals(expResult, result); + } + +} diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/SISTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/SISTest.java new file mode 100644 index 000000000..34953554c --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/tools/SISTest.java @@ -0,0 +1,69 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class SISTest { + // + + private SIS sis; + // + + @Test + public void testAll() throws Exception { + // + sis = new SIS(); + // + int mtLv = 0; + // + sis.initValues( mtLv, "TEST", System.out, System.err, FileUtils.getTempDirectory().getAbsolutePath(), "Test", "ABC", true, true ); + // + String fFName = sis.getfullFileName(); + sis.info( fFName ); + sis.info( "aaabbbcccdddeefff" ); + // + assertEquals( 33, fFName.length() ); + assertEquals( "Z", fFName.substring( 0, 1 ) ); + assertEquals( "_Test_ABC.txt", fFName.substring( fFName.length() - 13, fFName.length() ) ); + // assertEquals( "", fFName ); + // assertEquals( "", tmpFld.getRoot().getAbsolutePath() ); + // + } + + @AfterEach + public void after() { + // + int mtLv = 0; + if ( sis != null ) sis.onStop( mtLv ); + // + // tmpFld.delete(); + // + } + + + +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/util/ArrayUtilTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/util/ArrayUtilTest.java new file mode 100644 index 000000000..3e8498820 --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/util/ArrayUtilTest.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import org.junit.jupiter.api.Test; + +public class ArrayUtilTest { + + @Test + public void testInvertPermutationInt(){ + assertArrayEquals( + new int[]{ 2, 4, 3, 0, 1 }, + ArrayUtil.invertPermutation(3, 4, 0, 2, 1) + ); + } + + @Test + public void testInvertPermutationLong(){ + assertArrayEquals( + new long[]{ 2, 4, 3, 0, 1 }, + ArrayUtil.invertPermutation(3L, 4L, 0L, 2L, 1L) + ); + } + +} diff --git a/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/util/OneTimeLoggerTest.java b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/util/OneTimeLoggerTest.java new file mode 100644 index 000000000..ead53d80e --- /dev/null +++ b/cavis-dnn/cavis-dnn-common/src/test/java/org/nd4j/common/util/OneTimeLoggerTest.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class OneTimeLoggerTest { + + @Test + public void testLogger1() throws Exception { + OneTimeLogger.info(log, "Format: {}; Pew: {};", 1, 2); + } + + @Test + public void testBuffer1() throws Exception { + assertTrue(OneTimeLogger.isEligible("Message here")); + + assertFalse(OneTimeLogger.isEligible("Message here")); + + assertTrue(OneTimeLogger.isEligible("Message here 23")); + } +} diff --git a/nd4j/nd4j-common/src/test/resources/somedir/afile.txt b/cavis-dnn/cavis-dnn-common/src/test/resources/somedir/afile.txt similarity index 100% rename from nd4j/nd4j-common/src/test/resources/somedir/afile.txt rename to cavis-dnn/cavis-dnn-common/src/test/resources/somedir/afile.txt diff --git a/cavis-dnn/cavis-dnn-core/build.gradle b/cavis-dnn/cavis-dnn-core/build.gradle new file mode 100644 index 000000000..18c322532 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/build.gradle @@ -0,0 +1,51 @@ + + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + + implementation projects.cavisDnn.cavisDnnTsne + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators + implementation projects.cavisDnn.cavisDnnModelimport + implementation projects.cavisDnn.cavisDnnApi + implementation 'org.slf4j:slf4j-api' + testImplementation 'ch.qos.logback:logback-classic' + implementation projects.cavisDnn.cavisDnnNn + implementation 'org.apache.commons:commons-math3' + implementation "commons-io:commons-io" + implementation "org.apache.commons:commons-compress" + + testImplementation projects.cavisNative.cavisNativeCommon + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testImplementation projects.cavisDnn.cavisDnnCommonTests + + implementation "org.apache.commons:commons-lang3" + + implementation "com.fasterxml.jackson.core:jackson-core" + + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" + + + + + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage + implementation projects.cavisUi.cavisUiComponents + + //provided "javax.xml.bind:jaxb-api:2.3.1 + implementation "javax.xml.bind:jaxb-api:2.3.1" + implementation "com.github.oshi:oshi-json" + implementation "com.github.oshi:oshi-core" + + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" + testImplementation "org.bytedeco:javacpp" + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators + testImplementation projects.cavisNative.cavisNativeBlas + testImplementation "it.unimi.dsi:fastutil:8.1.1" + testImplementation "com.google.guava:guava" + +} + diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/datasets/test/TestDataSetIterator.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/datasets/test/TestDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/datasets/test/TestDataSetIterator.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/datasets/test/TestDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/datasets/vectorizer/Vectorizer.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/datasets/vectorizer/Vectorizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/datasets/vectorizer/Vectorizer.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/datasets/vectorizer/Vectorizer.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/evaluation/EvaluationTools.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/evaluation/EvaluationTools.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/evaluation/EvaluationTools.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/evaluation/EvaluationTools.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/DeviceMetric.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/DeviceMetric.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/DeviceMetric.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/DeviceMetric.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/DiskInfo.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/DiskInfo.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/DiskInfo.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/DiskInfo.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/HardwareMetric.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/HardwareMetric.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/HardwareMetric.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/HardwareMetric.java index 6af3190e4..cff151a1d 100644 --- a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/HardwareMetric.java +++ b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/HardwareMetric.java @@ -25,8 +25,8 @@ import org.nd4j.linalg.api.environment.Nd4jEnvironment; import org.nd4j.linalg.api.ops.performance.PerformanceTracker; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.api.memory.MemcpyDirection; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import oshi.json.SystemInfo; import oshi.json.hardware.CentralProcessor; import oshi.json.hardware.GlobalMemory; diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoFilePrintListener.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoFilePrintListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoFilePrintListener.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoFilePrintListener.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoPrintListener.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoPrintListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoPrintListener.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/SystemInfoPrintListener.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemPolling.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/SystemPolling.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemPolling.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/SystemPolling.java index 74e030b2f..954be2065 100644 --- a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/listener/SystemPolling.java +++ b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/listener/SystemPolling.java @@ -21,8 +21,8 @@ package org.deeplearning4j.core.listener; import lombok.extern.slf4j.Slf4j; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import oshi.json.SystemInfo; import java.io.File; diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/DataSetLoader.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/DataSetLoader.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/DataSetLoader.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/DataSetLoader.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/MultiDataSetLoader.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/MultiDataSetLoader.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/MultiDataSetLoader.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/MultiDataSetLoader.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/RecordReaderFileBatchLoader.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/impl/RecordReaderFileBatchLoader.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/RecordReaderFileBatchLoader.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/impl/RecordReaderFileBatchLoader.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedDataSetLoader.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedDataSetLoader.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedDataSetLoader.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedDataSetLoader.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedMultiDataSetLoader.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedMultiDataSetLoader.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedMultiDataSetLoader.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/loader/impl/SerializedMultiDataSetLoader.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/parallelism/AsyncIterator.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/parallelism/AsyncIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/parallelism/AsyncIterator.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/parallelism/AsyncIterator.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/Persistable.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/Persistable.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/Persistable.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/Persistable.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorage.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorage.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorage.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorage.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageEvent.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageEvent.java similarity index 75% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageEvent.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageEvent.java index 783b084df..7f8254c04 100644 --- a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageEvent.java +++ b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageEvent.java @@ -20,10 +20,8 @@ package org.deeplearning4j.core.storage; -import lombok.AllArgsConstructor; import lombok.Data; -@AllArgsConstructor @Data public class StatsStorageEvent { private final StatsStorage statsStorage; @@ -32,4 +30,16 @@ public class StatsStorageEvent { private final String typeID; private final String workerID; private final long timestamp; + + public StatsStorageEvent(StatsStorage statsStorage, StatsStorageListener.EventType eventType, + String sessionID, String typeID, String workerID, long timestamp) { + + this.statsStorage = statsStorage; + this.eventType = eventType; + this.sessionID = sessionID; + this.typeID = typeID; + this.workerID = workerID; + this.timestamp = timestamp; + } + } diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageListener.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageListener.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageListener.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouter.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouter.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouter.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouter.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouterProvider.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouterProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouterProvider.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StatsStorageRouterProvider.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StorageMetaData.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StorageMetaData.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StorageMetaData.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StorageMetaData.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StorageType.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StorageType.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/StorageType.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/StorageType.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/CollectionStatsStorageRouter.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/impl/CollectionStatsStorageRouter.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/CollectionStatsStorageRouter.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/impl/CollectionStatsStorageRouter.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/RemoteUIStatsStorageRouter.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/impl/RemoteUIStatsStorageRouter.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/RemoteUIStatsStorageRouter.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/impl/RemoteUIStatsStorageRouter.java index 980cfea1c..b67798056 100644 --- a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/impl/RemoteUIStatsStorageRouter.java +++ b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/impl/RemoteUIStatsStorageRouter.java @@ -27,7 +27,7 @@ import org.deeplearning4j.core.storage.Persistable; import org.deeplearning4j.core.storage.StatsStorageRouter; import org.deeplearning4j.core.storage.StorageMetaData; import org.deeplearning4j.core.storage.StorageType; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import javax.xml.bind.DatatypeConverter; import java.io.*; diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/listener/RoutingIterationListener.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/listener/RoutingIterationListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/storage/listener/RoutingIterationListener.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/storage/listener/RoutingIterationListener.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/ui/UiConnectionInfo.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/ui/UiConnectionInfo.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/ui/UiConnectionInfo.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/ui/UiConnectionInfo.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ModelGuesser.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/ModelGuesser.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ModelGuesser.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/ModelGuesser.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ModelGuesserException.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/ModelGuesserException.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ModelGuesserException.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/ModelGuesserException.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/MovingWindowMatrix.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/MovingWindowMatrix.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/MovingWindowMatrix.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/MovingWindowMatrix.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ThreadUtils.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/ThreadUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/ThreadUtils.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/ThreadUtils.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/UIDProvider.java b/cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/UIDProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/java/org/deeplearning4j/core/util/UIDProvider.java rename to cavis-dnn/cavis-dnn-core/src/main/java/org/deeplearning4j/core/util/UIDProvider.java diff --git a/deeplearning4j/deeplearning4j-core/src/main/resources/iris.dat b/cavis-dnn/cavis-dnn-core/src/main/resources/iris.dat old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/main/resources/iris.dat rename to cavis-dnn/cavis-dnn-core/src/main/resources/iris.dat diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/AssertTestsExtendBaseClass.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/AssertTestsExtendBaseClass.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/AssertTestsExtendBaseClass.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/AssertTestsExtendBaseClass.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/LayerHelperValidationUtil.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/LayerHelperValidationUtil.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/LayerHelperValidationUtil.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/LayerHelperValidationUtil.java index dd9b3af6a..cc1220762 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/LayerHelperValidationUtil.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/LayerHelperValidationUtil.java @@ -145,7 +145,7 @@ public class LayerHelperValidationUtil { System.out.println(p1); System.out.println(p2); } - assertTrue(maxRE < t.getMaxRelError(),s + " - param changed during forward pass: " + p); + assertTrue(maxRE < t.getMaxRelError(), s + " - param changed during forward pass: " + p); } for( int i=0; i[]{ int.class, int.class, int.class }, + 45, 175, 200); + + TestAbstract rectangle = DL4JClassLoading.createNewInstance( + rectangleClassName, + Object.class, + new Class[]{ int.class, int.class, TestAbstract.class }, + 10, 15, color); + + /* Then */ + assertNotNull(color); + assertEquals(colorClassName, color.getClass().getName()); + + assertNotNull(rectangle); + assertEquals(rectangleClassName, rectangle.getClass().getName()); + } +} diff --git a/deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/dummies/TestAbstract.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/common/config/dummies/TestAbstract.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/dummies/TestAbstract.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/common/config/dummies/TestAbstract.java diff --git a/deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/dummies/TestColor.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/common/config/dummies/TestColor.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/dummies/TestColor.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/common/config/dummies/TestColor.java diff --git a/deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/dummies/TestDummy.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/common/config/dummies/TestDummy.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/dummies/TestDummy.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/common/config/dummies/TestDummy.java diff --git a/deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/dummies/TestRectangle.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/common/config/dummies/TestRectangle.java similarity index 100% rename from deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/dummies/TestRectangle.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/common/config/dummies/TestRectangle.java diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/MnistFetcherTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/MnistFetcherTest.java new file mode 100644 index 000000000..f45a76fe7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/MnistFetcherTest.java @@ -0,0 +1,179 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.base.MnistFetcher; +import org.deeplearning4j.common.resources.DL4JResources; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.conditions.Conditions; + +import java.io.File; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@org.junit.jupiter.api.Timeout(300) +public class MnistFetcherTest extends BaseDL4JTest { + + @TempDir + public File testDir; + + @BeforeAll + public void setup() throws Exception { + DL4JResources.setBaseDirectory(testDir); + } + + @AfterAll + public void after() { + DL4JResources.resetBaseDirectoryLocation(); + } + + @Test + public void testMnist() throws Exception { + DataSetIterator iter = new MnistDataSetIterator(32, 60000, false, true, false, -1); + int count = 0; + while(iter.hasNext()){ + DataSet ds = iter.next(); + INDArray arr = ds.getFeatures().sum(1); + int countMatch = Nd4j.getExecutioner().execAndReturn(new MatchCondition(arr, Conditions.equals(0))).z().getInt(0); + assertEquals(0, countMatch); + count++; + } + assertEquals(60000/32, count); + + count = 0; + iter = new MnistDataSetIterator(32, false, 12345); + while(iter.hasNext()){ + DataSet ds = iter.next(); + INDArray arr = ds.getFeatures().sum(1); + int countMatch = Nd4j.getExecutioner().execAndReturn(new MatchCondition(arr, Conditions.equals(0))).z().getInt(0); + assertEquals(0, countMatch); + count++; + } + assertEquals((int)Math.ceil(10000/32.0), count); + } + + @Test + public void testMnistDataFetcher() throws Exception { + MnistFetcher mnistFetcher = new MnistFetcher(); + File mnistDir = mnistFetcher.downloadAndUntar(); + + assertTrue(mnistDir.isDirectory()); + } + +// @Test + public void testMnistSubset() throws Exception { + final int numExamples = 100; + + MnistDataSetIterator iter1 = new MnistDataSetIterator(10, numExamples, false, true, true, 123); + int examples1 = 0; + int itCount1 = 0; + while (iter1.hasNext()) { + itCount1++; + examples1 += iter1.next().numExamples(); + } + assertEquals(10, itCount1); + assertEquals(100, examples1); + + MnistDataSetIterator iter2 = new MnistDataSetIterator(10, numExamples, false, true, true, 123); + int examples2 = 0; + int itCount2 = 0; + for (int i = 0; i < 10; i++) { + itCount2++; + examples2 += iter2.next().numExamples(); + } + assertFalse(iter2.hasNext()); + assertEquals(10, itCount2); + assertEquals(100, examples2); + + MnistDataSetIterator iter3 = new MnistDataSetIterator(19, numExamples, false, true, true, 123); + int examples3 = 0; + int itCount3 = 0; + while (iter3.hasNext()) { + itCount3++; + examples3 += iter3.next().numExamples(); + } + assertEquals(100, examples3); + assertEquals((int)Math.ceil(100/19.0), itCount3); + + MnistDataSetIterator iter4 = new MnistDataSetIterator(32, true, 12345); + int count4 = 0; + while(iter4.hasNext()){ + count4 += iter4.next().numExamples(); + } + assertEquals(60000, count4); + } + + @Test + public void testSubsetRepeatability() throws Exception { + + DataSetIterator it = new MnistDataSetIterator(1, 1, false, false, true, 0); + DataSet d1 = it.next(); + for( int i=0; i<10; i++ ) { + it.reset(); + DataSet d2 = it.next(); + assertEquals(d1.get(0).getFeatures(), d2.get(0).getFeatures()); + } + + //Check larger number: + it = new MnistDataSetIterator(8, 32, false, false, true, 12345); + Set featureLabelSet = new HashSet<>(); + while(it.hasNext()){ + DataSet ds = it.next(); + INDArray f = ds.getFeatures(); + INDArray l = ds.getLabels(); + + for( int i=0; i flSet2 = new HashSet<>(); + while(it.hasNext()){ + DataSet ds = it.next(); + INDArray f = ds.getFeatures(); + INDArray l = ds.getLabels(); + + for( int j=0; j dsList = new ArrayList<>(); + while (iter.hasNext()) { + dsList.add(iter.next()); + } + + assertEquals(3, dsList.size()); //3 files + for (int i = 0; i < 3; i++) { + DataSet ds = dsList.get(i); + INDArray features = ds.getFeatures(); + INDArray labels = ds.getLabels(); + assertEquals(1, features.size(0)); //1 example in mini-batch + assertEquals(1, labels.size(0)); + assertEquals(3, features.size(1)); //3 values per line/time step + assertEquals(4, labels.size(1)); //1 value per line, but 4 possible values -> one-hot vector + assertEquals(4, features.size(2)); //sequence length = 4 + assertEquals(4, labels.size(2)); + } + + //Check features vs. expected: + INDArray expF0 = Nd4j.create(1, 3, 4); + expF0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0, 1, 2})); + expF0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {10, 11, 12})); + expF0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {20, 21, 22})); + expF0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {30, 31, 32})); + assertEquals(dsList.get(0).getFeatures(), expF0); + + INDArray expF1 = Nd4j.create(1, 3, 4); + expF1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {100, 101, 102})); + expF1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {110, 111, 112})); + expF1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {120, 121, 122})); + expF1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {130, 131, 132})); + assertEquals(dsList.get(1).getFeatures(), expF1); + + INDArray expF2 = Nd4j.create(1, 3, 4); + expF2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {200, 201, 202})); + expF2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {210, 211, 212})); + expF2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {220, 221, 222})); + expF2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {230, 231, 232})); + assertEquals(dsList.get(2).getFeatures(), expF2); + + //Check labels vs. expected: + INDArray expL0 = Nd4j.create(1, 4, 4); + expL0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {1, 0, 0, 0})); + expL0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + expL0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {0, 0, 1, 0})); + expL0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {0, 0, 0, 1})); + assertEquals(dsList.get(0).getLabels(), expL0); + + INDArray expL1 = Nd4j.create(1, 4, 4); + expL1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0, 0, 0, 1})); + expL1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {0, 0, 1, 0})); + expL1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + expL1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {1, 0, 0, 0})); + assertEquals(dsList.get(1).getLabels(), expL1); + + INDArray expL2 = Nd4j.create(1, 4, 4); + expL2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + expL2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {1, 0, 0, 0})); + expL2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {0, 0, 0, 1})); + expL2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {0, 0, 1, 0})); + assertEquals(dsList.get(2).getLabels(), expL2); + } + + @Test + public void testSequenceRecordReaderMeta() throws Exception { + File rootDir = temporaryFolder; + //need to manually extract + for (int i = 0; i < 3; i++) { + FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); + FileUtils.copyFile(Resources.asFile(String.format("csvsequencelabels_%d.txt", i)), new File(rootDir, String.format("csvsequencelabels_%d.txt", i))); + } + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iter = + new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); + + iter.setCollectMetaData(true); + + assertEquals(3, iter.inputColumns()); + assertEquals(4, iter.totalOutcomes()); + + while (iter.hasNext()) { + DataSet ds = iter.next(); + List meta = ds.getExampleMetaData(RecordMetaData.class); + DataSet fromMeta = iter.loadFromMetaData(meta); + + assertEquals(ds, fromMeta); + } + } + + @Test + public void testSequenceRecordReaderRegression() throws Exception { + //need to manually extract + File rootDir = temporaryFolder; + for (int i = 0; i < 3; i++) { + FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); + } + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iter = + new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 0, true); + + assertEquals(3, iter.inputColumns()); + assertEquals(3, iter.totalOutcomes()); + + List dsList = new ArrayList<>(); + while (iter.hasNext()) { + dsList.add(iter.next()); + } + + assertEquals(3, dsList.size()); //3 files + for (int i = 0; i < 3; i++) { + DataSet ds = dsList.get(i); + INDArray features = ds.getFeatures(); + INDArray labels = ds.getLabels(); + assertArrayEquals(new long[] {1, 3, 4}, features.shape()); //1 examples, 3 values, 4 time steps + assertArrayEquals(new long[] {1, 3, 4}, labels.shape()); + + assertEquals(features, labels); + } + + //Also test regression + reset from a single reader: + featureReader.reset(); + iter = new SequenceRecordReaderDataSetIterator(featureReader, 1, 0, 2, true); + int count = 0; + while (iter.hasNext()) { + DataSet ds = iter.next(); + assertEquals(2, ds.getFeatures().size(1)); + assertEquals(1, ds.getLabels().size(1)); + count++; + } + assertEquals(3, count); + + + iter.reset(); + count = 0; + while (iter.hasNext()) { + iter.next(); + count++; + } + assertEquals(3, count); + } + + @Test + public void testSequenceRecordReaderMultiRegression() throws Exception { + File rootDir = temporaryFolder; + //need to manually extract + for (int i = 0; i < 3; i++) { + FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); + } + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + + SequenceRecordReader reader = new CSVSequenceRecordReader(1, ","); + reader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iter = + new SequenceRecordReaderDataSetIterator(reader, 1, 2, 1, true); + + assertEquals(1, iter.inputColumns()); + assertEquals(2, iter.totalOutcomes()); + + List dsList = new ArrayList<>(); + while (iter.hasNext()) { + dsList.add(iter.next()); + } + + assertEquals(3, dsList.size()); //3 files + for (int i = 0; i < 3; i++) { + DataSet ds = dsList.get(i); + INDArray features = ds.getFeatures(); + INDArray labels = ds.getLabels(); + assertArrayEquals(new long[] {1, 1, 4}, features.shape()); //1 examples, 1 values, 4 time steps + assertArrayEquals(new long[] {1, 2, 4}, labels.shape()); + + INDArray f2d = features.get(point(0), all(), all()).transpose(); + INDArray l2d = labels.get(point(0), all(), all()).transpose(); + + switch (i){ + case 0: + assertEquals(Nd4j.create(new double[]{0,10,20,30}, new int[]{4,1}).castTo(DataType.FLOAT), f2d); + assertEquals(Nd4j.create(new double[][]{{1,2}, {11,12}, {21,22}, {31,32}}).castTo(DataType.FLOAT), l2d); + break; + case 1: + assertEquals(Nd4j.create(new double[]{100,110,120,130}, new int[]{4,1}).castTo(DataType.FLOAT), f2d); + assertEquals(Nd4j.create(new double[][]{{101,102}, {111,112}, {121,122}, {131,132}}).castTo(DataType.FLOAT), l2d); + break; + case 2: + assertEquals(Nd4j.create(new double[]{200,210,220,230}, new int[]{4,1}).castTo(DataType.FLOAT), f2d); + assertEquals(Nd4j.create(new double[][]{{201,202}, {211,212}, {221,222}, {231,232}}).castTo(DataType.FLOAT), l2d); + break; + default: + throw new RuntimeException(); + } + } + + + iter.reset(); + int count = 0; + while (iter.hasNext()) { + iter.next(); + count++; + } + assertEquals(3, count); + } + + + + @Test + public void testSequenceRecordReaderReset() throws Exception { + File rootDir = temporaryFolder; + //need to manually extract + for (int i = 0; i < 3; i++) { + FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); + FileUtils.copyFile(Resources.asFile(String.format("csvsequencelabels_%d.txt", i)), new File(rootDir, String.format("csvsequencelabels_%d.txt", i))); + } + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iter = + new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); + + assertEquals(3, iter.inputColumns()); + assertEquals(4, iter.totalOutcomes()); + + int nResets = 5; + for (int i = 0; i < nResets; i++) { + iter.reset(); + int count = 0; + while (iter.hasNext()) { + DataSet ds = iter.next(); + INDArray features = ds.getFeatures(); + INDArray labels = ds.getLabels(); + assertArrayEquals(new long[] {1, 3, 4}, features.shape()); + assertArrayEquals(new long[] {1, 4, 4}, labels.shape()); + count++; + } + assertEquals(3, count); + } + } + + + + @Test + public void testCSVLoadingRegression() throws Exception { + int nLines = 30; + int nFeatures = 5; + int miniBatchSize = 10; + int labelIdx = 0; + + String path = "rr_csv_test_rand.csv"; + Pair p = makeRandomCSV(path, nLines, nFeatures); + double[][] data = p.getFirst(); + RecordReader testReader = new CSVRecordReader(); + testReader.initialize(new FileSplit(p.getSecond())); + + DataSetIterator iter = new RecordReaderDataSetIterator(testReader, miniBatchSize, labelIdx, labelIdx, true); + int miniBatch = 0; + while (iter.hasNext()) { + DataSet test = iter.next(); + INDArray features = test.getFeatures(); + INDArray labels = test.getLabels(); + assertArrayEquals(new long[] {miniBatchSize, nFeatures}, features.shape()); + assertArrayEquals(new long[] {miniBatchSize, 1}, labels.shape()); + + int startRow = miniBatch * miniBatchSize; + for (int i = 0; i < miniBatchSize; i++) { + double labelExp = data[startRow + i][labelIdx]; + double labelAct = labels.getDouble(i); + assertEquals(labelExp, labelAct, 1e-5f); + + int featureCount = 0; + for (int j = 0; j < nFeatures + 1; j++) { + if (j == labelIdx) + continue; + double featureExp = data[startRow + i][j]; + double featureAct = features.getDouble(i, featureCount++); + assertEquals(featureExp, featureAct, 1e-5f); + } + } + + miniBatch++; + } + assertEquals(nLines / miniBatchSize, miniBatch); + } + + + public Pair makeRandomCSV(String tempFile, int nLines, int nFeatures) throws IOException { + File temp = temporaryFolder; + temp.mkdirs(); + temp.deleteOnExit(); + Random rand = new Random(12345); + + double[][] dArr = new double[nLines][nFeatures + 1]; + + try (PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(temp)))) { + for (int i = 0; i < nLines; i++) { + dArr[i][0] = rand.nextDouble(); //First column: label + out.print(dArr[i][0]); + for (int j = 0; j < nFeatures; j++) { + dArr[i][j + 1] = rand.nextDouble(); + out.print("," + dArr[i][j + 1]); + } + out.println(); + } + } catch (IOException e) { + log.error("",e); + } + + return new Pair<>(dArr,temp); + } + + @Test + public void testVariableLengthSequence() throws Exception { + File rootDir = temporaryFolder; + //need to manually extract + for (int i = 0; i < 3; i++) { + FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); + FileUtils.copyFile(Resources.asFile(String.format("csvsequencelabelsShort_%d.txt", i)), new File(rootDir, String.format("csvsequencelabelsShort_%d.txt", i))); + } + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabelsShort_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); + featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iterAlignStart = new SequenceRecordReaderDataSetIterator(featureReader, + labelReader, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_START); + + SequenceRecordReaderDataSetIterator iterAlignEnd = new SequenceRecordReaderDataSetIterator(featureReader2, + labelReader2, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); + + assertEquals(3, iterAlignStart.inputColumns()); + assertEquals(4, iterAlignStart.totalOutcomes()); + + assertEquals(3, iterAlignEnd.inputColumns()); + assertEquals(4, iterAlignEnd.totalOutcomes()); + + List dsListAlignStart = new ArrayList<>(); + while (iterAlignStart.hasNext()) { + dsListAlignStart.add(iterAlignStart.next()); + } + + List dsListAlignEnd = new ArrayList<>(); + while (iterAlignEnd.hasNext()) { + dsListAlignEnd.add(iterAlignEnd.next()); + } + + assertEquals(3, dsListAlignStart.size()); //3 files + assertEquals(3, dsListAlignEnd.size()); //3 files + + for (int i = 0; i < 3; i++) { + DataSet ds = dsListAlignStart.get(i); + INDArray features = ds.getFeatures(); + INDArray labels = ds.getLabels(); + assertEquals(1, features.size(0)); //1 example in mini-batch + assertEquals(1, labels.size(0)); + assertEquals(3, features.size(1)); //3 values per line/time step + assertEquals(4, labels.size(1)); //1 value per line, but 4 possible values -> one-hot vector + assertEquals(4, features.size(2)); //sequence length = 4 + assertEquals(4, labels.size(2)); + + DataSet ds2 = dsListAlignEnd.get(i); + features = ds2.getFeatures(); + labels = ds2.getLabels(); + assertEquals(1, features.size(0)); //1 example in mini-batch + assertEquals(1, labels.size(0)); + assertEquals(3, features.size(1)); //3 values per line/time step + assertEquals(4, labels.size(1)); //1 value per line, but 4 possible values -> one-hot vector + assertEquals(4, features.size(2)); //sequence length = 4 + assertEquals(4, labels.size(2)); + } + + //Check features vs. expected: + //Here: labels always longer than features -> same features for align start and align end + INDArray expF0 = Nd4j.create(1, 3, 4); + expF0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0, 1, 2})); + expF0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {10, 11, 12})); + expF0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {20, 21, 22})); + expF0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {30, 31, 32})); + assertEquals(expF0, dsListAlignStart.get(0).getFeatures()); + assertEquals(expF0, dsListAlignEnd.get(0).getFeatures()); + + INDArray expF1 = Nd4j.create(1, 3, 4); + expF1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {100, 101, 102})); + expF1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {110, 111, 112})); + expF1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {120, 121, 122})); + expF1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {130, 131, 132})); + assertEquals(expF1, dsListAlignStart.get(1).getFeatures()); + assertEquals(expF1, dsListAlignEnd.get(1).getFeatures()); + + INDArray expF2 = Nd4j.create(1, 3, 4); + expF2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {200, 201, 202})); + expF2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {210, 211, 212})); + expF2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {220, 221, 222})); + expF2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {230, 231, 232})); + assertEquals(expF2, dsListAlignStart.get(2).getFeatures()); + assertEquals(expF2, dsListAlignEnd.get(2).getFeatures()); + + //Check features mask array: + INDArray featuresMaskExpected = null; //null: equivalent to all 1s (i.e., present for all time steps) + for (int i = 0; i < 3; i++) { + INDArray featuresMaskStart = dsListAlignStart.get(i).getFeaturesMaskArray(); + INDArray featuresMaskEnd = dsListAlignEnd.get(i).getFeaturesMaskArray(); + assertEquals(featuresMaskExpected, featuresMaskStart); + assertEquals(featuresMaskExpected, featuresMaskEnd); + } + + + //Check labels vs. expected: + //First: aligning start + INDArray expL0 = Nd4j.create(1, 4, 4); + expL0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {1, 0, 0, 0})); + expL0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + assertEquals(expL0, dsListAlignStart.get(0).getLabels()); + + INDArray expL1 = Nd4j.create(1, 4, 4); + expL1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + assertEquals(expL1, dsListAlignStart.get(1).getLabels()); + + INDArray expL2 = Nd4j.create(1, 4, 4); + expL2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0, 0, 0, 1})); + expL2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {0, 0, 1, 0})); + expL2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + assertEquals(expL2, dsListAlignStart.get(2).getLabels()); + + //Second: align end + INDArray expL0end = Nd4j.create(1, 4, 4); + expL0end.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {1, 0, 0, 0})); + expL0end.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + assertEquals(expL0end, dsListAlignEnd.get(0).getLabels()); + + INDArray expL1end = Nd4j.create(1, 4, 4); + expL1end.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + assertEquals(expL1end, dsListAlignEnd.get(1).getLabels()); + + INDArray expL2end = Nd4j.create(1, 4, 4); + expL2end.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {0, 0, 0, 1})); + expL2end.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {0, 0, 1, 0})); + expL2end.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {0, 1, 0, 0})); + assertEquals(expL2end, dsListAlignEnd.get(2).getLabels()); + + //Check labels mask array + INDArray[] labelsMaskExpectedStart = new INDArray[] {Nd4j.create(new float[] {1, 1, 0, 0}, new int[] {1, 4}), + Nd4j.create(new float[] {1, 0, 0, 0}, new int[] {1, 4}), + Nd4j.create(new float[] {1, 1, 1, 0}, new int[] {1, 4})}; + INDArray[] labelsMaskExpectedEnd = new INDArray[] {Nd4j.create(new float[] {0, 0, 1, 1}, new int[] {1, 4}), + Nd4j.create(new float[] {0, 0, 0, 1}, new int[] {1, 4}), + Nd4j.create(new float[] {0, 1, 1, 1}, new int[] {1, 4})}; + + for (int i = 0; i < 3; i++) { + INDArray labelsMaskStart = dsListAlignStart.get(i).getLabelsMaskArray(); + INDArray labelsMaskEnd = dsListAlignEnd.get(i).getLabelsMaskArray(); + assertEquals(labelsMaskExpectedStart[i], labelsMaskStart); + assertEquals(labelsMaskExpectedEnd[i], labelsMaskEnd); + } + } + + @Test + public void testSequenceRecordReaderSingleReader() throws Exception { + File rootDir = temporaryFolder; + //need to manually extract + for (int i = 0; i < 3; i++) { + FileUtils.copyFile(Resources.asFile(String.format("csvsequenceSingle_%d.txt", i)), new File(rootDir, String.format("csvsequenceSingle_%d.txt", i))); + } + String path = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequenceSingle_%d.txt"); + + SequenceRecordReader reader = new CSVSequenceRecordReader(1, ","); + reader.initialize(new NumberedFileInputSplit(path, 0, 2)); + SequenceRecordReaderDataSetIterator iteratorClassification = + new SequenceRecordReaderDataSetIterator(reader, 1, 3, 0, false); + + assertTrue(iteratorClassification.hasNext()); + + SequenceRecordReader reader2 = new CSVSequenceRecordReader(1, ","); + reader2.initialize(new NumberedFileInputSplit(path, 0, 2)); + SequenceRecordReaderDataSetIterator iteratorRegression = + new SequenceRecordReaderDataSetIterator(reader2, 1, 1, 0, true); + + INDArray expF0 = Nd4j.create(1, 2, 4); + expF0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {1, 2})); + expF0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {11, 12})); + expF0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {21, 22})); + expF0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {31, 32})); + + INDArray expF1 = Nd4j.create(1, 2, 4); + expF1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {101, 102})); + expF1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {111, 112})); + expF1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {121, 122})); + expF1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {131, 132})); + + INDArray expF2 = Nd4j.create(1, 2, 4); + expF2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {201, 202})); + expF2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {211, 212})); + expF2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {221, 222})); + expF2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {231, 232})); + + INDArray[] expF = new INDArray[] {expF0, expF1, expF2}; + + //Expected out for classification: + INDArray expOut0 = Nd4j.create(1, 3, 4); + expOut0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {1, 0, 0})); + expOut0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {0, 1, 0})); + expOut0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {0, 0, 1})); + expOut0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {1, 0, 0})); + + INDArray expOut1 = Nd4j.create(1, 3, 4); + expOut1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0, 1, 0})); + expOut1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {0, 0, 1})); + expOut1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {1, 0, 0})); + expOut1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {0, 0, 1})); + + INDArray expOut2 = Nd4j.create(1, 3, 4); + expOut2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0, 1, 0})); + expOut2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {1, 0, 0})); + expOut2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {0, 1, 0})); + expOut2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {0, 0, 1})); + INDArray[] expOutClassification = new INDArray[] {expOut0, expOut1, expOut2}; + + //Expected out for regression: + INDArray expOutR0 = Nd4j.create(1, 1, 4); + expOutR0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {0})); + expOutR0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {1})); + expOutR0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {2})); + expOutR0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {0})); + + INDArray expOutR1 = Nd4j.create(1, 1, 4); + expOutR1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {1})); + expOutR1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {2})); + expOutR1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {0})); + expOutR1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {2})); + + INDArray expOutR2 = Nd4j.create(1, 1, 4); + expOutR2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] {1})); + expOutR2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] {0})); + expOutR2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] {1})); + expOutR2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] {2})); + INDArray[] expOutRegression = new INDArray[] {expOutR0, expOutR1, expOutR2}; + + int countC = 0; + while (iteratorClassification.hasNext()) { + DataSet ds = iteratorClassification.next(); + INDArray f = ds.getFeatures(); + INDArray l = ds.getLabels(); + assertNull(ds.getFeaturesMaskArray()); + assertNull(ds.getLabelsMaskArray()); + + assertArrayEquals(new long[] {1, 2, 4}, f.shape()); + assertArrayEquals(new long[] {1, 3, 4}, l.shape()); //One-hot representation + + assertEquals(expF[countC], f); + assertEquals(expOutClassification[countC++], l); + } + assertEquals(3, countC); + assertEquals(3, iteratorClassification.totalOutcomes()); + + int countF = 0; + while (iteratorRegression.hasNext()) { + DataSet ds = iteratorRegression.next(); + INDArray f = ds.getFeatures(); + INDArray l = ds.getLabels(); + assertNull(ds.getFeaturesMaskArray()); + assertNull(ds.getLabelsMaskArray()); + + assertArrayEquals(new long[] {1, 2, 4}, f.shape()); + assertArrayEquals(new long[] {1, 1, 4}, l.shape()); //Regression (single output) + + assertEquals(expF[countF], f); + assertEquals(expOutRegression[countF++], l); + } + assertEquals(3, countF); + assertEquals(1, iteratorRegression.totalOutcomes()); + } + + @Test + public void testSequenceRecordReaderSingleReaderWithEmptySequenceThrows() throws Exception { + assertThrows(ZeroLengthSequenceException.class, () -> { + SequenceRecordReader reader = new CSVSequenceRecordReader(1, ","); + reader.initialize(new FileSplit(Resources.asFile("empty.txt"))); + new SequenceRecordReaderDataSetIterator(reader, 1, -1, 1, true).next(); + }); + } + + @Test + public void testSequenceRecordReaderTwoReadersWithEmptyFeatureSequenceThrows() throws Exception { + assertThrows(ZeroLengthSequenceException.class, () -> { + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new FileSplit(Resources.asFile("empty.txt"))); + labelReader.initialize( + new FileSplit(Resources.asFile("csvsequencelabels_0.txt"))); + new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, -1, true).next(); + }); + } + + @Test + public void testSequenceRecordReaderTwoReadersWithEmptyLabelSequenceThrows() throws Exception { + assertThrows(ZeroLengthSequenceException.class, () -> { + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + + File f = Resources.asFile("csvsequence_0.txt"); + featureReader.initialize(new FileSplit(f)); + labelReader.initialize(new FileSplit(Resources.asFile("empty.txt"))); + + new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, -1, true).next(); + }); + } + + @Test + public void testSequenceRecordReaderSingleReaderMetaData() throws Exception { + File rootDir = temporaryFolder; + //need to manually extract + for (int i = 0; i < 3; i++) { + FileUtils.copyFile(Resources.asFile(String.format("csvsequenceSingle_%d.txt", i)), new File(rootDir, String.format("csvsequenceSingle_%d.txt", i))); + } + String path = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequenceSingle_%d.txt"); + + SequenceRecordReader reader = new CSVSequenceRecordReader(1, ","); + reader.initialize(new NumberedFileInputSplit(path, 0, 2)); + SequenceRecordReaderDataSetIterator iteratorClassification = + new SequenceRecordReaderDataSetIterator(reader, 1, 3, 0, false); + + SequenceRecordReader reader2 = new CSVSequenceRecordReader(1, ","); + reader2.initialize(new NumberedFileInputSplit(path, 0, 2)); + SequenceRecordReaderDataSetIterator iteratorRegression = + new SequenceRecordReaderDataSetIterator(reader2, 1, 1, 0, true); + + iteratorClassification.setCollectMetaData(true); + iteratorRegression.setCollectMetaData(true); + + while (iteratorClassification.hasNext()) { + DataSet ds = iteratorClassification.next(); + DataSet fromMeta = iteratorClassification.loadFromMetaData(ds.getExampleMetaData(RecordMetaData.class)); + assertEquals(ds, fromMeta); + } + + while (iteratorRegression.hasNext()) { + DataSet ds = iteratorRegression.next(); + DataSet fromMeta = iteratorRegression.loadFromMetaData(ds.getExampleMetaData(RecordMetaData.class)); + assertEquals(ds, fromMeta); + } + } + + + @Test + public void testSeqRRDSIArrayWritableOneReader() { + + List> sequence1 = new ArrayList<>(); + sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {1, 2, 3}, new long[]{1,3})), + new IntWritable(0))); + sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {4, 5, 6}, new long[]{1,3})), + new IntWritable(1))); + List> sequence2 = new ArrayList<>(); + sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {7, 8, 9}, new long[]{1,3})), + new IntWritable(2))); + sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {10, 11, 12}, new long[]{1,3})), + new IntWritable(3))); + + + SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); + + SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, 4, 1, false); + + DataSet ds = iter.next(); + + INDArray expFeatures = Nd4j.create(2, 3, 2); //2 examples, 3 values per time step, 2 time steps + expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] {{1, 4}, {2, 5}, {3, 6}})); + expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] {{7, 10}, {8, 11}, {9, 12}})); + + INDArray expLabels = Nd4j.create(2, 4, 2); + expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] {{1, 0}, {0, 1}, {0, 0}, {0, 0}})); + expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] {{0, 0}, {0, 0}, {1, 0}, {0, 1}})); + + assertEquals(expFeatures, ds.getFeatures()); + assertEquals(expLabels, ds.getLabels()); + } + + @Test + public void testSeqRRDSIArrayWritableOneReaderRegression() { + //Regression, where the output is an array writable + List> sequence1 = new ArrayList<>(); + sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {1, 2, 3}, new long[]{1,3})), + new NDArrayWritable(Nd4j.create(new double[] {100, 200, 300}, new long[]{1,3})))); + sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {4, 5, 6}, new long[]{1,3})), + new NDArrayWritable(Nd4j.create(new double[] {400, 500, 600}, new long[]{1,3})))); + List> sequence2 = new ArrayList<>(); + sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {7, 8, 9}, new long[]{1,3})), + new NDArrayWritable(Nd4j.create(new double[] {700, 800, 900}, new long[]{1,3})))); + sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {10, 11, 12}, new long[]{1,3})), + new NDArrayWritable(Nd4j.create(new double[] {1000, 1100, 1200}, new long[]{1,3})))); + + + SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); + + SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, -1, 1, true); + + DataSet ds = iter.next(); + + INDArray expFeatures = Nd4j.create(2, 3, 2); //2 examples, 3 values per time step, 2 time steps + expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] {{1, 4}, {2, 5}, {3, 6}})); + expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] {{7, 10}, {8, 11}, {9, 12}})); + + INDArray expLabels = Nd4j.create(2, 3, 2); + expLabels.tensorAlongDimension(0, 1, 2) + .assign(Nd4j.create(new double[][] {{100, 400}, {200, 500}, {300, 600}})); + expLabels.tensorAlongDimension(1, 1, 2) + .assign(Nd4j.create(new double[][] {{700, 1000}, {800, 1100}, {900, 1200}})); + + assertEquals(expFeatures, ds.getFeatures()); + assertEquals(expLabels, ds.getLabels()); + } + + @Test + public void testSeqRRDSIMultipleArrayWritablesOneReader() { + //Input with multiple array writables: + + List> sequence1 = new ArrayList<>(); + sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {1, 2, 3}, new long[]{1,3})), + new NDArrayWritable(Nd4j.create(new double[] {100, 200, 300}, new long[]{1,3})), new IntWritable(0))); + sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {4, 5, 6}, new long[]{1,3})), + new NDArrayWritable(Nd4j.create(new double[] {400, 500, 600}, new long[]{1,3})), new IntWritable(1))); + List> sequence2 = new ArrayList<>(); + sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {7, 8, 9}, new long[]{1,3})), + new NDArrayWritable(Nd4j.create(new double[] {700, 800, 900}, new long[]{1,3})), new IntWritable(2))); + sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {10, 11, 12}, new long[]{1,3})), + new NDArrayWritable(Nd4j.create(new double[] {1000, 1100, 1200}, new long[]{1,3})), new IntWritable(3))); + + + SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); + + SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, 4, 2, false); + + DataSet ds = iter.next(); + + INDArray expFeatures = Nd4j.create(2, 6, 2); //2 examples, 6 values per time step, 2 time steps + expFeatures.tensorAlongDimension(0, 1, 2).assign( + Nd4j.create(new double[][] {{1, 4}, {2, 5}, {3, 6}, {100, 400}, {200, 500}, {300, 600}})); + expFeatures.tensorAlongDimension(1, 1, 2).assign( + Nd4j.create(new double[][] {{7, 10}, {8, 11}, {9, 12}, {700, 1000}, {800, 1100}, {900, 1200}})); + + INDArray expLabels = Nd4j.create(2, 4, 2); + expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] {{1, 0}, {0, 1}, {0, 0}, {0, 0}})); + expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] {{0, 0}, {0, 0}, {1, 0}, {0, 1}})); + + assertEquals(expFeatures, ds.getFeatures()); + assertEquals(expLabels, ds.getLabels()); + } + + @Test + public void testSeqRRDSIArrayWritableTwoReaders() { + List> sequence1 = new ArrayList<>(); + sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {1, 2, 3}, new long[]{1,3})), + new IntWritable(100))); + sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {4, 5, 6}, new long[]{1,3})), + new IntWritable(200))); + List> sequence2 = new ArrayList<>(); + sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {7, 8, 9}, new long[]{1,3})), + new IntWritable(300))); + sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {10, 11, 12}, new long[]{1,3})), + new IntWritable(400))); + SequenceRecordReader rrFeatures = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); + + List> sequence1L = new ArrayList<>(); + sequence1L.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {100, 200, 300}, new long[]{1,3})), + new IntWritable(101))); + sequence1L.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {400, 500, 600}, new long[]{1,3})), + new IntWritable(201))); + List> sequence2L = new ArrayList<>(); + sequence2L.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {700, 800, 900}, new long[]{1,3})), + new IntWritable(301))); + sequence2L.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] {1000, 1100, 1200}, new long[]{1,3})), + new IntWritable(401))); + SequenceRecordReader rrLabels = new CollectionSequenceRecordReader(Arrays.asList(sequence1L, sequence2L)); + + SequenceRecordReaderDataSetIterator iter = + new SequenceRecordReaderDataSetIterator(rrFeatures, rrLabels, 2, -1, true); + + INDArray expFeatures = Nd4j.create(2, 4, 2); //2 examples, 4 values per time step, 2 time steps + expFeatures.tensorAlongDimension(0, 1, 2) + .assign(Nd4j.create(new double[][] {{1, 4}, {2, 5}, {3, 6}, {100, 200}})); + expFeatures.tensorAlongDimension(1, 1, 2) + .assign(Nd4j.create(new double[][] {{7, 10}, {8, 11}, {9, 12}, {300, 400}})); + + INDArray expLabels = Nd4j.create(2, 4, 2); + expLabels.tensorAlongDimension(0, 1, 2) + .assign(Nd4j.create(new double[][] {{100, 400}, {200, 500}, {300, 600}, {101, 201}})); + expLabels.tensorAlongDimension(1, 1, 2) + .assign(Nd4j.create(new double[][] {{700, 1000}, {800, 1100}, {900, 1200}, {301, 401}})); + + DataSet ds = iter.next(); + assertEquals(expFeatures, ds.getFeatures()); + assertEquals(expLabels, ds.getLabels()); + } + + @Test + public void testRecordReaderMetaData() throws Exception { + + RecordReader csv = new CSVRecordReader(); + csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + int batchSize = 10; + int labelIdx = 4; + int numClasses = 3; + + RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(csv, batchSize, labelIdx, numClasses); + rrdsi.setCollectMetaData(true); + + while (rrdsi.hasNext()) { + DataSet ds = rrdsi.next(); + List meta = ds.getExampleMetaData(RecordMetaData.class); + int i = 0; + for (RecordMetaData m : meta) { + Record r = csv.loadFromMetaData(m); + INDArray row = ds.getFeatures().getRow(i); +// if(i <= 3) { +// System.out.println(m.getLocation() + "\t" + r.getRecord() + "\t" + row); +// } + + for (int j = 0; j < 4; j++) { + double exp = r.getRecord().get(j).toDouble(); + double act = row.getDouble(j); + assertEquals(exp, act, 1e-6, "Failed on idx: " + j); + } + i++; + } +// System.out.println(); + + DataSet fromMeta = rrdsi.loadFromMetaData(meta); + assertEquals(ds, fromMeta); + } + } + + @Test + public void testRRDSIwithAsync() throws Exception { + RecordReader csv = new CSVRecordReader(); + csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + int batchSize = 10; + int labelIdx = 4; + int numClasses = 3; + + RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(csv, batchSize, labelIdx, numClasses); + AsyncDataSetIterator adsi = new AsyncDataSetIterator(rrdsi, 8, true); + while (adsi.hasNext()) { + DataSet ds = adsi.next(); + + } + + } + + + + @Test + public void testRecordReaderDataSetIteratorNDArrayWritableLabels() { + + Collection> data = new ArrayList<>(); + + data.add(Arrays.asList(new DoubleWritable(0), new DoubleWritable(1), + new NDArrayWritable(Nd4j.create(new double[] {1.1, 2.1, 3.1}, new long[]{1,3})))); + data.add(Arrays.asList(new DoubleWritable(2), new DoubleWritable(3), + new NDArrayWritable(Nd4j.create(new double[] {4.1, 5.1, 6.1}, new long[]{1,3})))); + data.add(Arrays.asList(new DoubleWritable(4), new DoubleWritable(5), + new NDArrayWritable(Nd4j.create(new double[] {7.1, 8.1, 9.1}, new long[]{1,3})))); + + RecordReader rr = new CollectionRecordReader(data); + int batchSize = 3; + int labelIndexFrom = 2; + int labelIndexTo = 2; + boolean regression = true; + DataSetIterator rrdsi = + new RecordReaderDataSetIterator(rr, batchSize, labelIndexFrom, labelIndexTo, regression); + + DataSet ds = rrdsi.next(); + INDArray expFeatures = Nd4j.create(new float[][] {{0, 1}, {2, 3}, {4, 5}}); + INDArray expLabels = Nd4j.create(new float[][] {{1.1f, 2.1f, 3.1f}, {4.1f, 5.1f, 6.1f}, {7.1f, 8.1f, 9.1f}}); + + assertEquals(expFeatures, ds.getFeatures()); + assertEquals(expLabels, ds.getLabels()); + + //ALSO: test if we have NDArrayWritables for BOTH the features and the labels + data = new ArrayList<>(); + + data.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] {0, 1}, new long[]{1,2})), + new NDArrayWritable(Nd4j.create(new double[] {1.1, 2.1, 3.1}, new long[]{1,3})))); + data.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] {2, 3}, new long[]{1,2})), + new NDArrayWritable(Nd4j.create(new double[] {4.1, 5.1, 6.1}, new long[]{1,3})))); + data.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] {4, 5}, new long[]{1,2})), + new NDArrayWritable(Nd4j.create(new double[] {7.1, 8.1, 9.1}, new long[]{1,3})))); + labelIndexFrom = 1; + labelIndexTo = 1; + + rr = new CollectionRecordReader(data); + rrdsi = new RecordReaderDataSetIterator(rr, batchSize, labelIndexFrom, labelIndexTo, regression); + + DataSet ds2 = rrdsi.next(); + assertEquals(expFeatures, ds2.getFeatures()); + assertEquals(expLabels, ds2.getLabels()); + } + + + @Test + //@Ignore + public void specialRRTest4() throws Exception { + RecordReader rr = new SpecialImageRecordReader(25000, 10, 3, 224, 224); + RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr, 128); + + int cnt = 0; + int examples = 0; + while (rrdsi.hasNext()) { + DataSet ds = rrdsi.next(); + assertEquals(128, ds.numExamples()); + for (int i = 0; i < ds.numExamples(); i++) { + INDArray example = ds.getFeatures().tensorAlongDimension(i, 1, 2, 3).dup(); + // assertEquals("Failed on DataSet [" + cnt + "], example [" + i + "]", (double) examples, example.meanNumber().doubleValue(), 0.01); + + // assertEquals("Failed on DataSet [" + cnt + "], example [" + i + "]", (double) examples, ds.getLabels().getRow(i).meanNumber().doubleValue(), 0.01); + examples++; + } + cnt++; + } + + } + + /* + @Test + public void specialRRTest1() throws Exception { + RecordReader rr = new SpecialImageRecordReader(250, 10,3, 224, 224); + DataSetIterator rrdsi = new ParallelRecordReaderDataSetIterator.Builder(rr) + .setBatchSize(10) + .numberOfWorkers(1) + .build(); + + int cnt = 0; + int examples = 0; + while (rrdsi.hasNext()) { + DataSet ds = rrdsi.next(); + for (int i = 0; i < ds.numExamples(); i++) { + INDArray example = ds.getFeatures().tensorAlongDimension(i, 1, 2, 3).dup(); + assertEquals("Failed on DataSet ["+ cnt + "], example ["+ i +"]",(double) examples, example.meanNumber().doubleValue(), 0.01); + examples++; + } + cnt++; + log.info("DataSet {} passed...", cnt); + } + + assertEquals(25, cnt); + } + + + @Test + public void specialRRTest2() throws Exception { + RecordReader rr = new SpecialImageRecordReader(250, 10,3, 224, 224); + DataSetIterator rrdsi = new ParallelRecordReaderDataSetIterator.Builder(rr) + .setBatchSize(10) + .numberOfWorkers(1) + .prefetchBufferSize(4) + .build(); + + rrdsi = new AsyncDataSetIterator(rrdsi); + + int cnt = 0; + int examples = 0; + while (rrdsi.hasNext()) { + DataSet ds = rrdsi.next(); + for (int i = 0; i < ds.numExamples(); i++) { + INDArray example = ds.getFeatures().tensorAlongDimension(i, 1, 2, 3).dup(); + assertEquals("Failed on DataSet ["+ cnt + "], example ["+ i +"]",(double) examples, example.meanNumber().doubleValue(), 0.01); + examples++; + } + cnt++; + } + + assertEquals(25, cnt); + } + + + @Test + public void specialRRTest3() throws Exception { + RecordReader rr = new SpecialImageRecordReader(400, 10,3, 224, 224); + DataSetIterator rrdsi = new ParallelRecordReaderDataSetIterator.Builder(rr) + .setBatchSize(128) + .numberOfWorkers(2) + .prefetchBufferSize(2) + .build(); + + log.info("DataType: {}", Nd4j.dataType() ); + + // rrdsi = new AsyncDataSetIterator(rrdsi); + + int cnt = 0; + int examples = 0; + while (rrdsi.hasNext()) { + DataSet ds = rrdsi.next(); + for (int i = 0; i < ds.numExamples(); i++) { + INDArray example = ds.getFeatures().tensorAlongDimension(i, 1, 2, 3).dup(); + assertEquals("Failed on DataSet ["+ cnt + "], example ["+ i +"]",(double) examples, example.meanNumber().doubleValue(), 0.01); + examples++; + } + cnt++; + } + + } + */ + + + @Test + public void testRecordReaderDataSetIteratorConcat() { + + //[DoubleWritable, DoubleWritable, NDArrayWritable([1,10]), IntWritable] -> concatenate to a [1,13] feature vector automatically. + + List l = Arrays.asList(new DoubleWritable(1), + new NDArrayWritable(Nd4j.create(new double[] {2, 3, 4})), new DoubleWritable(5), + new NDArrayWritable(Nd4j.create(new double[] {6, 7, 8})), new IntWritable(9), + new IntWritable(1)); + + RecordReader rr = new CollectionRecordReader(Collections.singletonList(l)); + + DataSetIterator iter = new RecordReaderDataSetIterator(rr, 1, 5, 3); + + DataSet ds = iter.next(); + INDArray expF = Nd4j.create(new float[] {1, 2, 3, 4, 5, 6, 7, 8, 9}, new int[]{1,9}); + INDArray expL = Nd4j.create(new float[] {0, 1, 0}, new int[]{1,3}); + + assertEquals(expF, ds.getFeatures()); + assertEquals(expL, ds.getLabels()); + } + + @Test + public void testRecordReaderDataSetIteratorConcat2() { + List l = new ArrayList<>(); + l.add(new IntWritable(0)); + l.add(new NDArrayWritable(Nd4j.arange(1, 9))); + l.add(new IntWritable(9)); + + RecordReader rr = new CollectionRecordReader(Collections.singletonList(l)); + DataSetIterator iter = new RecordReaderDataSetIterator(rr, 1); + + DataSet ds = iter.next(); + INDArray expF = Nd4j.create(new float[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, new int[]{1,10}); + + assertEquals(expF, ds.getFeatures()); + } + + @Test + public void testRecordReaderDataSetIteratorDisjointFeatures() { + + //Idea: input vector is like [f,f,f,f,l,l,f,f] or similar - i.e., label writables aren't start/end + + List l = Arrays.asList(new DoubleWritable(1), + new NDArrayWritable(Nd4j.create(new float[] {2, 3, 4}, new long[]{1,3})), new DoubleWritable(5), + new NDArrayWritable(Nd4j.create(new float[] {6, 7, 8}, new long[]{1,3}))); + + INDArray expF = Nd4j.create(new float[] {1, 6, 7, 8}, new long[]{1,4}); + INDArray expL = Nd4j.create(new float[] {2, 3, 4, 5}, new long[]{1,4}); + + RecordReader rr = new CollectionRecordReader(Collections.singletonList(l)); + + DataSetIterator iter = new RecordReaderDataSetIterator(rr, 1, 1, 2, true); + + DataSet ds = iter.next(); + assertEquals(expF, ds.getFeatures()); + assertEquals(expL, ds.getLabels()); + } + + @Test + public void testNormalizerPrefetchReset() throws Exception { + //Check NPE fix for: https://github.com/eclipse/deeplearning4j/issues/4214 + RecordReader csv = new CSVRecordReader(); + csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + int batchSize = 3; + + DataSetIterator iter = new RecordReaderDataSetIterator(csv, batchSize, 4, 4, true); + + DataNormalization normalizer = new NormalizerMinMaxScaler(0, 1); + normalizer.fit(iter); + iter.setPreProcessor(normalizer); + + iter.inputColumns(); //Prefetch + iter.totalOutcomes(); + iter.hasNext(); + iter.reset(); + iter.next(); + } + + @Test + public void testReadingFromStream() throws Exception { + + for(boolean b : new boolean[]{false, true}) { + int batchSize = 1; + int labelIndex = 4; + int numClasses = 3; + InputStream dataFile = Resources.asStream("iris.txt"); + RecordReader recordReader = new CSVRecordReader(0, ','); + recordReader.initialize(new InputStreamInputSplit(dataFile)); + + assertTrue(recordReader.hasNext()); + assertFalse(recordReader.resetSupported()); + + DataSetIterator iterator; + if(b){ + iterator = new RecordReaderDataSetIterator.Builder(recordReader, batchSize) + .classification(labelIndex, numClasses) + .build(); + } else { + iterator = new RecordReaderDataSetIterator(recordReader, batchSize, labelIndex, numClasses); + } + assertFalse(iterator.resetSupported()); + + int count = 0; + while (iterator.hasNext()) { + assertNotNull(iterator.next()); + count++; + } + + assertEquals(150, count); + + try { + iterator.reset(); + fail("Expected exception"); + } catch (Exception e) { + //expected + } + } + } + + + @Test + public void testImagesRRDSI() throws Exception { + File parentDir = temporaryFolder; + parentDir.deleteOnExit(); + String str1 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Zico/"); + String str2 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Ziwang_Xu/"); + + File f2 = new File(str2); + File f1 = new File(str1); + f1.mkdirs(); + f2.mkdirs(); + + TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f1.getPath(), "Zico_0001.jpg")), + new ClassPathResource("lfwtest/Zico/Zico_0001.jpg").getInputStream()); + TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f2.getPath(), "Ziwang_Xu_0001.jpg")), + new ClassPathResource("lfwtest/Ziwang_Xu/Ziwang_Xu_0001.jpg").getInputStream()); + + + Random r = new Random(12345); + ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator(); + + ImageRecordReader rr1 = new ImageRecordReader(28, 28, 3, labelMaker); + rr1.initialize(new FileSplit(parentDir)); + + + RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr1,2); + DataSet ds = rrdsi.next(); + assertArrayEquals(new long[]{2, 3, 28, 28}, ds.getFeatures().shape()); + assertArrayEquals(new long[]{2, 2}, ds.getLabels().shape()); + + + //Check the same thing via the builder: + rr1.reset(); + rrdsi = new RecordReaderDataSetIterator.Builder(rr1, 2) + .classification(1,2) + .build(); + + + ds = rrdsi.next(); + assertArrayEquals(new long[]{2, 3, 28, 28}, ds.getFeatures().shape()); + assertArrayEquals(new long[]{2, 2}, ds.getLabels().shape()); + } + + + + @Test + public void testSeqRRDSINoLabels(){ + List> sequence1 = new ArrayList<>(); + sequence1.add(Arrays.asList((Writable) new DoubleWritable(1), new DoubleWritable(2))); + sequence1.add(Arrays.asList((Writable) new DoubleWritable(3), new DoubleWritable(4))); + sequence1.add(Arrays.asList((Writable) new DoubleWritable(5), new DoubleWritable(6))); + List> sequence2 = new ArrayList<>(); + sequence2.add(Arrays.asList((Writable) new DoubleWritable(10), new DoubleWritable(20))); + sequence2.add(Arrays.asList((Writable) new DoubleWritable(30), new DoubleWritable(40))); + SequenceRecordReader rrFeatures = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); + + SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rrFeatures, 2, -1, -1); + + DataSet ds = iter.next(); + assertNotNull(ds.getFeatures()); + assertNull(ds.getLabels()); + } + + + @Test + public void testCollectMetaData(){ + RecordReaderDataSetIterator trainIter = new RecordReaderDataSetIterator.Builder(new CollectionRecordReader(Collections.>emptyList()), 1) + .collectMetaData(true) + .build(); + assertTrue(trainIter.isCollectMetaData()); + trainIter.setCollectMetaData(false); + assertFalse(trainIter.isCollectMetaData()); + trainIter.setCollectMetaData(true); + assertTrue(trainIter.isCollectMetaData()); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIteratorTest.java new file mode 100644 index 000000000..7b163892e --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIteratorTest.java @@ -0,0 +1,932 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.datavec; + + +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.datavec.api.conf.Configuration; +import org.datavec.api.io.labels.ParentPathLabelGenerator; +import org.datavec.api.records.Record; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.reader.BaseRecordReader; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; +import org.datavec.api.split.CollectionInputSplit; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.split.NumberedFileInputSplit; +import org.datavec.api.util.ndarray.RecordConverter; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Writable; +import org.datavec.image.recordreader.ImageRecordReader; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.common.resources.Resources; + +import java.io.*; +import java.net.URI; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.nd4j.linalg.indexing.NDArrayIndex.all; +import static org.nd4j.linalg.indexing.NDArrayIndex.interval; +import static org.nd4j.linalg.indexing.NDArrayIndex.point; + +@Timeout(300) +public class RecordReaderMultiDataSetIteratorTest extends BaseDL4JTest { + + @TempDir + public File temporaryFolder; + + @Test + public void testsBasic() throws Exception { + //Load details from CSV files; single input/output -> compare to RecordReaderDataSetIterator + RecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); + RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr, 10, 4, 3); + + RecordReader rr2 = new CSVRecordReader(0, ','); + rr2.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + MultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10).addReader("reader", rr2) + .addInput("reader", 0, 3).addOutputOneHot("reader", 4, 3).build(); + + while (rrdsi.hasNext()) { + DataSet ds = rrdsi.next(); + INDArray fds = ds.getFeatures(); + INDArray lds = ds.getLabels(); + + MultiDataSet mds = rrmdsi.next(); + assertEquals(1, mds.getFeatures().length); + assertEquals(1, mds.getLabels().length); + assertNull(mds.getFeaturesMaskArrays()); + assertNull(mds.getLabelsMaskArrays()); + INDArray fmds = mds.getFeatures(0); + INDArray lmds = mds.getLabels(0); + + assertNotNull(fmds); + assertNotNull(lmds); + + assertEquals(fds, fmds); + assertEquals(lds, lmds); + } + assertFalse(rrmdsi.hasNext()); + + //need to manually extract + File rootDir = temporaryFolder; + for (int i = 0; i < 3; i++) { + new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); + } + + //Load time series from CSV sequence files; compare to SequenceRecordReaderDataSetIterator + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iter = + new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); + + SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); + featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + MultiDataSetIterator srrmdsi = new RecordReaderMultiDataSetIterator.Builder(1) + .addSequenceReader("in", featureReader2).addSequenceReader("out", labelReader2).addInput("in") + .addOutputOneHot("out", 0, 4).build(); + + while (iter.hasNext()) { + DataSet ds = iter.next(); + INDArray fds = ds.getFeatures(); + INDArray lds = ds.getLabels(); + + MultiDataSet mds = srrmdsi.next(); + assertEquals(1, mds.getFeatures().length); + assertEquals(1, mds.getLabels().length); + assertNull(mds.getFeaturesMaskArrays()); + assertNull(mds.getLabelsMaskArrays()); + INDArray fmds = mds.getFeatures(0); + INDArray lmds = mds.getLabels(0); + + assertNotNull(fmds); + assertNotNull(lmds); + + assertEquals(fds, fmds); + assertEquals(lds, lmds); + } + assertFalse(srrmdsi.hasNext()); + } + + @Test + public void testsBasicMeta() throws Exception { + //As per testBasic - but also loading metadata + RecordReader rr2 = new CSVRecordReader(0, ','); + rr2.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10) + .addReader("reader", rr2).addInput("reader", 0, 3).addOutputOneHot("reader", 4, 3).build(); + + rrmdsi.setCollectMetaData(true); + + int count = 0; + while (rrmdsi.hasNext()) { + MultiDataSet mds = rrmdsi.next(); + MultiDataSet fromMeta = rrmdsi.loadFromMetaData(mds.getExampleMetaData(RecordMetaData.class)); + assertEquals(mds, fromMeta); + count++; + } + assertEquals(150 / 10, count); + } + + @Test + public void testSplittingCSV() throws Exception { + //Here's the idea: take Iris, and split it up into 2 inputs and 2 output arrays + //Inputs: columns 0 and 1-2 + //Outputs: columns 3, and 4->OneHot + //need to manually extract + RecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); + RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr, 10, 4, 3); + + RecordReader rr2 = new CSVRecordReader(0, ','); + rr2.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + MultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10).addReader("reader", rr2) + .addInput("reader", 0, 0).addInput("reader", 1, 2).addOutput("reader", 3, 3) + .addOutputOneHot("reader", 4, 3).build(); + + while (rrdsi.hasNext()) { + DataSet ds = rrdsi.next(); + INDArray fds = ds.getFeatures(); + INDArray lds = ds.getLabels(); + + MultiDataSet mds = rrmdsi.next(); + assertEquals(2, mds.getFeatures().length); + assertEquals(2, mds.getLabels().length); + assertNull(mds.getFeaturesMaskArrays()); + assertNull(mds.getLabelsMaskArrays()); + INDArray[] fmds = mds.getFeatures(); + INDArray[] lmds = mds.getLabels(); + + assertNotNull(fmds); + assertNotNull(lmds); + for (int i = 0; i < fmds.length; i++) + assertNotNull(fmds[i]); + for (int i = 0; i < lmds.length; i++) + assertNotNull(lmds[i]); + + //Get the subsets of the original iris data + INDArray expIn1 = fds.get(all(), interval(0,0,true)); + INDArray expIn2 = fds.get(all(), interval(1, 2, true)); + INDArray expOut1 = fds.get(all(), interval(3,3,true)); + INDArray expOut2 = lds; + + assertEquals(expIn1, fmds[0]); + assertEquals(expIn2, fmds[1]); + assertEquals(expOut1, lmds[0]); + assertEquals(expOut2, lmds[1]); + } + assertFalse(rrmdsi.hasNext()); + } + + @Test + public void testSplittingCSVMeta() throws Exception { + //Here's the idea: take Iris, and split it up into 2 inputs and 2 output arrays + //Inputs: columns 0 and 1-2 + //Outputs: columns 3, and 4->OneHot + RecordReader rr2 = new CSVRecordReader(0, ','); + rr2.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10) + .addReader("reader", rr2).addInput("reader", 0, 0).addInput("reader", 1, 2) + .addOutput("reader", 3, 3).addOutputOneHot("reader", 4, 3).build(); + rrmdsi.setCollectMetaData(true); + + int count = 0; + while (rrmdsi.hasNext()) { + MultiDataSet mds = rrmdsi.next(); + MultiDataSet fromMeta = rrmdsi.loadFromMetaData(mds.getExampleMetaData(RecordMetaData.class)); + assertEquals(mds, fromMeta); + count++; + } + assertEquals(150 / 10, count); + } + + @Test + public void testSplittingCSVSequence() throws Exception { + //Idea: take CSV sequences, and split "csvsequence_i.txt" into two separate inputs; keep "csvSequencelables_i.txt" + // as standard one-hot output + //need to manually extract + File rootDir = temporaryFolder; + for (int i = 0; i < 3; i++) { + new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); + } + + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iter = + new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); + + SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); + featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + MultiDataSetIterator srrmdsi = new RecordReaderMultiDataSetIterator.Builder(1) + .addSequenceReader("seq1", featureReader2).addSequenceReader("seq2", labelReader2) + .addInput("seq1", 0, 1).addInput("seq1", 2, 2).addOutputOneHot("seq2", 0, 4).build(); + + while (iter.hasNext()) { + DataSet ds = iter.next(); + INDArray fds = ds.getFeatures(); + INDArray lds = ds.getLabels(); + + MultiDataSet mds = srrmdsi.next(); + assertEquals(2, mds.getFeatures().length); + assertEquals(1, mds.getLabels().length); + assertNull(mds.getFeaturesMaskArrays()); + assertNull(mds.getLabelsMaskArrays()); + INDArray[] fmds = mds.getFeatures(); + INDArray[] lmds = mds.getLabels(); + + assertNotNull(fmds); + assertNotNull(lmds); + for (int i = 0; i < fmds.length; i++) + assertNotNull(fmds[i]); + for (int i = 0; i < lmds.length; i++) + assertNotNull(lmds[i]); + + INDArray expIn1 = fds.get(all(), NDArrayIndex.interval(0, 1, true), all()); + INDArray expIn2 = fds.get(all(), NDArrayIndex.interval(2, 2, true), all()); + + assertEquals(expIn1, fmds[0]); + assertEquals(expIn2, fmds[1]); + assertEquals(lds, lmds[0]); + } + assertFalse(srrmdsi.hasNext()); + } + + @Test + public void testSplittingCSVSequenceMeta() throws Exception { + //Idea: take CSV sequences, and split "csvsequence_i.txt" into two separate inputs; keep "csvSequencelables_i.txt" + // as standard one-hot output + //need to manually extract + File rootDir = temporaryFolder; + for (int i = 0; i < 3; i++) { + new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); + } + + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); + featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + RecordReaderMultiDataSetIterator srrmdsi = new RecordReaderMultiDataSetIterator.Builder(1) + .addSequenceReader("seq1", featureReader2).addSequenceReader("seq2", labelReader2) + .addInput("seq1", 0, 1).addInput("seq1", 2, 2).addOutputOneHot("seq2", 0, 4).build(); + + srrmdsi.setCollectMetaData(true); + + int count = 0; + while (srrmdsi.hasNext()) { + MultiDataSet mds = srrmdsi.next(); + MultiDataSet fromMeta = srrmdsi.loadFromMetaData(mds.getExampleMetaData(RecordMetaData.class)); + assertEquals(mds, fromMeta); + count++; + } + assertEquals(3, count); + } + + + @Test + public void testInputValidation() { + + //Test: no readers + try { + MultiDataSetIterator r = new RecordReaderMultiDataSetIterator.Builder(1).addInput("something") + .addOutput("something").build(); + fail("Should have thrown exception"); + } catch (Exception e) { + } + + //Test: reference to reader that doesn't exist + try { + RecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + MultiDataSetIterator r = new RecordReaderMultiDataSetIterator.Builder(1).addReader("iris", rr) + .addInput("thisDoesntExist", 0, 3).addOutputOneHot("iris", 4, 3).build(); + fail("Should have thrown exception"); + } catch (Exception e) { + } + + //Test: no inputs or outputs + try { + RecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + MultiDataSetIterator r = new RecordReaderMultiDataSetIterator.Builder(1).addReader("iris", rr).build(); + fail("Should have thrown exception"); + } catch (Exception e) { + } + } + + @Test + public void testVariableLengthTS() throws Exception { + //need to manually extract + File rootDir = temporaryFolder; + for (int i = 0; i < 3; i++) { + new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); + } + + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabelsShort_%d.txt"); + + //Set up SequenceRecordReaderDataSetIterators for comparison + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); + featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iterAlignStart = new SequenceRecordReaderDataSetIterator(featureReader, + labelReader, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_START); + + SequenceRecordReaderDataSetIterator iterAlignEnd = new SequenceRecordReaderDataSetIterator(featureReader2, + labelReader2, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); + + + //Set up + SequenceRecordReader featureReader3 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader3 = new CSVSequenceRecordReader(1, ","); + featureReader3.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader3.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReader featureReader4 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader4 = new CSVSequenceRecordReader(1, ","); + featureReader4.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader4.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + RecordReaderMultiDataSetIterator rrmdsiStart = new RecordReaderMultiDataSetIterator.Builder(1) + .addSequenceReader("in", featureReader3).addSequenceReader("out", labelReader3).addInput("in") + .addOutputOneHot("out", 0, 4) + .sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_START).build(); + + RecordReaderMultiDataSetIterator rrmdsiEnd = new RecordReaderMultiDataSetIterator.Builder(1) + .addSequenceReader("in", featureReader4).addSequenceReader("out", labelReader4).addInput("in") + .addOutputOneHot("out", 0, 4) + .sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_END).build(); + + + while (iterAlignStart.hasNext()) { + DataSet dsStart = iterAlignStart.next(); + DataSet dsEnd = iterAlignEnd.next(); + + MultiDataSet mdsStart = rrmdsiStart.next(); + MultiDataSet mdsEnd = rrmdsiEnd.next(); + + assertEquals(1, mdsStart.getFeatures().length); + assertEquals(1, mdsStart.getLabels().length); + //assertEquals(1, mdsStart.getFeaturesMaskArrays().length); //Features data is always longer -> don't need mask arrays for it + assertEquals(1, mdsStart.getLabelsMaskArrays().length); + + assertEquals(1, mdsEnd.getFeatures().length); + assertEquals(1, mdsEnd.getLabels().length); + //assertEquals(1, mdsEnd.getFeaturesMaskArrays().length); + assertEquals(1, mdsEnd.getLabelsMaskArrays().length); + + + assertEquals(dsStart.getFeatures(), mdsStart.getFeatures(0)); + assertEquals(dsStart.getLabels(), mdsStart.getLabels(0)); + assertEquals(dsStart.getLabelsMaskArray(), mdsStart.getLabelsMaskArray(0)); + + assertEquals(dsEnd.getFeatures(), mdsEnd.getFeatures(0)); + assertEquals(dsEnd.getLabels(), mdsEnd.getLabels(0)); + assertEquals(dsEnd.getLabelsMaskArray(), mdsEnd.getLabelsMaskArray(0)); + } + assertFalse(rrmdsiStart.hasNext()); + assertFalse(rrmdsiEnd.hasNext()); + } + + + @Test + public void testVariableLengthTSMeta() throws Exception { + //need to manually extract + File rootDir = temporaryFolder; + for (int i = 0; i < 3; i++) { + new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); + new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); + } + //Set up SequenceRecordReaderDataSetIterators for comparison + + String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabelsShort_%d.txt"); + + //Set up + SequenceRecordReader featureReader3 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader3 = new CSVSequenceRecordReader(1, ","); + featureReader3.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader3.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReader featureReader4 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader4 = new CSVSequenceRecordReader(1, ","); + featureReader4.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader4.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + RecordReaderMultiDataSetIterator rrmdsiStart = new RecordReaderMultiDataSetIterator.Builder(1) + .addSequenceReader("in", featureReader3).addSequenceReader("out", labelReader3).addInput("in") + .addOutputOneHot("out", 0, 4) + .sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_START).build(); + + RecordReaderMultiDataSetIterator rrmdsiEnd = new RecordReaderMultiDataSetIterator.Builder(1) + .addSequenceReader("in", featureReader4).addSequenceReader("out", labelReader4).addInput("in") + .addOutputOneHot("out", 0, 4) + .sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_END).build(); + + rrmdsiStart.setCollectMetaData(true); + rrmdsiEnd.setCollectMetaData(true); + + int count = 0; + while (rrmdsiStart.hasNext()) { + MultiDataSet mdsStart = rrmdsiStart.next(); + MultiDataSet mdsEnd = rrmdsiEnd.next(); + + MultiDataSet mdsStartFromMeta = + rrmdsiStart.loadFromMetaData(mdsStart.getExampleMetaData(RecordMetaData.class)); + MultiDataSet mdsEndFromMeta = rrmdsiEnd.loadFromMetaData(mdsEnd.getExampleMetaData(RecordMetaData.class)); + + assertEquals(mdsStart, mdsStartFromMeta); + assertEquals(mdsEnd, mdsEndFromMeta); + + count++; + } + assertFalse(rrmdsiStart.hasNext()); + assertFalse(rrmdsiEnd.hasNext()); + assertEquals(3, count); + } + + @Test + public void testImagesRRDMSI() throws Exception { + File parentDir = temporaryFolder; + parentDir.deleteOnExit(); + String str1 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Zico/"); + String str2 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Ziwang_Xu/"); + + File f1 = new File(str1); + File f2 = new File(str2); + f1.mkdirs(); + f2.mkdirs(); + + TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f1.getPath(), "Zico_0001.jpg")), + new ClassPathResource("lfwtest/Zico/Zico_0001.jpg").getInputStream()); + TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f2.getPath(), "Ziwang_Xu_0001.jpg")), + new ClassPathResource("lfwtest/Ziwang_Xu/Ziwang_Xu_0001.jpg").getInputStream()); + + + int outputNum = 2; + Random r = new Random(12345); + ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator(); + + ImageRecordReader rr1 = new ImageRecordReader(10, 10, 1, labelMaker); + ImageRecordReader rr1s = new ImageRecordReader(5, 5, 1, labelMaker); + + rr1.initialize(new FileSplit(parentDir)); + rr1s.initialize(new FileSplit(parentDir)); + + + MultiDataSetIterator trainDataIterator = new RecordReaderMultiDataSetIterator.Builder(1).addReader("rr1", rr1) + .addReader("rr1s", rr1s).addInput("rr1", 0, 0).addInput("rr1s", 0, 0) + .addOutputOneHot("rr1s", 1, outputNum).build(); + + //Now, do the same thing with ImageRecordReader, and check we get the same results: + ImageRecordReader rr1_b = new ImageRecordReader(10, 10, 1, labelMaker); + ImageRecordReader rr1s_b = new ImageRecordReader(5, 5, 1, labelMaker); + rr1_b.initialize(new FileSplit(parentDir)); + rr1s_b.initialize(new FileSplit(parentDir)); + + DataSetIterator dsi1 = new RecordReaderDataSetIterator(rr1_b, 1, 1, 2); + DataSetIterator dsi2 = new RecordReaderDataSetIterator(rr1s_b, 1, 1, 2); + + for (int i = 0; i < 2; i++) { + MultiDataSet mds = trainDataIterator.next(); + + DataSet d1 = dsi1.next(); + DataSet d2 = dsi2.next(); + + assertEquals(d1.getFeatures(), mds.getFeatures(0)); + assertEquals(d2.getFeatures(), mds.getFeatures(1)); + assertEquals(d1.getLabels(), mds.getLabels(0)); + } + } + + @Test + public void testImagesRRDMSI_Batched() throws Exception { + File parentDir = temporaryFolder; + parentDir.deleteOnExit(); + String str1 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Zico/"); + String str2 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Ziwang_Xu/"); + + File f1 = new File(str1); + File f2 = new File(str2); + f1.mkdirs(); + f2.mkdirs(); + + TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f1.getPath(), "Zico_0001.jpg")), + new ClassPathResource("lfwtest/Zico/Zico_0001.jpg").getInputStream()); + TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f2.getPath(), "Ziwang_Xu_0001.jpg")), + new ClassPathResource("lfwtest/Ziwang_Xu/Ziwang_Xu_0001.jpg").getInputStream()); + + int outputNum = 2; + ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator(); + + ImageRecordReader rr1 = new ImageRecordReader(10, 10, 1, labelMaker); + ImageRecordReader rr1s = new ImageRecordReader(5, 5, 1, labelMaker); + + URI[] uris = new FileSplit(parentDir).locations(); + + rr1.initialize(new CollectionInputSplit(uris)); + rr1s.initialize(new CollectionInputSplit(uris)); + + MultiDataSetIterator trainDataIterator = new RecordReaderMultiDataSetIterator.Builder(2).addReader("rr1", rr1) + .addReader("rr1s", rr1s).addInput("rr1", 0, 0).addInput("rr1s", 0, 0) + .addOutputOneHot("rr1s", 1, outputNum).build(); + + //Now, do the same thing with ImageRecordReader, and check we get the same results: + ImageRecordReader rr1_b = new ImageRecordReader(10, 10, 1, labelMaker); + ImageRecordReader rr1s_b = new ImageRecordReader(5, 5, 1, labelMaker); + rr1_b.initialize(new FileSplit(parentDir)); + rr1s_b.initialize(new FileSplit(parentDir)); + + DataSetIterator dsi1 = new RecordReaderDataSetIterator(rr1_b, 2, 1, 2); + DataSetIterator dsi2 = new RecordReaderDataSetIterator(rr1s_b, 2, 1, 2); + + MultiDataSet mds = trainDataIterator.next(); + + DataSet d1 = dsi1.next(); + DataSet d2 = dsi2.next(); + + assertEquals(d1.getFeatures(), mds.getFeatures(0)); + assertEquals(d2.getFeatures(), mds.getFeatures(1)); + assertEquals(d1.getLabels(), mds.getLabels(0)); + + //Check label assignment: + + File currentFile = rr1_b.getCurrentFile(); + INDArray expLabels; + if(currentFile.getAbsolutePath().contains("Zico")){ + expLabels = Nd4j.create(new double[][] {{0, 1}, {1, 0}}); + } else { + expLabels = Nd4j.create(new double[][] {{1, 0}, {0, 1}}); + } + + assertEquals(expLabels, d1.getLabels()); + assertEquals(expLabels, d2.getLabels()); + } + + + + + @Test + public void testTimeSeriesRandomOffset() { + //2 in, 2 out, 3 total sequences of length [1,3,5] + + List> seq1 = + Arrays.asList(Arrays.asList(new DoubleWritable(1.0), new DoubleWritable(2.0))); + List> seq2 = + Arrays.asList(Arrays.asList(new DoubleWritable(10.0), new DoubleWritable(11.0)), + Arrays.asList(new DoubleWritable(20.0), new DoubleWritable(21.0)), + Arrays.asList(new DoubleWritable(30.0), new DoubleWritable(31.0))); + List> seq3 = + Arrays.asList(Arrays.asList(new DoubleWritable(100.0), new DoubleWritable(101.0)), + Arrays.asList(new DoubleWritable(200.0), new DoubleWritable(201.0)), + Arrays.asList(new DoubleWritable(300.0), new DoubleWritable(301.0)), + Arrays.asList(new DoubleWritable(400.0), new DoubleWritable(401.0)), + Arrays.asList(new DoubleWritable(500.0), new DoubleWritable(501.0))); + + Collection>> seqs = Arrays.asList(seq1, seq2, seq3); + + SequenceRecordReader rr = new CollectionSequenceRecordReader(seqs); + + RecordReaderMultiDataSetIterator rrmdsi = + new RecordReaderMultiDataSetIterator.Builder(3).addSequenceReader("rr", rr).addInput("rr", 0, 0) + .addOutput("rr", 1, 1).timeSeriesRandomOffset(true, 1234L).build(); + + + Random r = new Random(1234); //Provides seed for each minibatch + long seed = r.nextLong(); + Random r2 = new Random(seed); //Use same RNG seed in new RNG for each minibatch + int expOffsetSeq1 = r2.nextInt(5 - 1 + 1); //0 to 4 inclusive + int expOffsetSeq2 = r2.nextInt(5 - 3 + 1); + int expOffsetSeq3 = 0; //Longest TS, always 0 + //With current seed: 3, 1, 0 + // System.out.println(expOffsetSeq1 + "\t" + expOffsetSeq2 + "\t" + expOffsetSeq3); + + MultiDataSet mds = rrmdsi.next(); + + INDArray expMask = Nd4j.create(new double[][] {{0, 0, 0, 1, 0}, {0, 1, 1, 1, 0}, {1, 1, 1, 1, 1}}); + + assertEquals(expMask, mds.getFeaturesMaskArray(0)); + assertEquals(expMask, mds.getLabelsMaskArray(0)); + + INDArray f = mds.getFeatures(0); + INDArray l = mds.getLabels(0); + + INDArray expF1 = Nd4j.create(new double[] {1.0}, new int[]{1,1}); + INDArray expL1 = Nd4j.create(new double[] {2.0}, new int[]{1,1}); + + INDArray expF2 = Nd4j.create(new double[] {10, 20, 30}, new int[]{1,3}); + INDArray expL2 = Nd4j.create(new double[] {11, 21, 31}, new int[]{1,3}); + + INDArray expF3 = Nd4j.create(new double[] {100, 200, 300, 400, 500}, new int[]{1,5}); + INDArray expL3 = Nd4j.create(new double[] {101, 201, 301, 401, 501}, new int[]{1,5}); + + assertEquals(expF1, f.get(point(0), all(), + NDArrayIndex.interval(expOffsetSeq1, expOffsetSeq1 + 1))); + assertEquals(expL1, l.get(point(0), all(), + NDArrayIndex.interval(expOffsetSeq1, expOffsetSeq1 + 1))); + + assertEquals(expF2, f.get(point(1), all(), + NDArrayIndex.interval(expOffsetSeq2, expOffsetSeq2 + 3))); + assertEquals(expL2, l.get(point(1), all(), + NDArrayIndex.interval(expOffsetSeq2, expOffsetSeq2 + 3))); + + assertEquals(expF3, f.get(point(2), all(), + NDArrayIndex.interval(expOffsetSeq3, expOffsetSeq3 + 5))); + assertEquals(expL3, l.get(point(2), all(), + NDArrayIndex.interval(expOffsetSeq3, expOffsetSeq3 + 5))); + } + + + @Test + public void testSeqRRDSIMasking(){ + //This also tests RecordReaderMultiDataSetIterator, by virtue of + List>> features = new ArrayList<>(); + List>> labels = new ArrayList<>(); + + features.add(Arrays.asList(l(new DoubleWritable(1)), l(new DoubleWritable(2)), l(new DoubleWritable(3)))); + features.add(Arrays.asList(l(new DoubleWritable(4)), l(new DoubleWritable(5)))); + + labels.add(Arrays.asList(l(new IntWritable(0)))); + labels.add(Arrays.asList(l(new IntWritable(1)))); + + CollectionSequenceRecordReader fR = new CollectionSequenceRecordReader(features); + CollectionSequenceRecordReader lR = new CollectionSequenceRecordReader(labels); + + SequenceRecordReaderDataSetIterator seqRRDSI = new SequenceRecordReaderDataSetIterator( + fR, lR, 2, 2, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); + + DataSet ds = seqRRDSI.next(); + + INDArray fMask = Nd4j.create(new double[][]{ + {1,1,1}, + {1,1,0}}); + + INDArray lMask = Nd4j.create(new double[][]{ + {0,0,1}, + {0,1,0}}); + + assertEquals(fMask, ds.getFeaturesMaskArray()); + assertEquals(lMask, ds.getLabelsMaskArray()); + + INDArray f = Nd4j.create(new double[][]{ + {1,2,3}, + {4,5,0}}); + + INDArray l = Nd4j.create(2,2,3); + l.putScalar(0,0,2, 1.0); + l.putScalar(1,1,1, 1.0); + + assertEquals(f, ds.getFeatures().get(all(), point(0), all())); + assertEquals(l, ds.getLabels()); + } + + private static List l(Writable... in){ + return Arrays.asList(in); + } + + + + @Test + public void testExcludeStringColCSV() throws Exception { + File csvFile = temporaryFolder; + + StringBuilder sb = new StringBuilder(); + for(int i=1; i<=10; i++ ){ + if(i > 1){ + sb.append("\n"); + } + sb.append("skip_").append(i).append(",").append(i).append(",").append(i + 0.5); + } + FileUtils.writeStringToFile(csvFile, sb.toString()); + + RecordReader rr = new CSVRecordReader(); + rr.initialize(new FileSplit(csvFile)); + + RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10) + .addReader("rr", rr) + .addInput("rr", 1, 1) + .addOutput("rr", 2, 2) + .build(); + + INDArray expFeatures = Nd4j.linspace(1,10,10).reshape(1,10).transpose(); + INDArray expLabels = Nd4j.linspace(1,10,10).addi(0.5).reshape(1,10).transpose(); + + MultiDataSet mds = rrmdsi.next(); + assertFalse(rrmdsi.hasNext()); + + assertEquals(expFeatures, mds.getFeatures(0).castTo(expFeatures.dataType())); + assertEquals(expLabels, mds.getLabels(0).castTo(expLabels.dataType())); + } + + + private static final int nX = 32; + private static final int nY = 32; + private static final int nZ = 28; + + + @Test + public void testRRMDSI5D() { + int batchSize = 5; + + CustomRecordReader recordReader = new CustomRecordReader(); + DataSetIterator dataIter = new RecordReaderDataSetIterator(recordReader, batchSize, + 1, /* Index of label in records */ + 2 /* number of different labels */); + + int count = 0; + while(dataIter.hasNext()){ + DataSet ds = dataIter.next(); + + int offset = 5*count; + for( int i=0; i<5; i++ ){ + INDArray act = ds.getFeatures().get(interval(i,i,true), all(), all(), all(), all()); + INDArray exp = Nd4j.valueArrayOf(new int[]{1, 1, nZ, nX, nY}, i + offset ); + assertEquals(exp, act); + } + count++; + } + + assertEquals(2, count); + } + + + static class CustomRecordReader extends BaseRecordReader { + + int n = 0; + + CustomRecordReader() { } + + @Override + public boolean batchesSupported() { + return false; + } + + @Override + public List> next(int num) { + throw new RuntimeException("Not implemented"); + } + + @Override + public List next() { + INDArray nd = Nd4j.create(new float[nZ*nY*nX], new int[] {1, 1, nZ, nY, nX }, 'c').assign(n); + final Listres = RecordConverter.toRecord(nd); + res.add(new IntWritable(0)); + n++; + return res; + } + + @Override + public boolean hasNext() { + return n<10; + } + + final static ArrayList labels = new ArrayList<>(2); + static { + labels.add("lbl0"); + labels.add("lbl1"); + } + @Override + public List getLabels() { + return labels; + } + + @Override + public void reset() { + n = 0; + } + + @Override + public boolean resetSupported() { + return true; + } + + @Override + public List record(URI uri, DataInputStream dataInputStream) { + return next(); + } + + @Override + public Record nextRecord() { + List r = next(); + return new org.datavec.api.records.impl.Record(r, null); + } + + @Override + public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException { + throw new RuntimeException("Not implemented"); + } + + @Override + public List loadFromMetaData(List recordMetaDatas) { + throw new RuntimeException("Not implemented"); + } + + @Override + public void close() { + } + + @Override + public void setConf(Configuration conf) { + } + + @Override + public Configuration getConf() { + return null; + } + + @Override + public void initialize(InputSplit split) { + n = 0; + } + @Override + public void initialize(Configuration conf, InputSplit split) { + n = 0; + } + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/tools/SpecialImageRecordReader.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/datavec/tools/SpecialImageRecordReader.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/tools/SpecialImageRecordReader.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/datavec/tools/SpecialImageRecordReader.java diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java new file mode 100644 index 000000000..f85c1fdf6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java @@ -0,0 +1,56 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.fetchers; + +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; + + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + + +/** + * @author saudet + */ +public class SvhnDataFetcherTest extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return 480_000_000L; //Shouldn't take this long but slow download or drive access on CI machines may need extra time. + } + + @Test + public void testSvhnDataFetcher() throws Exception { + assumeTrue(isIntegrationTests()); //Ignore unless integration tests - CI can get caught up on slow disk access + + SvhnDataFetcher fetch = new SvhnDataFetcher(); + File path = fetch.getDataSetPath(DataSetType.TRAIN); + File path2 = fetch.getDataSetPath(DataSetType.TEST); + File path3 = fetch.getDataSetPath(DataSetType.VALIDATION); + + assertTrue(path.isDirectory()); + assertTrue(path2.isDirectory()); + assertTrue(path3.isDirectory()); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIteratorTest.java similarity index 89% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIteratorTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIteratorTest.java index af42f61ea..a67078e8a 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIteratorTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIteratorTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.datasets.iterator; import org.apache.commons.lang3.RandomUtils; @@ -25,42 +26,43 @@ import org.junit.jupiter.api.Test; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.common.primitives.Pair; + import java.util.Iterator; import java.util.concurrent.atomic.AtomicInteger; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Abstract Data Set Iterator Test") -class AbstractDataSetIteratorTest extends BaseDL4JTest { +public class AbstractDataSetIteratorTest extends BaseDL4JTest { @Test - @DisplayName("Next") - void next() throws Exception { + public void next() throws Exception { int numFeatures = 128; int batchSize = 10; int numRows = 1000; AtomicInteger cnt = new AtomicInteger(0); FloatsDataSetIterator iterator = new FloatsDataSetIterator(floatIterable(numRows, numFeatures), batchSize); + assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { DataSet dataSet = iterator.next(); + INDArray features = dataSet.getFeatures(); + assertEquals(batchSize, features.rows()); assertEquals(numFeatures, features.columns()); cnt.incrementAndGet(); } + assertEquals(numRows / batchSize, cnt.get()); } + protected static Iterable> floatIterable(final int totalRows, final int numColumns) { return new Iterable>() { - @Override public Iterator> iterator() { return new Iterator>() { - private AtomicInteger cnt = new AtomicInteger(0); @Override @@ -70,8 +72,8 @@ class AbstractDataSetIteratorTest extends BaseDL4JTest { @Override public Pair next() { - float[] features = new float[numColumns]; - float[] labels = new float[numColumns]; + float features[] = new float[numColumns]; + float labels[] = new float[numColumns]; for (int i = 0; i < numColumns; i++) { features[i] = (float) i; labels[i] = RandomUtils.nextFloat(0, 5); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIteratorTest.java new file mode 100644 index 000000000..e999aee23 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIteratorTest.java @@ -0,0 +1,238 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.iterator; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.callbacks.InterleavedDataSetCallback; +import org.deeplearning4j.datasets.iterator.tools.VariableTimeseriesGenerator; +import org.deeplearning4j.nn.util.TestDataSetConsumer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +@Slf4j +public class AsyncDataSetIteratorTest extends BaseDL4JTest { + private ExistingDataSetIterator backIterator; + private static final int TEST_SIZE = 100; + private static final int ITERATIONS = 10; + + // time spent in consumer thread, milliseconds + private static final long EXECUTION_TIME = 5; + private static final long EXECUTION_SMALL = 1; + + @BeforeEach + public void setUp() throws Exception { + List iterable = new ArrayList<>(); + for (int i = 0; i < TEST_SIZE; i++) { + iterable.add(new DataSet(Nd4j.create(new float[100]), Nd4j.create(new float[10]))); + } + + backIterator = new ExistingDataSetIterator(iterable); + } + + @Test + public void hasNext1() throws Exception { + for (int iter = 0; iter < ITERATIONS; iter++) { + for (int prefetchSize = 2; prefetchSize <= 8; prefetchSize++) { + AsyncDataSetIterator iterator = new AsyncDataSetIterator(backIterator, prefetchSize); + int cnt = 0; + while (iterator.hasNext()) { + DataSet ds = iterator.next(); + + assertNotEquals(null, ds); + cnt++; + } + + assertEquals(TEST_SIZE, cnt, "Failed on iteration: " + iter + ", prefetchSize: " + prefetchSize); + iterator.shutdown(); + } + } + } + + @Test + public void hasNextWithResetAndLoad() throws Exception { + int[] prefetchSizes; + if(isIntegrationTests()){ + prefetchSizes = new int[]{2, 3, 4, 5, 6, 7, 8}; + } else { + prefetchSizes = new int[]{2, 3, 8}; + } + + + for (int iter = 0; iter < ITERATIONS; iter++) { + for(int prefetchSize : prefetchSizes){ + AsyncDataSetIterator iterator = new AsyncDataSetIterator(backIterator, prefetchSize); + TestDataSetConsumer consumer = new TestDataSetConsumer(EXECUTION_SMALL); + int cnt = 0; + while (iterator.hasNext()) { + DataSet ds = iterator.next(); + consumer.consumeOnce(ds, false); + + cnt++; + if (cnt == TEST_SIZE / 2) + iterator.reset(); + } + + assertEquals(TEST_SIZE + (TEST_SIZE / 2), cnt); + iterator.shutdown(); + } + } + } + + + @Test + public void testWithLoad() { + + for (int iter = 0; iter < ITERATIONS; iter++) { + AsyncDataSetIterator iterator = new AsyncDataSetIterator(backIterator, 8); + TestDataSetConsumer consumer = new TestDataSetConsumer(iterator, EXECUTION_TIME); + + consumer.consumeWhileHasNext(true); + + assertEquals(TEST_SIZE, consumer.getCount()); + iterator.shutdown(); + } + } + + @Test + public void testWithException() { + Assertions.assertThrows(ArrayIndexOutOfBoundsException.class, () -> { + ExistingDataSetIterator crashingIterator = new ExistingDataSetIterator(new IterableWithException(100)); + AsyncDataSetIterator iterator = new AsyncDataSetIterator(crashingIterator, 8); + + TestDataSetConsumer consumer = new TestDataSetConsumer(iterator, EXECUTION_SMALL); + consumer.consumeWhileHasNext(true); + iterator.shutdown(); + }); + } + + + + private class IterableWithException implements Iterable { + private final AtomicLong counter = new AtomicLong(0); + private final int crashIteration; + + public IterableWithException(int iteration) { + crashIteration = iteration; + } + + @Override + public Iterator iterator() { + counter.set(0); + return new Iterator() { + @Override + public boolean hasNext() { + return true; + } + + @Override + public DataSet next() { + if (counter.incrementAndGet() >= crashIteration) + throw new ArrayIndexOutOfBoundsException("Thrown as expected"); + + return new DataSet(Nd4j.create(10), Nd4j.create(10)); + } + + @Override + public void remove() { + + } + }; + } + } + + + @Test + public void testVariableTimeSeries1() throws Exception { + int numBatches = isIntegrationTests() ? 1000 : 100; + int batchSize = isIntegrationTests() ? 32 : 8; + int timeStepsMin = 10; + int timeStepsMax = isIntegrationTests() ? 500 : 100; + int valuesPerTimestep = isIntegrationTests() ? 128 : 16; + + AsyncDataSetIterator adsi = new AsyncDataSetIterator( + new VariableTimeseriesGenerator(1192, numBatches, batchSize, valuesPerTimestep, timeStepsMin, timeStepsMax, 10), 2, true); + + for (int e = 0; e < 10; e++) { + int cnt = 0; + while (adsi.hasNext()) { + DataSet ds = adsi.next(); + + //log.info("Features ptr: {}", AtomicAllocator.getInstance().getPointer(mds.getFeatures()[0].data()).address()); + assertEquals((double) cnt, ds.getFeatures().meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals( (double) cnt + 0.25, + ds.getLabels().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.5, + ds.getFeaturesMaskArray().meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.75, + ds.getLabelsMaskArray().meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + + cnt++; + } + + adsi.reset(); +// log.info("Epoch {} finished...", e); + } + } + + @Test + public void testVariableTimeSeries2() throws Exception { + AsyncDataSetIterator adsi = + new AsyncDataSetIterator(new VariableTimeseriesGenerator(1192, 100, 32, 128, 100, 100, 100), 2, + true, new InterleavedDataSetCallback(2 * 2)); + + + for (int e = 0; e < 5; e++) { + int cnt = 0; + while (adsi.hasNext()) { + + DataSet ds = adsi.next(); + ds.detach(); + + //log.info("Features ptr: {}", AtomicAllocator.getInstance().getPointer(mds.getFeatures()[0].data()).address()); + assertEquals((double) cnt, + ds.getFeatures().meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.25, + ds.getLabels().meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.5, + ds.getFeaturesMaskArray().meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.75, + ds.getLabelsMaskArray().meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + + cnt++; + } + + adsi.reset(); +// log.info("Epoch {} finished...", e); + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIteratorTest.java new file mode 100644 index 000000000..2952382b7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIteratorTest.java @@ -0,0 +1,165 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.iterator; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.tools.VariableMultiTimeseriesGenerator; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +public class AsyncMultiDataSetIteratorTest extends BaseDL4JTest { + + /** + * THIS TEST SHOULD BE ALWAYS RUN WITH DOUBLE PRECISION, WITHOUT ANY EXCLUSIONS + * + * @throws Exception + */ + @Test + public void testVariableTimeSeries1() throws Exception { + int numBatches = isIntegrationTests() ? 1000 : 100; + int batchSize = isIntegrationTests() ? 32 : 8; + int timeStepsMin = 10; + int timeStepsMax = isIntegrationTests() ? 500 : 100; + int valuesPerTimestep = isIntegrationTests() ? 128 : 16; + + val iterator = new VariableMultiTimeseriesGenerator(1192, numBatches, batchSize, valuesPerTimestep, timeStepsMin, timeStepsMax, 10); + iterator.reset(); + iterator.hasNext(); + val amdsi = new AsyncMultiDataSetIterator(iterator, 2, true); + + for (int e = 0; e < 10; e++) { + int cnt = 0; + while (amdsi.hasNext()) { + MultiDataSet mds = amdsi.next(); + + + //log.info("Features ptr: {}", AtomicAllocator.getInstance().getPointer(mds.getFeatures()[0].data()).address()); + assertEquals( (double) cnt, + mds.getFeatures()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.25, + mds.getLabels()[0].meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.5, + mds.getFeaturesMaskArrays()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals( (double) cnt + 0.75, + mds.getLabelsMaskArrays()[0].meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + + cnt++; + } + + amdsi.reset(); + log.info("Epoch {} finished...", e); + } + } + + + @Test + public void testVariableTimeSeries2() throws Exception { + int numBatches = isIntegrationTests() ? 1000 : 100; + int batchSize = isIntegrationTests() ? 32 : 8; + int timeStepsMin = 10; + int timeStepsMax = isIntegrationTests() ? 500 : 100; + int valuesPerTimestep = isIntegrationTests() ? 128 : 16; + + val iterator = new VariableMultiTimeseriesGenerator(1192, numBatches, batchSize, valuesPerTimestep, timeStepsMin, timeStepsMax, 10); + + for (int e = 0; e < 10; e++) { + iterator.reset(); + iterator.hasNext(); + val amdsi = new AsyncMultiDataSetIterator(iterator, 2, true); + + int cnt = 0; + while (amdsi.hasNext()) { + MultiDataSet mds = amdsi.next(); + + + //log.info("Features ptr: {}", AtomicAllocator.getInstance().getPointer(mds.getFeatures()[0].data()).address()); + assertEquals((double) cnt, + mds.getFeatures()[0].meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.25, + mds.getLabels()[0].meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.5, + mds.getFeaturesMaskArrays()[0].meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + assertEquals((double) cnt + 0.75, + mds.getLabelsMaskArrays()[0].meanNumber().doubleValue(), 1e-10, "Failed on epoch " + e + "; iteration: " + cnt + ";"); + + cnt++; + } + } + } +/* + @Test + public void testResetBug() throws Exception { + // /home/raver119/develop/dl4j-examples/src/main/resources/uci/train/features + + SequenceRecordReader trainFeatures = new CSVSequenceRecordReader(); + trainFeatures.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/train/features" + "/%d.csv", 0, 449)); + RecordReader trainLabels = new CSVRecordReader(); + trainLabels.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/train/labels" + "/%d.csv", 0, 449)); + + int miniBatchSize = 10; + int numLabelClasses = 6; + MultiDataSetIterator trainData = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize) + .addSequenceReader("features", trainFeatures) + .addReader("labels", trainLabels) + .addInput("features") + .addOutputOneHot("labels", 0, numLabelClasses) + .build(); + + //Normalize the training data + MultiDataNormalization normalizer = new MultiNormalizerStandardize(); + normalizer.fit(trainData); //Collect training data statistics + trainData.reset(); + + + SequenceRecordReader testFeatures = new CSVSequenceRecordReader(); + testFeatures.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/test/features" + "/%d.csv", 0, 149)); + RecordReader testLabels = new CSVRecordReader(); + testLabels.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/test/labels" + "/%d.csv", 0, 149)); + + MultiDataSetIterator testData = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize) + .addSequenceReader("features", testFeatures) + .addReader("labels", testLabels) + .addInput("features") + .addOutputOneHot("labels", 0, numLabelClasses) + .build(); + + System.out.println("-------------- HASH 1----------------"); + testData.reset(); + while(testData.hasNext()){ + System.out.println(Arrays.hashCode(testData.next().getFeatures(0).data().asFloat())); + } + + System.out.println("-------------- HASH 2 ----------------"); + testData.reset(); + testData.hasNext(); //***** Remove this (or move to after async creation), and we get expected results ***** + val adsi = new AsyncMultiDataSetIterator(testData, 4, true); //OR remove this (keeping hasNext) and we get expected results + //val adsi = new AsyncShieldMultiDataSetIterator(testData); + while(adsi.hasNext()){ + System.out.println(Arrays.hashCode(adsi.next().getFeatures(0).data().asFloat())); + } + } + */ +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessorTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessorTests.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessorTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessorTests.java index 3d4286b53..36ac3c338 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessorTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessorTests.java @@ -22,7 +22,6 @@ package org.deeplearning4j.datasets.iterator; import org.deeplearning4j.BaseDL4JTest; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.api.MultiDataSet; import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor; @@ -31,7 +30,6 @@ import org.nd4j.linalg.factory.Nd4j; import static org.junit.jupiter.api.Assertions.assertEquals; -@NativeTag public class CombinedPreProcessorTests extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java new file mode 100644 index 000000000..d4d0e28a1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java @@ -0,0 +1,367 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.iterator; + +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.image.loader.CifarLoader; +import org.datavec.image.loader.LFWLoader; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; +import org.deeplearning4j.datasets.fetchers.DataSetType; +import org.deeplearning4j.datasets.iterator.impl.*; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.CollectScoresIterationListener; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.*; + +public class DataSetIteratorTest extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return 360000; //Should run quickly; increased to large timeout due to occasonal slow CI downloads + } + + @Test + public void testBatchSizeOfOneIris() throws Exception { + //Test for (a) iterators returning correct number of examples, and + //(b) Labels are a proper one-hot vector (i.e., sum is 1.0) + + //Iris: + DataSetIterator iris = new IrisDataSetIterator(1, 5); + int irisC = 0; + while (iris.hasNext()) { + irisC++; + DataSet ds = iris.next(); + assertTrue(ds.getLabels().sum(Integer.MAX_VALUE).getDouble(0) == 1.0); + } + assertEquals(5, irisC); + } + + @Test + public void testBatchSizeOfOneMnist() throws Exception { + + //MNIST: + DataSetIterator mnist = new MnistDataSetIterator(1, 5); + int mnistC = 0; + while (mnist.hasNext()) { + mnistC++; + DataSet ds = mnist.next(); + assertTrue(ds.getLabels().sum(Integer.MAX_VALUE).getDouble(0) == 1.0); + } + assertEquals(5, mnistC); + } + + @Test + public void testMnist() throws Exception { + ClassPathResource cpr = new ClassPathResource("mnist_first_200.txt"); + CSVRecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new FileSplit(cpr.getTempFileFromArchive())); + RecordReaderDataSetIterator dsi = new RecordReaderDataSetIterator(rr, 10, 0, 10); + + MnistDataSetIterator iter = new MnistDataSetIterator(10, 200, false, true, false, 0); + + while (dsi.hasNext()) { + DataSet dsExp = dsi.next(); + DataSet dsAct = iter.next(); + + INDArray fExp = dsExp.getFeatures(); + fExp.divi(255); + INDArray lExp = dsExp.getLabels(); + + INDArray fAct = dsAct.getFeatures(); + INDArray lAct = dsAct.getLabels(); + + assertEquals(fExp, fAct.castTo(fExp.dataType())); + assertEquals(lExp, lAct.castTo(lExp.dataType())); + } + assertFalse(iter.hasNext()); + } + + @Test + public void testLfwIterator() throws Exception { + int numExamples = 1; + int row = 28; + int col = 28; + int channels = 1; + LFWDataSetIterator iter = new LFWDataSetIterator(numExamples, new int[] {row, col, channels}, true); + assertTrue(iter.hasNext()); + DataSet data = iter.next(); + assertEquals(numExamples, data.getLabels().size(0)); + assertEquals(row, data.getFeatures().size(2)); + } + + @Test + public void testTinyImageNetIterator() throws Exception { + int numClasses = 200; + int row = 64; + int col = 64; + int channels = 3; + TinyImageNetDataSetIterator iter = new TinyImageNetDataSetIterator(1, DataSetType.TEST); + assertTrue(iter.hasNext()); + DataSet data = iter.next(); + assertEquals(numClasses, data.getLabels().size(1)); + assertArrayEquals(new long[]{1, channels, row, col}, data.getFeatures().shape()); + } + + @Test + public void testTinyImageNetIterator2() throws Exception { + int numClasses = 200; + int row = 224; + int col = 224; + int channels = 3; + TinyImageNetDataSetIterator iter = new TinyImageNetDataSetIterator(1, new int[]{row, col}, DataSetType.TEST); + assertTrue(iter.hasNext()); + DataSet data = iter.next(); + assertEquals(numClasses, data.getLabels().size(1)); + assertArrayEquals(new long[]{1, channels, row, col}, data.getFeatures().shape()); + } + + @Test + public void testLfwModel() throws Exception { + final int numRows = 28; + final int numColumns = 28; + int numChannels = 3; + int outputNum = LFWLoader.NUM_LABELS; + int numSamples = LFWLoader.NUM_IMAGES; + int batchSize = 2; + int seed = 123; + int listenerFreq = 1; + + LFWDataSetIterator lfw = new LFWDataSetIterator(batchSize, numSamples, + new int[] {numRows, numColumns, numChannels}, outputNum, false, true, 1.0, new Random(seed)); + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(numChannels).nOut(6) + .weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .stride(1, 1).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .build()) + .setInputType(InputType.convolutionalFlat(numRows, numColumns, numChannels)) + ; + + MultiLayerNetwork model = new MultiLayerNetwork(builder.build()); + model.init(); + + model.setListeners(new ScoreIterationListener(listenerFreq)); + + model.fit(lfw.next()); + + DataSet dataTest = lfw.next(); + INDArray output = model.output(dataTest.getFeatures()); + Evaluation eval = new Evaluation(outputNum); + eval.eval(dataTest.getLabels(), output); +// System.out.println(eval.stats()); + } + + @Test + public void testCifar10Iterator() throws Exception { + int numExamples = 1; + int row = 32; + int col = 32; + int channels = 3; + Cifar10DataSetIterator iter = new Cifar10DataSetIterator(numExamples); + assertTrue(iter.hasNext()); + DataSet data = iter.next(); + assertEquals(numExamples, data.getLabels().size(0)); + assertEquals(channels * row * col, data.getFeatures().ravel().length()); + } + + + @Test //@Ignore //Ignored for now - CIFAR iterator needs work - https://github.com/eclipse/deeplearning4j/issues/4673 + public void testCifarModel() throws Exception { + // Streaming + runCifar(false); + + // Preprocess + runCifar(true); + } + + public void runCifar(boolean preProcessCifar) throws Exception { + final int height = 32; + final int width = 32; + int channels = 3; + int outputNum = CifarLoader.NUM_LABELS; + int batchSize = 5; + int seed = 123; + int listenerFreq = 1; + + Cifar10DataSetIterator cifar = new Cifar10DataSetIterator(batchSize); + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(channels).nOut(6).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .build()) + + .setInputType(InputType.convolutionalFlat(height, width, channels)); + + MultiLayerNetwork model = new MultiLayerNetwork(builder.build()); + model.init(); + + //model.setListeners(Arrays.asList((TrainingListener) new ScoreIterationListener(listenerFreq))); + + CollectScoresIterationListener listener = new CollectScoresIterationListener(listenerFreq); + model.setListeners(listener); + + model.fit(cifar); + + cifar = new Cifar10DataSetIterator(batchSize); + Evaluation eval = new Evaluation(cifar.getLabels()); + while (cifar.hasNext()) { + DataSet testDS = cifar.next(batchSize); + INDArray output = model.output(testDS.getFeatures()); + eval.eval(testDS.getLabels(), output); + } +// System.out.println(eval.stats(true)); + listener.exportScores(System.out); + } + + + @Test + public void testIteratorDataSetIteratorCombining() { + //Test combining of a bunch of small (size 1) data sets together + + int batchSize = 3; + int numBatches = 4; + + int featureSize = 5; + int labelSize = 6; + + Nd4j.getRandom().setSeed(12345); + + List orig = new ArrayList<>(); + for (int i = 0; i < batchSize * numBatches; i++) { + INDArray features = Nd4j.rand(1, featureSize); + INDArray labels = Nd4j.rand(1, labelSize); + orig.add(new DataSet(features, labels)); + } + + DataSetIterator iter = new IteratorDataSetIterator(orig.iterator(), batchSize); + int count = 0; + while (iter.hasNext()) { + DataSet ds = iter.next(); + assertArrayEquals(new long[] {batchSize, featureSize}, ds.getFeatures().shape()); + assertArrayEquals(new long[] {batchSize, labelSize}, ds.getLabels().shape()); + + List fList = new ArrayList<>(); + List lList = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + DataSet dsOrig = orig.get(count * batchSize + i); + fList.add(dsOrig.getFeatures()); + lList.add(dsOrig.getLabels()); + } + + INDArray fExp = Nd4j.vstack(fList); + INDArray lExp = Nd4j.vstack(lList); + + assertEquals(fExp, ds.getFeatures()); + assertEquals(lExp, ds.getLabels()); + + count++; + } + + assertEquals(count, numBatches); + } + + @Test + public void testIteratorDataSetIteratorSplitting() { + //Test splitting large data sets into smaller ones + + int origBatchSize = 4; + int origNumDSs = 3; + + int batchSize = 3; + int numBatches = 4; + + int featureSize = 5; + int labelSize = 6; + + Nd4j.getRandom().setSeed(12345); + + List orig = new ArrayList<>(); + for (int i = 0; i < origNumDSs; i++) { + INDArray features = Nd4j.rand(origBatchSize, featureSize); + INDArray labels = Nd4j.rand(origBatchSize, labelSize); + orig.add(new DataSet(features, labels)); + } + + + List expected = new ArrayList<>(); + expected.add(new DataSet(orig.get(0).getFeatures().getRows(0, 1, 2), + orig.get(0).getLabels().getRows(0, 1, 2))); + expected.add(new DataSet( + Nd4j.vstack(orig.get(0).getFeatures().getRows(3), + orig.get(1).getFeatures().getRows(0, 1)), + Nd4j.vstack(orig.get(0).getLabels().getRows(3), orig.get(1).getLabels().getRows(0, 1)))); + expected.add(new DataSet( + Nd4j.vstack(orig.get(1).getFeatures().getRows(2, 3), + orig.get(2).getFeatures().getRows(0)), + Nd4j.vstack(orig.get(1).getLabels().getRows(2, 3), orig.get(2).getLabels().getRows(0)))); + expected.add(new DataSet(orig.get(2).getFeatures().getRows(1, 2, 3), + orig.get(2).getLabels().getRows(1, 2, 3))); + + + DataSetIterator iter = new IteratorDataSetIterator(orig.iterator(), batchSize); + int count = 0; + while (iter.hasNext()) { + DataSet ds = iter.next(); + assertEquals(expected.get(count), ds); + + count++; + } + + assertEquals(count, numBatches); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetSplitterTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetSplitterTests.java similarity index 77% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetSplitterTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetSplitterTests.java index 6816c18aa..26302914e 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetSplitterTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetSplitterTests.java @@ -23,10 +23,7 @@ package org.deeplearning4j.datasets.iterator; import lombok.val; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.tools.DataSetGenerator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; @@ -37,8 +34,6 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.FILE_IO) public class DataSetSplitterTests extends BaseDL4JTest { @Test public void testSplitter_1() throws Exception { @@ -58,8 +53,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { int cnt = 0; while (train.hasNext()) { val data = train.next().getFeatures(); - - assertEquals( (float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); gcntTrain++; global++; } @@ -69,7 +63,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { while (test.hasNext()) { val data = test.next().getFeatures(); - assertEquals( (float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); gcntTest++; global++; } @@ -99,7 +93,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { while (train.hasNext()) { val data = train.next().getFeatures(); - assertEquals((float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); gcntTrain++; global++; } @@ -109,7 +103,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { if (e % 2 == 0) while (test.hasNext()) { val data = test.next().getFeatures(); - assertEquals( (float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); gcntTest++; global++; } @@ -118,50 +112,48 @@ public class DataSetSplitterTests extends BaseDL4JTest { assertEquals(700 * numEpochs + (300 * numEpochs / 2), global); } - @Test() + @Test public void testSplitter_3() throws Exception { - assertThrows(ND4JIllegalStateException.class, () -> { - val back = new DataSetGenerator(1000, new int[]{32, 100}, new int[]{32, 5}); + assertThrows(ND4JIllegalStateException.class, () -> { + val back = new DataSetGenerator(1000, new int[]{32, 100}, new int[]{32, 5}); - val splitter = new DataSetIteratorSplitter(back, 1000, 0.7); + val splitter = new DataSetIteratorSplitter(back, 1000, 0.7); - val train = splitter.getTrainIterator(); - val test = splitter.getTestIterator(); - val numEpochs = 10; + val train = splitter.getTrainIterator(); + val test = splitter.getTestIterator(); + val numEpochs = 10; - int gcntTrain = 0; - int gcntTest = 0; - int global = 0; - // emulating epochs here - for (int e = 0; e < numEpochs; e++) { - int cnt = 0; - while (train.hasNext()) { - val data = train.next().getFeatures(); + int gcntTrain = 0; + int gcntTest = 0; + int global = 0; + // emulating epochs here + for (int e = 0; e < numEpochs; e++) { + int cnt = 0; + while (train.hasNext()) { + val data = train.next().getFeatures(); - assertEquals((float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); - gcntTrain++; - global++; - } + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); + gcntTrain++; + global++; + } - train.reset(); + train.reset(); - while (test.hasNext()) { - val data = test.next().getFeatures(); - assertEquals((float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); - gcntTest++; - global++; - } - - // shifting underlying iterator by one - train.hasNext(); - back.shift(); - } - - assertEquals(1000 * numEpochs, global); - }); + while (test.hasNext()) { + val data = test.next().getFeatures(); + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); + gcntTest++; + global++; + } + // shifting underlying iterator by one + train.hasNext(); + back.shift(); + } + assertEquals(1000 * numEpochs, global); + }); } @Test @@ -181,7 +173,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { partIterator.reset(); while (partIterator.hasNext()) { val data = partIterator.next().getFeatures(); - assertEquals((float) perEpoch, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals( (float) perEpoch, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); //gcntTrain++; global++; cnt++; @@ -214,7 +206,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { int cnt = 0; val data = partIterator.next().getFeatures(); - assertEquals((float) perEpoch, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) perEpoch, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); //gcntTrain++; global++; cnt++; @@ -253,11 +245,10 @@ public class DataSetSplitterTests extends BaseDL4JTest { trained = true; val ds = trainIter.next(); assertNotNull(ds); - - assertEquals(globalIter, ds.getFeatures().getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals( (double) globalIter, ds.getFeatures().getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); globalIter++; } - assertTrue(trained,"Failed at epoch [" + e + "]"); + assertTrue(trained, "Failed at epoch [" + e + "]"); assertEquals(800, globalIter); @@ -269,10 +260,10 @@ public class DataSetSplitterTests extends BaseDL4JTest { val ds = testIter.next(); assertNotNull(ds); - assertEquals(globalIter, ds.getFeatures().getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals((double) globalIter, ds.getFeatures().getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); globalIter++; } - assertTrue(tested,"Failed at epoch [" + e + "]"); + assertTrue(tested, "Failed at epoch [" + e + "]"); assertEquals(900, globalIter); // validation set is used every 5 epochs @@ -284,10 +275,10 @@ public class DataSetSplitterTests extends BaseDL4JTest { val ds = validationIter.next(); assertNotNull(ds); - assertEquals(globalIter, ds.getFeatures().getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals((double) globalIter, ds.getFeatures().getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); globalIter++; } - assertTrue(validated,"Failed at epoch [" + e + "]"); + assertTrue(validated, "Failed at epoch [" + e + "]"); } // all 3 iterators have exactly 1000 elements combined @@ -319,7 +310,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { int farCnt = (1000 / 2) * (partNumber) + cnt; val data = iteratorList.get(partNumber).next().getFeatures(); - assertEquals((float) farCnt, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) farCnt, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); cnt++; global++; } @@ -328,8 +319,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { cnt = 0; while (iteratorList.get(0).hasNext()) { val data = iteratorList.get(0).next().getFeatures(); - - assertEquals((float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); global++; } } @@ -347,8 +337,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { int cnt = 0; while (iteratorList.get(partNumber).hasNext()) { val data = iteratorList.get(partNumber).next().getFeatures(); - - assertEquals( (float) (500*partNumber + cnt), data.getFloat(0), 1e-5,"Train failed on iteration " + cnt); + assertEquals((float) (500*partNumber + cnt), data.getFloat(0), 1e-5, "Train failed on iteration " + cnt); cnt++; } } @@ -372,7 +361,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { while (iteratorList.get(partNumber).hasNext()) { val data = iteratorList.get(partNumber).next().getFeatures(); - assertEquals( (float) (500*partNumber + cnt), data.getFloat(0), 1e-5,"Train failed on iteration " + cnt); + assertEquals((float) (500*partNumber + cnt), data.getFloat(0), 1e-5, "Train failed on iteration " + cnt); cnt++; } } @@ -397,7 +386,7 @@ public class DataSetSplitterTests extends BaseDL4JTest { val ds = validationIter.next(); assertNotNull(ds); - assertEquals((float) valCnt + 90, ds.getFeatures().getFloat(0), 1e-5,"Validation failed on iteration " + valCnt); + assertEquals((float) valCnt + 90, ds.getFeatures().getFloat(0), 1e-5, "Validation failed on iteration " + valCnt); valCnt++; } assertEquals(5, valCnt); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIteratorTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIteratorTests.java similarity index 93% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIteratorTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIteratorTests.java index c0db1401c..f614cc4ce 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIteratorTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIteratorTests.java @@ -22,12 +22,9 @@ package org.deeplearning4j.datasets.iterator; import lombok.extern.slf4j.Slf4j; import lombok.val; -import lombok.var; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.tools.SimpleVariableGenerator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.api.DataSet; import java.util.ArrayList; @@ -38,7 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j -@Tag(TagNames.JAVA_ONLY) public class DummyBlockDataSetIteratorTests extends BaseDL4JTest { @Test @@ -50,7 +46,7 @@ public class DummyBlockDataSetIteratorTests extends BaseDL4JTest { assertTrue(iterator.hasAnything()); val list = new ArrayList(8); - var datasets = iterator.next(3); + DataSet[] datasets = iterator.next(3); assertNotNull(datasets); assertEquals(3, datasets.length); list.addAll(Arrays.asList(datasets)); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIteratorTest.java new file mode 100644 index 000000000..d95c63ce7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIteratorTest.java @@ -0,0 +1,99 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.iterator; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class EarlyTerminationDataSetIteratorTest extends BaseDL4JTest { + + int minibatchSize = 10; + int numExamples = 105; + + @Test + public void testNextAndReset() throws Exception { + + int terminateAfter = 2; + + DataSetIterator iter = new MnistDataSetIterator(minibatchSize, numExamples); + EarlyTerminationDataSetIterator earlyEndIter = new EarlyTerminationDataSetIterator(iter, terminateAfter); + + assertTrue(earlyEndIter.hasNext()); + int batchesSeen = 0; + List seenData = new ArrayList<>(); + while (earlyEndIter.hasNext()) { + DataSet path = earlyEndIter.next(); + assertFalse(path == null); + seenData.add(path); + batchesSeen++; + } + assertEquals(batchesSeen, terminateAfter); + + //check data is repeated after reset + earlyEndIter.reset(); + batchesSeen = 0; + while (earlyEndIter.hasNext()) { + DataSet path = earlyEndIter.next(); + assertEquals(seenData.get(batchesSeen).getFeatures(), path.getFeatures()); + assertEquals(seenData.get(batchesSeen).getLabels(), path.getLabels()); + batchesSeen++; + } + } + + @Test + public void testNextNum() throws IOException { + int terminateAfter = 1; + + DataSetIterator iter = new MnistDataSetIterator(minibatchSize, numExamples); + EarlyTerminationDataSetIterator earlyEndIter = new EarlyTerminationDataSetIterator(iter, terminateAfter); + + earlyEndIter.next(10); + assertEquals(false, earlyEndIter.hasNext()); + + earlyEndIter.reset(); + assertEquals(true, earlyEndIter.hasNext()); + + } + + @Test + public void testCallstoNextNotAllowed() throws IOException { + int terminateAfter = 1; + + DataSetIterator iter = new MnistDataSetIterator(minibatchSize, numExamples); + EarlyTerminationDataSetIterator earlyEndIter = new EarlyTerminationDataSetIterator(iter, terminateAfter); + + earlyEndIter.next(10); + iter.reset(); + Assertions.assertThrows(RuntimeException.class, () -> { + earlyEndIter.next(10); + }); + } + } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIteratorTest.java new file mode 100644 index 000000000..b05240ac7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIteratorTest.java @@ -0,0 +1,115 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.iterator; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.adapter.MultiDataSetIteratorAdapter; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class EarlyTerminationMultiDataSetIteratorTest extends BaseDL4JTest { + + int minibatchSize = 5; + int numExamples = 105; + + @Test + public void testNextAndReset() throws Exception { + + int terminateAfter = 2; + + MultiDataSetIterator iter = + new MultiDataSetIteratorAdapter(new MnistDataSetIterator(minibatchSize, numExamples)); + + int count = 0; + List seenMDS = new ArrayList<>(); + while (count < terminateAfter) { + seenMDS.add(iter.next()); + count++; + } + iter.reset(); + + EarlyTerminationMultiDataSetIterator earlyEndIter = + new EarlyTerminationMultiDataSetIterator(iter, terminateAfter); + + assertTrue(earlyEndIter.hasNext()); + count = 0; + while (earlyEndIter.hasNext()) { + MultiDataSet path = earlyEndIter.next(); + assertEquals(path.getFeatures()[0], seenMDS.get(count).getFeatures()[0]); + assertEquals(path.getLabels()[0], seenMDS.get(count).getLabels()[0]); + count++; + } + assertEquals(count, terminateAfter); + + //check data is repeated + earlyEndIter.reset(); + count = 0; + while (earlyEndIter.hasNext()) { + MultiDataSet path = earlyEndIter.next(); + assertEquals(path.getFeatures()[0], seenMDS.get(count).getFeatures()[0]); + assertEquals(path.getLabels()[0], seenMDS.get(count).getLabels()[0]); + count++; + } + } + + @Test + public void testNextNum() throws IOException { + int terminateAfter = 1; + + MultiDataSetIterator iter = + new MultiDataSetIteratorAdapter(new MnistDataSetIterator(minibatchSize, numExamples)); + EarlyTerminationMultiDataSetIterator earlyEndIter = + new EarlyTerminationMultiDataSetIterator(iter, terminateAfter); + + earlyEndIter.next(10); + assertEquals(false, earlyEndIter.hasNext()); + + earlyEndIter.reset(); + assertEquals(true, earlyEndIter.hasNext()); + } + + @Test + public void testCallstoNextNotAllowed() throws IOException { + int terminateAfter = 1; + + MultiDataSetIterator iter = + new MultiDataSetIteratorAdapter(new MnistDataSetIterator(minibatchSize, numExamples)); + EarlyTerminationMultiDataSetIterator earlyEndIter = + new EarlyTerminationMultiDataSetIterator(iter, terminateAfter); + + earlyEndIter.next(10); + iter.reset(); + Assertions.assertThrows(RuntimeException.class, () -> { + earlyEndIter.next(10); + }); + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIteratorTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIteratorTests.java similarity index 94% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIteratorTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIteratorTests.java index 3a48641d8..f2339eb93 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIteratorTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIteratorTests.java @@ -24,21 +24,16 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.tools.DataSetGenerator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.FILE_IO) +@Timeout(300) public class JointMultiDataSetIteratorTests extends BaseDL4JTest { - @Test () - @Timeout(20000L) + @Test public void testJMDSI_1() { val iter0 = new DataSetGenerator(32, new int[]{3, 3}, new int[]{2, 2}); val iter1 = new DataSetGenerator(32, new int[]{3, 3, 3}, new int[]{2, 2, 2}); @@ -82,8 +77,7 @@ public class JointMultiDataSetIteratorTests extends BaseDL4JTest { } - @Test () - @Timeout(20000L) + @Test public void testJMDSI_2() { val iter0 = new DataSetGenerator(32, new int[]{3, 3}, new int[]{2, 2}); val iter1 = new DataSetGenerator(32, new int[]{3, 3, 3}, new int[]{2, 2, 2}); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/JointParallelDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/JointParallelDataSetIteratorTest.java new file mode 100644 index 000000000..6705f6430 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/JointParallelDataSetIteratorTest.java @@ -0,0 +1,199 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.iterator; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.parallel.JointParallelDataSetIterator; +import org.deeplearning4j.datasets.iterator.tools.SimpleVariableGenerator; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.enums.InequalityHandling; + + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@Slf4j +public class JointParallelDataSetIteratorTest extends BaseDL4JTest { + + /** + * Simple test, checking datasets alignment. They all should have the same data for the same cycle + * + * + * @throws Exception + */ + @Test + public void testJointIterator1() throws Exception { + DataSetIterator iteratorA = new SimpleVariableGenerator(119, 100, 32, 100, 10); + DataSetIterator iteratorB = new SimpleVariableGenerator(119, 100, 32, 100, 10); + + JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator.Builder(InequalityHandling.STOP_EVERYONE) + .addSourceIterator(iteratorA).addSourceIterator(iteratorB).build(); + + int cnt = 0; + int example = 0; + while (jpdsi.hasNext()) { + DataSet ds = jpdsi.next(); + assertNotNull( ds, "Failed on iteration " + cnt); + +// ds.detach(); + //ds.migrate(); + + assertEquals( (double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + assertEquals( (double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + + cnt++; + if (cnt % 2 == 0) + example++; + } + + assertEquals(100, example); + assertEquals(200, cnt); + } + + + /** + * This test checks for pass_null scenario, so in total we should have 300 real datasets + 100 nulls + * @throws Exception + */ + @Test + public void testJointIterator2() throws Exception { + DataSetIterator iteratorA = new SimpleVariableGenerator(119, 200, 32, 100, 10); + DataSetIterator iteratorB = new SimpleVariableGenerator(119, 100, 32, 100, 10); + + JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator.Builder(InequalityHandling.PASS_NULL) + .addSourceIterator(iteratorA).addSourceIterator(iteratorB).build(); + + int cnt = 0; + int example = 0; + int nulls = 0; + while (jpdsi.hasNext()) { + DataSet ds = jpdsi.next(); + if (cnt < 200) + assertNotNull( ds, "Failed on iteration " + cnt); + + if (ds == null) + nulls++; + + if (cnt % 2 == 2) { + assertEquals( (double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + assertEquals((double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + } + + + cnt++; + if (cnt % 2 == 0) + example++; + } + + assertEquals(100, nulls); + assertEquals(200, example); + assertEquals(400, cnt); + } + + /** + * Testing relocate + * + * @throws Exception + */ + @Test + public void testJointIterator3() throws Exception { + DataSetIterator iteratorA = new SimpleVariableGenerator(119, 200, 32, 100, 10); + DataSetIterator iteratorB = new SimpleVariableGenerator(119, 100, 32, 100, 10); + + JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator.Builder(InequalityHandling.RELOCATE) + .addSourceIterator(iteratorA).addSourceIterator(iteratorB).build(); + + int cnt = 0; + int example = 0; + while (jpdsi.hasNext()) { + DataSet ds = jpdsi.next(); + assertNotNull( ds, "Failed on iteration " + cnt); + + assertEquals((double) example, ds.getFeatures().meanNumber().doubleValue(),0.001, "Failed on iteration " + cnt); + assertEquals((double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + + + cnt++; + if (cnt < 200) { + if (cnt % 2 == 0) + example++; + } else + example++; + } + + + assertEquals(300, cnt); + assertEquals(200, example); + } + + /** + * Testing relocate + * + * @throws Exception + */ + @Test + public void testJointIterator4() throws Exception { + DataSetIterator iteratorA = new SimpleVariableGenerator(119, 200, 32, 100, 10); + DataSetIterator iteratorB = new SimpleVariableGenerator(119, 100, 32, 100, 10); + + JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator.Builder(InequalityHandling.RESET) + .addSourceIterator(iteratorA).addSourceIterator(iteratorB).build(); + + int cnt = 0; + int cnt_sec = 0; + int example_sec = 0; + int example = 0; + while (jpdsi.hasNext()) { + DataSet ds = jpdsi.next(); + assertNotNull( ds, "Failed on iteration " + cnt); + + if (cnt % 2 == 0) { + assertEquals( (double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + assertEquals((double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + } else { + if (cnt <= 200) { + assertEquals( (double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + assertEquals( (double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt); + } else { + assertEquals( (double) example_sec, ds.getFeatures().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt + ", second iteration " + cnt_sec); + assertEquals((double) example_sec + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001, "Failed on iteration " + cnt + ", second iteration " + cnt_sec); + } + } + + cnt++; + if (cnt % 2 == 0) + example++; + + if (cnt > 201 && cnt % 2 == 1) { + cnt_sec++; + example_sec++; + } + + } + + + assertEquals(400, cnt); + assertEquals(200, example); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/LoaderIteratorTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/LoaderIteratorTests.java similarity index 95% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/LoaderIteratorTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/LoaderIteratorTests.java index 9ccf3ef48..da97c5cd7 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/LoaderIteratorTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/LoaderIteratorTests.java @@ -23,13 +23,10 @@ package org.deeplearning4j.datasets.iterator; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.loader.DataSetLoaderIterator; import org.deeplearning4j.datasets.iterator.loader.MultiDataSetLoaderIterator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.loader.Loader; import org.nd4j.common.loader.LocalFileSourceFactory; import org.nd4j.common.loader.Source; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.MultiDataSet; @@ -44,12 +41,11 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.FILE_IO) + public class LoaderIteratorTests extends BaseDL4JTest { @Test - public void testDSLoaderIter() { + public void testDSLoaderIter(){ for(boolean r : new boolean[]{false, true}) { List l = Arrays.asList("3", "0", "1"); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/MultiDataSetSplitterTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/MultiDataSetSplitterTests.java similarity index 78% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/MultiDataSetSplitterTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/MultiDataSetSplitterTests.java index 5601b682e..27ffe5bba 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/MultiDataSetSplitterTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/MultiDataSetSplitterTests.java @@ -24,10 +24,7 @@ import lombok.val; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.tools.DataSetGenerator; import org.deeplearning4j.datasets.iterator.tools.MultiDataSetGenerator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; import org.nd4j.linalg.exception.ND4JIllegalStateException; @@ -36,8 +33,7 @@ import java.util.List; import java.util.Random; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.FILE_IO) + public class MultiDataSetSplitterTests extends BaseDL4JTest { @Test @@ -59,7 +55,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { while (train.hasNext()) { val data = train.next().getFeatures(0); - assertEquals( (float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals( (float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); gcntTrain++; global++; } @@ -69,7 +65,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { while (test.hasNext()) { val data = test.next().getFeatures(0); - assertEquals( (float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals( (float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); gcntTest++; global++; } @@ -100,7 +96,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { while (train.hasNext()) { val data = train.next().getFeatures(0); - assertEquals( (float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); gcntTrain++; global++; } @@ -110,7 +106,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { if (e % 2 == 0) while (test.hasNext()) { val data = test.next().getFeatures(0); - assertEquals((float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals( (float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); gcntTest++; global++; } @@ -119,49 +115,46 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { assertEquals(700 * numEpochs + (300 * numEpochs / 2), global); } - @Test() + @Test public void testSplitter_3() throws Exception { - assertThrows(ND4JIllegalStateException.class,() -> { - val back = new MultiDataSetGenerator(1000, new int[]{32, 100}, new int[]{32, 5}); + assertThrows(ND4JIllegalStateException.class, () -> { + val back = new MultiDataSetGenerator(1000, new int[]{32, 100}, new int[]{32, 5}); + val splitter = new MultiDataSetIteratorSplitter(back, 1000, 0.7); + val train = splitter.getTrainIterator(); + val test = splitter.getTestIterator(); + val numEpochs = 10; - val splitter = new MultiDataSetIteratorSplitter(back, 1000, 0.7); + int gcntTrain = 0; + int gcntTest = 0; + int global = 0; + // emulating epochs here + for (int e = 0; e < numEpochs; e++) { + int cnt = 0; + while (train.hasNext()) { + val data = train.next().getFeatures(0); - val train = splitter.getTrainIterator(); - val test = splitter.getTestIterator(); - val numEpochs = 10; + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); + gcntTrain++; + global++; + } - int gcntTrain = 0; - int gcntTest = 0; - int global = 0; - // emulating epochs here - for (int e = 0; e < numEpochs; e++){ - int cnt = 0; - while (train.hasNext()) { - val data = train.next().getFeatures(0); - - assertEquals((float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); - gcntTrain++; - global++; - } - - train.reset(); + train.reset(); - while (test.hasNext()) { - val data = test.next().getFeatures(0); - assertEquals( (float) cnt++, data.getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); - gcntTest++; - global++; - } + while (test.hasNext()) { + val data = test.next().getFeatures(0); + assertEquals((float) cnt++, data.getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); + gcntTest++; + global++; + } - // shifting underlying iterator by one - train.hasNext(); - back.shift(); - } - - assertEquals(1000 * numEpochs, global); - }); + // shifting underlying iterator by one + train.hasNext(); + back.shift(); + } + assertEquals(1000 * numEpochs, global); + }); } @Test @@ -192,11 +185,11 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { assertNotNull(ds); for (int i = 0; i < ds.getFeatures().length; ++i) { - assertEquals( (double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals((double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); } globalIter++; } - assertTrue(trained,"Failed at epoch [" + e + "]"); + assertTrue(trained, "Failed at epoch [" + e + "]"); assertEquals(800, globalIter); @@ -209,11 +202,11 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { assertNotNull(ds); for (int i = 0; i < ds.getFeatures().length; ++i) { - assertEquals((double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals((double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); } globalIter++; } - assertTrue(tested,"Failed at epoch [" + e + "]"); + assertTrue(tested, "Failed at epoch [" + e + "]"); assertEquals(900, globalIter); // validation set is used every 5 epochs @@ -226,11 +219,11 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { assertNotNull(ds); for (int i = 0; i < ds.getFeatures().length; ++i) { - assertEquals( (double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals((double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); } globalIter++; } - assertTrue(validated,"Failed at epoch [" + e + "]"); + assertTrue(validated, "Failed at epoch [" + e + "]"); } // all 3 iterators have exactly 1000 elements combined @@ -263,7 +256,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { val data = partIterator.next().getFeatures(); for (int i = 0; i < data.length; ++i) { - assertEquals((float) perEpoch, data[i].getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals( (float) perEpoch, data[i].getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); } //gcntTrain++; global++; @@ -305,12 +298,11 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { assertNotNull(ds); for (int i = 0; i < ds.getFeatures().length; ++i) { - assertEquals((double) globalIter, - ds.getFeatures()[i].getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals((double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); } globalIter++; } - assertTrue(trained,"Failed at epoch [" + e + "]"); + assertTrue(trained, "Failed at epoch [" + e + "]"); assertEquals(800, globalIter); @@ -322,11 +314,11 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { val ds = testIter.next(); assertNotNull(ds); for (int i = 0; i < ds.getFeatures().length; ++i) { - assertEquals((double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals((double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); } globalIter++; } - assertTrue(tested,"Failed at epoch [" + e + "]"); + assertTrue(tested, "Failed at epoch [" + e + "]"); assertEquals(900, globalIter); // validation set is used every 5 epochs @@ -339,12 +331,11 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { assertNotNull(ds); for (int i = 0; i < ds.getFeatures().length; ++i) { - assertEquals((double) globalIter, - ds.getFeatures()[i].getDouble(0), 1e-5f,"Failed at iteration [" + globalIter + "]"); + assertEquals((double) globalIter, ds.getFeatures()[i].getDouble(0), 1e-5f, "Failed at iteration [" + globalIter + "]"); } globalIter++; } - assertTrue(validated,"Failed at epoch [" + e + "]"); + assertTrue(validated, "Failed at epoch [" + e + "]"); } // all 3 iterators have exactly 1000 elements combined @@ -376,7 +367,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { int farCnt = (1000 / 2) * (partNumber) + cnt; val data = iteratorList.get(partNumber).next().getFeatures(); for (int i = 0; i < data.length; ++i) { - assertEquals( (float) farCnt, data[i].getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals( (float) farCnt, data[i].getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); } cnt++; global++; @@ -387,8 +378,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { while (iteratorList.get(0).hasNext()) { val data = iteratorList.get(0).next().getFeatures(); for (int i = 0; i < data.length; ++i) { - assertEquals((float) cnt++, - data[i].getFloat(0), 1e-5,"Train failed on iteration " + cnt + "; epoch: " + e); + assertEquals((float) cnt++,data[i].getFloat(0), 1e-5, "Train failed on iteration " + cnt + "; epoch: " + e); } global++; } @@ -408,7 +398,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { while (iteratorList.get(partNumber).hasNext()) { val data = iteratorList.get(partNumber).next().getFeatures(); for (int i = 0; i < data.length; ++i) { - assertEquals( (float) (500 * partNumber + cnt), data[i].getFloat(0), 1e-5,"Train failed on iteration " + cnt); + assertEquals((float) (500 * partNumber + cnt), data[i].getFloat(0), 1e-5, "Train failed on iteration " + cnt); } cnt++; } @@ -433,8 +423,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { while (iteratorList.get(partNumber).hasNext()) { val data = iteratorList.get(partNumber).next().getFeatures(); for (int i = 0; i < data.length; ++i) { - assertEquals( (float) (500 * partNumber + cnt), - data[i].getFloat(0), 1e-5,"Train failed on iteration " + cnt); + assertEquals((float) (500 * partNumber + cnt), data[i].getFloat(0), 1e-5, "Train failed on iteration " + cnt); } cnt++; } @@ -460,8 +449,7 @@ public class MultiDataSetSplitterTests extends BaseDL4JTest { val ds = validationIter.next(); assertNotNull(ds); for (int i = 0; i < ds.getFeatures().length; ++i) { - assertEquals((float) valCnt + 90, - ds.getFeatures()[i].getFloat(0), 1e-5,"Validation failed on iteration " + valCnt); + assertEquals((float) valCnt + 90,ds.getFeatures()[i].getFloat(0), 1e-5, "Validation failed on iteration " + valCnt); } valCnt++; } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIteratorTest.java similarity index 86% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIteratorTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIteratorTest.java index a8baa825a..3b221afd9 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIteratorTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIteratorTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.datasets.iterator; import org.datavec.api.records.reader.RecordReader; @@ -25,36 +26,31 @@ import org.datavec.api.split.FileSplit; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; import org.deeplearning4j.nn.util.TestDataSetConsumer; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.Timeout; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.io.ClassPathResource; import org.nd4j.common.resources.Resources; + import java.util.Iterator; import java.util.concurrent.atomic.AtomicLong; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Multiple Epochs Iterator Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class MultipleEpochsIteratorTest extends BaseDL4JTest { +import static org.junit.jupiter.api.Assertions.*; + +@Timeout(300) +public class MultipleEpochsIteratorTest extends BaseDL4JTest { @Test - @DisplayName("Test Next And Reset") - void testNextAndReset() throws Exception { + public void testNextAndReset() throws Exception { int epochs = 3; + RecordReader rr = new CSVRecordReader(); rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); DataSetIterator iter = new RecordReaderDataSetIterator(rr, 150); MultipleEpochsIterator multiIter = new MultipleEpochsIterator(epochs, iter); + assertTrue(multiIter.hasNext()); while (multiIter.hasNext()) { DataSet path = multiIter.next(); @@ -64,15 +60,18 @@ class MultipleEpochsIteratorTest extends BaseDL4JTest { } @Test - @DisplayName("Test Load Full Data Set") - void testLoadFullDataSet() throws Exception { + public void testLoadFullDataSet() throws Exception { int epochs = 3; + RecordReader rr = new CSVRecordReader(); rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); DataSetIterator iter = new RecordReaderDataSetIterator(rr, 150); DataSet ds = iter.next(50); + assertEquals(50, ds.getFeatures().size(0)); + MultipleEpochsIterator multiIter = new MultipleEpochsIterator(epochs, ds); + assertTrue(multiIter.hasNext()); int count = 0; while (multiIter.hasNext()) { @@ -86,26 +85,28 @@ class MultipleEpochsIteratorTest extends BaseDL4JTest { } @Test - @DisplayName("Test Load Batch Data Set") - void testLoadBatchDataSet() throws Exception { + public void testLoadBatchDataSet() throws Exception { int epochs = 2; + RecordReader rr = new CSVRecordReader(); rr.initialize(new FileSplit(new ClassPathResource("iris.txt").getFile())); DataSetIterator iter = new RecordReaderDataSetIterator(rr, 150, 4, 3); DataSet ds = iter.next(20); assertEquals(20, ds.getFeatures().size(0)); MultipleEpochsIterator multiIter = new MultipleEpochsIterator(epochs, ds); + while (multiIter.hasNext()) { DataSet path = multiIter.next(10); assertNotNull(path); assertEquals(10, path.numExamples(), 0.0); } + assertEquals(epochs, multiIter.epochs); } + @Test - @DisplayName("Test MEDI With Load 1") - void testMEDIWithLoad1() throws Exception { + public void testMEDIWithLoad1() throws Exception { ExistingDataSetIterator iter = new ExistingDataSetIterator(new IterableWithoutException(100)); MultipleEpochsIterator iterator = new MultipleEpochsIterator(10, iter, 24); TestDataSetConsumer consumer = new TestDataSetConsumer(iterator, 1); @@ -114,39 +115,38 @@ class MultipleEpochsIteratorTest extends BaseDL4JTest { } @Test - @DisplayName("Test MEDI With Load 2") - void testMEDIWithLoad2() throws Exception { + public void testMEDIWithLoad2() throws Exception { ExistingDataSetIterator iter = new ExistingDataSetIterator(new IterableWithoutException(100)); MultipleEpochsIterator iterator = new MultipleEpochsIterator(10, iter, 24); TestDataSetConsumer consumer = new TestDataSetConsumer(iterator, 2); long num1 = 0; + for (; num1 < 150; num1++) { consumer.consumeOnce(iterator.next(), true); } iterator.reset(); + long num2 = consumer.consumeWhileHasNext(true); assertEquals((10 * 100) + 150, num1 + num2); } @Test - @DisplayName("Test MEDI With Load 3") - void testMEDIWithLoad3() throws Exception { + public void testMEDIWithLoad3() throws Exception { ExistingDataSetIterator iter = new ExistingDataSetIterator(new IterableWithoutException(10000)); MultipleEpochsIterator iterator = new MultipleEpochsIterator(iter, 24, 136); TestDataSetConsumer consumer = new TestDataSetConsumer(iterator, 2); long num1 = 0; + while (iterator.hasNext()) { consumer.consumeOnce(iterator.next(), true); num1++; } + assertEquals(136, num1); } - @DisplayName("Iterable Without Exception") private class IterableWithoutException implements Iterable { - private final AtomicLong counter = new AtomicLong(0); - private final int datasets; public IterableWithoutException(int datasets) { @@ -157,7 +157,6 @@ class MultipleEpochsIteratorTest extends BaseDL4JTest { public Iterator iterator() { counter.set(0); return new Iterator() { - @Override public boolean hasNext() { return counter.get() < datasets; @@ -171,6 +170,7 @@ class MultipleEpochsIteratorTest extends BaseDL4JTest { @Override public void remove() { + } }; } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/RandomDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/RandomDataSetIteratorTest.java new file mode 100644 index 000000000..ada39e036 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/RandomDataSetIteratorTest.java @@ -0,0 +1,84 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.datasets.iterator; + +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RandomDataSetIteratorTest extends BaseDL4JTest { + + @Test + public void testDSI(){ + DataSetIterator iter = new RandomDataSetIterator(5, new long[]{3,4}, new long[]{3,5}, RandomDataSetIterator.Values.RANDOM_UNIFORM, + RandomDataSetIterator.Values.ONE_HOT); + + int count = 0; + while(iter.hasNext()){ + count++; + DataSet ds = iter.next(); + + assertArrayEquals(new long[]{3,4}, ds.getFeatures().shape()); + assertArrayEquals(new long[]{3,5}, ds.getLabels().shape()); + + assertTrue(ds.getFeatures().minNumber().doubleValue() >= 0.0 && ds.getFeatures().maxNumber().doubleValue() <= 1.0); + assertEquals(Nd4j.ones(3), ds.getLabels().sum(1)); + } + assertEquals(5, count); + } + + @Test + public void testMDSI(){ + Nd4j.getRandom().setSeed(12345); + MultiDataSetIterator iter = new RandomMultiDataSetIterator.Builder(5) + .addFeatures(new long[]{3,4}, RandomMultiDataSetIterator.Values.INTEGER_0_100) + .addFeatures(new long[]{3,5}, RandomMultiDataSetIterator.Values.BINARY) + .addLabels(new long[]{3,6}, RandomMultiDataSetIterator.Values.ZEROS) + .build(); + + int count = 0; + while(iter.hasNext()){ + count++; + MultiDataSet mds = iter.next(); + + assertEquals(2, mds.numFeatureArrays()); + assertEquals(1, mds.numLabelsArrays()); + assertArrayEquals(new long[]{3,4}, mds.getFeatures(0).shape()); + assertArrayEquals(new long[]{3,5}, mds.getFeatures(1).shape()); + assertArrayEquals(new long[]{3,6}, mds.getLabels(0).shape()); + + assertTrue(mds.getFeatures(0).minNumber().doubleValue() >= 0 && mds.getFeatures(0).maxNumber().doubleValue() <= 100.0 + && mds.getFeatures(0).maxNumber().doubleValue() > 2.0); + assertTrue(mds.getFeatures(1).minNumber().doubleValue() == 0.0 && mds.getFeatures(1).maxNumber().doubleValue() == 1.0); + assertEquals(0.0, mds.getLabels(0).sumNumber().doubleValue(), 0.0); + } + assertEquals(5, count); + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/SamplingTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/SamplingTest.java similarity index 76% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/SamplingTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/SamplingTest.java index 0d40b1054..6a95c33a8 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/SamplingTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/SamplingTest.java @@ -17,34 +17,27 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.datasets.iterator; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** * @author Adam Gibson */ -@DisplayName("Sampling Test") -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.NDARRAY_ETL) -class SamplingTest extends BaseDL4JTest { +public class SamplingTest extends BaseDL4JTest { @Test - @DisplayName("Test Sample") - void testSample() throws Exception { + public void testSample() throws Exception { DataSetIterator iter = new MnistDataSetIterator(10, 10); - // batch size and total + //batch size and total DataSetIterator sampling = new SamplingDataSetIterator(iter.next(), 10, 10); assertEquals(10, sampling.next().numExamples()); } + } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestAsyncIterator.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestAsyncIterator.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestAsyncIterator.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestAsyncIterator.java index 49ab99495..199953dbc 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestAsyncIterator.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestAsyncIterator.java @@ -22,11 +22,8 @@ package org.deeplearning4j.datasets.iterator; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.DataSetPreProcessor; @@ -35,13 +32,9 @@ import org.nd4j.linalg.factory.Nd4j; import java.util.List; - import static org.junit.jupiter.api.Assertions.*; -@Disabled -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.NDARRAY_ETL) +////@Ignore public class TestAsyncIterator extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestEmnistDataSetIterator.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestEmnistDataSetIterator.java similarity index 94% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestEmnistDataSetIterator.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestEmnistDataSetIterator.java index bbf465e47..aa0224b4b 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestEmnistDataSetIterator.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestEmnistDataSetIterator.java @@ -23,12 +23,7 @@ package org.deeplearning4j.datasets.iterator; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.impl.EmnistDataSetIterator; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -37,22 +32,17 @@ import org.nd4j.linalg.factory.Nd4j; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.NDARRAY_ETL) public class TestEmnistDataSetIterator extends BaseDL4JTest { - - @Override public DataType getDataType(){ return DataType.FLOAT; } @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) public void testEmnistDataSetIterator() throws Exception { + + int batchSize = 128; EmnistDataSetIterator.Set[] sets; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestFileIterators.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestFileIterators.java similarity index 89% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestFileIterators.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestFileIterators.java index 61a40a77a..c2c8ca166 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/TestFileIterators.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/TestFileIterators.java @@ -23,14 +23,8 @@ package org.deeplearning4j.datasets.iterator; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.file.FileDataSetIterator; import org.deeplearning4j.datasets.iterator.file.FileMultiDataSetIterator; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.MultiDataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; @@ -38,24 +32,23 @@ import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; import org.nd4j.linalg.factory.Nd4j; import java.io.File; -import java.nio.file.Path; import java.util.*; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -@Disabled -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.NDARRAY_ETL) public class TestFileIterators extends BaseDL4JTest { + @TempDir + public File folder; + @TempDir + public File folder2; @Test - public void testFileDataSetIterator(@TempDir Path folder, @TempDir Path testDir2) throws Exception { + public void testFileDataSetIterator() throws Exception { - File f = folder.toFile(); + File f = folder; DataSet d1 = new DataSet(Nd4j.linspace(1, 10, 10).reshape(10,1), Nd4j.linspace(101, 110, 10).reshape(10,1)); @@ -84,12 +77,9 @@ public class TestFileIterators extends BaseDL4JTest { //Test multiple directories - File f2a = new File(testDir2.toFile(),"folder1"); - f2a.mkdirs(); - File f2b = new File(testDir2.toFile(),"folder2"); - f2b.mkdirs(); - File f2c = new File(testDir2.toFile(),"folder3"); - f2c.mkdirs(); + File f2a = new File(folder2, "f2a"); + File f2b = new File(folder2, "f2b"); + File f2c = new File(folder2, "f2c"); d1.save(new File(f2a, "d1.bin")); d2.save(new File(f2a, "d2.bin")); d3.save(new File(f2b, "d3.bin")); @@ -143,9 +133,7 @@ public class TestFileIterators extends BaseDL4JTest { //Test batch size != saved size - File f4 = new File(folder.toFile(),"newFolder"); - f4.mkdirs(); - f = f4; + f = new File(folder, "f"); d1.save(new File(f, "d1.bin")); d2.save(new File(f, "d2.bin")); d3.save(new File(f, "d3.bin")); @@ -170,8 +158,8 @@ public class TestFileIterators extends BaseDL4JTest { } @Test - public void testFileMultiDataSetIterator(@TempDir Path folder) throws Exception { - File f = folder.toFile(); + public void testFileMultiDataSetIterator() throws Exception { + File f = folder; MultiDataSet d1 = new org.nd4j.linalg.dataset.MultiDataSet(Nd4j.linspace(1, 10, 10).reshape(10,1), Nd4j.linspace(101, 110, 10).reshape(10,1)); @@ -199,11 +187,9 @@ public class TestFileIterators extends BaseDL4JTest { assertEquals(exp, act); //Test multiple directories - File newDir = new File(folder.toFile(),"folder2"); - newDir.mkdirs(); - File f2a = new File(newDir,"folder-1"); - File f2b = new File(newDir,"folder-2"); - File f2c = new File(newDir,"folder-3"); + File f2a = new File(folder2, "2-f2a"); + File f2b = new File(folder2, "2-f2b"); + File f2c = new File(folder2, "2-f2C"); d1.save(new File(f2a, "d1.bin")); d2.save(new File(f2a, "d2.bin")); d3.save(new File(f2b, "d3.bin")); @@ -254,8 +240,6 @@ public class TestFileIterators extends BaseDL4JTest { //Test batch size != saved size - f = new File(folder.toFile(),"newolder"); - f.mkdirs(); d1.save(new File(f, "d1.bin")); d2.save(new File(f, "d2.bin")); d3.save(new File(f, "d3.bin")); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/DataSetGenerator.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/DataSetGenerator.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/DataSetGenerator.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/DataSetGenerator.java index 3ab97e477..f472701c9 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/DataSetGenerator.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/DataSetGenerator.java @@ -30,7 +30,7 @@ import org.nd4j.linalg.factory.Nd4j; import java.util.List; import java.util.concurrent.atomic.AtomicLong; -public class DataSetGenerator implements DataSetIterator { +public class DataSetGenerator implements DataSetIterator{ protected final int[] shapeFeatures; protected final int[] shapeLabels; protected final long totalBatches; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/MultiDataSetGenerator.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/MultiDataSetGenerator.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/MultiDataSetGenerator.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/MultiDataSetGenerator.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/SimpleVariableGenerator.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/SimpleVariableGenerator.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/SimpleVariableGenerator.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/SimpleVariableGenerator.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/VariableMultiTimeseriesGenerator.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/VariableMultiTimeseriesGenerator.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/VariableMultiTimeseriesGenerator.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/VariableMultiTimeseriesGenerator.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/VariableTimeseriesGenerator.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/VariableTimeseriesGenerator.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/VariableTimeseriesGenerator.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/datasets/iterator/tools/VariableTimeseriesGenerator.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java index 8155b36ea..b4e790ea1 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStopping.java @@ -50,13 +50,8 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.optimize.api.BaseTrainingListener; import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.deeplearning4j.optimize.solvers.BaseOptimizer; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.evaluation.classification.Evaluation; import org.nd4j.evaluation.classification.ROCBinary; import org.nd4j.evaluation.regression.RegressionEvaluation.Metric; @@ -75,22 +70,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.nio.file.Path; import java.util.*; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) public class TestEarlyStopping extends BaseDL4JTest { + @TempDir + public File testDir; @Override public DataType getDataType(){ @@ -102,7 +91,7 @@ public class TestEarlyStopping extends BaseDL4JTest { DataSetIterator irisIter = new IrisDataSetIterator(150, 150); - for( int i = 0; i < 6; i++ ) { + for( int i=0; i<6; i++ ) { Nd4j.getRandom().setSeed(12345); ScoreCalculator sc; @@ -192,7 +181,7 @@ public class TestEarlyStopping extends BaseDL4JTest { } } assertEquals(bestEpoch, out.getEpochCount(),msg); - assertEquals( bestScore, result.getBestModelScore(), 1e-5,msg); + assertEquals(bestScore, result.getBestModelScore(), 1e-5, msg); //Check that best score actually matches (returned model vs. manually calculated score) MultiLayerNetwork bestNetwork = result.getBestModel(); @@ -223,7 +212,7 @@ public class TestEarlyStopping extends BaseDL4JTest { default: throw new RuntimeException(); } - assertEquals(result.getBestModelScore(), score, 1e-2,msg); + assertEquals(result.getBestModelScore(), score, 1e-2, msg); } } @@ -855,7 +844,7 @@ public class TestEarlyStopping extends BaseDL4JTest { } @Test - public void testEarlyStoppingMaximizeScore(@TempDir Path testDir) throws Exception { + public void testEarlyStoppingMaximizeScore() throws Exception { Nd4j.getRandom().setSeed(12345); int outputs = 2; @@ -893,7 +882,7 @@ public class TestEarlyStopping extends BaseDL4JTest { .build()) .build(); - File f = testDir.toFile(); + File f = testDir; EarlyStoppingModelSaver saver = new LocalFileModelSaver(f.getAbsolutePath()); EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder() diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStoppingCompGraph.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStoppingCompGraph.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStoppingCompGraph.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStoppingCompGraph.java index a70abd2fa..1a02ffd7f 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStoppingCompGraph.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/earlystopping/TestEarlyStoppingCompGraph.java @@ -45,10 +45,7 @@ import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.evaluation.classification.Evaluation; import org.nd4j.evaluation.regression.RegressionEvaluation.Metric; import org.nd4j.linalg.activations.Activation; @@ -70,13 +67,6 @@ import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) public class TestEarlyStoppingCompGraph extends BaseDL4JTest { @Override diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvalJsonTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvalJsonTest.java new file mode 100644 index 000000000..901a0e883 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvalJsonTest.java @@ -0,0 +1,284 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval; + +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; +import org.nd4j.evaluation.curves.Histogram; +import org.nd4j.evaluation.curves.PrecisionRecallCurve; +import org.nd4j.evaluation.curves.RocCurve; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; + + +public class EvalJsonTest extends BaseDL4JTest { + + @Test + public void testSerdeEmpty() { + boolean print = false; + + org.nd4j.evaluation.IEvaluation[] arr = new org.nd4j.evaluation.IEvaluation[] {new Evaluation(), new EvaluationBinary(), new ROCBinary(10), + new ROCMultiClass(10), new RegressionEvaluation(3), new RegressionEvaluation(), + new EvaluationCalibration()}; + + for (org.nd4j.evaluation.IEvaluation e : arr) { + String json = e.toJson(); + String stats = e.stats(); + if (print) { + System.out.println(e.getClass() + "\n" + json + "\n\n"); + } + + IEvaluation fromJson = (IEvaluation) org.nd4j.evaluation.BaseEvaluation.fromJson(json, org.nd4j.evaluation.BaseEvaluation.class); + assertEquals(e.toJson(), fromJson.toJson()); + } + } + + @Test + public void testSerde() { + boolean print = false; + Nd4j.getRandom().setSeed(12345); + + Evaluation evaluation = new Evaluation(); + EvaluationBinary evaluationBinary = new EvaluationBinary(); + ROC roc = new ROC(2); + ROCBinary roc2 = new ROCBinary(2); + ROCMultiClass roc3 = new ROCMultiClass(2); + RegressionEvaluation regressionEvaluation = new RegressionEvaluation(); + EvaluationCalibration ec = new EvaluationCalibration(); + + + org.nd4j.evaluation.IEvaluation[] arr = new org.nd4j.evaluation.IEvaluation[] {evaluation, evaluationBinary, roc, roc2, roc3, regressionEvaluation, ec}; + + INDArray evalLabel = Nd4j.create(10, 3); + for (int i = 0; i < 10; i++) { + evalLabel.putScalar(i, i % 3, 1.0); + } + INDArray evalProb = Nd4j.rand(10, 3); + evalProb.diviColumnVector(evalProb.sum(true,1)); + evaluation.eval(evalLabel, evalProb); + roc3.eval(evalLabel, evalProb); + ec.eval(evalLabel, evalProb); + + evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 3), 0.5)); + evalProb = Nd4j.rand(10, 3); + evaluationBinary.eval(evalLabel, evalProb); + roc2.eval(evalLabel, evalProb); + + evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 1), 0.5)); + evalProb = Nd4j.rand(10, 1); + roc.eval(evalLabel, evalProb); + + regressionEvaluation.eval(Nd4j.rand(10, 3), Nd4j.rand(10, 3)); + + + + for (org.nd4j.evaluation.IEvaluation e : arr) { + String json = e.toJson(); + if (print) { + System.out.println(e.getClass() + "\n" + json + "\n\n"); + } + + IEvaluation fromJson = (IEvaluation) BaseEvaluation.fromJson(json, org.nd4j.evaluation.BaseEvaluation.class); + assertEquals(e.toJson(), fromJson.toJson()); + } + } + + @Test + public void testSerdeExactRoc() { + Nd4j.getRandom().setSeed(12345); + boolean print = false; + + ROC roc = new ROC(0); + ROCBinary roc2 = new ROCBinary(0); + ROCMultiClass roc3 = new ROCMultiClass(0); + + + org.nd4j.evaluation.IEvaluation[] arr = new org.nd4j.evaluation.IEvaluation[] {roc, roc2, roc3}; + + INDArray evalLabel = Nd4j.create(100, 3); + for (int i = 0; i < 100; i++) { + evalLabel.putScalar(i, i % 3, 1.0); + } + INDArray evalProb = Nd4j.rand(100, 3); + evalProb.diviColumnVector(evalProb.sum(1)); + roc3.eval(evalLabel, evalProb); + + evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(100, 3), 0.5)); + evalProb = Nd4j.rand(100, 3); + roc2.eval(evalLabel, evalProb); + + evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(100, 1), 0.5)); + evalProb = Nd4j.rand(100, 1); + roc.eval(evalLabel, evalProb); + + for (org.nd4j.evaluation.IEvaluation e : arr) { + System.out.println(e.getClass()); + String json = e.toJson(); + String stats = e.stats(); + if (print) { + System.out.println(json + "\n\n"); + } + org.nd4j.evaluation.IEvaluation fromJson = BaseEvaluation.fromJson(json, org.nd4j.evaluation.BaseEvaluation.class); + assertEquals(e, fromJson); + + if (fromJson instanceof ROC) { + //Shouldn't have probAndLabel, but should have stored AUC and AUPRC + assertNull(((ROC) fromJson).getProbAndLabel()); + assertTrue(((ROC) fromJson).calculateAUC() > 0.0); + assertTrue(((ROC) fromJson).calculateAUCPR() > 0.0); + + assertEquals(((ROC) e).getRocCurve(), ((ROC) fromJson).getRocCurve()); + assertEquals(((ROC) e).getPrecisionRecallCurve(), ((ROC) fromJson).getPrecisionRecallCurve()); + } else if (e instanceof ROCBinary) { + org.nd4j.evaluation.classification.ROC[] rocs = ((ROCBinary) fromJson).getUnderlying(); + org.nd4j.evaluation.classification.ROC[] origRocs = ((ROCBinary) e).getUnderlying(); + // for(ROC r : rocs ){ + for (int i = 0; i < origRocs.length; i++) { + org.nd4j.evaluation.classification.ROC r = rocs[i]; + org.nd4j.evaluation.classification.ROC origR = origRocs[i]; + //Shouldn't have probAndLabel, but should have stored AUC and AUPRC, AND stored curves + assertNull(r.getProbAndLabel()); + assertEquals(origR.calculateAUC(), origR.calculateAUC(), 1e-6); + assertEquals(origR.calculateAUCPR(), origR.calculateAUCPR(), 1e-6); + assertEquals(origR.getRocCurve(), origR.getRocCurve()); + assertEquals(origR.getPrecisionRecallCurve(), origR.getPrecisionRecallCurve()); + } + + } else if (e instanceof ROCMultiClass) { + org.nd4j.evaluation.classification.ROC[] rocs = ((ROCMultiClass) fromJson).getUnderlying(); + org.nd4j.evaluation.classification.ROC[] origRocs = ((ROCMultiClass) e).getUnderlying(); + for (int i = 0; i < origRocs.length; i++) { + org.nd4j.evaluation.classification.ROC r = rocs[i]; + org.nd4j.evaluation.classification.ROC origR = origRocs[i]; + //Shouldn't have probAndLabel, but should have stored AUC and AUPRC, AND stored curves + assertNull(r.getProbAndLabel()); + assertEquals(origR.calculateAUC(), origR.calculateAUC(), 1e-6); + assertEquals(origR.calculateAUCPR(), origR.calculateAUCPR(), 1e-6); + assertEquals(origR.getRocCurve(), origR.getRocCurve()); + assertEquals(origR.getPrecisionRecallCurve(), origR.getPrecisionRecallCurve()); + } + } + } + } + + @Test + public void testJsonYamlCurves() { + ROC roc = new ROC(0); + + INDArray evalLabel = + Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(100, 1), 0.5)); + INDArray evalProb = Nd4j.rand(100, 1); + roc.eval(evalLabel, evalProb); + + RocCurve c = roc.getRocCurve(); + PrecisionRecallCurve prc = roc.getPrecisionRecallCurve(); + + String json1 = c.toJson(); + String json2 = prc.toJson(); + + RocCurve c2 = RocCurve.fromJson(json1); + PrecisionRecallCurve prc2 = PrecisionRecallCurve.fromJson(json2); + + assertEquals(c, c2); + assertEquals(prc, prc2); + + // System.out.println(json1); + + //Also test: histograms + + EvaluationCalibration ec = new EvaluationCalibration(); + + evalLabel = Nd4j.create(10, 3); + for (int i = 0; i < 10; i++) { + evalLabel.putScalar(i, i % 3, 1.0); + } + evalProb = Nd4j.rand(10, 3); + evalProb.diviColumnVector(evalProb.sum(1)); + ec.eval(evalLabel, evalProb); + + Histogram[] histograms = new Histogram[] {ec.getResidualPlotAllClasses(), ec.getResidualPlot(0), + ec.getResidualPlot(1), ec.getProbabilityHistogramAllClasses(), ec.getProbabilityHistogram(0), + ec.getProbabilityHistogram(1)}; + + for (Histogram h : histograms) { + String json = h.toJson(); + String yaml = h.toYaml(); + + Histogram h2 = Histogram.fromJson(json); + Histogram h3 = Histogram.fromYaml(yaml); + + assertEquals(h, h2); + assertEquals(h2, h3); + } + + } + + @Test + public void testJsonWithCustomThreshold() { + + //Evaluation - binary threshold + Evaluation e = new Evaluation(0.25); + String json = e.toJson(); + String yaml = e.toYaml(); + + Evaluation eFromJson = Evaluation.fromJson(json); + Evaluation eFromYaml = Evaluation.fromYaml(yaml); + + assertEquals(0.25, eFromJson.getBinaryDecisionThreshold(), 1e-6); + assertEquals(0.25, eFromYaml.getBinaryDecisionThreshold(), 1e-6); + + + //Evaluation: custom cost array + INDArray costArray = Nd4j.create(new double[] {1.0, 2.0, 3.0}); + Evaluation e2 = new Evaluation(costArray); + + json = e2.toJson(); + yaml = e2.toYaml(); + + eFromJson = Evaluation.fromJson(json); + eFromYaml = Evaluation.fromYaml(yaml); + + assertEquals(e2.getCostArray(), eFromJson.getCostArray()); + assertEquals(e2.getCostArray(), eFromYaml.getCostArray()); + + + + //EvaluationBinary - per-output binary threshold + INDArray threshold = Nd4j.create(new double[] {1.0, 0.5, 0.25}); + EvaluationBinary eb = new EvaluationBinary(threshold); + + json = eb.toJson(); + yaml = eb.toYaml(); + + EvaluationBinary ebFromJson = EvaluationBinary.fromJson(json); + EvaluationBinary ebFromYaml = EvaluationBinary.fromYaml(yaml); + + assertEquals(threshold, ebFromJson.getDecisionThreshold()); + assertEquals(threshold, ebFromYaml.getDecisionThreshold()); + + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvalTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvalTest.java new file mode 100644 index 000000000..c33a69c87 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvalTest.java @@ -0,0 +1,633 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval; + +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.FloatWritable; +import org.datavec.api.writable.Writable; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; +import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; +import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; +import org.deeplearning4j.datasets.iterator.IteratorMultiDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.*; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.EvaluativeListener; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.SplitTestAndTrain; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.resources.Resources; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +public class EvalTest extends BaseDL4JTest { + + @Test + public void testIris() { + + // Network config + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(42) + .updater(new Sgd(1e-6)).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(2).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).build()) + + .build(); + + // Instantiate model + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + model.addListeners(new ScoreIterationListener(1)); + + // Train-test split + DataSetIterator iter = new IrisDataSetIterator(150, 150); + DataSet next = iter.next(); + next.shuffle(); + SplitTestAndTrain trainTest = next.splitTestAndTrain(5, new Random(42)); + + // Train + DataSet train = trainTest.getTrain(); + train.normalizeZeroMeanZeroUnitVariance(); + + // Test + DataSet test = trainTest.getTest(); + test.normalizeZeroMeanZeroUnitVariance(); + INDArray testFeature = test.getFeatures(); + INDArray testLabel = test.getLabels(); + + // Fitting model + model.fit(train); + // Get predictions from test feature + INDArray testPredictedLabel = model.output(testFeature); + + // Eval with class number + org.nd4j.evaluation.classification.Evaluation eval = new org.nd4j.evaluation.classification.Evaluation(3); //// Specify class num here + eval.eval(testLabel, testPredictedLabel); + double eval1F1 = eval.f1(); + double eval1Acc = eval.accuracy(); + + // Eval without class number + org.nd4j.evaluation.classification.Evaluation eval2 = new org.nd4j.evaluation.classification.Evaluation(); //// No class num + eval2.eval(testLabel, testPredictedLabel); + double eval2F1 = eval2.f1(); + double eval2Acc = eval2.accuracy(); + + //Assert the two implementations give same f1 and accuracy (since one batch) + assertTrue(eval1F1 == eval2F1 && eval1Acc == eval2Acc); + + org.nd4j.evaluation.classification.Evaluation evalViaMethod = model.evaluate(new ListDataSetIterator<>(Collections.singletonList(test))); + checkEvaluationEquality(eval, evalViaMethod); + +// System.out.println(eval.getConfusionMatrix().toString()); +// System.out.println(eval.getConfusionMatrix().toCSV()); +// System.out.println(eval.getConfusionMatrix().toHTML()); +// System.out.println(eval.confusionToString()); + + eval.getConfusionMatrix().toString(); + eval.getConfusionMatrix().toCSV(); + eval.getConfusionMatrix().toHTML(); + eval.confusionToString(); + } + + private static void assertMapEquals(Map first, Map second) { + assertEquals(first.keySet(), second.keySet()); + for (Integer i : first.keySet()) { + assertEquals(first.get(i), second.get(i)); + } + } + + private static void checkEvaluationEquality(org.nd4j.evaluation.classification.Evaluation evalExpected, org.nd4j.evaluation.classification.Evaluation evalActual) { + assertEquals(evalExpected.accuracy(), evalActual.accuracy(), 1e-3); + assertEquals(evalExpected.f1(), evalActual.f1(), 1e-3); + assertEquals(evalExpected.getNumRowCounter(), evalActual.getNumRowCounter(), 1e-3); + assertMapEquals(evalExpected.falseNegatives(), evalActual.falseNegatives()); + assertMapEquals(evalExpected.falsePositives(), evalActual.falsePositives()); + assertMapEquals(evalExpected.trueNegatives(), evalActual.trueNegatives()); + assertMapEquals(evalExpected.truePositives(), evalActual.truePositives()); + assertEquals(evalExpected.precision(), evalActual.precision(), 1e-3); + assertEquals(evalExpected.recall(), evalActual.recall(), 1e-3); + assertEquals(evalExpected.falsePositiveRate(), evalActual.falsePositiveRate(), 1e-3); + assertEquals(evalExpected.falseNegativeRate(), evalActual.falseNegativeRate(), 1e-3); + assertEquals(evalExpected.falseAlarmRate(), evalActual.falseAlarmRate(), 1e-3); + assertEquals(evalExpected.getConfusionMatrix(), evalActual.getConfusionMatrix()); + } + + @Test + public void testEvaluationWithMetaData() throws Exception { + + RecordReader csv = new CSVRecordReader(); + csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); + + int batchSize = 10; + int labelIdx = 4; + int numClasses = 3; + + RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(csv, batchSize, labelIdx, numClasses); + + NormalizerStandardize ns = new NormalizerStandardize(); + ns.fit(rrdsi); + rrdsi.setPreProcessor(ns); + rrdsi.reset(); + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)) + .list() + .layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(4).nOut(3).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + for (int i = 0; i < 4; i++) { + net.fit(rrdsi); + rrdsi.reset(); + } + + org.nd4j.evaluation.classification.Evaluation e = new org.nd4j.evaluation.classification.Evaluation(); + rrdsi.setCollectMetaData(true); //*** New: Enable collection of metadata (stored in the DataSets) *** + + while (rrdsi.hasNext()) { + DataSet ds = rrdsi.next(); + List meta = ds.getExampleMetaData(RecordMetaData.class); //*** New - cross dependencies here make types difficult, usid Object internally in DataSet for this*** + + INDArray out = net.output(ds.getFeatures()); + e.eval(ds.getLabels(), out, meta); //*** New - evaluate and also store metadata *** + } + +// System.out.println(e.stats()); + e.stats(); + +// System.out.println("\n\n*** Prediction Errors: ***"); + + List errors = e.getPredictionErrors(); //*** New - get list of prediction errors from evaluation *** + List metaForErrors = new ArrayList<>(); + for (org.nd4j.evaluation.meta.Prediction p : errors) { + metaForErrors.add((RecordMetaData) p.getRecordMetaData()); + } + DataSet ds = rrdsi.loadFromMetaData(metaForErrors); //*** New - dynamically load a subset of the data, just for prediction errors *** + INDArray output = net.output(ds.getFeatures()); + + int count = 0; + for (org.nd4j.evaluation.meta.Prediction t : errors) { + String s = t + "\t\tRaw Data: " + + csv.loadFromMetaData((RecordMetaData) t.getRecordMetaData()).getRecord() //*** New - load subset of data from MetaData object (usually batched for efficiency) *** + + "\tNormalized: " + ds.getFeatures().getRow(count) + "\tLabels: " + + ds.getLabels().getRow(count) + "\tNetwork predictions: " + output.getRow(count); +// System.out.println(s); + count++; + } + + int errorCount = errors.size(); + double expAcc = 1.0 - errorCount / 150.0; + assertEquals(expAcc, e.accuracy(), 1e-5); + + org.nd4j.evaluation.classification.ConfusionMatrix confusion = e.getConfusionMatrix(); + int[] actualCounts = new int[3]; + int[] predictedCounts = new int[3]; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + int entry = confusion.getCount(i, j); //(actual,predicted) + List list = e.getPredictions(i, j); + assertEquals(entry, list.size()); + + actualCounts[i] += entry; + predictedCounts[j] += entry; + } + } + + for (int i = 0; i < 3; i++) { + List actualClassI = e.getPredictionsByActualClass(i); + List predictedClassI = e.getPredictionByPredictedClass(i); + assertEquals(actualCounts[i], actualClassI.size()); + assertEquals(predictedCounts[i], predictedClassI.size()); + } + + + //Finally: test doEvaluation methods + rrdsi.reset(); + org.nd4j.evaluation.classification.Evaluation e2 = new org.nd4j.evaluation.classification.Evaluation(); + net.doEvaluation(rrdsi, e2); + for (int i = 0; i < 3; i++) { + List actualClassI = e2.getPredictionsByActualClass(i); + List predictedClassI = e2.getPredictionByPredictedClass(i); + assertEquals(actualCounts[i], actualClassI.size()); + assertEquals(predictedCounts[i], predictedClassI.size()); + } + + ComputationGraph cg = net.toComputationGraph(); + rrdsi.reset(); + e2 = new org.nd4j.evaluation.classification.Evaluation(); + cg.doEvaluation(rrdsi, e2); + for (int i = 0; i < 3; i++) { + List actualClassI = e2.getPredictionsByActualClass(i); + List predictedClassI = e2.getPredictionByPredictedClass(i); + assertEquals(actualCounts[i], actualClassI.size()); + assertEquals(predictedCounts[i], predictedClassI.size()); + } + + } + + private static void apply(org.nd4j.evaluation.classification.Evaluation e, int nTimes, INDArray predicted, INDArray actual) { + for (int i = 0; i < nTimes; i++) { + e.eval(actual, predicted); + } + } + + @Test + public void testEvalSplitting(){ + //Test for "tbptt-like" functionality + + for(WorkspaceMode ws : WorkspaceMode.values()) { + System.out.println("Starting test for workspace mode: " + ws); + + int nIn = 4; + int layerSize = 5; + int nOut = 6; + int tbpttLength = 10; + int tsLength = 5 * tbpttLength + tbpttLength / 2; + + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder() + .seed(12345) + .trainingWorkspaceMode(ws) + .inferenceWorkspaceMode(ws) + .list() + .layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).build()) + .layer(new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut) + .activation(Activation.SOFTMAX) + .build()) + .build(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .seed(12345) + .trainingWorkspaceMode(ws) + .inferenceWorkspaceMode(ws) + .list() + .layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).build()) + .layer(new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut) + .activation(Activation.SOFTMAX).build()) + .tBPTTLength(10) + .backpropType(BackpropType.TruncatedBPTT) + .build(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + net2.setParams(net1.params()); + + for(boolean useMask : new boolean[]{false, true}) { + + INDArray in1 = Nd4j.rand(new int[]{3, nIn, tsLength}); + INDArray out1 = TestUtils.randomOneHotTimeSeries(3, nOut, tsLength); + + INDArray in2 = Nd4j.rand(new int[]{5, nIn, tsLength}); + INDArray out2 = TestUtils.randomOneHotTimeSeries(5, nOut, tsLength); + + INDArray lMask1 = null; + INDArray lMask2 = null; + if(useMask){ + lMask1 = Nd4j.create(3, tsLength); + lMask2 = Nd4j.create(5, tsLength); + Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask1, 0.5)); + Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask2, 0.5)); + } + + List l = Arrays.asList(new DataSet(in1, out1, null, lMask1), new DataSet(in2, out2, null, lMask2)); + DataSetIterator iter = new ExistingDataSetIterator(l); + +// System.out.println("Net 1 eval"); + org.nd4j.evaluation.IEvaluation[] e1 = net1.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation()); +// System.out.println("Net 2 eval"); + org.nd4j.evaluation.IEvaluation[] e2 = net2.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation()); + + assertEquals(e1[0], e2[0]); + assertEquals(e1[1], e2[1]); + assertEquals(e1[2], e2[2]); + } + } + } + + @Test + public void testEvalSplittingCompGraph(){ + //Test for "tbptt-like" functionality + + for(WorkspaceMode ws : WorkspaceMode.values()) { + System.out.println("Starting test for workspace mode: " + ws); + + int nIn = 4; + int layerSize = 5; + int nOut = 6; + int tbpttLength = 10; + int tsLength = 5 * tbpttLength + tbpttLength / 2; + + ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder() + .seed(12345) + .trainingWorkspaceMode(ws) + .inferenceWorkspaceMode(ws) + .graphBuilder() + .addInputs("in") + .addLayer("0", new LSTM.Builder().nIn(nIn).nOut(layerSize).build(), "in") + .addLayer("1", new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut) + .activation(Activation.SOFTMAX) + .build(), "0") + .setOutputs("1") + .build(); + + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() + .seed(12345) + .trainingWorkspaceMode(ws) + .inferenceWorkspaceMode(ws) + .graphBuilder() + .addInputs("in") + .addLayer("0", new LSTM.Builder().nIn(nIn).nOut(layerSize).build(), "in") + .addLayer("1", new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut) + .activation(Activation.SOFTMAX) + .build(), "0") + .setOutputs("1") + .tBPTTLength(10) + .backpropType(BackpropType.TruncatedBPTT) + .build(); + + ComputationGraph net1 = new ComputationGraph(conf1); + net1.init(); + + ComputationGraph net2 = new ComputationGraph(conf2); + net2.init(); + + net2.setParams(net1.params()); + + for (boolean useMask : new boolean[]{false, true}) { + + INDArray in1 = Nd4j.rand(new int[]{3, nIn, tsLength}); + INDArray out1 = TestUtils.randomOneHotTimeSeries(3, nOut, tsLength); + + INDArray in2 = Nd4j.rand(new int[]{5, nIn, tsLength}); + INDArray out2 = TestUtils.randomOneHotTimeSeries(5, nOut, tsLength); + + INDArray lMask1 = null; + INDArray lMask2 = null; + if (useMask) { + lMask1 = Nd4j.create(3, tsLength); + lMask2 = Nd4j.create(5, tsLength); + Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask1, 0.5)); + Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask2, 0.5)); + } + + List l = Arrays.asList(new DataSet(in1, out1), new DataSet(in2, out2)); + DataSetIterator iter = new ExistingDataSetIterator(l); + +// System.out.println("Eval net 1"); + org.nd4j.evaluation.IEvaluation[] e1 = net1.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation()); +// System.out.println("Eval net 2"); + org.nd4j.evaluation.IEvaluation[] e2 = net2.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation()); + + assertEquals(e1[0], e2[0]); + assertEquals(e1[1], e2[1]); + assertEquals(e1[2], e2[2]); + } + } + } + + @Test + public void testEvalSplitting2(){ + List> seqFeatures = new ArrayList<>(); + List step = Arrays.asList(new FloatWritable(0), new FloatWritable(0), new FloatWritable(0)); + for( int i=0; i<30; i++ ){ + seqFeatures.add(step); + } + List> seqLabels = Collections.singletonList(Collections.singletonList(new FloatWritable(0))); + + SequenceRecordReader fsr = new CollectionSequenceRecordReader(Collections.singletonList(seqFeatures)); + SequenceRecordReader lsr = new CollectionSequenceRecordReader(Collections.singletonList(seqLabels)); + + + DataSetIterator testData = new SequenceRecordReaderDataSetIterator(fsr, lsr, 1, -1, true, + SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123) + .list() + .layer(0, new LSTM.Builder().activation(Activation.TANH).nIn(3).nOut(3).build()) + .layer(1, new RnnOutputLayer.Builder().activation(Activation.SIGMOID).lossFunction(LossFunctions.LossFunction.XENT) + .nIn(3).nOut(1).build()) + .backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(10).tBPTTBackwardLength(10) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + net.evaluate(testData); + } + + @Test + public void testEvaluativeListenerSimple(){ + //Sanity check: https://github.com/eclipse/deeplearning4j/issues/5351 + + // Network config + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(42) + .updater(new Sgd(1e-6)).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(2).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).build()) + .build(); + + // Instantiate model + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + // Train-test split + DataSetIterator iter = new IrisDataSetIterator(30, 150); + DataSetIterator iterTest = new IrisDataSetIterator(30, 150); + + net.setListeners(new EvaluativeListener(iterTest, 3)); + + for( int i=0; i<3; i++ ){ + net.fit(iter); + } + } + + @Test + public void testMultiOutputEvalSimple(){ + Nd4j.getRandom().setSeed(12345); + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345) + .graphBuilder() + .addInputs("in") + .addLayer("out1", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build(), "in") + .addLayer("out2", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build(), "in") + .setOutputs("out1", "out2") + .build(); + + ComputationGraph cg = new ComputationGraph(conf); + cg.init(); + + List list = new ArrayList<>(); + DataSetIterator iter = new IrisDataSetIterator(30, 150); + while(iter.hasNext()){ + DataSet ds = iter.next(); + list.add(new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[]{ds.getFeatures()}, new INDArray[]{ds.getLabels(), ds.getLabels()})); + } + + org.nd4j.evaluation.classification.Evaluation e = new org.nd4j.evaluation.classification.Evaluation(); + org.nd4j.evaluation.regression.RegressionEvaluation e2 = new org.nd4j.evaluation.regression.RegressionEvaluation(); + Map evals = new HashMap<>(); + evals.put(0, new org.nd4j.evaluation.IEvaluation[]{e}); + evals.put(1, new org.nd4j.evaluation.IEvaluation[]{e2}); + + cg.evaluate(new IteratorMultiDataSetIterator(list.iterator(), 30), evals); + + assertEquals(150, e.getNumRowCounter()); + assertEquals(150, e2.getExampleCountPerColumn().getInt(0)); + } + + @Test + public void testMultiOutputEvalCG(){ + //Simple sanity check on evaluation + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .graphBuilder() + .addInputs("in") + .layer("0", new EmbeddingSequenceLayer.Builder().nIn(10).nOut(10).build(), "in") + .layer("1", new LSTM.Builder().nIn(10).nOut(10).build(), "0") + .layer("2", new LSTM.Builder().nIn(10).nOut(10).build(), "0") + .layer("out1", new RnnOutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build(), "1") + .layer("out2", new RnnOutputLayer.Builder().nIn(10).nOut(20).activation(Activation.SOFTMAX).build(), "2") + .setOutputs("out1", "out2") + .build(); + + ComputationGraph cg = new ComputationGraph(conf); + cg.init(); + + org.nd4j.linalg.dataset.MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet( + new INDArray[]{Nd4j.create(10, 1, 10)}, + new INDArray[]{Nd4j.create(10, 10, 10), Nd4j.create(10, 20, 10)}); + + Map m = new HashMap<>(); + m.put(0, new org.nd4j.evaluation.IEvaluation[]{new org.nd4j.evaluation.classification.Evaluation()}); + m.put(1, new org.nd4j.evaluation.IEvaluation[]{new org.nd4j.evaluation.classification.Evaluation()}); + + cg.evaluate(new SingletonMultiDataSetIterator(mds), m); + } + + @Test + public void testInvalidEvaluation(){ + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + + .list() + .layer(new DenseLayer.Builder().nIn(4).nOut(10).build()) + .layer(new OutputLayer.Builder().nIn(10).nOut(3).lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.RELU).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSetIterator iter = new IrisDataSetIterator(150, 150); + try { + net.evaluate(iter); + fail("Expected exception"); + } catch (IllegalStateException e){ + assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("Evaluation")); + } + + try { + net.evaluateROC(iter, 0); + fail("Expected exception"); + } catch (IllegalStateException e){ + assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROC")); + } + + try { + net.evaluateROCMultiClass(iter, 0); + fail("Expected exception"); + } catch (IllegalStateException e){ + assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROCMultiClass")); + } + + ComputationGraph cg = net.toComputationGraph(); + try { + cg.evaluate(iter); + fail("Expected exception"); + } catch (IllegalStateException e){ + assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("Evaluation")); + } + + try { + cg.evaluateROC(iter, 0); + fail("Expected exception"); + } catch (IllegalStateException e){ + assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROC")); + } + + try { + cg.evaluateROCMultiClass(iter, 0); + fail("Expected exception"); + } catch (IllegalStateException e){ + assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROCMultiClass")); + } + + + //Disable validation, and check same thing: + net.getLayerWiseConfigurations().setValidateOutputLayerConfig(false); + net.evaluate(iter); + net.evaluateROCMultiClass(iter, 0); + + cg.getConfiguration().setValidateOutputLayerConfig(false); + cg.evaluate(iter); + cg.evaluateROCMultiClass(iter, 0); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvaluationToolsTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvaluationToolsTests.java similarity index 95% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvaluationToolsTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvaluationToolsTests.java index 09fd51696..70271cd95 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvaluationToolsTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/EvaluationToolsTests.java @@ -29,10 +29,7 @@ import org.deeplearning4j.nn.conf.layers.DenseLayer; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.api.DataSet; @@ -43,11 +40,7 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; import java.util.Arrays; import java.util.Random; -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.JACKSON_SERDE) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) + public class EvaluationToolsTests extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/ROCTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/ROCTest.java new file mode 100644 index 000000000..629ce0d9b --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/ROCTest.java @@ -0,0 +1,138 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.evaluation.curves.PrecisionRecallCurve; +import org.nd4j.evaluation.curves.RocCurve; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +public class ROCTest extends BaseDL4JTest { + + private static Map expTPR; + private static Map expFPR; + + static { + expTPR = new HashMap<>(); + double totalPositives = 5.0; + expTPR.put(0 / 10.0, 5.0 / totalPositives); //All 10 predicted as class 1, of which 5 of 5 are correct + expTPR.put(1 / 10.0, 5.0 / totalPositives); + expTPR.put(2 / 10.0, 5.0 / totalPositives); + expTPR.put(3 / 10.0, 5.0 / totalPositives); + expTPR.put(4 / 10.0, 5.0 / totalPositives); + expTPR.put(5 / 10.0, 5.0 / totalPositives); + expTPR.put(6 / 10.0, 4.0 / totalPositives); //Threshold: 0.4 -> last 4 predicted; last 5 actual + expTPR.put(7 / 10.0, 3.0 / totalPositives); + expTPR.put(8 / 10.0, 2.0 / totalPositives); + expTPR.put(9 / 10.0, 1.0 / totalPositives); + expTPR.put(10 / 10.0, 0.0 / totalPositives); + + expFPR = new HashMap<>(); + double totalNegatives = 5.0; + expFPR.put(0 / 10.0, 5.0 / totalNegatives); //All 10 predicted as class 1, but all 5 true negatives are predicted positive + expFPR.put(1 / 10.0, 4.0 / totalNegatives); //1 true negative is predicted as negative; 4 false positives + expFPR.put(2 / 10.0, 3.0 / totalNegatives); //2 true negatives are predicted as negative; 3 false positives + expFPR.put(3 / 10.0, 2.0 / totalNegatives); + expFPR.put(4 / 10.0, 1.0 / totalNegatives); + expFPR.put(5 / 10.0, 0.0 / totalNegatives); + expFPR.put(6 / 10.0, 0.0 / totalNegatives); + expFPR.put(7 / 10.0, 0.0 / totalNegatives); + expFPR.put(8 / 10.0, 0.0 / totalNegatives); + expFPR.put(9 / 10.0, 0.0 / totalNegatives); + expFPR.put(10 / 10.0, 0.0 / totalNegatives); + } + + @Test + public void RocEvalSanityCheck() { + + DataSetIterator iter = new IrisDataSetIterator(150, 150); + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).seed(12345) + .list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()).layer(1, + new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + NormalizerStandardize ns = new NormalizerStandardize(); + DataSet ds = iter.next(); + ns.fit(ds); + ns.transform(ds); + + iter.setPreProcessor(ns); + + for (int i = 0; i < 10; i++) { + net.fit(ds); + } + + for (int steps : new int[] {32, 0}) { //Steps = 0: exact + System.out.println("steps: " + steps); + + iter.reset(); + ds = iter.next(); + INDArray f = ds.getFeatures(); + INDArray l = ds.getLabels(); + INDArray out = net.output(f); + // System.out.println(f); + // System.out.println(out); + ROCMultiClass manual = new ROCMultiClass(steps); + manual.eval(l, out); + + iter.reset(); + ROCMultiClass roc = net.evaluateROCMultiClass(iter, steps); + + + for (int i = 0; i < 3; i++) { + double rocExp = manual.calculateAUC(i); + double rocAct = roc.calculateAUC(i); + assertEquals(rocExp, rocAct, 1e-6); + + RocCurve rc = roc.getRocCurve(i); + RocCurve rm = manual.getRocCurve(i); + + assertEquals(rc, rm); + } + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/RegressionEvalTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/RegressionEvalTest.java new file mode 100644 index 000000000..23e69502c --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/eval/RegressionEvalTest.java @@ -0,0 +1,139 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.nd4j.linalg.indexing.NDArrayIndex.all; +import static org.nd4j.linalg.indexing.NDArrayIndex.interval; + +public class RegressionEvalTest extends BaseDL4JTest { + + @Test + public void testRegressionEvalMethods() { + + //Basic sanity check + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.ZERO).list() + .layer(0, new OutputLayer.Builder().activation(Activation.TANH) + .lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(5).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray f = Nd4j.zeros(4, 10); + INDArray l = Nd4j.ones(4, 5); + + DataSet ds = new DataSet(f, l); + DataSetIterator iter = new ExistingDataSetIterator(Collections.singletonList(ds)); + org.nd4j.evaluation.regression.RegressionEvaluation re = net.evaluateRegression(iter); + + for (int i = 0; i < 5; i++) { + assertEquals(1.0, re.meanSquaredError(i), 1e-6); + assertEquals(1.0, re.meanAbsoluteError(i), 1e-6); + } + + + ComputationGraphConfiguration graphConf = + new NeuralNetConfiguration.Builder().weightInit(WeightInit.ZERO).graphBuilder() + .addInputs("in").addLayer("0", new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE) + .activation(Activation.TANH).nIn(10).nOut(5).build(), "in") + .setOutputs("0").build(); + + ComputationGraph cg = new ComputationGraph(graphConf); + cg.init(); + + RegressionEvaluation re2 = cg.evaluateRegression(iter); + + for (int i = 0; i < 5; i++) { + assertEquals(1.0, re2.meanSquaredError(i), 1e-6); + assertEquals(1.0, re2.meanAbsoluteError(i), 1e-6); + } + } + + @Test + public void testRegressionEvalPerOutputMasking() { + + INDArray l = Nd4j.create(new double[][] {{1, 2, 3}, {10, 20, 30}, {-5, -10, -20}}); + + INDArray predictions = Nd4j.zeros(l.shape()); + + INDArray mask = Nd4j.create(new double[][] {{0, 1, 1}, {1, 1, 0}, {0, 1, 0}}); + + + RegressionEvaluation re = new RegressionEvaluation(); + + re.eval(l, predictions, mask); + + double[] mse = new double[] {(10 * 10) / 1.0, (2 * 2 + 20 * 20 + 10 * 10) / 3, (3 * 3) / 1.0}; + + double[] mae = new double[] {10.0, (2 + 20 + 10) / 3.0, 3.0}; + + double[] rmse = new double[] {10.0, Math.sqrt((2 * 2 + 20 * 20 + 10 * 10) / 3.0), 3.0}; + + for (int i = 0; i < 3; i++) { + assertEquals(mse[i], re.meanSquaredError(i), 1e-6); + assertEquals(mae[i], re.meanAbsoluteError(i), 1e-6); + assertEquals(rmse[i], re.rootMeanSquaredError(i), 1e-6); + } + } + + @Test + public void testRegressionEvalTimeSeriesSplit(){ + + INDArray out1 = Nd4j.rand(new int[]{3, 5, 20}); + INDArray outSub1 = out1.get(all(), all(), interval(0,10)); + INDArray outSub2 = out1.get(all(), all(), interval(10, 20)); + + INDArray label1 = Nd4j.rand(new int[]{3, 5, 20}); + INDArray labelSub1 = label1.get(all(), all(), interval(0,10)); + INDArray labelSub2 = label1.get(all(), all(), interval(10, 20)); + + RegressionEvaluation e1 = new RegressionEvaluation(); + RegressionEvaluation e2 = new RegressionEvaluation(); + + e1.eval(label1, out1); + + e2.eval(labelSub1, outSub1); + e2.eval(labelSub2, outSub2); + + assertEquals(e1, e2); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidConfigurations.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidConfigurations.java similarity index 94% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidConfigurations.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidConfigurations.java index 16cacfd5c..0a09599bb 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidConfigurations.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidConfigurations.java @@ -30,10 +30,7 @@ import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.factory.Nd4j; @@ -41,10 +38,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; @Slf4j -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) public class TestInvalidConfigurations extends BaseDL4JTest { public static MultiLayerNetwork getDensePlusOutput(int nIn, int nOut) { @@ -363,99 +356,87 @@ public class TestInvalidConfigurations extends BaseDL4JTest { } } - @Test() + @Test public void testCnnInvalidKernel() { assertThrows(IllegalStateException.class, () -> { new ConvolutionLayer.Builder().kernelSize(3, 0).build(); - }); } - @Test() + @Test public void testCnnInvalidKernel2() { - assertThrows(IllegalArgumentException.class, () -> { + assertThrows(IllegalStateException.class, () -> { new ConvolutionLayer.Builder().kernelSize(2, 2, 2).build(); - }); } - @Test() + @Test public void testCnnInvalidStride() { - assertThrows(IllegalStateException.class,() -> { + assertThrows(IllegalStateException.class, () -> { new ConvolutionLayer.Builder().kernelSize(3, 3).stride(0, 1).build(); - }); } - @Test() + @Test public void testCnnInvalidStride2() { - assertThrows(IllegalArgumentException.class,() -> { + assertThrows(IllegalArgumentException.class, () -> { new ConvolutionLayer.Builder().kernelSize(3, 3).stride(1).build(); - }); } - @Test() + @Test public void testCnnInvalidPadding() { - assertThrows(IllegalArgumentException.class,() -> { + assertThrows(IllegalArgumentException.class, () -> { new ConvolutionLayer.Builder().kernelSize(3, 3).stride(1, 1).padding(-1, 0).build(); - }); } - @Test() + @Test public void testCnnInvalidPadding2() { - assertThrows(IllegalArgumentException.class,() -> { + assertThrows(IllegalArgumentException.class, () -> { new ConvolutionLayer.Builder().kernelSize(3, 3).stride(1, 1).padding(0, 0, 0).build(); - }); } - @Test() + @Test public void testSubsamplingInvalidKernel() { - assertThrows(IllegalStateException.class,() -> { + assertThrows(IllegalStateException.class, () -> { new SubsamplingLayer.Builder().kernelSize(3, 0).build(); - }); } - @Test() + @Test public void testSubsamplingInvalidKernel2() { - assertThrows(IllegalArgumentException.class,() -> { + assertThrows(IllegalArgumentException.class, () -> { new SubsamplingLayer.Builder().kernelSize(2).build(); - }); } - @Test() + @Test public void testSubsamplingInvalidStride() { - assertThrows(IllegalStateException.class,() -> { + assertThrows(IllegalStateException.class, () -> { new SubsamplingLayer.Builder().kernelSize(3, 3).stride(0, 1).build(); - }); } - @Test() + @Test public void testSubsamplingInvalidStride2() { - assertThrows(RuntimeException.class,() -> { + assertThrows(RuntimeException.class, () -> { new SubsamplingLayer.Builder().kernelSize(3, 3).stride(1, 1, 1).build(); - }); } - @Test() + @Test public void testSubsamplingInvalidPadding() { - assertThrows(IllegalArgumentException.class,() -> { + assertThrows(IllegalArgumentException.class, () -> { new SubsamplingLayer.Builder().kernelSize(3, 3).stride(1, 1).padding(-1, 0).build(); - }); } - @Test() + @Test public void testSubsamplingInvalidPadding2() { - assertThrows(RuntimeException.class,() -> { + assertThrows(RuntimeException.class, () -> { new SubsamplingLayer.Builder().kernelSize(3, 3).stride(1, 1).padding(0).build(); - }); } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidInput.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidInput.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidInput.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidInput.java index 52a139167..7d958355a 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidInput.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestInvalidInput.java @@ -29,10 +29,7 @@ import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -42,10 +39,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) public class TestInvalidInput extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestRecordReaders.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestRecordReaders.java new file mode 100644 index 000000000..868ec0809 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/exceptions/TestRecordReaders.java @@ -0,0 +1,131 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.exceptions; + +import org.datavec.api.records.reader.impl.collection.CollectionRecordReader; +import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader; +import org.datavec.api.writable.DoubleWritable; +import org.datavec.api.writable.IntWritable; +import org.datavec.api.writable.Writable; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; +import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; +import org.deeplearning4j.exception.DL4JException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestRecordReaders extends BaseDL4JTest { + + @Test + public void testClassIndexOutsideOfRangeRRDSI() { + Collection> c = new ArrayList<>(); + c.add(Arrays.asList(new DoubleWritable(0.5), new IntWritable(0))); + c.add(Arrays.asList(new DoubleWritable(1.0), new IntWritable(2))); + + CollectionRecordReader crr = new CollectionRecordReader(c); + + RecordReaderDataSetIterator iter = new RecordReaderDataSetIterator(crr, 2, 1, 2); + + try { + Assertions.assertThrows(Exception.class, + () -> { + DataSet ds = iter.next(); + }); + } catch (Exception e) { + assertTrue(e.getMessage().contains("to one-hot"), e.getMessage()); + } + } + + @Test + public void testClassIndexOutsideOfRangeRRMDSI() { + + Collection>> c = new ArrayList<>(); + Collection> seq1 = new ArrayList<>(); + seq1.add(Arrays.asList(new DoubleWritable(0.0), new IntWritable(0))); + seq1.add(Arrays.asList(new DoubleWritable(0.0), new IntWritable(1))); + c.add(seq1); + + Collection> seq2 = new ArrayList<>(); + seq2.add(Arrays.asList(new DoubleWritable(0.0), new IntWritable(0))); + seq2.add(Arrays.asList(new DoubleWritable(0.0), new IntWritable(2))); + c.add(seq2); + + CollectionSequenceRecordReader csrr = new CollectionSequenceRecordReader(c); + DataSetIterator dsi = new SequenceRecordReaderDataSetIterator(csrr, 2, 2, 1); + + try { + Assertions.assertThrows(Exception.class, + () -> { + DataSet ds = dsi.next(); + }); + } catch (Exception e) { + assertTrue(e.getMessage().contains("to one-hot"), e.getMessage()); + } + } + + @Test + public void testClassIndexOutsideOfRangeRRMDSI_MultipleReaders() { + + Collection>> c1 = new ArrayList<>(); + Collection> seq1 = new ArrayList<>(); + seq1.add(Arrays.asList(new DoubleWritable(0.0))); + seq1.add(Arrays.asList(new DoubleWritable(0.0))); + c1.add(seq1); + + Collection> seq2 = new ArrayList<>(); + seq2.add(Arrays.asList(new DoubleWritable(0.0))); + seq2.add(Arrays.asList(new DoubleWritable(0.0))); + c1.add(seq2); + + Collection>> c2 = new ArrayList<>(); + Collection> seq1a = new ArrayList<>(); + seq1a.add(Arrays.asList(new IntWritable(0))); + seq1a.add(Arrays.asList(new IntWritable(1))); + c2.add(seq1a); + + Collection> seq2a = new ArrayList<>(); + seq2a.add(Arrays.asList(new IntWritable(0))); + seq2a.add(Arrays.asList(new IntWritable(2))); + c2.add(seq2a); + + CollectionSequenceRecordReader csrr = new CollectionSequenceRecordReader(c1); + CollectionSequenceRecordReader csrrLabels = new CollectionSequenceRecordReader(c2); + DataSetIterator dsi = new SequenceRecordReaderDataSetIterator(csrr, csrrLabels, 2, 2); + + try { + Assertions.assertThrows(Exception.class, + () -> { + DataSet ds = dsi.next(); + }); + } catch (Exception e) { + assertTrue(e.getMessage().contains("to one-hot"), e.getMessage()); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java new file mode 100644 index 000000000..f39be0929 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java @@ -0,0 +1,460 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.gradientcheck; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.graph.AttentionVertex; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +////@Ignore +public class AttentionLayerTest extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return 90000L; + } + + @Test + public void testSelfAttentionLayer() { + int nIn = 3; + int nOut = 2; + int tsLength = 4; + int layerSize = 4; + + for (int mb : new int[]{1, 3}) { + for (boolean inputMask : new boolean[]{false, true}) { + for (boolean projectInput : new boolean[]{false, true}) { + INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{mb, nIn, tsLength}); + INDArray labels = TestUtils.randomOneHot(mb, nOut); + String maskType = (inputMask ? "inputMask" : "none"); + + INDArray inMask = null; + if (inputMask) { + inMask = Nd4j.ones(mb, tsLength); + for (int i = 0; i < mb; i++) { + int firstMaskedStep = tsLength - 1 - i; + if (firstMaskedStep == 0) { + firstMaskedStep = tsLength; + } + for (int j = firstMaskedStep; j < tsLength; j++) { + inMask.putScalar(i, j, 0.0); + } + } + } + + String name = "testSelfAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; + System.out.println("Starting test: " + name); + + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .list() + .layer(new LSTM.Builder().nOut(layerSize).build()) + .layer( projectInput ? + new SelfAttentionLayer.Builder().nOut(4).nHeads(2).projectInput(true).build() + : new SelfAttentionLayer.Builder().nHeads(1).projectInput(false).build() + ) + .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build()) + .layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.recurrent(nIn)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in) + .labels(labels).inputMask(inMask).subset(true).maxPerParam(100)); + assertTrue(gradOK, name); + } + } + } + } + + @Test + public void testLearnedSelfAttentionLayer() { + int nIn = 3; + int nOut = 2; + int tsLength = 4; + int layerSize = 4; + int numQueries = 3; + + for (boolean inputMask : new boolean[]{false, true}) { + for (int mb : new int[]{3, 1}) { + for (boolean projectInput : new boolean[]{false, true}) { + INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{mb, nIn, tsLength}); + INDArray labels = TestUtils.randomOneHot(mb, nOut); + String maskType = (inputMask ? "inputMask" : "none"); + + INDArray inMask = null; + if (inputMask) { + inMask = Nd4j.ones(mb, tsLength); + for (int i = 0; i < mb; i++) { + int firstMaskedStep = tsLength - 1 - i; + if (firstMaskedStep == 0) { + firstMaskedStep = tsLength; + } + for (int j = firstMaskedStep; j < tsLength; j++) { + inMask.putScalar(i, j, 0.0); + } + } + } + + String name = "testLearnedSelfAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; + System.out.println("Starting test: " + name); + + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .list() + .layer(new LSTM.Builder().nOut(layerSize).build()) + .layer( projectInput ? + new LearnedSelfAttentionLayer.Builder().nOut(4).nHeads(2).nQueries(numQueries).projectInput(true).build() + : new LearnedSelfAttentionLayer.Builder().nHeads(1).nQueries(numQueries).projectInput(false).build() + ) + .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build()) + .layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.recurrent(nIn)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in) + .labels(labels).inputMask(inMask).subset(true).maxPerParam(100)); + assertTrue(gradOK, name); + } + } + } + } + + @Test + public void testLearnedSelfAttentionLayer_differentMiniBatchSizes() { + int nIn = 3; + int nOut = 2; + int tsLength = 4; + int layerSize = 4; + int numQueries = 3; + + Random r = new Random(12345); + for (boolean inputMask : new boolean[]{false, true}) { + for (boolean projectInput : new boolean[]{false, true}) { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .list() + .layer(new LSTM.Builder().nOut(layerSize).build()) + .layer( projectInput ? + new LearnedSelfAttentionLayer.Builder().nOut(4).nHeads(2).nQueries(numQueries).projectInput(true).build() + : new LearnedSelfAttentionLayer.Builder().nHeads(1).nQueries(numQueries).projectInput(false).build() + ) + .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build()) + .layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.recurrent(nIn)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + for (int mb : new int[]{3, 1}) { + INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{mb, nIn, tsLength}); + INDArray labels = TestUtils.randomOneHot(mb, nOut); + String maskType = (inputMask ? "inputMask" : "none"); + + INDArray inMask = null; + if (inputMask) { + inMask = Nd4j.ones(DataType.INT, mb, tsLength); + for (int i = 0; i < mb; i++) { + int firstMaskedStep = tsLength - 1 - i; + if (firstMaskedStep == 0) { + firstMaskedStep = tsLength; + } + for (int j = firstMaskedStep; j < tsLength; j++) { + inMask.putScalar(i, j, 0.0); + } + } + } + + String name = "testLearnedSelfAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; + System.out.println("Starting test: " + name); + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in) + .labels(labels).inputMask(inMask).subset(true).maxPerParam(100)); + assertTrue(gradOK, name); + } + } + } + } + + @Test + public void testRecurrentAttentionLayer_differingTimeSteps(){ + int nIn = 9; + int nOut = 5; + int layerSize = 8; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.IDENTITY) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .list() + .layer(new LSTM.Builder().nOut(layerSize).build()) + .layer(new RecurrentAttentionLayer.Builder().nIn(layerSize).nOut(layerSize).nHeads(1).projectInput(false).hasBias(false).build()) + .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build()) + .layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.recurrent(nIn)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + final INDArray initialInput = Nd4j.rand(new int[]{8, nIn, 7}); + final INDArray goodNextInput = Nd4j.rand(new int[]{8, nIn, 7}); + final INDArray badNextInput = Nd4j.rand(new int[]{8, nIn, 12}); + + final INDArray labels = Nd4j.rand(new int[]{8, nOut}); + + net.fit(initialInput, labels); + net.fit(goodNextInput, labels); + + Assertions.assertThrows(IllegalArgumentException.class, () -> { + net.fit(badNextInput, labels); + }, "This layer only supports fixed length mini-batches. Expected 7 time steps but got 12."); + + } + + @Test + public void testRecurrentAttentionLayer() { + int nIn = 4; + int nOut = 2; + int tsLength = 3; + int layerSize = 3; + + for (int mb : new int[]{3, 1}) { + for (boolean inputMask : new boolean[]{true, false}) { + INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{mb, nIn, tsLength}); + INDArray labels = TestUtils.randomOneHot(mb, nOut); + String maskType = (inputMask ? "inputMask" : "none"); + + INDArray inMask = null; + if (inputMask) { + inMask = Nd4j.ones(mb, tsLength); + for (int i = 0; i < mb; i++) { + int firstMaskedStep = tsLength - 1 - i; + if (firstMaskedStep == 0) { + firstMaskedStep = tsLength; + } + for (int j = firstMaskedStep; j < tsLength; j++) { + inMask.putScalar(i, j, 0.0); + } + } + } + + String name = "testRecurrentAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType; + System.out.println("Starting test: " + name); + + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.IDENTITY) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .list() + .layer(new LSTM.Builder().nOut(layerSize).build()) + .layer(new RecurrentAttentionLayer.Builder().nIn(layerSize).nOut(layerSize).nHeads(1).projectInput(false).hasBias(false).build()) + .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build()) + .layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.recurrent(nIn)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + //System.out.println("Original"); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in) + .labels(labels).inputMask(inMask).subset(true).maxPerParam(100)); + assertTrue(gradOK, name); + } + } + } + + @Test + public void testAttentionVertex() { + int nIn = 3; + int nOut = 2; + int tsLength = 3; + int layerSize = 3; + + Random r = new Random(12345); + for (boolean inputMask : new boolean[]{false, true}) { + for (int mb : new int[]{3, 1}) { + for (boolean projectInput : new boolean[]{false, true}) { + INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{mb, nIn, tsLength}); + INDArray labels = TestUtils.randomOneHot(mb, nOut); + String maskType = (inputMask ? "inputMask" : "none"); + + INDArray inMask = null; + if (inputMask) { + inMask = Nd4j.ones(mb, tsLength); + for (int i = 0; i < mb; i++) { + int firstMaskedStep = tsLength - 1 - i; + if (firstMaskedStep == 0) { + firstMaskedStep = tsLength; + } + for (int j = firstMaskedStep; j < tsLength; j++) { + inMask.putScalar(i, j, 0.0); + } + } + } + + String name = "testAttentionVertex() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; + System.out.println("Starting test: " + name); + + + ComputationGraphConfiguration graph = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .graphBuilder() + .addInputs("input") + .addLayer("rnnKeys", new SimpleRnn.Builder().nOut(layerSize).build(), "input") + .addLayer("rnnQueries", new SimpleRnn.Builder().nOut(layerSize).build(), "input") + .addLayer("rnnValues", new SimpleRnn.Builder().nOut(layerSize).build(), "input") + .addVertex("attention", + projectInput ? + new AttentionVertex.Builder().nOut(4).nHeads(2).projectInput(true).nInQueries(layerSize).nInKeys(layerSize).nInValues(layerSize).build() + : new AttentionVertex.Builder().nOut(3).nHeads(1).projectInput(false).nInQueries(layerSize).nInKeys(layerSize).nInValues(layerSize).build(), "rnnQueries", "rnnKeys", "rnnValues") + .addLayer("pooling", new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build(), "attention") + .addLayer("output", new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "pooling") + .setOutputs("output") + .setInputTypes(InputType.recurrent(nIn)) + .build(); + + ComputationGraph net = new ComputationGraph(graph); + net.init(); + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{in}) + .labels(new INDArray[]{labels}).inputMask(inMask != null ? new INDArray[]{inMask} : null).subset(true).maxPerParam(100)); + assertTrue( gradOK, name); + } + } + } + } + + @Test + public void testAttentionVertexSameInput() { + int nIn = 3; + int nOut = 2; + int tsLength = 4; + int layerSize = 4; + + Random r = new Random(12345); + for (boolean inputMask : new boolean[]{false, true}) { + for (int mb : new int[]{3, 1}) { + for (boolean projectInput : new boolean[]{false, true}) { + INDArray in = Nd4j.rand(new int[]{mb, nIn, tsLength}); + INDArray labels = TestUtils.randomOneHot(mb, nOut); + String maskType = (inputMask ? "inputMask" : "none"); + + INDArray inMask = null; + if (inputMask) { + inMask = Nd4j.ones(mb, tsLength); + for (int i = 0; i < mb; i++) { + int firstMaskedStep = tsLength - 1 - i; + if (firstMaskedStep == 0) { + firstMaskedStep = tsLength; + } + for (int j = firstMaskedStep; j < tsLength; j++) { + inMask.putScalar(i, j, 0.0); + } + } + } + + String name = "testAttentionVertex() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; + System.out.println("Starting test: " + name); + + + ComputationGraphConfiguration graph = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .graphBuilder() + .addInputs("input") + .addLayer("rnn", new SimpleRnn.Builder().activation(Activation.TANH).nOut(layerSize).build(), "input") + .addVertex("attention", + projectInput ? + new AttentionVertex.Builder().nOut(4).nHeads(2).projectInput(true).nInQueries(layerSize).nInKeys(layerSize).nInValues(layerSize).build() + : new AttentionVertex.Builder().nOut(4).nHeads(1).projectInput(false).nInQueries(layerSize).nInKeys(layerSize).nInValues(layerSize).build(), "rnn", "rnn", "rnn") + .addLayer("pooling", new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build(), "attention") + .addLayer("output", new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "pooling") + .setOutputs("output") + .setInputTypes(InputType.recurrent(nIn)) + .build(); + + ComputationGraph net = new ComputationGraph(graph); + net.init(); + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{in}) + .labels(new INDArray[]{labels}).inputMask(inMask != null ? new INDArray[]{inMask} : null)); + assertTrue(gradOK, name); + } + } + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java new file mode 100644 index 000000000..6eb8c4e25 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java @@ -0,0 +1,597 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.gradientcheck; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.distribution.UniformDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * + */ +public class BNGradientCheckTest extends BaseDL4JTest { + + static { + Nd4j.setDataType(DataType.DOUBLE); + } + + @Override + public long getTimeoutMilliseconds() { + return 90000L; + } + + @Test + public void testGradient2dSimple() { + DataNormalization scaler = new NormalizerMinMaxScaler(); + DataSetIterator iter = new IrisDataSetIterator(150, 150); + scaler.fit(iter); + iter.setPreProcessor(scaler); + DataSet ds = iter.next(); + INDArray input = ds.getFeatures(); + INDArray labels = ds.getLabels(); + + for (boolean useLogStd : new boolean[]{true, false}) { + + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder().updater(new NoOp()) + .dataType(DataType.DOUBLE) + .seed(12345L) + .dist(new NormalDistribution(0, 1)).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3) + .activation(Activation.IDENTITY).build()) + .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).nOut(3).build()) + .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3).build()); + + MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); + mln.init(); + +// for (int j = 0; j < mln.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); + + //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc + //i.e., runningMean = decay * runningMean + (1-decay) * batchMean + //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" + Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input) + .labels(labels).excludeParams(excludeParams)); + + assertTrue(gradOK); + TestUtils.testModelSerialization(mln); + } + } + + @Test + public void testGradientCnnSimple() { + Nd4j.getRandom().setSeed(12345); + int minibatch = 10; + int depth = 1; + int hw = 4; + int nOut = 4; + INDArray input = Nd4j.rand(new int[]{minibatch, depth, hw, hw}); + INDArray labels = Nd4j.zeros(minibatch, nOut); + Random r = new Random(12345); + for (int i = 0; i < minibatch; i++) { + labels.putScalar(i, r.nextInt(nOut), 1.0); + } + + for (boolean useLogStd : new boolean[]{true, false}) { + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()).seed(12345L) + .dist(new NormalDistribution(0, 2)).list() + .layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nIn(depth).nOut(2) + .activation(Activation.IDENTITY).build()) + .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).build()) + .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(nOut).build()) + .setInputType(InputType.convolutional(hw, hw, depth)); + + MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); + mln.init(); + +// for (int j = 0; j < mln.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); + + //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc + //i.e., runningMean = decay * runningMean + (1-decay) * batchMean + //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" + Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input) + .labels(labels).excludeParams(excludeParams)); + + assertTrue(gradOK); + TestUtils.testModelSerialization(mln); + } + } + + @Test + public void testGradientBNWithCNNandSubsampling() { + //Parameterized test, testing combinations of: + // (a) activation function + // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') + // (c) Loss function (with specified output activations) + // (d) l1 and l2 values + Activation[] activFns = {Activation.SIGMOID, Activation.TANH, Activation.IDENTITY}; + boolean[] characteristic = {true}; //If true: run some backprop steps first + + LossFunctions.LossFunction[] lossFunctions = + {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE}; + Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH}; //i.e., lossFunctions[i] used with outputActivations[i] here + + double[] l2vals = {0.0, 0.1, 0.1}; + double[] l1vals = {0.0, 0.0, 0.2}; //i.e., use l2vals[j] with l1vals[j] + + Nd4j.getRandom().setSeed(12345); + int minibatch = 4; + int depth = 2; + int hw = 5; + int nOut = 2; + INDArray input = Nd4j.rand(new int[]{minibatch, depth, hw, hw}).muli(5).subi(2.5); + INDArray labels = TestUtils.randomOneHot(minibatch, nOut); + + DataSet ds = new DataSet(input, labels); + Random rng = new Random(12345); + for (boolean useLogStd : new boolean[]{true, false}) { + for (Activation afn : activFns) { + for (boolean doLearningFirst : characteristic) { + for (int i = 0; i < lossFunctions.length; i++) { + for (int j = 0; j < l2vals.length; j++) { + //Skip 2 of every 3 tests: from 24 cases to 8, still with decent coverage + if (rng.nextInt(3) != 0) + continue; + + LossFunctions.LossFunction lf = lossFunctions[i]; + Activation outputActivation = outputActivations[i]; + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(12345) + .dataType(DataType.DOUBLE) + .l2(l2vals[j]) + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT) + .updater(new NoOp()) + .dist(new UniformDistribution(-2, 2)).seed(12345L).list() + .layer(0, new ConvolutionLayer.Builder(2, 2).stride(1, 1).nOut(3) + .activation(afn).build()) + .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).build()) + .layer(2, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(1, 1).build()) + .layer(3, new BatchNormalization()) + .layer(4, new ActivationLayer.Builder().activation(afn).build()) + .layer(5, new OutputLayer.Builder(lf).activation(outputActivation).nOut(nOut) + .build()) + .setInputType(InputType.convolutional(hw, hw, depth)); + + MultiLayerConfiguration conf = builder.build(); + + MultiLayerNetwork mln = new MultiLayerNetwork(conf); + mln.init(); + String name = new Object() { + }.getClass().getEnclosingMethod().getName(); + +// System.out.println("Num params: " + mln.numParams()); + + if (doLearningFirst) { + //Run a number of iterations of learning + mln.setInput(ds.getFeatures()); + mln.setLabels(ds.getLabels()); + mln.computeGradientAndScore(); + double scoreBefore = mln.score(); + for (int k = 0; k < 20; k++) + mln.fit(ds); + mln.computeGradientAndScore(); + double scoreAfter = mln.score(); + //Can't test in 'characteristic mode of operation' if not learning + String msg = name + + " - score did not (sufficiently) decrease during learning - activationFn=" + + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + + ", scoreAfter=" + scoreAfter + ")"; + assertTrue( scoreAfter < 0.9 * scoreBefore, msg); + } + + System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + + ", outputActivation=" + outputActivation + ", doLearningFirst=" + + doLearningFirst + ", l1=" + l1vals[j] + ", l2=" + l2vals[j]); +// for (int k = 0; k < mln.getnLayers(); k++) +// System.out.println("Layer " + k + " # params: " + mln.getLayer(k).numParams()); + + //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc + //i.e., runningMean = decay * runningMean + (1-decay) * batchMean + //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" + Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "3_mean", "3_var", "1_log10stdev", "3_log10stdev")); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input) + .labels(labels).excludeParams(excludeParams).subset(true).maxPerParam(25)); //Most params are in output layer, only these should be skipped with this threshold + + assertTrue(gradOK); + TestUtils.testModelSerialization(mln); + } + } + } + } + } + } + + + @Test + public void testGradientDense() { + //Parameterized test, testing combinations of: + // (a) activation function + // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') + // (c) Loss function (with specified output activations) + // (d) l1 and l2 values + Activation[] activFns = {Activation.TANH, Activation.IDENTITY}; + boolean[] characteristic = {true}; //If true: run some backprop steps first + + LossFunctions.LossFunction[] lossFunctions = + {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE}; + Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH}; //i.e., lossFunctions[i] used with outputActivations[i] here + + double[] l2vals = {0.0, 0.1}; + double[] l1vals = {0.0, 0.2}; //i.e., use l2vals[j] with l1vals[j] + + Nd4j.getRandom().setSeed(12345); + int minibatch = 10; + int nIn = 5; + int nOut = 3; + INDArray input = Nd4j.rand(new int[]{minibatch, nIn}); + INDArray labels = Nd4j.zeros(minibatch, nOut); + Random r = new Random(12345); + for (int i = 0; i < minibatch; i++) { + labels.putScalar(i, r.nextInt(nOut), 1.0); + } + + DataSet ds = new DataSet(input, labels); + + for (boolean useLogStd : new boolean[]{true, false}) { + for (Activation afn : activFns) { + for (boolean doLearningFirst : characteristic) { + for (int i = 0; i < lossFunctions.length; i++) { + for (int j = 0; j < l2vals.length; j++) { + LossFunctions.LossFunction lf = lossFunctions[i]; + Activation outputActivation = outputActivations[i]; + + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .l2(l2vals[j]) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT) + .updater(new NoOp()) + .dist(new UniformDistribution(-2, 2)).seed(12345L).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(4) + .activation(afn).build()) + .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).build()) + .layer(2, new DenseLayer.Builder().nIn(4).nOut(4).build()) + .layer(3, new BatchNormalization.Builder().useLogStd(useLogStd).build()) + .layer(4, new OutputLayer.Builder(lf) + .activation(outputActivation).nOut(nOut) + .build()); + + MultiLayerConfiguration conf = builder.build(); + + MultiLayerNetwork mln = new MultiLayerNetwork(conf); + mln.init(); + + String name = new Object() { + }.getClass().getEnclosingMethod().getName(); + + if (doLearningFirst) { + //Run a number of iterations of learning + mln.setInput(ds.getFeatures()); + mln.setLabels(ds.getLabels()); + mln.computeGradientAndScore(); + double scoreBefore = mln.score(); + for (int k = 0; k < 10; k++) + mln.fit(ds); + mln.computeGradientAndScore(); + double scoreAfter = mln.score(); + //Can't test in 'characteristic mode of operation' if not learning + String msg = name + + " - score did not (sufficiently) decrease during learning - activationFn=" + + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + + ", scoreAfter=" + scoreAfter + ")"; + assertTrue( scoreAfter < 0.8 * scoreBefore, msg); + } + + System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + + ", outputActivation=" + outputActivation + ", doLearningFirst=" + + doLearningFirst + ", l1=" + l1vals[j] + ", l2=" + l2vals[j]); +// for (int k = 0; k < mln.getnLayers(); k++) +// System.out.println("Layer " + k + " # params: " + mln.getLayer(k).numParams()); + + //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc + //i.e., runningMean = decay * runningMean + (1-decay) * batchMean + //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" + Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "3_mean", "3_var", "1_log10stdev", "3_log10stdev")); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input) + .labels(labels).excludeParams(excludeParams)); + + assertTrue(gradOK); + TestUtils.testModelSerialization(mln); + } + } + } + } + } + } + + @Test + public void testGradient2dFixedGammaBeta() { + DataNormalization scaler = new NormalizerMinMaxScaler(); + DataSetIterator iter = new IrisDataSetIterator(150, 150); + scaler.fit(iter); + iter.setPreProcessor(scaler); + DataSet ds = iter.next(); + INDArray input = ds.getFeatures(); + INDArray labels = ds.getLabels(); + + for (boolean useLogStd : new boolean[]{true, false}) { + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp()) + .dataType(DataType.DOUBLE) + .seed(12345L) + .dist(new NormalDistribution(0, 1)).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY).build()) + .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).lockGammaBeta(true).gamma(2.0).beta(0.5).nOut(3) + .build()) + .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3).build()); + + MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); + mln.init(); + +// for (int j = 0; j < mln.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); + + //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc + //i.e., runningMean = decay * runningMean + (1-decay) * batchMean + //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" + Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input) + .labels(labels).excludeParams(excludeParams)); + + assertTrue(gradOK); + TestUtils.testModelSerialization(mln); + } + } + + @Test + public void testGradientCnnFixedGammaBeta() { + Nd4j.getRandom().setSeed(12345); + int minibatch = 10; + int depth = 1; + int hw = 4; + int nOut = 4; + INDArray input = Nd4j.rand(new int[]{minibatch, depth, hw, hw}); + INDArray labels = Nd4j.zeros(minibatch, nOut); + Random r = new Random(12345); + for (int i = 0; i < minibatch; i++) { + labels.putScalar(i, r.nextInt(nOut), 1.0); + } + + for (boolean useLogStd : new boolean[]{true, false}) { + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp()) + .dataType(DataType.DOUBLE) + .seed(12345L) + .dist(new NormalDistribution(0, 2)).list() + .layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nIn(depth).nOut(2) + .activation(Activation.IDENTITY).build()) + .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).lockGammaBeta(true).gamma(2.0).beta(0.5).build()) + .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(nOut).build()) + .setInputType(InputType.convolutional(hw, hw, depth)); + + MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); + mln.init(); + +// for (int j = 0; j < mln.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); + + //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc + //i.e., runningMean = decay * runningMean + (1-decay) * batchMean + //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" + Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input) + .labels(labels).excludeParams(excludeParams)); + + assertTrue(gradOK); + TestUtils.testModelSerialization(mln); + } + } + + @Test + public void testBatchNormCompGraphSimple() { + + int numClasses = 2; + int height = 3; + int width = 3; + int channels = 1; + long seed = 123; + + int minibatchSize = 3; + + for (boolean useLogStd : new boolean[]{true, false}) { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed).updater(new NoOp()) + .dataType(DataType.DOUBLE) + .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in") + .setInputTypes(InputType.convolutional(height, width, channels)) + .addLayer("bn", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "in") + .addLayer("out", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(numClasses).build(), "bn") + .setOutputs("out").build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + + Random r = new Random(12345); + INDArray input = Nd4j.rand(new int[]{minibatchSize, channels, height, width}); //Order: examples, channels, height, width + INDArray labels = Nd4j.zeros(minibatchSize, numClasses); + for (int i = 0; i < minibatchSize; i++) { + labels.putScalar(new int[]{i, r.nextInt(numClasses)}, 1.0); + } + + //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc + //i.e., runningMean = decay * runningMean + (1-decay) * batchMean + //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" + Set excludeParams = new HashSet<>(Arrays.asList("bn_mean", "bn_var")); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{input}) + .labels(new INDArray[]{labels}).excludeParams(excludeParams)); + + assertTrue(gradOK); + TestUtils.testModelSerialization(net); + } + } + + + @Test + public void testGradientBNWithCNNandSubsamplingCompGraph() { + //Parameterized test, testing combinations of: + // (a) activation function + // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') + // (c) Loss function (with specified output activations) + // (d) l1 and l2 values + Activation[] activFns = {Activation.TANH, Activation.IDENTITY}; + boolean doLearningFirst = true; + + LossFunctions.LossFunction[] lossFunctions = {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD}; + Activation[] outputActivations = {Activation.SOFTMAX}; //i.e., lossFunctions[i] used with outputActivations[i] here + + double[] l2vals = {0.0, 0.1}; + double[] l1vals = {0.0, 0.2}; //i.e., use l2vals[j] with l1vals[j] + + Nd4j.getRandom().setSeed(12345); + int minibatch = 10; + int depth = 2; + int hw = 5; + int nOut = 3; + INDArray input = Nd4j.rand(new int[]{minibatch, depth, hw, hw}); + INDArray labels = Nd4j.zeros(minibatch, nOut); + Random r = new Random(12345); + for (int i = 0; i < minibatch; i++) { + labels.putScalar(i, r.nextInt(nOut), 1.0); + } + + DataSet ds = new DataSet(input, labels); + + for (boolean useLogStd : new boolean[]{true, false}) { + for (Activation afn : activFns) { + for (int i = 0; i < lossFunctions.length; i++) { + for (int j = 0; j < l2vals.length; j++) { + LossFunctions.LossFunction lf = lossFunctions[i]; + Activation outputActivation = outputActivations[i]; + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .dataType(DataType.DOUBLE) + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT) + .updater(new NoOp()) + .dist(new UniformDistribution(-2, 2)).seed(12345L).graphBuilder() + .addInputs("in") + .addLayer("0", new ConvolutionLayer.Builder(2, 2).stride(1, 1).nOut(3) + .activation(afn).build(), "in") + .addLayer("1", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "0") + .addLayer("2", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(1, 1).build(), "1") + .addLayer("3", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "2") + .addLayer("4", new ActivationLayer.Builder().activation(afn).build(), "3") + .addLayer("5", new OutputLayer.Builder(lf).activation(outputActivation) + .nOut(nOut).build(), "4") + .setOutputs("5").setInputTypes(InputType.convolutional(hw, hw, depth)) + .build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + String name = new Object() { + }.getClass().getEnclosingMethod().getName(); + + if (doLearningFirst) { + //Run a number of iterations of learning + net.setInput(0, ds.getFeatures()); + net.setLabels(ds.getLabels()); + net.computeGradientAndScore(); + double scoreBefore = net.score(); + for (int k = 0; k < 20; k++) + net.fit(ds); + net.computeGradientAndScore(); + double scoreAfter = net.score(); + //Can't test in 'characteristic mode of operation' if not learning + String msg = name + + " - score did not (sufficiently) decrease during learning - activationFn=" + + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + + ", scoreAfter=" + scoreAfter + ")"; + assertTrue( scoreAfter < 0.9 * scoreBefore, msg); + } + + System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + + ", outputActivation=" + outputActivation + ", doLearningFirst=" + + doLearningFirst + ", l1=" + l1vals[j] + ", l2=" + l2vals[j]); +// for (int k = 0; k < net.getNumLayers(); k++) +// System.out.println("Layer " + k + " # params: " + net.getLayer(k).numParams()); + + //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc + //i.e., runningMean = decay * runningMean + (1-decay) * batchMean + //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" + Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "3_mean", "3_var", "1_log10stdev", "3_log10stdev")); + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{input}) + .labels(new INDArray[]{labels}).excludeParams(excludeParams)); + + assertTrue(gradOK); + TestUtils.testModelSerialization(net); + } + } + } + } + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java new file mode 100644 index 000000000..cdd11b6f9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java @@ -0,0 +1,534 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.gradientcheck; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.RNNFormat; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.convolutional.Cropping1D; +import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.util.Convolution1DUtils; +import org.deeplearning4j.util.ConvolutionUtils; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class CNN1DGradientCheckTest extends BaseDL4JTest { + private static final boolean PRINT_RESULTS = true; + private static final boolean RETURN_ON_FIRST_FAILURE = false; + private static final double DEFAULT_EPS = 1e-6; + private static final double DEFAULT_MAX_REL_ERROR = 1e-3; + private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; + + static { + Nd4j.setDataType(DataType.DOUBLE); + } + + @Override + public long getTimeoutMilliseconds() { + return 180000; + } + + @Test + public void testCnn1DWithLocallyConnected1D() { + Nd4j.getRandom().setSeed(1337); + + int[] minibatchSizes = {2, 3}; + int length = 7; + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int finalNOut = 4; + + int[] kernels = {1}; + int stride = 1; + int padding = 0; + + Activation[] activations = {Activation.SIGMOID}; + + for (Activation afn : activations) { + for (int minibatchSize : minibatchSizes) { + for (int kernel : kernels) { + INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length}); + INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length); + for (int i = 0; i < minibatchSize; i++) { + for (int j = 0; j < length; j++) { + labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0); + } + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list() + .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel) + .stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1) + .rnnDataFormat(RNNFormat.NCW) + .build()) + .layer(new LocallyConnected1D.Builder().activation(afn).kernelSize(kernel) + .stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2).hasBias(false) + .build()) + .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .setInputType(InputType.recurrent(convNIn, length)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "Minibatch=" + minibatchSize + ", activationFn=" + + afn + ", kernel = " + kernel; + + if (PRINT_RESULTS) { + System.out.println(msg); +// for (int j = 0; j < net.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + + } + } + } + + + @Test + public void testCnn1DWithCropping1D() { + Nd4j.getRandom().setSeed(1337); + + int[] minibatchSizes = {1, 3}; + int length = 7; + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int finalNOut = 4; + + + int[] kernels = {1, 2, 4}; + int stride = 1; + + int padding = 0; + int cropping = 1; + int croppedLength = length - 2 * cropping; + + Activation[] activations = {Activation.SIGMOID}; + SubsamplingLayer.PoolingType[] poolingTypes = + new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX, + SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM}; + + for (Activation afn : activations) { + for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { + for (int minibatchSize : minibatchSizes) { + for (int kernel : kernels) { + INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length}); + INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, croppedLength); + for (int i = 0; i < minibatchSize; i++) { + for (int j = 0; j < croppedLength; j++) { + labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0); + } + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list() + .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel) + .stride(stride).padding(padding).nOut(convNOut1) + .build()) + .layer(new Cropping1D.Builder(cropping).build()) + .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel) + .stride(stride).padding(padding).nOut(convNOut2) + .build()) + .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .setInputType(InputType.recurrent(convNIn, length,RNNFormat.NCW)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + + afn + ", kernel = " + kernel; + + if (PRINT_RESULTS) { + System.out.println(msg); +// for (int j = 0; j < net.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } + } + } + } + + + @Test + public void testCnn1DWithZeroPadding1D() { + Nd4j.getRandom().setSeed(1337); + + int[] minibatchSizes = {1, 3}; + int length = 7; + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int finalNOut = 4; + + + int[] kernels = {1, 2, 4}; + int stride = 1; + int pnorm = 2; + + int padding = 0; + int zeroPadding = 2; + int paddedLength = length + 2 * zeroPadding; + + Activation[] activations = {Activation.SIGMOID}; + SubsamplingLayer.PoolingType[] poolingTypes = + new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX, + SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM}; + + for (Activation afn : activations) { + for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { + for (int minibatchSize : minibatchSizes) { + for (int kernel : kernels) { + INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length}); + INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, paddedLength); + for (int i = 0; i < minibatchSize; i++) { + for (int j = 0; j < paddedLength; j++) { + labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0); + } + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list() + .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel) + .stride(stride).padding(padding).nOut(convNOut1) + .build()) + .layer(new ZeroPadding1DLayer.Builder(zeroPadding).build()) + .layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel) + .stride(stride).padding(padding).nOut(convNOut2) + .build()) + .layer(new ZeroPadding1DLayer.Builder(0).build()) + .layer(new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel) + .stride(stride).padding(padding).pnorm(pnorm).build()) + .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .setInputType(InputType.recurrent(convNIn, length,RNNFormat.NCW)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + + afn + ", kernel = " + kernel; + + if (PRINT_RESULTS) { + System.out.println(msg); +// for (int j = 0; j < net.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + TestUtils.testModelSerialization(net); + } + } + } + } + } + + + @Test + public void testCnn1DWithSubsampling1D() { + Nd4j.getRandom().setSeed(12345); + + int[] minibatchSizes = {1, 3}; + int length = 7; + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int finalNOut = 4; + + int[] kernels = {1, 2, 4}; + int stride = 1; + int padding = 0; + int pnorm = 2; + + Activation[] activations = {Activation.SIGMOID, Activation.TANH}; + SubsamplingLayer.PoolingType[] poolingTypes = + new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX, + SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM}; + + for (Activation afn : activations) { + for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { + for (int minibatchSize : minibatchSizes) { + for (int kernel : kernels) { + INDArray input = Nd4j.rand(new int[]{minibatchSize, convNIn, length}); + INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length); + for (int i = 0; i < minibatchSize; i++) { + for (int j = 0; j < length; j++) { + labels.putScalar(new int[]{i, i % finalNOut, j}, 1.0); + } + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list() + .layer(0, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel) + .stride(stride).padding(padding).nOut(convNOut1) + .build()) + .layer(1, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel) + .stride(stride).padding(padding).nOut(convNOut2) + .build()) + .layer(2, new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel) + .stride(stride).padding(padding).pnorm(pnorm).build()) + .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .setInputType(InputType.recurrent(convNIn, length,RNNFormat.NCW)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + + afn + ", kernel = " + kernel; + + if (PRINT_RESULTS) { + System.out.println(msg); +// for (int j = 0; j < net.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + TestUtils.testModelSerialization(net); + } + } + } + } + } + + @Test + public void testCnn1dWithMasking(){ + int length = 12; + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int finalNOut = 3; + + int pnorm = 2; + + SubsamplingLayer.PoolingType[] poolingTypes = + new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG}; + + for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { + for(ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Same, ConvolutionMode.Truncate}) { + for( int stride : new int[]{1, 2}){ + String s = cm + ", stride=" + stride + ", pooling=" + poolingType; + log.info("Starting test: " + s); + Nd4j.getRandom().setSeed(12345); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .activation(Activation.TANH) + .dist(new NormalDistribution(0, 1)).convolutionMode(cm) + .seed(12345) + .list() + .layer(new Convolution1DLayer.Builder().kernelSize(2) + .rnnDataFormat(RNNFormat.NCW) + .stride(stride).nIn(convNIn).nOut(convNOut1) + .build()) + .layer(new Subsampling1DLayer.Builder(poolingType).kernelSize(2) + .stride(stride).pnorm(pnorm).build()) + .layer(new Convolution1DLayer.Builder().kernelSize(2) + .rnnDataFormat(RNNFormat.NCW) + .stride(stride).nIn(convNOut1).nOut(convNOut2) + .build()) + .layer(new GlobalPoolingLayer(PoolingType.AVG)) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .setInputType(InputType.recurrent(convNIn, length)).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray f = Nd4j.rand(new int[]{2, convNIn, length}); + INDArray fm = Nd4j.create(2, length); + fm.get(NDArrayIndex.point(0), NDArrayIndex.all()).assign(1); + fm.get(NDArrayIndex.point(1), NDArrayIndex.interval(0,6)).assign(1); + + INDArray label = TestUtils.randomOneHot(2, finalNOut); + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(f) + .labels(label).inputMask(fm)); + + assertTrue(gradOK, s); + TestUtils.testModelSerialization(net); + + //TODO also check that masked step values don't impact forward pass, score or gradients + + DataSet ds = new DataSet(f,label,fm,null); + double scoreBefore = net.score(ds); + net.setInput(f); + net.setLabels(label); + net.setLayerMaskArrays(fm, null); + net.computeGradientAndScore(); + INDArray gradBefore = net.getFlattenedGradients().dup(); + f.putScalar(1, 0, 10, 10.0); + f.putScalar(1, 1, 11, 20.0); + double scoreAfter = net.score(ds); + net.setInput(f); + net.setLabels(label); + net.setLayerMaskArrays(fm, null); + net.computeGradientAndScore(); + INDArray gradAfter = net.getFlattenedGradients().dup(); + + assertEquals(scoreBefore, scoreAfter, 1e-6); + assertEquals(gradBefore, gradAfter); + } + } + } + } + + @Test + public void testCnn1Causal() throws Exception { + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int finalNOut = 3; + + int[] lengths = {11, 12, 13, 9, 10, 11}; + int[] kernels = {2, 3, 2, 4, 2, 3}; + int[] dilations = {1, 1, 2, 1, 2, 1}; + int[] strides = {1, 2, 1, 2, 1, 1}; + boolean[] masks = {false, true, false, true, false, true}; + boolean[] hasB = {true, false, true, false, true, true}; + for (int i = 0; i < lengths.length; i++) { + int length = lengths[i]; + int k = kernels[i]; + int d = dilations[i]; + int st = strides[i]; + boolean mask = masks[i]; + boolean hasBias = hasB[i]; + //TODO has bias + String s = "k=" + k + ", s=" + st + " d=" + d + ", seqLen=" + length; + log.info("Starting test: " + s); + Nd4j.getRandom().setSeed(12345); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .activation(Activation.TANH) + .weightInit(new NormalDistribution(0, 1)) + .seed(12345) + .list() + .layer(new Convolution1DLayer.Builder().kernelSize(k) + .dilation(d) + .hasBias(hasBias) + .convolutionMode(ConvolutionMode.Causal) + .stride(st).nOut(convNOut1) + .build()) + .layer(new Convolution1DLayer.Builder().kernelSize(k) + .dilation(d) + .convolutionMode(ConvolutionMode.Causal) + .stride(st).nOut(convNOut2) + .build()) + .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .setInputType(InputType.recurrent(convNIn, length,RNNFormat.NCW)).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray f = Nd4j.rand(DataType.DOUBLE, 2, convNIn, length); + INDArray fm = null; + if (mask) { + fm = Nd4j.create(2, length); + fm.get(NDArrayIndex.point(0), NDArrayIndex.all()).assign(1); + fm.get(NDArrayIndex.point(1), NDArrayIndex.interval(0, length - 2)).assign(1); + } + + long outSize1 = Convolution1DUtils.getOutputSize(length, k, st, 0, ConvolutionMode.Causal, d); + long outSize2 = Convolution1DUtils.getOutputSize(outSize1, k, st, 0, ConvolutionMode.Causal, d); + + INDArray label = TestUtils.randomOneHotTimeSeries(2, finalNOut, (int)outSize2); + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(f) + .labels(label).inputMask(fm)); + + assertTrue(gradOK, s); + TestUtils.testModelSerialization(net); + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java new file mode 100644 index 000000000..f7a9375f8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java @@ -0,0 +1,642 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.gradientcheck; + +import lombok.extern.java.Log; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.convolutional.Cropping3D; +import org.deeplearning4j.nn.conf.preprocessor.Cnn3DToFeedForwardPreProcessor; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Log +public class CNN3DGradientCheckTest extends BaseDL4JTest { + private static final boolean PRINT_RESULTS = true; + private static final boolean RETURN_ON_FIRST_FAILURE = false; + private static final double DEFAULT_EPS = 1e-6; + private static final double DEFAULT_MAX_REL_ERROR = 1e-3; + private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; + + static { + Nd4j.setDataType(DataType.DOUBLE); + } + + @Override + public long getTimeoutMilliseconds() { + return 90000L; + } + + @Test + public void testCnn3DPlain() { + Nd4j.getRandom().setSeed(1337); + + // Note: we checked this with a variety of parameters, but it takes a lot of time. + int[] depths = {6}; + int[] heights = {6}; + int[] widths = {6}; + + + int[] minibatchSizes = {3}; + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int denseNOut = 5; + int finalNOut = 42; + + + int[][] kernels = {{2, 2, 2}}; + int[][] strides = {{1, 1, 1}}; + + Activation[] activations = {Activation.SIGMOID}; + + ConvolutionMode[] modes = {ConvolutionMode.Truncate, ConvolutionMode.Same}; + + for (Activation afn : activations) { + for (int miniBatchSize : minibatchSizes) { + for (int depth : depths) { + for (int height : heights) { + for (int width : widths) { + for (ConvolutionMode mode : modes) { + for (int[] kernel : kernels) { + for (int[] stride : strides) { + for (Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) { + + int outDepth = mode == ConvolutionMode.Same ? + depth / stride[0] : (depth - kernel[0]) / stride[0] + 1; + int outHeight = mode == ConvolutionMode.Same ? + height / stride[1] : (height - kernel[1]) / stride[1] + 1; + int outWidth = mode == ConvolutionMode.Same ? + width / stride[2] : (width - kernel[2]) / stride[2] + 1; + + INDArray input; + if(df == Convolution3D.DataFormat.NDHWC){ + input = Nd4j.rand(new int[]{miniBatchSize, depth, height, width, convNIn}); + } else { + input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width}); + } + INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); + for (int i = 0; i < miniBatchSize; i++) { + labels.putScalar(new int[]{i, i % finalNOut}, 1.0); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL) + .dist(new NormalDistribution(0, 1)) + .list() + .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel) + .stride(stride).nIn(convNIn).nOut(convNOut1).hasBias(false) + .convolutionMode(mode).dataFormat(df) + .build()) + .layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1) + .nIn(convNOut1).nOut(convNOut2).hasBias(false) + .convolutionMode(mode).dataFormat(df) + .build()) + .layer(2, new DenseLayer.Builder().nOut(denseNOut).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .inputPreProcessor(2, + new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, + convNOut2, df == Convolution3D.DataFormat.NCDHW)) + .setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "DataFormat = " + df + ", minibatch size = " + miniBatchSize + ", activationFn=" + afn + + ", kernel = " + Arrays.toString(kernel) + ", stride = " + + Arrays.toString(stride) + ", mode = " + mode.toString() + + ", input depth " + depth + ", input height " + height + + ", input width " + width; + + if (PRINT_RESULTS) { + log.info(msg); +// for (int j = 0; j < net.getnLayers(); j++) { +// log.info("Layer " + j + " # params: " + net.getLayer(j).numParams()); +// } + } + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input) + .labels(labels).subset(true).maxPerParam(128)); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } + } + } + } + } + } + } + } + } + + @Test + public void testCnn3DZeroPadding() { + Nd4j.getRandom().setSeed(42); + + int depth = 4; + int height = 4; + int width = 4; + + + int[] minibatchSizes = {3}; + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int denseNOut = 5; + int finalNOut = 42; + + + int[] kernel = {2, 2, 2}; + int[] zeroPadding = {1, 1, 2, 2, 3, 3}; + + Activation[] activations = {Activation.SIGMOID}; + + ConvolutionMode[] modes = {ConvolutionMode.Truncate, ConvolutionMode.Same}; + + for (Activation afn : activations) { + for (int miniBatchSize : minibatchSizes) { + for (ConvolutionMode mode : modes) { + + int outDepth = mode == ConvolutionMode.Same ? + depth : (depth - kernel[0]) + 1; + int outHeight = mode == ConvolutionMode.Same ? + height : (height - kernel[1]) + 1; + int outWidth = mode == ConvolutionMode.Same ? + width : (width - kernel[2]) + 1; + + outDepth += zeroPadding[0] + zeroPadding[1]; + outHeight += zeroPadding[2] + zeroPadding[3]; + outWidth += zeroPadding[4] + zeroPadding[5]; + + INDArray input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width}); + INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); + for (int i = 0; i < miniBatchSize; i++) { + labels.putScalar(new int[]{i, i % finalNOut}, 1.0); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL) + .dist(new NormalDistribution(0, 1)) + .list() + .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel) + .nIn(convNIn).nOut(convNOut1).hasBias(false) + .convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW) + .build()) + .layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1) + .nIn(convNOut1).nOut(convNOut2).hasBias(false) + .convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW) + .build()) + .layer(2, new ZeroPadding3DLayer.Builder(zeroPadding).build()) + .layer(3, new DenseLayer.Builder().nOut(denseNOut).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .inputPreProcessor(3, + new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, + convNOut2, true)) + .setInputType(InputType.convolutional3D(depth, height, width, convNIn)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn + + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString() + + ", input depth " + depth + ", input height " + height + + ", input width " + width; + + if (PRINT_RESULTS) { + log.info(msg); +// for (int j = 0; j < net.getnLayers(); j++) { +// log.info("Layer " + j + " # params: " + net.getLayer(j).numParams()); +// } + } + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input) + .labels(labels).subset(true).maxPerParam(512)); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + + } + } + } + + + @Test + public void testCnn3DPooling() { + Nd4j.getRandom().setSeed(42); + + int depth = 4; + int height = 4; + int width = 4; + + + int[] minibatchSizes = {3}; + int convNIn = 2; + int convNOut = 4; + int denseNOut = 5; + int finalNOut = 42; + + int[] kernel = {2, 2, 2}; + + Activation[] activations = {Activation.SIGMOID}; + + Subsampling3DLayer.PoolingType[] poolModes = {Subsampling3DLayer.PoolingType.AVG}; + + ConvolutionMode[] modes = {ConvolutionMode.Truncate}; + + for (Activation afn : activations) { + for (int miniBatchSize : minibatchSizes) { + for (Subsampling3DLayer.PoolingType pool : poolModes) { + for (ConvolutionMode mode : modes) { + for (Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) { + + int outDepth = depth / kernel[0]; + int outHeight = height / kernel[1]; + int outWidth = width / kernel[2]; + + INDArray input = Nd4j.rand( + df == Convolution3D.DataFormat.NCDHW ? new int[]{miniBatchSize, convNIn, depth, height, width} + : new int[]{miniBatchSize, depth, height, width, convNIn}); + INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); + for (int i = 0; i < miniBatchSize; i++) { + labels.putScalar(new int[]{i, i % finalNOut}, 1.0); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .dist(new NormalDistribution(0, 1)) + .list() + .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1) + .nIn(convNIn).nOut(convNOut).hasBias(false) + .convolutionMode(mode).dataFormat(df) + .build()) + .layer(1, new Subsampling3DLayer.Builder(kernel) + .poolingType(pool).convolutionMode(mode).dataFormat(df).build()) + .layer(2, new DenseLayer.Builder().nOut(denseNOut).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .inputPreProcessor(2, + new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth,convNOut, df)) + .setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn + + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString() + + ", input depth " + depth + ", input height " + height + + ", input width " + width + ", dataFormat=" + df; + + if (PRINT_RESULTS) { + log.info(msg); + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, + DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, + RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } + } + } + } + } + + @Test + public void testCnn3DUpsampling() { + Nd4j.getRandom().setSeed(42); + + int depth = 2; + int height = 2; + int width = 2; + + + int[] minibatchSizes = {3}; + int convNIn = 2; + int convNOut = 4; + int denseNOut = 5; + int finalNOut = 42; + + + int[] upsamplingSize = {2, 2, 2}; + + Activation[] activations = {Activation.SIGMOID}; + + + ConvolutionMode[] modes = {ConvolutionMode.Truncate}; + + for (Activation afn : activations) { + for (int miniBatchSize : minibatchSizes) { + for (ConvolutionMode mode : modes) { + for(Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) { + + int outDepth = depth * upsamplingSize[0]; + int outHeight = height * upsamplingSize[1]; + int outWidth = width * upsamplingSize[2]; + + INDArray input = df == Convolution3D.DataFormat.NCDHW ? Nd4j.rand(miniBatchSize, convNIn, depth, height, width) : Nd4j.rand(miniBatchSize, depth, height, width, convNIn); + INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); + for (int i = 0; i < miniBatchSize; i++) { + labels.putScalar(new int[]{i, i % finalNOut}, 1.0); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL) + .dist(new NormalDistribution(0, 1)) + .seed(12345) + .list() + .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1) + .nIn(convNIn).nOut(convNOut).hasBias(false) + .convolutionMode(mode).dataFormat(df) + .build()) + .layer(1, new Upsampling3D.Builder(upsamplingSize[0]).dataFormat(df).build()) + .layer(2, new DenseLayer.Builder().nOut(denseNOut).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .inputPreProcessor(2, + new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, + convNOut, true)) + .setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn + + ", kernel = " + Arrays.toString(upsamplingSize) + ", mode = " + mode.toString() + + ", input depth " + depth + ", input height " + height + + ", input width " + width; + + if (PRINT_RESULTS) { + log.info(msg); +// for (int j = 0; j < net.getnLayers(); j++) { +// log.info("Layer " + j + " # params: " + net.getLayer(j).numParams()); +// } + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, + DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, + RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } + } + } + } + + @Test + public void testCnn3DCropping() { + Nd4j.getRandom().setSeed(42); + + int depth = 6; + int height = 6; + int width = 6; + + + int[] minibatchSizes = {3}; + int convNIn = 2; + int convNOut1 = 3; + int convNOut2 = 4; + int denseNOut = 5; + int finalNOut = 8; + + + int[] kernel = {1, 1, 1}; + int[] cropping = {0, 0, 1, 1, 2, 2}; + + Activation[] activations = {Activation.SIGMOID}; + + ConvolutionMode[] modes = {ConvolutionMode.Same}; + + for (Activation afn : activations) { + for (int miniBatchSize : minibatchSizes) { + for (ConvolutionMode mode : modes) { + + int outDepth = mode == ConvolutionMode.Same ? + depth : (depth - kernel[0]) + 1; + int outHeight = mode == ConvolutionMode.Same ? + height : (height - kernel[1]) + 1; + int outWidth = mode == ConvolutionMode.Same ? + width : (width - kernel[2]) + 1; + + outDepth -= cropping[0] + cropping[1]; + outHeight -= cropping[2] + cropping[3]; + outWidth -= cropping[4] + cropping[5]; + + INDArray input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width}); + INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); + for (int i = 0; i < miniBatchSize; i++) { + labels.putScalar(new int[]{i, i % finalNOut}, 1.0); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL) + .dist(new NormalDistribution(0, 1)) + .list() + .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel) + .nIn(convNIn).nOut(convNOut1).hasBias(false) + .convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW) + .build()) + .layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1) + .nIn(convNOut1).nOut(convNOut2).hasBias(false) + .convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW) + .build()) + .layer(2, new Cropping3D.Builder(cropping).build()) + .layer(3, new DenseLayer.Builder().nOut(denseNOut).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .inputPreProcessor(3, + new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, + convNOut2, true)) + .setInputType(InputType.convolutional3D(depth, height, width, convNIn)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn + + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString() + + ", input depth " + depth + ", input height " + height + + ", input width " + width; + + if (PRINT_RESULTS) { + log.info(msg); +// for (int j = 0; j < net.getnLayers(); j++) { +// log.info("Layer " + j + " # params: " + net.getLayer(j).numParams()); +// } + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, + DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, + RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + + } + } + } + + @Test + public void testDeconv3d() { + Nd4j.getRandom().setSeed(12345); + // Note: we checked this with a variety of parameters, but it takes a lot of time. + int[] depths = {8, 8, 9}; + int[] heights = {8, 9, 9}; + int[] widths = {8, 8, 9}; + + + int[][] kernels = {{2, 2, 2}, {3, 3, 3}, {2, 3, 2}}; + int[][] strides = {{1, 1, 1}, {1, 1, 1}, {2, 2, 2}}; + + Activation[] activations = {Activation.SIGMOID, Activation.TANH, Activation.IDENTITY}; + + ConvolutionMode[] modes = {ConvolutionMode.Truncate, ConvolutionMode.Same, ConvolutionMode.Same}; + int[] mbs = {1, 3, 2}; + Convolution3D.DataFormat[] dataFormats = new Convolution3D.DataFormat[]{Convolution3D.DataFormat.NCDHW, Convolution3D.DataFormat.NDHWC, Convolution3D.DataFormat.NCDHW}; + + int convNIn = 2; + int finalNOut = 2; + int[] deconvOut = {2, 3, 4}; + + for (int i = 0; i < activations.length; i++) { + Activation afn = activations[i]; + int miniBatchSize = mbs[i]; + int depth = depths[i]; + int height = heights[i]; + int width = widths[i]; + ConvolutionMode mode = modes[i]; + int[] kernel = kernels[i]; + int[] stride = strides[i]; + Convolution3D.DataFormat df = dataFormats[i]; + int dOut = deconvOut[i]; + + INDArray input; + if (df == Convolution3D.DataFormat.NDHWC) { + input = Nd4j.rand(new int[]{miniBatchSize, depth, height, width, convNIn}); + } else { + input = Nd4j.rand(new int[]{miniBatchSize, convNIn, depth, height, width}); + } + INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); + for (int j = 0; j < miniBatchSize; j++) { + labels.putScalar(new int[]{j, j % finalNOut}, 1.0); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .weightInit(new NormalDistribution(0, 0.1)) + .list() + .layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel) + .stride(stride).nIn(convNIn).nOut(dOut).hasBias(false) + .convolutionMode(mode).dataFormat(df) + .build()) + .layer(1, new Deconvolution3D.Builder().activation(afn).kernelSize(kernel) + .stride(stride).nOut(dOut).hasBias(false) + .convolutionMode(mode).dataFormat(df) + .build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(finalNOut).build()) + .setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build(); + + String json = conf.toJson(); + MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, c2); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "DataFormat = " + df + ", minibatch size = " + miniBatchSize + ", activationFn=" + afn + + ", kernel = " + Arrays.toString(kernel) + ", stride = " + + Arrays.toString(stride) + ", mode = " + mode.toString() + + ", input depth " + depth + ", input height " + height + + ", input width " + width; + + if (PRINT_RESULTS) { + log.info(msg); + } + + boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input) + .labels(labels).subset(true).maxPerParam(64)); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java new file mode 100644 index 000000000..b3a9e1020 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java @@ -0,0 +1,1324 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.gradientcheck; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.CNN2DFormat; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Arrays; + +import static org.deeplearning4j.nn.conf.ConvolutionMode.Same; +import static org.deeplearning4j.nn.conf.ConvolutionMode.Truncate; +import static org.junit.jupiter.api.Assertions.*; + +public class CNNGradientCheckTest extends BaseDL4JTest { + private static final boolean PRINT_RESULTS = true; + private static final boolean RETURN_ON_FIRST_FAILURE = false; + private static final double DEFAULT_EPS = 1e-6; + private static final double DEFAULT_MAX_REL_ERROR = 1e-3; + private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; + + static { + Nd4j.setDataType(DataType.DOUBLE); + } + + private CNN2DFormat format; + + public CNNGradientCheckTest(CNN2DFormat format){ + this.format = format; + } + + public static Object[] params(){ + return CNN2DFormat.values(); + } + + @Override + public long getTimeoutMilliseconds() { + return 999990000L; + } + + @Test + public void testGradientCNNMLN() { + if(this.format != CNN2DFormat.NCHW) //Only test NCHW due to flat input format... + return; + + //Parameterized test, testing combinations of: + // (a) activation function + // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') + // (c) Loss function (with specified output activations) + Activation[] activFns = {Activation.SIGMOID, Activation.TANH}; + boolean[] characteristic = {false, true}; //If true: run some backprop steps first + + LossFunctions.LossFunction[] lossFunctions = + {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE}; + Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH}; //i.e., lossFunctions[i] used with outputActivations[i] here + + DataSet ds = new IrisDataSetIterator(150, 150).next(); + ds.normalizeZeroMeanZeroUnitVariance(); + INDArray input = ds.getFeatures(); + INDArray labels = ds.getLabels(); + + for (Activation afn : activFns) { + for (boolean doLearningFirst : characteristic) { + for (int i = 0; i < lossFunctions.length; i++) { + LossFunctions.LossFunction lf = lossFunctions[i]; + Activation outputActivation = outputActivations[i]; + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp()) + .weightInit(WeightInit.XAVIER).seed(12345L).list() + .layer(0, new ConvolutionLayer.Builder(1, 1).nOut(6).activation(afn).build()) + .layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).build()) + .setInputType(InputType.convolutionalFlat(1, 4, 1)); + + MultiLayerConfiguration conf = builder.build(); + + MultiLayerNetwork mln = new MultiLayerNetwork(conf); + mln.init(); + String name = new Object() { + }.getClass().getEnclosingMethod().getName(); + + if (doLearningFirst) { + //Run a number of iterations of learning + mln.setInput(ds.getFeatures()); + mln.setLabels(ds.getLabels()); + mln.computeGradientAndScore(); + double scoreBefore = mln.score(); + for (int j = 0; j < 10; j++) + mln.fit(ds); + mln.computeGradientAndScore(); + double scoreAfter = mln.score(); + //Can't test in 'characteristic mode of operation' if not learning + String msg = name + " - score did not (sufficiently) decrease during learning - activationFn=" + + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + + ", scoreAfter=" + scoreAfter + ")"; + assertTrue(scoreAfter < 0.9 * scoreBefore, msg); + } + + if (PRINT_RESULTS) { + System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + + outputActivation + ", doLearningFirst=" + doLearningFirst); +// for (int j = 0; j < mln.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); + } + + boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK); + TestUtils.testModelSerialization(mln); + } + } + } + } + + + @Test + public void testGradientCNNL1L2MLN() { + if(this.format != CNN2DFormat.NCHW) //Only test NCHW due to flat input format... + return; + + //Parameterized test, testing combinations of: + // (a) activation function + // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') + // (c) Loss function (with specified output activations) + + DataSet ds = new IrisDataSetIterator(150, 150).next(); + ds.normalizeZeroMeanZeroUnitVariance(); + INDArray input = ds.getFeatures(); + INDArray labels = ds.getLabels(); + + //use l2vals[i] with l1vals[i] + double[] l2vals = {0.4, 0.0, 0.4, 0.4}; + double[] l1vals = {0.0, 0.0, 0.5, 0.0}; + double[] biasL2 = {0.0, 0.0, 0.0, 0.2}; + double[] biasL1 = {0.0, 0.0, 0.6, 0.0}; + Activation[] activFns = {Activation.SIGMOID, Activation.TANH, Activation.ELU, Activation.SOFTPLUS}; + boolean[] characteristic = {false, true, false, true}; //If true: run some backprop steps first + + LossFunctions.LossFunction[] lossFunctions = + {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE, LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE}; + Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH, Activation.SOFTMAX, Activation.IDENTITY}; //i.e., lossFunctions[i] used with outputActivations[i] here + + for( int i=0; i (mb,4,2,2) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(2 * 2 * 4) + .nOut(nOut).build()) + .setInputType(InputType.convolutionalFlat(height, width, inputDepth)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + + afn; + + if (PRINT_RESULTS) { + System.out.println(msg); +// for (int j = 0; j < net.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); + } + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } + } + + @Test + public void testCnnWithSpaceToBatch() { + Nd4j.getRandom().setSeed(12345); + int nOut = 4; + + int[] minibatchSizes = {2, 4}; + int width = 5; + int height = 5; + int inputDepth = 1; + + int[] kernel = {2, 2}; + int[] blocks = {2, 2}; + + String[] activations = {"sigmoid", "tanh"}; + SubsamplingLayer.PoolingType[] poolingTypes = + new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX, + SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM}; + + boolean nchw = format == CNN2DFormat.NCHW; + for (String afn : activations) { + for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { + for (int minibatchSize : minibatchSizes) { + long[] inShape = nchw ? new long[]{minibatchSize, inputDepth, height, width} : new long[]{minibatchSize, height, width, inputDepth}; + INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); + INDArray labels = Nd4j.zeros(4 * minibatchSize, nOut); + for (int i = 0; i < 4 * minibatchSize; i++) { + labels.putScalar(new int[]{i, i % nOut}, 1.0); + } + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()).weightInit(new NormalDistribution(0, 1)) + .list() + .layer(new ConvolutionLayer.Builder(kernel) + .nIn(inputDepth).nOut(3) + .dataFormat(format) + .build()) + .layer(new SpaceToBatchLayer.Builder(blocks) + .dataFormat(format) + .build()) //trivial space to batch + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .nOut(nOut).build()) + .setInputType(InputType.convolutional(height, width, inputDepth, format)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = format + " - poolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + + afn; + + if (PRINT_RESULTS) { + System.out.println(msg); +// for (int j = 0; j < net.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); + } + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + //Also check compgraph: + ComputationGraph cg = net.toComputationGraph(); + gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(cg).inputs(new INDArray[]{input}) + .labels(new INDArray[]{labels})); + assertTrue(gradOK, msg + " - compgraph"); + + TestUtils.testModelSerialization(net); + } + } + } + } + + + @Test + public void testCnnWithUpsampling() { + Nd4j.getRandom().setSeed(12345); + int nOut = 4; + + int[] minibatchSizes = {1, 3}; + int width = 5; + int height = 5; + int inputDepth = 1; + + int[] kernel = {2, 2}; + int[] stride = {1, 1}; + int[] padding = {0, 0}; + int size = 2; + + boolean nchw = format == CNN2DFormat.NCHW; + + for (int minibatchSize : minibatchSizes) { + long[] inShape = nchw ? new long[]{minibatchSize, inputDepth, height, width} : new long[]{minibatchSize, height, width, inputDepth}; + INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); + INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .updater(new NoOp()) + .dist(new NormalDistribution(0, 1)) + .list().layer(new ConvolutionLayer.Builder(kernel, + stride, padding).nIn(inputDepth) + .dataFormat(format) + .nOut(3).build())//output: (5-2+0)/1+1 = 4 + .layer(new Upsampling2D.Builder().size(size).dataFormat(format).build()) //output: 4*2 =8 -> 8x8x3 + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(8 * 8 * 3) + .nOut(4).build()) + .setInputType(InputType.convolutional(height, width, inputDepth, format)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "Upsampling - minibatch=" + minibatchSize; + + if (PRINT_RESULTS) { + System.out.println(msg); +// for (int j = 0; j < net.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } + + + @Test + public void testCnnWithSubsampling() { + Nd4j.getRandom().setSeed(12345); + int nOut = 4; + + int[] minibatchSizes = {1, 3}; + int width = 5; + int height = 5; + int inputDepth = 1; + + int[] kernel = {2, 2}; + int[] stride = {1, 1}; + int[] padding = {0, 0}; + int pnorm = 2; + + Activation[] activations = {Activation.SIGMOID, Activation.TANH}; + SubsamplingLayer.PoolingType[] poolingTypes = + new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX, + SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM}; + + boolean nchw = format == CNN2DFormat.NCHW; + + for (Activation afn : activations) { + for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { + for (int minibatchSize : minibatchSizes) { + long[] inShape = nchw ? new long[]{minibatchSize, inputDepth, height, width} : new long[]{minibatchSize, height, width, inputDepth}; + INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); + INDArray labels = Nd4j.zeros(minibatchSize, nOut); + for (int i = 0; i < minibatchSize; i++) { + labels.putScalar(new int[]{i, i % nOut}, 1.0); + } + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().updater(new NoOp()) + .dataType(DataType.DOUBLE) + .dist(new NormalDistribution(0, 1)) + .list().layer(0, + new ConvolutionLayer.Builder(kernel, + stride, padding).nIn(inputDepth) + .dataFormat(format) + .nOut(3).build())//output: (5-2+0)/1+1 = 4 + .layer(1, new SubsamplingLayer.Builder(poolingType) + .dataFormat(format) + .kernelSize(kernel).stride(stride).padding(padding) + .pnorm(pnorm).build()) //output: (4-2+0)/1+1 =3 -> 3x3x3 + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3 * 3 * 3) + .nOut(4).build()) + .setInputType(InputType.convolutional(height, width, inputDepth, format)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = format + " - poolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + + afn; + + if (PRINT_RESULTS) { + System.out.println(msg); +// for (int j = 0; j < net.getnLayers(); j++) +// System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); + } + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } + } + } + + @Test + public void testCnnWithSubsamplingV2() { + Nd4j.getRandom().setSeed(12345); + int nOut = 4; + + int[] minibatchSizes = {1, 3}; + int width = 5; + int height = 5; + int inputDepth = 1; + + int[] kernel = {2, 2}; + int[] stride = {1, 1}; + int[] padding = {0, 0}; + int pNorm = 3; + + Activation[] activations = {Activation.SIGMOID, Activation.TANH}; + SubsamplingLayer.PoolingType[] poolingTypes = + new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX, + SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM}; + + boolean nchw = format == CNN2DFormat.NCHW; + + for (Activation afn : activations) { + for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { + for (int minibatchSize : minibatchSizes) { + long[] inShape = nchw ? new long[]{minibatchSize, inputDepth, height, width} : new long[]{minibatchSize, height, width, inputDepth}; + INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); + INDArray labels = Nd4j.zeros(minibatchSize, nOut); + for (int i = 0; i < minibatchSize; i++) { + labels.putScalar(new int[]{i, i % nOut}, 1.0); + } + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().updater(new NoOp()) + .dataType(DataType.DOUBLE) + .dist(new NormalDistribution(0, 1)) + .list().layer(0, + new ConvolutionLayer.Builder(kernel, + stride, padding).nIn(inputDepth).dataFormat(format) + .nOut(3).build())//output: (5-2+0)/1+1 = 4 + .layer(1, new SubsamplingLayer.Builder(poolingType).dataFormat(format) + .kernelSize(kernel).stride(stride).padding(padding) + .pnorm(pNorm).build()) //output: (4-2+0)/1+1 =3 -> 3x3x3 + .layer(2, new ConvolutionLayer.Builder(kernel, stride, padding).dataFormat(format) + .nIn(3).nOut(2).build()) //Output: (3-2+0)/1+1 = 2 + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(2 * 2 * 2) + .nOut(4).build()) + .setInputType(InputType.convolutional(height, width, inputDepth, format)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + + afn; + System.out.println(msg); + + boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); + + assertTrue(gradOK, msg); + + TestUtils.testModelSerialization(net); + } + } + } + } + + @Test + public void testCnnLocallyConnected2D() { + int nOut = 3; + int width = 5; + int height = 5; + + Nd4j.getRandom().setSeed(12345); + + int[] inputDepths = new int[]{1, 2, 4}; + Activation[] activations = {Activation.SIGMOID, Activation.TANH, Activation.SOFTPLUS}; + int[] minibatch = {2, 1, 3}; + + boolean nchw = format == CNN2DFormat.NCHW; + + for( int i=0; i 0,"Could not generate non-zero mask after " + attempts + " attempts"); + assertTrue(lm.sumNumber().intValue() > 0, "Could not generate non-zero mask after " + attempts + " attempts"); boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{f}) .labels(new INDArray[]{l}).labelMask(new INDArray[]{lm})); @@ -510,7 +503,7 @@ public class GradientCheckTestsMasking extends BaseDL4JTest { double score2 = net.score(new DataSet(f,l,null,lm)); - assertEquals(score, score2, 1e-8,String.valueOf(i)); + assertEquals(score, score2, 1e-8, String.valueOf(i)); } } } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java similarity index 95% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java index 3721c40fc..3ab2efd59 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LRNGradientCheckTests.java @@ -30,10 +30,7 @@ import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; import org.deeplearning4j.nn.conf.layers.LocalResponseNormalization; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -45,10 +42,6 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag public class LRNGradientCheckTests extends BaseDL4JTest { private static final boolean PRINT_RESULTS = true; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java index 708d7a6e9..00fef6150 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LSTMGradientCheckTests.java @@ -31,10 +31,7 @@ import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.conf.preprocessor.RnnToCnnPreProcessor; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -46,10 +43,6 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag public class LSTMGradientCheckTests extends BaseDL4JTest { private static final boolean PRINT_RESULTS = true; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java index a411243a2..bc85841e3 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/LossFunctionGradientCheck.java @@ -34,10 +34,7 @@ import org.deeplearning4j.nn.conf.layers.DenseLayer; import org.deeplearning4j.nn.conf.layers.LossLayer; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.activations.impl.ActivationIdentity; import org.nd4j.linalg.api.buffer.DataType; @@ -52,24 +49,18 @@ import org.nd4j.linalg.lossfunctions.ILossFunction; import org.nd4j.linalg.lossfunctions.impl.*; import org.nd4j.common.primitives.Pair; import org.nd4j.common.util.ArrayUtil; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Random; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.nd4j.linalg.indexing.NDArrayIndex.all; import static org.nd4j.linalg.indexing.NDArrayIndex.point; @Slf4j -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -@Tag(TagNames.LOSS_FUNCTIONS) public class LossFunctionGradientCheck extends BaseDL4JTest { static { @@ -250,7 +241,7 @@ public class LossFunctionGradientCheck extends BaseDL4JTest { } } - assertEquals(0, failed.size(),"Tests failed"); + assertEquals( 0, failed.size(), "Tests failed"); } @Test @@ -350,15 +341,16 @@ public class LossFunctionGradientCheck extends BaseDL4JTest { // Serialize and de-serialize loss function // to ensure that we carry the parameters through // the serializer. - try { + try{ ObjectMapper m = NeuralNetConfiguration.mapper(); String s = m.writeValueAsString(lossFunctions[i]); ILossFunction lf2 = m.readValue(s, lossFunctions[i].getClass()); lossFunctions[i] = lf2; - } catch (IOException ex) { + } catch(IOException ex) { ex.printStackTrace(); - assertEquals(0, 1,"Tests failed: serialization of " + lossFunctions[i]); + assertTrue(false, "Tests failed: serialization of " + lossFunctions[i]); } + Nd4j.getRandom().setSeed(12345); MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .dataType(DataType.DOUBLE) @@ -418,7 +410,7 @@ public class LossFunctionGradientCheck extends BaseDL4JTest { System.out.println(s); } - assertEquals(0, failed.size(),"Tests failed"); + assertEquals( 0, failed.size(), "Tests failed"); } @Test @@ -726,6 +718,6 @@ public class LossFunctionGradientCheck extends BaseDL4JTest { System.out.println(s); } - assertEquals(0, failed.size(),"Tests failed"); + assertEquals(0, failed.size(), "Tests failed"); } } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java index 279756e9d..c9e65579b 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/NoBiasGradientCheckTests.java @@ -28,10 +28,7 @@ import org.deeplearning4j.nn.conf.distribution.NormalDistribution; import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -43,10 +40,6 @@ import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag public class NoBiasGradientCheckTests extends BaseDL4JTest { private static final boolean PRINT_RESULTS = true; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java index 3253d5802..12a1340e2 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/OutputLayerGradientChecks.java @@ -28,10 +28,7 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.distribution.NormalDistribution; import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -47,11 +44,6 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertTrue; - -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag public class OutputLayerGradientChecks extends BaseDL4JTest { private static final boolean PRINT_RESULTS = true; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java index 69ae75016..d1cbd5955 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/RnnGradientChecks.java @@ -35,11 +35,7 @@ import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.conf.layers.recurrent.TimeDistributed; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -51,10 +47,6 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag public class RnnGradientChecks extends BaseDL4JTest { private static final boolean PRINT_RESULTS = true; @@ -69,7 +61,7 @@ public class RnnGradientChecks extends BaseDL4JTest { } @Test - @Disabled("AB 2019/06/24 - Ignored to get to all passing baseline to prevent regressions via CI - see issue #7912") + ////@Ignore("AB 2019/06/24 - Ignored to get to all passing baseline to prevent regressions via CI - see issue #7912") public void testBidirectionalWrapper() { int nIn = 3; @@ -153,7 +145,7 @@ public class RnnGradientChecks extends BaseDL4JTest { } @Test - @Disabled("AB 2019/06/24 - Ignored to get to all passing baseline to prevent regressions via CI - see issue #7912") + //@Ignore("AB 2019/06/24 - Ignored to get to all passing baseline to prevent regressions via CI - see issue #7912") public void testSimpleRnn() { int nOut = 5; @@ -233,7 +225,7 @@ public class RnnGradientChecks extends BaseDL4JTest { } @Test - @Disabled("AB 2019/06/24 - Ignored to get to all passing baseline to prevent regressions via CI - see issue #7912") + //@Ignore("AB 2019/06/24 - Ignored to get to all passing baseline to prevent regressions via CI - see issue #7912") public void testLastTimeStepLayer(){ int nIn = 3; int nOut = 5; @@ -296,7 +288,7 @@ public class RnnGradientChecks extends BaseDL4JTest { boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in) .labels(labels).inputMask(inMask).subset(true).maxPerParam(16)); - assertTrue(gradOK, name); + assertTrue( gradOK, name); TestUtils.testModelSerialization(net); } } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java index 014029830..25d594d9a 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/UtilLayerGradientChecks.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.conf.layers.util.MaskLayer; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -53,11 +50,6 @@ import java.util.Set; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag - public class UtilLayerGradientChecks extends BaseDL4JTest { static { diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java index 4bfe39170..ec9fdab25 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/VaeGradientCheckTests.java @@ -29,10 +29,7 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.conf.layers.variational.*; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.activations.impl.ActivationTanH; import org.nd4j.linalg.api.buffer.DataType; @@ -48,11 +45,6 @@ import java.util.Arrays; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag - public class VaeGradientCheckTests extends BaseDL4JTest { private static final boolean PRINT_RESULTS = false; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java similarity index 86% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java index 68de9caf0..85e513076 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/gradientcheck/YoloGradientCheckTests.java @@ -35,18 +35,8 @@ import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; import org.deeplearning4j.nn.conf.layers.objdetect.Yolo2OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -55,53 +45,40 @@ import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.dataset.api.preprocessor.ImagePreProcessingScaler; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.linalg.factory.Nd4jBackend; import org.nd4j.linalg.learning.config.NoOp; import java.io.File; import java.io.FileOutputStream; import java.io.InputStream; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.FILE_IO) -@NativeTag public class YoloGradientCheckTests extends BaseDL4JTest { static { Nd4j.setDataType(DataType.DOUBLE); } - - @TempDir Path testDir; - - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(CNN2DFormat format : CNN2DFormat.values()) { - args.add(Arguments.of(format,nd4jBackend)); - } - } - return args.stream(); + private CNN2DFormat format; + public YoloGradientCheckTests(CNN2DFormat format){ + this.format = format; } + public static Object[] params(){ + return CNN2DFormat.values(); + } + + @TempDir + public File testDir; + @Override public long getTimeoutMilliseconds() { return 90000L; } - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.YoloGradientCheckTests#params") - public void testYoloOutputLayer(CNN2DFormat format,Nd4jBackend backend) { + @Test + public void testYoloOutputLayer() { int depthIn = 2; int c = 3; int b = 3; @@ -173,18 +150,18 @@ public class YoloGradientCheckTests extends BaseDL4JTest { .minAbsoluteError(1e-6) .labels(labels).subset(true).maxPerParam(100)); - assertTrue(gradOK,msg); + assertTrue(gradOK, msg); TestUtils.testModelSerialization(net); } } - private static INDArray yoloLabels(int mb, int c, int h, int w) { + private static INDArray yoloLabels(int mb, int c, int h, int w){ int labelDepth = 4 + c; INDArray labels = Nd4j.zeros(mb, labelDepth, h, w); //put 1 object per minibatch, at positions (0,0), (1,1) etc. //Positions for label boxes: (1,1) to (2,2), (2,2) to (4,4) etc - for( int i = 0; i < mb; i++) { + for( int i=0; i trying to predict 1 or -1 + Activation.SIGMOID, //kld -> probab so should be between 0 and 1 + Activation.SOFTMAX, //kld + softmax + Activation.TANH, //l1 + Activation.SOFTMAX, //l1 + softmax + Activation.TANH, //l2 + Activation.SOFTMAX, //l2 + softmax + Activation.IDENTITY, //mae + Activation.SOFTMAX, //mae + softmax + Activation.IDENTITY, //mape + Activation.SOFTMAX, //mape + softmax + Activation.SOFTMAX, //mcxent + Activation.IDENTITY, //mse + Activation.SOFTMAX, //mse + softmax + Activation.SIGMOID, //msle - requires positive labels/activations due to log + Activation.SOFTMAX, //msle + softmax + Activation.SIGMOID, //nll + Activation.SOFTMAX, //nll + softmax + Activation.SIGMOID, //poisson - requires positive predictions due to log... not sure if this is the best option + Activation.TANH, //squared hinge + Activation.SIGMOID, //f-measure (binary, single sigmoid output) + Activation.SOFTMAX //f-measure (binary, 2-label softmax output) + }; + + int[] nOut = new int[] {1, //xent + 3, //xent + 5, //cosine + 3, //hinge + 3, //kld + 3, //kld + softmax + 3, //l1 + 3, //l1 + softmax + 3, //l2 + 3, //l2 + softmax + 3, //mae + 3, //mae + softmax + 3, //mape + 3, //mape + softmax + 3, //mcxent + 3, //mse + 3, //mse + softmax + 3, //msle + 3, //msle + softmax + 3, //nll + 3, //nll + softmax + 3, //poisson + 3, //squared hinge + 1, //f-measure (binary, single sigmoid output) + 2, //f-measure (binary, 2-label softmax output) + }; + + for (int i = 0; i < lossFunctions.length; i++) { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(Updater.ADAM).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(nOut[i]).activation(Activation.TANH).build()) + .layer(1, new LossLayer.Builder().lossFunction(lossFunctions[i]) + .activation(outputActivationFn[i]).build()) + .validateOutputLayerConfig(false).build(); + + String json = conf.toJson(); + String yaml = conf.toYaml(); + + MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json); + MultiLayerConfiguration fromYaml = MultiLayerConfiguration.fromYaml(yaml); + + assertEquals(conf, fromJson); + assertEquals(conf, fromYaml); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/MultiLayerNeuralNetConfigurationTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/MultiLayerNeuralNetConfigurationTest.java new file mode 100644 index 000000000..33d8856cd --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/MultiLayerNeuralNetConfigurationTest.java @@ -0,0 +1,429 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.exception.DL4JInvalidConfigException; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.conf.weightnoise.DropConnect; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.*; +import java.util.Arrays; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +public class MultiLayerNeuralNetConfigurationTest extends BaseDL4JTest { + + @TempDir + public File testDir; + + @Test + public void testJson() throws Exception { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new DenseLayer.Builder().dist(new NormalDistribution(1, 1e-1)).build()) + .inputPreProcessor(0, new CnnToFeedForwardPreProcessor()).build(); + + String json = conf.toJson(); + MultiLayerConfiguration from = MultiLayerConfiguration.fromJson(json); + assertEquals(conf.getConf(0), from.getConf(0)); + + Properties props = new Properties(); + props.put("json", json); + String key = props.getProperty("json"); + assertEquals(json, key); + File f = new File(testDir, "props"); + f.deleteOnExit(); + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f)); + props.store(bos, ""); + bos.flush(); + bos.close(); + BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f)); + Properties props2 = new Properties(); + props2.load(bis); + bis.close(); + assertEquals(props2.getProperty("json"), props.getProperty("json")); + String json2 = props2.getProperty("json"); + MultiLayerConfiguration conf3 = MultiLayerConfiguration.fromJson(json2); + assertEquals(conf.getConf(0), conf3.getConf(0)); + + } + + @Test + public void testConvnetJson() { + final int numRows = 76; + final int numColumns = 76; + int nChannels = 3; + int outputNum = 6; + int seed = 123; + + //setup the network + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .l1(1e-1).l2(2e-4).weightNoise(new DropConnect(0.5)).miniBatch(true) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list() + .layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .build()) + .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .build()) + .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()) + .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .build()) + + .setInputType(InputType.convolutional(numRows, numColumns, nChannels)); + + MultiLayerConfiguration conf = builder.build(); + String json = conf.toJson(); + MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, conf2); + } + + @Test + public void testUpsamplingConvnetJson() { + final int numRows = 76; + final int numColumns = 76; + int nChannels = 3; + int outputNum = 6; + int seed = 123; + + //setup the network + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .l1(1e-1).l2(2e-4).dropOut(0.5).miniBatch(true) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list() + .layer(new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(new Upsampling2D.Builder().size(2).build()) + .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(new Upsampling2D.Builder().size(2).build()) + .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()) + .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .build()) + + .setInputType(InputType.convolutional(numRows, numColumns, nChannels)); + + MultiLayerConfiguration conf = builder.build(); + String json = conf.toJson(); + MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, conf2); + } + + @Test + public void testGlobalPoolingJson() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()) + .dist(new NormalDistribution(0, 1.0)).seed(12345L).list() + .layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nOut(5).build()) + .layer(1, new GlobalPoolingLayer.Builder().poolingType(PoolingType.PNORM).pnorm(3).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(3).build()) + .setInputType(InputType.convolutional(32, 32, 1)).build(); + + String str = conf.toJson(); + MultiLayerConfiguration fromJson = conf.fromJson(str); + + assertEquals(conf, fromJson); + } + + + @Test + public void testYaml() throws Exception { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new DenseLayer.Builder().dist(new NormalDistribution(1, 1e-1)).build()) + .inputPreProcessor(0, new CnnToFeedForwardPreProcessor()).build(); + String json = conf.toYaml(); + MultiLayerConfiguration from = MultiLayerConfiguration.fromYaml(json); + assertEquals(conf.getConf(0), from.getConf(0)); + + Properties props = new Properties(); + props.put("json", json); + String key = props.getProperty("json"); + assertEquals(json, key); + File f = new File(testDir, "props"); + f.deleteOnExit(); + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f)); + props.store(bos, ""); + bos.flush(); + bos.close(); + BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f)); + Properties props2 = new Properties(); + props2.load(bis); + bis.close(); + assertEquals(props2.getProperty("json"), props.getProperty("json")); + String yaml = props2.getProperty("json"); + MultiLayerConfiguration conf3 = MultiLayerConfiguration.fromYaml(yaml); + assertEquals(conf.getConf(0), conf3.getConf(0)); + + } + + @Test + public void testClone() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().build()) + .layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).build()) + .inputPreProcessor(1, new CnnToFeedForwardPreProcessor()).build(); + + MultiLayerConfiguration conf2 = conf.clone(); + + assertEquals(conf, conf2); + assertNotSame(conf, conf2); + assertNotSame(conf.getConfs(), conf2.getConfs()); + for (int i = 0; i < conf.getConfs().size(); i++) { + assertNotSame(conf.getConf(i), conf2.getConf(i)); + } + assertNotSame(conf.getInputPreProcessors(), conf2.getInputPreProcessors()); + for (Integer layer : conf.getInputPreProcessors().keySet()) { + assertNotSame(conf.getInputPreProcess(layer), conf2.getInputPreProcess(layer)); + } + } + + @Test + public void testRandomWeightInit() { + MultiLayerNetwork model1 = new MultiLayerNetwork(getConf()); + model1.init(); + + Nd4j.getRandom().setSeed(12345L); + MultiLayerNetwork model2 = new MultiLayerNetwork(getConf()); + model2.init(); + + float[] p1 = model1.params().data().asFloat(); + float[] p2 = model2.params().data().asFloat(); + System.out.println(Arrays.toString(p1)); + System.out.println(Arrays.toString(p2)); + + org.junit.jupiter.api.Assertions.assertArrayEquals(p1, p2, 0.0f); + } + + @Test + public void testTrainingListener() { + MultiLayerNetwork model1 = new MultiLayerNetwork(getConf()); + model1.init(); + model1.addListeners( new ScoreIterationListener(1)); + + MultiLayerNetwork model2 = new MultiLayerNetwork(getConf()); + model2.addListeners( new ScoreIterationListener(1)); + model2.init(); + + Layer[] l1 = model1.getLayers(); + for (int i = 0; i < l1.length; i++) + assertTrue(l1[i].getListeners() != null && l1[i].getListeners().size() == 1); + + Layer[] l2 = model2.getLayers(); + for (int i = 0; i < l2.length; i++) + assertTrue(l2[i].getListeners() != null && l2[i].getListeners().size() == 1); + } + + + private static MultiLayerConfiguration getConf() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345l).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2) + .dist(new NormalDistribution(0, 1)).build()) + .layer(1, new OutputLayer.Builder().nIn(2).nOut(1) + .activation(Activation.TANH) + .dist(new NormalDistribution(0, 1)).lossFunction(LossFunctions.LossFunction.MSE).build()) + .build(); + return conf; + } + + @Test + public void testInvalidConfig() { + + try { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list() + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + fail("No exception thrown for invalid configuration"); + } catch (IllegalStateException e) { + //OK + log.error("",e); + } catch (Throwable e) { + log.error("",e); + fail("Unexpected exception thrown for invalid config"); + } + + try { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list() + .layer(1, new DenseLayer.Builder().nIn(3).nOut(4).build()) + .layer(2, new OutputLayer.Builder().nIn(4).nOut(5).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + fail("No exception thrown for invalid configuration"); + } catch (IllegalStateException e) { + //OK + log.info(e.toString()); + } catch (Throwable e) { + log.error("",e); + fail("Unexpected exception thrown for invalid config"); + } + + try { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list() + .layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()) + .layer(2, new OutputLayer.Builder().nIn(4).nOut(5).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + fail("No exception thrown for invalid configuration"); + } catch (IllegalStateException e) { + //OK + log.info(e.toString()); + } catch (Throwable e) { + log.error("",e); + fail("Unexpected exception thrown for invalid config"); + } + } + + @Test + public void testListOverloads() { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list() + .layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()) + .layer(1, new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DenseLayer dl = (DenseLayer) conf.getConf(0).getLayer(); + assertEquals(3, dl.getNIn()); + assertEquals(4, dl.getNOut()); + OutputLayer ol = (OutputLayer) conf.getConf(1).getLayer(); + assertEquals(4, ol.getNIn()); + assertEquals(5, ol.getNOut()); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list() + .layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()) + .layer(1, new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build()) + .build(); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder().seed(12345) + .list(new DenseLayer.Builder().nIn(3).nOut(4).build(), + new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build()) + .build(); + MultiLayerNetwork net3 = new MultiLayerNetwork(conf3); + net3.init(); + + + assertEquals(conf, conf2); + assertEquals(conf, conf3); + } + + + @Test + public void testBiasLr() { + //setup the network + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new Adam(1e-2)) + .biasUpdater(new Adam(0.5)).list() + .layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(1, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()) + .layer(2, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(10) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28, 28, 1)).build(); + + org.deeplearning4j.nn.conf.layers.BaseLayer l0 = (BaseLayer) conf.getConf(0).getLayer(); + org.deeplearning4j.nn.conf.layers.BaseLayer l1 = (BaseLayer) conf.getConf(1).getLayer(); + org.deeplearning4j.nn.conf.layers.BaseLayer l2 = (BaseLayer) conf.getConf(2).getLayer(); + org.deeplearning4j.nn.conf.layers.BaseLayer l3 = (BaseLayer) conf.getConf(3).getLayer(); + + assertEquals(0.5, ((Adam)l0.getUpdaterByParam("b")).getLearningRate(), 1e-6); + assertEquals(1e-2, ((Adam)l0.getUpdaterByParam("W")).getLearningRate(), 1e-6); + + assertEquals(0.5, ((Adam)l1.getUpdaterByParam("b")).getLearningRate(), 1e-6); + assertEquals(1e-2, ((Adam)l1.getUpdaterByParam("W")).getLearningRate(), 1e-6); + + assertEquals(0.5, ((Adam)l2.getUpdaterByParam("b")).getLearningRate(), 1e-6); + assertEquals(1e-2, ((Adam)l2.getUpdaterByParam("W")).getLearningRate(), 1e-6); + + assertEquals(0.5, ((Adam)l3.getUpdaterByParam("b")).getLearningRate(), 1e-6); + assertEquals(1e-2, ((Adam)l3.getUpdaterByParam("W")).getLearningRate(), 1e-6); + } + + + @Test + public void testInvalidOutputLayer(){ + /* + Test case (invalid configs) + 1. nOut=1 + softmax + 2. mcxent + tanh + 3. xent + softmax + 4. xent + relu + 5. mcxent + sigmoid + */ + + LossFunctions.LossFunction[] lf = new LossFunctions.LossFunction[]{ + LossFunctions.LossFunction.MCXENT, LossFunctions.LossFunction.MCXENT, LossFunctions.LossFunction.XENT, + LossFunctions.LossFunction.XENT, LossFunctions.LossFunction.MCXENT}; + int[] nOut = new int[]{1, 3, 3, 3, 3}; + Activation[] activations = new Activation[]{Activation.SOFTMAX, Activation.TANH, Activation.SOFTMAX, Activation.RELU, Activation.SIGMOID}; + for( int i=0; i r = net.getLayer(0).conf().getLayer().getRegularizationByParam("b"); + assertEquals(0, r.size()); + + r = net.getLayer(1).conf().getLayer().getRegularizationByParam("beta"); + assertTrue(r == null || r.isEmpty()); + r = net.getLayer(1).conf().getLayer().getRegularizationByParam("gamma"); + assertTrue(r == null || r.isEmpty()); + r = net.getLayer(1).conf().getLayer().getRegularizationByParam("mean"); + assertTrue(r == null || r.isEmpty()); + r = net.getLayer(1).conf().getLayer().getRegularizationByParam("var"); + assertTrue(r == null || r.isEmpty()); + assertEquals(l2, TestUtils.getL2(net.getLayer(2).conf().getLayer().getRegularizationByParam("W")), 1e-4); + r = net.getLayer(2).conf().getLayer().getRegularizationByParam("b"); + assertTrue(r == null || r.isEmpty()); + } + + @Test + public void testLayerPretrainConfig() { + boolean pretrain = true; + + VariationalAutoencoder layer = new VariationalAutoencoder.Builder() + .nIn(10).nOut(5).updater(new Sgd(1e-1)) + .lossFunction(LossFunctions.LossFunction.KL_DIVERGENCE).build(); + + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().seed(42).layer(layer).build(); + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/constraints/TestConstraints.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/constraints/TestConstraints.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/constraints/TestConstraints.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/constraints/TestConstraints.java index 470dbfaa1..fda02a451 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/constraints/TestConstraints.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/constraints/TestConstraints.java @@ -41,10 +41,7 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -57,8 +54,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class TestConstraints extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/dropout/TestDropout.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/dropout/TestDropout.java new file mode 100644 index 000000000..5c06f2adc --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/dropout/TestDropout.java @@ -0,0 +1,615 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.dropout; + +import lombok.Data; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.graph.LayerVertex; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.DropoutLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.conditions.Conditions; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.primitives.Pair; +import org.nd4j.linalg.schedule.MapSchedule; +import org.nd4j.linalg.schedule.ScheduleType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.nd4j.linalg.indexing.NDArrayIndex.all; +import static org.nd4j.linalg.indexing.NDArrayIndex.point; + +public class TestDropout extends BaseDL4JTest { + + @Test + public void testBasicConfig(){ + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dropOut(0.6) + .list() + .layer(new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(new DenseLayer.Builder().nIn(10).nOut(10).dropOut(0.7).build()) + .layer(new DenseLayer.Builder().nIn(10).nOut(10).dropOut(new AlphaDropout(0.5)).build()) + .build(); + + assertEquals(new Dropout(0.6), conf.getConf(0).getLayer().getIDropout()); + assertEquals(new Dropout(0.7), conf.getConf(1).getLayer().getIDropout()); + assertEquals(new AlphaDropout(0.5), conf.getConf(2).getLayer().getIDropout()); + + + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() + .dropOut(0.6) + .graphBuilder() + .addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in") + .addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).dropOut(0.7).build(), "0") + .addLayer("2", new DenseLayer.Builder().nIn(10).nOut(10).dropOut(new AlphaDropout(0.5)).build(), "1") + .setOutputs("2") + .build(); + + assertEquals(new Dropout(0.6), ((LayerVertex)conf2.getVertices().get("0")).getLayerConf().getLayer().getIDropout()); + assertEquals(new Dropout(0.7), ((LayerVertex)conf2.getVertices().get("1")).getLayerConf().getLayer().getIDropout()); + assertEquals(new AlphaDropout(0.5), ((LayerVertex)conf2.getVertices().get("2")).getLayerConf().getLayer().getIDropout()); + } + + @Test + public void testCalls(){ + + CustomDropout d1 = new CustomDropout(); + CustomDropout d2 = new CustomDropout(); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new DenseLayer.Builder().nIn(4).nOut(3).dropOut(d1).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).dropOut(d2).nIn(3).nOut(3).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + List l = new ArrayList<>(); + l.add(new DataSet(Nd4j.rand(5,4), Nd4j.rand(5,3))); + l.add(new DataSet(Nd4j.rand(5,4), Nd4j.rand(5,3))); + l.add(new DataSet(Nd4j.rand(5,4), Nd4j.rand(5,3))); + + DataSetIterator iter = new ExistingDataSetIterator(l); + + net.fit(iter); + net.fit(iter); + + List> expList = Arrays.asList( + new Pair<>(0, 0), + new Pair<>(1, 0), + new Pair<>(2, 0), + new Pair<>(3, 1), + new Pair<>(4, 1), + new Pair<>(5, 1)); + + assertEquals(expList, d1.getAllCalls()); + assertEquals(expList, d2.getAllCalls()); + + assertEquals(expList, d1.getAllReverseCalls()); + assertEquals(expList, d2.getAllReverseCalls()); + + + d1 = new CustomDropout(); + d2 = new CustomDropout(); + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() + .graphBuilder() + .addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(4).nOut(3).dropOut(d1).build(), "in") + .addLayer("1", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).dropOut(d2).nIn(3).nOut(3).build(), "0") + .setOutputs("1") + .build(); + + ComputationGraph net2 = new ComputationGraph(conf2); + net2.init(); + + net2.fit(iter); + net2.fit(iter); + + assertEquals(expList, d1.getAllCalls()); + assertEquals(expList, d2.getAllCalls()); + } + + @Data + public static class CustomDropout implements IDropout{ + private List> allCalls = new ArrayList<>(); + private List> allReverseCalls = new ArrayList<>(); + + @Override + public INDArray applyDropout(INDArray inputActivations, INDArray result, int iteration, int epoch, LayerWorkspaceMgr workspaceMgr) { + allCalls.add(new Pair<>(iteration, epoch)); + return inputActivations; + } + + @Override + public INDArray backprop(INDArray gradAtOutput, INDArray gradAtInput, int iteration, int epoch) { + allReverseCalls.add(new Pair<>(iteration, epoch)); + return gradAtInput; + } + + @Override + public void clear() { + + } + + @Override + public IDropout clone() { + return this; + } + } + + @Test + public void testSerialization(){ + + IDropout[] dropouts = new IDropout[]{ + new Dropout(0.5), + new AlphaDropout(0.5), + new GaussianDropout(0.1), + new GaussianNoise(0.1)}; + + for(IDropout id : dropouts) { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dropOut(id) + .list() + .layer(new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(3).nOut(3).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + TestUtils.testModelSerialization(net); + + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() + .dropOut(id) + .graphBuilder() + .addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "in") + .addLayer("1", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(3).nOut(3).build(), "0") + .setOutputs("1") + .build(); + + ComputationGraph net2 = new ComputationGraph(conf2); + net2.init(); + + TestUtils.testModelSerialization(net2); + } + } + + @Test + public void testDropoutValues(){ + Nd4j.getRandom().setSeed(12345); + + Dropout d = new Dropout(0.5); + + INDArray in = Nd4j.ones(10, 10); + INDArray out = d.applyDropout(in, Nd4j.create(10,10), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + + assertEquals(in, Nd4j.ones(10, 10)); + + int countZeros = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(0))).getInt(0); + int countTwos = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(2))).getInt(0); + + assertEquals(100, countZeros + countTwos); //Should only be 0 or 2 + //Stochastic, but this should hold for most cases + assertTrue(countZeros >= 25 && countZeros <= 75); + assertTrue(countTwos >= 25 && countTwos <= 75); + + //Test schedule: + d = new Dropout(new MapSchedule.Builder(ScheduleType.ITERATION).add(0, 0.5).add(5, 0.1).build()); + for( int i=0; i<10; i++ ) { + out = d.applyDropout(in, Nd4j.create(in.shape()), i, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + assertEquals(in, Nd4j.ones(10, 10)); + countZeros = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(0))).getInt(0); + + if(i < 5){ + countTwos = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(2))).getInt(0); + assertEquals(100, countZeros + countTwos, String.valueOf(i)); //Should only be 0 or 2 + //Stochastic, but this should hold for most cases + assertTrue(countZeros >= 25 && countZeros <= 75); + assertTrue(countTwos >= 25 && countTwos <= 75); + } else { + int countInverse = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(1.0/0.1))).getInt(0); + assertEquals(100, countZeros + countInverse); //Should only be 0 or 10 + //Stochastic, but this should hold for most cases + assertTrue(countZeros >= 80); + assertTrue(countInverse <= 20); + } + } + } + + @Test + public void testGaussianDropoutValues(){ + Nd4j.getRandom().setSeed(12345); + + GaussianDropout d = new GaussianDropout(0.1); //sqrt(0.1/(1-0.1)) = 0.3333 stdev + + INDArray in = Nd4j.ones(50, 50); + INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + + assertEquals(in, Nd4j.ones(50, 50)); + + double mean = out.meanNumber().doubleValue(); + double stdev = out.stdNumber().doubleValue(); + + assertEquals(1.0, mean, 0.05); + assertEquals(0.333, stdev, 0.02); + } + + @Test + public void testGaussianNoiseValues(){ + Nd4j.getRandom().setSeed(12345); + + GaussianNoise d = new GaussianNoise(0.1); //sqrt(0.1/(1-0.1)) = 0.3333 stdev + + INDArray in = Nd4j.ones(50, 50); + INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + + assertEquals(in, Nd4j.ones(50, 50)); + + double mean = out.meanNumber().doubleValue(); + double stdev = out.stdNumber().doubleValue(); + + assertEquals(1.0, mean, 0.05); + assertEquals(0.1, stdev, 0.01); + } + + @Test + public void testAlphaDropoutValues(){ + Nd4j.getRandom().setSeed(12345); + + double p = 0.4; + AlphaDropout d = new AlphaDropout(p); + + double SELU_ALPHA = 1.6732632423543772; + double SELU_LAMBDA = 1.0507009873554804; + double alphaPrime = - SELU_LAMBDA * SELU_ALPHA; + double a = 1.0 / Math.sqrt((p + alphaPrime * alphaPrime * p * (1-p))); + double b = -1.0 / Math.sqrt(p + alphaPrime * alphaPrime * p * (1-p)) * (1-p) * alphaPrime; + + double actA = d.a(p); + double actB = d.b(p); + + assertEquals(a, actA, 1e-6); + assertEquals(b, actB, 1e-6); + + INDArray in = Nd4j.ones(10, 10); + INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + + int countValueDropped = 0; + int countEqn = 0; + double eqn = a * 1 + b; + double valueDropped = a * alphaPrime + b; + for(int i=0; i<100; i++ ){ + double v = out.getDouble(i); + if(v >= valueDropped - 1e-6 && v <= valueDropped + 1e-6){ + countValueDropped++; + } else if(v >= eqn - 1e-6 && v <= eqn + 1e-6){ + countEqn++; + } + + } + + assertEquals(100, countValueDropped + countEqn); + assertTrue(countValueDropped >= 25 && countValueDropped <= 75); + assertTrue(countEqn >= 25 && countEqn <= 75); + } + + + @Test + public void testSpatialDropout5DValues(){ + Nd4j.getRandom().setSeed(12345); + + SpatialDropout d = new SpatialDropout(0.5); + + INDArray in = Nd4j.ones(10, 10, 5, 5, 5); + INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + + assertEquals(in, Nd4j.ones(10, 10, 5, 5, 5)); + + //Now, we expect all values for a given depth to be the same... 0 or 2 + int countZero = 0; + int countTwo = 0; + for( int i=0; i<10; i++ ){ + for( int j=0; j<10; j++ ){ + double value = out.getDouble(i,j,0,0,0); + assertTrue( value == 0 || value == 2.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,5,}, value); + INDArray act = out.get(point(i), point(j), all(), all(),all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countTwo++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 25 && countZero <= 75); + assertTrue(countTwo >= 25 && countTwo <= 75); + + //Test schedule: + d = new SpatialDropout(new MapSchedule.Builder(ScheduleType.ITERATION).add(0, 0.5).add(5, 0.1).build()); + for( int i=0; i<10; i++ ) { + out = d.applyDropout(in, Nd4j.create(in.shape()), i, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + assertEquals(in, Nd4j.ones(10, 10, 5, 5, 5)); + + if(i < 5){ + countZero = 0; + countTwo = 0; + for( int m=0; m<10; m++ ){ + for( int j=0; j<10; j++ ){ + double value = out.getDouble(m,j,0,0,0); + assertTrue( value == 0 || value == 2.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,5,}, value); + INDArray act = out.get(point(m), point(j), all(), all(), all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countTwo++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 25 && countZero <= 75); + assertTrue(countTwo >= 25 && countTwo <= 75); + } else { + countZero = 0; + int countInverse = 0; + for( int m=0; m<10; m++ ){ + for( int j=0; j<10; j++ ){ + double value = out.getDouble(m,j,0,0,0); + assertTrue( value == 0 || value == 10.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,5,}, value); + INDArray act = out.get(point(m), point(j), all(), all(),all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countInverse++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 80); + assertTrue(countInverse <= 20); + } + } + } + + + @Test + public void testSpatialDropoutValues(){ + Nd4j.getRandom().setSeed(12345); + + SpatialDropout d = new SpatialDropout(0.5); + + INDArray in = Nd4j.ones(10, 10, 5, 5); + INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + + assertEquals(in, Nd4j.ones(10, 10, 5, 5)); + + //Now, we expect all values for a given depth to be the same... 0 or 2 + int countZero = 0; + int countTwo = 0; + for( int i=0; i<10; i++ ){ + for( int j=0; j<10; j++ ){ + double value = out.getDouble(i,j,0,0); + assertTrue( value == 0 || value == 2.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,}, value); + INDArray act = out.get(point(i), point(j), all(), all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countTwo++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 25 && countZero <= 75); + assertTrue(countTwo >= 25 && countTwo <= 75); + + //Test schedule: + d = new SpatialDropout(new MapSchedule.Builder(ScheduleType.ITERATION).add(0, 0.5).add(5, 0.1).build()); + for( int i=0; i<10; i++ ) { + out = d.applyDropout(in, Nd4j.create(in.shape()), i, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + assertEquals(in, Nd4j.ones(10, 10, 5, 5)); + + if(i < 5){ + countZero = 0; + countTwo = 0; + for( int m=0; m<10; m++ ){ + for( int j=0; j<10; j++ ){ + double value = out.getDouble(m,j,0,0); + assertTrue( value == 0 || value == 2.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,}, value); + INDArray act = out.get(point(m), point(j), all(), all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countTwo++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 25 && countZero <= 75); + assertTrue(countTwo >= 25 && countTwo <= 75); + } else { + countZero = 0; + int countInverse = 0; + for( int m=0; m<10; m++ ){ + for( int j=0; j<10; j++ ){ + double value = out.getDouble(m,j,0,0); + assertTrue( value == 0 || value == 10.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,}, value); + INDArray act = out.get(point(m), point(j), all(), all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countInverse++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 80); + assertTrue(countInverse <= 20); + } + } + } + + @Test + public void testSpatialDropoutValues3D(){ + Nd4j.getRandom().setSeed(12345); + + SpatialDropout d = new SpatialDropout(0.5); + + INDArray in = Nd4j.ones(10, 8, 12); + INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + + assertEquals(in, Nd4j.ones(10, 8, 12)); + + //Now, we expect all values for a given depth to be the same... 0 or 2 + int countZero = 0; + int countTwo = 0; + for( int i=0; i<10; i++ ){ + for( int j=0; j<8; j++ ){ + double value = out.getDouble(i,j,0); + assertTrue( value == 0 || value == 2.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{12}, value); + INDArray act = out.get(point(i), point(j), all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countTwo++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 20 && countZero <= 60); + assertTrue(countTwo >= 20 && countTwo <= 60); + + //Test schedule: + d = new SpatialDropout(new MapSchedule.Builder(ScheduleType.ITERATION).add(0, 0.5).add(5, 0.1).build()); + for( int i=0; i<10; i++ ) { + out = d.applyDropout(in, Nd4j.create(in.shape()), i, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); + assertEquals(in, Nd4j.ones(10, 8, 12)); + + if(i < 5){ + countZero = 0; + countTwo = 0; + for( int m=0; m<10; m++ ){ + for( int j=0; j<8; j++ ){ + double value = out.getDouble(m,j,0); + assertTrue( value == 0 || value == 2.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{12}, value); + INDArray act = out.get(point(m), point(j), all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countTwo++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 20 && countZero <= 60); + assertTrue(countTwo >= 20 && countTwo <= 60); + } else { + countZero = 0; + int countInverse = 0; + for( int m=0; m<10; m++ ){ + for( int j=0; j<8; j++ ){ + double value = out.getDouble(m,j,0); + assertTrue( value == 0 || value == 10.0); + INDArray exp = Nd4j.valueArrayOf(new int[]{12}, value); + INDArray act = out.get(point(m), point(j), all()); + assertEquals(exp, act); + + if(value == 0.0){ + countZero++; + } else { + countInverse++; + } + } + } + + //Stochastic, but this should hold for most cases + assertTrue(countZero >= 60); + assertTrue(countInverse <= 15); + } + } + } + + @Test + public void testSpatialDropoutJSON(){ + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new DropoutLayer.Builder(new SpatialDropout(0.5)).build()) + .build(); + + String asJson = conf.toJson(); + MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(asJson); + + assertEquals(conf, fromJson); + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertexTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertexTest.java new file mode 100644 index 000000000..941309304 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertexTest.java @@ -0,0 +1,710 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.graph; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.ActivationLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.impl.ActivationSigmoid; +import org.nd4j.linalg.activations.impl.ActivationTanH; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.rng.distribution.impl.UniformDistribution; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; +import org.nd4j.linalg.ops.transforms.Transforms; +import org.nd4j.common.primitives.Pair; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +public class ElementWiseVertexTest extends BaseDL4JTest { + @Test + public void testElementWiseVertexNumParams() { + /* + * https://github.com/eclipse/deeplearning4j/pull/3514#issuecomment-307754386 + * from @agibsonccc: check for the basics: like 0 numParams + */ + + ElementWiseVertex.Op ops[] = new ElementWiseVertex.Op[] {ElementWiseVertex.Op.Add, + ElementWiseVertex.Op.Subtract, ElementWiseVertex.Op.Product}; + + for (ElementWiseVertex.Op op : ops) { + ElementWiseVertex ewv = new ElementWiseVertex(op); + Assertions.assertEquals(0, ewv.numParams(true)); + Assertions.assertEquals(0, ewv.numParams(false)); + } + } + + @Test + public void testElementWiseVertexForwardAdd() { + int batchsz = 24; + int featuresz = 17; + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder() + .addInputs("input1", "input2", "input3") + .addLayer("denselayer", + new DenseLayer.Builder().nIn(featuresz).nOut(1).activation(Activation.IDENTITY) + .build(), + "input1") + /* denselayer is not actually used, but it seems that you _need_ to have trainable parameters, otherwise, you get + * Invalid shape: Requested INDArray shape [1, 0] contains dimension size values < 1 (all dimensions must be 1 or more) + * at org.nd4j.linalg.factory.Nd4j.checkShapeValues(Nd4j.java:4877) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4867) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4820) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:3948) + * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:409) + * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:341) + */ + .addVertex("elementwiseAdd", new ElementWiseVertex(ElementWiseVertex.Op.Add), "input1", + "input2", "input3") + .addLayer("Add", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), + "elementwiseAdd") + .setOutputs("Add", "denselayer").build(); + + ComputationGraph cg = new ComputationGraph(cgc); + cg.init(); + + + INDArray input1 = Nd4j.rand(batchsz, featuresz); + INDArray input2 = Nd4j.rand(batchsz, featuresz); + INDArray input3 = Nd4j.rand(batchsz, featuresz); + + INDArray target = input1.dup().addi(input2).addi(input3); + + INDArray output = cg.output(input1, input2, input3)[0]; + INDArray squared = output.sub(target.castTo(output.dataType())); + double rms = squared.mul(squared).sumNumber().doubleValue(); + Assertions.assertEquals(0.0, rms, this.epsilon); + } + + @Test + public void testElementWiseVertexForwardProduct() { + int batchsz = 24; + int featuresz = 17; + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder() + .addInputs("input1", "input2", "input3") + .addLayer("denselayer", + new DenseLayer.Builder().nIn(featuresz).nOut(1).activation(Activation.IDENTITY) + .build(), + "input1") + /* denselayer is not actually used, but it seems that you _need_ to have trainable parameters, otherwise, you get + * Invalid shape: Requested INDArray shape [1, 0] contains dimension size values < 1 (all dimensions must be 1 or more) + * at org.nd4j.linalg.factory.Nd4j.checkShapeValues(Nd4j.java:4877) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4867) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4820) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:3948) + * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:409) + * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:341) + */ + .addVertex("elementwiseProduct", new ElementWiseVertex(ElementWiseVertex.Op.Product), "input1", + "input2", "input3") + .addLayer("Product", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), + "elementwiseProduct") + .setOutputs("Product", "denselayer").build(); + + ComputationGraph cg = new ComputationGraph(cgc); + cg.init(); + + + INDArray input1 = Nd4j.rand(batchsz, featuresz); + INDArray input2 = Nd4j.rand(batchsz, featuresz); + INDArray input3 = Nd4j.rand(batchsz, featuresz); + + INDArray target = input1.dup().muli(input2).muli(input3); + + INDArray output = cg.output(input1, input2, input3)[0]; + INDArray squared = output.sub(target.castTo(output.dataType())); + double rms = squared.mul(squared).sumNumber().doubleValue(); + Assertions.assertEquals(0.0, rms, this.epsilon); + } + + @Test + public void testElementWiseVertexForwardSubtract() { + int batchsz = 24; + int featuresz = 17; + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder() + .addInputs("input1", "input2") + .addLayer("denselayer", + new DenseLayer.Builder().nIn(featuresz).nOut(1).activation(Activation.IDENTITY) + .build(), + "input1") + /* denselayer is not actually used, but it seems that you _need_ to have trainable parameters, otherwise, you get + * Invalid shape: Requested INDArray shape [1, 0] contains dimension size values < 1 (all dimensions must be 1 or more) + * at org.nd4j.linalg.factory.Nd4j.checkShapeValues(Nd4j.java:4877) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4867) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4820) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:3948) + * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:409) + * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:341) + */ + .addVertex("elementwiseSubtract", new ElementWiseVertex(ElementWiseVertex.Op.Subtract), + "input1", "input2") + .addLayer("Subtract", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), + "elementwiseSubtract") + .setOutputs("Subtract", "denselayer").build(); + + ComputationGraph cg = new ComputationGraph(cgc); + cg.init(); + + + INDArray input1 = Nd4j.rand(batchsz, featuresz); + INDArray input2 = Nd4j.rand(batchsz, featuresz); + + INDArray target = input1.dup().subi(input2); + + INDArray output = cg.output(input1, input2)[0]; + INDArray squared = output.sub(target); + double rms = Math.sqrt(squared.mul(squared).sumNumber().doubleValue()); + Assertions.assertEquals(0.0, rms, this.epsilon); + } + + @Test + public void testElementWiseVertexFullAdd() { + int batchsz = 24; + int featuresz = 17; + int midsz = 13; + int outputsz = 11; + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER) + .dataType(DataType.DOUBLE) + .biasInit(0.0).updater(new Sgd()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder() + .addInputs("input1", "input2", "input3") + .addLayer("dense1", + new DenseLayer.Builder().nIn(featuresz).nOut(midsz) + .activation(new ActivationTanH()).build(), + "input1") + .addLayer("dense2", + new DenseLayer.Builder().nIn(featuresz).nOut(midsz) + .activation(new ActivationTanH()).build(), + "input2") + .addLayer("dense3", + new DenseLayer.Builder().nIn(featuresz).nOut(midsz) + .activation(new ActivationTanH()).build(), + "input3") + .addVertex("elementwiseAdd", new ElementWiseVertex(ElementWiseVertex.Op.Add), "dense1", + "dense2", "dense3") + .addLayer("output", + new OutputLayer.Builder().nIn(midsz).nOut(outputsz) + .activation(new ActivationSigmoid()) + .lossFunction(LossFunction.MSE).build(), + "elementwiseAdd") + .setOutputs("output").build(); + + ComputationGraph cg = new ComputationGraph(cgc); + cg.init(); + INDArray input1 = Nd4j.rand(new int[] {batchsz, featuresz}, new UniformDistribution(-1, 1)); + INDArray input2 = Nd4j.rand(new int[] {batchsz, featuresz}, new UniformDistribution(-1, 1)); + INDArray input3 = Nd4j.rand(new int[] {batchsz, featuresz}, new UniformDistribution(-1, 1)); + INDArray target = nullsafe(Nd4j.rand(new int[] {batchsz, outputsz}, new UniformDistribution(0, 1))); + cg.setInputs(input1, input2, input3); + cg.setLabels(target); + + cg.computeGradientAndScore(); + + // Let's figure out what our params are now. + Map params = cg.paramTable(); + INDArray dense1_W = nullsafe(params.get("dense1_W")); + INDArray dense1_b = nullsafe(params.get("dense1_b")); + INDArray dense2_W = nullsafe(params.get("dense2_W")); + INDArray dense2_b = nullsafe(params.get("dense2_b")); + INDArray dense3_W = nullsafe(params.get("dense3_W")); + INDArray dense3_b = nullsafe(params.get("dense3_b")); + INDArray output_W = nullsafe(params.get("output_W")); + INDArray output_b = nullsafe(params.get("output_b")); + + // Now, let's calculate what we expect the output to be. + + INDArray mh = input1.mmul(dense1_W).addi(dense1_b.repmat(batchsz, 1)); + INDArray m = (Transforms.tanh(mh)); + + INDArray nh = input2.mmul(dense2_W).addi(dense2_b.repmat(batchsz, 1)); + INDArray n = (Transforms.tanh(nh)); + + INDArray oh = input3.mmul(dense3_W).addi(dense3_b.repmat(batchsz, 1)); + INDArray o = (Transforms.tanh(oh)); + + INDArray middle = Nd4j.zeros(batchsz, midsz); + middle.addi(m).addi(n).addi(o); + + + INDArray expect = Nd4j.zeros(batchsz, outputsz); + expect.addi(Transforms.sigmoid(middle.mmul(output_W).addi(output_b.repmat(batchsz, 1)))); + + + INDArray output = nullsafe(cg.output(input1, input2, input3)[0]); + + Assertions.assertEquals(0.0, mse(output, expect), this.epsilon); + + Pair pgd = cg.gradientAndScore(); + + double score = pgd.getSecond(); + Assertions.assertEquals(score, mse(output, target), this.epsilon); + + Map gradients = pgd.getFirst().gradientForVariable(); + /* + * So. Let's say we have inputs a, b, c + * mh = a W1 + b1 + * m = tanh(mh) + * + * nh = b W2 + b2 + * n = tanh(nh) + * + * oh = c W3 + b3 + * o = tanh(oh) + * + * s = m+n+o + * + * yh = s W4 + b4 + * y = sigmoid(yh) + * + * E = (y-t)^2 + * dE/dy = 2 (y-t) + * + * dy/dyh = y * (1-y) + * dE/dyh = 2 * y * (1-y) * (y-t) + * + * dyh/dW4 = s.transpose() + * dyh/db4 = Nd4j.ones(1, batchsz) + * dyh/ds = W4.tranpose() + * + * ds/dm = Nd4j.ones(1, midsz) + * + * dm/dmh = 1-(m^2) + * + * dmh/dW1 = a.transpose() + * dmh/db1 = Nd4j.ones(1, batchsz) + * + */ + + INDArray y = output; + INDArray s = middle; + INDArray W4 = output_W; + + INDArray dEdy = Nd4j.zeros(target.shape()); + dEdy.addi(y).subi(target).muli(2); // This should be of size batchsz x outputsz + dEdy.divi(target.shape()[1]); // Why? Because the LossFunction divides by the _element size_ of the output. + + INDArray dydyh = y.mul(y.mul(-1).add(1)); // This is of size batchsz x outputsz + INDArray dEdyh = dydyh.mul(dEdy); + + INDArray dyhdW4 = s.transpose(); + INDArray dEdW4 = nullsafe(dyhdW4.mmul(dEdyh)); + + INDArray dyhdb4 = Nd4j.ones(1, batchsz); + INDArray dEdb4 = nullsafe(dyhdb4.mmul(dEdyh)); + + INDArray dyhds = W4.transpose(); + INDArray dEds = dEdyh.mmul(dyhds); + + INDArray dsdm = Nd4j.ones(batchsz, midsz); + INDArray dEdm = dsdm.mul(dEds); + INDArray dmdmh = (m.mul(m)).mul(-1).add(1); + INDArray dEdmh = dmdmh.mul(dEdm); + INDArray dmhdW1 = input1.transpose(); + INDArray dEdW1 = nullsafe(dmhdW1.mmul(dEdmh)); + INDArray dmhdb1 = Nd4j.ones(1, batchsz); + INDArray dEdb1 = nullsafe(dmhdb1.mmul(dEdmh)); + + INDArray dsdn = Nd4j.ones(batchsz, midsz); + INDArray dEdn = dsdn.mul(dEds); + INDArray dndnh = (n.mul(n)).mul(-1).add(1); + INDArray dEdnh = dndnh.mul(dEdn); + INDArray dnhdW2 = input2.transpose(); + INDArray dEdW2 = nullsafe(dnhdW2.mmul(dEdnh)); + INDArray dnhdb2 = Nd4j.ones(1, batchsz); + INDArray dEdb2 = nullsafe(dnhdb2.mmul(dEdnh)); + + INDArray dsdo = Nd4j.ones(batchsz, midsz); + INDArray dEdo = dsdo.mul(dEds); + INDArray dodoh = (o.mul(o)).mul(-1).add(1); + INDArray dEdoh = dodoh.mul(dEdo); + INDArray dohdW3 = input3.transpose(); + INDArray dEdW3 = nullsafe(dohdW3.mmul(dEdoh)); + INDArray dohdb3 = Nd4j.ones(1, batchsz); + INDArray dEdb3 = nullsafe(dohdb3.mmul(dEdoh)); + + + Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_W")), dEdW4), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_b")), dEdb4), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_W")), dEdW1), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_b")), dEdb1), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_W")), dEdW2), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_b")), dEdb2), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense3_W")), dEdW3), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense3_b")), dEdb3), this.epsilon); + } + + @Test + public void testElementWiseVertexFullProduct() { + int batchsz = 24; + int featuresz = 17; + int midsz = 13; + int outputsz = 11; + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER) + .dataType(DataType.DOUBLE) + .biasInit(0.0).updater(new Sgd()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder() + .addInputs("input1", "input2", "input3") + .addLayer("dense1", + new DenseLayer.Builder().nIn(featuresz).nOut(midsz) + .activation(new ActivationTanH()).build(), + "input1") + .addLayer("dense2", + new DenseLayer.Builder().nIn(featuresz).nOut(midsz) + .activation(new ActivationTanH()).build(), + "input2") + .addLayer("dense3", + new DenseLayer.Builder().nIn(featuresz).nOut(midsz) + .activation(new ActivationTanH()).build(), + "input3") + .addVertex("elementwiseProduct", new ElementWiseVertex(ElementWiseVertex.Op.Product), "dense1", + "dense2", "dense3") + .addLayer("output", + new OutputLayer.Builder().nIn(midsz).nOut(outputsz) + .activation(new ActivationSigmoid()) + .lossFunction(LossFunction.MSE).build(), + "elementwiseProduct") + .setOutputs("output").build(); + + ComputationGraph cg = new ComputationGraph(cgc); + cg.init(); + INDArray input1 = Nd4j.rand(new int[] {batchsz, featuresz}, new UniformDistribution(-1, 1)); + INDArray input2 = Nd4j.rand(new int[] {batchsz, featuresz}, new UniformDistribution(-1, 1)); + INDArray input3 = Nd4j.rand(new int[] {batchsz, featuresz}, new UniformDistribution(-1, 1)); + INDArray target = nullsafe(Nd4j.rand(new int[] {batchsz, outputsz}, new UniformDistribution(0, 1))); + cg.setInputs(input1, input2, input3); + cg.setLabels(target); + + cg.computeGradientAndScore(); + + // Let's figure out what our params are now. + Map params = cg.paramTable(); + INDArray dense1_W = nullsafe(params.get("dense1_W")); + INDArray dense1_b = nullsafe(params.get("dense1_b")); + INDArray dense2_W = nullsafe(params.get("dense2_W")); + INDArray dense2_b = nullsafe(params.get("dense2_b")); + INDArray dense3_W = nullsafe(params.get("dense3_W")); + INDArray dense3_b = nullsafe(params.get("dense3_b")); + INDArray output_W = nullsafe(params.get("output_W")); + INDArray output_b = nullsafe(params.get("output_b")); + + // Now, let's calculate what we expect the output to be. + + INDArray mh = input1.mmul(dense1_W).addi(dense1_b.repmat(batchsz, 1)); + INDArray m = (Transforms.tanh(mh)); + + INDArray nh = input2.mmul(dense2_W).addi(dense2_b.repmat(batchsz, 1)); + INDArray n = (Transforms.tanh(nh)); + + INDArray oh = input3.mmul(dense3_W).addi(dense3_b.repmat(batchsz, 1)); + INDArray o = (Transforms.tanh(oh)); + + INDArray middle = Nd4j.ones(batchsz, midsz); + middle.muli(m).muli(n).muli(o); + + + INDArray expect = Nd4j.zeros(batchsz, outputsz); + expect.addi(Transforms.sigmoid(middle.mmul(output_W).addi(output_b.repmat(batchsz, 1)))); + + + INDArray output = nullsafe(cg.output(input1, input2, input3)[0]); + + Assertions.assertEquals(0.0, mse(output, expect), this.epsilon); + + Pair pgd = cg.gradientAndScore(); + + double score = pgd.getSecond(); + Assertions.assertEquals(score, mse(output, target), this.epsilon); + + Map gradients = pgd.getFirst().gradientForVariable(); + /* + * So. Let's say we have inputs a, b, c + * mh = a W1 + b1 + * m = tanh(mh) + * + * nh = b W2 + b2 + * n = tanh(nh) + * + * oh = c W3 + b3 + * o = tanh(oh) + * + * s = m*n*o + * + * yh = s W4 + b4 + * y = sigmoid(yh) + * + * E = (y-t)^2 + * dE/dy = 2 (y-t) + * + * dy/dyh = y * (1-y) + * dE/dyh = 2 * y * (1-y) * (y-t) + * + * dyh/dW4 = s.transpose() + * dyh/db4 = Nd4j.ones(1, batchsz) + * dyh/ds = W4.tranpose() + * + * ds/dm = Nd4j.ones(1, midsz).mul(o).mul(n) // Basically the _rest_ of the middle layers + * + * dm/dmh = 1-(m^2) + * + * dmh/dW1 = a.transpose() + * dmh/db1 = Nd4j.ones(1, batchsz) + * + */ + + INDArray y = output; + INDArray s = middle; + INDArray W4 = output_W; + + INDArray dEdy = Nd4j.zeros(target.shape()); + dEdy.addi(y).subi(target).muli(2); // This should be of size batchsz x outputsz + dEdy.divi(target.shape()[1]); // Why? Because the LossFunction divides by the _element size_ of the output. + + INDArray dydyh = y.mul(y.mul(-1).add(1)); // This is of size batchsz x outputsz + INDArray dEdyh = dydyh.mul(dEdy); + + INDArray dyhdW4 = s.transpose(); + INDArray dEdW4 = nullsafe(dyhdW4.mmul(dEdyh)); + + INDArray dyhdb4 = Nd4j.ones(1, batchsz); + INDArray dEdb4 = nullsafe(dyhdb4.mmul(dEdyh)); + + INDArray dyhds = W4.transpose(); + INDArray dEds = dEdyh.mmul(dyhds); + + INDArray dsdm = Nd4j.ones(batchsz, midsz).muli(n).muli(o); + INDArray dEdm = dsdm.mul(dEds); + INDArray dmdmh = (m.mul(m)).mul(-1).add(1); + INDArray dEdmh = dmdmh.mul(dEdm); + INDArray dmhdW1 = input1.transpose(); + INDArray dEdW1 = nullsafe(dmhdW1.mmul(dEdmh)); + INDArray dmhdb1 = Nd4j.ones(1, batchsz); + INDArray dEdb1 = nullsafe(dmhdb1.mmul(dEdmh)); + + INDArray dsdn = Nd4j.ones(batchsz, midsz).muli(m).muli(o); + INDArray dEdn = dsdn.mul(dEds); + INDArray dndnh = (n.mul(n)).mul(-1).add(1); + INDArray dEdnh = dndnh.mul(dEdn); + INDArray dnhdW2 = input2.transpose(); + INDArray dEdW2 = nullsafe(dnhdW2.mmul(dEdnh)); + INDArray dnhdb2 = Nd4j.ones(1, batchsz); + INDArray dEdb2 = nullsafe(dnhdb2.mmul(dEdnh)); + + INDArray dsdo = Nd4j.ones(batchsz, midsz).muli(m).muli(n); + INDArray dEdo = dsdo.mul(dEds); + INDArray dodoh = (o.mul(o)).mul(-1).add(1); + INDArray dEdoh = dodoh.mul(dEdo); + INDArray dohdW3 = input3.transpose(); + INDArray dEdW3 = nullsafe(dohdW3.mmul(dEdoh)); + INDArray dohdb3 = Nd4j.ones(1, batchsz); + INDArray dEdb3 = nullsafe(dohdb3.mmul(dEdoh)); + + Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_W")), dEdW4), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_b")), dEdb4), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_W")), dEdW1), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_b")), dEdb1), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_W")), dEdW2), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_b")), dEdb2), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense3_W")), dEdW3), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense3_b")), dEdb3), this.epsilon); + } + + @Test + public void testElementWiseVertexFullSubtract() { + int batchsz = 24; + int featuresz = 17; + int midsz = 13; + int outputsz = 11; + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER) + .dataType(DataType.DOUBLE) + .biasInit(0.0).updater(new Sgd()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder() + .addInputs("input1", "input2") + .addLayer("dense1", + new DenseLayer.Builder().nIn(featuresz).nOut(midsz) + .activation(new ActivationTanH()).build(), + "input1") + .addLayer("dense2", + new DenseLayer.Builder().nIn(featuresz).nOut(midsz) + .activation(new ActivationTanH()).build(), + "input2") + .addVertex("elementwiseSubtract", new ElementWiseVertex(ElementWiseVertex.Op.Subtract), + "dense1", "dense2") + .addLayer("output", + new OutputLayer.Builder().nIn(midsz).nOut(outputsz) + .activation(new ActivationSigmoid()) + .lossFunction(LossFunction.MSE).build(), + "elementwiseSubtract") + .setOutputs("output").build(); + + ComputationGraph cg = new ComputationGraph(cgc); + cg.init(); + INDArray input1 = Nd4j.rand(new int[] {batchsz, featuresz}, new UniformDistribution(-1, 1)); + INDArray input2 = Nd4j.rand(new int[] {batchsz, featuresz}, new UniformDistribution(-1, 1)); + INDArray target = nullsafe(Nd4j.rand(new int[] {batchsz, outputsz}, new UniformDistribution(0, 1))); + cg.setInputs(input1, input2); + cg.setLabels(target); + + cg.computeGradientAndScore(); + + // Let's figure out what our params are now. + Map params = cg.paramTable(); + INDArray dense1_W = nullsafe(params.get("dense1_W")); + INDArray dense1_b = nullsafe(params.get("dense1_b")); + INDArray dense2_W = nullsafe(params.get("dense2_W")); + INDArray dense2_b = nullsafe(params.get("dense2_b")); + INDArray output_W = nullsafe(params.get("output_W")); + INDArray output_b = nullsafe(params.get("output_b")); + + // Now, let's calculate what we expect the output to be. + + INDArray mh = input1.mmul(dense1_W).addi(dense1_b.repmat(batchsz, 1)); + INDArray m = (Transforms.tanh(mh)); + + INDArray nh = input2.mmul(dense2_W).addi(dense2_b.repmat(batchsz, 1)); + INDArray n = (Transforms.tanh(nh)); + + INDArray middle = Nd4j.zeros(batchsz, midsz); + middle.addi(m).subi(n); + + + INDArray expect = Nd4j.zeros(batchsz, outputsz); + expect.addi(Transforms.sigmoid(middle.mmul(output_W).addi(output_b.repmat(batchsz, 1)))); + + + INDArray output = nullsafe(cg.output(input1, input2)[0]); + + Assertions.assertEquals(0.0, mse(output, expect), this.epsilon); + + Pair pgd = cg.gradientAndScore(); + + double score = pgd.getSecond(); + Assertions.assertEquals(score, mse(output, target), this.epsilon); + + Map gradients = pgd.getFirst().gradientForVariable(); + /* + * So. Let's say we have inputs a, b, c + * mh = a W1 + b1 + * m = tanh(mh) + * + * nh = b W2 + b2 + * n = tanh(nh) + * + * s = m-n + * + * yh = s W4 + b4 + * y = sigmoid(yh) + * + * E = (y-t)^2 + * dE/dy = 2 (y-t) + * + * dy/dyh = y * (1-y) + * dE/dyh = 2 * y * (1-y) * (y-t) + * + * dyh/dW4 = s.transpose() + * dyh/db4 = Nd4j.ones(1, batchsz) + * dyh/ds = W4.tranpose() + * + * ds/dm = Nd4j.ones(1, midsz) + * ds/dn = Nd4j.ones(1, midsz).muli(-1) + * + * dm/dmh = 1-(m^2) + * + * dmh/dW1 = a.transpose() + * dmh/db1 = Nd4j.ones(1, batchsz) + * + */ + + INDArray y = output; + INDArray s = middle; + INDArray W4 = output_W; + + INDArray dEdy = Nd4j.zeros(target.shape()); + dEdy.addi(y).subi(target).muli(2); // This should be of size batchsz x outputsz + dEdy.divi(target.shape()[1]); // Why? Because the LossFunction divides by the _element size_ of the output. + + INDArray dydyh = y.mul(y.mul(-1).add(1)); // This is of size batchsz x outputsz + INDArray dEdyh = dydyh.mul(dEdy); + + INDArray dyhdW4 = s.transpose(); + INDArray dEdW4 = nullsafe(dyhdW4.mmul(dEdyh)); + + INDArray dyhdb4 = Nd4j.ones(1, batchsz); + INDArray dEdb4 = nullsafe(dyhdb4.mmul(dEdyh)); + + INDArray dyhds = W4.transpose(); + INDArray dEds = dEdyh.mmul(dyhds); + + INDArray dsdm = Nd4j.ones(batchsz, midsz); + INDArray dEdm = dsdm.mul(dEds); + INDArray dmdmh = (m.mul(m)).mul(-1).add(1); + INDArray dEdmh = dmdmh.mul(dEdm); + INDArray dmhdW1 = input1.transpose(); + INDArray dEdW1 = nullsafe(dmhdW1.mmul(dEdmh)); + INDArray dmhdb1 = Nd4j.ones(1, batchsz); + INDArray dEdb1 = nullsafe(dmhdb1.mmul(dEdmh)); + + INDArray dsdn = Nd4j.ones(batchsz, midsz).muli(-1); + INDArray dEdn = dsdn.mul(dEds); + INDArray dndnh = (n.mul(n)).mul(-1).add(1); + INDArray dEdnh = dndnh.mul(dEdn); + INDArray dnhdW2 = input2.transpose(); + INDArray dEdW2 = nullsafe(dnhdW2.mmul(dEdnh)); + INDArray dnhdb2 = Nd4j.ones(1, batchsz); + INDArray dEdb2 = nullsafe(dnhdb2.mmul(dEdnh)); + + + Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_W")), dEdW4), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_b")), dEdb4), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_W")), dEdW1), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_b")), dEdb1), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_W")), dEdW2), this.epsilon); + Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_b")), dEdb2), this.epsilon); + } + + + private static double mse(INDArray output, INDArray target) { + double mse_expect = Transforms.pow(output.sub(target), 2.0).sumNumber().doubleValue() + / (output.columns() * output.rows()); + return mse_expect; + } + + private static T nullsafe(T obj) { + if (obj == null) + throw new NullPointerException(); + T clean = obj; + return clean; + } + + private double epsilon = 1e-10; +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/graph/ShiftVertexTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/graph/ShiftVertexTest.java new file mode 100644 index 000000000..766854407 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/graph/ShiftVertexTest.java @@ -0,0 +1,262 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.graph; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.ActivationLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.BaseActivationFunction; +import org.nd4j.linalg.activations.impl.ActivationSigmoid; +import org.nd4j.linalg.activations.impl.ActivationTanH; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; +import org.nd4j.common.primitives.Pair; + +import java.util.Map; +import java.util.TreeMap; + +public class ShiftVertexTest extends BaseDL4JTest { + @Test + public void testShiftVertexNumParamsTrue() { + /* + * https://github.com/eclipse/deeplearning4j/pull/3514#issuecomment-307754386 + * from @agibsonccc: check for the basics: like 0 numParams + */ + + ShiftVertex sv = new ShiftVertex(0.7); // The 0.7 doesn't really matter. + Assertions.assertEquals(0, sv.numParams(true)); + } + + @Test + public void testShiftVertexNumParamsFalse() { + /* + * https://github.com/eclipse/deeplearning4j/pull/3514#issuecomment-307754386 + * from @agibsonccc: check for the basics: like 0 numParams + */ + + ShiftVertex sv = new ShiftVertex(0.7); // The 0.7 doesn't really matter. + Assertions.assertEquals(0, sv.numParams(false)); + } + + @Test + public void testGet() { + ShiftVertex sv = new ShiftVertex(0.7); + Assertions.assertEquals(0.7, sv.getShiftFactor(), this.epsilon); + } + + @Test + public void testSimple() { + /* + * This function _simply_ tests whether ShiftVertex is _in fact_ adding the shift value to it's inputs. + */ + // Just first n primes / 10. + INDArray input = Nd4j + .create(new double[][] {{0.2, 0.3, 0.5}, {0.7, 1.1, 1.3}, {1.7, 1.9, 2.3}, {2.9, 3.1, 3.7}}); + double sf = 4.1; + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input") + .addLayer("denselayer", + new DenseLayer.Builder().nIn(input.columns()).nOut(1) + .activation(Activation.IDENTITY).build(), + "input") + /* denselayer is not actually used, but it seems that you _need_ to have trainable parameters, otherwise, you get + * Invalid shape: Requested INDArray shape [1, 0] contains dimension size values < 1 (all dimensions must be 1 or more) + * at org.nd4j.linalg.factory.Nd4j.checkShapeValues(Nd4j.java:4877) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4867) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:4820) + * at org.nd4j.linalg.factory.Nd4j.create(Nd4j.java:3948) + * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:409) + * at org.deeplearning4j.nn.graph.ComputationGraph.init(ComputationGraph.java:341) + */ + .addLayer("identityinputactivation", + new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), "input") + .addVertex("shiftvertex", new ShiftVertex(sf), "identityinputactivation") + .addLayer("identityshiftvertex", + new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), + "shiftvertex") + .setOutputs("identityshiftvertex", "denselayer").build(); + + ComputationGraph cg = new ComputationGraph(cgc); + cg.init(); + + // We can call outputSingle, because we only have a single output layer. It has nothing to do with minibatches. + INDArray output = cg.output(true, input)[0]; + INDArray target = Nd4j.zeros(input.shape()); + target.addi(input); + target.addi(sf); + + INDArray squared = output.sub(target); + double rms = squared.mul(squared).sumNumber().doubleValue(); + Assertions.assertEquals(0.0, rms, this.epsilon); + } + + @Test + public void testComprehensive() { + /* + * This function tests ShiftVertex more comprehensively. Specifically, it verifies that the lossfunction works as + * expected on a ComputationGraph _with_ a ShiftVertex and it verifies that the derivatives produced by + * back propagating work as expected. + */ + BaseActivationFunction a1 = new ActivationTanH(); + BaseActivationFunction a2 = new ActivationSigmoid(); + // Just first n primes / 10. + INDArray input = Nd4j + .create(new double[][] {{0.2, 0.3, 0.5}, {0.7, 1.1, 1.3}, {1.7, 1.9, 2.3}, {2.9, 3.1, 3.7}}); + double sf = 4.1; + // Actually, given that I'm using a sigmoid on the output, + // these should really be between 0 and 1 + INDArray target = Nd4j.create(new double[][] {{0.05, 0.10, 0.15, 0.20, 0.25}, {0.30, 0.35, 0.40, 0.45, 0.50}, + {0.55, 0.60, 0.65, 0.70, 0.75}, {0.80, 0.85, 0.90, 0.95, 0.99}}); + + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER) + .dataType(DataType.DOUBLE) + .updater(new Sgd(0.01)) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder() + .addInputs("input") + .addLayer("denselayer", + new DenseLayer.Builder().nIn(input.columns()).nOut(input.columns()) + .activation(a1).build(), + "input") + .addVertex("shiftvertex", new ShiftVertex(sf), "denselayer") + .addLayer("output", + new OutputLayer.Builder().nIn(input.columns()).nOut(target.columns()) + .activation(a2).lossFunction(LossFunction.MSE).build(), + "shiftvertex") + .setOutputs("output").build(); + ComputationGraph cg = new ComputationGraph(cgc); + cg.init(); + cg.setInput(0, input); + cg.setLabel(0, target); + cg.computeGradientAndScore(); + double score_dl4j = cg.score(); + Map weights = cg.paramTable(); + Gradient g = cg.gradient(); + Map gradients = g.gradientForVariable(); + Map manual_gradients = new TreeMap(); + + INDArray W = nullsafe(weights.get("denselayer_W")); + INDArray b = nullsafe(weights.get("denselayer_b")); + INDArray V = nullsafe(weights.get("output_W")); + INDArray c = nullsafe(weights.get("output_b")); + + Map manual_weights = new TreeMap(); + manual_weights.put("denselayer_W", W); + manual_weights.put("denselayer_b", b); + manual_weights.put("output_W", V); + manual_weights.put("output_b", c); + + // First things first, let's calculate the score. + long batchsz = input.shape()[0]; + INDArray z = input.castTo(W.dataType()).mmul(W).add(b.repmat(batchsz, 1)); + INDArray a = a1.getActivation(z.dup(), true).add(sf); // activation modifies it's input!! + INDArray q = a.mmul(V).add(c.repmat(batchsz, 1)); + INDArray o = nullsafe(a2.getActivation(q.dup(), true)); + double score_manual = sum_errors(o, target) / (o.columns() * o.rows()); + + /* + * So. We have + * z5 = input1 * W15 + input2 * W25 + input3 * W35 + b5 + * a5 = activation(z5) + sr + * q9 = a1 * V19 + a2 * V29 + a3 * V39 + c9 + * o9 = activation(q9) + * + * dE/do = 2(o-t) + * doj/dqj = activation'(qj) + * dqj/dVij = ai dqj/dai = Vij dqj/dbj = 1 + * + * dq1/dv11 = a1 dq2/dV12 = a1 dq3/dV13 = a1 ... + * dq1/dv21 = a2 dq2... + */ + INDArray dEdo = target.like(); //Nd4j.zeros(target.shape()); + dEdo.addi(o.castTo(dEdo.dataType())).subi(target).muli(2); // This should be of size batchsz x outputsz + dEdo.divi(target.shape()[1]); // Why? Because the LossFunction divides by the _element size_ of the output. + + Pair derivs2 = a2.backprop(q, dEdo); + INDArray dEdq = derivs2.getFirst(); // This should be of size batchsz x outputsz (dE/do * do/dq) this _should_ be o * (1-o) * dE/do for Sigmoid. + // Should be o = q^3 do/dq = 3 q^2 for Cube. + /* + INDArray dodq = q.mul(q).mul(3); + INDArray tbv = dodq.mul(dEdo); + System.err.println("----"); + System.err.println(q); + System.err.println(o); + System.err.println(tbv); + System.err.println(dEdq); + */ + + INDArray dqdc = Nd4j.ones(1, batchsz); + INDArray dEdc = dqdc.mmul(dEdq); // This should be of size 1 x outputsz + INDArray dEdV = a.transpose().mmul(dEdq); + INDArray dEda = dEdq.mmul(V.transpose()); // This should be dEdo * dodq * dqda + + Pair derivs1 = a1.backprop(z, dEda); + INDArray dEdz = derivs1.getFirst(); + INDArray dzdb = Nd4j.ones(1, batchsz); + INDArray dEdb = dzdb.mmul(dEdz); + INDArray dEdW = input.transpose().mmul(dEdz); + + manual_gradients.put("output_b", dEdc); + manual_gradients.put("output_W", dEdV); + manual_gradients.put("denselayer_b", dEdb); + manual_gradients.put("denselayer_W", dEdW); + + double summse = Math.pow((score_manual - score_dl4j), 2); + int denominator = 1; + + for (Map.Entry mesi : gradients.entrySet()) { + String name = mesi.getKey(); + INDArray dl4j_gradient = nullsafe(mesi.getValue()); + INDArray manual_gradient = nullsafe(manual_gradients.get(name)); + double se = sum_errors(dl4j_gradient, manual_gradient); + summse += se; + denominator += dl4j_gradient.columns() * dl4j_gradient.rows(); + } + + Assertions.assertEquals(0.0, summse / denominator, this.epsilon); + + } + + private static double sum_errors(INDArray a, INDArray b) { + INDArray o = a.sub(b.castTo(a.dataType())); + return o.mul(o).sumNumber().doubleValue(); + } + + private static T nullsafe(T obj) { + if (obj == null) + throw new NullPointerException(); + T clean = obj; + return clean; + } + + private double epsilon = 1e-10; +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java new file mode 100644 index 000000000..96a2bc739 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java @@ -0,0 +1,232 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.dropout.Dropout; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.activations.impl.ActivationSoftmax; +import org.nd4j.linalg.activations.impl.ActivationTanH; +import org.nd4j.linalg.convolution.Convolution; +import org.nd4j.linalg.learning.config.AdaGrad; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; + +import java.io.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Jeffrey Tang. + */ +public class LayerBuilderTest extends BaseDL4JTest { + final double DELTA = 1e-15; + + int numIn = 10; + int numOut = 5; + double drop = 0.3; + IActivation act = new ActivationSoftmax(); + PoolingType poolType = PoolingType.MAX; + int[] kernelSize = new int[] {2, 2}; + int[] stride = new int[] {2, 2}; + int[] padding = new int[] {1, 1}; + int k = 1; + Convolution.Type convType = Convolution.Type.VALID; + LossFunction loss = LossFunction.MCXENT; + WeightInit weight = WeightInit.XAVIER; + double corrupt = 0.4; + double sparsity = 0.3; + double corruptionLevel = 0.5; + double dropOut = 0.1; + IUpdater updater = new AdaGrad(); + GradientNormalization gradNorm = GradientNormalization.ClipL2PerParamType; + double gradNormThreshold = 8; + + @Test + public void testLayer() throws Exception { + DenseLayer layer = new DenseLayer.Builder().activation(act).weightInit(weight).dropOut(dropOut) + .updater(updater).gradientNormalization(gradNorm) + .gradientNormalizationThreshold(gradNormThreshold).build(); + + checkSerialization(layer); + + assertEquals(act, layer.getActivationFn()); + assertEquals(weight.getWeightInitFunction(), layer.getWeightInitFn()); + assertEquals(new Dropout(dropOut), layer.getIDropout()); + assertEquals(updater, layer.getIUpdater()); + assertEquals(gradNorm, layer.getGradientNormalization()); + assertEquals(gradNormThreshold, layer.getGradientNormalizationThreshold(), 0.0); + } + + @Test + public void testFeedForwardLayer() throws Exception { + DenseLayer ff = new DenseLayer.Builder().nIn(numIn).nOut(numOut).build(); + + checkSerialization(ff); + + assertEquals(numIn, ff.getNIn()); + assertEquals(numOut, ff.getNOut()); + } + + @Test + public void testConvolutionLayer() throws Exception { + ConvolutionLayer conv = new ConvolutionLayer.Builder(kernelSize, stride, padding).build(); + + checkSerialization(conv); + + // assertEquals(convType, conv.getConvolutionType()); + assertArrayEquals(kernelSize, conv.getKernelSize()); + assertArrayEquals(stride, conv.getStride()); + assertArrayEquals(padding, conv.getPadding()); + } + + @Test + public void testSubsamplingLayer() throws Exception { + SubsamplingLayer sample = + new SubsamplingLayer.Builder(poolType, stride).kernelSize(kernelSize).padding(padding).build(); + + checkSerialization(sample); + + assertArrayEquals(padding, sample.getPadding()); + assertArrayEquals(kernelSize, sample.getKernelSize()); + assertEquals(poolType, sample.getPoolingType()); + assertArrayEquals(stride, sample.getStride()); + } + + @Test + public void testOutputLayer() throws Exception { + OutputLayer out = new OutputLayer.Builder(loss).build(); + + checkSerialization(out); + } + + @Test + public void testRnnOutputLayer() throws Exception { + RnnOutputLayer out = new RnnOutputLayer.Builder(loss).build(); + + checkSerialization(out); + } + + @Test + public void testAutoEncoder() throws Exception { + AutoEncoder enc = new AutoEncoder.Builder().corruptionLevel(corruptionLevel).sparsity(sparsity).build(); + + checkSerialization(enc); + + assertEquals(corruptionLevel, enc.getCorruptionLevel(), DELTA); + assertEquals(sparsity, enc.getSparsity(), DELTA); + } + + @Test + public void testGravesLSTM() throws Exception { + GravesLSTM glstm = new GravesLSTM.Builder().forgetGateBiasInit(1.5).activation(Activation.TANH).nIn(numIn) + .nOut(numOut).build(); + + checkSerialization(glstm); + + assertEquals(glstm.getForgetGateBiasInit(), 1.5, 0.0); + assertEquals(glstm.nIn, numIn); + assertEquals(glstm.nOut, numOut); + assertTrue(glstm.getActivationFn() instanceof ActivationTanH); + } + + @Test + public void testGravesBidirectionalLSTM() throws Exception { + final GravesBidirectionalLSTM glstm = new GravesBidirectionalLSTM.Builder().forgetGateBiasInit(1.5) + .activation(Activation.TANH).nIn(numIn).nOut(numOut).build(); + + checkSerialization(glstm); + + assertEquals(1.5, glstm.getForgetGateBiasInit(), 0.0); + assertEquals(glstm.nIn, numIn); + assertEquals(glstm.nOut, numOut); + assertTrue(glstm.getActivationFn() instanceof ActivationTanH); + } + + @Test + public void testEmbeddingLayer() throws Exception { + EmbeddingLayer el = new EmbeddingLayer.Builder().nIn(10).nOut(5).build(); + checkSerialization(el); + + assertEquals(10, el.getNIn()); + assertEquals(5, el.getNOut()); + } + + @Test + public void testBatchNormLayer() throws Exception { + BatchNormalization bN = new BatchNormalization.Builder().nIn(numIn).nOut(numOut).gamma(2).beta(1).decay(0.5) + .lockGammaBeta(true).build(); + + checkSerialization(bN); + + assertEquals(numIn, bN.nIn); + assertEquals(numOut, bN.nOut); + assertEquals(true, bN.isLockGammaBeta()); + assertEquals(0.5, bN.decay, 1e-4); + assertEquals(2, bN.gamma, 1e-4); + assertEquals(1, bN.beta, 1e-4); + } + + @Test + public void testActivationLayer() throws Exception { + ActivationLayer activationLayer = new ActivationLayer.Builder().activation(act).build(); + + checkSerialization(activationLayer); + + assertEquals(act, activationLayer.activationFn); + } + + private void checkSerialization(Layer layer) throws Exception { + NeuralNetConfiguration confExpected = new NeuralNetConfiguration.Builder().layer(layer).build(); + NeuralNetConfiguration confActual; + + // check Java serialization + byte[] data; + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutput out = new ObjectOutputStream(bos)) { + out.writeObject(confExpected); + data = bos.toByteArray(); + } + try (ByteArrayInputStream bis = new ByteArrayInputStream(data); ObjectInput in = new ObjectInputStream(bis)) { + confActual = (NeuralNetConfiguration) in.readObject(); + } + assertEquals(confExpected.getLayer(), confActual.getLayer(), "unequal Java serialization"); + + // check JSON + String json = confExpected.toJson(); + confActual = NeuralNetConfiguration.fromJson(json); + assertEquals(confExpected.getLayer(), confActual.getLayer(), "unequal JSON serialization"); + + // check YAML + String yaml = confExpected.toYaml(); + confActual = NeuralNetConfiguration.fromYaml(yaml); + assertEquals(confExpected.getLayer(), confActual.getLayer(), "unequal YAML serialization"); + + // check the layer's use of callSuper on equals method + confActual.getLayer().setIDropout(new Dropout(new java.util.Random().nextDouble())); + assertNotEquals( confExpected.getLayer(), confActual.getLayer(), "broken equals method (missing callSuper?)"); + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java new file mode 100644 index 000000000..635926f7c --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java @@ -0,0 +1,426 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.Distribution; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.distribution.UniformDistribution; +import org.deeplearning4j.nn.conf.dropout.Dropout; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInitDistribution; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.learning.config.AdaDelta; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.RmsProp; +import org.nd4j.linalg.schedule.MapSchedule; +import org.nd4j.linalg.schedule.ScheduleType; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class LayerConfigTest extends BaseDL4JTest { + + @Test + public void testLayerName() { + + String name1 = "genisys"; + String name2 = "bill"; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).name(name1).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).name(name2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(name1, conf.getConf(0).getLayer().getLayerName()); + assertEquals(name2, conf.getConf(1).getLayer().getLayerName()); + + } + + @Test + public void testActivationLayerwiseOverride() { + //Without layerwise override: + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.RELU).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals("relu", ((BaseLayer) conf.getConf(0).getLayer()).getActivationFn().toString()); + assertEquals("relu", ((BaseLayer) conf.getConf(1).getLayer()).getActivationFn().toString()); + + //With + conf = new NeuralNetConfiguration.Builder().activation(Activation.RELU).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).activation(Activation.TANH).build()).build(); + + net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals("relu", ((BaseLayer) conf.getConf(0).getLayer()).getActivationFn().toString()); + assertEquals("tanh", ((BaseLayer) conf.getConf(1).getLayer()).getActivationFn().toString()); + } + + + @Test + public void testWeightBiasInitLayerwiseOverride() { + //Without layerwise override: + final Distribution defaultDistribution = new NormalDistribution(0, 1.0); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dist(defaultDistribution).biasInit(1).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(new WeightInitDistribution(defaultDistribution), ((BaseLayer) conf.getConf(0).getLayer()).getWeightInitFn()); + assertEquals(new WeightInitDistribution(defaultDistribution), ((BaseLayer) conf.getConf(1).getLayer()).getWeightInitFn()); + + assertEquals(1, ((BaseLayer) conf.getConf(0).getLayer()).getBiasInit(), 0.0); + assertEquals(1, ((BaseLayer) conf.getConf(1).getLayer()).getBiasInit(), 0.0); + + //With: + final Distribution overriddenDistribution = new UniformDistribution(0, 1); + conf = new NeuralNetConfiguration.Builder() + .dist(defaultDistribution).biasInit(1).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, + new DenseLayer.Builder().nIn(2).nOut(2) + .dist(overriddenDistribution).biasInit(0).build()) + .build(); + + net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(new WeightInitDistribution(defaultDistribution), ((BaseLayer) conf.getConf(0).getLayer()).getWeightInitFn()); + assertEquals(new WeightInitDistribution(overriddenDistribution), ((BaseLayer) conf.getConf(1).getLayer()).getWeightInitFn()); + + assertEquals(1, ((BaseLayer) conf.getConf(0).getLayer()).getBiasInit(), 0.0); + assertEquals(0, ((BaseLayer) conf.getConf(1).getLayer()).getBiasInit(), 0.0); + } + + /* + @Test + public void testLrL1L2LayerwiseOverride() { + //Idea: Set some common values for all layers. Then selectively override + // the global config, and check they actually work. + + //Learning rate without layerwise override: + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(0.3).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(0.3, ((BaseLayer) conf.getConf(0).getLayer()).getLearningRate(), 0.0); + assertEquals(0.3, ((BaseLayer) conf.getConf(1).getLayer()).getLearningRate(), 0.0); + + //With: + conf = new NeuralNetConfiguration.Builder().learningRate(0.3).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).learningRate(0.2).build()).build(); + + net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(0.3, ((BaseLayer) conf.getConf(0).getLayer()).getLearningRate(), 0.0); + assertEquals(0.2, ((BaseLayer) conf.getConf(1).getLayer()).getLearningRate(), 0.0); + + //L1 and L2 without layerwise override: + conf = new NeuralNetConfiguration.Builder().l1(0.1).l2(0.2).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(0.1, ((BaseLayer) conf.getConf(0).getLayer()).getL1(), 0.0); + assertEquals(0.1, ((BaseLayer) conf.getConf(1).getLayer()).getL1(), 0.0); + assertEquals(0.2, ((BaseLayer) conf.getConf(0).getLayer()).getL2(), 0.0); + assertEquals(0.2, ((BaseLayer) conf.getConf(1).getLayer()).getL2(), 0.0); + + //L1 and L2 with layerwise override: + conf = new NeuralNetConfiguration.Builder().l1(0.1).l2(0.2).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l1(0.9).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.8).build()).build(); + net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(0.9, ((BaseLayer) conf.getConf(0).getLayer()).getL1(), 0.0); + assertEquals(0.1, ((BaseLayer) conf.getConf(1).getLayer()).getL1(), 0.0); + assertEquals(0.2, ((BaseLayer) conf.getConf(0).getLayer()).getL2(), 0.0); + assertEquals(0.8, ((BaseLayer) conf.getConf(1).getLayer()).getL2(), 0.0); + }*/ + + + + @Test + public void testDropoutLayerwiseOverride() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dropOut(1.0).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(new Dropout(1.0), conf.getConf(0).getLayer().getIDropout()); + assertEquals(new Dropout(1.0), conf.getConf(1).getLayer().getIDropout()); + + conf = new NeuralNetConfiguration.Builder().dropOut(1.0).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).dropOut(2.0).build()).build(); + + net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(new Dropout(1.0), conf.getConf(0).getLayer().getIDropout()); + assertEquals(new Dropout(2.0), conf.getConf(1).getLayer().getIDropout()); + } + + @Test + public void testMomentumLayerwiseOverride() { + Map testMomentumAfter = new HashMap<>(); + testMomentumAfter.put(0, 0.1); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .updater(new Nesterovs(1.0, new MapSchedule(ScheduleType.ITERATION, testMomentumAfter))) + .list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(0.1, ((Nesterovs)((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); + assertEquals(0.1, ((Nesterovs)((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); + + Map testMomentumAfter2 = new HashMap<>(); + testMomentumAfter2.put(0, 0.2); + + conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(1.0, new MapSchedule(ScheduleType.ITERATION, testMomentumAfter) )) + .list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder() + .nIn(2).nOut(2).updater(new Nesterovs(1.0, new MapSchedule(ScheduleType.ITERATION, testMomentumAfter2))).build()) + .build(); + + net = new MultiLayerNetwork(conf); + net.init(); + assertEquals(0.1, ((Nesterovs)((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); + assertEquals(0.2, ((Nesterovs)((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0,0), 0.0); + } + + @Test + public void testUpdaterRhoRmsDecayLayerwiseOverride() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new AdaDelta(0.5, 0.9)).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new AdaDelta(0.01,0.9)).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertTrue(((BaseLayer) conf.getConf(0).getLayer()).getIUpdater() instanceof AdaDelta); + assertTrue(((BaseLayer) conf.getConf(1).getLayer()).getIUpdater() instanceof AdaDelta); + assertEquals(0.5, ((AdaDelta)((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getRho(), 0.0); + assertEquals(0.01, ((AdaDelta)((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getRho(), 0.0); + + conf = new NeuralNetConfiguration.Builder().updater(new RmsProp(1.0, 2.0, RmsProp.DEFAULT_RMSPROP_EPSILON)).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).updater(new RmsProp(1.0, 1.0, RmsProp.DEFAULT_RMSPROP_EPSILON)).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new AdaDelta(0.5,AdaDelta.DEFAULT_ADADELTA_EPSILON)).build()) + .build(); + + net = new MultiLayerNetwork(conf); + net.init(); + + assertTrue(((BaseLayer) conf.getConf(0).getLayer()).getIUpdater() instanceof RmsProp); + assertTrue(((BaseLayer) conf.getConf(1).getLayer()).getIUpdater() instanceof AdaDelta); + assertEquals(1.0, ((RmsProp) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getRmsDecay(), 0.0); + assertEquals(0.5, ((AdaDelta) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getRho(), 0.0); + } + + + @Test + public void testUpdaterAdamParamsLayerwiseOverride() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .updater(new Adam(1.0, 0.5, 0.5, 1e-8)) + .list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new Adam(1.0, 0.6, 0.7, 1e-8)).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta1(), 0.0); + assertEquals(0.6, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta1(), 0.0); + assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta2(), 0.0); + assertEquals(0.7, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta2(), 0.0); + } + + @Test + public void testGradientNormalizationLayerwiseOverride() { + + //Learning rate without layerwise override: + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, + ((BaseLayer) conf.getConf(0).getLayer()).getGradientNormalization()); + assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, + ((BaseLayer) conf.getConf(1).getLayer()).getGradientNormalization()); + assertEquals(10, ((BaseLayer) conf.getConf(0).getLayer()).getGradientNormalizationThreshold(), 0.0); + assertEquals(10, ((BaseLayer) conf.getConf(1).getLayer()).getGradientNormalizationThreshold(), 0.0); + + //With: + conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2) + .gradientNormalization(GradientNormalization.None) + .gradientNormalizationThreshold(2.5).build()) + .build(); + + net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, + ((BaseLayer) conf.getConf(0).getLayer()).getGradientNormalization()); + assertEquals(GradientNormalization.None, ((BaseLayer) conf.getConf(1).getLayer()).getGradientNormalization()); + assertEquals(10, ((BaseLayer) conf.getConf(0).getLayer()).getGradientNormalizationThreshold(), 0.0); + assertEquals(2.5, ((BaseLayer) conf.getConf(1).getLayer()).getGradientNormalizationThreshold(), 0.0); + } + + + /* + @Test + public void testLearningRatePolicyExponential() { + double lr = 2; + double lrDecayRate = 5; + int iterations = 1; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr) + .updater(Updater.SGD) + .learningRateDecayPolicy(LearningRatePolicy.Exponential).lrPolicyDecayRate(lrDecayRate).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(LearningRatePolicy.Exponential, conf.getConf(0).getLearningRatePolicy()); + assertEquals(LearningRatePolicy.Exponential, conf.getConf(1).getLearningRatePolicy()); + assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); + assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); + } + + @Test + public void testLearningRatePolicyInverse() { + double lr = 2; + double lrDecayRate = 5; + double power = 3; + int iterations = 1; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) + .learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(lrDecayRate) + .lrPolicyPower(power).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(LearningRatePolicy.Inverse, conf.getConf(0).getLearningRatePolicy()); + assertEquals(LearningRatePolicy.Inverse, conf.getConf(1).getLearningRatePolicy()); + assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); + assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); + assertEquals(power, conf.getConf(0).getLrPolicyPower(), 0.0); + assertEquals(power, conf.getConf(1).getLrPolicyPower(), 0.0); + } + + + @Test + public void testLearningRatePolicySteps() { + double lr = 2; + double lrDecayRate = 5; + double steps = 4; + int iterations = 1; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) + .learningRateDecayPolicy(LearningRatePolicy.Step).lrPolicyDecayRate(lrDecayRate) + .lrPolicySteps(steps).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(LearningRatePolicy.Step, conf.getConf(0).getLearningRatePolicy()); + assertEquals(LearningRatePolicy.Step, conf.getConf(1).getLearningRatePolicy()); + assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); + assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); + assertEquals(steps, conf.getConf(0).getLrPolicySteps(), 0.0); + assertEquals(steps, conf.getConf(1).getLrPolicySteps(), 0.0); + } + + @Test + public void testLearningRatePolicyPoly() { + double lr = 2; + double lrDecayRate = 5; + double power = 3; + int iterations = 1; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) + .learningRateDecayPolicy(LearningRatePolicy.Poly).lrPolicyDecayRate(lrDecayRate) + .lrPolicyPower(power).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(LearningRatePolicy.Poly, conf.getConf(0).getLearningRatePolicy()); + assertEquals(LearningRatePolicy.Poly, conf.getConf(1).getLearningRatePolicy()); + assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); + assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); + assertEquals(power, conf.getConf(0).getLrPolicyPower(), 0.0); + assertEquals(power, conf.getConf(1).getLrPolicyPower(), 0.0); + } + + @Test + public void testLearningRatePolicySigmoid() { + double lr = 2; + double lrDecayRate = 5; + double steps = 4; + int iterations = 1; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) + .learningRateDecayPolicy(LearningRatePolicy.Sigmoid).lrPolicyDecayRate(lrDecayRate) + .lrPolicySteps(steps).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(LearningRatePolicy.Sigmoid, conf.getConf(0).getLearningRatePolicy()); + assertEquals(LearningRatePolicy.Sigmoid, conf.getConf(1).getLearningRatePolicy()); + assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); + assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); + assertEquals(steps, conf.getConf(0).getLrPolicySteps(), 0.0); + assertEquals(steps, conf.getConf(1).getLrPolicySteps(), 0.0); + } + +*/ +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java new file mode 100644 index 000000000..4b60f98c4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java @@ -0,0 +1,213 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.Updater; +import org.deeplearning4j.nn.conf.distribution.Distribution; +import org.deeplearning4j.nn.conf.distribution.GaussianDistribution; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.weightnoise.DropConnect; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.weights.WeightInitDistribution; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.RmsProp; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.schedule.MapSchedule; +import org.nd4j.linalg.schedule.ScheduleType; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class LayerConfigValidationTest extends BaseDL4JTest { + + + @Test + public void testDropConnect() { + // Warning thrown only since some layers may not have l1 or l2 + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).weightNoise(new DropConnect(0.5)) + .list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + } + + + @Test + public void testL1L2NotSet() { + // Warning thrown only since some layers may not have l1 or l2 + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)) + .list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + } + + @Test + //@Ignore //Old assumption: throw exception on l1 but no regularization. Current design: warn, not exception + public void testRegNotSetL1Global() { + assertThrows(IllegalStateException.class, () -> { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).l1(0.5).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + }); + } + + @Test + //@Ignore //Old assumption: throw exception on l1 but no regularization. Current design: warn, not exception + public void testRegNotSetL2Local() { + assertThrows(IllegalStateException.class, () -> { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.5).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + }); + } + + @Test + public void testWeightInitDistNotSet() { + // Warning thrown only since global dist can be set with a different weight init locally + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).dist(new GaussianDistribution(1e-3, 2)) + .list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + } + + @Test + public void testNesterovsNotSetGlobal() { + // Warnings only thrown + Map testMomentumAfter = new HashMap<>(); + testMomentumAfter.put(0, 0.1); + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().updater(new Nesterovs(1.0, new MapSchedule(ScheduleType.ITERATION, testMomentumAfter))).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + } + + @Test + public void testCompGraphNullLayer() { + ComputationGraphConfiguration.GraphBuilder gb = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.01)) + .seed(42).miniBatch(false).l1(0.2).l2(0.2) + /* Graph Builder */ + .updater(Updater.RMSPROP).graphBuilder().addInputs("in") + .addLayer("L" + 1, + new GravesLSTM.Builder().nIn(20).updater(Updater.RMSPROP).nOut(10) + .weightInit(WeightInit.XAVIER) + .dropOut(0.4).l1(0.3).activation(Activation.SIGMOID).build(), + "in") + .addLayer("output", + new RnnOutputLayer.Builder().nIn(20).nOut(10).activation(Activation.SOFTMAX) + .weightInit(WeightInit.RELU_UNIFORM).build(), + "L" + 1) + .setOutputs("output"); + ComputationGraphConfiguration conf = gb.build(); + ComputationGraph cg = new ComputationGraph(conf); + cg.init(); + } + + + @Test + public void testPredefinedConfigValues() { + double expectedMomentum = 0.9; + double expectedAdamMeanDecay = 0.9; + double expectedAdamVarDecay = 0.999; + double expectedRmsDecay = 0.95; + Distribution expectedDist = new NormalDistribution(0, 1); + double expectedL1 = 0.0; + double expectedL2 = 0.0; + + // Nesterovs Updater + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(0.9)) + .list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.5).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new Nesterovs(0.3, 0.4)).build()).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + BaseLayer layerConf = (BaseLayer) net.getLayer(0).conf().getLayer(); + assertEquals(expectedMomentum, ((Nesterovs) layerConf.getIUpdater()).getMomentum(), 1e-3); + assertNull(TestUtils.getL1Reg(layerConf.getRegularization())); + assertEquals(0.5, TestUtils.getL2(layerConf), 1e-3); + + BaseLayer layerConf1 = (BaseLayer) net.getLayer(1).conf().getLayer(); + assertEquals(0.4, ((Nesterovs) layerConf1.getIUpdater()).getMomentum(), 1e-3); + + // Adam Updater + conf = new NeuralNetConfiguration.Builder().updater(new Adam(0.3)) + .weightInit(new WeightInitDistribution(expectedDist)).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.5).l1(0.3).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); + net = new MultiLayerNetwork(conf); + net.init(); + + layerConf = (BaseLayer) net.getLayer(0).conf().getLayer(); + assertEquals(0.3, TestUtils.getL1(layerConf), 1e-3); + assertEquals(0.5, TestUtils.getL2(layerConf), 1e-3); + + layerConf1 = (BaseLayer) net.getLayer(1).conf().getLayer(); + assertEquals(expectedAdamMeanDecay, ((Adam) layerConf1.getIUpdater()).getBeta1(), 1e-3); + assertEquals(expectedAdamVarDecay, ((Adam) layerConf1.getIUpdater()).getBeta2(), 1e-3); + assertEquals(new WeightInitDistribution(expectedDist), layerConf1.getWeightInitFn()); + assertNull(TestUtils.getL1Reg(layerConf1.getRegularization())); + assertNull(TestUtils.getL2Reg(layerConf1.getRegularization())); + + //RMSProp Updater + conf = new NeuralNetConfiguration.Builder().updater(new RmsProp(0.3)).list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) + .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new RmsProp(0.3, 0.4, RmsProp.DEFAULT_RMSPROP_EPSILON)).build()).build(); + net = new MultiLayerNetwork(conf); + net.init(); + + layerConf = (BaseLayer) net.getLayer(0).conf().getLayer(); + assertEquals(expectedRmsDecay, ((RmsProp) layerConf.getIUpdater()).getRmsDecay(), 1e-3); + assertNull(TestUtils.getL1Reg(layerConf.getRegularization())); + assertNull(TestUtils.getL2Reg(layerConf.getRegularization())); + + layerConf1 = (BaseLayer) net.getLayer(1).conf().getLayer(); + assertEquals(0.4, ((RmsProp) layerConf1.getIUpdater()).getRmsDecay(), 1e-3); + + + } + +} + + diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/misc/TestGraphVertex.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/misc/TestGraphVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/misc/TestGraphVertex.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/misc/TestGraphVertex.java diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CNNProcessorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CNNProcessorTest.java new file mode 100644 index 000000000..79878cd4c --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CNNProcessorTest.java @@ -0,0 +1,309 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.preprocessor; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.*; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + +import static org.junit.jupiter.api.Assertions.*; + +/** + **/ + +public class CNNProcessorTest extends BaseDL4JTest { + private static int rows = 28; + private static int cols = 28; + private static INDArray in2D = Nd4j.create(DataType.FLOAT, 1, 784); + private static INDArray in3D = Nd4j.create(DataType.FLOAT, 20, 784, 7); + private static INDArray in4D = Nd4j.create(DataType.FLOAT, 20, 1, 28, 28); + + + @Test + public void testFeedForwardToCnnPreProcessor() { + FeedForwardToCnnPreProcessor convProcessor = new FeedForwardToCnnPreProcessor(rows, cols, 1); + + INDArray check2to4 = convProcessor.preProcess(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); + int val2to4 = check2to4.shape().length; + assertTrue(val2to4 == 4); + assertEquals(Nd4j.create(DataType.FLOAT, 1, 1, 28, 28), check2to4); + + INDArray check4to4 = convProcessor.preProcess(in4D, -1, LayerWorkspaceMgr.noWorkspaces()); + int val4to4 = check4to4.shape().length; + assertTrue(val4to4 == 4); + assertEquals(Nd4j.create(DataType.FLOAT, 20, 1, 28, 28), check4to4); + } + + @Test + public void testFeedForwardToCnnPreProcessor2() { + int[] nRows = {1, 5, 20}; + int[] nCols = {1, 5, 20}; + int[] nDepth = {1, 3}; + int[] nMiniBatchSize = {1, 5}; + for (int rows : nRows) { + for (int cols : nCols) { + for (int d : nDepth) { + FeedForwardToCnnPreProcessor convProcessor = new FeedForwardToCnnPreProcessor(rows, cols, d); + + for (int miniBatch : nMiniBatchSize) { + long[] ffShape = new long[] {miniBatch, rows * cols * d}; + INDArray rand = Nd4j.rand(ffShape); + INDArray ffInput_c = Nd4j.create(DataType.FLOAT, ffShape, 'c'); + INDArray ffInput_f = Nd4j.create(DataType.FLOAT, ffShape, 'f'); + ffInput_c.assign(rand); + ffInput_f.assign(rand); + assertEquals(ffInput_c, ffInput_f); + + //Test forward pass: + INDArray convAct_c = convProcessor.preProcess(ffInput_c, -1, LayerWorkspaceMgr.noWorkspaces()); + INDArray convAct_f = convProcessor.preProcess(ffInput_f, -1, LayerWorkspaceMgr.noWorkspaces()); + long[] convShape = {miniBatch, d, rows, cols}; + assertArrayEquals(convShape, convAct_c.shape()); + assertArrayEquals(convShape, convAct_f.shape()); + assertEquals(convAct_c, convAct_f); + + //Check values: + //CNN reshaping (for each example) takes a 1d vector and converts it to 3d + // (4d total, for minibatch data) + //1d vector is assumed to be rows from channels 0 concatenated, followed by channels 1, etc + for (int ex = 0; ex < miniBatch; ex++) { + for (int r = 0; r < rows; r++) { + for (int c = 0; c < cols; c++) { + for (int depth = 0; depth < d; depth++) { + int origPosition = depth * (rows * cols) + r * cols + c; //pos in vector + double vecValue = ffInput_c.getDouble(ex, origPosition); + double convValue = convAct_c.getDouble(ex, depth, r, c); + assertEquals(vecValue, convValue, 0.0); + } + } + } + } + + //Test backward pass: + //Idea is that backward pass should do opposite to forward pass + INDArray epsilon4_c = Nd4j.create(DataType.FLOAT, convShape, 'c'); + INDArray epsilon4_f = Nd4j.create(DataType.FLOAT, convShape, 'f'); + epsilon4_c.assign(convAct_c); + epsilon4_f.assign(convAct_f); + INDArray epsilon2_c = convProcessor.backprop(epsilon4_c, -1, LayerWorkspaceMgr.noWorkspaces()); + INDArray epsilon2_f = convProcessor.backprop(epsilon4_f, -1, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(ffInput_c, epsilon2_c); + assertEquals(ffInput_c, epsilon2_f); + } + } + } + } + } + + + @Test + public void testFeedForwardToCnnPreProcessorBackprop() { + FeedForwardToCnnPreProcessor convProcessor = new FeedForwardToCnnPreProcessor(rows, cols, 1); + convProcessor.preProcess(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); + + INDArray check2to2 = convProcessor.backprop(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); + int val2to2 = check2to2.shape().length; + assertTrue(val2to2 == 2); + assertEquals(Nd4j.create(DataType.FLOAT, 1, 784), check2to2); + } + + @Test + public void testCnnToFeedForwardProcessor() { + CnnToFeedForwardPreProcessor convProcessor = new CnnToFeedForwardPreProcessor(rows, cols, 1); + + INDArray check2to4 = convProcessor.backprop(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); + int val2to4 = check2to4.shape().length; + assertTrue(val2to4 == 4); + assertEquals(Nd4j.create(DataType.FLOAT, 1, 1, 28, 28), check2to4); + + INDArray check4to4 = convProcessor.backprop(in4D, -1, LayerWorkspaceMgr.noWorkspaces()); + int val4to4 = check4to4.shape().length; + assertTrue(val4to4 == 4); + assertEquals(Nd4j.create(DataType.FLOAT, 20, 1, 28, 28), check4to4); + } + + @Test + public void testCnnToFeedForwardPreProcessorBackprop() { + CnnToFeedForwardPreProcessor convProcessor = new CnnToFeedForwardPreProcessor(rows, cols, 1); + convProcessor.preProcess(in4D, -1, LayerWorkspaceMgr.noWorkspaces()); + + INDArray check2to2 = convProcessor.preProcess(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); + int val2to2 = check2to2.shape().length; + assertTrue(val2to2 == 2); + assertEquals(Nd4j.create(DataType.FLOAT, 1, 784), check2to2); + + INDArray check4to2 = convProcessor.preProcess(in4D, -1, LayerWorkspaceMgr.noWorkspaces()); + int val4to2 = check4to2.shape().length; + assertTrue(val4to2 == 2); + assertEquals(Nd4j.create(DataType.FLOAT, 20, 784), check4to2); + } + + @Test + public void testCnnToFeedForwardPreProcessor2() { + int[] nRows = {1, 5, 20}; + int[] nCols = {1, 5, 20}; + int[] nDepth = {1, 3}; + int[] nMiniBatchSize = {1, 5}; + for (int rows : nRows) { + for (int cols : nCols) { + for (int d : nDepth) { + CnnToFeedForwardPreProcessor convProcessor = new CnnToFeedForwardPreProcessor(rows, cols, d); + + for (int miniBatch : nMiniBatchSize) { + long[] convActShape = new long[] {miniBatch, d, rows, cols}; + INDArray rand = Nd4j.rand(convActShape); + INDArray convInput_c = Nd4j.create(DataType.FLOAT, convActShape, 'c'); + INDArray convInput_f = Nd4j.create(DataType.FLOAT, convActShape, 'f'); + convInput_c.assign(rand); + convInput_f.assign(rand); + assertEquals(convInput_c, convInput_f); + + //Test forward pass: + INDArray ffAct_c = convProcessor.preProcess(convInput_c, -1, LayerWorkspaceMgr.noWorkspaces()); + INDArray ffAct_f = convProcessor.preProcess(convInput_f, -1, LayerWorkspaceMgr.noWorkspaces()); + long[] ffActShape = {miniBatch, d * rows * cols}; + assertArrayEquals(ffActShape, ffAct_c.shape()); + assertArrayEquals(ffActShape, ffAct_f.shape()); + assertEquals(ffAct_c, ffAct_f); + + //Check values: + //CNN reshaping (for each example) takes a 1d vector and converts it to 3d + // (4d total, for minibatch data) + //1d vector is assumed to be rows from channels 0 concatenated, followed by channels 1, etc + for (int ex = 0; ex < miniBatch; ex++) { + for (int r = 0; r < rows; r++) { + for (int c = 0; c < cols; c++) { + for (int depth = 0; depth < d; depth++) { + int vectorPosition = depth * (rows * cols) + r * cols + c; //pos in vector after reshape + double vecValue = ffAct_c.getDouble(ex, vectorPosition); + double convValue = convInput_c.getDouble(ex, depth, r, c); + assertEquals(convValue, vecValue, 0.0); + } + } + } + } + + //Test backward pass: + //Idea is that backward pass should do opposite to forward pass + INDArray epsilon2_c = Nd4j.create(DataType.FLOAT, ffActShape, 'c'); + INDArray epsilon2_f = Nd4j.create(DataType.FLOAT, ffActShape, 'f'); + epsilon2_c.assign(ffAct_c); + epsilon2_f.assign(ffAct_c); + INDArray epsilon4_c = convProcessor.backprop(epsilon2_c, -1, LayerWorkspaceMgr.noWorkspaces()); + INDArray epsilon4_f = convProcessor.backprop(epsilon2_f, -1, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(convInput_c, epsilon4_c); + assertEquals(convInput_c, epsilon4_f); + } + } + } + } + } + + @Test + public void testInvalidInputShape(){ + + NeuralNetConfiguration.Builder builder = new NeuralNetConfiguration.Builder() + .seed(123) + .miniBatch(true) + .cacheMode(CacheMode.DEVICE) + .updater(new Nesterovs(0.9)) + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT); + + int[] kernelArray = new int[]{3,3}; + int[] strideArray = new int[]{1,1}; + int[] zeroPaddingArray = new int[]{0,0}; + int processWidth = 4; + + NeuralNetConfiguration.ListBuilder listBuilder = builder.list(); // Building the DL4J network + + listBuilder = listBuilder.layer(0, new ConvolutionLayer.Builder(kernelArray, strideArray, zeroPaddingArray) + .name("cnn1") + .convolutionMode(ConvolutionMode.Strict) + .nIn(2) // 2 input channels + .nOut(processWidth) + .weightInit(WeightInit.XAVIER_UNIFORM) + .activation(Activation.RELU) + .biasInit(1e-2).build()); + + listBuilder = listBuilder.layer(1, new ConvolutionLayer.Builder(kernelArray, strideArray, zeroPaddingArray) + .name("cnn2") + .convolutionMode(ConvolutionMode.Strict) + .nOut(processWidth) + .weightInit(WeightInit.XAVIER_UNIFORM) + .activation(Activation.RELU) + .biasInit(1e-2) + .build()); + + listBuilder = listBuilder.layer(2, new ConvolutionLayer.Builder(kernelArray, strideArray, zeroPaddingArray) + .name("cnn3") + .convolutionMode(ConvolutionMode.Strict) + .nOut(processWidth) + .weightInit(WeightInit.XAVIER_UNIFORM) + .activation(Activation.RELU).build()); + + listBuilder = listBuilder.layer(3, new ConvolutionLayer.Builder(kernelArray, strideArray, zeroPaddingArray) + .name("cnn4") + .convolutionMode(ConvolutionMode.Strict) + .nOut(processWidth) + .weightInit(WeightInit.XAVIER_UNIFORM) + .activation(Activation.RELU).build()); + + listBuilder = listBuilder + .layer(4, new OutputLayer.Builder(LossFunctions.LossFunction.MSE) + .name("output") + .nOut(1) + .activation(Activation.TANH) + .build()); + + MultiLayerConfiguration conf = listBuilder + + + .setInputType(InputType.convolutional(20, 10, 2)) + .build(); + + // For some reason, this model works + MultiLayerNetwork niceModel = new MultiLayerNetwork(conf); + niceModel.init(); + + niceModel.output(Nd4j.create(DataType.FLOAT, 1, 2, 20, 10)); //Valid + + try { + niceModel.output(Nd4j.create(DataType.FLOAT, 1, 2, 10, 20)); + fail("Expected exception"); + } catch (IllegalStateException e){ + //OK + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CustomPreprocessorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CustomPreprocessorTest.java new file mode 100644 index 000000000..36bfbc95f --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CustomPreprocessorTest.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.preprocessor; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.preprocessor.custom.MyCustomPreprocessor; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.introspect.AnnotatedClass; +import com.fasterxml.jackson.databind.jsontype.NamedType; + +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CustomPreprocessorTest extends BaseDL4JTest { + + @Test + public void testCustomPreprocessor() { + //Second: let's create a MultiLayerCofiguration with one, and check JSON and YAML config actually works... + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(10) + .activation(Activation.SOFTMAX).nOut(10).build()) + .inputPreProcessor(0, new MyCustomPreprocessor()) + .build(); + + String json = conf.toJson(); + String yaml = conf.toYaml(); + +// System.out.println(json); + + MultiLayerConfiguration confFromJson = MultiLayerConfiguration.fromJson(json); + assertEquals(conf, confFromJson); + + MultiLayerConfiguration confFromYaml = MultiLayerConfiguration.fromYaml(yaml); + assertEquals(conf, confFromYaml); + + assertTrue(confFromJson.getInputPreProcess(0) instanceof MyCustomPreprocessor); + + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/TestPreProcessors.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/TestPreProcessors.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/TestPreProcessors.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/TestPreProcessors.java index 99f0417e3..3f6741b89 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/TestPreProcessors.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/TestPreProcessors.java @@ -34,10 +34,7 @@ import org.deeplearning4j.nn.layers.convolution.ConvolutionLayer; import org.deeplearning4j.nn.layers.feedforward.dense.DenseLayer; import org.deeplearning4j.nn.modelimport.keras.preprocessors.ReshapePreprocessor; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -48,8 +45,6 @@ import java.util.Arrays; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class TestPreProcessors extends BaseDL4JTest { @Test @@ -200,8 +195,8 @@ public class TestPreProcessors extends BaseDL4JTest { INDArray act3d_f = Nd4j.create(activations3dc.shape(), 'f'); act3d_f.assign(activations3dc); - assertEquals(activations2dc, proc.backprop(act3d_c, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()), msg); - assertEquals(activations2dc, proc.backprop(act3d_f, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()), msg); + assertEquals( activations2dc, proc.backprop(act3d_c, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()), msg); + assertEquals( activations2dc, proc.backprop(act3d_f, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()), msg); } } @@ -250,13 +245,13 @@ public class TestPreProcessors extends BaseDL4JTest { //Check shape of outputs: val prod = nChannels * inputHeight * inputWidth; INDArray activationsRnn = proc.preProcess(activationsCnn, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(new long[] {miniBatchSize, prod, timeSeriesLength}, - activationsRnn.shape(),msg); + assertArrayEquals( new long[] {miniBatchSize, prod, timeSeriesLength}, + activationsRnn.shape(), msg); //Check backward pass. Given that activations and epsilons have same shape, they should //be opposite operations - i.e., get the same thing back out INDArray twiceProcessed = proc.backprop(activationsRnn, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activationsCnn.shape(), twiceProcessed.shape(),msg); + assertArrayEquals(activationsCnn.shape(), twiceProcessed.shape(), msg); assertEquals(activationsCnn, twiceProcessed, msg); //Second way to check: compare to ComposableInputPreProcessor(CNNtoFF, FFtoRNN) diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/custom/MyCustomPreprocessor.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/custom/MyCustomPreprocessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/custom/MyCustomPreprocessor.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/custom/MyCustomPreprocessor.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/weightnoise/TestWeightNoise.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/weightnoise/TestWeightNoise.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/weightnoise/TestWeightNoise.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/weightnoise/TestWeightNoise.java index 1449c8d04..f495e8fb0 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/weightnoise/TestWeightNoise.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/conf/weightnoise/TestWeightNoise.java @@ -109,7 +109,6 @@ public class TestWeightNoise extends BaseDL4JTest { @Test public void testCalls() { - List trainData = new ArrayList<>(); trainData.add(new DataSet(Nd4j.rand(5, 10), Nd4j.rand(5, 10))); trainData.add(new DataSet(Nd4j.rand(5, 10), Nd4j.rand(5, 10))); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java index 0752670b9..039c3e4e6 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/dtypes/DTypeTests.java @@ -142,12 +142,10 @@ import org.deeplearning4j.nn.modelimport.keras.preprocessors.TensorFlowCnnToFeed import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.nn.weights.WeightInitDistribution; -import org.junit.AfterClass; + +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.activations.impl.ActivationSoftmax; import org.nd4j.linalg.api.buffer.DataType; @@ -161,8 +159,8 @@ import org.nd4j.linalg.learning.config.NoOp; import org.nd4j.linalg.lossfunctions.LossFunctions; import org.nd4j.linalg.lossfunctions.impl.LossNegativeLogLikelihood; import org.nd4j.linalg.profiler.ProfilerConfig; -import org.nd4j.shade.guava.collect.ImmutableSet; -import org.nd4j.shade.guava.reflect.ClassPath; +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.ClassPath; import java.io.IOException; import java.lang.reflect.Modifier; @@ -174,8 +172,6 @@ import java.util.Set; @Slf4j @Disabled -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class DTypeTests extends BaseDL4JTest { protected static Set> seenLayers = new HashSet<>(); @@ -195,12 +191,12 @@ public class DTypeTests extends BaseDL4JTest { return 9999999L; } - @AfterClass + @AfterAll public static void after() { ImmutableSet info; try { //Dependency note: this ClassPath class was added in Guava 14 - info = org.nd4j.shade.guava.reflect.ClassPath.from(DTypeTests.class.getClassLoader()) + info = com.google.common.reflect.ClassPath.from(DTypeTests.class.getClassLoader()) .getTopLevelClassesRecursive("org.deeplearning4j"); } catch (IOException e) { //Should never happen @@ -570,7 +566,7 @@ public class DTypeTests extends BaseDL4JTest { List ff = net.feedForward(in); for (int i = 0; i < ff.size(); i++) { String s = msg + " - layer " + (i - 1) + " - " + (i == 0 ? "input" : net.getLayer(i - 1).conf().getLayer().getClass().getSimpleName()); - assertEquals(networkDtype, ff.get(i).dataType(), s); + assertEquals(networkDtype, ff.get(i).dataType(), msg); } net.setInput(in); @@ -702,7 +698,7 @@ public class DTypeTests extends BaseDL4JTest { } @Test - @Disabled + ////@Ignore public void testDtypesModelVsGlobalDtypeCnn1d() { //Nd4jCpu.Environment.getInstance().setUseMKLDNN(false); Nd4j.getEnvironment().setDebug(true); @@ -1424,7 +1420,7 @@ public class DTypeTests extends BaseDL4JTest { net.init(); INDArray out = net.output(in); - assertEquals(networkDtype, out.dataType(), msg); + assertEquals( networkDtype, out.dataType(), msg); List ff = net.feedForward(in); for (int i = 0; i < ff.size(); i++) { String s = msg + " - layer " + (i - 1) + " - " + (i == 0 ? "input" : net.getLayer(i - 1).conf().getLayer().getClass().getSimpleName()); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/ComputationGraphTestRNN.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/ComputationGraphTestRNN.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/ComputationGraphTestRNN.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/ComputationGraphTestRNN.java index 899a18993..2bddca70a 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/ComputationGraphTestRNN.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/ComputationGraphTestRNN.java @@ -39,10 +39,7 @@ import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; import org.deeplearning4j.nn.gradient.Gradient; import org.deeplearning4j.nn.layers.recurrent.BaseRecurrentLayer; import org.deeplearning4j.nn.layers.recurrent.GravesLSTM; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -61,8 +58,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class ComputationGraphTestRNN extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphCNN.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphCNN.java new file mode 100644 index 000000000..96532aa69 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphCNN.java @@ -0,0 +1,190 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.graph; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator; +import org.deeplearning4j.exception.DL4JInvalidConfigException; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.CNN2DFormat; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + + +////@Ignore +public class TestCompGraphCNN extends BaseDL4JTest { + + protected ComputationGraphConfiguration conf; + protected ComputationGraph graph; + protected DataSetIterator dataSetIterator; + protected DataSet ds; + + protected static ComputationGraphConfiguration getMultiInputGraphConfig() { + ComputationGraphConfiguration conf = + new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .graphBuilder().addInputs("input") + .setInputTypes(InputType.convolutional(32, 32, 3)) + .addLayer("cnn1", + new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(3).nOut(3) + .build(), + "input") + .addLayer("cnn2", + new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(3).nOut(3) + .build(), + "input") + .addLayer("max1", + new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) + .stride(1, 1).kernelSize(2, 2).build(), + "cnn1", "cnn2") + .addLayer("dnn1", new DenseLayer.Builder().nOut(7).build(), "max1") + .addLayer("output", new OutputLayer.Builder().nIn(7).nOut(10).activation(Activation.SOFTMAX).build(), "dnn1") + .setOutputs("output").build(); + + return conf; + } + + protected static DataSetIterator getDS() { + + List list = new ArrayList<>(5); + for (int i = 0; i < 5; i++) { + INDArray f = Nd4j.create(1, 32 * 32 * 3); + INDArray l = Nd4j.create(1, 10); + l.putScalar(i, 1.0); + list.add(new DataSet(f, l)); + } + return new ListDataSetIterator(list, 5); + } + + protected static int getNumParams() { + return 2 * (3 * 1 * 4 * 4 * 3 + 3) + (7 * 14 * 14 * 6 + 7) + (7 * 10 + 10); + } + + @BeforeEach + ////@Ignore + public void beforeDo() { + conf = getMultiInputGraphConfig(); + graph = new ComputationGraph(conf); + graph.init(); + + dataSetIterator = getDS(); + ds = dataSetIterator.next(); + + } + + @Test + public void testConfigBasic() { + //Check the order. there are 2 possible valid orders here + int[] order = graph.topologicalSortOrder(); + int[] expOrder1 = new int[] {0, 1, 2, 4, 3, 5, 6}; //First of 2 possible valid orders + int[] expOrder2 = new int[] {0, 2, 1, 4, 3, 5, 6}; //Second of 2 possible valid orders + boolean orderOK = Arrays.equals(expOrder1, order) || Arrays.equals(expOrder2, order); + assertTrue(orderOK); + + INDArray params = graph.params(); + assertNotNull(params); + + // confirm param shape is what is expected + int nParams = getNumParams(); + assertEquals(nParams, params.length()); + + INDArray arr = Nd4j.linspace(0, nParams, nParams, DataType.FLOAT).reshape(1, nParams); + assertEquals(nParams, arr.length()); + + // params are set + graph.setParams(arr); + params = graph.params(); + assertEquals(arr, params); + + //Number of inputs and outputs: + assertEquals(1, graph.getNumInputArrays()); + assertEquals(1, graph.getNumOutputArrays()); + + } + + @Test + public void testCNNComputationGraphKernelTooLarge() { + Assertions.assertThrows(DL4JInvalidConfigException.class, () -> { + int imageWidth = 23; + int imageHeight = 19; + int nChannels = 1; + int classes = 2; + int numSamples = 200; + + int kernelHeight = 3; + int kernelWidth = imageWidth; + + + DataSet trainInput; + + ComputationGraphConfiguration conf = + new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .seed(123).graphBuilder().addInputs("input") + .setInputTypes(InputType.convolutional(nChannels, imageWidth, + imageHeight)) + .addLayer("conv1", new ConvolutionLayer.Builder() + .kernelSize(kernelHeight, kernelWidth).stride(1, 1) + .dataFormat(CNN2DFormat.NCHW) + .nIn(nChannels).nOut(2).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build(), "input") + .addLayer("pool1", + new SubsamplingLayer.Builder() + .dataFormat(CNN2DFormat.NCHW) + .poolingType(SubsamplingLayer.PoolingType.MAX) + .kernelSize(imageHeight - kernelHeight + 1, 1) + .stride(1, 1).build(), + "conv1") + .addLayer("output", new OutputLayer.Builder().nOut(classes).activation(Activation.SOFTMAX).build(), "pool1") + .setOutputs("output").build(); + + + ComputationGraph model = new ComputationGraph(conf); + model.init(); + + + INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); + INDArray emptyLables = Nd4j.zeros(numSamples, classes); + + trainInput = new DataSet(emptyFeatures, emptyLables); + + model.fit(trainInput); + }); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphUnsupervised.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphUnsupervised.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphUnsupervised.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphUnsupervised.java index b947c0d86..a17979bf2 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphUnsupervised.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphUnsupervised.java @@ -32,10 +32,7 @@ import org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistr import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -52,10 +49,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) public class TestCompGraphUnsupervised extends BaseDL4JTest { @Override diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestComputationGraphNetwork.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestComputationGraphNetwork.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestComputationGraphNetwork.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestComputationGraphNetwork.java index 7d452db56..6b1191a51 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestComputationGraphNetwork.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestComputationGraphNetwork.java @@ -63,10 +63,6 @@ import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.deeplearning4j.util.ModelSerializer; import org.junit.jupiter.api.*; import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.activations.impl.ActivationIdentity; import org.nd4j.linalg.api.buffer.DataType; @@ -90,17 +86,17 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.*; import static org.deeplearning4j.nn.conf.layers.recurrent.Bidirectional.Mode.CONCAT; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class TestComputationGraphNetwork extends BaseDL4JTest { + @TempDir + public File testDir; + private static ComputationGraphConfiguration getIrisGraphConfiguration() { return new NeuralNetConfiguration.Builder().seed(12345) .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder() @@ -124,7 +120,8 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { private static OpExecutioner.ProfilingMode origMode; - @BeforeAll public static void beforeClass(){ + @BeforeAll + public static void beforeClass(){ origMode = Nd4j.getExecutioner().getProfilingMode(); } @@ -292,8 +289,6 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { } @Test - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.LONG_TEST) public void testIrisFit() { ComputationGraphConfiguration configuration = getIrisGraphConfiguration(); @@ -327,10 +322,8 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { assertEquals(paramsMLN, paramsGraph); } - @Test() - @Timeout(300000) - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.LONG_TEST) + @Test + @Timeout(300) public void testIrisFitMultiDataSetIterator() throws Exception { RecordReader rr = new CSVRecordReader(0, ','); @@ -599,8 +592,6 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { } @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) public void testPreTraining() { ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() @@ -861,10 +852,10 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { DataSetIterator iter = new IrisDataSetIterator(1, 1); Gradient expectedGradient = new DefaultGradient(); - expectedGradient.setGradientFor("first_W", Nd4j.ones(4, 5).castTo(Nd4j.defaultFloatingPointType())); - expectedGradient.setGradientFor("first_b", Nd4j.ones(1, 5).castTo(Nd4j.defaultFloatingPointType())); - expectedGradient.setGradientFor("output_W", Nd4j.ones(5, 3).castTo(Nd4j.defaultFloatingPointType())); - expectedGradient.setGradientFor("output_b", Nd4j.ones(1, 3).castTo(Nd4j.defaultFloatingPointType())); + expectedGradient.setGradientFor("first_W", Nd4j.ones(4, 5)); + expectedGradient.setGradientFor("first_b", Nd4j.ones(1, 5)); + expectedGradient.setGradientFor("output_W", Nd4j.ones(5, 3)); + expectedGradient.setGradientFor("output_b", Nd4j.ones(1, 3)); ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder() @@ -883,11 +874,11 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { assertEquals(expectedGradient.getGradientFor("first_W"), actualGradient.getGradientFor("first_W")); // Update params with set - net.setParam("first_W", Nd4j.ones(4, 5).castTo(Nd4j.defaultFloatingPointType())); - net.setParam("first_b", Nd4j.ones(1, 5).castTo(Nd4j.defaultFloatingPointType())); - net.setParam("output_W", Nd4j.ones(5, 3).castTo(Nd4j.defaultFloatingPointType())); - net.setParam("output_b", Nd4j.ones(1, 3).castTo(Nd4j.defaultFloatingPointType())); - INDArray actualParams = net.params().castTo(Nd4j.defaultFloatingPointType()); + net.setParam("first_W", Nd4j.ones(4, 5)); + net.setParam("first_b", Nd4j.ones(1, 5)); + net.setParam("output_W", Nd4j.ones(5, 3)); + net.setParam("output_b", Nd4j.ones(1, 3)); + INDArray actualParams = net.params(); // Confirm params assertEquals(Nd4j.ones(1, 43), actualParams); @@ -1184,26 +1175,24 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { g.calcRegularizationScore(false); } - @Test() + @Test public void testErrorNoOutputLayer() { - assertThrows(DL4JException.class,() -> { - ComputationGraphConfiguration c = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in") - .addLayer("dense", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in").setOutputs("dense") - .build(); - ComputationGraph cg = new ComputationGraph(c); - cg.init(); + ComputationGraphConfiguration c = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in") + .addLayer("dense", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in").setOutputs("dense") + .build(); - INDArray f = Nd4j.create(1, 10); - INDArray l = Nd4j.create(1, 10); + ComputationGraph cg = new ComputationGraph(c); + cg.init(); - cg.setInputs(f); - cg.setLabels(l); + INDArray f = Nd4j.create(1, 10); + INDArray l = Nd4j.create(1, 10); + cg.setInputs(f); + cg.setLabels(l); + Assertions.assertThrows( DL4JException.class, () -> { cg.computeGradientAndScore(); }); - - } @@ -1271,8 +1260,6 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { } @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) public void testEpochCounter() throws Exception { ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() @@ -1310,8 +1297,6 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { } @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) public void testSummary() { int V_WIDTH = 130; int V_HEIGHT = 130; @@ -1531,7 +1516,7 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { //Hack output layer to be identity mapping graph.getOutputLayer(0).setParam("W", Nd4j.eye(input.length())); graph.getOutputLayer(0).setParam("b", Nd4j.zeros(input.length())); - assertEquals(Nd4j.create(expected).reshape(1,expected.length), graph.outputSingle(input),"Incorrect output"); + assertEquals(Nd4j.create(expected).reshape(1,expected.length), graph.outputSingle(input), "Incorrect output"); } private static INDArray getInputArray4d(float[] inputArr) { @@ -1729,8 +1714,8 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { MultiLayerTest.CheckModelsListener listener = new MultiLayerTest.CheckModelsListener(); net.setListeners(listener); - INDArray f = Nd4j.create(DataType.DOUBLE,1,10); - INDArray l = Nd4j.create(DataType.DOUBLE,1,10); + INDArray f = Nd4j.create(1,10); + INDArray l = Nd4j.create(1,10); DataSet ds = new DataSet(f,l); MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(f,l); @@ -1788,14 +1773,14 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { for(String s : exp.keySet()){ boolean allowed = ((org.deeplearning4j.nn.layers.feedforward.dense.DenseLayer)cg.getLayer(s)).isInputModificationAllowed(); // System.out.println(s + "\t" + allowed); - assertEquals( exp.get(s), allowed,s); + assertEquals(exp.get(s), allowed, s); } } @Test public void testCompGraphDropoutOutputLayers(){ - //https://github.com/eclipse/deeplearning4j/issues/6326 + //https://github.com/deeplearning4j/deeplearning4j/issues/6326 ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() .dropOut(0.8) .graphBuilder() @@ -1833,7 +1818,7 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { @Test public void testCompGraphDropoutOutputLayers2() { - //https://github.com/eclipse/deeplearning4j/issues/6326 + //https://github.com/deeplearning4j/deeplearning4j/issues/6326 ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() .dropOut(0.8) .graphBuilder() @@ -1992,7 +1977,7 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { @Test public void testVerticesAndMasking7027(){ - //https://github.com/eclipse/deeplearning4j/issues/7027 + //https://github.com/deeplearning4j/deeplearning4j/issues/7027 int inputSize = 300; int hiddenSize = 100; int dataSize = 10; @@ -2033,7 +2018,7 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { @Test public void testCompGraphUpdaterBlocks(){ //Check that setting learning rate results in correct rearrangement of updater state within updater blocks - //https://github.com/eclipse/deeplearning4j/issues/6809#issuecomment-463892644 + //https://github.com/deeplearning4j/deeplearning4j/issues/6809#issuecomment-463892644 double lr = 1e-3; ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() @@ -2129,10 +2114,9 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { } @Test - @Execution(ExecutionMode.SAME_THREAD) - @Tag(TagNames.NEEDS_VERIFY) - @Disabled public void testCompGraphInputReuse() { + Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); + int inputSize = 5; int outputSize = 6; int layerSize = 3; @@ -2147,8 +2131,7 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { .setOutputs("out") .addLayer("0",new DenseLayer.Builder().nIn(inputSize).nOut(layerSize).build(),"in") .addVertex("combine", new MergeVertex(), "0", "0", "0") - .addLayer("out",new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(3*layerSize) - .nOut(outputSize) + .addLayer("out",new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(3*layerSize).nOut(outputSize) .activation(Activation.SIGMOID).build(),"combine") .build(); @@ -2157,8 +2140,8 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { int dataSize = 11; - INDArray features = Nd4j.rand(DataType.DOUBLE,new int[] {dataSize, inputSize}); - INDArray labels = Nd4j.rand(DataType.DOUBLE,new int[] {dataSize, outputSize}); + INDArray features = Nd4j.rand(new int[] {dataSize, inputSize}); + INDArray labels = Nd4j.rand(new int[] {dataSize, outputSize}); boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{features}) .labels(new INDArray[]{labels})); @@ -2207,7 +2190,7 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { } @Test - public void testMergeNchw(@TempDir Path testDir) throws Exception { + public void testMergeNchw() throws Exception { ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() .convolutionMode(ConvolutionMode.Same) .graphBuilder() @@ -2234,7 +2217,7 @@ public class TestComputationGraphNetwork extends BaseDL4JTest { INDArray[] in = new INDArray[]{Nd4j.rand(DataType.FLOAT, 1, 32, 32, 3)}; INDArray out = cg.outputSingle(in); - File dir = testDir.toFile(); + File dir = testDir; File f = new File(dir, "net.zip"); cg.save(f); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestSetGetParameters.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestSetGetParameters.java new file mode 100644 index 000000000..ec5c47894 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestSetGetParameters.java @@ -0,0 +1,83 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.graph; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.*; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestSetGetParameters extends BaseDL4JTest { + + @Test + public void testInitWithParamsCG() { + + Nd4j.getRandom().setSeed(12345); + + //Create configuration. Doesn't matter if this doesn't actually work for forward/backward pass here + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder() + .addInputs("in").addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in") + .addLayer("1", new GravesLSTM.Builder().nIn(10).nOut(10).build(), "in") + .addLayer("2", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build(), "in") + .addLayer("3", new ConvolutionLayer.Builder().nIn(10).nOut(10).kernelSize(2, 2).stride(2, 2) + .padding(2, 2).build(), "in") + .addLayer("4", new OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "3") + .addLayer("5", new OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "0") + .addLayer("6", new RnnOutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "1", + "2") + .setOutputs("4", "5", "6").build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + INDArray params = net.params(); + + + ComputationGraph net2 = new ComputationGraph(conf); + net2.init(params, true); + + ComputationGraph net3 = new ComputationGraph(conf); + net3.init(params, false); + + assertEquals(params, net2.params()); + assertEquals(params, net3.params()); + + assertFalse(params == net2.params()); //Different objects due to clone + assertTrue(params == net3.params()); //Same object due to clone + + + Map paramsMap = net.paramTable(); + Map paramsMap2 = net2.paramTable(); + Map paramsMap3 = net3.paramTable(); + for (String s : paramsMap.keySet()) { + assertEquals(paramsMap.get(s), paramsMap2.get(s)); + assertEquals(paramsMap.get(s), paramsMap3.get(s)); + } + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestVariableLengthTSCG.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestVariableLengthTSCG.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestVariableLengthTSCG.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestVariableLengthTSCG.java index 4c02886b2..a39ac53b5 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestVariableLengthTSCG.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/TestVariableLengthTSCG.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; import org.deeplearning4j.nn.gradient.Gradient; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -53,8 +50,7 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestVariableLengthTSCG extends BaseDL4JTest { @Test @@ -229,7 +225,7 @@ public class TestVariableLengthTSCG extends BaseDL4JTest { INDArray g1s = g1map.get(s); INDArray g2s = g2map.get(s); - assertNotEquals(g1s, g2s, s); + assertNotEquals( g1s, g2s, s); } //Modify the values at the masked time step, and check that neither the gradients, score or activations change @@ -335,7 +331,8 @@ public class TestVariableLengthTSCG extends BaseDL4JTest { net.computeGradientAndScore(); double score = net.score(); - assertEquals( expScore, score, 0.1,msg); + + assertEquals(expScore, score, 0.1, msg); } } } @@ -370,7 +367,7 @@ public class TestVariableLengthTSCG extends BaseDL4JTest { } } - INDArray input = Nd4j.rand(new int[] {miniBatch, nIn, tsLength}); + INDArray input = Nd4j.rand(miniBatch, nIn, tsLength); ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L) diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/graphnodes/TestGraphNodes.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/graphnodes/TestGraphNodes.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/graphnodes/TestGraphNodes.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/graphnodes/TestGraphNodes.java index 12653f6ea..de4010554 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/graphnodes/TestGraphNodes.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/graph/graphnodes/TestGraphNodes.java @@ -40,10 +40,7 @@ import org.deeplearning4j.nn.graph.vertex.GraphVertex; import org.deeplearning4j.nn.graph.vertex.impl.*; import org.deeplearning4j.nn.transferlearning.TransferLearning; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.api.MultiDataSet; @@ -60,8 +57,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class TestGraphNodes extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/ActivationLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/ActivationLayerTest.java new file mode 100644 index 000000000..0c7375cfd --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/ActivationLayerTest.java @@ -0,0 +1,329 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ActivationLayer; +import org.deeplearning4j.nn.conf.layers.AutoEncoder; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.impl.ActivationELU; +import org.nd4j.linalg.activations.impl.ActivationRationalTanh; +import org.nd4j.linalg.activations.impl.ActivationSoftmax; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + */ + +public class ActivationLayerTest extends BaseDL4JTest { + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @Test + public void testInputTypes() { + org.deeplearning4j.nn.conf.layers.ActivationLayer l = + new org.deeplearning4j.nn.conf.layers.ActivationLayer.Builder().activation(Activation.RELU) + .build(); + + + InputType in1 = InputType.feedForward(20); + InputType in2 = InputType.convolutional(28, 28, 1); + + assertEquals(in1, l.getOutputType(0, in1)); + assertEquals(in2, l.getOutputType(0, in2)); + assertNull(l.getPreProcessorForInputType(in1)); + assertNull(l.getPreProcessorForInputType(in2)); + } + + @Test + public void testDenseActivationLayer() throws Exception { + DataSetIterator iter = new MnistDataSetIterator(2, 2); + DataSet next = iter.next(); + + // Run without separate activation layer + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new DenseLayer.Builder().nIn(28 * 28 * 1).nOut(10).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).nIn(10).nOut(10).build()) + .build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + network.fit(next); + + + // Run with separate activation layer + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new DenseLayer.Builder().nIn(28 * 28 * 1).nOut(10).activation(Activation.IDENTITY) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.ActivationLayer.Builder() + .activation(Activation.RELU).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10) + .build()) + .build(); + + MultiLayerNetwork network2 = new MultiLayerNetwork(conf2); + network2.init(); + network2.fit(next); + + // check parameters + assertEquals(network.getLayer(0).getParam("W"), network2.getLayer(0).getParam("W")); + assertEquals(network.getLayer(1).getParam("W"), network2.getLayer(2).getParam("W")); + assertEquals(network.getLayer(0).getParam("b"), network2.getLayer(0).getParam("b")); + assertEquals(network.getLayer(1).getParam("b"), network2.getLayer(2).getParam("b")); + + // check activations + network.init(); + network.setInput(next.getFeatures()); + List activations = network.feedForward(true); + + network2.init(); + network2.setInput(next.getFeatures()); + List activations2 = network2.feedForward(true); + + assertEquals(activations.get(1).reshape(activations2.get(2).shape()), activations2.get(2)); + assertEquals(activations.get(2), activations2.get(3)); + + + } + + @Test + public void testAutoEncoderActivationLayer() throws Exception { + + int minibatch = 3; + int nIn = 5; + int layerSize = 5; + int nOut = 3; + + INDArray next = Nd4j.rand(new int[] {minibatch, nIn}); + INDArray labels = Nd4j.zeros(minibatch, nOut); + for (int i = 0; i < minibatch; i++) { + labels.putScalar(i, i % nOut, 1.0); + } + + // Run without separate activation layer + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new AutoEncoder.Builder().nIn(nIn).nOut(layerSize).corruptionLevel(0.0) + .activation(Activation.SIGMOID).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.RECONSTRUCTION_CROSSENTROPY) + .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut) + .build()) + .build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + network.fit(next, labels); //Labels are necessary for this test: layer activation function affect pretraining results, otherwise + + + // Run with separate activation layer + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new AutoEncoder.Builder().nIn(nIn).nOut(layerSize).corruptionLevel(0.0) + .activation(Activation.IDENTITY).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.ActivationLayer.Builder() + .activation(Activation.SIGMOID).build()) + .layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.RECONSTRUCTION_CROSSENTROPY) + .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut) + .build()) + .build(); + + MultiLayerNetwork network2 = new MultiLayerNetwork(conf2); + network2.init(); + network2.fit(next, labels); + + // check parameters + assertEquals(network.getLayer(0).getParam("W"), network2.getLayer(0).getParam("W")); + assertEquals(network.getLayer(1).getParam("W"), network2.getLayer(2).getParam("W")); + assertEquals(network.getLayer(0).getParam("b"), network2.getLayer(0).getParam("b")); + assertEquals(network.getLayer(1).getParam("b"), network2.getLayer(2).getParam("b")); + + // check activations + network.init(); + network.setInput(next); + List activations = network.feedForward(true); + + network2.init(); + network2.setInput(next); + List activations2 = network2.feedForward(true); + + assertEquals(activations.get(1).reshape(activations2.get(2).shape()), activations2.get(2)); + assertEquals(activations.get(2), activations2.get(3)); + + + } + + @Test + public void testCNNActivationLayer() throws Exception { + DataSetIterator iter = new MnistDataSetIterator(2, 2); + DataSet next = iter.next(); + + // Run without separate activation layer + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(1).nOut(20) + .activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).nOut(10).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + network.fit(next); + + + // Run with separate activation layer + MultiLayerConfiguration conf2 = + new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .seed(123).list() + .layer(0, new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(1).nOut(20) + .activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER) + .build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.ActivationLayer.Builder() + .activation(Activation.RELU).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .nOut(10).build()) + + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + MultiLayerNetwork network2 = new MultiLayerNetwork(conf2); + network2.init(); + network2.fit(next); + + // check parameters + assertEquals(network.getLayer(0).getParam("W"), network2.getLayer(0).getParam("W")); + assertEquals(network.getLayer(1).getParam("W"), network2.getLayer(2).getParam("W")); + assertEquals(network.getLayer(0).getParam("b"), network2.getLayer(0).getParam("b")); + + // check activations + network.init(); + network.setInput(next.getFeatures()); + List activations = network.feedForward(true); + + network2.init(); + network2.setInput(next.getFeatures()); + List activations2 = network2.feedForward(true); + + assertEquals(activations.get(1).reshape(activations2.get(2).shape()), activations2.get(2)); + assertEquals(activations.get(2), activations2.get(3)); + } + + + @Test + public void testActivationInheritance() { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .weightInit(WeightInit.XAVIER) + .activation(Activation.RATIONALTANH) + .list() + .layer(new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(new ActivationLayer()) + .layer(new ActivationLayer.Builder().build()) + .layer(new ActivationLayer.Builder().activation(Activation.ELU).build()) + .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(10).nOut(10).build()) + .build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + assertNotNull(((ActivationLayer)network.getLayer(1).conf().getLayer()).getActivationFn()); + + assertTrue(((DenseLayer)network.getLayer(0).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); + assertTrue(((ActivationLayer)network.getLayer(1).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); + assertTrue(((ActivationLayer)network.getLayer(2).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); + assertTrue(((ActivationLayer)network.getLayer(3).conf().getLayer()).getActivationFn() instanceof ActivationELU); + assertTrue(((OutputLayer)network.getLayer(4).conf().getLayer()).getActivationFn() instanceof ActivationSoftmax); + } + + @Test + public void testActivationInheritanceCG() { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .weightInit(WeightInit.XAVIER) + .activation(Activation.RATIONALTANH) + .graphBuilder() + .addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in") + .addLayer("1", new ActivationLayer(), "0") + .addLayer("2", new ActivationLayer.Builder().build(), "1") + .addLayer("3", new ActivationLayer.Builder().activation(Activation.ELU).build(), "2") + .addLayer("4", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "3") + .setOutputs("4") + .build(); + + ComputationGraph network = new ComputationGraph(conf); + network.init(); + + assertNotNull(((ActivationLayer)network.getLayer("1").conf().getLayer()).getActivationFn()); + + assertTrue(((DenseLayer)network.getLayer("0").conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); + assertTrue(((ActivationLayer)network.getLayer("1").conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); + assertTrue(((ActivationLayer)network.getLayer("2").conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); + assertTrue(((ActivationLayer)network.getLayer("3").conf().getLayer()).getActivationFn() instanceof ActivationELU); + assertTrue(((OutputLayer)network.getLayer("4").conf().getLayer()).getActivationFn() instanceof ActivationSoftmax); + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/AutoEncoderTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/AutoEncoderTest.java new file mode 100644 index 000000000..f841d1454 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/AutoEncoderTest.java @@ -0,0 +1,79 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.graph.MergeVertex; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.AutoEncoder; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +public class AutoEncoderTest extends BaseDL4JTest { + + @Test + public void sanityCheckIssue5662(){ + int mergeSize = 50; + int encdecSize = 25; + int in1Size = 20; + int in2Size = 15; + int hiddenSize = 10; + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .weightInit(WeightInit.XAVIER) + .graphBuilder() + .addInputs("in1", "in2") + .addLayer("1", new DenseLayer.Builder().nOut(mergeSize).build(), "in1") + .addLayer("2", new DenseLayer.Builder().nOut(mergeSize).build(), "in2") + .addVertex("merge", new MergeVertex(), "1", "2") + .addLayer("e",new AutoEncoder.Builder().nOut(encdecSize).corruptionLevel(0.2).build(),"merge") + .addLayer("hidden",new AutoEncoder.Builder().nOut(hiddenSize).build(),"e") + .addLayer("decoder",new AutoEncoder.Builder().nOut(encdecSize).corruptionLevel(0.2).build(),"hidden") + .addLayer("L4", new DenseLayer.Builder().nOut(mergeSize).build(), "decoder") + .addLayer("out1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(in1Size).build(),"L4") + .addLayer("out2",new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(in2Size).build(),"L4") + .setOutputs("out1","out2") + .setInputTypes(InputType.feedForward(in1Size), InputType.feedForward(in2Size)) + + .build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + + MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet( + new INDArray[]{Nd4j.create(1, in1Size), Nd4j.create(1, in2Size)}, + new INDArray[]{Nd4j.create(1, in1Size), Nd4j.create(1, in2Size)}); + + net.summary(InputType.feedForward(in1Size), InputType.feedForward(in2Size)); + net.fit(new SingletonMultiDataSetIterator(mds)); + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/BaseLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/BaseLayerTest.java new file mode 100644 index 000000000..bc1b2db87 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/BaseLayerTest.java @@ -0,0 +1,106 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +public class BaseLayerTest extends BaseDL4JTest { + + protected INDArray weight = Nd4j.create(new double[] {0.10, -0.20, -0.15, 0.05}, new int[] {2, 2}); + protected INDArray bias = Nd4j.create(new double[] {0.5, 0.5}, new int[] {1, 2}); + protected Map paramTable; + + @BeforeEach + public void doBefore() { + paramTable = new HashMap<>(); + paramTable.put("W", weight); + paramTable.put("b", bias); + + } + + @Test + public void testSetExistingParamsConvolutionSingleLayer() { + Layer layer = configureSingleLayer(); + assertNotEquals(paramTable, layer.paramTable()); + + layer.setParamTable(paramTable); + assertEquals(paramTable, layer.paramTable()); + } + + + @Test + public void testSetExistingParamsDenseMultiLayer() { + MultiLayerNetwork net = configureMultiLayer(); + + for (Layer layer : net.getLayers()) { + assertNotEquals(paramTable, layer.paramTable()); + layer.setParamTable(paramTable); + assertEquals(paramTable, layer.paramTable()); + } + } + + + public Layer configureSingleLayer() { + int nIn = 2; + int nOut = 2; + + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .layer(new ConvolutionLayer.Builder().nIn(nIn).nOut(nOut).build()).build(); + + val numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + return conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + } + + + public MultiLayerNetwork configureMultiLayer() { + int nIn = 2; + int nOut = 2; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(nOut).build()) + .layer(1, new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build()).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + return net; + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/CacheModeTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/CacheModeTest.java new file mode 100644 index 000000000..d4eea7a49 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/CacheModeTest.java @@ -0,0 +1,173 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.conf.*; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CacheModeTest extends BaseDL4JTest { + + @Test + public void testConvCacheModeSimple(){ + + MultiLayerConfiguration conf1 = getConf(CacheMode.NONE); + MultiLayerConfiguration conf2 = getConf(CacheMode.DEVICE); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + INDArray in = Nd4j.rand(3, 28*28); + INDArray labels = TestUtils.randomOneHot(3, 10); + + INDArray out1 = net1.output(in); + INDArray out2 = net2.output(in); + assertEquals(out1, out2); + + assertEquals(net1.params(), net2.params()); + net1.fit(in, labels); + net2.fit(in, labels); + assertEquals(net1.params(), net2.params()); + } + + private static MultiLayerConfiguration getConf(CacheMode cacheMode){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .inferenceWorkspaceMode(WorkspaceMode.ENABLED) + .trainingWorkspaceMode(WorkspaceMode.ENABLED) + .seed(12345) + .cacheMode(cacheMode) + .list() + .layer(new ConvolutionLayer.Builder().nOut(3).build()) + .layer(new ConvolutionLayer.Builder().nOut(3).build()) + .layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) + .build(); + + return conf; + } + + @Test + public void testLSTMCacheModeSimple(){ + + for(boolean graves : new boolean[]{true, false}) { + + MultiLayerConfiguration conf1 = getConfLSTM(CacheMode.NONE, graves); + MultiLayerConfiguration conf2 = getConfLSTM(CacheMode.DEVICE, graves); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + INDArray in = Nd4j.rand(new int[]{3, 3, 10}); + INDArray labels = TestUtils.randomOneHotTimeSeries(3, 10, 10); + + INDArray out1 = net1.output(in); + INDArray out2 = net2.output(in); + assertEquals(out1, out2); + + assertEquals(net1.params(), net2.params()); + net1.fit(in, labels); + net2.fit(in, labels); + assertEquals(net1.params(), net2.params()); + } + } + + private static MultiLayerConfiguration getConfLSTM(CacheMode cacheMode, boolean graves){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .inferenceWorkspaceMode(WorkspaceMode.ENABLED) + .trainingWorkspaceMode(WorkspaceMode.ENABLED) + .seed(12345) + .cacheMode(cacheMode) + .list() + .layer(graves ? + new GravesLSTM.Builder().nIn(3).nOut(3).build() : + new LSTM.Builder().nIn(3).nOut(3).build()) + .layer(graves ? + new GravesLSTM.Builder().nIn(3).nOut(3).build() : + new LSTM.Builder().nIn(3).nOut(3).build()) + .layer(new RnnOutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).build()) + .build(); + + return conf; + } + + + @Test + public void testConvCacheModeSimpleCG(){ + + ComputationGraphConfiguration conf1 = getConfCG(CacheMode.NONE); + ComputationGraphConfiguration conf2 = getConfCG(CacheMode.DEVICE); + + ComputationGraph net1 = new ComputationGraph(conf1); + net1.init(); + ComputationGraph net2 = new ComputationGraph(conf2); + net2.init(); + + INDArray in = Nd4j.rand(3, 28*28); + INDArray labels = TestUtils.randomOneHot(3, 10); + + INDArray out1 = net1.outputSingle(in); + INDArray out2 = net2.outputSingle(in); + assertEquals(out1, out2); + + assertEquals(net1.params(), net2.params()); + net1.fit(new DataSet(in, labels)); + net2.fit(new DataSet(in, labels)); + assertEquals(net1.params(), net2.params()); + } + + private static ComputationGraphConfiguration getConfCG(CacheMode cacheMode){ + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .inferenceWorkspaceMode(WorkspaceMode.ENABLED) + .trainingWorkspaceMode(WorkspaceMode.ENABLED) + .seed(12345) + .cacheMode(cacheMode) + .graphBuilder() + .addInputs("in") + .layer("0", new ConvolutionLayer.Builder().nOut(3).build(), "in") + .layer("1", new ConvolutionLayer.Builder().nOut(3).build(), "0") + .layer("2", new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).build(), "1") + .setOutputs("2") + .setInputTypes(InputType.convolutionalFlat(28, 28, 1)) + .build(); + + return conf; + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/CenterLossOutputLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/CenterLossOutputLayerTest.java new file mode 100644 index 000000000..73bd4c333 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/CenterLossOutputLayerTest.java @@ -0,0 +1,149 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; + +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +public class CenterLossOutputLayerTest extends BaseDL4JTest { + + private ComputationGraph getGraph(int numLabels, double lambda) { + Nd4j.getRandom().setSeed(12345); + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .dist(new NormalDistribution(0, 1)).updater(new NoOp()) + .graphBuilder().addInputs("input1") + .addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.RELU).build(), + "input1") + .addLayer("lossLayer", new CenterLossOutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(numLabels) + .lambda(lambda).activation(Activation.SOFTMAX).build(), "l1") + .setOutputs("lossLayer").build(); + + ComputationGraph graph = new ComputationGraph(conf); + graph.init(); + return graph; + } + + public ComputationGraph getCNNMnistConfig() { + + int nChannels = 1; // Number of input channels + int outputNum = 10; // The number of possible outcomes + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) // Training iterations as above + .l2(0.0005).weightInit(WeightInit.XAVIER) + .updater(new Nesterovs(0.01, 0.9)) + .graphBuilder().addInputs("input") + .setInputTypes(InputType.convolutionalFlat(28, 28, 1)) + .addLayer("0", new ConvolutionLayer.Builder(5, 5) + //nIn and nOut specify channels. nIn here is the nChannels and nOut is the number of filters to be applied + .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build(), + "input") + .addLayer("1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2) + .stride(2, 2).build(), "0") + .addLayer("2", new ConvolutionLayer.Builder(5, 5) + //Note that nIn need not be specified in later layers + .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "1") + .addLayer("3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2) + .stride(2, 2).build(), "2") + .addLayer("4", new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build(), "3") + .addLayer("output", + new org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer.Builder( + LossFunction.MCXENT).nOut(outputNum) + .activation(Activation.SOFTMAX).build(), + "4") + .setOutputs("output").build(); + + ComputationGraph graph = new ComputationGraph(conf); + graph.init(); + return graph; + } + + @Test + public void testLambdaConf() { + double[] lambdas = new double[] {0.1, 0.01}; + double[] results = new double[2]; + int numClasses = 2; + + INDArray input = Nd4j.rand(150, 4); + INDArray labels = Nd4j.zeros(150, numClasses); + Random r = new Random(12345); + for (int i = 0; i < 150; i++) { + labels.putScalar(i, r.nextInt(numClasses), 1.0); + } + ComputationGraph graph; + + for (int i = 0; i < lambdas.length; i++) { + graph = getGraph(numClasses, lambdas[i]); + graph.setInput(0, input); + graph.setLabel(0, labels); + graph.computeGradientAndScore(); + results[i] = graph.score(); + } + + assertNotEquals(results[0], results[1]); + } + + + + @Test + ////@Ignore //Should be run manually + public void testMNISTConfig() throws Exception { + int batchSize = 64; // Test batch size + DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345); + + ComputationGraph net = getCNNMnistConfig(); + net.init(); + net.setListeners(new ScoreIterationListener(1)); + + for (int i = 0; i < 50; i++) { + net.fit(mnistTrain.next()); + Thread.sleep(1000); + } + + Thread.sleep(100000); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/DropoutLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/DropoutLayerTest.java new file mode 100644 index 000000000..cee20827c --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/DropoutLayerTest.java @@ -0,0 +1,305 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.dropout.Dropout; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.DropoutLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + */ +public class DropoutLayerTest extends BaseDL4JTest { + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @Test + public void testInputTypes() { + DropoutLayer config = new DropoutLayer.Builder(0.5).build(); + + InputType in1 = InputType.feedForward(20); + InputType in2 = InputType.convolutional(28, 28, 1); + + assertEquals(in1, config.getOutputType(0, in1)); + assertEquals(in2, config.getOutputType(0, in2)); + assertNull(config.getPreProcessorForInputType(in1)); + assertNull(config.getPreProcessorForInputType(in2)); + } + + @Test + public void testDropoutLayerWithoutTraining() throws Exception { + MultiLayerConfiguration confIntegrated = new NeuralNetConfiguration.Builder().seed(3648) + .list().layer(0, + new ConvolutionLayer.Builder(1, 1).stride(1, 1).nIn(1).nOut(1).dropOut(0.25) + .activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER) + .build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .weightInit(WeightInit.XAVIER).dropOut(0.25) + .nOut(4).build()) + .setInputType(InputType.convolutionalFlat(2, 2, 1)).build(); + + MultiLayerNetwork netIntegrated = new MultiLayerNetwork(confIntegrated); + netIntegrated.init(); + netIntegrated.getLayer(0).setParam("W", Nd4j.eye(1)); + netIntegrated.getLayer(0).setParam("b", Nd4j.zeros(1, 1)); + netIntegrated.getLayer(1).setParam("W", Nd4j.eye(4)); + netIntegrated.getLayer(1).setParam("b", Nd4j.zeros(4, 1)); + + MultiLayerConfiguration confSeparate = + new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .seed(3648) + .list().layer(0, + new DropoutLayer.Builder(0.25) + .build()) + .layer(1, new ConvolutionLayer.Builder(1, 1).stride(1, 1).nIn(1).nOut(1) + .activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER) + .build()) + .layer(2, new DropoutLayer.Builder(0.25).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .nOut(4).build()) + + .setInputType(InputType.convolutionalFlat(2, 2, 1)).build(); + + MultiLayerNetwork netSeparate = new MultiLayerNetwork(confSeparate); + netSeparate.init(); + netSeparate.getLayer(1).setParam("W", Nd4j.eye(1)); + netSeparate.getLayer(1).setParam("b", Nd4j.zeros(1, 1)); + netSeparate.getLayer(3).setParam("W", Nd4j.eye(4)); + netSeparate.getLayer(3).setParam("b", Nd4j.zeros(4, 1)); + + //Disable input modification for this test: + for(Layer l : netIntegrated.getLayers()){ + l.allowInputModification(false); + } + for(Layer l : netSeparate.getLayers()){ + l.allowInputModification(false); + } + + INDArray in = Nd4j.arange(1, 5).reshape(1,4); + Nd4j.getRandom().setSeed(12345); + List actTrainIntegrated = netIntegrated.feedForward(in.dup(), true); + Nd4j.getRandom().setSeed(12345); + List actTrainSeparate = netSeparate.feedForward(in.dup(), true); + Nd4j.getRandom().setSeed(12345); + List actTestIntegrated = netIntegrated.feedForward(in.dup(), false); + Nd4j.getRandom().setSeed(12345); + List actTestSeparate = netSeparate.feedForward(in.dup(), false); + + //Check masks: + INDArray maskIntegrated = ((Dropout)netIntegrated.getLayer(0).conf().getLayer().getIDropout()).getMask(); + INDArray maskSeparate = ((Dropout)netSeparate.getLayer(0).conf().getLayer().getIDropout()).getMask(); + assertEquals(maskIntegrated, maskSeparate); + + + + + assertEquals(actTrainIntegrated.get(1), actTrainSeparate.get(2)); + assertEquals(actTrainIntegrated.get(2), actTrainSeparate.get(4)); + assertEquals(actTestIntegrated.get(1), actTestSeparate.get(2)); + assertEquals(actTestIntegrated.get(2), actTestSeparate.get(4)); + } + + @Test + public void testDropoutLayerWithDenseMnist() throws Exception { + DataSetIterator iter = new MnistDataSetIterator(2, 2); + DataSet next = iter.next(); + + // Run without separate activation layer + MultiLayerConfiguration confIntegrated = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new DenseLayer.Builder().nIn(28 * 28 * 1).nOut(10) + .activation(Activation.RELU).weightInit( + WeightInit.XAVIER) + .build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).dropOut(0.25) + .nIn(10).nOut(10).build()) + .build(); + + MultiLayerNetwork netIntegrated = new MultiLayerNetwork(confIntegrated); + netIntegrated.init(); + netIntegrated.fit(next); + + // Run with separate activation layer + MultiLayerConfiguration confSeparate = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new DenseLayer.Builder().nIn(28 * 28 * 1).nOut(10).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new DropoutLayer.Builder(0.25).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10) + .build()) + .build(); + + MultiLayerNetwork netSeparate = new MultiLayerNetwork(confSeparate); + netSeparate.init(); + netSeparate.fit(next); + + //Disable input modification for this test: + for(Layer l : netIntegrated.getLayers()){ + l.allowInputModification(false); + } + for(Layer l : netSeparate.getLayers()){ + l.allowInputModification(false); + } + + // check parameters + assertEquals(netIntegrated.getLayer(0).getParam("W"), netSeparate.getLayer(0).getParam("W")); + assertEquals(netIntegrated.getLayer(0).getParam("b"), netSeparate.getLayer(0).getParam("b")); + assertEquals(netIntegrated.getLayer(1).getParam("W"), netSeparate.getLayer(2).getParam("W")); + assertEquals(netIntegrated.getLayer(1).getParam("b"), netSeparate.getLayer(2).getParam("b")); + + // check activations + netIntegrated.setInput(next.getFeatures()); + netSeparate.setInput(next.getFeatures()); + + Nd4j.getRandom().setSeed(12345); + List actTrainIntegrated = netIntegrated.feedForward(true); + Nd4j.getRandom().setSeed(12345); + List actTrainSeparate = netSeparate.feedForward(true); + assertEquals(actTrainIntegrated.get(1), actTrainSeparate.get(1)); + assertEquals(actTrainIntegrated.get(2), actTrainSeparate.get(3)); + + Nd4j.getRandom().setSeed(12345); + List actTestIntegrated = netIntegrated.feedForward(false); + Nd4j.getRandom().setSeed(12345); + List actTestSeparate = netSeparate.feedForward(false); + assertEquals(actTestIntegrated.get(1), actTrainSeparate.get(1)); + assertEquals(actTestIntegrated.get(2), actTestSeparate.get(3)); + } + + @Test + public void testDropoutLayerWithConvMnist() throws Exception { + Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); //Set to double datatype - MKL-DNN not used for CPU (otherwise different strides due to Dl4J impl permutes) + DataSetIterator iter = new MnistDataSetIterator(2, 2); + DataSet next = iter.next(); + + // Run without separate activation layer + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration confIntegrated = new NeuralNetConfiguration.Builder().seed(123) + .list().layer(0, + new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(1).nOut(20) + .activation(Activation.TANH).weightInit(WeightInit.XAVIER) + .build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).dropOut(0.5) + .nOut(10).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + // Run with separate activation layer + Nd4j.getRandom().setSeed(12345); + + //Manually configure preprocessors + //This is necessary, otherwise CnnToFeedForwardPreprocessor will be in different locatinos + //i.e., dropout on 4d activations in latter, and dropout on 2d activations in former + Map preProcessorMap = new HashMap<>(); + preProcessorMap.put(1, new CnnToFeedForwardPreProcessor(13, 13, 20)); + + MultiLayerConfiguration confSeparate = new NeuralNetConfiguration.Builder().seed(123).list() + .layer(0, new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(1).nOut(20) + .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()) + .layer(1, new DropoutLayer.Builder(0.5).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()) + .inputPreProcessors(preProcessorMap) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + + Nd4j.getRandom().setSeed(12345); + MultiLayerNetwork netIntegrated = new MultiLayerNetwork(confIntegrated); + netIntegrated.init(); + + Nd4j.getRandom().setSeed(12345); + MultiLayerNetwork netSeparate = new MultiLayerNetwork(confSeparate); + netSeparate.init(); + + assertEquals(netIntegrated.params(), netSeparate.params()); + + Nd4j.getRandom().setSeed(12345); + netIntegrated.fit(next); + + Nd4j.getRandom().setSeed(12345); + netSeparate.fit(next); + + assertEquals(netIntegrated.params(), netSeparate.params()); + + // check parameters + assertEquals(netIntegrated.getLayer(0).getParam("W"), netSeparate.getLayer(0).getParam("W")); + assertEquals(netIntegrated.getLayer(0).getParam("b"), netSeparate.getLayer(0).getParam("b")); + assertEquals(netIntegrated.getLayer(1).getParam("W"), netSeparate.getLayer(2).getParam("W")); + assertEquals(netIntegrated.getLayer(1).getParam("b"), netSeparate.getLayer(2).getParam("b")); + + // check activations + netIntegrated.setInput(next.getFeatures().dup()); + netSeparate.setInput(next.getFeatures().dup()); + + Nd4j.getRandom().setSeed(12345); + List actTrainIntegrated = netIntegrated.feedForward(true); + Nd4j.getRandom().setSeed(12345); + List actTrainSeparate = netSeparate.feedForward(true); + assertEquals(actTrainIntegrated.get(1), actTrainSeparate.get(1)); + assertEquals(actTrainIntegrated.get(2), actTrainSeparate.get(3)); + + netIntegrated.setInput(next.getFeatures().dup()); + netSeparate.setInput(next.getFeatures().dup()); + Nd4j.getRandom().setSeed(12345); + List actTestIntegrated = netIntegrated.feedForward(false); + Nd4j.getRandom().setSeed(12345); + List actTestSeparate = netSeparate.feedForward(false); + assertEquals(actTestIntegrated.get(1), actTestSeparate.get(1)); + assertEquals(actTestIntegrated.get(2), actTestSeparate.get(3)); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerTest.java new file mode 100644 index 000000000..c3543e167 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerTest.java @@ -0,0 +1,377 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; +import org.deeplearning4j.nn.transferlearning.TransferLearning; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +public class FrozenLayerTest extends BaseDL4JTest { + + /* + A model with a few frozen layers == + Model with non frozen layers set with the output of the forward pass of the frozen layers + */ + @Test + public void testFrozen() { + DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.IDENTITY); + + FineTuneConfiguration finetune = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build(); + + MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(overallConf.clone().list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()) + .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build()); + + modelToFineTune.init(); + List ff = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false); + INDArray asFrozenFeatures = ff.get(2); + + MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(finetune) + .setFeatureExtractor(1).build(); + + INDArray paramsLastTwoLayers = + Nd4j.hstack(modelToFineTune.getLayer(2).params(), modelToFineTune.getLayer(3).params()); + MultiLayerNetwork notFrozen = new MultiLayerNetwork(overallConf.clone().list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build(), paramsLastTwoLayers); + + // assertEquals(modelNow.getLayer(2).conf(), notFrozen.getLayer(0).conf()); //Equal, other than names + // assertEquals(modelNow.getLayer(3).conf(), notFrozen.getLayer(1).conf()); //Equal, other than names + + //Check: forward pass + INDArray outNow = modelNow.output(randomData.getFeatures()); + INDArray outNotFrozen = notFrozen.output(asFrozenFeatures); + assertEquals(outNow, outNotFrozen); + + for (int i = 0; i < 5; i++) { + notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); + modelNow.fit(randomData); + } + + INDArray expected = Nd4j.hstack(modelToFineTune.getLayer(0).params(), modelToFineTune.getLayer(1).params(), + notFrozen.params()); + INDArray act = modelNow.params(); + assertEquals(expected, act); + } + + + @Test + public void cloneMLNFrozen() { + + DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.IDENTITY); + MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(overallConf.list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()) + .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build()); + + modelToFineTune.init(); + INDArray asFrozenFeatures = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false).get(2); + MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).setFeatureExtractor(1).build(); + + MultiLayerNetwork clonedModel = modelNow.clone(); + + //Check json + assertEquals(modelNow.getLayerWiseConfigurations().toJson(), clonedModel.getLayerWiseConfigurations().toJson()); + + //Check params + assertEquals(modelNow.params(), clonedModel.params()); + + MultiLayerNetwork notFrozen = new MultiLayerNetwork( + overallConf.list().layer(0, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build(), + Nd4j.hstack(modelToFineTune.getLayer(2).params(), modelToFineTune.getLayer(3).params())); + + int i = 0; + while (i < 5) { + notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); + modelNow.fit(randomData); + clonedModel.fit(randomData); + i++; + } + + INDArray expectedParams = Nd4j.hstack(modelToFineTune.getLayer(0).params(), + modelToFineTune.getLayer(1).params(), notFrozen.params()); + assertEquals(expectedParams, modelNow.params()); + assertEquals(expectedParams, clonedModel.params()); + + } + + + @Test + public void testFrozenCompGraph() { + DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.IDENTITY); + + ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In") + .addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0") + .addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1") + .addLayer("layer3", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "layer2") + .setOutputs("layer3").build()); + + modelToFineTune.init(); + INDArray asFrozenFeatures = modelToFineTune.feedForward(randomData.getFeatures(), false).get("layer1"); + + ComputationGraph modelNow = + new TransferLearning.GraphBuilder(modelToFineTune).setFeatureExtractor("layer1").build(); + + ComputationGraph notFrozen = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .addLayer("layer0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer0In") + .addLayer("layer1", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "layer0") + .setOutputs("layer1").build()); + + notFrozen.init(); + notFrozen.setParams(Nd4j.hstack(modelToFineTune.getLayer("layer2").params(), + modelToFineTune.getLayer("layer3").params())); + + int i = 0; + while (i < 5) { + notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); + modelNow.fit(randomData); + i++; + } + + assertEquals(Nd4j.hstack(modelToFineTune.getLayer("layer0").params(), + modelToFineTune.getLayer("layer1").params(), notFrozen.params()), modelNow.params()); + } + + @Test + public void cloneCompGraphFrozen() { + + DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.IDENTITY); + + ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In") + .addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0") + .addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1") + .addLayer("layer3", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "layer2") + .setOutputs("layer3").build()); + + modelToFineTune.init(); + INDArray asFrozenFeatures = modelToFineTune.feedForward(randomData.getFeatures(), false).get("layer1"); + ComputationGraph modelNow = + new TransferLearning.GraphBuilder(modelToFineTune).setFeatureExtractor("layer1").build(); + + ComputationGraph clonedModel = modelNow.clone(); + + //Check json + assertEquals(clonedModel.getConfiguration().toJson(), modelNow.getConfiguration().toJson()); + + //Check params + assertEquals(modelNow.params(), clonedModel.params()); + + ComputationGraph notFrozen = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .addLayer("layer0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer0In") + .addLayer("layer1", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "layer0") + .setOutputs("layer1").build()); + notFrozen.init(); + notFrozen.setParams(Nd4j.hstack(modelToFineTune.getLayer("layer2").params(), + modelToFineTune.getLayer("layer3").params())); + + + int i = 0; + while (i < 5) { + notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); + modelNow.fit(randomData); + clonedModel.fit(randomData); + i++; + } + + INDArray expectedParams = Nd4j.hstack(modelToFineTune.getLayer("layer0").params(), + modelToFineTune.getLayer("layer1").params(), notFrozen.params()); + assertEquals(expectedParams, modelNow.params()); + assertEquals(expectedParams, clonedModel.params()); + } + + + @Test + public void testFrozenLayerInstantiation() { + //We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if + // they were initialized via the builder + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()) + .layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10) + .nOut(10).build()) + .build(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, + new org.deeplearning4j.nn.conf.layers.misc.FrozenLayer(new DenseLayer.Builder().nIn(10).nOut(10) + .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())) + .layer(1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayer( + new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build())) + .layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10) + .nOut(10).build()) + .build(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + assertEquals(net1.params(), net2.params()); + + + String json = conf2.toJson(); + MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json); + + assertEquals(conf2, fromJson); + + MultiLayerNetwork net3 = new MultiLayerNetwork(fromJson); + net3.init(); + + INDArray input = Nd4j.rand(10, 10); + + INDArray out2 = net2.output(input); + INDArray out3 = net3.output(input); + + assertEquals(out2, out3); + } + + @Test + public void testFrozenLayerInstantiationCompGraph() { + + //We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if + // they were initialized via the builder + ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder() + .addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build(), "in") + .addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build(), "0") + .addLayer("2", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10) + .nOut(10).build(), + "1") + .setOutputs("2").build(); + + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder() + .addInputs("in") + .addLayer("0", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayer.Builder() + .layer(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()) + .build(), "in") + .addLayer("1", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayer.Builder() + .layer(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()) + .build(), "0") + .addLayer("2", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10) + .nOut(10).build(), + "1") + .setOutputs("2").build(); + + ComputationGraph net1 = new ComputationGraph(conf1); + net1.init(); + ComputationGraph net2 = new ComputationGraph(conf2); + net2.init(); + + assertEquals(net1.params(), net2.params()); + + + String json = conf2.toJson(); + ComputationGraphConfiguration fromJson = ComputationGraphConfiguration.fromJson(json); + + assertEquals(conf2, fromJson); + + ComputationGraph net3 = new ComputationGraph(fromJson); + net3.init(); + + INDArray input = Nd4j.rand(10, 10); + + INDArray out2 = net2.outputSingle(input); + INDArray out3 = net3.outputSingle(input); + + assertEquals(out2, out3); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackpropTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackpropTest.java new file mode 100644 index 000000000..dce5daebd --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackpropTest.java @@ -0,0 +1,396 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.graph.MergeVertex; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; +import org.deeplearning4j.nn.transferlearning.TransferLearning; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@Slf4j +public class FrozenLayerWithBackpropTest extends BaseDL4JTest { + + @Test + public void testFrozenWithBackpropLayerInstantiation() { + //We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if + // they were initialized via the builder + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()) + .layer(2, new OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10) + .nOut(10).build()) + .build(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, + new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(10).nOut(10) + .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())) + .layer(1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build())) + .layer(2, new OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10) + .nOut(10).build()) + .build(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + assertEquals(net1.params(), net2.params()); + + + String json = conf2.toJson(); + MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json); + + assertEquals(conf2, fromJson); + + MultiLayerNetwork net3 = new MultiLayerNetwork(fromJson); + net3.init(); + + INDArray input = Nd4j.rand(10, 10); + + INDArray out2 = net2.output(input); + INDArray out3 = net3.output(input); + + assertEquals(out2, out3); + } + + @Test + public void testFrozenLayerInstantiationCompGraph() { + + //We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if + // they were initialized via the builder + ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder() + .addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build(), "in") + .addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build(), "0") + .addLayer("2", new OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10) + .nOut(10).build(), + "1") + .setOutputs("2").build(); + + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder() + .addInputs("in") + .addLayer("0", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()), "in") + .addLayer("1", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).build()), "0") + .addLayer("2", new OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10) + .nOut(10).build(), + "1") + .setOutputs("2").build(); + + ComputationGraph net1 = new ComputationGraph(conf1); + net1.init(); + ComputationGraph net2 = new ComputationGraph(conf2); + net2.init(); + + assertEquals(net1.params(), net2.params()); + + + String json = conf2.toJson(); + ComputationGraphConfiguration fromJson = ComputationGraphConfiguration.fromJson(json); + + assertEquals(conf2, fromJson); + + ComputationGraph net3 = new ComputationGraph(fromJson); + net3.init(); + + INDArray input = Nd4j.rand(10, 10); + + INDArray out2 = net2.outputSingle(input); + INDArray out3 = net3.outputSingle(input); + + assertEquals(out2, out3); + } + + @Test + public void testMultiLayerNetworkFrozenLayerParamsAfterBackprop() { + Nd4j.getRandom().setSeed(12345); + DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1)); + + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder() + .seed(12345) + .weightInit(WeightInit.XAVIER) + .updater(new Sgd(2)) + .list() + .layer(new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(3).nOut(4).build())) + .layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(4).nOut(2).build())) + .layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(2).nOut(1).build())) + .build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf1); + network.init(); + INDArray unfrozenLayerParams = network.getLayer(0).params().dup(); + INDArray frozenLayerParams1 = network.getLayer(1).params().dup(); + INDArray frozenLayerParams2 = network.getLayer(2).params().dup(); + INDArray frozenOutputLayerParams = network.getLayer(3).params().dup(); + + for (int i = 0; i < 100; i++) { + network.fit(randomData); + } + + assertNotEquals(unfrozenLayerParams, network.getLayer(0).params()); + assertEquals(frozenLayerParams1, network.getLayer(1).params()); + assertEquals(frozenLayerParams2, network.getLayer(2).params()); + assertEquals(frozenOutputLayerParams, network.getLayer(3).params()); + + } + + @Test + public void testComputationGraphFrozenLayerParamsAfterBackprop() { + Nd4j.getRandom().setSeed(12345); + + DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1)); + String frozenBranchName = "B1-"; + String unfrozenBranchName = "B2-"; + + String initialLayer = "initial"; + + String frozenBranchUnfrozenLayer0 = frozenBranchName + "0"; + String frozenBranchFrozenLayer1 = frozenBranchName + "1"; + String frozenBranchFrozenLayer2 = frozenBranchName + "2"; + String frozenBranchOutput = frozenBranchName + "Output"; + + + String unfrozenLayer0 = unfrozenBranchName + "0"; + String unfrozenLayer1 = unfrozenBranchName + "1"; + String unfrozenBranch2 = unfrozenBranchName + "Output"; + + ComputationGraphConfiguration computationGraphConf = new NeuralNetConfiguration.Builder() + .updater(new Sgd(2.0)) + .seed(12345) + .graphBuilder() + .addInputs("input") + .addLayer(initialLayer, new DenseLayer.Builder().nIn(4).nOut(4).build(),"input") + .addLayer(frozenBranchUnfrozenLayer0, new DenseLayer.Builder().nIn(4).nOut(3).build(),initialLayer) + .addLayer(frozenBranchFrozenLayer1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(3).nOut(4).build()),frozenBranchUnfrozenLayer0) + .addLayer(frozenBranchFrozenLayer2, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(4).nOut(2).build()),frozenBranchFrozenLayer1) + .addLayer(unfrozenLayer0, new DenseLayer.Builder().nIn(4).nOut(4).build(),initialLayer) + .addLayer(unfrozenLayer1, new DenseLayer.Builder().nIn(4).nOut(2).build(),unfrozenLayer0) + .addLayer(unfrozenBranch2, new DenseLayer.Builder().nIn(2).nOut(1).build(),unfrozenLayer1) + .addVertex("merge", new MergeVertex(), frozenBranchFrozenLayer2, unfrozenBranch2) + .addLayer(frozenBranchOutput,new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(3).nOut(1).build()),"merge") + .setOutputs(frozenBranchOutput) + .build(); + + ComputationGraph computationGraph = new ComputationGraph(computationGraphConf); + computationGraph.init(); + INDArray unfrozenLayerParams = computationGraph.getLayer(frozenBranchUnfrozenLayer0).params().dup(); + INDArray frozenLayerParams1 = computationGraph.getLayer(frozenBranchFrozenLayer1).params().dup(); + INDArray frozenLayerParams2 = computationGraph.getLayer(frozenBranchFrozenLayer2).params().dup(); + INDArray frozenOutputLayerParams = computationGraph.getLayer(frozenBranchOutput).params().dup(); + + for (int i = 0; i < 100; i++) { + computationGraph.fit(randomData); + } + + assertNotEquals(unfrozenLayerParams, computationGraph.getLayer(frozenBranchUnfrozenLayer0).params()); + assertEquals(frozenLayerParams1, computationGraph.getLayer(frozenBranchFrozenLayer1).params()); + assertEquals(frozenLayerParams2, computationGraph.getLayer(frozenBranchFrozenLayer2).params()); + assertEquals(frozenOutputLayerParams, computationGraph.getLayer(frozenBranchOutput).params()); + + } + + /** + * Frozen layer should have same results as a layer with Sgd updater with learning rate set to 0 + */ + @Test + public void testFrozenLayerVsSgd() { + Nd4j.getRandom().setSeed(12345); + DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1)); + + MultiLayerConfiguration confSgd = new NeuralNetConfiguration.Builder() + .seed(12345) + .weightInit(WeightInit.XAVIER) + .updater(new Sgd(2)) + .list() + .layer(0,new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1,new DenseLayer.Builder().updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).nIn(3).nOut(4).build()) + .layer(2,new DenseLayer.Builder().updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).nIn(4).nOut(2).build()) + .layer(3,new OutputLayer.Builder(LossFunctions.LossFunction.MSE).updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).activation(Activation.TANH).nIn(2).nOut(1).build()) + .build(); + + MultiLayerConfiguration confFrozen = new NeuralNetConfiguration.Builder() + .seed(12345) + .weightInit(WeightInit.XAVIER) + .updater(new Sgd(2)) + .list() + .layer(0,new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1,new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(3).nOut(4).build())) + .layer(2,new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(4).nOut(2).build())) + .layer(3,new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(2).nOut(1).build())) + .build(); + MultiLayerNetwork frozenNetwork = new MultiLayerNetwork(confFrozen); + frozenNetwork.init(); + INDArray unfrozenLayerParams = frozenNetwork.getLayer(0).params().dup(); + INDArray frozenLayerParams1 = frozenNetwork.getLayer(1).params().dup(); + INDArray frozenLayerParams2 = frozenNetwork.getLayer(2).params().dup(); + INDArray frozenOutputLayerParams = frozenNetwork.getLayer(3).params().dup(); + + MultiLayerNetwork sgdNetwork = new MultiLayerNetwork(confSgd); + sgdNetwork.init(); + INDArray unfrozenSgdLayerParams = sgdNetwork.getLayer(0).params().dup(); + INDArray frozenSgdLayerParams1 = sgdNetwork.getLayer(1).params().dup(); + INDArray frozenSgdLayerParams2 = sgdNetwork.getLayer(2).params().dup(); + INDArray frozenSgdOutputLayerParams = sgdNetwork.getLayer(3).params().dup(); + + for (int i = 0; i < 100; i++) { + frozenNetwork.fit(randomData); + } + for (int i = 0; i < 100; i++) { + sgdNetwork.fit(randomData); + } + + assertEquals(frozenNetwork.getLayer(0).params(), sgdNetwork.getLayer(0).params()); + assertEquals(frozenNetwork.getLayer(1).params(), sgdNetwork.getLayer(1).params()); + assertEquals(frozenNetwork.getLayer(2).params(), sgdNetwork.getLayer(2).params()); + assertEquals(frozenNetwork.getLayer(3).params(), sgdNetwork.getLayer(3).params()); + + } + + @Test + public void testComputationGraphVsSgd() { + Nd4j.getRandom().setSeed(12345); + DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1)); + String frozenBranchName = "B1-"; + String unfrozenBranchName = "B2-"; + + String initialLayer = "initial"; + + String frozenBranchUnfrozenLayer0 = frozenBranchName + "0"; + String frozenBranchFrozenLayer1 = frozenBranchName + "1"; + String frozenBranchFrozenLayer2 = frozenBranchName + "2"; + String frozenBranchOutput = frozenBranchName + "Output"; + + + String unfrozenLayer0 = unfrozenBranchName + "0"; + String unfrozenLayer1 = unfrozenBranchName + "1"; + String unfrozenBranch2 = unfrozenBranchName + "Output"; + + ComputationGraphConfiguration computationGraphConf = new NeuralNetConfiguration.Builder() + .updater(new Sgd(2.0)) + .seed(12345) + .graphBuilder() + .addInputs("input") + .addLayer(initialLayer,new DenseLayer.Builder().nIn(4).nOut(4).build(),"input") + .addLayer(frozenBranchUnfrozenLayer0,new DenseLayer.Builder().nIn(4).nOut(3).build(), initialLayer) + .addLayer(frozenBranchFrozenLayer1,new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(3).nOut(4).build()),frozenBranchUnfrozenLayer0) + .addLayer(frozenBranchFrozenLayer2, + new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new DenseLayer.Builder().nIn(4).nOut(2).build()),frozenBranchFrozenLayer1) + .addLayer(unfrozenLayer0,new DenseLayer.Builder().nIn(4).nOut(4).build(),initialLayer) + .addLayer(unfrozenLayer1,new DenseLayer.Builder().nIn(4).nOut(2).build(),unfrozenLayer0) + .addLayer(unfrozenBranch2,new DenseLayer.Builder().nIn(2).nOut(1).build(),unfrozenLayer1) + .addVertex("merge",new MergeVertex(), frozenBranchFrozenLayer2, unfrozenBranch2) + .addLayer(frozenBranchOutput, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop( + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(3).nOut(1).build()),"merge") + .setOutputs(frozenBranchOutput) + .build(); + + ComputationGraphConfiguration computationSgdGraphConf = new NeuralNetConfiguration.Builder() + .updater(new Sgd(2.0)) + .seed(12345) + .graphBuilder() + .addInputs("input") + .addLayer(initialLayer, new DenseLayer.Builder().nIn(4).nOut(4).build(),"input") + .addLayer(frozenBranchUnfrozenLayer0,new DenseLayer.Builder().nIn(4).nOut(3).build(),initialLayer) + .addLayer(frozenBranchFrozenLayer1,new DenseLayer.Builder().updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).nIn(3).nOut(4).build(),frozenBranchUnfrozenLayer0) + .addLayer(frozenBranchFrozenLayer2,new DenseLayer.Builder().updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).nIn(4).nOut(2).build(),frozenBranchFrozenLayer1) + .addLayer(unfrozenLayer0,new DenseLayer.Builder().nIn(4).nOut(4).build(),initialLayer) + .addLayer(unfrozenLayer1,new DenseLayer.Builder().nIn(4).nOut(2).build(),unfrozenLayer0) + .addLayer(unfrozenBranch2,new DenseLayer.Builder().nIn(2).nOut(1).build(),unfrozenLayer1) + .addVertex("merge",new MergeVertex(), frozenBranchFrozenLayer2, unfrozenBranch2) + .addLayer(frozenBranchOutput,new OutputLayer.Builder(LossFunctions.LossFunction.MSE).updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).activation(Activation.TANH).nIn(3).nOut(1).build(),"merge") + .setOutputs(frozenBranchOutput) + .build(); + + ComputationGraph frozenComputationGraph = new ComputationGraph(computationGraphConf); + frozenComputationGraph.init(); + INDArray unfrozenLayerParams = frozenComputationGraph.getLayer(frozenBranchUnfrozenLayer0).params().dup(); + INDArray frozenLayerParams1 = frozenComputationGraph.getLayer(frozenBranchFrozenLayer1).params().dup(); + INDArray frozenLayerParams2 = frozenComputationGraph.getLayer(frozenBranchFrozenLayer2).params().dup(); + INDArray frozenOutputLayerParams = frozenComputationGraph.getLayer(frozenBranchOutput).params().dup(); + + ComputationGraph sgdComputationGraph = new ComputationGraph(computationSgdGraphConf); + sgdComputationGraph.init(); + INDArray unfrozenSgdLayerParams = sgdComputationGraph.getLayer(frozenBranchUnfrozenLayer0).params().dup(); + INDArray frozenSgdLayerParams1 = sgdComputationGraph.getLayer(frozenBranchFrozenLayer1).params().dup(); + INDArray frozenSgdLayerParams2 = sgdComputationGraph.getLayer(frozenBranchFrozenLayer2).params().dup(); + INDArray frozenSgdOutputLayerParams = sgdComputationGraph.getLayer(frozenBranchOutput).params().dup(); + + for (int i = 0; i < 100; i++) { + frozenComputationGraph.fit(randomData); + } + for (int i = 0; i < 100; i++) { + sgdComputationGraph.fit(randomData); + } + + assertEquals(frozenComputationGraph.getLayer(frozenBranchUnfrozenLayer0).params(), sgdComputationGraph.getLayer(frozenBranchUnfrozenLayer0).params()); + assertEquals(frozenComputationGraph.getLayer(frozenBranchFrozenLayer1).params(), sgdComputationGraph.getLayer(frozenBranchFrozenLayer1).params()); + assertEquals(frozenComputationGraph.getLayer(frozenBranchFrozenLayer2).params(), sgdComputationGraph.getLayer(frozenBranchFrozenLayer2).params()); + assertEquals(frozenComputationGraph.getLayer(frozenBranchOutput).params(), sgdComputationGraph.getLayer(frozenBranchOutput).params()); + + } + + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/OutputLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/OutputLayerTest.java new file mode 100644 index 000000000..fcd509494 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/OutputLayerTest.java @@ -0,0 +1,575 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.*; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.layers.recurrent.RnnOutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; + +import java.util.Collections; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +public class OutputLayerTest extends BaseDL4JTest { + + @Test + public void testSetParams() { + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT) + .updater(new Sgd(1e-1)) + .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder().nIn(4).nOut(3) + .weightInit(WeightInit.ZERO).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + + long numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + OutputLayer l = (OutputLayer) conf.getLayer().instantiate(conf, + Collections.singletonList(new ScoreIterationListener(1)), 0, params, true, params.dataType()); + params = l.params(); + l.setParams(params); + assertEquals(params, l.params()); + } + + @Test + public void testOutputLayersRnnForwardPass() { + //Test output layer with RNNs ( + //Expect all outputs etc. to be 2d + int nIn = 2; + int nOut = 5; + int layerSize = 4; + int timeSeriesLength = 6; + int miniBatchSize = 3; + + Random r = new Random(12345L); + INDArray input = Nd4j.zeros(miniBatchSize, nIn, timeSeriesLength); + for (int i = 0; i < miniBatchSize; i++) { + for (int j = 0; j < nIn; j++) { + for (int k = 0; k < timeSeriesLength; k++) { + input.putScalar(new int[] {i, j, k}, r.nextDouble() - 0.5); + } + } + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).list() + .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize) + .dist(new NormalDistribution(0, 1)).activation(Activation.TANH) + .updater(new NoOp()).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut) + .dist(new NormalDistribution(0, 1)) + .updater(new NoOp()).build()) + .inputPreProcessor(1, new RnnToFeedForwardPreProcessor()).build(); + + MultiLayerNetwork mln = new MultiLayerNetwork(conf); + mln.init(); + + INDArray out2d = mln.feedForward(input).get(2); + assertArrayEquals(out2d.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut}); + + INDArray out = mln.output(input); + assertArrayEquals(out.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut}); + + INDArray preout = mln.output(input); + assertArrayEquals(preout.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut}); + + //As above, but for RnnOutputLayer. Expect all activations etc. to be 3d + + MultiLayerConfiguration confRnn = new NeuralNetConfiguration.Builder().seed(12345L).list() + .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize) + .dist(new NormalDistribution(0, 1)).activation(Activation.TANH) + .updater(new NoOp()).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder(LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut) + .dist(new NormalDistribution(0, 1)) + .updater(new NoOp()).build()) + .build(); + + MultiLayerNetwork mlnRnn = new MultiLayerNetwork(confRnn); + mln.init(); + + INDArray out3d = mlnRnn.feedForward(input).get(2); + assertArrayEquals(out3d.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength}); + + INDArray outRnn = mlnRnn.output(input); + assertArrayEquals(outRnn.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength}); + + INDArray preoutRnn = mlnRnn.output(input); + assertArrayEquals(preoutRnn.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength}); + } + + @Test + public void testRnnOutputLayerIncEdgeCases() { + //Basic test + test edge cases: timeSeriesLength==1, miniBatchSize==1, both + int[] tsLength = {5, 1, 5, 1}; + int[] miniBatch = {7, 7, 1, 1}; + int nIn = 3; + int nOut = 6; + int layerSize = 4; + + FeedForwardToRnnPreProcessor proc = new FeedForwardToRnnPreProcessor(); + + for (int t = 0; t < tsLength.length; t++) { + Nd4j.getRandom().setSeed(12345); + int timeSeriesLength = tsLength[t]; + int miniBatchSize = miniBatch[t]; + + Random r = new Random(12345L); + INDArray input = Nd4j.zeros(miniBatchSize, nIn, timeSeriesLength); + for (int i = 0; i < miniBatchSize; i++) { + for (int j = 0; j < nIn; j++) { + for (int k = 0; k < timeSeriesLength; k++) { + input.putScalar(new int[] {i, j, k}, r.nextDouble() - 0.5); + } + } + } + INDArray labels3d = Nd4j.zeros(miniBatchSize, nOut, timeSeriesLength); + for (int i = 0; i < miniBatchSize; i++) { + for (int j = 0; j < timeSeriesLength; j++) { + int idx = r.nextInt(nOut); + labels3d.putScalar(new int[] {i, idx, j}, 1.0f); + } + } + INDArray labels2d = proc.backprop(labels3d, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).list() + .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize) + .dist(new NormalDistribution(0, 1)) + .activation(Activation.TANH).updater(new NoOp()).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut) + .dist(new NormalDistribution(0, 1)) + .updater(new NoOp()).build()) + .inputPreProcessor(1, new RnnToFeedForwardPreProcessor()) + .build(); + + MultiLayerNetwork mln = new MultiLayerNetwork(conf); + mln.init(); + + INDArray out2d = mln.feedForward(input).get(2); + INDArray out3d = proc.preProcess(out2d, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()); + + MultiLayerConfiguration confRnn = new NeuralNetConfiguration.Builder().seed(12345L).list() + .layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize) + .dist(new NormalDistribution(0, 1)) + .activation(Activation.TANH).updater(new NoOp()).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder(LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut) + .dist(new NormalDistribution(0, 1)) + .updater(new NoOp()).build()) + .build(); + + MultiLayerNetwork mlnRnn = new MultiLayerNetwork(confRnn); + mlnRnn.init(); + + INDArray outRnn = mlnRnn.feedForward(input).get(2); + + mln.setLabels(labels2d); + mlnRnn.setLabels(labels3d); + + + mln.computeGradientAndScore(); + mlnRnn.computeGradientAndScore(); + + //score is average over all examples. + //However: OutputLayer version has miniBatch*timeSeriesLength "examples" (after reshaping) + //RnnOutputLayer has miniBatch examples + //Hence: expect difference in scores by factor of timeSeriesLength + double score = mln.score() * timeSeriesLength; + double scoreRNN = mlnRnn.score(); + + assertTrue(!Double.isNaN(score)); + assertTrue(!Double.isNaN(scoreRNN)); + + double relError = Math.abs(score - scoreRNN) / (Math.abs(score) + Math.abs(scoreRNN)); + System.out.println(relError); + assertTrue(relError < 1e-6); + + //Check labels and inputs for output layer: + OutputLayer ol = (OutputLayer) mln.getOutputLayer(); + assertArrayEquals(ol.getInput().shape(), new long[] {miniBatchSize * timeSeriesLength, layerSize}); + assertArrayEquals(ol.getLabels().shape(), new long[] {miniBatchSize * timeSeriesLength, nOut}); + + RnnOutputLayer rnnol = (RnnOutputLayer) mlnRnn.getOutputLayer(); + //assertArrayEquals(rnnol.getInput().shape(),new int[]{miniBatchSize,layerSize,timeSeriesLength}); + //Input may be set by BaseLayer methods. Thus input may end up as reshaped 2d version instead of original 3d version. + //Not ideal, but everything else works. + assertArrayEquals(rnnol.getLabels().shape(), new long[] {miniBatchSize, nOut, timeSeriesLength}); + + //Check shapes of output for both: + assertArrayEquals(out2d.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut}); + + INDArray out = mln.output(input); + assertArrayEquals(out.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut}); + + INDArray preout = mln.output(input); + assertArrayEquals(preout.shape(), new long[] {miniBatchSize * timeSeriesLength, nOut}); + + + INDArray outFFRnn = mlnRnn.feedForward(input).get(2); + assertArrayEquals(outFFRnn.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength}); + + INDArray outRnn2 = mlnRnn.output(input); + assertArrayEquals(outRnn2.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength}); + + INDArray preoutRnn = mlnRnn.output(input); + assertArrayEquals(preoutRnn.shape(), new long[] {miniBatchSize, nOut, timeSeriesLength}); + } + } + + + @Test + public void testCompareRnnOutputRnnLoss(){ + Nd4j.getRandom().setSeed(12345); + + int timeSeriesLength = 4; + int nIn = 5; + int layerSize = 6; + int nOut = 6; + int miniBatchSize = 3; + + MultiLayerConfiguration conf1 = + new NeuralNetConfiguration.Builder().seed(12345L) + .updater(new NoOp()) + .list() + .layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH) + .dist(new NormalDistribution(0, 1.0)) + .updater(new NoOp()).build()) + .layer(new DenseLayer.Builder().nIn(layerSize).nOut(nOut).activation(Activation.IDENTITY).build()) + .layer(new RnnLossLayer.Builder(LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .build()) + .build(); + + MultiLayerNetwork mln = new MultiLayerNetwork(conf1); + mln.init(); + + + MultiLayerConfiguration conf2 = + new NeuralNetConfiguration.Builder().seed(12345L) + .updater(new NoOp()) + .list() + .layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH) + .dist(new NormalDistribution(0, 1.0)) + .updater(new NoOp()).build()) + .layer(new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder(LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .nIn(layerSize).nOut(nOut) + .build()) + .build(); + + MultiLayerNetwork mln2 = new MultiLayerNetwork(conf2); + mln2.init(); + + mln2.setParams(mln.params()); + + INDArray in = Nd4j.rand(new int[]{miniBatchSize, nIn, timeSeriesLength}); + + INDArray out1 = mln.output(in); + INDArray out2 = mln.output(in); + + assertEquals(out1, out2); + + Random r = new Random(12345); + INDArray labels = Nd4j.create(miniBatchSize, nOut, timeSeriesLength); + for( int i=0; i= 0 && max <= 1.0); + + INDArray sum = out.sum(1); + assertEquals(Nd4j.ones(DataType.FLOAT,2,4,5), sum); + } + + @Test + public void testOutputLayerDefaults(){ + + new NeuralNetConfiguration.Builder().list() + .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder().nIn(10).nOut(10).build()) + .build(); + + new NeuralNetConfiguration.Builder().list() + .layer(new org.deeplearning4j.nn.conf.layers.LossLayer.Builder().build()) + .build(); + + new NeuralNetConfiguration.Builder().list() + .layer(new org.deeplearning4j.nn.conf.layers.CnnLossLayer.Builder().build()) + .build(); + + new NeuralNetConfiguration.Builder().list() + .layer(new org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer.Builder().build()) + .build(); + + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/RepeatVectorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/RepeatVectorTest.java new file mode 100644 index 000000000..2aa98575e --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/RepeatVectorTest.java @@ -0,0 +1,72 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.misc.RepeatVector; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RepeatVectorTest extends BaseDL4JTest { + + private int REPEAT = 4; + + + private Layer getRepeatVectorLayer() { + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().seed(123) + .dataType(DataType.DOUBLE) + .layer(new RepeatVector.Builder(REPEAT).build()).build(); + return conf.getLayer().instantiate(conf, null, 0, + null, false, DataType.DOUBLE); + } + + @Test + public void testRepeatVector() { + + double[] arr = new double[] {1., 2., 3., 1., 2., 3., 1., 2., 3., 1., 2., 3.}; + INDArray expectedOut = Nd4j.create(arr, new long[] {1, 3, REPEAT}, 'f'); + INDArray input = Nd4j.create(new double[] {1., 2., 3.}, new long[] {1, 3}); + Layer layer = getRepeatVectorLayer(); + + INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + assertTrue(Arrays.equals(expectedOut.shape(), output.shape())); + assertEquals(expectedOut, output); + + INDArray epsilon = Nd4j.ones(1,3,4); + + Pair out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); + INDArray outEpsilon = out.getSecond(); + INDArray expectedEpsilon = Nd4j.create(new double[] {4., 4., 4.}, new long[] {1, 3}); + assertEquals(expectedEpsilon, outEpsilon); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/SeedTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/SeedTest.java similarity index 80% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/SeedTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/SeedTest.java index acc1dfe10..a9b3ee532 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/SeedTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/SeedTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.layers; import org.deeplearning4j.BaseDL4JTest; @@ -24,48 +25,45 @@ import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; import org.deeplearning4j.nn.api.Layer; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.AutoEncoder; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** */ -@DisplayName("Seed Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.RNG) -class SeedTest extends BaseDL4JTest { + +public class SeedTest extends BaseDL4JTest { private DataSetIterator irisIter = new IrisDataSetIterator(50, 50); - private DataSet data = irisIter.next(); + @Test - @DisplayName("Test Auto Encoder Seed") - void testAutoEncoderSeed() { - AutoEncoder layerType = new AutoEncoder.Builder().nIn(4).nOut(3).corruptionLevel(0.0).activation(Activation.SIGMOID).build(); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(layerType).seed(123).build(); + public void testAutoEncoderSeed() { + AutoEncoder layerType = new AutoEncoder.Builder().nIn(4).nOut(3).corruptionLevel(0.0) + .activation(Activation.SIGMOID).build(); + + NeuralNetConfiguration conf = + new NeuralNetConfiguration.Builder().layer(layerType).seed(123).build(); + long numParams = conf.getLayer().initializer().numParams(conf); INDArray params = Nd4j.create(1, numParams); Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); layer.setBackpropGradientsViewArray(Nd4j.create(1, numParams)); layer.fit(data.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); + layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); double score = layer.score(); INDArray parameters = layer.params(); layer.setParams(parameters); layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); + double score2 = layer.score(); assertEquals(parameters, layer.params()); assertEquals(score, score2, 1e-4); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/TestDropout.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/TestDropout.java new file mode 100644 index 000000000..67f66fb21 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/TestDropout.java @@ -0,0 +1,118 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.UniformDistribution; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.iter.NdIndexIterator; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.lang.reflect.Field; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestDropout extends BaseDL4JTest { + + @Test + public void testDropoutSimple() throws Exception { + //Testing dropout with a single layer + //Layer input: values should be set to either 0.0 or 2.0x original value + + int nIn = 8; + int nOut = 8; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .updater(new Sgd()) + .dropOut(0.5).list() + .layer(0, new OutputLayer.Builder().activation(Activation.IDENTITY) + .lossFunction(LossFunctions.LossFunction.MSE).nIn(nIn).nOut(nOut) + .weightInit(WeightInit.XAVIER).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + net.getLayer(0).getParam("W").assign(Nd4j.eye(nIn)); + + int nTests = 15; + + Nd4j.getRandom().setSeed(12345); + int noDropoutCount = 0; + for (int i = 0; i < nTests; i++) { + INDArray in = Nd4j.rand(1, nIn); + INDArray out = Nd4j.rand(1, nOut); + INDArray inCopy = in.dup(); + + List l = net.feedForward(in, true); + + INDArray postDropout = l.get(l.size() - 1); + //Dropout occurred. Expect inputs to be either scaled 2x original, or set to 0.0 (with dropout = 0.5) + for (int j = 0; j < inCopy.length(); j++) { + double origValue = inCopy.getDouble(j); + double doValue = postDropout.getDouble(j); + if (doValue > 0.0) { + //Input was kept -> should be scaled by factor of (1.0/0.5 = 2) + assertEquals(origValue * 2.0, doValue, 0.0001); + } + } + + //Do forward pass + //(1) ensure dropout ISN'T being applied for forward pass at test time + //(2) ensure dropout ISN'T being applied for test time scoring + //If dropout is applied at test time: outputs + score will differ between passes + INDArray in2 = Nd4j.rand(1, nIn); + INDArray out2 = Nd4j.rand(1, nOut); + INDArray outTest1 = net.output(in2, false); + INDArray outTest2 = net.output(in2, false); + INDArray outTest3 = net.output(in2, false); + assertEquals(outTest1, outTest2); + assertEquals(outTest1, outTest3); + + double score1 = net.score(new DataSet(in2, out2), false); + double score2 = net.score(new DataSet(in2, out2), false); + double score3 = net.score(new DataSet(in2, out2), false); + assertEquals(score1, score2, 0.0); + assertEquals(score1, score3, 0.0); + } + + if (noDropoutCount >= nTests / 3) { + //at 0.5 dropout ratio and more than a few inputs, expect only a very small number of instances where + //no dropout occurs, just due to random chance + fail("Too many instances of dropout not being applied"); + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsNetMNISTTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsNetMNISTTest.java new file mode 100644 index 000000000..18c285baf --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsNetMNISTTest.java @@ -0,0 +1,101 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.capsule; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ActivationLayer; +import org.deeplearning4j.nn.conf.layers.CapsuleLayer; +import org.deeplearning4j.nn.conf.layers.CapsuleStrengthLayer; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.LossLayer; +import org.deeplearning4j.nn.conf.layers.PrimaryCapsules; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.linalg.activations.impl.ActivationSoftmax; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.lossfunctions.impl.LossNegativeLogLikelihood; + +////@Ignore("AB - ignored due to excessive runtime. Keep for manual debugging when required") +@Tag("long-running") +public class CapsNetMNISTTest extends BaseDL4JTest { + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @Test + public void testCapsNetOnMNIST(){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(123) + .updater(new Adam()) + .list() + .layer(new ConvolutionLayer.Builder() + .nOut(16) + .kernelSize(9, 9) + .stride(3, 3) + .build()) + .layer(new PrimaryCapsules.Builder(8, 8) + .kernelSize(7, 7) + .stride(2, 2) + .build()) + .layer(new CapsuleLayer.Builder(10, 16, 3).build()) + .layer(new CapsuleStrengthLayer.Builder().build()) + .layer(new ActivationLayer.Builder(new ActivationSoftmax()).build()) + .layer(new LossLayer.Builder(new LossNegativeLogLikelihood()).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) + .build(); + + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + int rngSeed = 12345; + try { + MnistDataSetIterator mnistTrain = new MnistDataSetIterator(64, true, rngSeed); + MnistDataSetIterator mnistTest = new MnistDataSetIterator(64, false, rngSeed); + + for (int i = 0; i < 2; i++) { + model.fit(mnistTrain); + } + + Evaluation eval = model.evaluate(mnistTest); + + assertTrue(eval.accuracy() > 0.95,"Accuracy not over 95%"); + assertTrue(eval.precision() > 0.95, "Precision not over 95%"); + assertTrue(eval.recall() > 0.95, "Recall not over 95%"); + assertTrue(eval.f1() > 0.95, "F1-score not over 95%"); + + } catch (IOException e){ + System.out.println("Could not load MNIST."); + } + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleLayerTest.java similarity index 78% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleLayerTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleLayerTest.java index e7f9f369c..70e503c42 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleLayerTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleLayerTest.java @@ -17,76 +17,84 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.layers.capsule; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; + import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.conf.MultiLayerConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.CapsuleLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Capsule Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class CapsuleLayerTest extends BaseDL4JTest { +public class CapsuleLayerTest extends BaseDL4JTest { @Override - public DataType getDataType() { + public DataType getDataType(){ return DataType.FLOAT; } @Test - @DisplayName("Test Output Type") - void testOutputType() { + public void testOutputType(){ CapsuleLayer layer = new CapsuleLayer.Builder(10, 16, 5).build(); + InputType in1 = InputType.recurrent(5, 8); + assertEquals(InputType.recurrent(10, 16), layer.getOutputType(0, in1)); } @Test - @DisplayName("Test Input Type") - void testInputType() { + public void testInputType(){ CapsuleLayer layer = new CapsuleLayer.Builder(10, 16, 5).build(); + InputType in1 = InputType.recurrent(5, 8); + layer.setNIn(in1, true); + assertEquals(5, layer.getInputCapsules()); assertEquals(8, layer.getInputCapsuleDimensions()); } @Test - @DisplayName("Test Config") - void testConfig() { + public void testConfig(){ CapsuleLayer layer1 = new CapsuleLayer.Builder(10, 16, 5).build(); + assertEquals(10, layer1.getCapsules()); assertEquals(16, layer1.getCapsuleDimensions()); assertEquals(5, layer1.getRoutings()); assertFalse(layer1.isHasBias()); + CapsuleLayer layer2 = new CapsuleLayer.Builder(10, 16, 5).hasBias(true).build(); + assertTrue(layer2.isHasBias()); + } @Test - @DisplayName("Test Layer") - void testLayer() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).list().layer(new CapsuleLayer.Builder(10, 16, 3).build()).setInputType(InputType.recurrent(10, 8)).build(); + public void testLayer(){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(123) + .list() + .layer(new CapsuleLayer.Builder(10, 16, 3).build()) + .setInputType(InputType.recurrent(10, 8)) + .build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); model.init(); + INDArray emptyFeatures = Nd4j.zeros(64, 10, 8); + long[] shape = model.output(emptyFeatures).shape(); - assertArrayEquals(new long[] { 64, 10, 16 }, shape); + + assertArrayEquals(new long[]{64, 10, 16}, shape); } } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleStrengthLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleStrengthLayerTest.java similarity index 75% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleStrengthLayerTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleStrengthLayerTest.java index a5b59200d..fac472d68 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleStrengthLayerTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsuleStrengthLayerTest.java @@ -17,52 +17,55 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.layers.capsule; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; + import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.conf.MultiLayerConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.CapsuleStrengthLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Capsule Strength Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class CapsuleStrengthLayerTest extends BaseDL4JTest { +public class CapsuleStrengthLayerTest extends BaseDL4JTest { @Override - public DataType getDataType() { + public DataType getDataType(){ return DataType.FLOAT; } @Test - @DisplayName("Test Output Type") - void testOutputType() { + public void testOutputType(){ CapsuleStrengthLayer layer = new CapsuleStrengthLayer.Builder().build(); + InputType in1 = InputType.recurrent(5, 8); + assertEquals(InputType.feedForward(5), layer.getOutputType(0, in1)); } @Test - @DisplayName("Test Layer") - void testLayer() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).list().layer(new CapsuleStrengthLayer.Builder().build()).setInputType(InputType.recurrent(5, 8)).build(); + public void testLayer(){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(123) + .list() + .layer(new CapsuleStrengthLayer.Builder().build()) + .setInputType(InputType.recurrent(5, 8)) + .build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); model.init(); + INDArray emptyFeatures = Nd4j.zeros(64, 5, 10); + long[] shape = model.output(emptyFeatures).shape(); - assertArrayEquals(new long[] { 64, 5 }, shape); + + assertArrayEquals(new long[]{64, 5}, shape); } } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/PrimaryCapsulesTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/PrimaryCapsulesTest.java new file mode 100644 index 000000000..5840ec85f --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/capsule/PrimaryCapsulesTest.java @@ -0,0 +1,129 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.capsule; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.PrimaryCapsules; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +public class PrimaryCapsulesTest extends BaseDL4JTest { + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @Test + public void testOutputType(){ + PrimaryCapsules layer = new PrimaryCapsules.Builder(8, 8) + .kernelSize(7, 7) + .stride(2, 2) + .build(); + + + InputType in1 = InputType.convolutional(7, 7, 16); + assertEquals(InputType.recurrent(8, 8), layer.getOutputType(0, in1)); + + } + + @Test + public void testInputType(){ + PrimaryCapsules layer = new PrimaryCapsules.Builder(8, 8) + .kernelSize(7, 7) + .stride(2, 2) + .build(); + InputType in1 = InputType.convolutional(7, 7, 16); + + + layer.setNIn(in1, true); + + assertEquals(8, layer.getCapsules()); + assertEquals(8, layer.getCapsuleDimensions()); + } + + @Test + public void testConfig(){ + PrimaryCapsules layer1 = new PrimaryCapsules.Builder(8, 10) + .kernelSize(5, 5) + .stride(4, 4) + .useLeakyReLU(0.5) + .build(); + + assertEquals(8, layer1.getCapsuleDimensions()); + assertEquals(10, layer1.getChannels()); + assertArrayEquals(new int[]{5, 5}, layer1.getKernelSize()); + assertArrayEquals(new int[]{4, 4}, layer1.getStride()); + assertArrayEquals(new int[]{0, 0}, layer1.getPadding()); + assertArrayEquals(new int[]{1, 1}, layer1.getDilation()); + assertTrue(layer1.isUseRelu()); + assertEquals(0.5, layer1.getLeak(), 0.001); + + PrimaryCapsules layer2 = new PrimaryCapsules.Builder(8, 10) + .kernelSize(5, 5) + .stride(4, 4) + .build(); + assertFalse(layer2.isUseRelu()); + + PrimaryCapsules layer3 = new PrimaryCapsules.Builder(8, 10) + .kernelSize(5, 5) + .stride(4, 4) + .useReLU() + .build(); + assertTrue(layer3.isUseRelu()); + assertEquals(0, layer3.getLeak(), 0.001); + + } + + @Test + public void testLayer(){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(123) + .list() + .layer(new PrimaryCapsules.Builder(8, 10) + .kernelSize(5, 5) + .stride(4, 4) + .useLeakyReLU(0.5) + .build()) + .setInputType(InputType.convolutional(20, 20, 20)) + .build(); + + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + INDArray emptyFeatures = Nd4j.zeros(64, 20, 20, 20); + + long[] shape = model.output(emptyFeatures).shape(); + + assertArrayEquals(new long[]{64, 160, 8}, shape); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvDataFormatTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvDataFormatTests.java new file mode 100644 index 000000000..7c07bfeb2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvDataFormatTests.java @@ -0,0 +1,1036 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ +package org.deeplearning4j.nn.layers.convolution; + +import lombok.*; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.exception.DL4JInvalidInputException; +import org.deeplearning4j.nn.api.MaskState; +import org.deeplearning4j.nn.conf.*; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.CnnLossLayer; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.ZeroPaddingLayer; +import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; +import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.ComposableInputPreProcessor; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.workspace.ArrayType; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.deeplearning4j.util.ConvolutionUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Timeout(300) +public class ConvDataFormatTests extends BaseDL4JTest { + + private final DataType dataType; + + public ConvDataFormatTests(DataType dataType){ + this.dataType = dataType; + } + + public static Object[] params(){ + return new DataType[]{DataType.FLOAT, DataType.DOUBLE}; + } + + @Test + public void testConv2d() { + try { + for (boolean helpers : new boolean[]{false, true}) { + for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getConv2dNet(CNN2DFormat.NCHW, true, cm)) + .net2(getConv2dNet(CNN2DFormat.NCHW, false, cm)) + .net3(getConv2dNet(CNN2DFormat.NHWC, true, cm)) + .net4(getConv2dNet(CNN2DFormat.NHWC, false, cm)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testSubsampling2d() { + try { + for (boolean helpers : new boolean[]{false, true}) { + for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getSubsampling2dNet(CNN2DFormat.NCHW, true, cm)) + .net2(getSubsampling2dNet(CNN2DFormat.NCHW, false, cm)) + .net3(getSubsampling2dNet(CNN2DFormat.NHWC, true, cm)) + .net4(getSubsampling2dNet(CNN2DFormat.NHWC, false, cm)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testDepthwiseConv2d() { + try { + for (boolean helpers : new boolean[]{false, true}) { + for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getDepthwiseConv2dNet(CNN2DFormat.NCHW, true, cm)) + .net2(getDepthwiseConv2dNet(CNN2DFormat.NCHW, false, cm)) + .net3(getDepthwiseConv2dNet(CNN2DFormat.NHWC, true, cm)) + .net4(getDepthwiseConv2dNet(CNN2DFormat.NHWC, false, cm)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testSeparableConv2d() { + try { + for (boolean helpers : new boolean[]{false, true}) { + for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getSeparableConv2dNet(CNN2DFormat.NCHW, true, cm)) + .net2(getSeparableConv2dNet(CNN2DFormat.NCHW, false, cm)) + .net3(getSeparableConv2dNet(CNN2DFormat.NHWC, true, cm)) + .net4(getSeparableConv2dNet(CNN2DFormat.NHWC, false, cm)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testDeconv2d() { + try { + for (boolean helpers : new boolean[]{false, true}) { + for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getDeconv2DNet2dNet(CNN2DFormat.NCHW, true, cm)) + .net2(getDeconv2DNet2dNet(CNN2DFormat.NCHW, false, cm)) + .net3(getDeconv2DNet2dNet(CNN2DFormat.NHWC, true, cm)) + .net4(getDeconv2DNet2dNet(CNN2DFormat.NHWC, false, cm)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testLRN() { + try { + for (boolean helpers : new boolean[]{false, true}) { + for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getLrnLayer(CNN2DFormat.NCHW, true, cm)) + .net2(getLrnLayer(CNN2DFormat.NCHW, false, cm)) + .net3(getLrnLayer(CNN2DFormat.NHWC, true, cm)) + .net4(getLrnLayer(CNN2DFormat.NHWC, false, cm)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testZeroPaddingLayer(){ + try { + for (boolean helpers : new boolean[]{false, true}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers" : "No helpers"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getZeroPaddingNet(CNN2DFormat.NCHW, true)) + .net2(getZeroPaddingNet(CNN2DFormat.NCHW, false)) + .net3(getZeroPaddingNet(CNN2DFormat.NHWC, true)) + .net4(getZeroPaddingNet(CNN2DFormat.NHWC, false)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testCropping2DLayer(){ + try { + for (boolean helpers : new boolean[]{false, true}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers" : "No helpers"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getCropping2dNet(CNN2DFormat.NCHW, true)) + .net2(getCropping2dNet(CNN2DFormat.NCHW, false)) + .net3(getCropping2dNet(CNN2DFormat.NHWC, true)) + .net4(getCropping2dNet(CNN2DFormat.NHWC, false)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testUpsampling2d(){ + try { + for (boolean helpers : new boolean[]{false, true}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers" : "No helpers"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getUpsamplingNet(CNN2DFormat.NCHW, true)) + .net2(getUpsamplingNet(CNN2DFormat.NCHW, false)) + .net3(getUpsamplingNet(CNN2DFormat.NHWC, true)) + .net4(getUpsamplingNet(CNN2DFormat.NHWC, false)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testBatchNormNet(){ + try { + for(boolean useLogStd : new boolean[]{true, false}) { + for (boolean helpers : new boolean[]{false, true}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = (helpers ? "With helpers" : "No helpers") + " - " + (useLogStd ? "logstd" : "std"); + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getBatchNormNet(useLogStd, CNN2DFormat.NCHW, true)) + .net2(getBatchNormNet(useLogStd, CNN2DFormat.NCHW, false)) + .net3(getBatchNormNet(useLogStd, CNN2DFormat.NHWC, true)) + .net4(getBatchNormNet(useLogStd, CNN2DFormat.NHWC, false)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testCnnLossLayer() { + try { + for (boolean helpers : new boolean[]{false, true}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers" : "No helpers"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labelsNHWC = TestUtils.randomOneHot(this.dataType,2*6*6, 3); + labelsNHWC = labelsNHWC.reshape(2,6,6,3); + INDArray labelsNCHW = labelsNHWC.permute(0,3,1,2).dup(); + + + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getCnnLossNet(CNN2DFormat.NCHW, true, ConvolutionMode.Same)) + .net2(getCnnLossNet(CNN2DFormat.NCHW, false, ConvolutionMode.Same)) + .net3(getCnnLossNet(CNN2DFormat.NHWC, true, ConvolutionMode.Same)) + .net4(getCnnLossNet(CNN2DFormat.NHWC, false, ConvolutionMode.Same)) + .inNCHW(inNCHW) + .labelsNCHW(labelsNCHW) + .labelsNHWC(labelsNHWC) + .testLayerIdx(1) + .nhwcOutput(true) + .build(); + + testHelper(tc); + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testSpaceToDepthNet(){ + try { + for (boolean helpers : new boolean[]{false, true}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers" : "No helpers"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getSpaceToDepthNet(CNN2DFormat.NCHW, true)) + .net2(getSpaceToDepthNet(CNN2DFormat.NCHW, false)) + .net3(getSpaceToDepthNet(CNN2DFormat.NHWC, true)) + .net4(getSpaceToDepthNet(CNN2DFormat.NHWC, false)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testSpaceToBatchNet(){ + try { + for (boolean helpers : new boolean[]{false, true}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers" : "No helpers"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 16, 16); + INDArray labels = TestUtils.randomOneHot(8, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getSpaceToBatchNet(CNN2DFormat.NCHW, true)) + .net2(getSpaceToBatchNet(CNN2DFormat.NCHW, false)) + .net3(getSpaceToBatchNet(CNN2DFormat.NHWC, true)) + .net4(getSpaceToBatchNet(CNN2DFormat.NHWC, false)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + @Test + public void testLocallyConnected() { + try { + for (boolean helpers : new boolean[]{false, true}) { + for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getLocallyConnectedNet(CNN2DFormat.NCHW, true, cm)) + .net2(getLocallyConnectedNet(CNN2DFormat.NCHW, false, cm)) + .net3(getLocallyConnectedNet(CNN2DFormat.NHWC, true, cm)) + .net4(getLocallyConnectedNet(CNN2DFormat.NHWC, false, cm)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + + @Test + public void testGlobalPooling() { + try { + for (boolean helpers : new boolean[]{false, true}) { + for (PoolingType pt : PoolingType.values()) { + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = helpers ? "With helpers (" + pt + ")" : "No helpers (" + pt + ")"; + System.out.println(" --- " + msg + " ---"); + + INDArray inNCHW = Nd4j.rand(this.dataType, 2, 3, 12, 12); + INDArray labels = TestUtils.randomOneHot(2, 10); + + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getGlobalPoolingNet(CNN2DFormat.NCHW, pt, true)) + .net2(getGlobalPoolingNet(CNN2DFormat.NCHW, pt, false)) + .net3(getGlobalPoolingNet(CNN2DFormat.NHWC, pt, true)) + .net4(getGlobalPoolingNet(CNN2DFormat.NHWC, pt, false)) + .inNCHW(inNCHW) + .labelsNCHW(labels) + .labelsNHWC(labels) + .testLayerIdx(1) + .build(); + + testHelper(tc); + } + } + } finally { + Nd4j.getEnvironment().allowHelpers(true); + } + } + + private MultiLayerNetwork getConv2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { + if (setOnLayerAlso) { + return getNetWithLayer(new ConvolutionLayer.Builder() + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .dataFormat(format) + .nOut(3) + .helperAllowFallback(false) + .build(), format, cm, null); + } else { + return getNetWithLayer(new ConvolutionLayer.Builder() + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .nOut(3) + .helperAllowFallback(false) + .build(), format, cm, null); + } + } + + private MultiLayerNetwork getSubsampling2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { + if (setOnLayerAlso) { + return getNetWithLayer(new SubsamplingLayer.Builder() + .kernelSize(2, 2) + .stride(1, 1) + .dataFormat(format) + .helperAllowFallback(false) + .build(), format, cm, null); + } else { + return getNetWithLayer(new SubsamplingLayer.Builder() + .kernelSize(2, 2) + .stride(1, 1) + .helperAllowFallback(false) + .build(), format, cm, null); + } + } + + private MultiLayerNetwork getSeparableConv2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { + if (setOnLayerAlso) { + return getNetWithLayer(new SeparableConvolution2D.Builder() + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .dataFormat(format) + .nOut(3) + .helperAllowFallback(false) + .build(), format, cm, null); + } else { + return getNetWithLayer(new SeparableConvolution2D.Builder() + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .nOut(3) + .helperAllowFallback(false) + .build(), format, cm, null); + } + } + + private MultiLayerNetwork getDepthwiseConv2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { + if (setOnLayerAlso) { + return getNetWithLayer(new DepthwiseConvolution2D.Builder() + .depthMultiplier(2) + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .dataFormat(format) + .nOut(3) + .helperAllowFallback(false) + .build(), format, cm, null); + } else { + return getNetWithLayer(new DepthwiseConvolution2D.Builder() + .depthMultiplier(2) + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .nOut(3) + .helperAllowFallback(false) + .build(), format, cm, null); + } + } + + private MultiLayerNetwork getLrnLayer(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { + if (setOnLayerAlso) { + return getNetWithLayer(new LocalResponseNormalization.Builder() + .dataFormat(format) + .helperAllowFallback(false) + .build(), format, cm, null); + } else { + return getNetWithLayer(new LocalResponseNormalization.Builder() + .helperAllowFallback(false) + .build(), format, cm, null); + } + } + + private MultiLayerNetwork getZeroPaddingNet(CNN2DFormat format, boolean setOnLayerAlso) { + if (setOnLayerAlso) { + return getNetWithLayer(new ZeroPaddingLayer.Builder(2,2) + .dataFormat(format).build(), format, ConvolutionMode.Same, null); + } else { + return getNetWithLayer(new ZeroPaddingLayer.Builder(2,2).build(), + format, ConvolutionMode.Same, null); + } + } + + private MultiLayerNetwork getCropping2dNet(CNN2DFormat format, boolean setOnLayerAlso) { + if (setOnLayerAlso) { + return getNetWithLayer(new Cropping2D.Builder(2,2) + .dataFormat(format).build(), format, ConvolutionMode.Same, null); + } else { + return getNetWithLayer(new Cropping2D.Builder(2,2) + .build(), format, ConvolutionMode.Same, null); + } + } + + private MultiLayerNetwork getUpsamplingNet(CNN2DFormat format, boolean setOnLayerAlso) { + if (setOnLayerAlso) { + return getNetWithLayer(new Upsampling2D.Builder(2) + .dataFormat(format).build(), format, ConvolutionMode.Same, null); + } else { + return getNetWithLayer(new Upsampling2D.Builder(2) + .build(), format, ConvolutionMode.Same, null); + } + } + + private MultiLayerNetwork getDeconv2DNet2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { + if (setOnLayerAlso) { + return getNetWithLayer(new Deconvolution2D.Builder().nOut(2) + .activation(Activation.TANH) + .kernelSize(2,2) + .dataFormat(format) + .stride(2,2) + .build(), format, cm, null); + } else { + return getNetWithLayer(new Deconvolution2D.Builder().nOut(2) + .activation(Activation.TANH) + .kernelSize(2,2) + .dataFormat(format) + .stride(2,2) + .build(), format, cm, null); + } + } + + private MultiLayerNetwork getBatchNormNet(boolean logStdev, CNN2DFormat format, boolean setOnLayerAlso) { + if (setOnLayerAlso) { + return getNetWithLayer(new BatchNormalization.Builder() + .useLogStd(logStdev) + .dataFormat(format) + .helperAllowFallback(false) + .nOut(3).build(), format, ConvolutionMode.Same, null); + } else { + return getNetWithLayer(new BatchNormalization.Builder() + .useLogStd(logStdev) + .helperAllowFallback(false) + .nOut(3).build(), format, ConvolutionMode.Same, null); + } + } + + private MultiLayerNetwork getSpaceToDepthNet(CNN2DFormat format, boolean setOnLayerAlso) { + if (setOnLayerAlso) { + return getNetWithLayer(new SpaceToDepthLayer.Builder() + .blocks(2) + .dataFormat(format) + .build(), format, ConvolutionMode.Same, null); + } else { + return getNetWithLayer(new SpaceToDepthLayer.Builder() + .blocks(2) + .build(), format, ConvolutionMode.Same, null); + } + } + + private MultiLayerNetwork getSpaceToBatchNet(CNN2DFormat format, boolean setOnLayerAlso) { + if (setOnLayerAlso) { + return getNetWithLayer(new SpaceToBatchLayer.Builder() + .blocks(2, 2) + .dataFormat(format) + .build(), format, ConvolutionMode.Same, InputType.convolutional(16, 16, 3, format)); + } else { + return getNetWithLayer(new SpaceToBatchLayer.Builder() + .blocks(2, 2) + .build(), format, ConvolutionMode.Same, InputType.convolutional(16, 16, 3, format)); + } + } + + private MultiLayerNetwork getLocallyConnectedNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { + if (setOnLayerAlso) { + return getNetWithLayer(new LocallyConnected2D.Builder() + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .dataFormat(format) + .nOut(3) + .build(), format, cm, null); + } else { + return getNetWithLayer(new LocallyConnected2D.Builder() + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .nOut(3) + .build(), format, cm, null); + } + } + + private MultiLayerNetwork getNetWithLayer(Layer layer, CNN2DFormat format, ConvolutionMode cm, InputType inputType) { + NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder() + .dataType(this.dataType) + .seed(12345) + .convolutionMode(cm) + .list() + .layer(new ConvolutionLayer.Builder() + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .nOut(3) + .helperAllowFallback(false) + .build()) + .layer(layer) + .layer(new OutputLayer.Builder().nOut(10) + .activation(Activation.SOFTMAX).build()) + .setInputType(inputType != null ? inputType : InputType.convolutional(12, 12, 3, format)); + + if(format == CNN2DFormat.NHWC && !(layer instanceof GlobalPoolingLayer)){ + //Add a preprocessor due to the differences in how NHWC and NCHW activations are flattened + //DL4J's flattening behaviour matches Keras (hence TF) for import compatibility + builder.inputPreProcessor(2, new ComposableInputPreProcessor(new NHWCToNCHWPreprocessor(), new CnnToFeedForwardPreProcessor())); + } + + MultiLayerNetwork net = new MultiLayerNetwork(builder.build()); + net.init(); + return net; + } + + private MultiLayerNetwork getGlobalPoolingNet(CNN2DFormat format, PoolingType pt, boolean setOnLayerAlso) { + if (setOnLayerAlso) { + return getNetWithLayer(new GlobalPoolingLayer.Builder(pt) + .poolingDimensions(format == CNN2DFormat.NCHW ? new int[]{2,3} : new int[]{1,2}) + .build(), format, ConvolutionMode.Same, null); + } else { + return getNetWithLayer(new GlobalPoolingLayer.Builder(pt) + .build(), format, ConvolutionMode.Same, null); + } + } + + private MultiLayerNetwork getCnnLossNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm){ + NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder() + .seed(12345) + .convolutionMode(cm) + .list() + .layer(new ConvolutionLayer.Builder() + .kernelSize(3, 3) + .stride(2, 2) + .activation(Activation.TANH) + .dataFormat(format) + .nOut(3) + .helperAllowFallback(false) + .build()); + if(setOnLayerAlso){ + builder.layer(new CnnLossLayer.Builder() + .format(format).activation(Activation.SOFTMAX).build()); + } else { + builder.layer(new CnnLossLayer.Builder() + .activation(Activation.SOFTMAX).build()); + } + + builder.setInputType(InputType.convolutional(12, 12, 3, format)); + + MultiLayerNetwork net = new MultiLayerNetwork(builder.build()); + net.init(); + return net; + } + + @AllArgsConstructor + @Data + @NoArgsConstructor + @Builder + private static class TestCase { + private String msg; + private MultiLayerNetwork net1; + private MultiLayerNetwork net2; + private MultiLayerNetwork net3; + private MultiLayerNetwork net4; + private INDArray inNCHW; + private INDArray labelsNCHW; + private INDArray labelsNHWC; + private int testLayerIdx; + private boolean nhwcOutput; + } + + public static void testHelper(TestCase tc) { + + tc.net2.params().assign(tc.net1.params()); + tc.net3.params().assign(tc.net1.params()); + tc.net4.params().assign(tc.net1.params()); + + //Test forward pass: + INDArray inNCHW = tc.inNCHW; + INDArray inNHWC = tc.inNCHW.permute(0, 2, 3, 1).dup(); + + INDArray l0_1 = tc.net1.feedForward(inNCHW).get(tc.testLayerIdx + 1); + INDArray l0_2 = tc.net2.feedForward(inNCHW).get(tc.testLayerIdx + 1); + INDArray l0_3 = tc.net3.feedForward(inNHWC).get(tc.testLayerIdx + 1); + INDArray l0_4 = tc.net4.feedForward(inNHWC).get(tc.testLayerIdx + 1); + + assertEquals( l0_1, l0_2, tc.msg); + if(l0_1.rank() == 4) { + assertEquals( l0_1, l0_3.permute(0, 3, 1, 2), tc.msg); + assertEquals(l0_1, l0_4.permute(0, 3, 1, 2), tc.msg); + } else { + assertEquals(l0_1, l0_3, tc.msg); + assertEquals(l0_1, l0_4, tc.msg); + } + + + INDArray out1 = tc.net1.output(inNCHW); + INDArray out2 = tc.net2.output(inNCHW); + INDArray out3 = tc.net3.output(inNHWC); + INDArray out4 = tc.net4.output(inNHWC); + + assertEquals(out1, out2, tc.msg); + if(!tc.nhwcOutput) { + assertEquals(out1, out3, tc.msg); + assertEquals(out1, out4, tc.msg); + } else { + assertEquals(out1, out3.permute(0,3,1,2), tc.msg); //NHWC to NCHW + assertEquals(out1, out4.permute(0,3,1,2), tc.msg); + } + + //Test backprop + Pair p1 = tc.net1.calculateGradients(inNCHW, tc.labelsNCHW, null, null); + Pair p2 = tc.net2.calculateGradients(inNCHW, tc.labelsNCHW, null, null); + Pair p3 = tc.net3.calculateGradients(inNHWC, tc.labelsNHWC, null, null); + Pair p4 = tc.net4.calculateGradients(inNHWC, tc.labelsNHWC, null, null); + + //Inpput gradients + assertEquals(p1.getSecond(), p2.getSecond(), tc.msg); + assertEquals(p1.getSecond(), p3.getSecond().permute(0,3,1,2), tc.msg); //Input gradients for NHWC input are also in NHWC format + assertEquals(p1.getSecond(), p4.getSecond().permute(0,3,1,2), tc.msg); + + List diff12 = differentGrads(p1.getFirst(), p2.getFirst()); + List diff13 = differentGrads(p1.getFirst(), p3.getFirst()); + List diff14 = differentGrads(p1.getFirst(), p4.getFirst()); + assertEquals( 0, diff12.size(), tc.msg + " " + diff12); + assertEquals(0, diff13.size(), tc.msg + " " + diff13); + assertEquals(0, diff14.size(), tc.msg + " " + diff14); + + assertEquals(p1.getFirst().gradientForVariable(), p2.getFirst().gradientForVariable(), tc.msg); + assertEquals(p1.getFirst().gradientForVariable(), p3.getFirst().gradientForVariable(), tc.msg); + assertEquals(p1.getFirst().gradientForVariable(), p4.getFirst().gradientForVariable(), tc.msg); + + tc.net1.fit(inNCHW, tc.labelsNCHW); + tc.net2.fit(inNCHW, tc.labelsNCHW); + tc.net3.fit(inNHWC, tc.labelsNHWC); + tc.net4.fit(inNHWC, tc.labelsNHWC); + + assertEquals(tc.net1.params(), tc.net2.params(), tc.msg); + assertEquals(tc.net1.params(), tc.net3.params(), tc.msg); + assertEquals(tc.net1.params(), tc.net4.params(), tc.msg); + + //Test serialization + MultiLayerNetwork net1a = TestUtils.testModelSerialization(tc.net1); + MultiLayerNetwork net2a = TestUtils.testModelSerialization(tc.net2); + MultiLayerNetwork net3a = TestUtils.testModelSerialization(tc.net3); + MultiLayerNetwork net4a = TestUtils.testModelSerialization(tc.net4); + + out1 = tc.net1.output(inNCHW); + assertEquals(out1, net1a.output(inNCHW), tc.msg); + assertEquals(out1, net2a.output(inNCHW), tc.msg); + if(!tc.nhwcOutput) { + assertEquals(out1, net3a.output(inNHWC), tc.msg); + assertEquals(out1, net4a.output(inNHWC), tc.msg); + } else { + assertEquals(out1, net3a.output(inNHWC).permute(0,3,1,2), tc.msg); //NHWC to NCHW + assertEquals(out1, net4a.output(inNHWC).permute(0,3,1,2), tc.msg); + } + + } + + private static List differentGrads(Gradient g1, Gradient g2) { + List differs = new ArrayList<>(); + Map m1 = g1.gradientForVariable(); + Map m2 = g2.gradientForVariable(); + for(String s : m1.keySet()){ + INDArray a1 = m1.get(s); + INDArray a2 = m2.get(s); + if(!a1.equals(a2)){ + differs.add(s); + } + } + return differs; + } + + + //Converts NHWC to NCHW activations + @EqualsAndHashCode + private static class NHWCToNCHWPreprocessor implements InputPreProcessor { + + @Override + public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) { + return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.permute(0,3,1,2)); + } + + @Override + public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) { + return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, output.permute(0,2,3,1)); + } + + @Override + public InputPreProcessor clone() { + return this; + } + + @Override + public InputType getOutputType(InputType inputType) { + InputType.InputTypeConvolutional c = (InputType.InputTypeConvolutional) inputType; + return InputType.convolutional(c.getHeight(), c.getWidth(), c.getChannels(), CNN2DFormat.NCHW); + } + + @Override + public Pair feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState, int minibatchSize) { + return null; + } + } + + + @Test + public void testWrongFormatIn(){ + + for(CNN2DFormat df : CNN2DFormat.values()) { + for(int i = 0; i < 4; i++) { + NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder() + .list(); + switch (i){ + case 0: + b.layer(new ConvolutionLayer.Builder().kernelSize(2,2).nIn(3).nOut(3).dataFormat(df).build()); + b.setInputType(InputType.convolutional(12,12,3,df)); + break; + case 1: + b.layer(new DepthwiseConvolution2D.Builder().kernelSize(2,2).nIn(3).nOut(3).dataFormat(df).build()); + b.setInputType(InputType.convolutional(12,12,3,df)); + break; + case 2: + b.layer(new Deconvolution2D.Builder().dataFormat(df).kernelSize(2,2).nIn(3).nOut(3).build()); + b.setInputType(InputType.convolutional(12,12,3,df)); + break; + case 3: + b.layer(new SeparableConvolution2D.Builder().dataFormat(df).kernelSize(2,2).nIn(3).nOut(3).build()); + b.setInputType(InputType.convolutional(12,12,3,df)); + break; + } + + + MultiLayerNetwork net = new MultiLayerNetwork(b.build()); + net.init(); + + INDArray in; + INDArray wrongFormatIn; + if(df == CNN2DFormat.NCHW){ + in = Nd4j.create(DataType.FLOAT, 5, 3, 12, 12); + wrongFormatIn = Nd4j.create(DataType.FLOAT, 5, 12, 12, 3); + } else { + in = Nd4j.create(DataType.FLOAT, 5, 12, 12, 3); + wrongFormatIn = Nd4j.create(DataType.FLOAT, 5, 3, 12, 12); + } + + net.output(in); + + try { + net.output(wrongFormatIn); + } catch (DL4JInvalidInputException e) { +// e.printStackTrace(); + String msg = e.getMessage(); + assertTrue(msg.contains(ConvolutionUtils.NCHW_NHWC_ERROR_MSG) || msg.contains("input array channels does not match CNN layer configuration"), msg); + } + } + } + + + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Convolution3DTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Convolution3DTest.java similarity index 76% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Convolution3DTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Convolution3DTest.java index fd9574445..596906d10 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Convolution3DTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Convolution3DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.layers.convolution; import org.deeplearning4j.BaseDL4JTest; @@ -27,72 +28,72 @@ import org.deeplearning4j.nn.conf.GradientNormalization; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.Convolution3D; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; + import java.util.Arrays; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** * @author Max Pumperla */ -@DisplayName("Convolution 3 D Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class Convolution3DTest extends BaseDL4JTest { +public class Convolution3DTest extends BaseDL4JTest { private int nExamples = 1; - private int nChannelsOut = 1; - private int nChannelsIn = 1; - private int inputDepth = 2 * 2; - private int inputWidth = 28 / 2; - private int inputHeight = 28 / 2; - private int[] kernelSize = new int[] { 2, 2, 2 }; - + private int[] kernelSize = new int[]{2, 2, 2}; private int outputDepth = inputDepth - kernelSize[0] + 1; - private int outputHeight = inputHeight - kernelSize[1] + 1; - private int outputWidth = inputWidth - kernelSize[2] + 1; private INDArray epsilon = Nd4j.ones(nExamples, nChannelsOut, outputDepth, outputHeight, outputWidth); + @Test - @DisplayName("Test Convolution 3 d Forward Same Mode") - void testConvolution3dForwardSameMode() { + public void testConvolution3dForwardSameMode() { + INDArray containedInput = getContainedData(); Convolution3DLayer layer = (Convolution3DLayer) getConvolution3DLayer(ConvolutionMode.Same); + assertTrue(layer.convolutionMode == ConvolutionMode.Same); + INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces()); + assertTrue(Arrays.equals(containedInput.shape(), containedOutput.shape())); + } @Test - @DisplayName("Test Convolution 3 d Forward Valid Mode") - void testConvolution3dForwardValidMode() throws Exception { + public void testConvolution3dForwardValidMode() throws Exception { + Convolution3DLayer layer = (Convolution3DLayer) getConvolution3DLayer(ConvolutionMode.Strict); + assertTrue(layer.convolutionMode == ConvolutionMode.Strict); + INDArray input = getData(); INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertTrue(Arrays.equals(new long[] { nExamples, nChannelsOut, outputDepth, outputWidth, outputHeight }, output.shape())); + + assertTrue(Arrays.equals(new long[]{nExamples, nChannelsOut, outputDepth, outputWidth, outputHeight}, + output.shape())); } private Layer getConvolution3DLayer(ConvolutionMode mode) { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123).layer(new Convolution3D.Builder().kernelSize(kernelSize).nIn(nChannelsIn).nOut(nChannelsOut).dataFormat(Convolution3D.DataFormat.NCDHW).convolutionMode(mode).hasBias(false).build()).build(); + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123) + .layer(new Convolution3D.Builder().kernelSize(kernelSize).nIn(nChannelsIn).nOut(nChannelsOut) + .dataFormat(Convolution3D.DataFormat.NCDHW).convolutionMode(mode).hasBias(false) + .build()) + .build(); long numParams = conf.getLayer().initializer().numParams(conf); INDArray params = Nd4j.ones(1, numParams); return conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); @@ -106,6 +107,7 @@ class Convolution3DTest extends BaseDL4JTest { } private INDArray getContainedData() { - return Nd4j.create(new double[] { 1., 2., 3., 4., 5., 6., 7., 8 }, new int[] { 1, 1, 2, 2, 2 }); + return Nd4j.create(new double[]{1., 2., 3., 4., 5., 6., 7., 8}, new int[]{1, 1, 2, 2, 2}); } + } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerSetupTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerSetupTest.java new file mode 100644 index 000000000..26fc3a4e3 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerSetupTest.java @@ -0,0 +1,496 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.convolution; + +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.image.recordreader.ImageRecordReader; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToCnnPreProcessor; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.params.BatchNormalizationParamInitializer; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.util.FeatureUtil; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Adam Gibson + */ +public class ConvolutionLayerSetupTest extends BaseDL4JTest { + + @TempDir + public File testDir; + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @Test + public void testConvolutionLayerSetup() { + MultiLayerConfiguration.Builder builder = inComplete(); + builder.setInputType(InputType.convolutionalFlat(28, 28, 1)); + MultiLayerConfiguration completed = complete().build(); + MultiLayerConfiguration test = builder.build(); + assertEquals(completed, test); + + } + + + @Test + public void testDenseToOutputLayer() { + Nd4j.getRandom().setSeed(12345); + final int numRows = 76; + final int numColumns = 76; + int nChannels = 3; + int outputNum = 6; + int seed = 123; + + //setup the network + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .l1(1e-1).l2(2e-4).dropOut(0.5).miniBatch(true) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list() + .layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .build()) + .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .build()) + .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()) + .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .build()) + + .setInputType(InputType.convolutional(numRows, numColumns, nChannels)); + + DataSet d = new DataSet(Nd4j.rand(new int[]{10, nChannels, numRows, numColumns}), + FeatureUtil.toOutcomeMatrix(new int[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, 6)); + MultiLayerNetwork network = new MultiLayerNetwork(builder.build()); + network.init(); + network.fit(d); + + } + + + @Test + public void testMnistLenet() throws Exception { + MultiLayerConfiguration.Builder incomplete = incompleteMnistLenet(); + incomplete.setInputType(InputType.convolutionalFlat(28, 28, 1)); + + MultiLayerConfiguration testConf = incomplete.build(); + assertEquals(800, ((FeedForwardLayer) testConf.getConf(4).getLayer()).getNIn()); + assertEquals(500, ((FeedForwardLayer) testConf.getConf(5).getLayer()).getNIn()); + + //test instantiation + DataSetIterator iter = new MnistDataSetIterator(10, 10); + MultiLayerNetwork network = new MultiLayerNetwork(testConf); + network.init(); + network.fit(iter.next()); + } + + + + @Test + public void testMultiChannel() throws Exception { + INDArray in = Nd4j.rand(new int[] {10, 3, 28, 28}); + INDArray labels = Nd4j.rand(10, 2); + DataSet next = new DataSet(in, labels); + + NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLFW(); + builder.setInputType(InputType.convolutional(28, 28, 3)); + MultiLayerConfiguration conf = builder.build(); + ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(2).getLayer(); + assertEquals(6, layer2.getNIn()); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + network.fit(next); + } + + @Test + public void testLRN() throws Exception { + List labels = new ArrayList<>(Arrays.asList("Zico", "Ziwang_Xu")); + File dir = testDir; + new ClassPathResource("lfwtest/").copyDirectory(dir); + String rootDir = dir.getAbsolutePath(); + + RecordReader reader = new ImageRecordReader(28, 28, 3); + reader.initialize(new FileSplit(new File(rootDir))); + DataSetIterator recordReader = new RecordReaderDataSetIterator(reader, 10, 1, labels.size()); + labels.remove("lfwtest"); + NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLRN(); + builder.setInputType(InputType.convolutional(28, 28, 3)); + + MultiLayerConfiguration conf = builder.build(); + + ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(3).getLayer(); + assertEquals(6, layer2.getNIn()); + + } + + + public MultiLayerConfiguration.Builder incompleteLRN() { + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder().seed(3) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + new int[] {5, 5}).nOut(6).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder( + new int[] {2, 2}).build()) + .layer(2, new LocalResponseNormalization.Builder().build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + new int[] {5, 5}).nOut(6).build()) + .layer(4, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder( + new int[] {2, 2}).build()) + .layer(5, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(2) + .activation(Activation.SOFTMAX).build()); + return builder; + } + + + public MultiLayerConfiguration.Builder incompleteLFW() { + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder().seed(3) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + new int[] {5, 5}).nOut(6).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder( + new int[] {2, 2}).build()) + .layer(2, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + new int[] {5, 5}).nOut(6).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder( + new int[] {2, 2}).build()) + .layer(4, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX) + .nOut(2).build()); + return builder; + } + + + + public MultiLayerConfiguration.Builder incompleteMnistLenet() { + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder().seed(3) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + new int[] {5, 5}).nIn(1).nOut(20).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder( + new int[] {2, 2}, new int[] {2, 2}).build()) + .layer(2, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + new int[] {5, 5}).nIn(20).nOut(50).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder( + new int[] {2, 2}, new int[] {2, 2}).build()) + .layer(4, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nOut(500) + .build()) + .layer(5, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .activation(Activation.SOFTMAX).nOut(10) + .build()); + return builder; + } + + public MultiLayerConfiguration mnistLenet() { + MultiLayerConfiguration builder = + new NeuralNetConfiguration.Builder().seed(3) + .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + new int[] {5, 5}).nIn(1).nOut(6).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder( + new int[] {5, 5}, new int[] {2, 2}).build()) + .layer(2, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + new int[] {5, 5}).nIn(1).nOut(6).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder( + new int[] {5, 5}, new int[] {2, 2}).build()) + .layer(4, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(150) + .nOut(10).build()) + .build(); + return builder; + } + + public MultiLayerConfiguration.Builder inComplete() { + int nChannels = 1; + int outputNum = 10; + int seed = 123; + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] {10, 10}, + new int[] {2, 2}).nIn(nChannels).nOut(6).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .build()) + ; + + return builder; + } + + + public MultiLayerConfiguration.Builder complete() { + final int numRows = 28; + final int numColumns = 28; + int nChannels = 1; + int outputNum = 10; + int seed = 123; + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] {10, 10}, + new int[] {2, 2}).nIn(nChannels).nOut(6).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nIn(5 * 5 * 1 * 6) //216 + .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .build()) + .inputPreProcessor(0, new FeedForwardToCnnPreProcessor(numRows, numColumns, nChannels)) + .inputPreProcessor(2, new CnnToFeedForwardPreProcessor(5, 5, 6)); + + return builder; + } + + + @Test + public void testDeconvolution() { + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list() + //out = stride * (in-1) + filter - 2*pad -> 2 * (28-1) + 2 - 0 = 56 -> 56x56x3 + .layer(0, new Deconvolution2D.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()) + //(56-2+2*1)/2+1 = 29 -> 29x29x3 + .layer(1, new SubsamplingLayer.Builder().kernelSize(2, 2).padding(1, 1).stride(2, 2).build()) + .layer(2, new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28, 28, 1)); + + MultiLayerConfiguration conf = builder.build(); + + assertNotNull(conf.getInputPreProcess(2)); + assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); + CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); + assertEquals(29, proc.getInputHeight()); + assertEquals(29, proc.getInputWidth()); + assertEquals(3, proc.getNumChannels()); + + assertEquals(29 * 29 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); + } + + @Test + public void testSubSamplingWithPadding() { + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list() + .layer(0, new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()) //(28-2+0)/2+1 = 14 + .layer(1, new SubsamplingLayer.Builder().kernelSize(2, 2).padding(1, 1).stride(2, 2).build()) //(14-2+2)/2+1 = 8 -> 8x8x3 + .layer(2, new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28, 28, 1)); + + MultiLayerConfiguration conf = builder.build(); + + assertNotNull(conf.getInputPreProcess(2)); + assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); + CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); + assertEquals(8, proc.getInputHeight()); + assertEquals(8, proc.getInputWidth()); + assertEquals(3, proc.getNumChannels()); + + assertEquals(8 * 8 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); + } + + @Test + public void testUpsampling() { + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list() + .layer(new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()) //(28-2+0)/2+1 = 14 + .layer(new Upsampling2D.Builder().size(3).build()) // 14 * 3 = 42! + .layer(new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28, 28, 1)); + + MultiLayerConfiguration conf = builder.build(); + + assertNotNull(conf.getInputPreProcess(2)); + assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); + CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); + assertEquals(42, proc.getInputHeight()); + assertEquals(42, proc.getInputWidth()); + assertEquals(3, proc.getNumChannels()); + + assertEquals(42 * 42 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); + } + + @Test + public void testSpaceToBatch() { + + int[] blocks = new int[] {2, 2}; + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list() + .layer(new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()) //(28-2+0)/2+1 = 14 + .layer(new SpaceToBatchLayer.Builder(blocks).build()) // Divide space dimensions by blocks, i.e. 14/2 = 7 + .layer(new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28, 28, 1)); + + MultiLayerConfiguration conf = builder.build(); + + assertNotNull(conf.getInputPreProcess(2)); + assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); + CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); + assertEquals(7, proc.getInputHeight()); + assertEquals(7, proc.getInputWidth()); + assertEquals(3, proc.getNumChannels()); + } + + @Test + public void testSpaceToDepth() { + + int blocks = 2; + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list() + //(28-2+0)/2+1 = 14 -> 14x14x3 out + .layer(new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()) + // Divide space dimensions by blocks, i.e. 14/2 = 7 -> 7x7x12 out (3x2x2 depth) + .layer(new SpaceToDepthLayer.Builder(blocks, SpaceToDepthLayer.DataFormat.NCHW).build()) + .layer(new OutputLayer.Builder().nIn(3 * 2 * 2).nOut(3).activation(Activation.SOFTMAX).build()) // nIn of the next layer gets multiplied by 2*2. + .setInputType(InputType.convolutional(28, 28, 1)); + + MultiLayerConfiguration conf = builder.build(); + + assertNotNull(conf.getInputPreProcess(2)); + assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); + CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); + assertEquals(7, proc.getInputHeight()); + assertEquals(7, proc.getInputWidth()); + assertEquals(12, proc.getNumChannels()); + + } + + + @Test + public void testCNNDBNMultiLayer() throws Exception { + DataSetIterator iter = new MnistDataSetIterator(2, 2); + DataSet next = iter.next(); + + // Run with separate activation layer + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .weightInit(WeightInit.XAVIER).list() + .layer(0, new ConvolutionLayer.Builder(new int[] {1, 1}, new int[] {1, 1}).nIn(1).nOut(6) + .activation(Activation.IDENTITY).build()) + .layer(1, new BatchNormalization.Builder().build()) + .layer(2, new ActivationLayer.Builder().activation(Activation.RELU).build()) + .layer(3, new DenseLayer.Builder().nIn(28 * 28 * 6).nOut(10).activation(Activation.IDENTITY) + .build()) + .layer(4, new BatchNormalization.Builder().nOut(10).build()) + .layer(5, new ActivationLayer.Builder().activation(Activation.RELU).build()) + .layer(6, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(10).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + network.setInput(next.getFeatures()); + INDArray activationsActual = network.output(next.getFeatures()); + assertEquals(10, activationsActual.shape()[1], 1e-2); + + network.fit(next); + INDArray actualGammaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.GAMMA); + INDArray actualBetaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.BETA); + assertTrue(actualGammaParam != null); + assertTrue(actualBetaParam != null); + } + + @Test + public void testSeparableConv2D() { + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list() + .layer( new SeparableConvolution2D.Builder(2, 2) + .depthMultiplier(2) + .padding(0, 0) + .stride(2, 2).nIn(1).nOut(3).build()) //(28-2+0)/2+1 = 14 + .layer( new SubsamplingLayer.Builder().kernelSize(2, 2).padding(1, 1).stride(2, 2).build()) //(14-2+2)/2+1 = 8 -> 8x8x3 + .layer(2, new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28, 28, 1)); + + MultiLayerConfiguration conf = builder.build(); + + assertNotNull(conf.getInputPreProcess(2)); + assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); + CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); + assertEquals(8, proc.getInputHeight()); + assertEquals(8, proc.getInputWidth()); + assertEquals(3, proc.getNumChannels()); + + assertEquals(8 * 8 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); + } + + @Test + public void testDeconv2D() { + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list() + //out = stride * (in-1) + filter - 2*pad -> 2 * (28-1) + 2 - 0 = 56 -> 56x56x3 + .layer( new Deconvolution2D.Builder(2, 2) + .padding(0, 0) + .stride(2, 2).nIn(1).nOut(3).build()) + //(56-2+2*1)/2+1 = 29 -> 29x29x3 + .layer( new SubsamplingLayer.Builder().kernelSize(2, 2).padding(1, 1).stride(2, 2).build()) + .layer(2, new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28, 28, 1)); + + MultiLayerConfiguration conf = builder.build(); + + assertNotNull(conf.getInputPreProcess(2)); + assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); + CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); + assertEquals(29, proc.getInputHeight()); + assertEquals(29, proc.getInputWidth()); + assertEquals(3, proc.getNumChannels()); + + assertEquals(29 * 29 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerTest.java new file mode 100644 index 000000000..6b68d6cea --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerTest.java @@ -0,0 +1,838 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.convolution; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.exception.DL4JException; +import org.deeplearning4j.exception.DL4JInvalidInputException; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.RNNFormat; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.Convolution1DLayer; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.weights.WeightInitNormal; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.junit.jupiter.api.Test; +import org.nd4j.enums.RnnDataFormat; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.impl.ActivationSoftmax; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.shape.Shape; +import org.nd4j.linalg.convolution.Convolution; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.INDArrayIndex; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.lossfunctions.impl.LossMCXENT; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Adam Gibson + */ +public class ConvolutionLayerTest extends BaseDL4JTest { + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @Test + public void testTwdFirstLayer() throws Exception { + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4) + .updater(new Nesterovs(0.9)).dropOut(0.5) + .list().layer(0, + new ConvolutionLayer.Builder(8, 8) //16 filters kernel size 8 stride 4 + .stride(4, 4).nOut(16).dropOut(0.5) + .activation(Activation.RELU).weightInit( + WeightInit.XAVIER) + .build()) + .layer(1, new ConvolutionLayer.Builder(4, 4) //32 filters kernel size 4 stride 2 + .stride(2, 2).nOut(32).dropOut(0.5).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(2, new DenseLayer.Builder() //fully connected with 256 rectified units + .nOut(256).activation(Activation.RELU).weightInit(WeightInit.XAVIER) + .dropOut(0.5).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS) //output layer + .nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)); + + DataSetIterator iter = new MnistDataSetIterator(10, 10); + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + DataSet ds = iter.next(); + for( int i=0; i<5; i++ ) { + network.fit(ds); + } + } + + @Test + public void testCNNSubComboWithMixedHW() { + int imageHeight = 20; + int imageWidth = 23; + int nChannels = 1; + int classes = 2; + int numSamples = 200; + + int kernelHeight = 3; + int kernelWidth = 3; + + DataSet trainInput; + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder() + .seed(123) + .list() + .layer(0, new ConvolutionLayer.Builder(kernelHeight, kernelWidth).stride(1, 1) + .nOut(2).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new SubsamplingLayer.Builder() + .poolingType(SubsamplingLayer.PoolingType.MAX) + .kernelSize(imageHeight - kernelHeight, 1).stride(1, 1).build()) + .layer(2, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, nChannels)); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); + INDArray emptyLables = Nd4j.zeros(numSamples, classes); + + trainInput = new DataSet(emptyFeatures, emptyLables); + model.fit(trainInput); + } + + @Test + public void testCausal1d() { + Nd4j.getEnvironment().setVerbose(true); + Nd4j.getEnvironment().setDebug(true); + //See: Fixes: https://github.com/eclipse/deeplearning4j/issues/9060 + double learningRate = 1e-3; + long seed = 123; + long timeSteps = 72; + long vectorLength = 64; + long batchSize = 1; + INDArray arr = Nd4j.randn(batchSize,vectorLength,timeSteps); + + MultiLayerConfiguration build = new NeuralNetConfiguration.Builder().seed(seed) + .activation(Activation.RELU) + .weightInit(new WeightInitNormal()) // better init + .updater(new Adam(learningRate)) + .list() + // block 1 + .layer(new Convolution1D.Builder() + .kernelSize(2) + .rnnDataFormat(RNNFormat.NCW) + .stride(1) + .nOut(14) + .convolutionMode(ConvolutionMode.Causal) + .dilation(4) + .build()) + .layer(new RnnLossLayer.Builder().dataFormat(RNNFormat.NCW) + .activation(new ActivationSoftmax()) + .lossFunction(new LossMCXENT()).build()) + .setInputType(InputType.recurrent(vectorLength,timeSteps,RNNFormat.NCW)) + .build(); + + MultiLayerNetwork network = new MultiLayerNetwork(build); + network.init(); + INDArray output = network.output(arr); + assertArrayEquals(new long[]{1,14,72},output.shape()); + System.out.println(output); + } + + @Test + public void testCNNTooLargeKernel() { + assertThrows(DL4JException.class, () -> { + int imageHeight = 20; + + int imageWidth = 23; + int nChannels = 1; + int classes = 2; + int numSamples = 200; + + int kernelHeight = imageHeight; + int kernelWidth = imageWidth + 1; + + DataSet trainInput; + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder() + .seed(123) + .list() + .layer(0, new ConvolutionLayer.Builder(kernelHeight, kernelWidth) //(img-kernel+2*padding)/stride + 1: must be >= 1. Therefore: with p=0, kernel <= img size + .stride(1, 1).nOut(2).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, nChannels)); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); + INDArray emptyLables = Nd4j.zeros(numSamples, classes); + + trainInput = new DataSet(emptyFeatures, emptyLables); + model.fit(trainInput); + }); + } + + @Test + public void testCNNZeroStride() { + assertThrows(Exception.class, () -> { + int imageHeight = 20; + int imageWidth = 23; + int nChannels = 1; + int classes = 2; + int numSamples = 200; + + int kernelHeight = imageHeight; + int kernelWidth = imageWidth; + + DataSet trainInput; + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder() + .seed(123) + .list() + .layer(0, new ConvolutionLayer.Builder(kernelHeight, kernelWidth).stride(1, 0) + .nOut(2).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).build()) + + .setInputType(InputType.convolutional(imageHeight, imageWidth, nChannels)); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); + INDArray emptyLables = Nd4j.zeros(numSamples, classes); + + trainInput = new DataSet(emptyFeatures, emptyLables); + model.fit(trainInput); + }); + } + + @Test + public void testCNNBiasInit() { + ConvolutionLayer cnn = new ConvolutionLayer.Builder().nIn(1).nOut(3).biasInit(1).build(); + + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(cnn).build(); + + val numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + + assertEquals(1, layer.getParam("b").size(0)); + } + + @Test + public void testCNNInputSetupMNIST() throws Exception { + INDArray input = getMnistData(); + Layer layer = getMNISTConfig(); + layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + + assertEquals(input, layer.input()); + assertArrayEquals(input.shape(), layer.input().shape()); + } + + @Test + public void testFeatureMapShapeMNIST() throws Exception { + int inputWidth = 28; + int[] stride = new int[] {1, 1}; + int[] padding = new int[] {0, 0}; + int[] kernelSize = new int[] {9, 9}; + int nChannelsIn = 1; + int depth = 20; + int featureMapWidth = (inputWidth + padding[1] * 2 - kernelSize[1]) / stride[1] + 1; + + INDArray input = getMnistData(); + + Layer layer = getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding); + INDArray convActivations = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + + assertEquals(featureMapWidth, convActivations.size(2)); + assertEquals(depth, convActivations.size(1)); + } + + @Test + public void testActivateResultsContained() { + Layer layer = getContainedConfig(); + INDArray input = getContainedData(); + INDArray expectedOutput = Nd4j.create(new float[] {0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f, + 0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f, + 0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f, + 0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f, + 0.99966465f, 0.99966465f, 0.99966465f}, new int[] {1, 2, 4, 4}); + + INDArray convActivations = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + + assertArrayEquals(expectedOutput.shape(), convActivations.shape()); + assertEquals(expectedOutput, convActivations); + } + + ////////////////////////////////////////////////////////////////////////////////// + + private static Layer getCNNConfig(int nIn, int nOut, int[] kernelSize, int[] stride, int[] padding) { + + ConvolutionLayer layer = new ConvolutionLayer.Builder(kernelSize, stride, padding).nIn(nIn).nOut(nOut) + .activation(Activation.SIGMOID).build(); + + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(layer).build(); + + val numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + return conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + } + + public Layer getMNISTConfig() { + int[] kernelSize = new int[] {9, 9}; + int[] stride = new int[] {1, 1}; + int[] padding = new int[] {1, 1}; + int nChannelsIn = 1; + int depth = 20; + + return getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding); + + } + + public INDArray getMnistData() throws Exception { + int inputWidth = 28; + int inputHeight = 28; + int nChannelsIn = 1; + int nExamples = 5; + + DataSetIterator data = new MnistDataSetIterator(nExamples, nExamples); + DataSet mnist = data.next(); + nExamples = mnist.numExamples(); + return mnist.getFeatures().reshape(nExamples, nChannelsIn, inputHeight, inputWidth); + } + + public Layer getContainedConfig() { + int[] kernelSize = new int[] {2, 2}; + int[] stride = new int[] {2, 2}; + int[] padding = new int[] {0, 0}; + int nChannelsIn = 1; + int depth = 2; + + INDArray W = Nd4j.create(new double[] {0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5}, new int[] {2, 1, 2, 2}); + INDArray b = Nd4j.create(new double[] {1, 1}); + Layer layer = getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding); + layer.setParam("W", W); + layer.setParam("b", b); + + return layer; + + } + + public INDArray getContainedData() { + INDArray ret = Nd4j.create(new float[] {1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4}, new int[] {1, 1, 8, 8}); + return ret; + } + + public INDArray getContainedCol() { + return Nd4j.create(new float[] {1, 1, 1, 1, 3, 3, 3, 3, 1, 1, 1, 1, 3, 3, 3, 3, 1, 1, 1, 1, 3, 3, 3, 3, 1, 1, + 1, 1, 3, 3, 3, 3, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, + 2, 2, 4, 4, 4, 4}, new int[] {1, 1, 2, 2, 4, 4}); + } + + + + ////////////////////////////////////////////////////////////////////////////////// + + + @Test + public void testCNNMLNPretrain() throws Exception { + // Note CNN does not do pretrain + int numSamples = 10; + int batchSize = 10; + DataSetIterator mnistIter = new MnistDataSetIterator(batchSize, numSamples, true); + + MultiLayerNetwork model = getCNNMLNConfig(false, true); + model.fit(mnistIter); + + mnistIter.reset(); + + MultiLayerNetwork model2 = getCNNMLNConfig(false, true); + model2.fit(mnistIter); + mnistIter.reset(); + + DataSet test = mnistIter.next(); + + Evaluation eval = new Evaluation(); + INDArray output = model.output(test.getFeatures()); + eval.eval(test.getLabels(), output); + double f1Score = eval.f1(); + + Evaluation eval2 = new Evaluation(); + INDArray output2 = model2.output(test.getFeatures()); + eval2.eval(test.getLabels(), output2); + double f1Score2 = eval2.f1(); + + assertEquals(f1Score, f1Score2, 1e-4); + + + } + + + @Test + public void testCNNMLNBackprop() throws Exception { + int numSamples = 10; + int batchSize = 10; + DataSetIterator mnistIter = new MnistDataSetIterator(batchSize, numSamples, true); + + MultiLayerNetwork model = getCNNMLNConfig(true, false); + model.fit(mnistIter); + + MultiLayerNetwork model2 = getCNNMLNConfig(true, false); + model2.fit(mnistIter); + + mnistIter.reset(); + DataSet test = mnistIter.next(); + + Evaluation eval = new Evaluation(); + INDArray output = model.output(test.getFeatures()); + eval.eval(test.getLabels(), output); + double f1Score = eval.f1(); + + Evaluation eval2 = new Evaluation(); + INDArray output2 = model2.output(test.getFeatures()); + eval2.eval(test.getLabels(), output2); + double f1Score2 = eval2.f1(); + + assertEquals(f1Score, f1Score2, 1e-4); + + } + + @Test + public void testGetSetParams() { + + MultiLayerNetwork net = getCNNMLNConfig(true, false); + + INDArray paramsOrig = net.params().dup(); + net.setParams(paramsOrig); + + INDArray params2 = net.params(); + + assertEquals(paramsOrig, params2); + } + + private static final int kH = 2; + private static final int kW = 2; + private static final int[] strides = {1, 1}; + private static final int[] pad = {0, 0}; + + private static final int miniBatch = 2; + private static final int inDepth = 2; + private static final int height = 3; + private static final int width = 3; + + private static final int outW = 2; + private static final int outH = 2; + + private static INDArray getInput() { + + /* + ----- Input images ----- + example 0: + channels 0 channels 1 + [ 0 1 2 [ 9 10 11 + 3 4 5 12 13 14 + 6 7 8] 15 16 17] + example 1: + [18 19 20 [27 28 29 + 21 22 23 30 31 32 + 24 25 26] 33 34 35] + */ + + INDArray input = Nd4j.create(new int[] {miniBatch, inDepth, height, width}, 'c'); + input.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{0, 1, 2}, {3, 4, 5}, {6, 7, 8}})); + input.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{9, 10, 11}, {12, 13, 14}, {15, 16, 17}})); + input.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{18, 19, 20}, {21, 22, 23}, {24, 25, 26}})); + input.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{27, 28, 29}, {30, 31, 32}, {33, 34, 35}})); + + return input; + } + + @Test + public void testCnnIm2ColReshaping() { + //This test: a bit unusual in that it tests the *assumptions* of the CNN implementation rather than the implementation itself + //Specifically, it tests the row and column orders after reshaping on im2col is reshaped (both forward and backward pass) + INDArray input = getInput(); + + //im2col in the required order: want [outW,outH,miniBatch,depthIn,kH,kW], but need to input [miniBatch,channels,kH,kW,outH,outW] + // given the current im2col implementation + //To get this: create an array of the order we want, permute it to the order required by im2col implementation, and then do im2col on that + //to get old order from required order: permute(2,3,4,5,1,2) + INDArray col = Nd4j.create(new int[] {miniBatch, outH, outW, inDepth, kH, kW}, 'c'); + INDArray col2 = col.permute(0, 3, 4, 5, 1, 2); + Convolution.im2col(input, kH, kW, strides[0], strides[1], pad[0], pad[1], false, col2); + + /* + Expected Output, im2col + - example 0 - + channels 0 channels 1 + h0,w0 h0,w1 h0,w0 h0,w1 + 0 1 1 2 9 10 10 11 + 3 4 4 5 12 13 13 14 + + h1,w0 h1,w1 h1,w0 h1,w1 + 3 4 4 5 12 13 13 14 + 6 7 7 8 15 16 16 17 + + - example 1 - + channels 0 channels 1 + h0,w0 h0,w1 h0,w0 h0,w1 + 18 19 19 20 27 28 28 29 + 21 22 22 23 30 31 31 32 + + h1,w0 h1,w1 h1,w0 h1,w1 + 21 22 22 23 30 31 31 32 + 24 25 25 26 33 34 34 35 + */ + + //Now, after reshaping im2col to 2d, we expect: + //Rows with order (wOut0,hOut0,mb0), (wOut1,hOut0,mb0), (wOut0,hOut1,mb0), (wOut1,hOut1,mb0), (wOut0,hOut0,mb1), ... + //Columns with order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), ... + + INDArray reshapedCol = Shape.newShapeNoCopy(col, new int[] {miniBatch * outH * outW, inDepth * kH * kW}, false); + + INDArray exp2d = Nd4j.create(outW * outH * miniBatch, inDepth * kH * kW); + exp2d.putRow(0, Nd4j.create(new double[] {0, 1, 3, 4, 9, 10, 12, 13})); //wOut0,hOut0,mb0 -> both depths, in order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), (d1,kh0,kw1), (d1,kh1,kw0), (d1,kh1,kw1) + exp2d.putRow(1, Nd4j.create(new double[] {1, 2, 4, 5, 10, 11, 13, 14})); //wOut1,hOut0,mb0 + exp2d.putRow(2, Nd4j.create(new double[] {3, 4, 6, 7, 12, 13, 15, 16})); //wOut0,hOut1,mb0 + exp2d.putRow(3, Nd4j.create(new double[] {4, 5, 7, 8, 13, 14, 16, 17})); //wOut1,hOut1,mb0 + exp2d.putRow(4, Nd4j.create(new double[] {18, 19, 21, 22, 27, 28, 30, 31})); //wOut0,hOut0,mb1 + exp2d.putRow(5, Nd4j.create(new double[] {19, 20, 22, 23, 28, 29, 31, 32})); //wOut1,hOut0,mb1 + exp2d.putRow(6, Nd4j.create(new double[] {21, 22, 24, 25, 30, 31, 33, 34})); //wOut0,hOut1,mb1 + exp2d.putRow(7, Nd4j.create(new double[] {22, 23, 25, 26, 31, 32, 34, 35})); //wOut1,hOut1,mb1 + + assertEquals(exp2d, reshapedCol); + + //Check the same thing for the backprop im2col (different order) + INDArray colBackprop = Nd4j.create(new int[] {miniBatch, outH, outW, inDepth, kH, kW}, 'c'); + INDArray colBackprop2 = colBackprop.permute(0, 3, 4, 5, 1, 2); + + Convolution.im2col(input, kH, kW, strides[0], strides[1], pad[0], pad[1], false, colBackprop2); + + INDArray reshapedColBackprop = Shape.newShapeNoCopy(colBackprop, + new int[] {miniBatch * outH * outW, inDepth * kH * kW}, false); + + //Rows with order (mb0,h0,w0), (mb0,h0,w1), (mb0,h1,w0), (mb0,h1,w1), (mb1,h0,w0), (mb1,h0,w1), (mb1,h1,w0), (mb1,h1,w1) + //Columns with order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), ... + + INDArray exp2dv2 = Nd4j.create(outW * outH * miniBatch, inDepth * kH * kW); + exp2dv2.putRow(0, Nd4j.create(new double[] {0, 1, 3, 4, 9, 10, 12, 13})); //wOut0,hOut0,mb0 -> both depths, in order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), (d1,kh0,kw1), (d1,kh1,kw0), (d1,kh1,kw1) + exp2dv2.putRow(1, Nd4j.create(new double[] {1, 2, 4, 5, 10, 11, 13, 14})); //wOut1,hOut0,mb0 + exp2dv2.putRow(2, Nd4j.create(new double[] {3, 4, 6, 7, 12, 13, 15, 16})); //wOut0,hOut1,mb0 + exp2dv2.putRow(3, Nd4j.create(new double[] {4, 5, 7, 8, 13, 14, 16, 17})); //wOut1,hOut1,mb0 + exp2dv2.putRow(4, Nd4j.create(new double[] {18, 19, 21, 22, 27, 28, 30, 31})); //wOut0,hOut0,mb1 + exp2dv2.putRow(5, Nd4j.create(new double[] {19, 20, 22, 23, 28, 29, 31, 32})); //wOut1,hOut0,mb1 + exp2dv2.putRow(6, Nd4j.create(new double[] {21, 22, 24, 25, 30, 31, 33, 34})); //wOut0,hOut1,mb1 + exp2dv2.putRow(7, Nd4j.create(new double[] {22, 23, 25, 26, 31, 32, 34, 35})); //wOut1,hOut1,mb1 + + assertEquals(exp2dv2, reshapedColBackprop); + } + + @Test + public void testDeltaReshaping() { + //As per above test: testing assumptions of cnn implementation... + + //Delta: initially shape [miniBatch,dOut,outH,outW] + //permute to [dOut,miniB,outH,outW] + //then reshape to [dOut,miniB*outH*outW] + //Expect columns of delta2d to be like: (mb0,h0,w0), (mb0,h0,w1), (mb1,h0,w2), (mb0,h1,w0), ... (mb1,...), ..., (mb2,...) + int miniBatch = 3; + int depth = 2; + int outW = 3; + int outH = 3; + + /* + ----- Input delta ----- + example 0: + channels 0 channels 1 + [ 0 1 2 [ 9 10 11 + 3 4 5 12 13 14 + 6 7 8] 15 16 17] + example 1: + [18 19 20 [27 28 29 + 21 22 23 30 31 32 + 24 25 26] 33 34 35] + example 2: + [36 37 38 [45 46 47 + 39 40 41 48 49 50 + 42 43 44] 51 52 53] + */ + + INDArray deltaOrig = Nd4j.create(new int[] {miniBatch, depth, outH, outW}, 'c'); + deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{0, 1, 2}, {3, 4, 5}, {6, 7, 8}})); + deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{9, 10, 11}, {12, 13, 14}, {15, 16, 17}})); + deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{18, 19, 20}, {21, 22, 23}, {24, 25, 26}})); + deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{27, 28, 29}, {30, 31, 32}, {33, 34, 35}})); + deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(2), NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{36, 37, 38}, {39, 40, 41}, {42, 43, 44}})); + deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(2), NDArrayIndex.point(1), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{45, 46, 47}, {48, 49, 50}, {51, 52, 53}})); + + + INDArray deltaPermute = deltaOrig.permute(1, 0, 2, 3).dup('c'); + INDArray delta2d = Shape.newShapeNoCopy(deltaPermute, new int[] {depth, miniBatch * outW * outH}, false); + + INDArray exp = Nd4j.create(new double[][] { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 18, 19, 20, 21, 22, 23, 24, 25, 26, 36, 37, 38, 39, 40, 41, 42, 43, + 44}, //depth0 + {9, 10, 11, 12, 13, 14, 15, 16, 17, 27, 28, 29, 30, 31, 32, 33, 34, 35, 45, 46, 47, 48, 49, 50, + 51, 52, 53} //depth1 + }).castTo(delta2d.dataType()); + + assertEquals(exp, delta2d); + } + + @Test + public void testWeightReshaping() { + //Test assumptions of weight reshaping + //Weights: originally c order, shape [outDepth, inDepth, kH, kw] + //permute (3,2,1,0) + + int depthOut = 2; + int depthIn = 3; + int kH = 2; + int kW = 2; + + /* + ----- Weights ----- + - dOut 0 - + dIn 0 dIn 1 dIn 2 + [ 0 1 [ 4 5 [ 8 9 + 2 3] 6 7] 10 11] + - dOut 1 - + [12 13 [16 17 [20 21 + 14 15] 18 19] 22 23] + */ + + INDArray weightOrig = Nd4j.create(new int[] {depthOut, depthIn, kH, kW}, 'c'); + weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{0, 1}, {2, 3}})); + weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{4, 5}, {6, 7}})); + weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(2), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{8, 9}, {10, 11}})); + weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{12, 13}, {14, 15}})); + weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{16, 17}, {18, 19}})); + weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(2), NDArrayIndex.all(), + NDArrayIndex.all()}, Nd4j.create(new double[][] {{20, 21}, {22, 23}})); + + INDArray weightPermute = weightOrig.permute(3, 2, 1, 0); + INDArray w2d = Shape.newShapeNoCopy(weightPermute, new int[] {depthIn * kH * kW, depthOut}, true); + + assertNotNull(w2d); + + //Expected order of weight rows, after reshaping: (kw0,kh0,din0), (kw1,kh0,din0), (kw0,kh1,din0), (kw1,kh1,din0), (kw0,kh0,din1), ... + INDArray wExp = Nd4j.create(new double[][] {{0, 12}, {1, 13}, {2, 14}, {3, 15}, {4, 16}, {5, 17}, {6, 18}, + {7, 19}, {8, 20}, {9, 21}, {10, 22}, {11, 23}}).castTo(DataType.FLOAT); + + assertEquals(wExp, w2d); + } + + ////////////////////////////////////////////////////////////////////////////////// + + private static MultiLayerNetwork getCNNMLNConfig(boolean backprop, boolean pretrain) { + int outputNum = 10; + int seed = 123; + + MultiLayerConfiguration.Builder conf = + new NeuralNetConfiguration.Builder().seed(seed) + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list() + .layer(0, new ConvolutionLayer.Builder(new int[] {10, 10}).nOut(6).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, + new int[] {2, 2}).stride(1, 1).build()) + .layer(2, new OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)); + + MultiLayerNetwork model = new MultiLayerNetwork(conf.build()); + model.init(); + + return model; + } + + + + @Test + public void test1dInputType(){ + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .convolutionMode(ConvolutionMode.Same) + .list() + .layer(new Convolution1DLayer.Builder().nOut(3).kernelSize(2).activation(Activation.TANH).build()) + .layer(new Subsampling1DLayer.Builder().kernelSize(2).stride(2).build()) + .layer(new Upsampling1D.Builder().size(2).build()) + .layer(new RnnOutputLayer.Builder().nOut(7).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.recurrent(10)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + List l = conf.getLayerActivationTypes(InputType.recurrent(10)); + assertEquals(InputType.recurrent(3, -1), l.get(0)); + assertEquals(InputType.recurrent(3, -1), l.get(1)); + assertEquals(InputType.recurrent(3, -1), l.get(2)); + assertEquals(InputType.recurrent(7, -1), l.get(3)); + + List l2 = conf.getLayerActivationTypes(InputType.recurrent(10, 6)); + assertEquals(InputType.recurrent(3, 6), l2.get(0)); + assertEquals(InputType.recurrent(3, 3), l2.get(1)); + assertEquals(InputType.recurrent(3, 6), l2.get(2)); + assertEquals(InputType.recurrent(7, 6), l2.get(3)); + + + INDArray in = Nd4j.create(2, 10, 6); + INDArray out = net.output(in); + assertArrayEquals(new long[]{2,7,6}, out.shape()); + } + + @Test + public void testDeconvBadInput(){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new Deconvolution2D.Builder().nIn(5).nOut(3).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray badInput = Nd4j.create(DataType.FLOAT, 1, 10, 5, 5); + try { + net.output(badInput); + } catch (DL4JInvalidInputException e){ + String msg = e.getMessage(); + assertTrue(msg.contains("Deconvolution2D") && msg.contains("input") && msg.contains("channels"), msg); + } + } + + @Test + public void testConv1dCausalAllowed(){ + new Convolution1DLayer.Builder().convolutionMode(ConvolutionMode.Causal).kernelSize(2).build(); + new Subsampling1DLayer.Builder().convolutionMode(ConvolutionMode.Causal).kernelSize(2).build(); + } + + @Test + public void testConv2dNoCausalAllowed(){ + + try{ + new ConvolutionLayer.Builder().convolutionMode(ConvolutionMode.Causal).build(); + fail("Expected exception"); + } catch (Throwable t){ + String m = t.getMessage().toLowerCase(); + assertTrue(m.contains("causal") && m.contains("1d"), m); + } + + try{ + new Deconvolution2D.Builder().convolutionMode(ConvolutionMode.Causal).build(); + fail("Expected exception"); + } catch (Throwable t){ + String m = t.getMessage().toLowerCase(); + assertTrue(m.contains("causal") && m.contains("1d"), m); + } + + try{ + new DepthwiseConvolution2D.Builder().convolutionMode(ConvolutionMode.Causal).build(); + fail("Expected exception"); + } catch (Throwable t){ + String m = t.getMessage().toLowerCase(); + assertTrue(m.contains("causal") && m.contains("1d"), m); + } + + try{ + new SeparableConvolution2D.Builder().convolutionMode(ConvolutionMode.Causal).build(); + fail("Expected exception"); + } catch (Throwable t){ + String m = t.getMessage().toLowerCase(); + assertTrue(m.contains("causal") && m.contains("1d"), m); + } + + try{ + new SubsamplingLayer.Builder().convolutionMode(ConvolutionMode.Causal).build(); + fail("Expected exception"); + } catch (Throwable t){ + String m = t.getMessage().toLowerCase(); + assertTrue(m.contains("causal") && m.contains("1d"), m); + } + } + + @Test + public void testConv3dNoCausalAllowed(){ + try{ + new Convolution3D.Builder().convolutionMode(ConvolutionMode.Causal).build(); + fail("Expected exception"); + } catch (Throwable t){ + String m = t.getMessage().toLowerCase(); + assertTrue(m.contains("causal") && m.contains("1d"), m); + } + + try{ + new Subsampling3DLayer.Builder().convolutionMode(ConvolutionMode.Causal).build(); + fail("Expected exception"); + } catch (Throwable t){ + String m = t.getMessage().toLowerCase(); + assertTrue(m.contains("causal") && m.contains("1d"), m); + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/LocallyConnectedLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/LocallyConnectedLayerTest.java new file mode 100644 index 000000000..fa8c88493 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/LocallyConnectedLayerTest.java @@ -0,0 +1,194 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.convolution; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.buffer.util.DataTypeUtil; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author Max Pumperla + */ +public class LocallyConnectedLayerTest extends BaseDL4JTest { + + @BeforeEach + public void before() { + DataTypeUtil.setDTypeForContext(DataType.DOUBLE); + Nd4j.factory().setDType(DataType.DOUBLE); + Nd4j.EPS_THRESHOLD = 1e-4; + } + + @Test + public void test2dForward(){ + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4) + .updater(new Nesterovs(0.9)).dropOut(0.5) + .list() + .layer(new LocallyConnected2D.Builder().kernelSize(8, 8).nIn(3) + .stride(4, 4).nOut(16).dropOut(0.5) + .convolutionMode(ConvolutionMode.Strict) + .setInputSize(28, 28) + .activation(Activation.RELU).weightInit( + WeightInit.XAVIER) + .build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS) //output layer + .nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 3)); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + INDArray input = Nd4j.ones(10, 3, 28, 28); + INDArray output = network.output(input, false); + + assertArrayEquals(new long[] {10, 10}, output.shape()); + } + + @Test + public void test1dForward(){ + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4) + .updater(new Nesterovs(0.9)).dropOut(0.5) + .list() + .layer(new LocallyConnected1D.Builder().kernelSize(4).nIn(3) + .stride(1).nOut(16).dropOut(0.5) + .convolutionMode(ConvolutionMode.Strict) + .setInputSize(28) + .activation(Activation.RELU).weightInit( + WeightInit.XAVIER) + .build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS) //output layer + .nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.recurrent(3, 8)); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + INDArray input = Nd4j.ones(10, 3, 8); + INDArray output = network.output(input, false);; + for (int i = 0; i < 100; i++) { // TODO: this falls flat for 1000 iterations on my machine + output = network.output(input, false); + } + + assertArrayEquals(new long[] {(8 - 4 + 1) * 10, 10}, output.shape()); + network.fit(input, output); + + } + + @Test + public void testLocallyConnected(){ + for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) { + Nd4j.setDefaultDataTypes(globalDtype, globalDtype); + for (DataType networkDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) { + assertEquals(globalDtype, Nd4j.dataType()); + assertEquals(globalDtype, Nd4j.defaultFloatingPointType()); + + for (int test = 0; test < 2; test++) { + String msg = "Global dtype: " + globalDtype + ", network dtype: " + networkDtype + ", test=" + test; + + ComputationGraphConfiguration.GraphBuilder b = new NeuralNetConfiguration.Builder() + .dataType(networkDtype) + .seed(123) + .updater(new NoOp()) + .weightInit(WeightInit.XAVIER) + .convolutionMode(ConvolutionMode.Same) + .graphBuilder(); + + INDArray[] in; + INDArray label; + switch (test){ + case 0: + b.addInputs("in") + .addLayer("1", new LSTM.Builder().nOut(5).build(), "in") + .addLayer("2", new LocallyConnected1D.Builder().kernelSize(2).nOut(4).build(), "1") + .addLayer("out", new RnnOutputLayer.Builder().nOut(10).build(), "2") + .setOutputs("out") + .setInputTypes(InputType.recurrent(5, 4)); + in = new INDArray[]{Nd4j.rand(networkDtype, 2, 5, 4)}; + label = TestUtils.randomOneHotTimeSeries(2, 10, 4).castTo(networkDtype); + break; + case 1: + b.addInputs("in") + .addLayer("1", new ConvolutionLayer.Builder().kernelSize(2,2).nOut(5).convolutionMode(ConvolutionMode.Same).build(), "in") + .addLayer("2", new LocallyConnected2D.Builder().kernelSize(2,2).nOut(5).build(), "1") + .addLayer("out", new OutputLayer.Builder().nOut(10).build(), "2") + .setOutputs("out") +// .setInputTypes(InputType.convolutional(28, 28, 1)); +// in = new INDArray[]{Nd4j.rand(networkDtype, 2, 1, 28, 28)}; + .setInputTypes(InputType.convolutional(8, 8, 1)); + in = new INDArray[]{Nd4j.rand(networkDtype, 2, 1, 8, 8)}; + label = TestUtils.randomOneHot(2, 10).castTo(networkDtype); + break; + default: + throw new RuntimeException(); + } + + ComputationGraph net = new ComputationGraph(b.build()); + net.init(); + + INDArray out = net.outputSingle(in); + assertEquals(networkDtype, out.dataType(), msg); + Map ff = net.feedForward(in, false); + for (Map.Entry e : ff.entrySet()) { + if (e.getKey().equals("in")) + continue; + String s = msg + " - layer: " + e.getKey(); + assertEquals(networkDtype, e.getValue().dataType(), s); + } + + net.setInputs(in); + net.setLabels(label); + net.computeGradientAndScore(); + + net.fit(new MultiDataSet(in, new INDArray[]{label})); + } + } + } + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepthTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepthTest.java similarity index 76% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepthTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepthTest.java index 8c9b4aac5..f69b0041e 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepthTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepthTest.java @@ -17,82 +17,79 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.layers.convolution; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.conf.layers.SpaceToDepthLayer; + import org.deeplearning4j.nn.api.Layer; import org.deeplearning4j.nn.conf.GradientNormalization; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + import java.util.Arrays; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Space To Depth Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class SpaceToDepthTest extends BaseDL4JTest { +public class SpaceToDepthTest extends BaseDL4JTest { private int mb = 1; - private int inDepth = 2; - private int inputWidth = 2; - private int inputHeight = 2; private int blockSize = 2; - private SpaceToDepthLayer.DataFormat dataFormat = SpaceToDepthLayer.DataFormat.NCHW; private int outDepth = inDepth * blockSize * blockSize; - private int outputHeight = inputHeight / blockSize; - private int outputWidth = inputWidth / blockSize; + private INDArray getContainedData() { - return Nd4j.create(new double[] { 1., 2., 3., 4., 5., 6., 7., 8. }, new int[] { mb, inDepth, inputHeight, inputWidth }, 'c'); + return Nd4j.create(new double[] {1., 2., 3., 4., 5., 6., 7., 8.}, + new int[] {mb, inDepth, inputHeight, inputWidth}, 'c'); } private INDArray getContainedOutput() { - return Nd4j.create(new double[] { 1., 5., 2., 6., 3., 7., 4., 8. }, new int[] { mb, outDepth, outputHeight, outputWidth }, 'c'); + return Nd4j.create(new double[] {1., 5., 2., 6., 3., 7., 4., 8.}, + new int[] {mb, outDepth, outputHeight, outputWidth}, 'c'); } private Layer getSpaceToDepthLayer() { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123).layer(new SpaceToDepthLayer.Builder(blockSize, dataFormat).build()).build(); + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123) + .layer(new SpaceToDepthLayer.Builder(blockSize, dataFormat).build()).build(); return conf.getLayer().instantiate(conf, null, 0, null, true, Nd4j.defaultFloatingPointType()); } @Test - @DisplayName("Test Space To Depth Forward") - void testSpaceToDepthForward() throws Exception { + public void testSpaceToDepthForward() throws Exception { INDArray containedInput = getContainedData(); INDArray containedExpectedOut = getContainedOutput(); Layer std = getSpaceToDepthLayer(); INDArray containedOutput = std.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces()); + assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape())); assertEquals(containedExpectedOut, containedOutput); } @Test - @DisplayName("Test Space To Depth Backward") - void testSpaceToDepthBackward() throws Exception { + public void testSpaceToDepthBackward() throws Exception { INDArray containedInputEpsilon = getContainedOutput(); + INDArray containedExpectedOut = getContainedData(); Layer std = getSpaceToDepthLayer(); + std.setInput(getContainedData(), LayerWorkspaceMgr.noWorkspaces()); INDArray containedOutput = std.backpropGradient(containedInputEpsilon, LayerWorkspaceMgr.noWorkspaces()).getRight(); + assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape())); assertEquals(containedExpectedOut, containedOutput); } -} +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java new file mode 100644 index 000000000..2fca7643a --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java @@ -0,0 +1,249 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.convolution; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; +import org.deeplearning4j.nn.gradient.DefaultGradient; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.params.DefaultParamInitializer; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.junit.jupiter.api.Test; +import org.nd4j.common.primitives.Pair; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Adam Gibson + */ +public class SubsamplingLayerTest extends BaseDL4JTest { + + private int nExamples = 1; + private int depth = 20; //channels & nOut + private int nChannelsIn = 1; + private int inputWidth = 28; + private int inputHeight = 28; + private int[] kernelSize = new int[] {2, 2}; + private int[] stride = new int[] {2, 2}; + + int featureMapWidth = (inputWidth - kernelSize[0]) / stride[0] + 1; + int featureMapHeight = (inputHeight - kernelSize[1]) / stride[0] + 1; + private INDArray epsilon = Nd4j.ones(nExamples, depth, featureMapHeight, featureMapWidth); + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @Test + public void testSubSampleMaxActivate() throws Exception { + INDArray containedExpectedOut = + Nd4j.create(new double[] {5., 7., 6., 8., 4., 7., 5., 9.}, new long[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType()); + INDArray containedInput = getContainedData(); + INDArray input = getData(); + Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX); + + INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces()); + assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape())); + assertEquals(containedExpectedOut, containedOutput); + + INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, featureMapWidth, featureMapHeight}, + output.shape())); + assertEquals(nChannelsIn, output.size(1), 1e-4); // channels retained + } + + @Test + public void testSubSampleMeanActivate() throws Exception { + INDArray containedExpectedOut = + Nd4j.create(new double[] {2., 4., 3., 5., 3.5, 6.5, 4.5, 8.5}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType()); + INDArray containedInput = getContainedData(); + INDArray input = getData(); + Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG); + + INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces()); + assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape())); + assertEquals(containedExpectedOut, containedOutput); + + INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, featureMapWidth, featureMapHeight}, + output.shape())); + assertEquals(nChannelsIn, output.size(1), 1e-4); // channels retained + } + + ////////////////////////////////////////////////////////////////////////////////// + + @Test + public void testSubSampleLayerMaxBackprop() throws Exception { + INDArray expectedContainedEpsilonInput = + Nd4j.create(new double[] {1., 1., 1., 1., 1., 1., 1., 1.}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType()); + + INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 1., + 0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0.}, + new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType()); + + INDArray input = getContainedData(); + + Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX); + layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + + Pair containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond()); + assertEquals(null, containedOutput.getFirst().getGradientFor("W")); + assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length); + + INDArray input2 = getData(); + layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces()); + long depth = input2.size(1); + + epsilon = Nd4j.ones(5, depth, featureMapHeight, featureMapWidth); + + Pair out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(input.shape().length, out.getSecond().shape().length); + assertEquals(depth, out.getSecond().size(1)); // channels retained + } + + @Test + public void testSubSampleLayerAvgBackprop() throws Exception { + INDArray expectedContainedEpsilonInput = + Nd4j.create(new double[] {1., 2., 3., 4., 5., 6., 7., 8.}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType()); + + INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {0.25, 0.25, 0.5, 0.5, 0.25, 0.25, 0.5, 0.5, + 0.75, 0.75, 1., 1., 0.75, 0.75, 1., 1., 1.25, 1.25, 1.5, 1.5, 1.25, 1.25, 1.5, 1.5, 1.75, 1.75, + 2., 2., 1.75, 1.75, 2., 2.}, new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType()); + INDArray input = getContainedData(); + + Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG); + layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + + Pair containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond()); + assertEquals(null, containedOutput.getFirst().getGradientFor("W")); + assertArrayEquals(expectedContainedEpsilonResult.shape(), containedOutput.getSecond().shape()); + + } + + + @Test + public void testSubSampleLayerSumBackprop() throws Exception { + assertThrows(UnsupportedOperationException.class, () -> { + Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.SUM); + INDArray input = getData(); + layer.setInput(input, LayerWorkspaceMgr.noWorkspaces()); + layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); + }); + } + + ////////////////////////////////////////////////////////////////////////////////// + + private Layer getSubsamplingLayer(SubsamplingLayer.PoolingType pooling) { + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123) + .layer(new SubsamplingLayer.Builder(pooling, new int[] {2, 2}).build()).build(); + + return conf.getLayer().instantiate(conf, null, 0, null, true, Nd4j.defaultFloatingPointType()); + } + + public INDArray getData() throws Exception { + DataSetIterator data = new MnistDataSetIterator(5, 5); + DataSet mnist = data.next(); + nExamples = mnist.numExamples(); + return mnist.getFeatures().reshape(nExamples, nChannelsIn, inputWidth, inputHeight); + } + + public INDArray getContainedData() { + INDArray ret = Nd4j.create(new double[] {1., 1., 3., 7., 5., 1., 3., 3., 2., 2., 8., 4., 2., 6., 4., 4., 3., 3., + 6., 7., 4., 4., 6., 7., 5., 5., 9., 8., 4., 4., 9., 8.}, new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType()); + return ret; + } + + private Gradient createPrevGradient() { + Gradient gradient = new DefaultGradient(); + INDArray pseudoGradients = Nd4j.ones(nExamples, nChannelsIn, inputHeight, inputWidth); + + gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, pseudoGradients); + gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, pseudoGradients); + return gradient; + } + + ////////////////////////////////////////////////////////////////////////////////// + + @Test + public void testSubTooLargeKernel() { + assertThrows(Exception.class, () -> { + int imageHeight = 20; + int imageWidth = 23; + int nChannels = 1; + int classes = 2; + int numSamples = 200; + + int kernelHeight = 3; + int kernelWidth = 3; + + DataSet trainInput; + MultiLayerConfiguration.Builder builder = + new NeuralNetConfiguration.Builder().seed(123).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder( + kernelHeight, kernelWidth).stride(1, 1).nOut(2) + .activation(Activation.RELU).weightInit( + WeightInit.XAVIER) + .build()) + .layer(1, new SubsamplingLayer.Builder() + .poolingType(SubsamplingLayer.PoolingType.MAX) + .kernelSize(imageHeight - kernelHeight + 2, 1) //imageHeight-kernelHeight+1 is ok: full height + .stride(1, 1).build()) + .layer(2, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).build()) + + .setInputType(InputType.convolutional(imageHeight, imageWidth, nChannels)); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); + INDArray emptyLables = Nd4j.zeros(numSamples, classes); + + trainInput = new DataSet(emptyFeatures, emptyLables); + model.fit(trainInput); + }); + } + + + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/TestConvolutionModes.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/TestConvolutionModes.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/TestConvolutionModes.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/TestConvolutionModes.java index 3d9232e09..6cc561ceb 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/TestConvolutionModes.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/TestConvolutionModes.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.util.ConvolutionUtils; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -51,8 +48,6 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class TestConvolutionModes extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling1DTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling1DTest.java similarity index 76% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling1DTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling1DTest.java index 3571d167a..0504c4fac 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling1DTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling1DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.layers.convolution; import lombok.val; @@ -27,84 +28,91 @@ import org.deeplearning4j.nn.conf.GradientNormalization; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.Upsampling1D; import org.deeplearning4j.nn.gradient.Gradient; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Pair; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + import java.util.Arrays; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** * @author Max Pumperla */ -@DisplayName("Upsampling 1 D Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class Upsampling1DTest extends BaseDL4JTest { +public class Upsampling1DTest extends BaseDL4JTest { private int nExamples = 1; - private int depth = 20; - private int nChannelsIn = 1; - private int inputLength = 28; - private int size = 2; - private int outputLength = inputLength * size; - private INDArray epsilon = Nd4j.ones(nExamples, depth, outputLength); + @Test - @DisplayName("Test Upsampling 1 D") - void testUpsampling1D() throws Exception { - double[] outArray = new double[] { 1., 1., 2., 2., 3., 3., 4., 4. }; - INDArray containedExpectedOut = Nd4j.create(outArray, new int[] { 1, 1, 8 }); + public void testUpsampling1D() throws Exception { + + double[] outArray = new double[] {1., 1., 2., 2., 3., 3., 4., 4.}; + INDArray containedExpectedOut = Nd4j.create(outArray, new int[] {1, 1, 8}); INDArray containedInput = getContainedData(); INDArray input = getData(); - Layer layer = getUpsampling1DLayer(); + Layer layer = getUpsampling1DLayer(); + INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces()); assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape())); assertEquals(containedExpectedOut, containedOutput); + INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertTrue(Arrays.equals(new long[] { nExamples, nChannelsIn, outputLength }, output.shape())); + assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, outputLength}, + output.shape())); assertEquals(nChannelsIn, output.size(1), 1e-4); } + @Test - @DisplayName("Test Upsampling 1 D Backprop") - void testUpsampling1DBackprop() throws Exception { - INDArray expectedContainedEpsilonInput = Nd4j.create(new double[] { 1., 3., 2., 6., 7., 2., 5., 5. }, new int[] { 1, 1, 8 }); - INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] { 4., 8., 9., 10. }, new int[] { 1, 1, 4 }); + public void testUpsampling1DBackprop() throws Exception { + INDArray expectedContainedEpsilonInput = + Nd4j.create(new double[] {1., 3., 2., 6., 7., 2., 5., 5.}, + new int[] {1, 1, 8}); + + INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {4., 8., 9., 10.}, + new int[] {1, 1, 4}); + INDArray input = getContainedData(); + Layer layer = getUpsampling1DLayer(); layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + Pair containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond()); assertEquals(null, containedOutput.getFirst().getGradientFor("W")); assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length); + INDArray input2 = getData(); layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces()); val depth = input2.size(1); + epsilon = Nd4j.ones(5, depth, outputLength); + Pair out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); assertEquals(input.shape().length, out.getSecond().shape().length); assertEquals(depth, out.getSecond().size(1)); } + private Layer getUpsampling1DLayer() { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123).layer(new Upsampling1D.Builder(size).build()).build(); - return conf.getLayer().instantiate(conf, null, 0, null, true, Nd4j.defaultFloatingPointType()); + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123) + .layer(new Upsampling1D.Builder(size).build()).build(); + return conf.getLayer().instantiate(conf, null, 0, + null, true, Nd4j.defaultFloatingPointType()); } public INDArray getData() throws Exception { @@ -116,7 +124,10 @@ class Upsampling1DTest extends BaseDL4JTest { } private INDArray getContainedData() { - INDArray ret = Nd4j.create(new double[] { 1., 2., 3., 4. }, new int[] { 1, 1, 4 }); + INDArray ret = Nd4j.create + (new double[] {1., 2., 3., 4.}, + new int[] {1, 1, 4}); return ret; } + } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling2DTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling2DTest.java similarity index 78% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling2DTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling2DTest.java index 1fae34297..a0ee3de55 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling2DTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.layers.convolution; import lombok.val; @@ -27,86 +28,92 @@ import org.deeplearning4j.nn.conf.GradientNormalization; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.Upsampling2D; import org.deeplearning4j.nn.gradient.Gradient; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Pair; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + import java.util.Arrays; + import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** * @author Max Pumperla */ -@DisplayName("Upsampling 2 D Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class Upsampling2DTest extends BaseDL4JTest { +public class Upsampling2DTest extends BaseDL4JTest { private int nExamples = 1; - private int depth = 20; - private int nChannelsIn = 1; - private int inputWidth = 28; - private int inputHeight = 28; private int size = 2; - private int outputWidth = inputWidth * size; - private int outputHeight = inputHeight * size; private INDArray epsilon = Nd4j.ones(nExamples, depth, outputHeight, outputWidth); + @Test - @DisplayName("Test Upsampling") - void testUpsampling() throws Exception { - double[] outArray = new double[] { 1., 1., 2., 2., 1., 1., 2., 2., 3., 3., 4., 4., 3., 3., 4., 4. }; - INDArray containedExpectedOut = Nd4j.create(outArray, new int[] { 1, 1, 4, 4 }); + public void testUpsampling() throws Exception { + + double[] outArray = new double[] {1., 1., 2., 2., 1., 1., 2., 2., 3., 3., 4., 4., 3., 3., 4., 4.}; + INDArray containedExpectedOut = Nd4j.create(outArray, new int[] {1, 1, 4, 4}); INDArray containedInput = getContainedData(); INDArray input = getData(); Layer layer = getUpsamplingLayer(); + INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces()); assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape())); assertEquals(containedExpectedOut, containedOutput); + INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertTrue(Arrays.equals(new long[] { nExamples, nChannelsIn, outputWidth, outputHeight }, output.shape())); + assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, outputWidth, outputHeight}, + output.shape())); assertEquals(nChannelsIn, output.size(1), 1e-4); } + @Test - @DisplayName("Test Upsampling 2 D Backprop") - void testUpsampling2DBackprop() throws Exception { - INDArray expectedContainedEpsilonInput = Nd4j.create(new double[] { 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1. }, new int[] { 1, 1, 4, 4 }); - INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] { 4., 4., 4., 4. }, new int[] { 1, 1, 2, 2 }); + public void testUpsampling2DBackprop() throws Exception { + INDArray expectedContainedEpsilonInput = + Nd4j.create(new double[] {1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.}, + new int[] {1, 1, 4, 4}); + + INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {4., 4., 4., 4.}, + new int[] {1, 1, 2, 2}); + INDArray input = getContainedData(); + Layer layer = getUpsamplingLayer(); layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); + Pair containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond()); assertEquals(null, containedOutput.getFirst().getGradientFor("W")); assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length); + INDArray input2 = getData(); layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces()); val depth = input2.size(1); + epsilon = Nd4j.ones(5, depth, outputHeight, outputWidth); + Pair out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); assertEquals(input.shape().length, out.getSecond().shape().length); assertEquals(depth, out.getSecond().size(1)); } + private Layer getUpsamplingLayer() { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123).layer(new Upsampling2D.Builder(size).build()).build(); + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123) + .layer(new Upsampling2D.Builder(size).build()).build(); return conf.getLayer().instantiate(conf, null, 0, null, true, Nd4j.defaultFloatingPointType()); } @@ -118,7 +125,10 @@ class Upsampling2DTest extends BaseDL4JTest { } private INDArray getContainedData() { - INDArray ret = Nd4j.create(new double[] { 1., 2., 3., 4. }, new int[] { 1, 1, 2, 2 }); + INDArray ret = Nd4j.create + (new double[] {1., 2., 3., 4.}, + new int[] {1, 1, 2, 2}); return ret; } + } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomActivation.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomActivation.java similarity index 87% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomActivation.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomActivation.java index b1be86a47..2f837fc2f 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomActivation.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomActivation.java @@ -26,26 +26,20 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.DenseLayer; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.layers.custom.testclasses.CustomActivation; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.learning.config.Sgd; import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.introspect.AnnotatedClass; -import org.nd4j.shade.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.introspect.AnnotatedClass; +import com.fasterxml.jackson.databind.jsontype.NamedType; import java.util.Collection; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) public class TestCustomActivation extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomLayers.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomLayers.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomLayers.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomLayers.java index 3db1efe4a..a0de7f2df 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomLayers.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/TestCustomLayers.java @@ -33,18 +33,15 @@ import org.deeplearning4j.nn.layers.custom.testclasses.CustomOutputLayer; import org.deeplearning4j.nn.layers.custom.testclasses.CustomOutputLayerImpl; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.introspect.AnnotatedClass; -import org.nd4j.shade.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.introspect.AnnotatedClass; +import com.fasterxml.jackson.databind.jsontype.NamedType; import java.util.Collection; import java.util.HashSet; @@ -53,9 +50,6 @@ import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) public class TestCustomLayers extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomActivation.java diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayer.java new file mode 100644 index 000000000..1eacc4d20 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayer.java @@ -0,0 +1,90 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.custom.testclasses; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.nn.api.ParamInitializer; +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; +import org.deeplearning4j.nn.conf.memory.LayerMemoryReport; +import org.deeplearning4j.nn.params.DefaultParamInitializer; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collection; +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CustomLayer extends FeedForwardLayer { + + private final double someCustomParameter; + + public CustomLayer(@JsonProperty("someCustomParameter") double someCustomParameter) { + this.someCustomParameter = someCustomParameter; + this.nIn = 10; + this.nOut = 10; + } + + @Override + public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf, + Collection trainingListeners, int layerIndex, INDArray layerParamsView, + boolean initializeParams, DataType networkDataType) { + CustomLayerImpl ret = new CustomLayerImpl(conf, networkDataType); + ret.setListeners(trainingListeners); + ret.setIndex(layerIndex); + ret.setParamsViewArray(layerParamsView); + Map paramTable = initializer().init(conf, layerParamsView, initializeParams); + ret.setParamTable(paramTable); + ret.setConf(conf); + return ret; + } + + @Override + public ParamInitializer initializer() { + return DefaultParamInitializer.getInstance(); + } + + @Override + public InputType getOutputType(int layerIndex, InputType inputType) { + return InputType.feedForward(10); + } + + @Override + public void setNIn(InputType inputType, boolean override) { + //No op + } + + @Override + public InputPreProcessor getPreProcessorForInputType(InputType inputType) { + return null; + } + + @Override + public LayerMemoryReport getMemoryReport(InputType inputType) { + throw new UnsupportedOperationException(); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayerImpl.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayerImpl.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayerImpl.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayerImpl.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayer.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayer.java index 64fb0416d..88972c96a 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayer.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayer.java @@ -68,7 +68,7 @@ public class CustomOutputLayer extends BaseOutputLayer { } @NoArgsConstructor - public static class Builder extends BaseOutputLayer.Builder { + public static class Builder extends BaseOutputLayer.Builder { public Builder(LossFunctions.LossFunction lossFunction) { super.lossFunction(lossFunction); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayerImpl.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayerImpl.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayerImpl.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomOutputLayerImpl.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseTest.java similarity index 78% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseTest.java index c1b1de017..2c4968e52 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.layers.feedforward.dense; import org.deeplearning4j.BaseDL4JTest; @@ -29,10 +30,7 @@ import org.deeplearning4j.nn.conf.layers.DenseLayer; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -40,85 +38,105 @@ import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.learning.config.Sgd; import org.nd4j.linalg.lossfunctions.LossFunctions; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Dense Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class DenseTest extends BaseDL4JTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DenseTest extends BaseDL4JTest { private int numSamples = 150; - private int batchSize = 150; - private DataSetIterator iter = new IrisDataSetIterator(batchSize, numSamples); - private DataSet data; @Test - @DisplayName("Test Dense Bias Init") - void testDenseBiasInit() { + public void testDenseBiasInit() { DenseLayer build = new DenseLayer.Builder().nIn(1).nOut(3).biasInit(1).build(); + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(build).build(); + long numParams = conf.getLayer().initializer().numParams(conf); INDArray params = Nd4j.create(1, numParams); Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true, Nd4j.defaultFloatingPointType()); + assertEquals(1, layer.getParam("b").size(0)); } @Test - @DisplayName("Test MLP Multi Layer Pretrain") - void testMLPMultiLayerPretrain() { + public void testMLPMultiLayerPretrain() { // Note CNN does not do pretrain MultiLayerNetwork model = getDenseMLNConfig(false, true); model.fit(iter); + MultiLayerNetwork model2 = getDenseMLNConfig(false, true); model2.fit(iter); iter.reset(); + DataSet test = iter.next(); + assertEquals(model.params(), model2.params()); + Evaluation eval = new Evaluation(); INDArray output = model.output(test.getFeatures()); eval.eval(test.getLabels(), output); double f1Score = eval.f1(); + Evaluation eval2 = new Evaluation(); INDArray output2 = model2.output(test.getFeatures()); eval2.eval(test.getLabels(), output2); double f1Score2 = eval2.f1(); + assertEquals(f1Score, f1Score2, 1e-4); + } @Test - @DisplayName("Test MLP Multi Layer Backprop") - void testMLPMultiLayerBackprop() { + public void testMLPMultiLayerBackprop() { MultiLayerNetwork model = getDenseMLNConfig(true, false); model.fit(iter); + MultiLayerNetwork model2 = getDenseMLNConfig(true, false); model2.fit(iter); iter.reset(); + DataSet test = iter.next(); + assertEquals(model.params(), model2.params()); + Evaluation eval = new Evaluation(); INDArray output = model.output(test.getFeatures()); eval.eval(test.getLabels(), output); double f1Score = eval.f1(); + Evaluation eval2 = new Evaluation(); INDArray output2 = model2.output(test.getFeatures()); eval2.eval(test.getLabels(), output2); double f1Score2 = eval2.f1(); + assertEquals(f1Score, f1Score2, 1e-4); + } - // //////////////////////////////////////////////////////////////////////////////// + + ////////////////////////////////////////////////////////////////////////////////// + private static MultiLayerNetwork getDenseMLNConfig(boolean backprop, boolean pretrain) { int numInputs = 4; int outputNum = 3; long seed = 6; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed).updater(new Sgd(1e-3)).l1(0.3).l2(1e-3).list().layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(numInputs).nOut(3).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(1, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(3).nOut(2).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).nIn(2).nOut(outputNum).activation(Activation.SOFTMAX).build()).build(); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed) + .updater(new Sgd(1e-3)).l1(0.3).l2(1e-3).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(numInputs).nOut(3) + .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(3).nOut(2) + .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).nIn(2).nOut(outputNum).activation(Activation.SOFTMAX).build()) + .build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); model.init(); return model; + } } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayerTest.java new file mode 100644 index 000000000..30e221c1a --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayerTest.java @@ -0,0 +1,814 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.feedforward.embedding; + +import lombok.EqualsAndHashCode; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.RNNFormat; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.EmbeddingLayer; +import org.deeplearning4j.nn.conf.layers.EmbeddingSequenceLayer; +import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.params.DefaultParamInitializer; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.weights.embeddings.EmbeddingInitializer; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.impl.ActivationIdentity; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.*; + +public class EmbeddingLayerTest extends BaseDL4JTest { + + @Test + public void testEmbeddingLayerConfig() { + + for (boolean hasBias : new boolean[]{true, false}) { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(0, new EmbeddingLayer.Builder().hasBias(hasBias).nIn(10).nOut(5).build()) + .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + Layer l0 = net.getLayer(0); + + assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingLayer.class, l0.getClass()); + assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn()); + assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut()); + + INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY); + INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY); + assertArrayEquals(new long[]{10, 5}, weights.shape()); + if (hasBias) { + assertArrayEquals(new long[]{1, 5}, bias.shape()); + } + } + } + + @Test + public void testEmbeddingSequenceLayerConfig() { + + int inputLength = 6; + int nIn = 10; + int embeddingDim = 5; + int nout = 4; + + for (boolean hasBias : new boolean[]{true, false}) { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(new EmbeddingSequenceLayer.Builder().hasBias(hasBias) + .inputLength(inputLength).nIn(nIn).nOut(embeddingDim).build()) + .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nout).activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + Layer l0 = net.getLayer(0); + + assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingSequenceLayer.class, l0.getClass()); + assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn()); + assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut()); + + INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY); + INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY); + assertArrayEquals(new long[]{10, 5}, weights.shape()); + if (hasBias) { + assertArrayEquals(new long[]{1, 5}, bias.shape()); + } + } + } + + @Test + public void testEmbeddingLongerSequencesForwardPass() { + + int nClassesIn = 10; + int inputLength = 6; + int embeddingDim = 5; + int nOut = 4; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(new EmbeddingSequenceLayer.Builder().inputLength(inputLength) + .hasBias(true).nIn(nClassesIn).nOut(embeddingDim).build()) + .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build()) + .build(); + + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + int batchSize = 3; + + INDArray inEmbedding = Nd4j.create(batchSize, inputLength); + + Random r = new Random(12345); + for (int i = 0; i < batchSize; i++) { + int classIdx = r.nextInt(nClassesIn); + inEmbedding.putScalar(i, classIdx); + } + + INDArray output = net.output(inEmbedding); + + assertArrayEquals(new long[]{batchSize, nOut, inputLength}, output.shape()); + } + + @Test + public void testEmbeddingSingleSequenceForwardPass() { + int nClassesIn = 10; + int embeddingDim = 5; + int nOut = 4; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(new EmbeddingSequenceLayer.Builder().inputLength(1) + .hasBias(true).nIn(nClassesIn).nOut(embeddingDim).build()) + .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build()) + .build(); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build()) + .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build()) + .inputPreProcessor(0, new RnnToFeedForwardPreProcessor()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net.init(); + net2.init(); + + net2.setParams(net.params().dup()); + + int batchSize = 3; + INDArray inEmbedding = Nd4j.create(batchSize, 1); + INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, 1); + + Random r = new Random(12345); + for (int i = 0; i < batchSize; i++) { + int classIdx = r.nextInt(nClassesIn); + inEmbedding.putScalar(i, classIdx); + inOneHot.putScalar(new int[]{i, classIdx, 0}, 1.0); + } + + List activationsDense = net2.feedForward(inOneHot, false); + List activationEmbedding = net.feedForward(inEmbedding, false); + + INDArray actD1 = activationsDense.get(1); + INDArray actE1 = activationEmbedding.get(1).reshape(batchSize, embeddingDim); + assertEquals(actD1, actE1); + + + INDArray actD2 = activationsDense.get(2); + INDArray actE2 = activationEmbedding.get(2).reshape(batchSize, nOut); + assertEquals(actD2, actE2); + } + + @Test + public void testEmbeddingForwardPass() { + //With the same parameters, embedding layer should have same activations as the equivalent one-hot representation + // input with a DenseLayer + + int nClassesIn = 10; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build()) + .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build()) + .build(); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build()) + .layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net.init(); + net2.init(); + + net2.setParams(net.params().dup()); + + int batchSize = 3; + INDArray inEmbedding = Nd4j.create(batchSize, 1); + INDArray inOneHot = Nd4j.create(batchSize, nClassesIn); + + Random r = new Random(12345); + for (int i = 0; i < batchSize; i++) { + int classIdx = r.nextInt(nClassesIn); + inEmbedding.putScalar(i, classIdx); + inOneHot.putScalar(new int[]{i, classIdx}, 1.0); + } + + List activationsEmbedding = net.feedForward(inEmbedding, false); + List activationsDense = net2.feedForward(inOneHot, false); + for (int i = 1; i < 3; i++) { + INDArray actE = activationsEmbedding.get(i); + INDArray actD = activationsDense.get(i); + assertEquals(actE, actD); + } + } + + @Test + public void testEmbeddingBackwardPass() { + //With the same parameters, embedding layer should have same activations as the equivalent one-hot representation + // input with a DenseLayer + + int nClassesIn = 10; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build()).layer(1, + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4) + .activation(Activation.SOFTMAX).build()) + .build(); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).list() + .layer(new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4) + .activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net.init(); + net2.init(); + + net2.setParams(net.params().dup()); + + int batchSize = 3; + INDArray inEmbedding = Nd4j.create(batchSize, 1); + INDArray inOneHot = Nd4j.create(batchSize, nClassesIn); + INDArray outLabels = Nd4j.create(batchSize, 4); + + Random r = new Random(12345); + for (int i = 0; i < batchSize; i++) { + int classIdx = r.nextInt(nClassesIn); + inEmbedding.putScalar(i, classIdx); + inOneHot.putScalar(new int[]{i, classIdx}, 1.0); + + int labelIdx = r.nextInt(4); + outLabels.putScalar(new int[]{i, labelIdx}, 1.0); + } + + net.setInput(inEmbedding); + net2.setInput(inOneHot); + net.setLabels(outLabels); + net2.setLabels(outLabels); + + net.computeGradientAndScore(); + net2.computeGradientAndScore(); + + assertEquals(net2.score(), net.score(), 1e-6); + + Map gradient = net.gradient().gradientForVariable(); + Map gradient2 = net2.gradient().gradientForVariable(); + assertEquals(gradient.size(), gradient2.size()); + + for (String s : gradient.keySet()) { + assertEquals(gradient2.get(s), gradient.get(s)); + } + } + + + @Test + public void testEmbeddingSequenceBackwardPass() { + int nClassesIn = 10; + int embeddingDim = 5; + int nOut = 4; + int inputLength = 1; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(new EmbeddingSequenceLayer.Builder().inputLength(inputLength) + .hasBias(true).nIn(nClassesIn).nOut(embeddingDim).build()) + .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.recurrent(nClassesIn,inputLength,RNNFormat.NCW)) + .build(); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list() + .layer(new DenseLayer.Builder().nIn(nClassesIn).nOut(embeddingDim).activation(Activation.IDENTITY).build()) + .layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.recurrent(nClassesIn,inputLength,RNNFormat.NCW)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net.init(); + net2.init(); + + net2.setParams(net.params().dup()); + + int batchSize = 3; + INDArray inEmbedding = Nd4j.create(batchSize, 1); + INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, 1); + INDArray outLabels = Nd4j.create(batchSize, 4, 1); + + Random r = new Random(1337); + for (int i = 0; i < batchSize; i++) { + int classIdx = r.nextInt(nClassesIn); + inEmbedding.putScalar(i, classIdx); + inOneHot.putScalar(new int[]{i, classIdx, 0}, 1.0); + + int labelIdx = r.nextInt(4); + outLabels.putScalar(new int[]{i, labelIdx, 0}, 1.0); + } + + net.setInput(inEmbedding); + net2.setInput(inOneHot); + net.setLabels(outLabels); + net2.setLabels(outLabels); + + net.computeGradientAndScore(); + net2.computeGradientAndScore(); + +// System.out.println(net.score() + "\t" + net2.score()); + assertEquals(net2.score(), net.score(), 1e-6); + + Map gradient = net.gradient().gradientForVariable(); + Map gradient2 = net2.gradient().gradientForVariable(); + assertEquals(gradient.size(), gradient2.size()); + + for (String s : gradient.keySet()) { + assertEquals(gradient2.get(s), gradient.get(s)); + } + } + + @Test + public void testEmbeddingLayerRNN() { + int nClassesIn = 10; + int batchSize = 3; + int timeSeriesLength = 8; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH) + .dataType(DataType.DOUBLE) + .list() + .layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build()) + .layer(1, new LSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build()) + .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4) + .activation(Activation.SOFTMAX).build()) + .inputPreProcessor(0, new RnnToFeedForwardPreProcessor()) + .inputPreProcessor(1, new FeedForwardToRnnPreProcessor()) + .setInputType(InputType.recurrent(nClassesIn,timeSeriesLength, RNNFormat.NCW)) + .build(); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .dataType(DataType.DOUBLE) + .list() + .layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build()) + .layer(1, new LSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build()) + .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4) + .activation(Activation.SOFTMAX).build()) + .inputPreProcessor(0, new RnnToFeedForwardPreProcessor()) + .inputPreProcessor(1, new FeedForwardToRnnPreProcessor()) + .setInputType(InputType.recurrent(nClassesIn,timeSeriesLength, RNNFormat.NCW)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net.init(); + net2.init(); + + net2.setParams(net.params().dup()); + + ; + INDArray inEmbedding = Nd4j.create(batchSize, 1, timeSeriesLength); + INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, timeSeriesLength); + INDArray outLabels = Nd4j.create(batchSize, 4, timeSeriesLength); + + Random r = new Random(12345); + for (int i = 0; i < batchSize; i++) { + for (int j = 0; j < timeSeriesLength; j++) { + int classIdx = r.nextInt(nClassesIn); + inEmbedding.putScalar(new int[]{i, 0, j}, classIdx); + inOneHot.putScalar(new int[]{i, classIdx, j}, 1.0); + + int labelIdx = r.nextInt(4); + outLabels.putScalar(new int[]{i, labelIdx, j}, 1.0); + } + } + + net.setInput(inEmbedding); + net2.setInput(inOneHot); + net.setLabels(outLabels); + net2.setLabels(outLabels); + + net.computeGradientAndScore(); + net2.computeGradientAndScore(); + +// System.out.println(net.score() + "\t" + net2.score()); + assertEquals(net2.score(), net.score(), 1e-5); + + Map gradient = net.gradient().gradientForVariable(); + Map gradient2 = net2.gradient().gradientForVariable(); + assertEquals(gradient.size(), gradient2.size()); + + for (String s : gradient.keySet()) { + assertEquals(gradient2.get(s), gradient.get(s)); + } + + } + + @Test + public void testEmbeddingLayerWithMasking() { + //Idea: have masking on the input with an embedding and dense layers on input + //Ensure that the parameter gradients for the inputs don't depend on the inputs when inputs are masked + + int[] miniBatchSizes = {1, 2, 5}; + int nIn = 2; + Random r = new Random(12345); + + int numInputClasses = 10; + int timeSeriesLength = 5; + + for (DataType maskDtype : new DataType[]{DataType.FLOAT, DataType.DOUBLE, DataType.INT}) { + for (int nExamples : miniBatchSizes) { + Nd4j.getRandom().setSeed(12345); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(0.1)).seed(12345).list() + .layer(0, new EmbeddingLayer.Builder().hasBias(true).activation(Activation.TANH).nIn(numInputClasses) + .nOut(5).build()) + .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build()) + .layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build()) + .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3) + .nOut(4).build()) + .inputPreProcessor(0, new RnnToFeedForwardPreProcessor()) + .inputPreProcessor(2, new FeedForwardToRnnPreProcessor()) + .setInputType(InputType.recurrent(numInputClasses,timeSeriesLength, RNNFormat.NCW)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(0.1)).seed(12345).list() + .layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(numInputClasses).nOut(5) + .build()) + .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build()) + .layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build()) + .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3) + .nOut(4).build()) + .inputPreProcessor(0, new RnnToFeedForwardPreProcessor()) + .inputPreProcessor(2, new FeedForwardToRnnPreProcessor()) + .setInputType(InputType.recurrent(numInputClasses,timeSeriesLength, RNNFormat.NCW)) + .build(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + net2.setParams(net.params().dup()); + + INDArray inEmbedding = Nd4j.zeros(nExamples, 1, timeSeriesLength); + INDArray inDense = Nd4j.zeros(nExamples, numInputClasses, timeSeriesLength); + + INDArray labels = Nd4j.zeros(nExamples, 4, timeSeriesLength); + + for (int i = 0; i < nExamples; i++) { + for (int j = 0; j < timeSeriesLength; j++) { + int inIdx = r.nextInt(numInputClasses); + inEmbedding.putScalar(new int[]{i, 0, j}, inIdx); + inDense.putScalar(new int[]{i, inIdx, j}, 1.0); + + int outIdx = r.nextInt(4); + labels.putScalar(new int[]{i, outIdx, j}, 1.0); + } + } + + INDArray inputMask = Nd4j.zeros(maskDtype, nExamples, timeSeriesLength); + for (int i = 0; i < nExamples; i++) { + for (int j = 0; j < timeSeriesLength; j++) { + inputMask.putScalar(new int[]{i, j}, (r.nextBoolean() ? 1.0 : 0.0)); + } + } + + net.setLayerMaskArrays(inputMask, null); + net2.setLayerMaskArrays(inputMask, null); + List actEmbedding = net.feedForward(inEmbedding, false); + List actDense = net2.feedForward(inDense, false); + for (int i = 1; i < actEmbedding.size(); i++) { + assertEquals(actDense.get(i), actEmbedding.get(i)); + } + + net.setLabels(labels); + net2.setLabels(labels); + net.computeGradientAndScore(); + net2.computeGradientAndScore(); + +// System.out.println(net.score() + "\t" + net2.score()); + assertEquals(net2.score(), net.score(), 1e-5); + + Map gradients = net.gradient().gradientForVariable(); + Map gradients2 = net2.gradient().gradientForVariable(); + assertEquals(gradients.keySet(), gradients2.keySet()); + for (String s : gradients.keySet()) { + assertEquals(gradients2.get(s), gradients.get(s)); + } + } + } + } + + + @Test + public void testW2VInits(){ + Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); + + for( int i=0; i<2; i++ ) { + + INDArray vectors = Nd4j.linspace(1,15,15, DataType.FLOAT).reshape(5,3); + + EmbeddingLayer el; + if(i == 0){ + el = new EmbeddingLayer.Builder().weightInit(vectors).build(); + } else { + el = new EmbeddingLayer.Builder().weightInit(new WordVectorsMockup()).build(); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345).list() + .layer(el) + .layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(3).nOut(3).build()) + .layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3) + .nOut(4).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray w = net.getParam("0_W"); + assertEquals(vectors, w); + + TestUtils.testModelSerialization(net); + + //Test same thing for embedding sequence layer: + EmbeddingSequenceLayer esl; + if(i == 0){ + esl = new EmbeddingSequenceLayer.Builder().weightInit(vectors).build(); + } else { + esl = new EmbeddingSequenceLayer.Builder().weightInit(new WordVectorsMockup()).build(); + } + + conf = new NeuralNetConfiguration.Builder() + .seed(12345).list() + .layer(esl) + .layer(new GlobalPoolingLayer()) + .layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(3).nOut(3).build()) + .layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3) + .nOut(4).build()) + .build(); + + net = new MultiLayerNetwork(conf); + net.init(); + + w = net.getParam("0_W"); + assertEquals(vectors, w); + + TestUtils.testModelSerialization(net); + } + } + + @Test + public void testEmbeddingSequenceLayerWithMasking() { + //Idea: have masking on the input with an embedding and dense layers on input + //Ensure that the parameter gradients for the inputs don't depend on the inputs when inputs are masked + + int[] miniBatchSizes = {1, 3}; + int nIn = 2; + Random r = new Random(12345); + + int numInputClasses = 10; + int timeSeriesLength = 5; + + for (DataType maskDtype : new DataType[]{DataType.FLOAT, DataType.DOUBLE, DataType.INT}) { + for (DataType inLabelDtype : new DataType[]{DataType.FLOAT, DataType.DOUBLE, DataType.INT}) { + for(int inputRank : new int[]{2, 3}) { + for (int nExamples : miniBatchSizes) { + Nd4j.getRandom().setSeed(12345); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(0.1)).seed(12345).list() + .layer(0, new EmbeddingSequenceLayer.Builder().hasBias(true).activation(Activation.TANH).nIn(numInputClasses) + .nOut(5).build()) + .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build()) + .layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build()) + .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3) + .nOut(4).build()) + .setInputType(InputType.recurrent(numInputClasses,timeSeriesLength,RNNFormat.NCW)).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(0.1)).seed(12345).list() + .layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(numInputClasses).nOut(5) + .build()) + .layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build()) + .layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).dataFormat(RNNFormat.NCW).build()) + .layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3) + .nOut(4).build()) + .setInputType(InputType.recurrent(numInputClasses,1,RNNFormat.NCW)).build(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + net2.setParams(net.params().dup()); + + INDArray inEmbedding = Nd4j.zeros(inLabelDtype, inputRank == 2 ? new long[]{nExamples, timeSeriesLength} : new long[]{nExamples, 1, timeSeriesLength}); + INDArray inDense = Nd4j.zeros(inLabelDtype, nExamples, numInputClasses, timeSeriesLength); + + INDArray labels = Nd4j.zeros(inLabelDtype, nExamples, 4, timeSeriesLength); + + for (int i = 0; i < nExamples; i++) { + for (int j = 0; j < timeSeriesLength; j++) { + int inIdx = r.nextInt(numInputClasses); + inEmbedding.putScalar(inputRank == 2 ? new int[]{i, j} : new int[]{i, 0, j}, inIdx); + inDense.putScalar(new int[]{i, inIdx, j}, 1.0); + + int outIdx = r.nextInt(4); + labels.putScalar(new int[]{i, outIdx, j}, 1.0); + } + } + + INDArray inputMask = Nd4j.zeros(maskDtype, nExamples, timeSeriesLength); + for (int i = 0; i < nExamples; i++) { + for (int j = 0; j < timeSeriesLength; j++) { + inputMask.putScalar(new int[]{i, j}, (r.nextBoolean() ? 1.0 : 0.0)); + } + } + + net.setLayerMaskArrays(inputMask, null); + net2.setLayerMaskArrays(inputMask, null); + List actEmbedding = net.feedForward(inEmbedding, false); + List actDense = net2.feedForward(inDense, false); + for (int i = 2; i < actEmbedding.size(); i++) { //Start from layer 2: EmbeddingSequence is 3d, first dense is 2d (before reshape) + assertEquals(actDense.get(i), actEmbedding.get(i)); + } + + net.setLabels(labels); + net2.setLabels(labels); + net.computeGradientAndScore(); + net2.computeGradientAndScore(); + + assertEquals(net2.score(), net.score(), 1e-5); + + Map gradients = net.gradient().gradientForVariable(); + Map gradients2 = net2.gradient().gradientForVariable(); + assertEquals(gradients.keySet(), gradients2.keySet()); + for (String s : gradients.keySet()) { + assertEquals(gradients2.get(s), gradients.get(s)); + } + } + } + } + } + } + + @EqualsAndHashCode + private static class WordVectorsMockup implements EmbeddingInitializer { + + @Override + public void loadWeightsInto(INDArray array) { + INDArray vectors = Nd4j.linspace(1,15,15, DataType.FLOAT).reshape(5,3); + array.assign(vectors); + } + + @Override + public long vocabSize() { + return 5; + } + + @Override + public int vectorSize() { + return 3; + } + + @Override + public boolean jsonSerializable() { + return true; + } + } + + @Test + public void testEmbeddingDefaultActivation(){ + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new EmbeddingLayer.Builder().nIn(10).nOut(10).build()) + .layer(new EmbeddingSequenceLayer.Builder().nIn(10).nOut(10).build()) + .build(); + + EmbeddingLayer l = (EmbeddingLayer) conf.getConf(0).getLayer(); + assertEquals(new ActivationIdentity(), l.getActivationFn()); + + EmbeddingSequenceLayer l2 = (EmbeddingSequenceLayer) conf.getConf(1).getLayer(); + assertEquals(new ActivationIdentity(), l2.getActivationFn()); + + } + + + @Test + public void testEmbeddingWeightInit(){ + // https://github.com/eclipse/deeplearning4j/issues/8663 + //The embedding layer weight initialization should be independent of the vocabulary size (nIn setting) + + for(WeightInit wi : new WeightInit[]{WeightInit.XAVIER, WeightInit.RELU, WeightInit.XAVIER_UNIFORM, WeightInit.LECUN_NORMAL}) { + + for (boolean seq : new boolean[]{false, true}) { + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345) + .list() + .layer(seq ? + new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100).nOut(100).build() : + new EmbeddingLayer.Builder().weightInit(wi).nIn(100).nOut(100).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .seed(12345) + .list() + .layer(seq ? + new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100).nOut(100).build() : + new EmbeddingLayer.Builder().weightInit(wi).nIn(100).nOut(100).build()) + .build(); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder() + .seed(12345) + .list() + .layer(seq ? + new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100000).nOut(100).build() : + new EmbeddingLayer.Builder().weightInit(wi).nIn(100000).nOut(100).build()) + .build(); + MultiLayerNetwork net3 = new MultiLayerNetwork(conf3); + net3.init(); + + INDArray p1 = net.params(); + INDArray p2 = net2.params(); + INDArray p3 = net3.params(); + boolean eq = p1.equalsWithEps(p2, 1e-4); + String str = (seq ? "EmbeddingSequenceLayer" : "EmbeddingLayer") + " - " + wi; + assertTrue(eq, str + " p1/p2 params not equal"); + + double m1 = p1.meanNumber().doubleValue(); + double s1 = p1.stdNumber().doubleValue(); + + double m3 = p3.meanNumber().doubleValue(); + double s3 = p3.stdNumber().doubleValue(); + + + + assertEquals(m1, m3, 0.1, str); + assertEquals(s1, s3, 0.1, str); + + double re = relErr(s1, s3); + assertTrue(re < 0.05, str + " - " + re); + } + } + + } + + public static double relErr(double d1, double d2){ + if(d1 == 0.0 && d2 == 0.0) + return 0.0; + return Math.abs(d1 - d2) / (Math.abs(d1) + Math.abs(d2)); + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java new file mode 100644 index 000000000..e50868b7a --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java @@ -0,0 +1,784 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.normalization; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.Updater; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.BatchNormalization; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.params.BatchNormalizationParamInitializer; +import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; +import org.deeplearning4j.nn.transferlearning.TransferLearning; +import org.deeplearning4j.nn.updater.MultiLayerUpdater; +import org.deeplearning4j.nn.updater.UpdaterBlock; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.buffer.util.DataTypeUtil; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp; +import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp; +import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp; +import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastSubOp; +import org.nd4j.linalg.api.shape.Shape; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.NoOpUpdater; +import org.nd4j.linalg.learning.RmsPropUpdater; +import org.nd4j.linalg.learning.config.AdaDelta; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.ops.transforms.Transforms; +import org.nd4j.common.primitives.Pair; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + */ +@Slf4j +public class BatchNormalizationTest extends BaseDL4JTest { + + static { + //Force Nd4j initialization, then set data type to double: + Nd4j.zeros(1); + DataTypeUtil.setDTypeForContext(DataType.DOUBLE); + } + + protected INDArray dnnInput = Nd4j.linspace(0, 31, 32, Nd4j.dataType()).reshape(2, 16); + protected INDArray dnnEpsilon = Nd4j.linspace(0, 31, 32, Nd4j.dataType()).reshape(2, 16); + + protected INDArray cnnInput = Nd4j.linspace(0, 63, 64, Nd4j.dataType()).reshape(2, 2, 4, 4); + protected INDArray cnnEpsilon = Nd4j.linspace(0, 63, 64, Nd4j.dataType()).reshape(2, 2, 4, 4); + + @BeforeEach + public void doBefore() { + } + + @Override + public long getTimeoutMilliseconds() { + return 90000L; + } + + @Test + public void testDnnForwardPass() { + int nOut = 10; + Layer l = getLayer(nOut, 0.0, false, -1, -1); + assertEquals(4 * nOut, l.numParams()); //Gamma, beta, global mean, global var + + INDArray randInput = Nd4j.rand(100, nOut); + INDArray output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces()); + + INDArray mean = output.mean(0); + INDArray stdev = output.std(false, 0); + +// System.out.println(Arrays.toString(mean.data().asFloat())); + + assertArrayEquals(new float[nOut], mean.data().asFloat(), 1e-6f); + assertEquals(Nd4j.ones(nOut), stdev); + + //If we fix gamma/beta: expect different mean and variance... + double gamma = 2.0; + double beta = 3.0; + l = getLayer(nOut, 0.0, true, gamma, beta); + assertEquals(2 * nOut, l.numParams()); //Should have only global mean/var parameters + output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces()); + mean = output.mean(0); + stdev = output.std(false, 0); + + assertEquals(Nd4j.valueArrayOf(mean.shape(), beta), mean); + assertEquals(Nd4j.valueArrayOf(stdev.shape(), gamma), stdev); + } + + protected static Layer getLayer(int nOut, double epsilon, boolean lockGammaBeta, double gamma, double beta) { + BatchNormalization.Builder b = new BatchNormalization.Builder().nOut(nOut).eps(epsilon); + if (lockGammaBeta) { + b.lockGammaBeta(true).gamma(gamma).beta(beta); + } + BatchNormalization bN = b.build(); + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(bN).build(); + + long numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = null; + if (numParams > 0) { + params = Nd4j.create(1, numParams); + } + Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true, params == null ? Nd4j.defaultFloatingPointType() : params.dataType()); + if (numParams > 0) { + layer.setBackpropGradientsViewArray(Nd4j.create(1, numParams)); + } + return layer; + } + + @Test + public void testDnnForwardBackward() { + double eps = 1e-5; + int nIn = 4; + int minibatch = 2; + Nd4j.getRandom().setSeed(12345); + INDArray input = Nd4j.rand('c', new int[]{minibatch, nIn}); + + //TODO: other values for gamma/beta + INDArray gamma = Nd4j.ones(1, nIn); + INDArray beta = Nd4j.zeros(1, nIn); + + Layer l = getLayer(nIn, eps, false, -1, -1); + + INDArray mean = input.mean(0); + INDArray var = input.var(false, 0); + INDArray xHat = input.subRowVector(mean).divRowVector(Transforms.sqrt(var.add(eps), true)); + INDArray outExpected = xHat.mulRowVector(gamma).addRowVector(beta); + + INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces()); + +// System.out.println(Arrays.toString(outExpected.data().asDouble())); +// System.out.println(Arrays.toString(out.data().asDouble())); + + assertEquals(outExpected, out); + + //------------------------------------------------------------- + //Check backprop + INDArray epsilon = Nd4j.rand(minibatch, nIn); //dL/dy + + INDArray dldgammaExp = epsilon.mul(xHat).sum(true, 0); + INDArray dldbetaExp = epsilon.sum(true, 0); + + INDArray dldxhat = epsilon.mulRowVector(gamma); + INDArray dldvar = dldxhat.mul(input.subRowVector(mean)).mul(-0.5) + .mulRowVector(Transforms.pow(var.add(eps), -3.0 / 2.0, true)).sum(0); + INDArray dldmu = dldxhat.mulRowVector(Transforms.pow(var.add(eps), -1.0 / 2.0, true)).neg().sum(0) + .add(dldvar.mul(input.subRowVector(mean).mul(-2.0).sum(0).div(minibatch))); + INDArray dldinExp = dldxhat.mulRowVector(Transforms.pow(var.add(eps), -1.0 / 2.0, true)) + .add(input.subRowVector(mean).mul(2.0 / minibatch).mulRowVector(dldvar)) + .addRowVector(dldmu.mul(1.0 / minibatch)); + + Pair p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); + + INDArray dldgamma = p.getFirst().getGradientFor("gamma"); + INDArray dldbeta = p.getFirst().getGradientFor("beta"); + + assertEquals(dldgammaExp, dldgamma); + assertEquals(dldbetaExp, dldbeta); + +// System.out.println("EPSILONS"); +// System.out.println(Arrays.toString(dldinExp.data().asDouble())); +// System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble())); + assertEquals(dldinExp, p.getSecond()); + } + + @Test + public void testCnnForwardPass() { + int nOut = 10; + Layer l = getLayer(nOut, 0.0, false, -1, -1); + assertEquals(4 * nOut, l.numParams()); //Gamma, beta, global mean, global var + int hw = 15; + + Nd4j.getRandom().setSeed(12345); + INDArray randInput = Nd4j.rand(new int[]{100, nOut, hw, hw}); + INDArray output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces()); + + assertEquals(4, output.rank()); + + INDArray mean = output.mean(0, 2, 3); + INDArray stdev = output.std(false, 0, 2, 3); + + assertArrayEquals(new float[nOut], mean.data().asFloat(), 1e-6f); + assertArrayEquals(Nd4j.ones(1, nOut).data().asFloat(), stdev.data().asFloat(), 1e-6f); + + //If we fix gamma/beta: expect different mean and variance... + double gamma = 2.0; + double beta = 3.0; + l = getLayer(nOut, 0.0, true, gamma, beta); + assertEquals(2 * nOut, l.numParams()); //Should have only global mean/var parameters + output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces()); + mean = output.mean(0, 2, 3); + stdev = output.std(false, 0, 2, 3); + + assertEquals(Nd4j.valueArrayOf(mean.shape(), beta), mean); + assertEquals(Nd4j.valueArrayOf(stdev.shape(), gamma), stdev); + } + + @Test + public void test2dVs4d() { + //Idea: 2d and 4d should be the same... + Nd4j.getRandom().setSeed(12345); + + int m = 2; + int h = 3; + int w = 3; + int nOut = 2; + + INDArray in = Nd4j.rand('c', m * h * w, nOut); + + INDArray in4 = in.dup(); + in4 = Shape.newShapeNoCopy(in4, new int[]{m, h, w, nOut}, false); + assertNotNull(in4); + in4 = in4.permute(0, 3, 1, 2).dup(); + INDArray arr = Nd4j.rand(1, m * h * w * nOut).reshape('f', h, w, m, nOut).permute(2, 3, 1, 0); + in4 = arr.assign(in4); + + Layer l1 = getLayer(nOut); + Layer l2 = getLayer(nOut); + + INDArray out2d = l1.activate(in.dup(), true, LayerWorkspaceMgr.noWorkspaces()); + INDArray out4d = l2.activate(in4.dup(), true, LayerWorkspaceMgr.noWorkspaces()); + + INDArray out4dAs2 = out4d.permute(0, 2, 3, 1).dup('c'); + out4dAs2 = Shape.newShapeNoCopy(out4dAs2, new int[]{m * h * w, nOut}, false); + + assertEquals(out2d, out4dAs2); + + //Test backprop: + INDArray epsilons2d = Nd4j.rand('c', m * h * w, nOut); + INDArray epsilons4d = epsilons2d.dup(); + epsilons4d = Shape.newShapeNoCopy(epsilons4d, new int[]{m, h, w, nOut}, false); + assertNotNull(epsilons4d); + epsilons4d = epsilons4d.permute(0, 3, 1, 2).dup(); + + Pair b2d = l1.backpropGradient(epsilons2d, LayerWorkspaceMgr.noWorkspaces()); + Pair b4d = l2.backpropGradient(epsilons4d, LayerWorkspaceMgr.noWorkspaces()); + + INDArray e4dAs2d = b4d.getSecond().permute(0, 2, 3, 1).dup('c'); + e4dAs2d = Shape.newShapeNoCopy(e4dAs2d, new int[]{m * h * w, nOut}, false); + + assertEquals(b2d.getSecond(), e4dAs2d); + } + + protected static Layer getLayer(int nOut) { + return getLayer(nOut, Nd4j.EPS_THRESHOLD, false, -1, -1); + } + + @Test + public void testCnnForwardBackward() { + double eps = 1e-5; + int nIn = 4; + int hw = 3; + int minibatch = 2; + Nd4j.getRandom().setSeed(12345); + INDArray input = Nd4j.rand('c', new int[]{minibatch, nIn, hw, hw}); + + //TODO: other values for gamma/beta + INDArray gamma = Nd4j.ones(1, nIn); + INDArray beta = Nd4j.zeros(1, nIn); + + Layer l = getLayer(nIn, eps, false, -1, -1); + + INDArray mean = input.mean(0, 2, 3); + INDArray var = input.var(false, 0, 2, 3); + INDArray xHat = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1)); + Nd4j.getExecutioner().exec(new BroadcastDivOp(xHat, Transforms.sqrt(var.add(eps), true), xHat, 1)); + + INDArray outExpected = Nd4j.getExecutioner().exec(new BroadcastMulOp(xHat, gamma, xHat.dup(), 1)); + Nd4j.getExecutioner().exec(new BroadcastAddOp(outExpected, beta, outExpected, 1)); + + INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces()); + +// System.out.println(Arrays.toString(outExpected.data().asDouble())); +// System.out.println(Arrays.toString(out.data().asDouble())); + + assertEquals(outExpected, out); + + //------------------------------------------------------------- + //Check backprop + INDArray epsilon = Nd4j.rand('c', new int[]{minibatch, nIn, hw, hw}); //dL/dy + + int effectiveMinibatch = minibatch * hw * hw; + + INDArray dldgammaExp = epsilon.mul(xHat).sum(0, 2, 3); + dldgammaExp = dldgammaExp.reshape(1, dldgammaExp.length()); + INDArray dldbetaExp = epsilon.sum(0, 2, 3); + dldbetaExp = dldbetaExp.reshape(1, dldbetaExp.length()); + + INDArray dldxhat = Nd4j.getExecutioner().exec(new BroadcastMulOp(epsilon, gamma, epsilon.dup(), 1)); //epsilon.mulRowVector(gamma); + + INDArray inputSubMean = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1)); + + INDArray dldvar = dldxhat.mul(inputSubMean).mul(-0.5); + dldvar = Nd4j.getExecutioner().exec( + new BroadcastMulOp(dldvar, Transforms.pow(var.add(eps), -3.0 / 2.0, true), dldvar.dup(), 1)); + dldvar = dldvar.sum(0, 2, 3); + + + INDArray dldmu = Nd4j + .getExecutioner().exec(new BroadcastMulOp(dldxhat, + Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1)) + .neg().sum(0, 2, 3); + dldmu = dldmu.add(dldvar.mul(inputSubMean.mul(-2.0).sum(0, 2, 3).div(effectiveMinibatch))); + + INDArray dldinExp = Nd4j.getExecutioner().exec( + new BroadcastMulOp(dldxhat, Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1)); + dldinExp = dldinExp.add(Nd4j.getExecutioner().exec( + new BroadcastMulOp(inputSubMean.mul(2.0 / effectiveMinibatch), dldvar, inputSubMean.dup(), 1))); + dldinExp = Nd4j.getExecutioner().exec( + new BroadcastAddOp(dldinExp, dldmu.mul(1.0 / effectiveMinibatch), dldinExp.dup(), 1)); + + Pair p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); + + INDArray dldgamma = p.getFirst().getGradientFor("gamma"); + INDArray dldbeta = p.getFirst().getGradientFor("beta"); + + assertEquals(dldgammaExp, dldgamma); + assertEquals(dldbetaExp, dldbeta); + + // System.out.println("EPSILONS"); + // System.out.println(Arrays.toString(dldinExp.data().asDouble())); + // System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble())); + assertEquals(dldinExp, p.getSecond()); + } + + @Test + public void testDBNBNMultiLayer() throws Exception { + DataSetIterator iter = new MnistDataSetIterator(2, 2); + DataSet next = iter.next(); + + // Run with separate activation layer + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new DenseLayer.Builder().nIn(28 * 28).nOut(10).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(1, new BatchNormalization.Builder().nOut(10).build()).layer(2, + new ActivationLayer.Builder() + .activation(Activation.RELU).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10) + .build()) + .build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + network.setInput(next.getFeatures()); + INDArray activationsActual = network.output(next.getFeatures()); + assertEquals(10, activationsActual.shape()[1], 1e-2); + + network.fit(next); + INDArray actualGammaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.GAMMA); + INDArray actualBetaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.BETA); + assertTrue(actualGammaParam != null); + assertTrue(actualBetaParam != null); + } + + @Test + public void testCNNBNActivationCombo() throws Exception { + DataSetIterator iter = new MnistDataSetIterator(2, 2); + DataSet next = iter.next(); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER) + .activation(Activation.IDENTITY).build()) + .layer(1, new BatchNormalization.Builder().build()) + .layer(2, new ActivationLayer.Builder().activation(Activation.RELU).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + network.fit(next); + + assertNotEquals(null, network.getLayer(0).getParam("W")); + assertNotEquals(null, network.getLayer(0).getParam("b")); + } + + + @Test + public void checkSerialization() throws Exception { + //Serialize the batch norm network (after training), and make sure we get same activations out as before + // i.e., make sure state is properly stored + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345) + .list() + .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER) + .activation(Activation.IDENTITY).build()) + .layer(1, new BatchNormalization.Builder().build()) + .layer(2, new ActivationLayer.Builder().activation(Activation.LEAKYRELU).build()) + .layer(3, new DenseLayer.Builder().nOut(10).activation(Activation.LEAKYRELU).build()) + .layer(4, new BatchNormalization.Builder().build()) + .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSetIterator iter = new MnistDataSetIterator(16, true, 12345); + for (int i = 0; i < 20; i++) { + net.fit(iter.next()); + } + + INDArray in = iter.next().getFeatures(); + + INDArray out = net.output(in, false); + INDArray out2 = net.output(in, false); + + assertEquals(out, out2); + + MultiLayerNetwork net2 = TestUtils.testModelSerialization(net); + + INDArray outDeser = net2.output(in, false); + + assertEquals(out, outDeser); + } + + @Test + public void testGradientAndUpdaters() throws Exception { + //Global mean/variance are part of the parameter vector. Expect 0 gradient, and no-op updater for these + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(Updater.RMSPROP).seed(12345).list() + .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER) + .activation(Activation.IDENTITY).build()) + .layer(1, new BatchNormalization.Builder().build()) + .layer(2, new ActivationLayer.Builder().activation(Activation.LEAKYRELU).build()) + .layer(3, new DenseLayer.Builder().nOut(10).activation(Activation.LEAKYRELU).build()) + .layer(4, new BatchNormalization.Builder().build()) + .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSetIterator iter = new MnistDataSetIterator(16, true, 12345); + + DataSet ds = iter.next(); + net.setInput(ds.getFeatures()); + net.setLabels(ds.getLabels()); + + net.computeGradientAndScore(); + + Gradient g = net.gradient(); + Map map = g.gradientForVariable(); + + org.deeplearning4j.nn.api.Updater u = net.getUpdater(); + + MultiLayerUpdater mlu = (MultiLayerUpdater) u; + List l = mlu.getUpdaterBlocks(); + assertNotNull(l); + assertEquals(5, l.size()); //Conv+bn (RMSProp), No-op (bn), RMSProp (dense, bn), no-op (bn), RMSProp (out) + + for (UpdaterBlock ub : l) { + + List list = ub.getLayersAndVariablesInBlock(); + for (UpdaterBlock.ParamState v : list) { + if (BatchNormalizationParamInitializer.GLOBAL_MEAN.equals(v.getParamName()) + || BatchNormalizationParamInitializer.GLOBAL_VAR.equals(v.getParamName()) + || BatchNormalizationParamInitializer.GLOBAL_LOG_STD.equals(v.getParamName())) { + assertTrue(ub.getGradientUpdater() instanceof NoOpUpdater); + } else { + assertTrue(ub.getGradientUpdater() instanceof RmsPropUpdater); + } + } + } + } + + + @Test + public void checkMeanVarianceEstimate() throws Exception { + Nd4j.getRandom().setSeed(12345); + //Check that the internal global mean/variance estimate is approximately correct + + for(boolean useLogStd : new boolean[]{true, false}) { + + //First, Mnist data as 2d input (NOT taking into account convolution property) + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(Updater.RMSPROP).seed(12345) + .list().layer(0, + new BatchNormalization.Builder().nIn(10).nOut(10).eps(1e-5).decay(0.95) + .useLogStd(useLogStd).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER) + .activation(Activation.IDENTITY).nIn(10).nOut(10).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + int minibatch = 32; + List list = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + list.add(new DataSet(Nd4j.rand(minibatch, 10), Nd4j.rand(minibatch, 10))); + } + + DataSetIterator iter = new ListDataSetIterator(list); + + INDArray expMean = Nd4j.valueArrayOf(new int[]{1, 10}, 0.5); + INDArray expVar = Nd4j.valueArrayOf(new int[]{1, 10}, 1 / 12.0); //Expected variance of U(0,1) distribution: 1/12 * (1-0)^2 = 0.0833 + + + for (int i = 0; i < 10; i++) { + iter.reset(); + net.fit(iter); + } + + INDArray estMean = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_MEAN); + INDArray estVar; + if(useLogStd){ + INDArray log10std = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_LOG_STD); + estVar = Nd4j.valueArrayOf(log10std.shape(), 10.0).castTo(log10std.dataType()); + Transforms.pow(estVar, log10std, false); // stdev = 10^(log10(stdev)) + estVar.muli(estVar); + } else { + estVar = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_VAR); + } + + float[] fMeanExp = expMean.data().asFloat(); + float[] fMeanAct = estMean.data().asFloat(); + float[] fVarExp = expVar.data().asFloat(); + float[] fVarAct = estVar.data().asFloat(); + + // System.out.println("Mean vs. estimated mean:"); + // System.out.println(Arrays.toString(fMeanExp)); + // System.out.println(Arrays.toString(fMeanAct)); + // + // System.out.println("Var vs. estimated var:"); + // System.out.println(Arrays.toString(fVarExp)); + // System.out.println(Arrays.toString(fVarAct)); + + assertArrayEquals(fMeanExp, fMeanAct, 0.02f); + assertArrayEquals(fVarExp, fVarAct, 0.02f); + } + } + + + @Test + public void checkMeanVarianceEstimateCNN() throws Exception { + + for(boolean useLogStd : new boolean[]{true, false}) { + Nd4j.getRandom().setSeed(12345); + //Check that the internal global mean/variance estimate is approximately correct + + //First, Mnist data as 2d input (NOT taking into account convolution property) + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(Updater.RMSPROP).seed(12345).list() + .layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(useLogStd).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER) + .activation(Activation.IDENTITY).nOut(10).build()) + .setInputType(InputType.convolutional(5, 5, 3)).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + int minibatch = 32; + List list = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + list.add(new DataSet(Nd4j.rand(new int[]{minibatch, 3, 5, 5}), Nd4j.rand(minibatch, 10))); + } + + DataSetIterator iter = new ListDataSetIterator(list); + + INDArray expMean = Nd4j.valueArrayOf(new int[]{1, 3}, 0.5); + INDArray expVar = Nd4j.valueArrayOf(new int[]{1, 3}, 1 / 12.0); //Expected variance of U(0,1) distribution: 1/12 * (1-0)^2 = 0.0833 + + + for (int i = 0; i < 10; i++) { + iter.reset(); + net.fit(iter); + } + + INDArray estMean = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_MEAN); + INDArray estVar; + if(useLogStd){ + INDArray log10std = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_LOG_STD); + estVar = Nd4j.valueArrayOf(log10std.shape(), 10.0).castTo(log10std.dataType()); + Transforms.pow(estVar, log10std, false); // stdev = 10^(log10(stdev)) + estVar.muli(estVar); + } else { + estVar = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_VAR); + } + + float[] fMeanExp = expMean.data().asFloat(); + float[] fMeanAct = estMean.data().asFloat(); + float[] fVarExp = expVar.data().asFloat(); + float[] fVarAct = estVar.data().asFloat(); + + // System.out.println("Mean vs. estimated mean:"); + // System.out.println(Arrays.toString(fMeanExp)); + // System.out.println(Arrays.toString(fMeanAct)); + // + // System.out.println("Var vs. estimated var:"); + // System.out.println(Arrays.toString(fVarExp)); + // System.out.println(Arrays.toString(fVarAct)); + + assertArrayEquals(fMeanExp, fMeanAct, 0.01f); + assertArrayEquals(fVarExp, fVarAct, 0.01f); + } + } + + @Test + public void checkMeanVarianceEstimateCNNCompareModes() throws Exception { + + Nd4j.getRandom().setSeed(12345); + //Check that the internal global mean/variance estimate is approximately correct + + //First, Mnist data as 2d input (NOT taking into account convolution property) + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(Updater.RMSPROP).seed(12345).list() + .layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(false).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER) + .activation(Activation.IDENTITY).nOut(10).build()) + .setInputType(InputType.convolutional(5, 5, 3)).build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(Updater.RMSPROP).seed(12345).list() + .layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(true).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER) + .activation(Activation.IDENTITY).nOut(10).build()) + .setInputType(InputType.convolutional(5, 5, 3)).build(); + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + int minibatch = 32; + for (int i = 0; i < 10; i++) { + DataSet ds = new DataSet(Nd4j.rand(new int[]{minibatch, 3, 5, 5}), Nd4j.rand(minibatch, 10)); + net.fit(ds); + net2.fit(ds); + + INDArray globalVar = net.getParam("0_" + BatchNormalizationParamInitializer.GLOBAL_VAR); + + INDArray log10std = net2.getParam("0_" + BatchNormalizationParamInitializer.GLOBAL_LOG_STD); + INDArray globalVar2 = Nd4j.valueArrayOf(log10std.shape(), 10.0).castTo(log10std.dataType()); + Transforms.pow(globalVar2, log10std, false); // stdev = 10^(log10(stdev)) + globalVar2.muli(globalVar2); + + assertEquals(globalVar, globalVar2); + } + } + + + @Test + public void testBatchNorm() throws Exception { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345) + .updater(new Adam(1e-3)) + .activation(Activation.TANH) + .list() + .layer(new ConvolutionLayer.Builder().nOut(5).kernelSize(2, 2).build()) + .layer(new BatchNormalization()) + .layer(new ConvolutionLayer.Builder().nOut(5).kernelSize(2, 2).build()) + .layer(new OutputLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nOut(10).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSetIterator iter = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(32, true, 12345), 10); + + net.fit(iter); + + MultiLayerNetwork net2 = new TransferLearning.Builder(net) + .fineTuneConfiguration(FineTuneConfiguration.builder() + .updater(new AdaDelta()) + .build()) + .removeOutputLayer() + .addLayer(new BatchNormalization.Builder().nOut(3380).build()) + .addLayer(new OutputLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nIn(3380).nOut(10).build()) + .build(); + + net2.fit(iter); + } + + @Test + public void testBatchNormRecurrentCnn1d() { + //Simple sanity check on CNN1D and RNN layers + + for (boolean rnn : new boolean[]{true, false}) { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345) + .weightInit(WeightInit.XAVIER) + .convolutionMode(ConvolutionMode.Same) + .list() + .layer(rnn ? new LSTM.Builder().nOut(3).build() : + new Convolution1DLayer.Builder().kernelSize(3).stride(1).nOut(3).build()) + .layer(new BatchNormalization()) + .layer(new RnnOutputLayer.Builder().nOut(3).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build()) + .setInputType(InputType.recurrent(3)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray in = Nd4j.rand(new int[]{1, 3, 5}); + INDArray label = Nd4j.rand(new int[]{1, 3, 5}); + + INDArray out = net.output(in); + assertArrayEquals(new long[]{1, 3, 5}, out.shape()); + + net.fit(in, label); + log.info("OK: {}", (rnn ? "rnn" : "cnn1d")); + } + } + + @Test + public void testInputValidation() { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new BatchNormalization.Builder().nIn(10).nOut(10).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray in1 = Nd4j.create(1, 10); + INDArray in2 = Nd4j.create(1, 5); + + INDArray out1 = net.output(in1); + try { + INDArray out2 = net.output(in2); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("expected input")); + } + } +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/LocalResponseTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/LocalResponseTest.java new file mode 100644 index 000000000..99fc1e5a3 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/normalization/LocalResponseTest.java @@ -0,0 +1,217 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.normalization; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.LocalResponseNormalization; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.primitives.Pair; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * + */ +public class LocalResponseTest extends BaseDL4JTest { + + private INDArray x = Nd4j.create(new double[] {0.88128096, -0.96666986, -0.61832994, 0.26418415, 0.05694608, + 0.2950289, 0.99222249, 0.24541704, 0.4219842, 0.96430975, 0.19299535, -0.06658337, -0.27603117, + 0.24216647, 0.21834095, 0.03863283, -0.82313406, -0.37236378, -0.77667993, 0.66295379, -0.34406275, + -0.25924176, 0.26652309, -0.58964926, -0.46907067, 0.34666502, 0.81208313, -0.17042427, -0.22470538, + 0.8348338, 0.50494033, 0.45004508, 0.58735144, -0.87217808, -0.74788797, -0.04363599, 0.72276866, + 0.52476895, -0.52383977, 0.1311436, 0.2628099, 0.77274454, 0.86400729, -0.35246921, -0.03399619, + -0.502312, 0.42834607, 0.85534132, 0.90083021, 0.24571614, 0.63058525, -0.82919437, 0.57236177, + -0.0913529, -0.7102778, 0.81631756, -0.89004314, 0.43995622, -0.26112801, -0.76135367, 0.65180862, + -0.54667377, 0.94908774, 0.59298772, 0.36457643, 0.58892179, -0.52951556, 0.31559938, -0.55268252, + 0.8272332, 0.37911707, -0.96299696, -0.40717798, 0.43324658, 0.2589654, -0.15605508, 0.96334064, + -0.31666604, 0.19781154, 0.09908111, 0.64796048, -0.99037546, 0.67919868, 0.43810204}, + new int[] {2, 7, 3, 2}); + + private INDArray activationsExpected = Nd4j.create(new double[] {0.52397668, -0.57476264, -0.3676528, 0.15707894, + 0.03385943, 0.17542371, 0.58992499, 0.14591768, 0.25090647, 0.57335907, 0.11475233, -0.03958985, + -0.16411273, 0.14398433, 0.12981956, 0.02297027, -0.48942304, -0.22139823, -0.46177959, 0.39418164, + -0.20457059, -0.15413573, 0.15846729, -0.3505919, -0.27889356, 0.20611978, 0.48284137, -0.10133155, + -0.13360347, 0.49636194, 0.30022132, 0.26758799, 0.34922296, -0.51858318, -0.4446843, -0.02594452, + 0.42974478, 0.31202248, -0.31146204, 0.07797609, 0.15626372, 0.4594543, 0.51370209, -0.20957276, + -0.02021335, -0.29866382, 0.25469059, 0.50856382, 0.53558689, 0.14609739, 0.37491882, -0.49301448, + 0.34031925, -0.05431537, -0.42228988, 0.48536259, -0.52917528, 0.26157826, -0.15526266, -0.45265958, + 0.38753596, -0.32503816, 0.56427884, 0.35256693, 0.21676543, 0.35014921, -0.31483513, 0.18764766, + -0.32859638, 0.49183461, 0.22540972, -0.57255536, -0.24210122, 0.25760418, 0.15397197, -0.0927838, + 0.57277, -0.18827969, 0.1176173, 0.05891332, 0.38526815, -0.58884346, 0.40383074, 0.26048511}, + new int[] {2, 7, 3, 2}); + + private INDArray epsilon = Nd4j.create(new double[] {-0.13515499, 0.96470547, -0.62253004, 0.80172491, -0.97510445, + -0.41198033, -0.4790071, 0.07551047, -0.01383764, -0.05797465, 0.21242172, 0.7145375, -0.17809176, + -0.11465316, -0.2066526, 0.21950938, 0.4627091, 0.30275798, 0.61443841, 0.75912178, -0.132248, + -0.82923287, 0.74962652, -0.88993639, 0.04406403, 0.32096064, -0.46400586, 0.1603231, 0.63007826, + 0.10626783, 0.08009516, 0.88297033, 0.11441587, 0.35862735, 0.40441504, -0.60132015, 0.87743825, + 0.09792926, 0.92742652, 0.6182847, -0.9602651, -0.19611064, 0.15762019, 0.00339905, -0.9238292, + 0.02451134, -0.44294646, -0.5450229, 0.87502575, -0.59481794, 0.65259099, -0.77772689, 0.53300053, + 0.11541174, 0.32667685, 0.99437004, -0.04084824, -0.45166185, 0.29513556, 0.53582036, 0.95541358, + -0.75714606, -0.63295805, -0.70315111, -0.6553846, -0.78824568, 0.84295344, -0.38352135, + -0.04541624, 0.17396702, 0.41530582, 0.11870354, 0.85787249, -0.94597596, 0.05792254, 0.04811822, + 0.04847952, -0.82953823, 0.8089835, 0.50185651, -0.88619858, -0.78598201, 0.27489874, 0.63673472}, + new int[] {2, 7, 3, 2}); + + private INDArray newEpsilonExpected = Nd4j.create(new double[] {-0.08033668, 0.57355404, -0.37014094, 0.47668865, + -0.57978398, -0.24495915, -0.28474802, 0.04490108, -0.00823483, -0.03448687, 0.12630466, 0.42485803, + -0.10589627, -0.06816553, -0.12287001, 0.13051508, 0.27510744, 0.18001786, 0.36528736, 0.45133191, + -0.07863599, -0.49303374, 0.44571424, -0.52912313, 0.02620371, 0.19082049, -0.27585581, 0.09532529, + 0.3746179, 0.06316902, 0.04761803, 0.52497554, 0.06804816, 0.21323238, 0.24044329, -0.35752413, + 0.52168733, 0.05821467, 0.55140609, 0.3676247, -0.57095432, -0.11660115, 0.09367896, 0.00202246, + -0.54928631, 0.01455687, -0.26336867, -0.3240425, 0.52023786, -0.35366109, 0.3879728, -0.46243483, + 0.31692421, 0.06862034, 0.19421607, 0.59124804, -0.0242459, -0.26852599, 0.17547797, 0.31857637, + 0.56804365, -0.45020312, -0.37634474, -0.41804832, -0.38966343, -0.4686695, 0.50119156, -0.22802454, + -0.02698562, 0.10343311, 0.24693431, 0.0706142, 0.5100745, -0.56245267, 0.03443092, 0.02860913, + 0.02883426, -0.49320197, 0.4810102, 0.29840365, -0.5269345, -0.46732581, 0.16344811, 0.37857518}, + new int[] {2, 7, 3, 2}); + + private INDArray activationsActual; + private Layer layer; + + @BeforeEach + public void doBefore() { + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123) + .layer(new LocalResponseNormalization.Builder().k(2).n(5).alpha(1e-4).beta(0.75).build()) + .build(); + + layer = new LocalResponseNormalization().instantiate(conf, null, 0, null, false, Nd4j.defaultFloatingPointType()); + activationsActual = layer.activate(x, false, LayerWorkspaceMgr.noWorkspaces()); + } + + @Test + public void testActivate() { + // Precision is off from the expected results because expected results generated in numpy + assertEquals(activationsExpected, activationsActual); + assertArrayEquals(activationsExpected.shape(), activationsActual.shape()); + } + + @Test + public void testBackpropGradient() { + Pair containedOutput = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); + + assertEquals(newEpsilonExpected.getDouble(8), containedOutput.getSecond().getDouble(8), 1e-4); + assertEquals(newEpsilonExpected.getDouble(20), containedOutput.getSecond().getDouble(20), 1e-4); + assertEquals(null, containedOutput.getFirst().getGradientFor("W")); + assertArrayEquals(newEpsilonExpected.shape(), containedOutput.getSecond().shape()); + } + + @Test + public void testRegularization() { + // Confirm a structure with regularization true will not throw an error + + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).l1(0.2) + .l2(0.1).seed(123) + .layer(new LocalResponseNormalization.Builder().k(2).n(5).alpha(1e-4).beta(0.75).build()) + .build(); + } + + @Test + public void testMultiCNNLayer() throws Exception { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list() + .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(1, new LocalResponseNormalization.Builder().build()).layer(2, + new DenseLayer.Builder() + .nOut(2).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(10) + .build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + DataSetIterator iter = new MnistDataSetIterator(2, 2); + DataSet next = iter.next(); + + network.fit(next); + } + + + @Test + public void testLrnManual() { + int wh = 5; + int depth = 6; + int minibatch = 3; + + int n = 4; + double k = 2.0; + double alpha = 1e-4; + double beta = 0.75; + + INDArray in = Nd4j.rand(new int[] {minibatch, depth, wh, wh}); + INDArray outExp = Nd4j.zeros(minibatch, depth, wh, wh); + + for (int m = 0; m < minibatch; m++) { + for (int x = 0; x < wh; x++) { + for (int y = 0; y < wh; y++) { + for (int i = 0; i < depth; i++) { + int jFrom = Math.max(0, i - n / 2); + int jTo = Math.min(depth - 1, i + n / 2); + double sum = 0.0; + for (int j = jFrom; j <= jTo; j++) { + double d = in.getDouble(m, j, x, y); + sum += d * d; + } + double out = in.getDouble(m, i, x, y) / Math.pow(k + alpha * sum, beta); + outExp.putScalar(m, i, x, y, out); + } + } + } + } + + LocalResponseNormalization lrn = new LocalResponseNormalization.Builder().build(); + NeuralNetConfiguration nnc = new NeuralNetConfiguration.Builder().layer(lrn).build(); + org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization layer = + (org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization) lrn.instantiate(nnc, + null, 0, null, false, Nd4j.defaultFloatingPointType()); + + INDArray outAct = layer.activate(in, true, LayerWorkspaceMgr.noWorkspaces()); + + assertEquals(outExp, outAct); + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/objdetect/TestYolo2OutputLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/objdetect/TestYolo2OutputLayer.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/objdetect/TestYolo2OutputLayer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/objdetect/TestYolo2OutputLayer.java index 89deea8dd..7fb8dc8af 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/objdetect/TestYolo2OutputLayer.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/objdetect/TestYolo2OutputLayer.java @@ -25,13 +25,7 @@ import org.apache.commons.io.IOUtils; import org.datavec.api.records.reader.RecordReader; import org.datavec.api.split.FileSplit; import org.deeplearning4j.nn.conf.GradientNormalization; -import org.junit.jupiter.api.Disabled; - - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.common.io.ClassPathResource; import org.datavec.image.recordreader.objdetect.ObjectDetectionRecordReader; @@ -65,18 +59,17 @@ import java.io.InputStream; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.net.URI; -import java.nio.file.Path; import java.util.Collections; import java.util.Comparator; import java.util.List; import static org.junit.jupiter.api.Assertions.*; import static org.nd4j.linalg.indexing.NDArrayIndex.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestYolo2OutputLayer extends BaseDL4JTest { - + @TempDir + public File tempDir; @Test public void testYoloActivateScoreBasic() { @@ -230,13 +223,12 @@ public class TestYolo2OutputLayer extends BaseDL4JTest { } @Test - public void testIOUCalc(@TempDir Path tempDir) throws Exception { + public void testIOUCalc() throws Exception { InputStream is1 = new ClassPathResource("yolo/VOC_SingleImage/JPEGImages/2007_009346.jpg").getInputStream(); InputStream is2 = new ClassPathResource("yolo/VOC_SingleImage/Annotations/2007_009346.xml").getInputStream(); - File dir = new File(tempDir.toFile(),"testYoloOverfitting"); - dir.mkdirs(); + File dir = new File(tempDir, "testYoloOverfitting"); File jpg = new File(dir, "JPEGImages"); File annot = new File(dir, "Annotations"); jpg.mkdirs(); @@ -434,8 +426,8 @@ public class TestYolo2OutputLayer extends BaseDL4JTest { @Test - @Disabled //TODO UNIGNORE THIS - IGNORED AS CRASHING JVM HENCE GETTING IN THE WAY OF FIXING OTHER PROBLEMS - public void testYoloOverfitting(@TempDir Path tempDir) throws Exception { + ////@Ignore //TODO UNIGNORE THIS - IGNORED AS CRASHING JVM HENCE GETTING IN THE WAY OF FIXING OTHER PROBLEMS + public void testYoloOverfitting() throws Exception { Nd4j.getRandom().setSeed(12345); InputStream is1 = new ClassPathResource("yolo/VOC_TwoImage/JPEGImages/2007_009346.jpg").getInputStream(); @@ -443,7 +435,7 @@ public class TestYolo2OutputLayer extends BaseDL4JTest { InputStream is3 = new ClassPathResource("yolo/VOC_TwoImage/JPEGImages/2008_003344.jpg").getInputStream(); InputStream is4 = new ClassPathResource("yolo/VOC_TwoImage/Annotations/2008_003344.xml").getInputStream(); - File dir = tempDir.toFile(); + File dir = tempDir; File jpg = new File(dir, "JPEGImages"); File annot = new File(dir, "Annotations"); jpg.mkdirs(); @@ -590,8 +582,8 @@ public class TestYolo2OutputLayer extends BaseDL4JTest { double p1 = o1.getClassPredictions().getDouble(idxCat); double c1 = o1.getConfidence(); assertEquals(idxCat, o1.getPredictedClass() ); - assertTrue(p1 >= 0.85,String.valueOf(p1)); - assertTrue(c1 >= 0.85,String.valueOf(c1)); + assertTrue( p1 >= 0.85, String.valueOf(p1)); + assertTrue( c1 >= 0.85, String.valueOf(c1)); assertEquals(cx1, o1.getCenterX(), 0.1); assertEquals(cy1, o1.getCenterY(), 0.1); assertEquals(wGrid1, o1.getWidth(), 0.2); @@ -602,8 +594,8 @@ public class TestYolo2OutputLayer extends BaseDL4JTest { double p2 = o2.getClassPredictions().getDouble(idxCat); double c2 = o2.getConfidence(); assertEquals(idxCat, o2.getPredictedClass() ); - assertTrue(p2 >= 0.85,String.valueOf(p2)); - assertTrue(c2 >= 0.85,String.valueOf(c2)); + assertTrue( p2 >= 0.85, String.valueOf(p2)); + assertTrue( c2 >= 0.85, String.valueOf(c2)); assertEquals(cx2, o2.getCenterX(), 0.1); assertEquals(cy2, o2.getCenterY(), 0.1); assertEquals(wGrid2, o2.getWidth(), 0.2); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayerTest.java new file mode 100644 index 000000000..1c9da8933 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayerTest.java @@ -0,0 +1,212 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.ocnn; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.gradientcheck.GradientCheckUtil; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.util.ModelSerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.impl.ActivationIdentity; +import org.nd4j.linalg.activations.impl.ActivationReLU; +import org.nd4j.linalg.activations.impl.ActivationSigmoid; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.schedule.ScheduleType; +import org.nd4j.linalg.schedule.StepSchedule; + +import java.io.File; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +public class OCNNOutputLayerTest extends BaseDL4JTest { + + private static final boolean PRINT_RESULTS = true; + private static final boolean RETURN_ON_FIRST_FAILURE = false; + private static final double DEFAULT_EPS = 1e-6; + private static final double DEFAULT_MAX_REL_ERROR = 1e-3; + private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; + + @TempDir + public File testDir; + static { + Nd4j.setDataType(DataType.DOUBLE); + } + + + @Test + public void testLayer() { + DataSetIterator dataSetIterator = getNormalizedIterator(); + boolean doLearningFirst = true; + MultiLayerNetwork network = getGradientCheckNetwork(2); + + + DataSet ds = dataSetIterator.next(); + INDArray arr = ds.getFeatures(); + network.setInput(arr); + + if (doLearningFirst) { + //Run a number of iterations of learning + network.setInput(arr); + network.setListeners(new ScoreIterationListener(1)); + network.computeGradientAndScore(); + double scoreBefore = network.score(); + for (int j = 0; j < 10; j++) + network.fit(ds); + network.computeGradientAndScore(); + double scoreAfter = network.score(); + //Can't test in 'characteristic mode of operation' if not learning + String msg = "testLayer() - score did not (sufficiently) decrease during learning - activationFn=" + + "relu" + ", lossFn=" + "ocnn" + ", " + "sigmoid" + + ", doLearningFirst=" + doLearningFirst + " (before=" + scoreBefore + + ", scoreAfter=" + scoreAfter + ")"; + // assertTrue(msg, scoreAfter < scoreBefore); + } + + if (PRINT_RESULTS) { + System.out.println("testLayer() - activationFn=" + "relu" + ", lossFn=" + + "ocnn" + "sigmoid" + ", doLearningFirst=" + + doLearningFirst); + for (int j = 0; j < network.getnLayers(); j++) + System.out.println("Layer " + j + " # params: " + network.getLayer(j).numParams()); + } + + boolean gradOK = GradientCheckUtil.checkGradients(network, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, + DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, ds.getFeatures(), ds.getLabels()); + + String msg = "testLayer() - activationFn=" + "relu" + ", lossFn=" + "ocnn" + + ",=" + "sigmoid" + ", doLearningFirst=" + doLearningFirst; + assertTrue(gradOK, msg); + + + + } + + + @Test + public void testLabelProbabilities() throws Exception { + Nd4j.getRandom().setSeed(42); + DataSetIterator dataSetIterator = getNormalizedIterator(); + MultiLayerNetwork network = getSingleLayer(); + DataSet next = dataSetIterator.next(); + DataSet filtered = next.filterBy(new int[]{0, 1}); + for (int i = 0; i < 10; i++) { + network.setEpochCount(i); + network.getLayerWiseConfigurations().setEpochCount(i); + network.fit(filtered); + } + + DataSet anomalies = next.filterBy(new int[] {2}); + INDArray output = network.output(anomalies.getFeatures()); + INDArray normalOutput = network.output(anomalies.getFeatures(),false); + assertEquals(output.lt(0.0).castTo(Nd4j.defaultFloatingPointType()).sumNumber().doubleValue(), + normalOutput.eq(0.0).castTo(Nd4j.defaultFloatingPointType()).sumNumber().doubleValue(),1e-1); + +// System.out.println("Labels " + anomalies.getLabels()); +// System.out.println("Anomaly output " + normalOutput); +// System.out.println(output); + + INDArray normalProbs = network.output(filtered.getFeatures()); + INDArray outputForNormalSamples = network.output(filtered.getFeatures(),false); + System.out.println("Normal probabilities " + normalProbs); + System.out.println("Normal raw output " + outputForNormalSamples); + + File tmpFile = new File(testDir.getAbsoluteFile(),"tmp-file-" + UUID.randomUUID().toString()); + ModelSerializer.writeModel(network,tmpFile,true); + tmpFile.deleteOnExit(); + + MultiLayerNetwork multiLayerNetwork = ModelSerializer.restoreMultiLayerNetwork(tmpFile); + assertEquals(network.params(),multiLayerNetwork.params()); + assertEquals(network.numParams(),multiLayerNetwork.numParams()); + + } + + + public DataSetIterator getNormalizedIterator() { + DataSetIterator dataSetIterator = new IrisDataSetIterator(150,150); + NormalizerStandardize normalizerStandardize = new NormalizerStandardize(); + normalizerStandardize.fit(dataSetIterator); + dataSetIterator.reset(); + dataSetIterator.setPreProcessor(normalizerStandardize); + return dataSetIterator; + } + + private MultiLayerNetwork getSingleLayer() { + int numHidden = 2; + + MultiLayerConfiguration configuration = new NeuralNetConfiguration.Builder() + .seed(12345) + .weightInit(WeightInit.XAVIER) + .miniBatch(true) + .updater(new Adam(0.1)) +// .updater(Nesterovs.builder() +// .momentum(0.1) +// .learningRateSchedule(new StepSchedule( +// ScheduleType.EPOCH, +// 1e-2, +// 0.1, +// 20)).build()) + .list(new DenseLayer.Builder().activation(new ActivationReLU()) + .nIn(4).nOut(2).build(), + new org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer.Builder() + .nIn(2).activation(new ActivationSigmoid()).initialRValue(0.1) + .nu(0.1) + .hiddenLayerSize(numHidden).build()) + .build(); + MultiLayerNetwork network = new MultiLayerNetwork(configuration); + network.init(); + network.setListeners(new ScoreIterationListener(1)); + return network; + } + + + public MultiLayerNetwork getGradientCheckNetwork(int numHidden) { + MultiLayerConfiguration configuration = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .seed(42).updater(new NoOp()).miniBatch(false) + .list(new DenseLayer.Builder().activation(new ActivationIdentity()).nIn(4).nOut(4).build(), + new org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer.Builder().nIn(4) + .nu(0.002).activation(new ActivationSigmoid()) + .hiddenLayerSize(numHidden).build()) + .build(); + MultiLayerNetwork network = new MultiLayerNetwork(configuration); + network.init(); + return network; + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingMaskingTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingMaskingTests.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingMaskingTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingMaskingTests.java index c2bcc6c47..f6ef09732 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingMaskingTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingMaskingTests.java @@ -29,10 +29,7 @@ import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.conf.layers.GlobalPoolingLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -47,8 +44,7 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.nd4j.linalg.indexing.NDArrayIndex.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class GlobalPoolingMaskingTests extends BaseDL4JTest { @Test @@ -292,7 +288,7 @@ public class GlobalPoolingMaskingTests extends BaseDL4JTest { INDArray outSubset = net.output(subset); INDArray outMaskedSubset = outMasked.getRow(i, true); - assertEquals(outSubset, outMaskedSubset, "minibatch: " + i); + assertEquals( outSubset, outMaskedSubset, "minibatch: " + i); } } } @@ -351,7 +347,7 @@ public class GlobalPoolingMaskingTests extends BaseDL4JTest { INDArray outSubset = net.output(subset); INDArray outMaskedSubset = outMasked.getRow(i, true); - assertEquals(outSubset, outMaskedSubset, "minibatch: " + i); + assertEquals( outSubset, outMaskedSubset, "minibatch: " + i); } } } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalTest.java new file mode 100644 index 000000000..2c9f0886e --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalTest.java @@ -0,0 +1,720 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.recurrent; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; +import org.deeplearning4j.earlystopping.scorecalc.DataSetLossCalculator; +import org.deeplearning4j.earlystopping.termination.MaxEpochsTerminationCondition; +import org.deeplearning4j.earlystopping.trainer.EarlyStoppingGraphTrainer; +import org.deeplearning4j.nn.conf.*; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM; +import org.deeplearning4j.nn.conf.layers.GravesLSTM; +import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; +import org.deeplearning4j.nn.conf.layers.recurrent.Bidirectional; +import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; +import org.deeplearning4j.nn.conf.layers.variational.BernoulliReconstructionDistribution; +import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.updater.MultiLayerUpdater; +import org.deeplearning4j.nn.updater.graph.ComputationGraphUpdater; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.util.ModelSerializer; +import org.deeplearning4j.util.TimeSeriesUtils; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.primitives.Pair; +import org.deeplearning4j.nn.workspace.ArrayType; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import static org.deeplearning4j.nn.conf.RNNFormat.NCW; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +public class BidirectionalTest extends BaseDL4JTest { + + private RNNFormat rnnDataFormat; + + public BidirectionalTest(RNNFormat rnnDataFormat){ + this.rnnDataFormat = rnnDataFormat; + } + + public static Object[] params(){ + return RNNFormat.values(); + } + @Test + public void compareImplementations(){ + for(WorkspaceMode wsm : WorkspaceMode.values()) { + log.info("*** Starting workspace mode: " + wsm); + + //Bidirectional(GravesLSTM) and GravesBidirectionalLSTM should be equivalent, given equivalent params + //Note that GravesBidirectionalLSTM implements ADD mode only + + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .trainingWorkspaceMode(wsm) + .inferenceWorkspaceMode(wsm) + .updater(new Adam()) + .list() + .layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())) + .layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())) + .layer(new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).dataFormat(rnnDataFormat) + .nIn(10).nOut(10).build()) + .build(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .trainingWorkspaceMode(wsm) + .inferenceWorkspaceMode(wsm) + .updater(new Adam()) + .list() + .layer(new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()) + .layer(new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()) + .layer(new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).dataFormat(rnnDataFormat) + .nIn(10).nOut(10).build()) + .build(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + assertEquals(net1.numParams(), net2.numParams()); + for (int i = 0; i < 3; i++) { + int n1 = (int)net1.getLayer(i).numParams(); + int n2 = (int)net2.getLayer(i).numParams(); + assertEquals(n1, n2); + } + + net2.setParams(net1.params()); //Assuming exact same layout here... + + INDArray in; + if (rnnDataFormat == NCW){ + in = Nd4j.rand(new int[]{3, 10, 5}); + }else{ + in = Nd4j.rand(new int[]{3, 5, 10}); + } + + INDArray out1 = net1.output(in); + INDArray out2 = net2.output(in); + + assertEquals(out1, out2); + + INDArray labels; + if (rnnDataFormat == NCW){ + labels = Nd4j.rand(new int[]{3, 10, 5}); + }else{ + labels = Nd4j.rand(new int[]{3, 5, 10}); + } + net1.setInput(in); + net1.setLabels(labels); + + net2.setInput(in); + net2.setLabels(labels); + + net1.computeGradientAndScore(); + net2.computeGradientAndScore(); + + //Ensure scores are equal: + assertEquals(net1.score(), net2.score(), 1e-6); + + //Ensure gradients are equal: + Gradient g1 = net1.gradient(); + Gradient g2 = net2.gradient(); + assertEquals(g1.gradient(), g2.gradient()); + + //Ensure updates are equal: + MultiLayerUpdater u1 = (MultiLayerUpdater) net1.getUpdater(); + MultiLayerUpdater u2 = (MultiLayerUpdater) net2.getUpdater(); + assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray()); + u1.update(net1, g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + u2.update(net2, g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(g1.gradient(), g2.gradient()); + assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray()); + + //Ensure params are equal, after fitting + net1.fit(in, labels); + net2.fit(in, labels); + + INDArray p1 = net1.params(); + INDArray p2 = net2.params(); + assertEquals(p1, p2); + } + } + + @Test + public void compareImplementationsCompGraph(){ +// for(WorkspaceMode wsm : WorkspaceMode.values()) { + for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) { + log.info("*** Starting workspace mode: " + wsm); + + //Bidirectional(GravesLSTM) and GravesBidirectionalLSTM should be equivalent, given equivalent params + //Note that GravesBidirectionalLSTM implements ADD mode only + + ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .updater(new Adam()) + .trainingWorkspaceMode(wsm) + .inferenceWorkspaceMode(wsm) + .graphBuilder() + .addInputs("in") + .layer("0", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).build()), "in") + .layer("1", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).build()), "0") + .layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE) + .nIn(10).nOut(10).build(), "1") + .setOutputs("2") + .build(); + + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .updater(new Adam()) + .trainingWorkspaceMode(wsm) + .inferenceWorkspaceMode(wsm) + .graphBuilder() + .addInputs("in") + .layer("0", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build(), "in") + .layer("1", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build(), "0") + .layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE) + .nIn(10).nOut(10).build(), "1") + .setOutputs("2") + .build(); + + ComputationGraph net1 = new ComputationGraph(conf1); + net1.init(); + + ComputationGraph net2 = new ComputationGraph(conf2); + net2.init(); + + assertEquals(net1.numParams(), net2.numParams()); + for (int i = 0; i < 3; i++) { + int n1 = (int)net1.getLayer(i).numParams(); + int n2 = (int)net2.getLayer(i).numParams(); + assertEquals(n1, n2); + } + + net2.setParams(net1.params()); //Assuming exact same layout here... + + INDArray in = Nd4j.rand(new int[]{3, 10, 5}); + + INDArray out1 = net1.outputSingle(in); + INDArray out2 = net2.outputSingle(in); + + assertEquals(out1, out2); + + INDArray labels = Nd4j.rand(new int[]{3, 10, 5}); + + net1.setInput(0,in); + net1.setLabels(labels); + + net2.setInput(0,in); + net2.setLabels(labels); + + net1.computeGradientAndScore(); + net2.computeGradientAndScore(); + + //Ensure scores are equal: + assertEquals(net1.score(), net2.score(), 1e-6); + + //Ensure gradients are equal: + Gradient g1 = net1.gradient(); + Gradient g2 = net2.gradient(); + assertEquals(g1.gradient(), g2.gradient()); + + //Ensure updates are equal: + ComputationGraphUpdater u1 = (ComputationGraphUpdater) net1.getUpdater(); + ComputationGraphUpdater u2 = (ComputationGraphUpdater) net2.getUpdater(); + assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray()); + u1.update(g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + u2.update(g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + assertEquals(g1.gradient(), g2.gradient()); + assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray()); + + //Ensure params are equal, after fitting + net1.fit(new DataSet(in, labels)); + net2.fit(new DataSet(in, labels)); + + INDArray p1 = net1.params(); + INDArray p2 = net2.params(); + assertEquals(p1, p2); + } + } + + + @Test + public void testSerialization() throws Exception { + + for(WorkspaceMode wsm : WorkspaceMode.values()) { + log.info("*** Starting workspace mode: " + wsm); + + Nd4j.getRandom().setSeed(12345); + + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .trainingWorkspaceMode(wsm) + .inferenceWorkspaceMode(wsm) + .updater(new Adam()) + .list() + .layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())) + .layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())) + .layer(new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE) + .nIn(10).nOut(10).dataFormat(rnnDataFormat).build()) + .build(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + + INDArray in; + INDArray labels; + + long[] inshape = rnnDataFormat == NCW ? new long[]{3, 10, 5} : new long[]{3, 5, 10}; + + in = Nd4j.rand(inshape); + labels = Nd4j.rand(inshape); + + net1.fit(in, labels); + + byte[] bytes; + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + ModelSerializer.writeModel(net1, baos, true); + bytes = baos.toByteArray(); + } + + + MultiLayerNetwork net2 = ModelSerializer.restoreMultiLayerNetwork(new ByteArrayInputStream(bytes), true); + + + in = Nd4j.rand(inshape); + labels = Nd4j.rand(inshape); + + INDArray out1 = net1.output(in); + INDArray out2 = net2.output(in); + + assertEquals(out1, out2); + + net1.setInput(in); + net2.setInput(in); + net1.setLabels(labels); + net2.setLabels(labels); + + net1.computeGradientAndScore(); + net2.computeGradientAndScore(); + + assertEquals(net1.score(), net2.score(), 1e-6); + assertEquals(net1.gradient().gradient(), net2.gradient().gradient()); + } + } + + + @Test + public void testSerializationCompGraph() throws Exception { + + for(WorkspaceMode wsm : WorkspaceMode.values()) { + log.info("*** Starting workspace mode: " + wsm); + + Nd4j.getRandom().setSeed(12345); + + ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder() + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .trainingWorkspaceMode(wsm) + .inferenceWorkspaceMode(wsm) + .updater(new Adam()) + .graphBuilder() + .addInputs("in") + .layer("0", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()), "in") + .layer("1", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()), "0") + .layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).dataFormat(rnnDataFormat) + .nIn(10).nOut(10).build(), "1") + .setOutputs("2") + .build(); + + ComputationGraph net1 = new ComputationGraph(conf1); + net1.init(); + long[] inshape = (rnnDataFormat == NCW)? new long[]{3, 10, 5}: new long[]{3, 5, 10}; + INDArray in = Nd4j.rand(inshape); + INDArray labels = Nd4j.rand(inshape); + + net1.fit(new DataSet(in, labels)); + + byte[] bytes; + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + ModelSerializer.writeModel(net1, baos, true); + bytes = baos.toByteArray(); + } + + + ComputationGraph net2 = ModelSerializer.restoreComputationGraph(new ByteArrayInputStream(bytes), true); + + + in = Nd4j.rand(inshape); + labels = Nd4j.rand(inshape); + + INDArray out1 = net1.outputSingle(in); + INDArray out2 = net2.outputSingle(in); + + assertEquals(out1, out2); + + net1.setInput(0, in); + net2.setInput(0, in); + net1.setLabels(labels); + net2.setLabels(labels); + + net1.computeGradientAndScore(); + net2.computeGradientAndScore(); + + assertEquals(net1.score(), net2.score(), 1e-6); + assertEquals(net1.gradient().gradient(), net2.gradient().gradient()); + } + } + + @Test + public void testSimpleBidirectional() { + + for (WorkspaceMode wsm : WorkspaceMode.values()) { + log.info("*** Starting workspace mode: " + wsm); + Nd4j.getRandom().setSeed(12345); + + Bidirectional.Mode[] modes = new Bidirectional.Mode[]{Bidirectional.Mode.CONCAT, Bidirectional.Mode.ADD, + Bidirectional.Mode.AVERAGE, Bidirectional.Mode.MUL}; + + long[] inshape = rnnDataFormat == NCW ? new long[]{3, 10, 6} : new long[]{3, 6, 10}; + INDArray in = Nd4j.rand(inshape); + + for (Bidirectional.Mode m : modes) { + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .trainingWorkspaceMode(wsm) + .inferenceWorkspaceMode(wsm) + .updater(new Adam()) + .list() + .layer(new Bidirectional(m, new SimpleRnn.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())) + .build(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .updater(new Adam()) + .list() + .layer(new SimpleRnn.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()) + .build(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2.clone()); + net2.init(); + MultiLayerNetwork net3 = new MultiLayerNetwork(conf2.clone()); + net3.init(); + + net2.setParam("0_W", net1.getParam("0_fW")); + net2.setParam("0_RW", net1.getParam("0_fRW")); + net2.setParam("0_b", net1.getParam("0_fb")); + + net3.setParam("0_W", net1.getParam("0_bW")); + net3.setParam("0_RW", net1.getParam("0_bRW")); + net3.setParam("0_b", net1.getParam("0_bb")); + + INDArray inReverse = TimeSeriesUtils.reverseTimeSeries(in, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT, rnnDataFormat); + INDArray out1 = net1.output(in); + INDArray out2 = net2.output(in); + INDArray out3 = TimeSeriesUtils.reverseTimeSeries(net3.output(inReverse), LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT, rnnDataFormat); + + INDArray outExp; + switch (m) { + case ADD: + outExp = out2.add(out3); + break; + case MUL: + outExp = out2.mul(out3); + break; + case AVERAGE: + outExp = out2.add(out3).muli(0.5); + break; + case CONCAT: + outExp = Nd4j.concat((rnnDataFormat == NCW)?1:2, out2, out3); + break; + default: + throw new RuntimeException(); + } + + assertEquals(outExp, out1, m.toString()); + + + //Check gradients: + if (m == Bidirectional.Mode.ADD || m == Bidirectional.Mode.CONCAT) { + + INDArray eps = Nd4j.rand(inshape); + + INDArray eps1; + if (m == Bidirectional.Mode.CONCAT) { + eps1 = Nd4j.concat((rnnDataFormat == NCW)?1:2, eps, eps); + } else { + eps1 = eps; + } + + net1.setInput(in); + net2.setInput(in); + net3.setInput(TimeSeriesUtils.reverseTimeSeries(in, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT, rnnDataFormat)); + net1.feedForward(true, false); + net2.feedForward(true, false); + net3.feedForward(true, false); + + Pair p1 = net1.backpropGradient(eps1, LayerWorkspaceMgr.noWorkspaces()); + Pair p2 = net2.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces()); + Pair p3 = net3.backpropGradient(TimeSeriesUtils.reverseTimeSeries(eps, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT, rnnDataFormat), LayerWorkspaceMgr.noWorkspaces()); + Gradient g1 = p1.getFirst(); + Gradient g2 = p2.getFirst(); + Gradient g3 = p3.getFirst(); + + for (boolean updates : new boolean[]{false, true}) { + if (updates) { + net1.getUpdater().update(net1, g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + net2.getUpdater().update(net2, g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + net3.getUpdater().update(net3, g3, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + } + + assertEquals(g2.gradientForVariable().get("0_W"), g1.gradientForVariable().get("0_fW")); + assertEquals(g2.gradientForVariable().get("0_RW"), g1.gradientForVariable().get("0_fRW")); + assertEquals(g2.gradientForVariable().get("0_b"), g1.gradientForVariable().get("0_fb")); + + assertEquals(g3.gradientForVariable().get("0_W"), g1.gradientForVariable().get("0_bW")); + assertEquals(g3.gradientForVariable().get("0_RW"), g1.gradientForVariable().get("0_bRW")); + assertEquals(g3.gradientForVariable().get("0_b"), g1.gradientForVariable().get("0_bb")); + } + + } + } + } + } + + + @Test + public void testSimpleBidirectionalCompGraph() { + + for (WorkspaceMode wsm : WorkspaceMode.values()) { + log.info("*** Starting workspace mode: " + wsm); + Nd4j.getRandom().setSeed(12345); + + Bidirectional.Mode[] modes = new Bidirectional.Mode[]{Bidirectional.Mode.CONCAT, Bidirectional.Mode.ADD, + Bidirectional.Mode.AVERAGE, Bidirectional.Mode.MUL}; + + + long[] inshape = rnnDataFormat == NCW ? new long[]{3, 10, 6} : new long[]{3, 6, 10}; + INDArray in = Nd4j.rand(inshape); + + + for (Bidirectional.Mode m : modes) { + ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .trainingWorkspaceMode(wsm) + .inferenceWorkspaceMode(wsm) + .updater(new Adam()) + .graphBuilder() + .addInputs("in") + .layer("0", new Bidirectional(m, new SimpleRnn.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()), "in") + .setOutputs("0") + .build(); + + ComputationGraph net1 = new ComputationGraph(conf1); + net1.init(); + + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .weightInit(WeightInit.XAVIER) + .updater(new Adam()) + .graphBuilder() + .addInputs("in") + .layer("0", new SimpleRnn.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build(), "in") + .setOutputs("0") + .build(); + + ComputationGraph net2 = new ComputationGraph(conf2.clone()); + net2.init(); + ComputationGraph net3 = new ComputationGraph(conf2.clone()); + net3.init(); + + net2.setParam("0_W", net1.getParam("0_fW")); + net2.setParam("0_RW", net1.getParam("0_fRW")); + net2.setParam("0_b", net1.getParam("0_fb")); + + net3.setParam("0_W", net1.getParam("0_bW")); + net3.setParam("0_RW", net1.getParam("0_bRW")); + net3.setParam("0_b", net1.getParam("0_bb")); + + + INDArray out1 = net1.outputSingle(in); + INDArray out2 = net2.outputSingle(in); + INDArray out3; + INDArray inReverse; + if (rnnDataFormat == RNNFormat.NWC){ + inReverse = TimeSeriesUtils.reverseTimeSeries(in.permute(0, 2, 1), LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT).permute(0, 2, 1); + out3 = net3.outputSingle(inReverse); + out3 = TimeSeriesUtils.reverseTimeSeries(out3.permute(0, 2, 1), LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT).permute(0, 2, 1); + + } + else{ + inReverse = TimeSeriesUtils.reverseTimeSeries(in, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT); + out3 = net3.outputSingle(inReverse); + out3 = TimeSeriesUtils.reverseTimeSeries(out3, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT); + + } + + INDArray outExp; + switch (m) { + case ADD: + outExp = out2.add(out3); + break; + case MUL: + outExp = out2.mul(out3); + break; + case AVERAGE: + outExp = out2.add(out3).muli(0.5); + break; + case CONCAT: + System.out.println(out2.shapeInfoToString()); + System.out.println(out3.shapeInfoToString()); + outExp = Nd4j.concat((rnnDataFormat == NCW)?1:2, out2, out3); + break; + default: + throw new RuntimeException(); + } + + assertEquals(outExp, out1, m.toString()); + + + //Check gradients: + if (m == Bidirectional.Mode.ADD || m == Bidirectional.Mode.CONCAT) { + + INDArray eps = Nd4j.rand(inshape); + + INDArray eps1; + if (m == Bidirectional.Mode.CONCAT) { + eps1 = Nd4j.concat((rnnDataFormat == NCW)?1:2, eps, eps); + } else { + eps1 = eps; + } + + INDArray epsReversed = (rnnDataFormat == NCW)? + TimeSeriesUtils.reverseTimeSeries(eps, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT): + TimeSeriesUtils.reverseTimeSeries(eps.permute(0, 2, 1), LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT) + .permute(0, 2, 1); + net1.outputSingle(true, false, in); + net2.outputSingle(true, false, in); + net3.outputSingle(true, false, inReverse); + + Gradient g1 = net1.backpropGradient(eps1); + Gradient g2 = net2.backpropGradient(eps); + Gradient g3 = net3.backpropGradient(epsReversed); + + for (boolean updates : new boolean[]{false, true}) { + if (updates) { + net1.getUpdater().update(g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + net2.getUpdater().update(g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + net3.getUpdater().update(g3, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); + } + + assertEquals(g2.gradientForVariable().get("0_W"), g1.gradientForVariable().get("0_fW")); + assertEquals(g2.gradientForVariable().get("0_RW"), g1.gradientForVariable().get("0_fRW")); + assertEquals(g2.gradientForVariable().get("0_b"), g1.gradientForVariable().get("0_fb")); + + assertEquals(g3.gradientForVariable().get("0_W"), g1.gradientForVariable().get("0_bW")); + assertEquals(g3.gradientForVariable().get("0_RW"), g1.gradientForVariable().get("0_bRW")); + assertEquals(g3.gradientForVariable().get("0_b"), g1.gradientForVariable().get("0_bb")); + } + } + } + } + } + + + @Test + public void testIssue5472(){ + //https://github.com/deeplearning4j/deeplearning4j/issues/5472 + + int in = 2; + int out = 2; + ComputationGraphConfiguration.GraphBuilder builder = new NeuralNetConfiguration.Builder() + .updater(new Adam(0.01)) + .activation(Activation.RELU) + .graphBuilder() + .addInputs("IN") + .setInputTypes(InputType.recurrent(in)) + .addLayer("AUTOENCODER", + new VariationalAutoencoder.Builder() + .encoderLayerSizes(64) + .decoderLayerSizes(64) + .nOut(7) + .pzxActivationFunction(Activation.IDENTITY) + .reconstructionDistribution(new BernoulliReconstructionDistribution(Activation.SIGMOID.getActivationFunction())).build(), + "IN") + .addLayer("RNN", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nOut(128).build()), "AUTOENCODER") + .addLayer("OUT", new RnnOutputLayer.Builder() + .nOut(out) + .activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "RNN") + .setOutputs("OUT") + + ; + + ComputationGraph net = new ComputationGraph(builder.build()); + net.init(); + + MultiDataSetIterator iterator = new SingletonMultiDataSetIterator(new MultiDataSet(Nd4j.create(10,in,5), Nd4j.create(10,out,5))); + + EarlyStoppingConfiguration.Builder b = new EarlyStoppingConfiguration.Builder<>() + .epochTerminationConditions(new MaxEpochsTerminationCondition(10)) + .scoreCalculator(new DataSetLossCalculator(iterator, true)) + .evaluateEveryNEpochs(1) + .modelSaver(new InMemoryModelSaver<>()); + + EarlyStoppingGraphTrainer earlyStoppingGraphTrainer = new EarlyStoppingGraphTrainer(b.build(), net, iterator, null); + earlyStoppingGraphTrainer.fit(); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTMTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTMTest.java new file mode 100644 index 000000000..b6f3e7a58 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTMTest.java @@ -0,0 +1,561 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.recurrent; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.CacheMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.RNNFormat; +import org.deeplearning4j.nn.conf.distribution.UniformDistribution; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.params.GravesBidirectionalLSTMParamInitializer; +import org.deeplearning4j.nn.params.GravesLSTMParamInitializer; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.impl.ActivationSigmoid; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.AdaGrad; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.primitives.Pair; + +import static org.junit.jupiter.api.Assertions.*; + +public class GravesBidirectionalLSTMTest extends BaseDL4JTest { + private double score = 0.0; + private RNNFormat rnnDataFormat; + + public GravesBidirectionalLSTMTest(RNNFormat rnnDataFormat){ + this.rnnDataFormat = rnnDataFormat; + } + + public static Object[] params(){ + return RNNFormat.values(); + } + @Test + public void testBidirectionalLSTMGravesForwardBasic() { + //Very basic test of forward prop. of LSTM layer with a time series. + //Essentially make sure it doesn't throw any exceptions, and provides output in the correct shape. + int nIn = 13; + int nHiddenUnits = 17; + + final NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn) + .nOut(nHiddenUnits).dataFormat(rnnDataFormat).activation(Activation.TANH).build()) + .build(); + + val numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + final GravesBidirectionalLSTM layer = + (GravesBidirectionalLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + + //Data: has shape [miniBatchSize,nIn,timeSeriesLength]; + //Output/activations has shape [miniBatchsize,nHiddenUnits,timeSeriesLength]; + if (rnnDataFormat == RNNFormat.NCW){ + final INDArray dataSingleExampleTimeLength1 = Nd4j.ones(1, nIn, 1); + final INDArray activations1 = layer.activate(dataSingleExampleTimeLength1, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations1.shape(), new long[] {1, nHiddenUnits, 1}); + + final INDArray dataMultiExampleLength1 = Nd4j.ones(10, nIn, 1); + final INDArray activations2 = layer.activate(dataMultiExampleLength1, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations2.shape(), new long[] {10, nHiddenUnits, 1}); + + final INDArray dataSingleExampleLength12 = Nd4j.ones(1, nIn, 12); + final INDArray activations3 = layer.activate(dataSingleExampleLength12, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations3.shape(), new long[] {1, nHiddenUnits, 12}); + + final INDArray dataMultiExampleLength15 = Nd4j.ones(10, nIn, 15); + final INDArray activations4 = layer.activate(dataMultiExampleLength15, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations4.shape(), new long[] {10, nHiddenUnits, 15}); + } + else{ + final INDArray dataSingleExampleTimeLength1 = Nd4j.ones(1, 1, nIn); + final INDArray activations1 = layer.activate(dataSingleExampleTimeLength1, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations1.shape(), new long[] {1, 1, nHiddenUnits}); + + final INDArray dataMultiExampleLength1 = Nd4j.ones(10, 1, nIn); + final INDArray activations2 = layer.activate(dataMultiExampleLength1, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations2.shape(), new long[] {10, 1, nHiddenUnits}); + + final INDArray dataSingleExampleLength12 = Nd4j.ones(1, 12, nIn); + final INDArray activations3 = layer.activate(dataSingleExampleLength12, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations3.shape(), new long[] {1, 12, nHiddenUnits}); + + final INDArray dataMultiExampleLength15 = Nd4j.ones(10, 15, nIn); + final INDArray activations4 = layer.activate(dataMultiExampleLength15, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations4.shape(), new long[] {10, 15, nHiddenUnits}); + } + + } + + @Test + public void testBidirectionalLSTMGravesBackwardBasic() { + //Very basic test of backprop for mini-batch + time series + //Essentially make sure it doesn't throw any exceptions, and provides output in the correct shape. + + testGravesBackwardBasicHelper(13, 3, 17, 10, 7); + testGravesBackwardBasicHelper(13, 3, 17, 1, 7); //Edge case: miniBatchSize = 1 + testGravesBackwardBasicHelper(13, 3, 17, 10, 1); //Edge case: timeSeriesLength = 1 + testGravesBackwardBasicHelper(13, 3, 17, 1, 1); //Edge case: both miniBatchSize = 1 and timeSeriesLength = 1 + } + + private void testGravesBackwardBasicHelper(int nIn, int nOut, int lstmNHiddenUnits, int miniBatchSize, + int timeSeriesLength) { + + INDArray inputData = (rnnDataFormat == RNNFormat.NCW)?Nd4j.ones(miniBatchSize, nIn, timeSeriesLength): + Nd4j.ones(miniBatchSize, timeSeriesLength, nIn); + + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn) + .nOut(lstmNHiddenUnits).dataFormat(rnnDataFormat) + .dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()) + .build(); + + long numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + GravesBidirectionalLSTM lstm = + (GravesBidirectionalLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + lstm.setBackpropGradientsViewArray(Nd4j.create(1, conf.getLayer().initializer().numParams(conf))); + //Set input, do a forward pass: + lstm.activate(inputData, false, LayerWorkspaceMgr.noWorkspaces()); + assertNotNull(lstm.input()); + + INDArray epsilon =(rnnDataFormat == RNNFormat.NCW)? Nd4j.ones(miniBatchSize, lstmNHiddenUnits, timeSeriesLength): + Nd4j.ones(miniBatchSize, timeSeriesLength, lstmNHiddenUnits); + + Pair out = lstm.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); + Gradient outGradient = out.getFirst(); + INDArray nextEpsilon = out.getSecond(); + + INDArray biasGradientF = outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS); + INDArray inWeightGradientF = + outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS); + INDArray recurrentWeightGradientF = outGradient + .getGradientFor(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS); + assertNotNull(biasGradientF); + assertNotNull(inWeightGradientF); + assertNotNull(recurrentWeightGradientF); + + INDArray biasGradientB = outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS); + INDArray inWeightGradientB = + outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS); + INDArray recurrentWeightGradientB = outGradient + .getGradientFor(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS); + assertNotNull(biasGradientB); + assertNotNull(inWeightGradientB); + assertNotNull(recurrentWeightGradientB); + + assertArrayEquals(biasGradientF.shape(), new long[] {1, 4 * lstmNHiddenUnits}); + assertArrayEquals(inWeightGradientF.shape(), new long[] {nIn, 4 * lstmNHiddenUnits}); + assertArrayEquals(recurrentWeightGradientF.shape(), new long[] {lstmNHiddenUnits, 4 * lstmNHiddenUnits + 3}); + + assertArrayEquals(biasGradientB.shape(), new long[] {1, 4 * lstmNHiddenUnits}); + assertArrayEquals(inWeightGradientB.shape(), new long[] {nIn, 4 * lstmNHiddenUnits}); + assertArrayEquals(recurrentWeightGradientB.shape(), new long[] {lstmNHiddenUnits, 4 * lstmNHiddenUnits + 3}); + + assertNotNull(nextEpsilon); + if (rnnDataFormat == RNNFormat.NCW) { + assertArrayEquals(nextEpsilon.shape(), new long[]{miniBatchSize, nIn, timeSeriesLength}); + }else{ + assertArrayEquals(nextEpsilon.shape(), new long[]{miniBatchSize, timeSeriesLength, nIn }); + } + + //Check update: + for (String s : outGradient.gradientForVariable().keySet()) { + lstm.update(outGradient.getGradientFor(s), s); + } + } + + @Test + public void testGravesBidirectionalLSTMForwardPassHelper() throws Exception { + //GravesBidirectionalLSTM.activateHelper() has different behaviour (due to optimizations) when forBackprop==true vs false + //But should otherwise provide identical activations + Nd4j.getRandom().setSeed(12345); + + final int nIn = 10; + final int layerSize = 15; + final int miniBatchSize = 4; + final int timeSeriesLength = 7; + + final NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn) + .nOut(layerSize) + .dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()) + .build(); + + long numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + final GravesBidirectionalLSTM lstm = + (GravesBidirectionalLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + final INDArray input = Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength}); + lstm.setInput(input, LayerWorkspaceMgr.noWorkspaces()); + + + final INDArray fwdPassFalse = LSTMHelpers.activateHelper(lstm, lstm.conf(), new ActivationSigmoid(), + lstm.input(), + lstm.getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS), + lstm.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS), + lstm.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS), false, null, null, + false, true, GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS, null, true, + null, CacheMode.NONE, LayerWorkspaceMgr.noWorkspaces(), true).fwdPassOutput; + + final INDArray[] fwdPassTrue = LSTMHelpers.activateHelper(lstm, lstm.conf(), new ActivationSigmoid(), + lstm.input(), + lstm.getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS), + lstm.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS), + lstm.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS), false, null, null, + true, true, GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS, null, true, null, + CacheMode.NONE, LayerWorkspaceMgr.noWorkspaces(), true).fwdPassOutputAsArrays; + + //I have no idea what the heck this does --Ben + for (int i = 0; i < timeSeriesLength; i++) { + final INDArray sliceFalse = fwdPassFalse.tensorAlongDimension(i, 1, 0); + final INDArray sliceTrue = fwdPassTrue[i]; + assertTrue(sliceFalse.equals(sliceTrue)); + } + } + + static private void reverseColumnsInPlace(final INDArray x) { + final long N = x.size(1); + final INDArray x2 = x.dup(); + + for (int t = 0; t < N; t++) { + final long b = N - t - 1; + //clone? + x.putColumn(t, x2.getColumn(b)); + } + } + + @Test + public void testGetSetParmas() { + final int nIn = 2; + final int layerSize = 3; + final int miniBatchSize = 2; + final int timeSeriesLength = 10; + + Nd4j.getRandom().setSeed(12345); + + final NeuralNetConfiguration confBidirectional = new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn) + .nOut(layerSize).dataFormat(rnnDataFormat) + .dist(new UniformDistribution(-0.1, 0.1)).activation(Activation.TANH).build()) + .build(); + + + long numParams = confBidirectional.getLayer().initializer().numParams(confBidirectional); + INDArray params = Nd4j.create(1, numParams); + final GravesBidirectionalLSTM bidirectionalLSTM = (GravesBidirectionalLSTM) confBidirectional.getLayer() + .instantiate(confBidirectional, null, 0, params, true, params.dataType()); + + + final INDArray sig = (rnnDataFormat == RNNFormat.NCW)?Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength}): + Nd4j.rand(new int[] {miniBatchSize, timeSeriesLength, nIn}); + + final INDArray act1 = bidirectionalLSTM.activate(sig, false, LayerWorkspaceMgr.noWorkspaces()); + + params = bidirectionalLSTM.params(); + + bidirectionalLSTM.setParams(params); + + final INDArray act2 = bidirectionalLSTM.activate(sig, false, LayerWorkspaceMgr.noWorkspaces()); + + assertArrayEquals(act2.data().asDouble(), act1.data().asDouble(), 1e-8); + + + } + + @Test + public void testSimpleForwardsAndBackwardsActivation() { + + final int nIn = 2; + final int layerSize = 3; + final int miniBatchSize = 1; + final int timeSeriesLength = 5; + + Nd4j.getRandom().setSeed(12345); + + final NeuralNetConfiguration confBidirectional = + new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder() + .nIn(nIn).nOut(layerSize).dataFormat(rnnDataFormat) + .dist(new UniformDistribution(-0.1, 0.1)) + .activation(Activation.TANH).updater(new NoOp()).build()) + .build(); + + final NeuralNetConfiguration confForwards = new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(layerSize).dataFormat(rnnDataFormat) + .weightInit(WeightInit.ZERO).activation(Activation.TANH).build()) + .build(); + + long numParams = confForwards.getLayer().initializer().numParams(confForwards); + INDArray params = Nd4j.create(1, numParams); + long numParamsBD = confBidirectional.getLayer().initializer().numParams(confBidirectional); + INDArray paramsBD = Nd4j.create(1, numParamsBD); + final GravesBidirectionalLSTM bidirectionalLSTM = (GravesBidirectionalLSTM) confBidirectional.getLayer() + .instantiate(confBidirectional, null, 0, paramsBD, true, params.dataType()); + final GravesLSTM forwardsLSTM = + (GravesLSTM) confForwards.getLayer().instantiate(confForwards, null, 0, params, true, params.dataType()); + + bidirectionalLSTM.setBackpropGradientsViewArray( + Nd4j.create(1, confBidirectional.getLayer().initializer().numParams(confBidirectional))); + forwardsLSTM.setBackpropGradientsViewArray( + Nd4j.create(1, confForwards.getLayer().initializer().numParams(confForwards))); + + + final INDArray sig = (rnnDataFormat == RNNFormat.NCW)?Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength}): + Nd4j.rand(new int[] {miniBatchSize, timeSeriesLength, nIn}); + final INDArray sigb = sig.dup(); + + if (rnnDataFormat == RNNFormat.NCW) { + reverseColumnsInPlace(sigb.slice(0)); + } + else{ + reverseColumnsInPlace(sigb.slice(0).permute(1, 0)); + } + + final INDArray recurrentWeightsF = bidirectionalLSTM + .getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS); + final INDArray inputWeightsF = + bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS); + final INDArray biasWeightsF = + bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS); + + final INDArray recurrentWeightsF2 = forwardsLSTM.getParam(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY); + final INDArray inputWeightsF2 = forwardsLSTM.getParam(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY); + final INDArray biasWeightsF2 = forwardsLSTM.getParam(GravesLSTMParamInitializer.BIAS_KEY); + + //assert that the forwards part of the bidirectional layer is equal to that of the regular LSTM + assertArrayEquals(recurrentWeightsF2.shape(), recurrentWeightsF.shape()); + assertArrayEquals(inputWeightsF2.shape(), inputWeightsF.shape()); + assertArrayEquals(biasWeightsF2.shape(), biasWeightsF.shape()); + + forwardsLSTM.setParam(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY, recurrentWeightsF); + forwardsLSTM.setParam(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY, inputWeightsF); + forwardsLSTM.setParam(GravesLSTMParamInitializer.BIAS_KEY, biasWeightsF); + + //copy forwards weights to make the forwards activations do the same thing + + final INDArray recurrentWeightsB = bidirectionalLSTM + .getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS); + final INDArray inputWeightsB = + bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS); + final INDArray biasWeightsB = + bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS); + + //assert that the forwards and backwards are the same shapes + assertArrayEquals(recurrentWeightsF.shape(), recurrentWeightsB.shape()); + assertArrayEquals(inputWeightsF.shape(), inputWeightsB.shape()); + assertArrayEquals(biasWeightsF.shape(), biasWeightsB.shape()); + + //zero out backwards layer + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS, + Nd4j.zeros(recurrentWeightsB.shape())); + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS, + Nd4j.zeros(inputWeightsB.shape())); + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS, + Nd4j.zeros(biasWeightsB.shape())); + + + forwardsLSTM.setInput(sig, LayerWorkspaceMgr.noWorkspaces()); + + //compare activations + final INDArray activation1 = forwardsLSTM.activate(sig, false, LayerWorkspaceMgr.noWorkspaces()).slice(0); + final INDArray activation2 = bidirectionalLSTM.activate(sig, false, LayerWorkspaceMgr.noWorkspaces()).slice(0); + + assertArrayEquals(activation1.data().asFloat(), activation2.data().asFloat(), 1e-5f); + + final INDArray randSig = (rnnDataFormat == RNNFormat.NCW)?Nd4j.rand(new int[] {1, layerSize, timeSeriesLength}): + Nd4j.rand(new int[] {1, timeSeriesLength, layerSize}); + INDArray randSigBackwards = randSig.dup(); + if (rnnDataFormat == RNNFormat.NCW){ + reverseColumnsInPlace(randSigBackwards.slice(0)); + }else{ + reverseColumnsInPlace(randSigBackwards.slice(0).permute(1, 0)); + } + + final Pair backprop1 = forwardsLSTM.backpropGradient(randSig, LayerWorkspaceMgr.noWorkspaces()); + final Pair backprop2 = bidirectionalLSTM.backpropGradient(randSig, LayerWorkspaceMgr.noWorkspaces()); + + //compare gradients + assertArrayEquals( + backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY).dup() + .data().asFloat(), + backprop2.getFirst() + .getGradientFor(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS) + .dup().data().asFloat(), + 1e-5f); + + assertArrayEquals( + backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY).dup().data() + .asFloat(), + backprop2.getFirst() + .getGradientFor(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS) + .dup().data().asFloat(), + 1e-5f); + + assertArrayEquals( + backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.BIAS_KEY).dup().data().asFloat(), + backprop2.getFirst().getGradientFor(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS) + .dup().data().asFloat(), + 1e-5f); + + //copy forwards to backwards + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS, + bidirectionalLSTM.getParam( + GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS)); + + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS, + bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS)); + + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS, + bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS)); + + //zero out forwards layer + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS, + Nd4j.zeros(recurrentWeightsB.shape())); + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS, + Nd4j.zeros(inputWeightsB.shape())); + bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS, + Nd4j.zeros(biasWeightsB.shape())); + + //run on reversed signal + final INDArray activation3 = bidirectionalLSTM.activate(sigb, false, LayerWorkspaceMgr.noWorkspaces()).slice(0); + + final INDArray activation3Reverse = activation3.dup(); + if (rnnDataFormat == RNNFormat.NCW){ + reverseColumnsInPlace(activation3Reverse); + } + else{ + reverseColumnsInPlace(activation3Reverse.permute(1, 0)); + } + + assertArrayEquals(activation3Reverse.shape(), activation1.shape()); + assertEquals(activation3Reverse, activation1); + + + //test backprop now + final INDArray refBackGradientReccurrent = + backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY); + + final INDArray refBackGradientInput = + backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY); + + final INDArray refBackGradientBias = backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.BIAS_KEY); + + //reverse weights only with backwards signal should yield same result as forwards weights with forwards signal + final Pair backprop3 = bidirectionalLSTM.backpropGradient(randSigBackwards, LayerWorkspaceMgr.noWorkspaces()); + + final INDArray backGradientRecurrent = backprop3.getFirst() + .getGradientFor(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS); + final INDArray backGradientInput = backprop3.getFirst() + .getGradientFor(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS); + final INDArray backGradientBias = + backprop3.getFirst().getGradientFor(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS); + + assertArrayEquals(refBackGradientBias.dup().data().asDouble(), backGradientBias.dup().data().asDouble(), 1e-6); + + assertArrayEquals(refBackGradientInput.dup().data().asDouble(), backGradientInput.dup().data().asDouble(), + 1e-6); + + assertArrayEquals(refBackGradientReccurrent.dup().data().asDouble(), + backGradientRecurrent.dup().data().asDouble(), 1e-6); + + final INDArray refEpsilon = backprop1.getSecond().dup(); + final INDArray backEpsilon = backprop3.getSecond().dup(); + + if (rnnDataFormat == RNNFormat.NCW) { + reverseColumnsInPlace(refEpsilon.slice(0)); + } + else{ + reverseColumnsInPlace(refEpsilon.slice(0).permute(1, 0)); + } + assertArrayEquals(backEpsilon.dup().data().asDouble(), refEpsilon.dup().data().asDouble(), 1e-6); + + } + + @Test + public void testSerialization() { + + final MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new AdaGrad(0.1)) + .l2(0.001) + .seed(12345).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder() + .activation(Activation.TANH).nIn(2).nOut(2) + .dist(new UniformDistribution(-0.05, 0.05)).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder() + .activation(Activation.TANH).nIn(2).nOut(2) + .dist(new UniformDistribution(-0.05, 0.05)).build()) + .layer(2, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder() + .activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT) + .nIn(2).nOut(2).build()) + .build(); + + + final String json1 = conf1.toJson(); + + final MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json1); + + final String json2 = conf1.toJson(); + + + assertEquals(json1, json2); + } + + @Test + public void testGateActivationFnsSanityCheck() { + for (String gateAfn : new String[] {"sigmoid", "hardsigmoid"}) { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .seed(12345).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder() + .gateActivationFunction(gateAfn).activation(Activation.TANH).nIn(2).nOut(2).dataFormat(rnnDataFormat) + .build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(2).dataFormat(rnnDataFormat) + .activation(Activation.TANH).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(gateAfn, ((org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM) net.getLayer(0).conf() + .getLayer()).getGateActivationFn().toString()); + + INDArray in = Nd4j.rand(new int[] {3, 2, 5}); + INDArray labels = Nd4j.rand(new int[] {3, 2, 5}); + if (rnnDataFormat == RNNFormat.NWC){ + in = in.permute(0, 2, 1); + labels = labels.permute(0, 2, 1); + } + net.fit(in, labels); + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTMTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTMTest.java new file mode 100644 index 000000000..80d3af6fe --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTMTest.java @@ -0,0 +1,280 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.recurrent; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.common.config.DL4JClassLoading; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.UniformDistribution; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.params.GravesLSTMParamInitializer; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.INDArrayIndex; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.primitives.Pair; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + + +public class GravesLSTMTest extends BaseDL4JTest { + + @Test + public void testLSTMGravesForwardBasic() { + //Very basic test of forward prop. of LSTM layer with a time series. + //Essentially make sure it doesn't throw any exceptions, and provides output in the correct shape. + + int nIn = 13; + int nHiddenUnits = 17; + + NeuralNetConfiguration conf = + new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn) + .nOut(nHiddenUnits).activation(Activation.TANH).build()) + .build(); + + val numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + GravesLSTM layer = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + + //Data: has shape [miniBatchSize,nIn,timeSeriesLength]; + //Output/activations has shape [miniBatchsize,nHiddenUnits,timeSeriesLength]; + + INDArray dataSingleExampleTimeLength1 = Nd4j.ones(1, nIn, 1); + INDArray activations1 = layer.activate(dataSingleExampleTimeLength1, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations1.shape(), new long[] {1, nHiddenUnits, 1}); + + INDArray dataMultiExampleLength1 = Nd4j.ones(10, nIn, 1); + INDArray activations2 = layer.activate(dataMultiExampleLength1, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations2.shape(), new long[] {10, nHiddenUnits, 1}); + + INDArray dataSingleExampleLength12 = Nd4j.ones(1, nIn, 12); + INDArray activations3 = layer.activate(dataSingleExampleLength12, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations3.shape(), new long[] {1, nHiddenUnits, 12}); + + INDArray dataMultiExampleLength15 = Nd4j.ones(10, nIn, 15); + INDArray activations4 = layer.activate(dataMultiExampleLength15, false, LayerWorkspaceMgr.noWorkspaces()); + assertArrayEquals(activations4.shape(), new long[] {10, nHiddenUnits, 15}); + } + + @Test + public void testLSTMGravesBackwardBasic() { + //Very basic test of backprop for mini-batch + time series + //Essentially make sure it doesn't throw any exceptions, and provides output in the correct shape. + + testGravesBackwardBasicHelper(13, 3, 17, 10, 7); + testGravesBackwardBasicHelper(13, 3, 17, 1, 7); //Edge case: miniBatchSize = 1 + testGravesBackwardBasicHelper(13, 3, 17, 10, 1); //Edge case: timeSeriesLength = 1 + testGravesBackwardBasicHelper(13, 3, 17, 1, 1); //Edge case: both miniBatchSize = 1 and timeSeriesLength = 1 + } + + private static void testGravesBackwardBasicHelper(int nIn, int nOut, int lstmNHiddenUnits, int miniBatchSize, + int timeSeriesLength) { + + INDArray inputData = Nd4j.ones(miniBatchSize, nIn, timeSeriesLength); + + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn) + .nOut(lstmNHiddenUnits) + .dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()) + .build(); + + val numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + GravesLSTM lstm = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + lstm.setBackpropGradientsViewArray(Nd4j.create(1, conf.getLayer().initializer().numParams(conf))); + //Set input, do a forward pass: + lstm.activate(inputData, false, LayerWorkspaceMgr.noWorkspaces()); + assertNotNull(lstm.input()); + + INDArray epsilon = Nd4j.ones(miniBatchSize, lstmNHiddenUnits, timeSeriesLength); + + Pair out = lstm.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); + Gradient outGradient = out.getFirst(); + INDArray nextEpsilon = out.getSecond(); + + INDArray biasGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.BIAS_KEY); + INDArray inWeightGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY); + INDArray recurrentWeightGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY); + assertNotNull(biasGradient); + assertNotNull(inWeightGradient); + assertNotNull(recurrentWeightGradient); + + assertArrayEquals(biasGradient.shape(), new long[] {1, 4 * lstmNHiddenUnits}); + assertArrayEquals(inWeightGradient.shape(), new long[] {nIn, 4 * lstmNHiddenUnits}); + assertArrayEquals(recurrentWeightGradient.shape(), new long[] {lstmNHiddenUnits, 4 * lstmNHiddenUnits + 3}); + + assertNotNull(nextEpsilon); + assertArrayEquals(nextEpsilon.shape(), new long[] {miniBatchSize, nIn, timeSeriesLength}); + + //Check update: + for (String s : outGradient.gradientForVariable().keySet()) { + lstm.update(outGradient.getGradientFor(s), s); + } + } + + @Test + public void testGravesLSTMForwardPassHelper() throws Exception { + //GravesLSTM.activateHelper() has different behaviour (due to optimizations) when forBackprop==true vs false + //But should otherwise provide identical activations + Nd4j.getRandom().setSeed(12345); + + int nIn = 10; + int layerSize = 15; + int miniBatchSize = 4; + int timeSeriesLength = 7; + + NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() + .layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(layerSize) + .dist(new UniformDistribution(0, 1)) + .activation(Activation.TANH).build()) + .build(); + + val numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + GravesLSTM lstm = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + INDArray input = Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength}); + lstm.setInput(input, LayerWorkspaceMgr.noWorkspaces()); + + Method actHelper = GravesLSTM.class.getDeclaredMethod("activateHelper", boolean.class, INDArray.class, + INDArray.class, boolean.class, LayerWorkspaceMgr.class); + actHelper.setAccessible(true); + + //Call activateHelper with both forBackprop == true, and forBackprop == false and compare + Class innerClass = DL4JClassLoading.loadClassByName("org.deeplearning4j.nn.layers.recurrent.FwdPassReturn"); + + Object oFalse = actHelper.invoke(lstm, false, null, null, false, LayerWorkspaceMgr.noWorkspacesImmutable()); //GravesLSTM.FwdPassReturn object; want fwdPassOutput INDArray + Object oTrue = actHelper.invoke(lstm, false, null, null, true, LayerWorkspaceMgr.noWorkspacesImmutable()); //want fwdPassOutputAsArrays object + + Field fwdPassOutput = innerClass.getDeclaredField("fwdPassOutput"); + fwdPassOutput.setAccessible(true); + + Field fwdPassOutputAsArrays = innerClass.getDeclaredField("fwdPassOutputAsArrays"); + fwdPassOutputAsArrays.setAccessible(true); + + INDArray fwdPassFalse = (INDArray) fwdPassOutput.get(oFalse); + INDArray[] fwdPassTrue = (INDArray[]) fwdPassOutputAsArrays.get(oTrue); + + for (int i = 0; i < timeSeriesLength; i++) { + INDArray sliceFalse = fwdPassFalse.tensorAlongDimension(i, 1, 0); + INDArray sliceTrue = fwdPassTrue[i]; + assertTrue(sliceFalse.equals(sliceTrue)); + } + } + + @Test + public void testSingleExample() { + Nd4j.getRandom().setSeed(12345); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(0.1)).seed(12345).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().activation(Activation.TANH) + .nIn(2).nOut(2).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(1) + .activation(Activation.TANH).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray in1 = Nd4j.rand(new int[] {1, 2, 4}); + INDArray in2 = Nd4j.rand(new int[] {1, 2, 5}); + in2.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4)}, in1); + + assertEquals(in1, in2.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4))); + + INDArray labels1 = Nd4j.rand(new int[] {1, 1, 4}); + INDArray labels2 = Nd4j.create(1, 1, 5); + labels2.put(new INDArrayIndex[] {NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4)}, labels1); + assertEquals(labels1, labels2.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4))); + + INDArray out1 = net.output(in1); + INDArray out2 = net.output(in2); + +// System.out.println(Arrays.toString(net.output(in1).data().asFloat())); +// System.out.println(Arrays.toString(net.output(in2).data().asFloat())); + + List activations1 = net.feedForward(in1); + List activations2 = net.feedForward(in2); + +// for (int i = 0; i < 3; i++) { +// System.out.println("-----\n" + i); +// System.out.println(Arrays.toString(activations1.get(i).dup().data().asDouble())); +// System.out.println(Arrays.toString(activations2.get(i).dup().data().asDouble())); +// +// System.out.println(activations1.get(i)); +// System.out.println(activations2.get(i)); +// } + + + + //Expect first 4 time steps to be indentical... + for (int i = 0; i < 4; i++) { + double d1 = out1.getDouble(i); + double d2 = out2.getDouble(i); + assertEquals(d1, d2, 0.0); + } + } + + + @Test + public void testGateActivationFnsSanityCheck() { + for (String gateAfn : new String[] {"sigmoid", "hardsigmoid"}) { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .seed(12345).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder() + .gateActivationFunction(gateAfn).activation(Activation.TANH).nIn(2).nOut(2) + .build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(2) + .activation(Activation.TANH).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(gateAfn, ((org.deeplearning4j.nn.conf.layers.GravesLSTM) net.getLayer(0).conf().getLayer()) + .getGateActivationFn().toString()); + + INDArray in = Nd4j.rand(new int[] {3, 2, 5}); + INDArray labels = Nd4j.rand(new int[] {3, 2, 5}); + + net.fit(in, labels); + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayerTest.java new file mode 100644 index 000000000..1508d4b62 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayerTest.java @@ -0,0 +1,121 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.recurrent; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.RNNFormat; +import org.deeplearning4j.nn.conf.layers.LSTM; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class MaskZeroLayerTest extends BaseDL4JTest { + private RNNFormat rnnDataFormat; + + public MaskZeroLayerTest(RNNFormat rnnDataFormat){ + this.rnnDataFormat = rnnDataFormat; + } + + public static Object[] params(){ + return RNNFormat.values(); + } + @Test + public void activate() { + + //GIVEN two examples where some of the timesteps are zero. + INDArray ex1 = Nd4j.create(new double[][]{ + new double[]{0, 3, 5}, + new double[]{0, 0, 2} + }); + INDArray ex2 = Nd4j.create(new double[][]{ + new double[]{0, 0, 2}, + new double[]{0, 0, 2} + }); + + // A LSTM which adds one for every non-zero timestep + org.deeplearning4j.nn.conf.layers.LSTM underlying = new org.deeplearning4j.nn.conf.layers.LSTM.Builder() + .activation(Activation.IDENTITY) + .gateActivationFunction(Activation.IDENTITY) + .nIn(2) + .nOut(1).dataFormat(rnnDataFormat) + .build(); + NeuralNetConfiguration conf = new NeuralNetConfiguration(); + conf.setLayer(underlying); + INDArray params = Nd4j.zeros(new int[]{1, 16}); + + //Set the biases to 1. + for (int i = 12; i < 16; i++) { + params.putScalar(i, 1.0); + } + Layer lstm = underlying.instantiate(conf, Collections.emptyList(), 0, params, false, params.dataType()); + double maskingValue = 0.0; + + MaskZeroLayer l = new MaskZeroLayer(lstm, maskingValue); + INDArray input = Nd4j.create(Arrays.asList(ex1, ex2), new int[]{2, 2, 3}); + if (rnnDataFormat == RNNFormat.NWC){ + input = input.permute(0, 2, 1); + } + //WHEN + INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces()); + if (rnnDataFormat == RNNFormat.NWC){ + out = out.permute(0, 2,1); + } + //THEN output should only be incremented for the non-zero timesteps + INDArray firstExampleOutput = out.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all()); + INDArray secondExampleOutput = out.get(NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all()); + + assertEquals(0.0, firstExampleOutput.getDouble(0), 1e-6); + assertEquals(1.0, firstExampleOutput.getDouble(1), 1e-6); + assertEquals(2.0, firstExampleOutput.getDouble(2), 1e-6); + + assertEquals(0.0, secondExampleOutput.getDouble(0), 1e-6); + assertEquals(0.0, secondExampleOutput.getDouble(1), 1e-6); + assertEquals(1.0, secondExampleOutput.getDouble(2), 1e-6); + + } + + @Test + public void testSerialization(){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new org.deeplearning4j.nn.conf.layers.util.MaskZeroLayer.Builder() + .setMaskValue(0.0).setUnderlying(new LSTM.Builder().nIn(4).nOut(5).dataFormat(rnnDataFormat).build()).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + TestUtils.testModelSerialization(net); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/RnnDataFormatTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/RnnDataFormatTests.java similarity index 81% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/RnnDataFormatTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/RnnDataFormatTests.java index 01c76aecc..2b5280339 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/RnnDataFormatTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/RnnDataFormatTests.java @@ -39,78 +39,62 @@ import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.conf.layers.util.MaskZeroLayer; import org.deeplearning4j.nn.gradient.Gradient; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Pair; -import org.nd4j.linalg.factory.Nd4jBackend; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; @AllArgsConstructor -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) public class RnnDataFormatTests extends BaseDL4JTest { + private final boolean helpers; + private boolean lastTimeStep; + private boolean maskZeros; - public static Stream params() { + public static List params(){ List ret = new ArrayList<>(); for (boolean helpers: new boolean[]{true, false}) for (boolean lastTimeStep: new boolean[]{true, false}) for (boolean maskZero: new boolean[]{true, false}) - for(Nd4jBackend backend : BaseNd4jTestWithBackends.BACKENDS) - ret.add(new Object[]{helpers, lastTimeStep, maskZero,backend}); - return ret.stream().map(Arguments::of); + ret.add(new Object[]{helpers, lastTimeStep, maskZero}); + return ret; } - @MethodSource("org.deeplearning4j.nn.layers.recurrent.RnnDataFormatTests#params") - @ParameterizedTest - public void testSimpleRnn(boolean helpers, - boolean lastTimeStep, - boolean maskZeros, - Nd4jBackend backend) { + @Test + public void testSimpleRnn() { try { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = "Helpers: " + helpers + ", lastTimeStep: " + lastTimeStep + ", maskZeros: " + maskZeros; - System.out.println(" --- " + msg + " ---"); + Nd4j.getRandom().setSeed(12345); + Nd4j.getEnvironment().allowHelpers(helpers); + String msg = "Helpers: " + helpers + ", lastTimeStep: " + lastTimeStep + ", maskZeros: " + maskZeros; + System.out.println(" --- " + msg + " ---"); - INDArray inNCW = Nd4j.rand(DataType.FLOAT, 2, 3, 12); + INDArray inNCW = Nd4j.rand(DataType.FLOAT, 2, 3, 12); - INDArray labelsNWC = (lastTimeStep) ?TestUtils.randomOneHot(2, 10): TestUtils.randomOneHot(2 * 12, 10).reshape(2, 12, 10); + INDArray labelsNWC = (lastTimeStep) ?TestUtils.randomOneHot(2, 10): TestUtils.randomOneHot(2 * 12, 10).reshape(2, 12, 10); - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSimpleRnnNet(RNNFormat.NCW, true, lastTimeStep, maskZeros)) - .net2(getSimpleRnnNet(RNNFormat.NCW, false, lastTimeStep, maskZeros)) - .net3(getSimpleRnnNet(RNNFormat.NWC, true, lastTimeStep, maskZeros)) - .net4(getSimpleRnnNet(RNNFormat.NWC, false, lastTimeStep, maskZeros)) - .inNCW(inNCW) - .labelsNCW((lastTimeStep)? labelsNWC: labelsNWC.permute(0, 2, 1)) - .labelsNWC(labelsNWC) - .testLayerIdx(1) - .build(); + TestCase tc = TestCase.builder() + .msg(msg) + .net1(getSimpleRnnNet(RNNFormat.NCW, true, lastTimeStep, maskZeros)) + .net2(getSimpleRnnNet(RNNFormat.NCW, false, lastTimeStep, maskZeros)) + .net3(getSimpleRnnNet(RNNFormat.NWC, true, lastTimeStep, maskZeros)) + .net4(getSimpleRnnNet(RNNFormat.NWC, false, lastTimeStep, maskZeros)) + .inNCW(inNCW) + .labelsNCW((lastTimeStep)? labelsNWC: labelsNWC.permute(0, 2, 1)) + .labelsNWC(labelsNWC) + .testLayerIdx(1) + .build(); - TestCase.testHelper(tc); + TestCase.testHelper(tc); } finally { @@ -118,11 +102,8 @@ public class RnnDataFormatTests extends BaseDL4JTest { } } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.RnnDataFormatTests#params") - public void testLSTM(boolean helpers, - boolean lastTimeStep, - boolean maskZeros,Nd4jBackend backend) { + @Test + public void testLSTM() { try { Nd4j.getRandom().setSeed(12345); @@ -155,13 +136,8 @@ public class RnnDataFormatTests extends BaseDL4JTest { } - @MethodSource("org.deeplearning4j.nn.layers.recurrent.RnnDataFormatTests#params") - @ParameterizedTest - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.LONG_TEST) - public void testGraveLSTM(boolean helpers, - boolean lastTimeStep, - boolean maskZeros,Nd4jBackend backend) { + @Test + public void testGraveLSTM() { try { Nd4j.getRandom().setSeed(12345); @@ -194,11 +170,8 @@ public class RnnDataFormatTests extends BaseDL4JTest { } - @MethodSource("org.deeplearning4j.nn.layers.recurrent.RnnDataFormatTests#params") - @ParameterizedTest - public void testGraveBiLSTM(boolean helpers, - boolean lastTimeStep, - boolean maskZeros,Nd4jBackend backend) { + @Test + public void testGraveBiLSTM() { try { Nd4j.getRandom().setSeed(12345); @@ -285,7 +258,7 @@ public class RnnDataFormatTests extends BaseDL4JTest { .layer(layer) .layer( (lastTimeStep)?new OutputLayer.Builder().activation(Activation.SOFTMAX).nOut(10).build(): - new RnnOutputLayer.Builder().activation(Activation.SOFTMAX).nOut(10).dataFormat(format).build() + new RnnOutputLayer.Builder().activation(Activation.SOFTMAX).nOut(10).dataFormat(format).build() ) .setInputType(InputType.recurrent(3, 12, format)); @@ -366,9 +339,9 @@ public class RnnDataFormatTests extends BaseDL4JTest { List diff12 = differentGrads(p1.getFirst(), p2.getFirst()); List diff13 = differentGrads(p1.getFirst(), p3.getFirst()); List diff14 = differentGrads(p1.getFirst(), p4.getFirst()); - assertEquals(0, diff12.size(),tc.msg + " " + diff12); - assertEquals(0, diff13.size(),tc.msg + " " + diff13); - assertEquals( 0, diff14.size(),tc.msg + " " + diff14); + assertEquals(0, diff12.size(), tc.msg + " " + diff12); + assertEquals(0, diff13.size(), tc.msg + " " + diff13); + assertEquals(0, diff14.size(), tc.msg + " " + diff14); assertEquals(p1.getFirst().gradientForVariable(), p2.getFirst().gradientForVariable(), tc.msg); assertEquals(p1.getFirst().gradientForVariable(), p3.getFirst().gradientForVariable(), tc.msg); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestLastTimeStepLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestLastTimeStepLayer.java similarity index 84% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestLastTimeStepLayer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestLastTimeStepLayer.java index 0b65b1eef..170ab285f 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestLastTimeStepLayer.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestLastTimeStepLayer.java @@ -33,28 +33,13 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.conf.layers.recurrent.LastTimeStep; import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.graph.ComputationGraph; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.enums.RnnDataFormat; -import org.nd4j.linalg.BaseNd4jTestWithBackends; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.Nd4jBackend; import org.nd4j.linalg.indexing.NDArrayIndex; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.nd4j.linalg.learning.config.AdaGrad; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - import static org.deeplearning4j.nn.api.OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT; import static org.deeplearning4j.nn.weights.WeightInit.XAVIER_UNIFORM; import static org.junit.jupiter.api.Assertions.*; @@ -62,23 +47,20 @@ import static org.nd4j.linalg.activations.Activation.IDENTITY; import static org.nd4j.linalg.activations.Activation.TANH; import static org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction.MSE; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -public class TestLastTimeStepLayer extends BaseDL4JTest { - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(RNNFormat rnnFormat : RNNFormat.values()) { - args.add(Arguments.of(rnnFormat,nd4jBackend)); - } - } - return args.stream(); +public class TestLastTimeStepLayer extends BaseDL4JTest { + private RNNFormat rnnDataFormat; + + public TestLastTimeStepLayer(RNNFormat rnnDataFormat){ + this.rnnDataFormat = rnnDataFormat; } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestLastTimeStepLayer#params") - public void testLastTimeStepVertex(RNNFormat rnnDataFormat,Nd4jBackend backend) { + public static Object[] params(){ + return RNNFormat.values(); + } + + @Test + public void testLastTimeStepVertex() { ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in") .addLayer("lastTS", new LastTimeStep(new SimpleRnn.Builder() @@ -140,9 +122,8 @@ public class TestLastTimeStepLayer extends BaseDL4JTest { TestUtils.testModelSerialization(graph); } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestLastTimeStepLayer#params") - public void testMaskingAndAllMasked(RNNFormat rnnDataFormat,Nd4jBackend backend) { + @Test + public void testMaskingAndAllMasked(){ ComputationGraphConfiguration.GraphBuilder builder = new NeuralNetConfiguration.Builder() .optimizationAlgo(STOCHASTIC_GRADIENT_DESCENT) .weightInit(XAVIER_UNIFORM) diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRecurrentWeightInit.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRecurrentWeightInit.java similarity index 93% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRecurrentWeightInit.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRecurrentWeightInit.java index 153a23cc3..951680ca7 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRecurrentWeightInit.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRecurrentWeightInit.java @@ -27,15 +27,11 @@ import org.deeplearning4j.nn.conf.layers.GravesLSTM; import org.deeplearning4j.nn.conf.layers.LSTM; import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestRecurrentWeightInit extends BaseDL4JTest { @Test @@ -91,10 +87,10 @@ public class TestRecurrentWeightInit extends BaseDL4JTest { double max = rw.maxNumber().doubleValue(); if(rwInit){ assertTrue(min >= 2.0, String.valueOf(min)); - assertTrue(max <= 3.0, String.valueOf(max)); + assertTrue( max <= 3.0, String.valueOf(max)); } else { assertTrue(min >= 0.0, String.valueOf(min)); - assertTrue(max <= 1.0, String.valueOf(max)); + assertTrue( max <= 1.0, String.valueOf(max)); } } } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRnnLayers.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRnnLayers.java similarity index 87% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRnnLayers.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRnnLayers.java index f8102c57d..b9f850453 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRnnLayers.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestRnnLayers.java @@ -35,52 +35,36 @@ import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.enums.RnnDataFormat; -import org.nd4j.linalg.BaseNd4jTestWithBackends; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.Nd4jBackend; import org.nd4j.linalg.learning.config.NoOp; import org.nd4j.linalg.lossfunctions.LossFunctions; import org.nd4j.common.primitives.Pair; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; -import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestRnnLayers extends BaseDL4JTest { + private RNNFormat rnnDataFormat; - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(RNNFormat rnnFormat : RNNFormat.values()) { - args.add(Arguments.of(rnnFormat,nd4jBackend)); - } - } - return args.stream(); + public TestRnnLayers(RNNFormat rnnDataFormat){ + this.rnnDataFormat = rnnDataFormat; } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestRnnLayers#params") - public void testTimeStepIs3Dimensional(RNNFormat rnnDataFormat,Nd4jBackend backend) { + public static Object[] params(){ + return RNNFormat.values(); + } + @Test + public void testTimeStepIs3Dimensional() { int nIn = 12; int nOut = 3; @@ -129,9 +113,8 @@ public class TestRnnLayers extends BaseDL4JTest { } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestRnnLayers#params") - public void testDropoutRecurrentLayers(RNNFormat rnnDataFormat,Nd4jBackend backend) { + @Test + public void testDropoutRecurrentLayers(){ Nd4j.getRandom().setSeed(12345); String[] layerTypes = new String[]{"graves", "lstm", "simple"}; @@ -228,11 +211,10 @@ public class TestRnnLayers extends BaseDL4JTest { } } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestRnnLayers#params") - public void testMismatchedInputLabelLength(RNNFormat rnnDataFormat,Nd4jBackend backend){ + @Test + public void testMismatchedInputLabelLength(){ - for( int i = 0; i < 2; i++) { + for( int i=0; i<2; i++ ){ NeuralNetConfiguration.ListBuilder lb = new NeuralNetConfiguration.Builder() diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestSimpleRnn.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestSimpleRnn.java similarity index 78% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestSimpleRnn.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestSimpleRnn.java index 77a1235ab..5fc4e8bb1 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestSimpleRnn.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestSimpleRnn.java @@ -28,50 +28,32 @@ import org.deeplearning4j.nn.conf.RNNFormat; import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.Nd4jBackend; import org.nd4j.linalg.learning.config.NoOp; import org.nd4j.linalg.ops.transforms.Transforms; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.nd4j.linalg.indexing.NDArrayIndex.all; -import static org.nd4j.linalg.indexing.NDArrayIndex.interval; import static org.nd4j.linalg.indexing.NDArrayIndex.point; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestSimpleRnn extends BaseDL4JTest { + private RNNFormat rnnDataFormat; - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(RNNFormat rnnFormat : RNNFormat.values()) { - args.add(Arguments.of(rnnFormat,nd4jBackend)); - } - } - return args.stream(); + public TestSimpleRnn(RNNFormat rnnDataFormat){ + this.rnnDataFormat = rnnDataFormat; } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestRnnLayers#params") - public void testSimpleRnn(RNNFormat rnnDataFormat, Nd4jBackend backend) { + public static Object[] params(){ + return RNNFormat.values(); + } + + @Test + public void testSimpleRnn(){ Nd4j.getRandom().setSeed(12345); int m = 3; @@ -86,7 +68,6 @@ public class TestSimpleRnn extends BaseDL4JTest { in = Nd4j.rand(DataType.FLOAT, m, tsLength, nIn); } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .updater(new NoOp()) .weightInit(WeightInit.XAVIER) @@ -139,9 +120,8 @@ public class TestSimpleRnn extends BaseDL4JTest { TestUtils.testModelSerialization(net); } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestRnnLayers#params") - public void testBiasInit(RNNFormat rnnDataFormat,Nd4jBackend backend) { + @Test + public void testBiasInit(){ Nd4j.getRandom().setSeed(12345); int nIn = 5; int layerSize = 6; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestTimeDistributed.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestTimeDistributed.java similarity index 85% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestTimeDistributed.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestTimeDistributed.java index b38e3e909..6c9a55ed2 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestTimeDistributed.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/TestTimeDistributed.java @@ -36,48 +36,30 @@ import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.conf.layers.recurrent.TimeDistributed; import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.Nd4jBackend; import org.nd4j.linalg.learning.config.Adam; import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - import static org.junit.jupiter.api.Assertions.assertEquals; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestTimeDistributed extends BaseDL4JTest { + private RNNFormat rnnDataFormat; - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(RNNFormat rnnFormat : RNNFormat.values()) { - args.add(Arguments.of(rnnFormat,nd4jBackend)); - } - } - return args.stream(); + public TestTimeDistributed(RNNFormat rnnDataFormat){ + this.rnnDataFormat = rnnDataFormat; } - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestTimeDistributed#params") - public void testTimeDistributed(RNNFormat rnnDataFormat,Nd4jBackend backend){ + public static Object[] params(){ + return RNNFormat.values(); + } + @Test + public void testTimeDistributed(){ for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.ENABLED, WorkspaceMode.NONE}) { MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder() @@ -147,12 +129,11 @@ public class TestTimeDistributed extends BaseDL4JTest { } - @MethodSource("org.deeplearning4j.nn.layers.recurrent.TestTimeDistributed#params") - @ParameterizedTest - public void testTimeDistributedDense(RNNFormat rnnDataFormat,Nd4jBackend backend) { + @Test + public void testTimeDistributedDense(){ - for( int rnnType = 0; rnnType < 3; rnnType++ ) { - for( int ffType = 0; ffType < 3; ffType++ ) { + for( int rnnType=0; rnnType<3; rnnType++ ) { + for( int ffType=0; ffType<3; ffType++ ) { Layer l0, l2; switch (rnnType) { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/SameDiffCustomLayerTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/SameDiffCustomLayerTests.java new file mode 100644 index 000000000..7b0f6c2cf --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/SameDiffCustomLayerTests.java @@ -0,0 +1,169 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.layers.samediff; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.graph.GraphVertex; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.inputs.InvalidInputTypeException; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.samediff.SDLayerParams; +import org.deeplearning4j.nn.conf.layers.samediff.SDVertexParams; +import org.deeplearning4j.nn.conf.layers.samediff.SameDiffVertex; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.nativeblas.NativeOpsHolder; + +import java.util.Map; + +@Slf4j +public class SameDiffCustomLayerTests extends BaseDL4JTest { + private DataType initialType; + + //public ExpectedException exceptionRule = ExpectedException.none(); + + @BeforeEach + public void before() { + Nd4j.create(1); + initialType = Nd4j.dataType(); + + Nd4j.setDataType(DataType.DOUBLE); + Nd4j.getRandom().setSeed(123); + } + + @AfterEach + public void after() { + Nd4j.setDataType(initialType); + + NativeOpsHolder.getInstance().getDeviceNativeOps().enableDebugMode(false); + NativeOpsHolder.getInstance().getDeviceNativeOps().enableVerboseMode(false); + } + + @Test + public void testInputValidationSameDiffLayer() { + final MultiLayerConfiguration config = new NeuralNetConfiguration.Builder().list() + .layer(new ValidatingSameDiffLayer()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.SIGMOID).nOut(2).build()) + .setInputType(InputType.feedForward(2)) + .build(); + + final MultiLayerNetwork net = new MultiLayerNetwork(config); + net.init(); + + final INDArray goodInput = Nd4j.rand(1, 2); + final INDArray badInput = Nd4j.rand(2, 2); + net.fit(goodInput, goodInput); + Assertions.assertThrows(IllegalArgumentException.class, () -> { + net.fit(badInput, badInput); + }); + } + + + @Test + public void testInputValidationSameDiffVertex(){ + final ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder().graphBuilder() + .addVertex("a", new ValidatingSameDiffVertex(), "input") + .addLayer("output", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.SIGMOID).nOut(2).build(), "a") + .addInputs("input") + .setInputTypes(InputType.feedForward(2)) + .setOutputs("output") + .build(); + + final ComputationGraph net = new ComputationGraph(config); + net.init(); + + final INDArray goodInput = Nd4j.rand(1, 2); + final INDArray badInput = Nd4j.rand(2, 2); + + net.fit(new INDArray[]{goodInput}, new INDArray[]{goodInput}); + + Assertions.assertThrows(IllegalArgumentException.class, () -> { + net.fit(new INDArray[]{badInput}, new INDArray[]{badInput}); + }); + } + + private class ValidatingSameDiffLayer extends org.deeplearning4j.nn.conf.layers.samediff.SameDiffLayer { + @Override + public void validateInput(INDArray input) { + Preconditions.checkArgument(input.size(0) < 2, "Expected Message"); + } + + @Override + public SDVariable defineLayer(SameDiff sameDiff, SDVariable layerInput, Map paramTable, SDVariable mask) { + return layerInput; + } + + @Override + public void defineParameters(SDLayerParams params) { } + + @Override + public void initializeParameters(Map params) { } + + @Override + public InputType getOutputType(int layerIndex, InputType inputType) { return inputType; } + } + + private class ValidatingSameDiffVertex extends SameDiffVertex { + @Override + public GraphVertex clone() { + return new ValidatingSameDiffVertex(); + } + + @Override + public InputType getOutputType(int layerIndex, InputType... vertexInputs) throws InvalidInputTypeException { + return vertexInputs[0]; + } + + @Override + public void validateInput(INDArray[] input) { + Preconditions.checkArgument(input[0].size(0) < 2, "Expected Message"); + } + + @Override + public SDVariable defineVertex(SameDiff sameDiff, Map layerInput, Map paramTable, Map maskVars) { + return layerInput.get("input"); + } + + @Override + public void defineParametersAndInputs(SDVertexParams params) { + params.defineInputs("input"); + } + + @Override + public void initializeParameters(Map params) {} + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffConv.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffConv.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffConv.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffConv.java index 00bc6c721..d9a331d0b 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffConv.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffConv.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.layers.samediff.testlayers.SameDiffConv; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.params.ConvolutionParamInitializer; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -51,13 +48,9 @@ import java.util.Map; import java.util.Random; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.Assume.assumeTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; @Slf4j -@NativeTag -@Tag(TagNames.SAMEDIFF) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) public class TestSameDiffConv extends BaseDL4JTest { private static final boolean PRINT_RESULTS = true; @@ -217,7 +210,7 @@ public class TestSameDiffConv extends BaseDL4JTest { INDArray out = net.output(in); INDArray outExp = net2.output(in); - assertEquals(outExp, out, msg); + assertEquals( outExp, out, msg); //Also check serialization: MultiLayerNetwork netLoaded = TestUtils.testModelSerialization(net); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDense.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDense.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDense.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDense.java index 13017e23e..5e1949f8a 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDense.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDense.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.layers.samediff.testlayers.SameDiffDense; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.params.DefaultParamInitializer; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -54,10 +51,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.SAMEDIFF) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) public class TestSameDiffDense extends BaseDL4JTest { private static final boolean PRINT_RESULTS = true; @@ -405,9 +398,9 @@ public class TestSameDiffDense extends BaseDL4JTest { netSD.fit(ds); netStandard.fit(ds); String s = String.valueOf(i); - assertEquals(netStandard.getFlattenedGradients(), netSD.getFlattenedGradients(), s); - assertEquals(netStandard.params(), netSD.params(), s); - assertEquals(netStandard.getUpdater().getStateViewArray(), netSD.getUpdater().getStateViewArray(), s); + assertEquals( netStandard.getFlattenedGradients(), netSD.getFlattenedGradients(), s); + assertEquals( netStandard.params(), netSD.params(), s); + assertEquals( netStandard.getUpdater().getStateViewArray(), netSD.getUpdater().getStateViewArray(), s); } //Sanity check on different minibatch sizes: diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDenseVertex.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDenseVertex.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDenseVertex.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDenseVertex.java index 862a5acaa..630ec1231 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDenseVertex.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffDenseVertex.java @@ -32,10 +32,7 @@ import org.deeplearning4j.nn.gradient.Gradient; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.layers.samediff.testlayers.SameDiffDenseVertex; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -49,10 +46,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @Slf4j -@NativeTag -@Tag(TagNames.SAMEDIFF) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) public class TestSameDiffDenseVertex extends BaseDL4JTest { @Test @@ -141,7 +134,7 @@ public class TestSameDiffDenseVertex extends BaseDL4JTest { INDArray i1 = m1.get(s); INDArray i2 = m2.get(s); - assertEquals(i2, i1, s); + assertEquals( i2, i1, s); } assertEquals(gStd.gradient(), gSD.gradient()); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffLambda.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffLambda.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffLambda.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffLambda.java index 1ff851b11..4afbc7e37 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffLambda.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffLambda.java @@ -34,10 +34,7 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.layers.samediff.testlayers.SameDiffSimpleLambdaLayer; import org.deeplearning4j.nn.layers.samediff.testlayers.SameDiffSimpleLambdaVertex; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -50,10 +47,6 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; import static org.junit.jupiter.api.Assertions.assertEquals; @Slf4j -@NativeTag -@Tag(TagNames.SAMEDIFF) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) public class TestSameDiffLambda extends BaseDL4JTest { private static final boolean PRINT_RESULTS = true; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffOutput.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffOutput.java similarity index 94% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffOutput.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffOutput.java index 7aa13f11c..2f0479b67 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffOutput.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/TestSameDiffOutput.java @@ -31,10 +31,7 @@ import org.deeplearning4j.nn.layers.samediff.testlayers.SameDiffMSELossLayer; import org.deeplearning4j.nn.layers.samediff.testlayers.SameDiffMSEOutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -45,10 +42,6 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; import static org.junit.jupiter.api.Assertions.assertEquals; @Slf4j -@NativeTag -@Tag(TagNames.SAMEDIFF) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) public class TestSameDiffOutput extends BaseDL4JTest { @Test @@ -173,8 +166,8 @@ public class TestSameDiffOutput extends BaseDL4JTest { netSD.fit(ds); netStd.fit(ds); String s = String.valueOf(i); - assertEquals(netStd.params(), netSD.params(), s); - assertEquals(netStd.getFlattenedGradients(), netSD.getFlattenedGradients(), s); + assertEquals( netStd.params(), netSD.params(), s); + assertEquals( netStd.getFlattenedGradients(), netSD.getFlattenedGradients(),s ); } //Test fit before output: diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/MinimalSameDiffDense.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/MinimalSameDiffDense.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/MinimalSameDiffDense.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/MinimalSameDiffDense.java index 72368a725..8864448b0 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/MinimalSameDiffDense.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/MinimalSameDiffDense.java @@ -21,6 +21,7 @@ package org.deeplearning4j.nn.layers.samediff.testlayers; import lombok.Data; +import lombok.EqualsAndHashCode; import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.samediff.SameDiffLayer; import org.deeplearning4j.nn.conf.layers.samediff.SDLayerParams; @@ -33,6 +34,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import java.util.Map; +@EqualsAndHashCode(callSuper = true) @Data public class MinimalSameDiffDense extends SameDiffLayer { diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java index 602f60a58..0049696de 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffConv.java @@ -38,7 +38,7 @@ import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.impl.layers.convolution.config.Conv2DConfig; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.*; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java index bf3745856..e49e6aca6 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDense.java @@ -34,7 +34,7 @@ import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.*; diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffDenseVertex.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSELossLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSELossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSELossLayer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSELossLayer.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSEOutputLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSEOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSEOutputLayer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffMSEOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffSimpleLambdaLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffSimpleLambdaLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffSimpleLambdaLayer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffSimpleLambdaLayer.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffSimpleLambdaVertex.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffSimpleLambdaVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffSimpleLambdaVertex.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/samediff/testlayers/SameDiffSimpleLambdaVertex.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestReconstructionDistributions.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestReconstructionDistributions.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestReconstructionDistributions.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestReconstructionDistributions.java index 5b1c12633..934ba63a8 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestReconstructionDistributions.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestReconstructionDistributions.java @@ -30,10 +30,7 @@ import org.deeplearning4j.nn.conf.layers.variational.BernoulliReconstructionDist import org.deeplearning4j.nn.conf.layers.variational.ExponentialReconstructionDistribution; import org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution; import org.deeplearning4j.nn.conf.layers.variational.ReconstructionDistribution; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -47,10 +44,6 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.RNG) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) public class TestReconstructionDistributions extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestVAE.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestVAE.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestVAE.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestVAE.java index ff82e1ef9..e61614a1b 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestVAE.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/layers/variational/TestVAE.java @@ -32,10 +32,7 @@ import org.deeplearning4j.nn.conf.weightnoise.WeightNoise; import org.deeplearning4j.nn.gradient.Gradient; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.activations.impl.ActivationTanH; import org.nd4j.linalg.api.ndarray.INDArray; @@ -54,10 +51,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.RNG) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) public class TestVAE extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java similarity index 92% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java index aba191181..175292211 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/CloseNetworkTests.java @@ -28,10 +28,7 @@ import org.deeplearning4j.nn.conf.inputs.InputType; import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -39,10 +36,7 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.learning.config.Adam; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.WORKSPACES) + public class CloseNetworkTests extends BaseDL4JTest { public static MultiLayerNetwork getTestNet() { @@ -98,14 +92,14 @@ public class CloseNetworkTests extends BaseDL4JTest { net.output(f); } catch (IllegalStateException e) { String msg = e.getMessage(); - assertTrue(msg.contains("released"),msg); + assertTrue(msg.contains("released"), msg); } try { net.fit(f, l); } catch (IllegalStateException e) { String msg = e.getMessage(); - assertTrue(msg.contains("released"),msg); + assertTrue(msg.contains("released"), msg); } } } @@ -146,14 +140,14 @@ public class CloseNetworkTests extends BaseDL4JTest { net.output(f); } catch (IllegalStateException e) { String msg = e.getMessage(); - assertTrue( msg.contains("released"),msg); + assertTrue(msg.contains("released"), msg); } try { net.fit(new INDArray[]{f}, new INDArray[]{l}); } catch (IllegalStateException e) { String msg = e.getMessage(); - assertTrue(msg.contains("released"),msg); + assertTrue(msg.contains("released"), msg); } } } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/LargeNetTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/LargeNetTest.java new file mode 100644 index 000000000..44d1a2098 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/LargeNetTest.java @@ -0,0 +1,110 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.misc; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.EmbeddingLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +//@Ignore //Ignored due to very large memory requirements +public class LargeNetTest extends BaseDL4JTest { + + //@Ignore + @Test + public void testLargeMultiLayerNetwork(){ + Nd4j.setDataType(DataType.FLOAT); + + //More than 2.1 billion parameters + //10M classes plus 300 vector size -> 3 billion elements + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new EmbeddingLayer.Builder().nIn(10_000_000).nOut(300).build()) + .layer(new OutputLayer.Builder().nIn(300).nOut(10).activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray params = net.params(); + long paramsLength = params.length(); + long expParamsLength = 10_000_000L * 300 + 300 * 10 + 10; + assertEquals(expParamsLength, paramsLength); + + long[] expW = new long[]{10_000_000, 300}; + assertArrayEquals(expW, net.getParam("0_W").shape()); + + long[] expW1 = new long[]{300, 10}; + assertArrayEquals(expW1, net.getParam("1_W").shape()); + + long[] expB1 = new long[]{1, 10}; + assertArrayEquals(expB1, net.getParam("1_b").shape()); + } + + //@Ignore + @Test + public void testLargeCompGraph(){ + Nd4j.setDataType(DataType.FLOAT); + + //More than 2.1 billion parameters + //10M classes plus 300 vector size -> 3 billion elements + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .graphBuilder() + .addInputs("in") + .layer("0", new EmbeddingLayer.Builder().nIn(10_000_000).nOut(300).build(), "in") + .layer("1", new OutputLayer.Builder().nIn(300).nOut(10).activation(Activation.SOFTMAX).build(), "0") + .setOutputs("1") + .build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + + INDArray params = net.params(); + long paramsLength = params.length(); + long expParamsLength = 10_000_000L * 300 + 300 * 10 + 10; + assertEquals(expParamsLength, paramsLength); + + long[] expW = new long[]{10_000_000, 300}; + assertArrayEquals(expW, net.getParam("0_W").shape()); + + long[] expW1 = new long[]{300, 10}; + assertArrayEquals(expW1, net.getParam("1_W").shape()); + + long[] expB1 = new long[]{1, 10}; + assertArrayEquals(expB1, net.getParam("1_b").shape()); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestLrChanges.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestLrChanges.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestLrChanges.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestLrChanges.java index 0a01d5efa..77f3a2342 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestLrChanges.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestLrChanges.java @@ -29,10 +29,7 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.conf.weightnoise.DropConnect; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -46,9 +43,7 @@ import org.nd4j.linalg.schedule.ExponentialSchedule; import org.nd4j.linalg.schedule.ScheduleType; import static org.junit.jupiter.api.Assertions.assertEquals; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.WORKSPACES) + public class TestLrChanges extends BaseDL4JTest { @Test @@ -66,7 +61,7 @@ public class TestLrChanges extends BaseDL4JTest { MultiLayerNetwork net = new MultiLayerNetwork(conf); net.init(); - for( int i = 0; i < 10; i++) { + for( int i=0; i<10; i++ ){ net.fit(Nd4j.rand(10,10), Nd4j.rand(10,10)); } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestMemoryReports.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestMemoryReports.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestMemoryReports.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestMemoryReports.java index 4d4f4c0bb..a7fcee172 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestMemoryReports.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestMemoryReports.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.conf.memory.MemoryType; import org.deeplearning4j.nn.conf.memory.MemoryUseMode; import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToCnnPreProcessor; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -53,9 +50,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.WORKSPACES) + public class TestMemoryReports extends BaseDL4JTest { public static List> getTestLayers() { @@ -266,7 +261,7 @@ public class TestMemoryReports extends BaseDL4JTest { @Test public void testPreprocessors() throws Exception { - //https://github.com/eclipse/deeplearning4j/issues/4223 + //https://github.com/deeplearning4j/deeplearning4j/issues/4223 File f = new ClassPathResource("4223/CompGraphConfig.json").getTempFileFromArchive(); String s = FileUtils.readFileToString(f, Charset.defaultCharset()); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestNetConversion.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestNetConversion.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestNetConversion.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestNetConversion.java index 04663963f..cd1ca1a28 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/TestNetConversion.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/TestNetConversion.java @@ -29,10 +29,7 @@ import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -41,8 +38,6 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; import static org.junit.jupiter.api.Assertions.assertEquals; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class TestNetConversion extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/WorkspaceTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/WorkspaceTests.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/WorkspaceTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/WorkspaceTests.java index 7fa8095d4..ad57a4688 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/WorkspaceTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/WorkspaceTests.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; @@ -59,10 +56,6 @@ import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.WORKSPACES) public class WorkspaceTests extends BaseDL4JTest { @BeforeEach @@ -95,7 +88,7 @@ public class WorkspaceTests extends BaseDL4JTest { @Test public void testWorkspaceIndependence() { - //https://github.com/eclipse/deeplearning4j/issues/4337 + //https://github.com/deeplearning4j/deeplearning4j/issues/4337 int depthIn = 2; int depthOut = 2; int nOut = 2; @@ -150,7 +143,7 @@ public class WorkspaceTests extends BaseDL4JTest { @Test public void testWithPreprocessorsCG() { - //https://github.com/eclipse/deeplearning4j/issues/4347 + //https://github.com/deeplearning4j/deeplearning4j/issues/4347 //Cause for the above issue was layerVertex.setInput() applying the preprocessor, with the result // not being detached properly from the workspace... diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/iter/WSTestDataSetIterator.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/iter/WSTestDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/iter/WSTestDataSetIterator.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/misc/iter/WSTestDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/mkldnn/ValidateMKLDNN.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/mkldnn/ValidateMKLDNN.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/mkldnn/ValidateMKLDNN.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/mkldnn/ValidateMKLDNN.java index 9cd15eba6..0f6337502 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/mkldnn/ValidateMKLDNN.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/mkldnn/ValidateMKLDNN.java @@ -34,11 +34,9 @@ import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.gradient.Gradient; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -52,10 +50,8 @@ import java.lang.reflect.Field; import java.util.Arrays; import java.util.Collections; -import static junit.framework.TestCase.*; -import static org.junit.Assume.assumeTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) +import static org.junit.jupiter.api.Assumptions.assumeTrue; + public class ValidateMKLDNN extends BaseDL4JTest { @@ -199,7 +195,8 @@ public class ValidateMKLDNN extends BaseDL4JTest { } } - @Test @Disabled //https://github.com/eclipse/deeplearning4j/issues/7272 + @Test + ////@Ignore //https://github.com/deeplearning4j/deeplearning4j/issues/7272 public void validateLRN() { //Only run test if using nd4j-native backend @@ -304,7 +301,7 @@ public class ValidateMKLDNN extends BaseDL4JTest { MultiLayerNetwork net = new MultiLayerNetwork(conf); net.init(); org.deeplearning4j.nn.layers.normalization.BatchNormalization bn = (org.deeplearning4j.nn.layers.normalization.BatchNormalization) net.getLayer(0); - assertNotNull(bn.getHelper()); + Assertions.assertNotNull(bn.getHelper()); System.out.println(bn.getHelper()); net.output(in, true); @@ -314,7 +311,7 @@ public class ValidateMKLDNN extends BaseDL4JTest { Field f = bn.getClass().getDeclaredField("helper"); f.setAccessible(true); f.set(bn, null); - assertNull(bn.getHelper()); + Assertions.assertNull(bn.getHelper()); net.output(in, true); bn.setInput(in, LayerWorkspaceMgr.noWorkspaces()); @@ -323,6 +320,6 @@ public class ValidateMKLDNN extends BaseDL4JTest { INDArray dldin_dl4j = p.getSecond(); INDArray dldin_helper = pcudnn.getSecond(); - assertTrue(dldin_dl4j.equalsWithEps(dldin_helper, 1e-5)); + Assertions.assertTrue(dldin_dl4j.equalsWithEps(dldin_helper, 1e-5)); } } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/BackPropMLPTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/BackPropMLPTest.java new file mode 100644 index 000000000..94f26b712 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/BackPropMLPTest.java @@ -0,0 +1,423 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.multilayer; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.params.DefaultParamInitializer; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.iter.NdIndexIterator; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.transforms.strict.SigmoidDerivative; +import org.nd4j.linalg.api.ops.impl.transforms.strict.TanhDerivative; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.exception.ND4JArraySizeException; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; +import org.nd4j.linalg.ops.transforms.Transforms; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class BackPropMLPTest extends BaseDL4JTest { + + @Test + public void testMLPTrivial() { + //Simplest possible case: 1 hidden layer, 1 hidden neuron, batch size of 1. + MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(new int[] {1}, Activation.SIGMOID)); + network.setListeners(new ScoreIterationListener(1)); + network.init(); + + DataSetIterator iter = new IrisDataSetIterator(1, 10); + + while (iter.hasNext()) + network.fit(iter.next()); + } + + @Test + public void testMLP() { + //Simple mini-batch test with multiple hidden layers + MultiLayerConfiguration conf = getIrisMLPSimpleConfig(new int[] {5, 4, 3}, Activation.SIGMOID); +// System.out.println(conf); + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + DataSetIterator iter = new IrisDataSetIterator(10, 100); + + while (iter.hasNext()) { + network.fit(iter.next()); + } + } + + @Test + public void testMLP2() { + //Simple mini-batch test with multiple hidden layers + MultiLayerConfiguration conf = getIrisMLPSimpleConfig(new int[] {5, 15, 3}, Activation.TANH); +// System.out.println(conf); + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + DataSetIterator iter = new IrisDataSetIterator(12, 120); + + while (iter.hasNext()) { + network.fit(iter.next()); + } + } + + @Test + public void testSingleExampleWeightUpdates() { + //Simplest possible case: 1 hidden layer, 1 hidden neuron, batch size of 1. + //Manually calculate weight updates (entirely outside of DL4J and ND4J) + // and compare expected and actual weights after backprop + + DataSetIterator iris = new IrisDataSetIterator(1, 10); + + MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(new int[] {1}, Activation.SIGMOID)); + network.init(); + + Layer[] layers = network.getLayers(); + + final boolean printCalculations = false; + + while (iris.hasNext()) { + DataSet data = iris.next(); + INDArray x = data.getFeatures(); + INDArray y = data.getLabels(); + float[] xFloat = asFloat(x); + float[] yFloat = asFloat(y); + + //Do forward pass: + INDArray l1Weights = layers[0].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); //Hidden layer + INDArray l2Weights = layers[1].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); //Output layer + INDArray l1Bias = layers[0].getParam(DefaultParamInitializer.BIAS_KEY).dup(); + INDArray l2Bias = layers[1].getParam(DefaultParamInitializer.BIAS_KEY).dup(); + float[] l1WeightsFloat = asFloat(l1Weights); + float[] l2WeightsFloat = asFloat(l2Weights); + float l1BiasFloat = l1Bias.getFloat(0); + float[] l2BiasFloatArray = asFloat(l2Bias); + + float hiddenUnitPreSigmoid = dotProduct(l1WeightsFloat, xFloat) + l1BiasFloat; //z=w*x+b + float hiddenUnitPostSigmoid = sigmoid(hiddenUnitPreSigmoid); //a=sigma(z) + + float[] outputPreSoftmax = new float[3]; + //Normally a matrix multiplication here, but only one hidden unit in this trivial example + for (int i = 0; i < 3; i++) { + outputPreSoftmax[i] = hiddenUnitPostSigmoid * l2WeightsFloat[i] + l2BiasFloatArray[i]; + } + float[] outputPostSoftmax = softmax(outputPreSoftmax); + + //Do backward pass: + float[] deltaOut = vectorDifference(outputPostSoftmax, yFloat); //out-labels + //deltaHidden = sigmaPrime(hiddenUnitZ) * sum_k (w_jk * \delta_k); here, only one j + float deltaHidden = 0.0f; + for (int i = 0; i < 3; i++) + deltaHidden += l2WeightsFloat[i] * deltaOut[i]; + deltaHidden *= derivOfSigmoid(hiddenUnitPreSigmoid); + + //Calculate weight/bias updates: + //dL/dW = delta * (activation of prev. layer) + //dL/db = delta + float[] dLdwOut = new float[3]; + for (int i = 0; i < dLdwOut.length; i++) + dLdwOut[i] = deltaOut[i] * hiddenUnitPostSigmoid; + float[] dLdwHidden = new float[4]; + for (int i = 0; i < dLdwHidden.length; i++) + dLdwHidden[i] = deltaHidden * xFloat[i]; + float[] dLdbOut = deltaOut; + float dLdbHidden = deltaHidden; + + if (printCalculations) { + System.out.println("deltaOut = " + Arrays.toString(deltaOut)); + System.out.println("deltaHidden = " + deltaHidden); + System.out.println("dLdwOut = " + Arrays.toString(dLdwOut)); + System.out.println("dLdbOut = " + Arrays.toString(dLdbOut)); + System.out.println("dLdwHidden = " + Arrays.toString(dLdwHidden)); + System.out.println("dLdbHidden = " + dLdbHidden); + } + + + //Calculate new parameters: + //w_i = w_i - (learningRate)/(batchSize) * sum_j (dL_j/dw_i) + //b_i = b_i - (learningRate)/(batchSize) * sum_j (dL_j/db_i) + //Which for batch size of one (here) is simply: + //w_i = w_i - learningRate * dL/dW + //b_i = b_i - learningRate * dL/db + float[] expectedL1WeightsAfter = new float[4]; + float[] expectedL2WeightsAfter = new float[3]; + float expectedL1BiasAfter = l1BiasFloat - 0.1f * dLdbHidden; + float[] expectedL2BiasAfter = new float[3]; + + for (int i = 0; i < 4; i++) + expectedL1WeightsAfter[i] = l1WeightsFloat[i] - 0.1f * dLdwHidden[i]; + for (int i = 0; i < 3; i++) + expectedL2WeightsAfter[i] = l2WeightsFloat[i] - 0.1f * dLdwOut[i]; + for (int i = 0; i < 3; i++) + expectedL2BiasAfter[i] = l2BiasFloatArray[i] - 0.1f * dLdbOut[i]; + + + //Finally, do back-prop on network, and compare parameters vs. expected parameters + network.fit(data); + + /* INDArray l1WeightsAfter = layers[0].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); //Hidden layer + INDArray l2WeightsAfter = layers[1].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); //Output layer + INDArray l1BiasAfter = layers[0].getParam(DefaultParamInitializer.BIAS_KEY).dup(); + INDArray l2BiasAfter = layers[1].getParam(DefaultParamInitializer.BIAS_KEY).dup(); + float[] l1WeightsFloatAfter = asFloat(l1WeightsAfter); + float[] l2WeightsFloatAfter = asFloat(l2WeightsAfter); + float l1BiasFloatAfter = l1BiasAfter.getFloat(0); + float[] l2BiasFloatAfter = asFloat(l2BiasAfter); + + if( printCalculations) { + System.out.println("Expected L1 weights = " + Arrays.toString(expectedL1WeightsAfter)); + System.out.println("Actual L1 weights = " + Arrays.toString(asFloat(l1WeightsAfter))); + System.out.println("Expected L2 weights = " + Arrays.toString(expectedL2WeightsAfter)); + System.out.println("Actual L2 weights = " + Arrays.toString(asFloat(l2WeightsAfter))); + System.out.println("Expected L1 bias = " + expectedL1BiasAfter); + System.out.println("Actual L1 bias = " + Arrays.toString(asFloat(l1BiasAfter))); + System.out.println("Expected L2 bias = " + Arrays.toString(expectedL2BiasAfter)); + System.out.println("Actual L2 bias = " + Arrays.toString(asFloat(l2BiasAfter))); + } + + + float eps = 1e-4f; + assertArrayEquals(l1WeightsFloatAfter,expectedL1WeightsAfter,eps); + assertArrayEquals(l2WeightsFloatAfter,expectedL2WeightsAfter,eps); + assertEquals(l1BiasFloatAfter,expectedL1BiasAfter,eps); + assertArrayEquals(l2BiasFloatAfter,expectedL2BiasAfter,eps); + */ +// System.out.println("\n\n--------------"); + } + } + + + @Test + public void testMLPGradientCalculation() { + testIrisMiniBatchGradients(1, new int[] {1}, Activation.SIGMOID); + testIrisMiniBatchGradients(1, new int[] {5}, Activation.SIGMOID); + testIrisMiniBatchGradients(12, new int[] {15, 25, 10}, Activation.SIGMOID); + testIrisMiniBatchGradients(50, new int[] {10, 50, 200, 50, 10}, Activation.TANH); + testIrisMiniBatchGradients(150, new int[] {30, 50, 20}, Activation.TANH); + } + + private static void testIrisMiniBatchGradients(int miniBatchSize, int[] hiddenLayerSizes, + Activation activationFunction) { + int totalExamples = 10 * miniBatchSize; + if (totalExamples > 150) { + totalExamples = miniBatchSize * (150 / miniBatchSize); + } + if (miniBatchSize > 150) { + fail(); + } + DataSetIterator iris = new IrisDataSetIterator(miniBatchSize, totalExamples); + + MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(hiddenLayerSizes, Activation.SIGMOID)); + network.init(); + + Layer[] layers = network.getLayers(); + int nLayers = layers.length; + + while (iris.hasNext()) { + DataSet data = iris.next(); + INDArray x = data.getFeatures(); + INDArray y = data.getLabels(); + + //Do forward pass: + INDArray[] layerWeights = new INDArray[nLayers]; + INDArray[] layerBiases = new INDArray[nLayers]; + for (int i = 0; i < nLayers; i++) { + layerWeights[i] = layers[i].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); + layerBiases[i] = layers[i].getParam(DefaultParamInitializer.BIAS_KEY).dup(); + } + + INDArray[] layerZs = new INDArray[nLayers]; + INDArray[] layerActivations = new INDArray[nLayers]; + for (int i = 0; i < nLayers; i++) { + INDArray layerInput = (i == 0 ? x : layerActivations[i - 1]); + layerZs[i] = layerInput.castTo(layerWeights[i].dataType()).mmul(layerWeights[i]).addiRowVector(layerBiases[i]); + layerActivations[i] = (i == nLayers - 1 ? doSoftmax(layerZs[i].dup()) : doSigmoid(layerZs[i].dup())); + } + + //Do backward pass: + INDArray[] deltas = new INDArray[nLayers]; + deltas[nLayers - 1] = layerActivations[nLayers - 1].sub(y.castTo(layerActivations[nLayers-1].dataType())); //Out - labels; shape=[miniBatchSize,nOut]; + assertArrayEquals(deltas[nLayers - 1].shape(), new long[] {miniBatchSize, 3}); + for (int i = nLayers - 2; i >= 0; i--) { + INDArray sigmaPrimeOfZ; + sigmaPrimeOfZ = doSigmoidDerivative(layerZs[i]); + INDArray epsilon = layerWeights[i + 1].mmul(deltas[i + 1].transpose()).transpose(); + deltas[i] = epsilon.mul(sigmaPrimeOfZ); + assertArrayEquals(deltas[i].shape(), new long[] {miniBatchSize, hiddenLayerSizes[i]}); + } + + INDArray[] dLdw = new INDArray[nLayers]; + INDArray[] dLdb = new INDArray[nLayers]; + for (int i = 0; i < nLayers; i++) { + INDArray prevActivations = (i == 0 ? x : layerActivations[i - 1]); + //Raw gradients, so not yet divided by mini-batch size (division is done in BaseUpdater) + dLdw[i] = deltas[i].transpose().castTo(prevActivations.dataType()).mmul(prevActivations).transpose(); //Shape: [nIn, nOut] + dLdb[i] = deltas[i].sum(true, 0); //Shape: [1,nOut] + + int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]); + int nOut = (i < nLayers - 1 ? hiddenLayerSizes[i] : 3); + assertArrayEquals(dLdw[i].shape(), new long[] {nIn, nOut}); + assertArrayEquals(dLdb[i].shape(), new long[] {1, nOut}); + } + + + //Calculate and get gradient, compare to expected + network.setInput(x); + network.setLabels(y); + network.computeGradientAndScore(); + Gradient gradient = network.gradientAndScore().getFirst(); + + float eps = 1e-4f; + for (int i = 0; i < hiddenLayerSizes.length; i++) { + String wKey = i + "_" + DefaultParamInitializer.WEIGHT_KEY; + String bKey = i + "_" + DefaultParamInitializer.BIAS_KEY; + INDArray wGrad = gradient.getGradientFor(wKey); + INDArray bGrad = gradient.getGradientFor(bKey); + float[] wGradf = asFloat(wGrad); + float[] bGradf = asFloat(bGrad); + float[] expWGradf = asFloat(dLdw[i]); + float[] expBGradf = asFloat(dLdb[i]); + assertArrayEquals(wGradf, expWGradf, eps); + assertArrayEquals(bGradf, expBGradf, eps); + } + } + } + + + /** Very simple back-prop config set up for Iris. + * Learning Rate = 0.1 + * No regularization, no Adagrad, no momentum etc. One iteration. + */ + private static MultiLayerConfiguration getIrisMLPSimpleConfig(int[] hiddenLayerSizes, + Activation activationFunction) { + NeuralNetConfiguration.ListBuilder lb = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .seed(12345L).list(); + + for (int i = 0; i < hiddenLayerSizes.length; i++) { + int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]); + lb.layer(i, new DenseLayer.Builder().nIn(nIn).nOut(hiddenLayerSizes[i]).weightInit(WeightInit.XAVIER) + .activation(activationFunction).build()); + } + + lb.layer(hiddenLayerSizes.length, + new OutputLayer.Builder(LossFunction.MCXENT).nIn(hiddenLayerSizes[hiddenLayerSizes.length - 1]) + .nOut(3).weightInit(WeightInit.XAVIER) + .activation(activationFunction.equals(Activation.IDENTITY) ? Activation.IDENTITY + : Activation.SOFTMAX) + .build()); + + return lb.build(); + } + + public static float[] asFloat(INDArray arr) { + long len = arr.length(); + if (len > Integer.MAX_VALUE) + throw new ND4JArraySizeException(); + float[] f = new float[(int) len]; + NdIndexIterator iterator = new NdIndexIterator('c', arr.shape()); + for (int i = 0; i < len; i++) { + f[i] = arr.getFloat(iterator.next()); + } + return f; + } + + public static float dotProduct(float[] x, float[] y) { + float sum = 0.0f; + for (int i = 0; i < x.length; i++) + sum += x[i] * y[i]; + return sum; + } + + public static float sigmoid(float in) { + return (float) (1.0 / (1.0 + Math.exp(-in))); + } + + public static float[] sigmoid(float[] in) { + float[] out = new float[in.length]; + for (int i = 0; i < in.length; i++) { + out[i] = sigmoid(in[i]); + } + return out; + } + + public static float derivOfSigmoid(float in) { + // float v = (float)( Math.exp(in) / Math.pow(1+Math.exp(in),2.0) ); + float v = in * (1 - in); + return v; + } + + public static float[] derivOfSigmoid(float[] in) { + float[] out = new float[in.length]; + for (int i = 0; i < in.length; i++) { + out[i] = derivOfSigmoid(in[i]); + } + return out; + } + + public static float[] softmax(float[] in) { + float[] out = new float[in.length]; + float sumExp = 0.0f; + for (int i = 0; i < in.length; i++) { + sumExp += Math.exp(in[i]); + } + for (int i = 0; i < in.length; i++) { + out[i] = (float) Math.exp(in[i]) / sumExp; + } + return out; + } + + public static float[] vectorDifference(float[] x, float[] y) { + float[] out = new float[x.length]; + for (int i = 0; i < x.length; i++) { + out[i] = x[i] - y[i]; + } + return out; + } + + public static INDArray doSoftmax(INDArray input) { + return Transforms.softmax(input, true); + } + + public static INDArray doSigmoid(INDArray input) { + return Transforms.sigmoid(input, true); + } + + public static INDArray doSigmoidDerivative(INDArray input) { + return Nd4j.getExecutioner().exec(new SigmoidDerivative(input.dup())); + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java new file mode 100644 index 000000000..e10f3180b --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java @@ -0,0 +1,1532 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.multilayer; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.exception.DL4JException; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.*; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.objdetect.Yolo2OutputLayer; +import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.RnnToCnnPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.gradient.DefaultGradient; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.layers.BaseOutputLayer; +import org.deeplearning4j.nn.params.DefaultParamInitializer; +import org.deeplearning4j.nn.transferlearning.TransferLearning; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.deeplearning4j.optimize.api.BaseTrainingListener; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.util.ModelSerializer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.executioner.OpExecutioner; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.SplitTestAndTrain; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.heartbeat.Heartbeat; +import org.nd4j.linalg.heartbeat.reports.Environment; +import org.nd4j.linalg.heartbeat.reports.Event; +import org.nd4j.linalg.heartbeat.reports.Task; +import org.nd4j.linalg.heartbeat.utils.EnvironmentUtils; +import org.nd4j.linalg.heartbeat.utils.TaskUtils; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.primitives.Pair; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +public class MultiLayerTest extends BaseDL4JTest { + + private static OpExecutioner.ProfilingMode origMode; + + @BeforeAll + public static void beforeClass(){ + origMode = Nd4j.getExecutioner().getProfilingMode(); + } + + @BeforeEach + public void before(){ + Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC); + } + + @AfterAll + public static void afterClass(){ + Nd4j.getExecutioner().setProfilingMode(origMode); + } + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @Test + public void testSetParams() { + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder() + .list().layer(0, + new DenseLayer.Builder().nIn(4).nOut(3) + .activation(Activation.TANH).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()) + .build(); + + MultiLayerNetwork network3 = new MultiLayerNetwork(conf); + network3.init(); + + INDArray params = network3.params(); + INDArray weights = network3.getLayer(0).getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); + INDArray bias = network3.getLayer(0).getParam(DefaultParamInitializer.BIAS_KEY).dup(); + network3.setParameters(params); + assertEquals(weights, network3.getLayer(0).getParam(DefaultParamInitializer.WEIGHT_KEY)); + assertEquals(bias, network3.getLayer(0).getParam(DefaultParamInitializer.BIAS_KEY)); + INDArray params4 = network3.params(); + assertEquals(params, params4); + } + + @Test + public void testBatchNorm() { + Nd4j.getRandom().setSeed(123); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(2, new BatchNormalization.Builder().nOut(2).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).nIn(2).nOut(3).build()) + .build(); + + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + network.setListeners(new ScoreIterationListener(1)); + + DataSetIterator iter = new IrisDataSetIterator(150, 150); + + DataSet next = iter.next(); + next.normalizeZeroMeanZeroUnitVariance(); + SplitTestAndTrain trainTest = next.splitTestAndTrain(110); + network.setLabels(trainTest.getTrain().getLabels()); + network.init(); + for( int i=0; i<5; i++ ) { + network.fit(trainTest.getTrain()); + } + + } + + @Test + public void testBackProp() { + Nd4j.getRandom().setSeed(123); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).nIn(2).nOut(3).build()) + .build(); + + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + network.setListeners(new ScoreIterationListener(1)); + + DataSetIterator iter = new IrisDataSetIterator(150, 150); + + DataSet next = iter.next(); + next.normalizeZeroMeanZeroUnitVariance(); + SplitTestAndTrain trainTest = next.splitTestAndTrain(110); + network.setInput(trainTest.getTrain().getFeatures()); + network.setLabels(trainTest.getTrain().getLabels()); + network.init(); + for( int i=0; i<5; i++ ) { + network.fit(trainTest.getTrain()); + } + + DataSet test = trainTest.getTest(); + Evaluation eval = new Evaluation(); + INDArray output = network.output(test.getFeatures()); + eval.eval(test.getLabels(), output); + log.info("Score " + eval.stats()); + } + + + + @Test + public void testGradientWithAsList() { + MultiLayerNetwork net1 = new MultiLayerNetwork(getConf()); + MultiLayerNetwork net2 = new MultiLayerNetwork(getConf()); + net1.init(); + net2.init(); + + DataSet x1 = new IrisDataSetIterator(1, 150).next(); + DataSet all = new IrisDataSetIterator(150, 150).next(); + DataSet x2 = all.asList().get(0); + + //x1 and x2 contain identical data + assertArrayEquals(asFloat(x1.getFeatures()), asFloat(x2.getFeatures()), 0.0f); + assertArrayEquals(asFloat(x1.getLabels()), asFloat(x2.getLabels()), 0.0f); + assertEquals(x1, x2); + + //Set inputs/outputs so gradient can be calculated: + net1.feedForward(x1.getFeatures()); + net2.feedForward(x2.getFeatures()); + ((BaseOutputLayer) net1.getLayer(1)).setLabels(x1.getLabels()); + ((BaseOutputLayer) net2.getLayer(1)).setLabels(x2.getLabels()); + + net1.gradient(); + net2.gradient(); + } + + /** + * This test intended only to test activateSelectedLayers method, it does not involves fully-working AutoEncoder. + */ + @Test + public void testSelectedActivations() { + // Train DeepAutoEncoder on very limited trainset + final int numRows = 28; + final int numColumns = 28; + int seed = 123; + int numSamples = 3; + int iterations = 1; + int listenerFreq = iterations / 5; + + log.info("Load data...."); + + float[][] trainingData = new float[numSamples][numColumns * numRows]; + Arrays.fill(trainingData[0], 0.95f); + Arrays.fill(trainingData[1], 0.5f); + Arrays.fill(trainingData[2], 0.05f); + + + + log.info("Build model...."); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed) + .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list() + .layer(0, new DenseLayer.Builder().nIn(numRows * numColumns).nOut(1000).build()) + .layer(1, new DenseLayer.Builder().nIn(1000).nOut(500).build()) + .layer(2, new DenseLayer.Builder().nIn(500).nOut(250).build()) + .layer(3, new DenseLayer.Builder().nIn(250).nOut(100).build()) + .layer(4, new DenseLayer.Builder().nIn(100).nOut(30).build()) //encoding stops + .layer(5, new DenseLayer.Builder().nIn(30).nOut(100).build()) //decoding starts + .layer(6, new DenseLayer.Builder().nIn(100).nOut(250).build()) + .layer(7, new DenseLayer.Builder().nIn(250).nOut(500).build()) + .layer(8, new DenseLayer.Builder().nIn(500).nOut(1000).build()) + .layer(9, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(1000) + .nOut(numRows * numColumns).activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + model.addListeners(new ScoreIterationListener(listenerFreq)); + + log.info("Train model...."); + int cnt = 0; + while (cnt < numSamples) { + INDArray input = Nd4j.create(trainingData[cnt]).reshape(1, -1); + model.fit(new DataSet(input, input)); + cnt++; + } + // Make two separate selective calls + + log.info("Testing full cycle..."); + + List comparableResult = model.feedForward(Nd4j.create(trainingData[0], new long[]{1, trainingData[0].length})); + + INDArray encodeResult = model.activateSelectedLayers(0, 4, Nd4j.create(trainingData[0], new long[]{1, trainingData[0].length})); + + log.info("Compare feedForward results with selectedActivation"); + + assertEquals(comparableResult.get(5), encodeResult); + + INDArray decodeResults = model.activateSelectedLayers(5, 9, encodeResult); + + + log.info("Decode results: " + decodeResults.columns() + " " + decodeResults); + log.info("Comparable results: " + comparableResult.get(10).columns() + " " + comparableResult.get(10)); + + assertEquals(comparableResult.get(10), decodeResults); + } + + private static MultiLayerConfiguration getConf() { + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().seed(12345L) + .list().layer(0, + new DenseLayer.Builder().nIn(4).nOut(3) + + .dist(new NormalDistribution(0,1)) + .build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + + .dist(new NormalDistribution(0, 1)).build()) + .build(); + return conf; + } + + public static float[] asFloat(INDArray arr) { + long len = arr.length(); + + float[] f = new float[(int) len]; + for (int i = 0; i < len; i++) + f[i] = arr.getFloat(i); + return f; + } + + @Test + public void testFeedForwardToLayer() { + + int nIn = 30; + int nOut = 25; + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT) + .updater(new Sgd(1e-3)) + .list().layer( + 0, new DenseLayer.Builder().nIn(nIn).nOut(600) + + .dist(new NormalDistribution(0,1e-5)) + .build()) + .layer(1, new DenseLayer.Builder() + .nIn(600).nOut(250) + .dist(new NormalDistribution(0, 1e-5)) + .build()) + .layer(2, new DenseLayer.Builder() + .nIn(250).nOut(100) + .dist(new NormalDistribution(0, 1e-5)) + .build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(100).nOut(25) + .activation(Activation.SOFTMAX) + .weightInit(new NormalDistribution(0, 1e-5)).build()) + .build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + + INDArray input = Nd4j.rand(5, nIn); + + List activations = network.feedForward(input); + assertEquals(5, activations.size()); //4 layers + input + + List activationsAll = network.feedForwardToLayer(3, input); + assertEquals(activations, activationsAll); + + for (int i = 3; i >= 0; i--) { + List activationsPartial = network.feedForwardToLayer(i, input); + assertEquals(i + 2, activationsPartial.size()); //i+2: for layer 3: input + activations of {0,1,2,3} -> 5 total = 3+2 + for (int j = 0; j <= i; j++) { + INDArray exp = activationsAll.get(j); + INDArray act = activationsPartial.get(j); + assertEquals(exp, act); + } + } + } + + + @Test + public void testBackpropGradient() { + //Testing: MultiLayerNetwork.backpropGradient() + //i.e., specifically without an output layer + + int nIn = 10; + int nOut = 40; + int miniBatch = 5; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .updater(new Sgd(0.1)).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(2, new DenseLayer.Builder().nIn(30).nOut(nOut).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + Nd4j.getRandom().setSeed(12345); + INDArray eps = Nd4j.rand(miniBatch, nOut); + INDArray input = Nd4j.rand(miniBatch, nIn); + + net.setInput(input); + net.feedForward(true, false); //Need to feed forward before backprop + + Pair pair = net.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces()); + INDArray epsOut = pair.getSecond(); + assertNotNull(epsOut); + assertArrayEquals(new long[] {miniBatch, nIn}, epsOut.shape()); + + Gradient g = pair.getFirst(); + Map gradMap = g.gradientForVariable(); + assertEquals(6, gradMap.size()); //3 layers, weight + bias gradients for each + + String[] expKeys = {"0_" + DefaultParamInitializer.WEIGHT_KEY, "0_" + DefaultParamInitializer.BIAS_KEY, + "1_" + DefaultParamInitializer.WEIGHT_KEY, "2_" + DefaultParamInitializer.BIAS_KEY, + "2_" + DefaultParamInitializer.WEIGHT_KEY, "2_" + DefaultParamInitializer.BIAS_KEY}; + Set keys = gradMap.keySet(); + for (String s : expKeys) { + assertTrue(keys.contains(s)); + } + + /* + System.out.println(pair); + + //Use updater to go from raw gradients -> updates + //Apply learning rate, gradient clipping, adagrad/momentum/rmsprop etc + Updater updater = UpdaterCreator.getUpdater(net); + updater.update(net, g, 0, miniBatch); + + StepFunction stepFunction = new NegativeGradientStepFunction(); + INDArray params = net.params(); + System.out.println(Arrays.toString(params.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 10)).dup().data().asFloat())); + stepFunction.step(params, g.gradient()); + net.setParams(params); //params() may not be in-place + System.out.println(Arrays.toString(params.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 10)).dup().data().asFloat())); + */ + } + + @Test + public void testLayerNames() { + int nIn = 10; + int nOut = 40; + + List layerNameList = new ArrayList<>(); + layerNameList.add("dnn1"); + layerNameList.add("dnn2"); + layerNameList.add("dnn3"); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .updater(new Sgd(0.1)).list() + .layer(0, new DenseLayer.Builder().name("dnn1").nIn(nIn).nOut(20).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(1, new DenseLayer.Builder().name("dnn2").nIn(20).nOut(30).activation(Activation.RELU) + .weightInit(WeightInit.XAVIER).build()) + .layer(2, new DenseLayer.Builder().name("dnn3").nIn(30).nOut(nOut) + .activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(layerNameList.get(0), net.getLayer(0).conf().getLayer().getLayerName()); + assertEquals(layerNameList, net.getLayerNames()); + BaseLayer b = (BaseLayer) net.getLayer(layerNameList.get(2)).conf().getLayer(); + assertEquals("softmax", b.getActivationFn().toString()); + } + + + @Test + public void testScoreExamples() { + Nd4j.getRandom().setSeed(12345); + int nIn = 5; + int nOut = 6; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()) + .build(); + + MultiLayerConfiguration confNoReg = new NeuralNetConfiguration.Builder().seed(12345) + .updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()) + .build(); + + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + MultiLayerNetwork netNoReg = new MultiLayerNetwork(confNoReg); + netNoReg.init(); + netNoReg.setParameters(net.params().dup()); + + //Score single example, and compare to scoreExamples: + INDArray input = Nd4j.rand(3, nIn); + INDArray output = Nd4j.rand(3, nOut); + DataSet ds = new DataSet(input, output); + + INDArray scoresWithRegularization = net.scoreExamples(ds, true); + INDArray scoresNoRegularization = net.scoreExamples(ds, false); + + assertArrayEquals(new long[] {3, 1}, scoresWithRegularization.shape()); + assertArrayEquals(new long[] {3, 1}, scoresNoRegularization.shape()); + + for (int i = 0; i < 3; i++) { + DataSet singleEx = new DataSet(input.getRow(i,true), output.getRow(i,true)); + double score = net.score(singleEx); + double scoreNoReg = netNoReg.score(singleEx); + + double scoreUsingScoreExamples = scoresWithRegularization.getDouble(i); + double scoreUsingScoreExamplesNoReg = scoresNoRegularization.getDouble(i); + assertEquals(score, scoreUsingScoreExamples, 1e-4); + assertEquals(scoreNoReg, scoreUsingScoreExamplesNoReg, 1e-4); + assertTrue(scoreUsingScoreExamples > scoreUsingScoreExamplesNoReg); //Regularization term increases score + + // System.out.println(score + "\t" + scoreUsingScoreExamples + "\t|\t" + scoreNoReg + "\t" + scoreUsingScoreExamplesNoReg); + } + } + + @Test + public void testDataSetScore() { + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .weightInit(WeightInit.XAVIER).seed(12345L).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.SIGMOID).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray in = Nd4j.create(new double[] {1.0, 2.0, 3.0, 4.0}, new long[]{1, 4}); + INDArray out = Nd4j.create(new double[] {1, 0, 0}, new long[]{1,3}); + + double score = net.score(new DataSet(in, out)); + } + + @Test + public void testDataSetScoreCNN() { + + int miniBatch = 3; + int depth = 2; + int width = 3; + int height = 3; + int nOut = 2; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345L).list().layer(0, new ConvolutionLayer.Builder(2, 2).nOut(1).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(2).build()) + .setInputType(InputType.convolutionalFlat(height, width, depth)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + Nd4j.getRandom().setSeed(12345); + Random r = new Random(12345); + INDArray input = Nd4j.rand(miniBatch, depth * width * height); + INDArray labels = Nd4j.create(miniBatch, nOut); + for (int i = 0; i < miniBatch; i++) { + labels.putScalar(new int[] {i, r.nextInt(nOut)}, 1.0); + } + + double score = net.score(new DataSet(input, labels)); + } + + @Test + public void testPredict() throws Exception { + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .weightInit(WeightInit.XAVIER).seed(12345L).list() + .layer(0, new DenseLayer.Builder().nIn(784).nOut(50).activation(Activation.RELU).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(50).nOut(10).build()) + .setInputType(InputType.convolutional(28, 28, 1)).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSetIterator ds = new MnistDataSetIterator(10, 10); + net.fit(ds); + + DataSetIterator testDs = new MnistDataSetIterator(1, 1); + DataSet testData = testDs.next(); + testData.setLabelNames(Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + String actualLables = testData.getLabelName(0); + List prediction = net.predict(testData); + assertTrue(actualLables != null); + assertTrue(prediction.get(0) != null); + } + + @Test + //@Ignore + public void testCid() throws Exception { + System.out.println(EnvironmentUtils.buildCId()); + + Environment environment = EnvironmentUtils.buildEnvironment(); + environment.setSerialVersionID(EnvironmentUtils.buildCId()); + + Task task = TaskUtils.buildTask(Nd4j.create(new double[] {1, 2, 3, 4, 5, 6}, new long[]{1,6})); + + Heartbeat.getInstance().reportEvent(Event.STANDALONE, environment, task); + + Thread.sleep(25000); + } + + @Test + public void testOutput() throws Exception { + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .weightInit(WeightInit.XAVIER).seed(12345L).list() + .layer(0, new DenseLayer.Builder().nIn(784).nOut(50).activation(Activation.RELU).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(50).nOut(10).build()) + .setInputType(InputType.convolutional(28, 28, 1)).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSetIterator fullData = new MnistDataSetIterator(1, 2); + net.fit(fullData); + + + fullData.reset(); + DataSet expectedSet = fullData.next(2); + INDArray expectedOut = net.output(expectedSet.getFeatures(), false); + + fullData.reset(); + + INDArray actualOut = net.output(fullData); + + assertEquals(expectedOut, actualOut); + } + + @Test + public void testGradientUpdate() throws Exception { + DataSetIterator iter = new IrisDataSetIterator(1, 1); + + Gradient expectedGradient = new DefaultGradient(); + expectedGradient.setGradientFor("0_W", Nd4j.ones(4, 5)); + expectedGradient.setGradientFor("0_b", Nd4j.ones(1, 5)); + expectedGradient.setGradientFor("1_W", Nd4j.ones(5, 3)); + expectedGradient.setGradientFor("1_b", Nd4j.ones(1, 3)); + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().updater(new Sgd(1.0)) + .activation(Activation.RELU).weightInit(WeightInit.XAVIER) + .list().layer(0, new DenseLayer.Builder().name("dnn1").nIn(4).nOut(5).build()) + .layer(1, new OutputLayer.Builder().name("output").nIn(5).nOut(3) + .activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER) + .build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + net.fit(iter.next()); + // TODO validate actual layer gradientView - issue getting var out of BaseLayer w/o adding MLN getter that gets confused with local gradient vars + Gradient actualGradient = net.gradient; + assertNotEquals(expectedGradient.getGradientFor("0_W"), actualGradient.getGradientFor("0_W")); + + net.update(expectedGradient); + actualGradient = net.gradient; + assertEquals(expectedGradient.getGradientFor("0_W"), actualGradient.getGradientFor("0_W")); + + // Update params with set + net.setParam("0_W", Nd4j.ones(4, 5)); + net.setParam("0_b", Nd4j.ones(1, 5)); + net.setParam("1_W", Nd4j.ones(5, 3)); + net.setParam("1_b", Nd4j.ones(1, 3)); + INDArray actualParams = net.params(); + + // Confirm params + assertEquals(expectedGradient.gradient(), actualParams); + + net.update(expectedGradient); + actualParams = net.params(); + assertEquals(Nd4j.ones(1, 43).addi(1), actualParams); + } + + + @Test + public void testCnnInvalidData() { + assertThrows(DL4JException.class, () -> { + int miniBatch = 3; + int depth = 2; + int width = 5; + int height = 5; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(2) + .nOut(2).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nOut(2).build()) + .setInputType(InputType.convolutional(height, width, depth)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray inputWrongDepth = Nd4j.rand(new int[]{miniBatch, 5, height, width}); //Order: examples, channels, height, width + net.feedForward(inputWrongDepth); + }); + } + + @Test + public void testApplyingPreTrainConfigAndParams() { + int nIn = 10; + int nOut = 10; + + // Test pretrain true + MultiLayerNetwork aePre = getAeModel(true, nIn, nOut); + int actualNP = (int)aePre.numParams(); + assertEquals(2 * (nIn * nOut + nOut) + nIn, actualNP); + INDArray params = aePre.params(); + assertEquals(params.length(), actualNP); // check num params + Map paramTable = aePre.paramTable(); + assertTrue(paramTable.containsKey("0_vb")); // check vb exists for pretrain layer + aePre.setParam("0_vb", Nd4j.ones(10)); + params = aePre.getParam("0_vb"); + assertEquals(Nd4j.ones(1,10), params); // check set params for vb + + + // Test pretrain false, expect same for true because its not changed when applying update + MultiLayerNetwork aeNoPre = getAeModel(false, nIn, nOut); + actualNP = (int)aeNoPre.numParams(); + assertEquals(2 * (nIn * nOut + nOut) + nIn, actualNP); + params = aeNoPre.params(); + assertEquals(params.length(), actualNP); + paramTable = aePre.paramTable(); + assertTrue(paramTable.containsKey("0_vb")); + } + + public MultiLayerNetwork getAeModel(boolean preTrain, int nIn, int nOut) { + MultiLayerConfiguration vae = new NeuralNetConfiguration.Builder() + .seed(42).updater(new NoOp()) + .weightInit(WeightInit.UNIFORM) + .list(new AutoEncoder.Builder() + .activation(Activation.IDENTITY).nOut(nIn).build(), + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.COSINE_PROXIMITY) + .activation(Activation.IDENTITY).nOut(nOut) + .build()) + .setInputType(InputType.feedForward(nOut)).build(); + MultiLayerNetwork network = new MultiLayerNetwork(vae); + network.init(); + return network; + } + + + @Test + public void testIterationCountAndPersistence() throws IOException { + Nd4j.getRandom().setSeed(123); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123) + .list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build(); + + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + DataSetIterator iter = new IrisDataSetIterator(50, 150); + + assertEquals(0, network.getLayerWiseConfigurations().getIterationCount()); + network.fit(iter); + assertEquals(3, network.getLayerWiseConfigurations().getIterationCount()); + iter.reset(); + network.fit(iter); + assertEquals(6, network.getLayerWiseConfigurations().getIterationCount()); + iter.reset(); + network.fit(iter.next()); + assertEquals(7, network.getLayerWiseConfigurations().getIterationCount()); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ModelSerializer.writeModel(network, baos, true); + byte[] asBytes = baos.toByteArray(); + + ByteArrayInputStream bais = new ByteArrayInputStream(asBytes); + MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(bais, true); + assertEquals(7, net.getLayerWiseConfigurations().getIterationCount()); + } + + + @Test + public void testBiasL1L2() { + + + Nd4j.getRandom().setSeed(123); + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .weightInit(WeightInit.XAVIER).activation(Activation.TANH).seed(123).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nIn(10).nOut(10) + .build()) + .build(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .l1Bias(0.1).l2Bias(0.2).weightInit(WeightInit.XAVIER).activation(Activation.TANH) + .seed(123).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nIn(10).nOut(10) + .build()) + .build(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + BaseLayer bl0 = (BaseLayer) net2.getLayer(0).conf().getLayer(); + assertEquals(0.1, TestUtils.getL1(bl0.getRegularizationBias()), 1e-6); + assertEquals(0.2, TestUtils.getL2(bl0.getRegularizationBias()), 1e-6); + + INDArray features = Nd4j.rand(10, 10); + INDArray labels = Nd4j.rand(10, 10); + + net2.setParams(net1.params().dup()); + + net1.setInput(features); + net1.setLabels(labels); + net2.setInput(features); + net2.setLabels(labels); + + net1.computeGradientAndScore(); + net2.computeGradientAndScore(); + + double r = net1.calcRegularizationScore(true); + assertEquals(0.0, r, 0.0); + + r = net2.calcRegularizationScore(true); + assertEquals(0.0, r, 0.0); + + + double s1 = net1.score(); + double s2 = net2.score(); + assertEquals(s1, s2, 1e-6); //Biases initialized to 0 -> should initially have same score + + for (int i = 0; i < 10; i++) { + net1.fit(features, labels); + } + + net2.setParams(net1.params().dup()); + net1.computeGradientAndScore(); + net2.computeGradientAndScore(); + + r = net1.calcRegularizationScore(true); + assertEquals(0.0, r, 0.0); + + r = net2.calcRegularizationScore(true); + assertTrue(r > 0.0); + + s1 = net1.score(); + s2 = net2.score(); + + assertNotEquals(s1, s2, 1e-6); //Scores should differ due to bias l1/l2 + + for (int i = 0; i < 2; i++) { + assertEquals(0.0, net1.getLayer(i).calcRegularizationScore(true), 0.0); + assertTrue(net2.getLayer(i).calcRegularizationScore(true) > 0.0); + } + } + + /* + Summary should pick up preprocessors set manually on inputs as well + */ + @Test + public void testSummary() { + int V_WIDTH = 130; + int V_HEIGHT = 130; + int V_NFRAMES = 150; + MultiLayerConfiguration confForArchitecture = + new NeuralNetConfiguration.Builder().seed(12345).l2(0.001) //l2 regularization on all layers + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .list() + .layer(0, new ConvolutionLayer.Builder(10, 10).nIn(3) //3 channels: RGB + .nOut(30).stride(4, 4).activation(Activation.RELU).weightInit( + WeightInit.RELU) + .updater(Updater.ADAGRAD).build()) //Output: (130-10+0)/4+1 = 31 -> 31*31*30 + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) + .kernelSize(3, 3).stride(2, 2).build()) //(31-3+0)/2+1 = 15 + .layer(2, new ConvolutionLayer.Builder(3, 3).nIn(30).nOut(10).stride(2, 2) + .activation(Activation.RELU).weightInit(WeightInit.RELU) + .updater(Updater.ADAGRAD).build()) //Output: (15-3+0)/2+1 = 7 -> 7*7*10 = 490 + .layer(3, new DenseLayer.Builder().activation(Activation.RELU).nIn(490).nOut(50) + .weightInit(WeightInit.RELU).updater(Updater.ADAGRAD) + .gradientNormalization( + GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).build()) + .layer(4, new GravesLSTM.Builder().activation(Activation.SOFTSIGN).nIn(50) + .nOut(50).weightInit(WeightInit.XAVIER).updater(Updater.ADAGRAD) + .gradientNormalization( + GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10) + .build()) + .layer(5, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(50).nOut(4) //4 possible shapes: circle, square, arc, line + .updater(Updater.ADAGRAD).weightInit(WeightInit.XAVIER) + .gradientNormalization( + GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).build()) + .inputPreProcessor(0, new RnnToCnnPreProcessor(V_HEIGHT, V_WIDTH, 3)) + .inputPreProcessor(3, new CnnToFeedForwardPreProcessor(7, 7, 10)) + .inputPreProcessor(4, new FeedForwardToRnnPreProcessor()) + .backpropType(BackpropType.TruncatedBPTT) + .tBPTTForwardLength(V_NFRAMES / 5).tBPTTBackwardLength(V_NFRAMES / 5).build(); + MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(confForArchitecture); + modelExpectedArch.init(); + MultiLayerNetwork modelMow = new TransferLearning.Builder(modelExpectedArch).setFeatureExtractor(2).build(); +// System.out.println(modelExpectedArch.summary()); +// System.out.println(modelMow.summary()); +// System.out.println(modelMow.summary(InputType.recurrent(V_HEIGHT*V_WIDTH*3))); + } + + @Test + public void testErrorNoOutputLayer() { + assertThrows(DL4JException.class, () -> { + MultiLayerConfiguration c = new NeuralNetConfiguration.Builder().list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(c); + net.init(); + + INDArray f = Nd4j.create(1, 10); + INDArray l = Nd4j.create(1, 10); + + net.setInput(f); + net.setLabels(l); + + net.computeGradientAndScore(); + }); + } + + + @Test + public void testSetParamTable() { + + Nd4j.getRandom().setSeed(123); + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(123).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(2, new LSTM.Builder().nIn(2).nOut(2).build()) + .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3) + .build()) + .build(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(987).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(2, new LSTM.Builder().nIn(2).nOut(2).build()) + .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3) + .build()) + .build(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); + net1.init(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); + net2.init(); + + assertNotEquals(net1.params(), net2.params()); + assertNotEquals(net1.paramTable(), net2.paramTable()); + + net1.setParamTable(net2.paramTable()); + assertEquals(net1.params(), net2.params()); + assertEquals(net1.paramTable(), net2.paramTable()); + } + + + @Test + public void testCompareLayerMethods(){ + //Simple test: compare .layer(int, Layer) and .layer(Layer) are identical + + MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(123).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(2, new LSTM.Builder().nIn(2).nOut(2).build()) + .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3) + .build()) + .build(); + + MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(123).list() + .layer(new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER) + .activation(Activation.TANH).build()) + .layer(new LSTM.Builder().nIn(2).nOut(2).build()) + .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3) + .build()) + .build(); + + assertEquals(conf1, conf2); + } + + + @Test + public void testEpochCounter() throws Exception { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(0, net.getLayerWiseConfigurations().getEpochCount()); + + + DataSetIterator iter = new IrisDataSetIterator(150, 150); + + for( int i=0; i<4; i++ ){ + assertEquals(i, net.getLayerWiseConfigurations().getEpochCount()); + net.fit(iter); + assertEquals(i+1, net.getLayerWiseConfigurations().getEpochCount()); + } + + assertEquals(4, net.getLayerWiseConfigurations().getEpochCount()); + + MultiLayerNetwork restored = TestUtils.testModelSerialization(net); + assertEquals(4, restored.getLayerWiseConfigurations().getEpochCount()); + } + + @Test + public void testInputClearance() throws Exception { + //Activations should be cleared - if not, it's possible for out of (workspace) scope arrays to be around + // which can cause a crash + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .convolutionMode(ConvolutionMode.Same) + .list() + .layer(new ConvolutionLayer.Builder().kernelSize(2,2).stride(1,1).nIn(1).nOut(1).build()) + .layer(new SubsamplingLayer.Builder().kernelSize(2,2).stride(1,1).build()) + .layer(new DenseLayer.Builder().nOut(10).build()) + .layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28,28,1)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray content = Nd4j.create(1,1,28,28); + + //Check output: + net.output(content); + for(org.deeplearning4j.nn.api.Layer l : net.getLayers()){ + assertNull(l.input()); + } + + //Check feedForward: + net.feedForward(content, false); + for(org.deeplearning4j.nn.api.Layer l : net.getLayers()){ + assertNull(l.input()); + } + } + + + @Test + public void testExternalErrors() { + //Simple test: same network, but in one case: one less layer (the OutputLayer), where the epsilons are passed in externally + // instead. Should get identical results + + for(WorkspaceMode ws : WorkspaceMode.values()) { + log.info("Workspace mode: " + ws); + + Nd4j.getRandom().setSeed(12345); + INDArray inData = Nd4j.rand(3, 10); + INDArray outData = Nd4j.rand(3, 10); + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration standard = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .trainingWorkspaceMode(ws) + .inferenceWorkspaceMode(ws) + .seed(12345).list() + .layer(new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(10) + .nOut(10).build()) + .build(); + MultiLayerNetwork s = new MultiLayerNetwork(standard); + s.init(); + + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration external = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .trainingWorkspaceMode(ws) + .inferenceWorkspaceMode(ws) + .seed(12345).list() + .layer(new DenseLayer.Builder().nIn(10).nOut(10).build()) + .build(); + + MultiLayerNetwork e = new MultiLayerNetwork(external); + e.init(); + + s.setInput(inData); + s.setLabels(outData); + s.computeGradientAndScore(); + Gradient sGrad = s.gradient(); + + s.setInput(inData); + s.feedForward(true, false); //FF without clearing inputs as we need them later + + e.setInput(inData); + e.feedForward(true, false); //FF without clearing inputs as we need them later + + org.deeplearning4j.nn.layers.OutputLayer ol = (org.deeplearning4j.nn.layers.OutputLayer) s.getLayer(1); + Pair olPairStd = ol.backpropGradient(null, LayerWorkspaceMgr.noWorkspaces()); + + INDArray olEpsilon = olPairStd.getSecond().detach(); + + e.setInput(inData); + e.feedForward(true, false); + Pair extErrorGrad = e.backpropGradient(olEpsilon, LayerWorkspaceMgr.noWorkspaces()); + + int nParamsDense = 10 * 10 + 10; + assertEquals(sGrad.gradient().get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nParamsDense)), + extErrorGrad.getFirst().gradient()); + + Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); + } + } + + @Test + public void testExternalErrors2(){ + Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC); + int nIn = 4; + int nOut = 3; + + for(WorkspaceMode ws : WorkspaceMode.values()) { +// System.out.println("***** WORKSPACE: " + ws); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .updater(new Adam(0.01)) + .trainingWorkspaceMode(ws) + .inferenceWorkspaceMode(ws) + .list() + .layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.RELU).build()) + .layer(new ActivationLayer.Builder().activation(Activation.IDENTITY).build()) + .inputPreProcessor(0, new RnnToFeedForwardPreProcessor()) + .inputPreProcessor(1, new FeedForwardToRnnPreProcessor()) + .build(); + + MultiLayerNetwork graph = new MultiLayerNetwork(conf); + graph.init(); + + final int minibatch = 5; + final int seqLen = 6; + + INDArray param = Nd4j.create(new double[]{0.54, 0.31, 0.98, -0.30, -0.66, -0.19, -0.29, -0.62, 0.13, -0.32, 0.01, -0.03, 0.00, 0.00, 0.00}).reshape(1, -1); + graph.setParams(param); + + INDArray input = Nd4j.rand(new int[]{minibatch, nIn, seqLen}, 12); + INDArray expected = Nd4j.ones(minibatch, nOut, seqLen); + + graph.setInput(input); + INDArray output = graph.feedForward(false, false).get(2); + INDArray error = output.sub(expected); + + for (org.deeplearning4j.nn.api.Layer l : graph.getLayers()) { + assertNotNull(l.input()); + assertFalse(l.input().isAttached()); + } + + // Compute Gradient + Pair gradient = graph.backpropGradient(error, LayerWorkspaceMgr.noWorkspaces()); + graph.getUpdater().update(graph, gradient.getFirst(), 0, 0, minibatch, LayerWorkspaceMgr.noWorkspaces()); + + Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); + } + + Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.DISABLED); + } + + @Test + public void testLayerSize(){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + + .list() + .layer(new ConvolutionLayer.Builder().kernelSize(2,2).nOut(6).build()) + .layer(new SubsamplingLayer.Builder().kernelSize(2,2).build()) + .layer(new DenseLayer.Builder().nOut(30).build()) + .layer(new OutputLayer.Builder().nOut(13).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28,28,3)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + assertEquals(6, net.layerSize(0)); + assertEquals(0, net.layerSize(1)); + assertEquals(30, net.layerSize(2)); + assertEquals(13, net.layerSize(3)); + + assertEquals(3, net.layerInputSize(0)); + assertEquals(0, net.layerInputSize(1)); + assertEquals(((FeedForwardLayer)net.getLayer(2).conf().getLayer()).getNIn(), net.layerInputSize(2)); + assertEquals(30, net.layerInputSize(3)); + } + + + @Test + public void testZeroParamNet() throws Exception { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new SubsamplingLayer.Builder().kernelSize(2,2).stride(2,2).build()) + .layer(new LossLayer.Builder().activation(Activation.SIGMOID).lossFunction(LossFunctions.LossFunction.MSE).build()) + .setInputType(InputType.convolutionalFlat(28,28,1)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSet ds = new MnistDataSetIterator(16, true, 12345).next(); + + INDArray out = net.output(ds.getFeatures()); + + INDArray labelTemp = Nd4j.create(out.shape()); + ds.setLabels(labelTemp); + + net.fit(ds); + + MultiLayerNetwork net2 = TestUtils.testModelSerialization(net); + INDArray out2 = net2.output(ds.getFeatures()); + assertEquals(out, out2); + } + + + @Test + public void testInputActivationGradient(){ + Nd4j.setDataType(DataType.DOUBLE); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .seed(12345) + .activation(Activation.TANH) + .list() + .layer(new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(new OutputLayer.Builder().nIn(10).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray in = Nd4j.rand(1, 10); + INDArray label = Nd4j.rand(1, 10); + + Pair p = net.calculateGradients(in, label, null, null); + + //Quick gradient check: + double eps = 1e-6; + double maxRelError = 1e-5; + for( int i=0; i<10; i++ ){ + double orig = in.getDouble(i); + in.putScalar(i, orig + eps); + double scorePlus = net.score(new DataSet(in, label)); + in.putScalar(i, orig - eps); + double scoreMinus = net.score(new DataSet(in, label)); + in.putScalar(i, orig); + + double expGrad = (scorePlus - scoreMinus) / (2.0 * eps); + double actGrad = p.getSecond().getDouble(i); + + double relError = (Math.abs(expGrad - actGrad)) / (Math.abs(expGrad) + Math.abs(actGrad)); + + String str = i + " - " + relError + " - exp=" + expGrad + ", act=" + actGrad; + assertTrue(relError < maxRelError, str); + } + } + + + @Test + public void testMultiLayerConfigurationActivationTypes(){ + + NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder() + .list() + .layer(new LSTM.Builder().nOut(6).build()) + .layer(new LSTM.Builder().nOut(7).build()) + .layer(new GlobalPoolingLayer()) + .layer(new OutputLayer.Builder().nOut(8).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.recurrent(10)); + + MultiLayerConfiguration conf = builder.build(); + + List outBuilder = builder.getLayerActivationTypes(); + List outConf = conf.getLayerActivationTypes(InputType.recurrent(10)); + + List exp = Arrays.asList( + InputType.recurrent(6), + InputType.recurrent(7), + InputType.feedForward(7), + InputType.feedForward(8) + ); + + + assertEquals(exp, outBuilder); + assertEquals(exp, outConf); + } + + @Test + public void testMultipleEpochsSimple(){ + //Mainly a simple sanity check on the preconditions in the method... + DataSetIterator iter = new IrisDataSetIterator(10, 150); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + net.fit(iter, 3); + + ComputationGraph g = net.toComputationGraph(); + g.fit(iter, 3); + } + + + @Test + public void testPretrainFitMethods(){ + + //The fit methods should *not* do layerwise pretraining: + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + + .list() + .layer(new VariationalAutoencoder.Builder() + .nIn(10).nOut(10).encoderLayerSizes(10).decoderLayerSizes(10).build()) + .layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build()) + + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + Set> exp = new HashSet<>(); + exp.add(MultiLayerNetwork.class); + + CheckModelsListener listener = new CheckModelsListener(); + net.setListeners(listener); + + INDArray f = Nd4j.create(1,10); + INDArray l = Nd4j.create(1,10); + DataSet ds = new DataSet(f,l); + MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(f,l); + + DataSetIterator iter = new ExistingDataSetIterator(Collections.singletonList(ds)); + net.fit(iter); + assertEquals(exp, listener.getModelClasses()); + + net.fit(ds); + assertEquals(exp, listener.getModelClasses()); + + net.fit(f, l); + assertEquals(exp, listener.getModelClasses()); + + net.fit(f, l, null, null); + assertEquals(exp, listener.getModelClasses()); + + net.fit(mds); + assertEquals(exp, listener.getModelClasses()); + + net.fit(new SingletonMultiDataSetIterator(mds)); + assertEquals(exp, listener.getModelClasses()); + } + + @Test + public void testINDArrayConfigCloning(){ + //INDArrays in config should be cloned to avoid threading issues + + int mb = 3; + int b = 4; + int c = 3; + int depth = b * (5 + c); + int w = 6; + int h = 6; + + INDArray bbPrior = Nd4j.rand(b, 2).muliRowVector(Nd4j.create(new double[]{w, h}).castTo(Nd4j.defaultFloatingPointType())); + + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .l2(0.01) + .list() + .layer(new ConvolutionLayer.Builder().nIn(depth).nOut(depth).kernelSize(1,1).build()) + .layer(new Yolo2OutputLayer.Builder() + .boundingBoxPriors(bbPrior) + .build()) + .build(); + + MultiLayerConfiguration conf2 = conf.clone(); + + INDArray bb1 = ((Yolo2OutputLayer)conf.getConf(1).getLayer()).getBoundingBoxes(); + INDArray bb2 = ((Yolo2OutputLayer)conf2.getConf(1).getLayer()).getBoundingBoxes(); + assertFalse(bb1 == bb2); + + assertEquals(bb1, bb2); + } + + @Data + public static class CheckModelsListener extends BaseTrainingListener { + + private Set> modelClasses = new HashSet<>(); + + @Override + public void iterationDone(Model model, int iteration, int epoch) { + modelClasses.add(model.getClass()); + } + } + + + @Test + public void testMLNUpdaterBlocks(){ + //Check that setting learning rate results in correct rearrangement of updater state within updater blocks + //https://github.com/deeplearning4j/deeplearning4j/issues/6809#issuecomment-463892644 + + double lr = 1e-3; + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345) + .weightInit(WeightInit.XAVIER) + .updater(new Adam(lr)) + .list() + .layer(new DenseLayer.Builder().nIn(5).nOut(3).build()) + .layer(new DenseLayer.Builder().nIn(3).nOut(2).build()) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(2).nOut(1) + .activation(Activation.SIGMOID).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray in = Nd4j.rand(1, 5); + INDArray lbl = Nd4j.rand(1,1); + + net.fit(new DataSet(in, lbl)); + + INDArray viewArray = net.getUpdater().getStateViewArray(); + INDArray viewArrayCopy = viewArray.dup(); + //Initially updater view array is set out like: + //[m0w, m0b, m1w, m1b, m2w, m2b][v0w, v0b, v1w, v1b, v2w, v2b] + long soFar = 0; + INDArray m0w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+5*3)).assign(0); //m0w + soFar += 5*3; + INDArray m0b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+3)).assign(1); //m0b + soFar += 3; + INDArray m1w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+3*2)).assign(2); //m1w + soFar += 3*2; + INDArray m1b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+2)).assign(3); //m1b + soFar += 2; + INDArray m2w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+2*1)).assign(4); //m2w + soFar += 2*1; + INDArray m2b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+1)).assign(5); //m2b + soFar += 1; + + INDArray v0w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+5*3)).assign(6); //v0w + soFar += 5*3; + INDArray v0b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+3)).assign(7); //v0b + soFar += 3; + INDArray v1w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+3*2)).assign(8); //v1w + soFar += 3*2; + INDArray v1b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+2)).assign(9); //v1b + soFar += 2; + INDArray v2w = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+2*1)).assign(10); //v2w + soFar += 2*1; + INDArray v2b = viewArray.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(soFar, soFar+1)).assign(11); //v2b + soFar += 1; + + + net.setLearningRate(0, 0.0); + + //Expect new updater state to look like: + //[m0w, m0b][v0w,v0b], [m1w, m1b, m2w, m2b][v1w, v1b, v2w, v2b] + INDArray exp = Nd4j.concat(1, m0w, m0b, v0w, v0b, + m1w, m1b, m2w, m2b, v1w, v1b, v2w, v2b); + + INDArray act = net.getUpdater().getStateViewArray(); +// System.out.println(exp); +// System.out.println(act); + + assertEquals(exp, act); + + //And set layer 1 LR: + net.setLearningRate(1, 0.2); + exp = Nd4j.concat(1, m0w, m0b, v0w, v0b, + m1w, m1b, v1w, v1b, + m2w, m2b, v2w, v2b); + assertEquals(exp, net.getUpdater().getStateViewArray()); + + + //Set all back to original LR and check again: + net.setLearningRate(1, lr); + net.setLearningRate(0, lr); + + exp = Nd4j.concat(1, m0w, m0b, m1w, m1b, m2w, m2b, v0w, v0b, v1w, v1b, v2w, v2b); + assertEquals(exp, net.getUpdater().getStateViewArray()); + + + //Finally, training sanity check (if things are wrong, we get -ve values in adam V, which causes NaNs) + net.getUpdater().getStateViewArray().assign(viewArrayCopy); + net.setLearningRate(0, 0.0); + + Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.NAN_PANIC); + net.fit(new DataSet(in, lbl)); + Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTestRNN.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTestRNN.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTestRNN.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTestRNN.java index c241f52c6..6f1b3f732 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTestRNN.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTestRNN.java @@ -42,10 +42,7 @@ import org.deeplearning4j.nn.layers.recurrent.LSTM; import org.deeplearning4j.nn.layers.recurrent.SimpleRnn; import org.deeplearning4j.nn.params.GravesLSTMParamInitializer; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -63,8 +60,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class MultiLayerTestRNN extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestMasking.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestMasking.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestMasking.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestMasking.java index 486f47dcb..420417296 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestMasking.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestMasking.java @@ -38,10 +38,7 @@ import org.deeplearning4j.nn.conf.preprocessor.CnnToRnnPreProcessor; import org.deeplearning4j.nn.conf.preprocessor.RnnToCnnPreProcessor; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.buffer.util.DataTypeUtil; @@ -58,8 +55,7 @@ import org.nd4j.linalg.lossfunctions.impl.*; import java.util.Collections; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestMasking extends BaseDL4JTest { static { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestSetGetParameters.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestSetGetParameters.java new file mode 100644 index 000000000..ff5efa35a --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestSetGetParameters.java @@ -0,0 +1,156 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.multilayer; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.layers.*; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestSetGetParameters extends BaseDL4JTest { + + @Test + public void testSetParameters() { + //Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this. + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new DenseLayer.Builder().nIn(9).nOut(10) + .dist(new NormalDistribution(0, 1)).build()) + .layer(1, new DenseLayer.Builder().nIn(10).nOut(11) + .dist(new NormalDistribution(0, 1)).build()) + .layer(2, new AutoEncoder.Builder().corruptionLevel(0.5).nIn(11).nOut(12) + .dist(new NormalDistribution(0, 1)).build()) + .layer(3, new OutputLayer.Builder(LossFunction.MSE).nIn(12).nOut(12) + .dist(new NormalDistribution(0, 1)).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray initParams = net.params().dup(); + Map initParams2 = net.paramTable(); + + net.setParams(net.params()); + + INDArray initParamsAfter = net.params(); + Map initParams2After = net.paramTable(); + + for (String s : initParams2.keySet()) { + assertTrue(initParams2.get(s).equals(initParams2After.get(s)), "Params differ: " + s); + } + + assertEquals(initParams, initParamsAfter); + + //Now, try the other way: get(set(random)) + INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape()); + net.setParams(randomParams.dup()); + + assertEquals(net.params(), randomParams); + } + + @Test + public void testSetParametersRNN() { + //Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this. + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new GravesLSTM.Builder().nIn(9).nOut(10) + .dist(new NormalDistribution(0, 1)).build()) + .layer(1, new GravesLSTM.Builder().nIn(10).nOut(11) + .dist(new NormalDistribution(0, 1)).build()) + .layer(2, new RnnOutputLayer.Builder(LossFunction.MSE) + .dist(new NormalDistribution(0, 1)).nIn(11).nOut(12).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray initParams = net.params().dup(); + Map initParams2 = net.paramTable(); + + net.setParams(net.params()); + + INDArray initParamsAfter = net.params(); + Map initParams2After = net.paramTable(); + + for (String s : initParams2.keySet()) { + assertTrue( initParams2.get(s).equals(initParams2After.get(s)), "Params differ: " + s); + } + + assertEquals(initParams, initParamsAfter); + + //Now, try the other way: get(set(random)) + INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape()); + net.setParams(randomParams.dup()); + + assertEquals(net.params(), randomParams); + } + + @Test + public void testInitWithParams() { + + Nd4j.getRandom().setSeed(12345); + + //Create configuration. Doesn't matter if this doesn't actually work for forward/backward pass here + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list() + .layer(0, new ConvolutionLayer.Builder().nIn(10).nOut(10).kernelSize(2, 2).stride(2, 2) + .padding(2, 2).build()) + .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(2, new GravesLSTM.Builder().nIn(10).nOut(10).build()) + .layer(3, new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build()) + .layer(4, new OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + INDArray params = net.params(); + + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf); + net2.init(params, true); + + MultiLayerNetwork net3 = new MultiLayerNetwork(conf); + net3.init(params, false); + + assertEquals(params, net2.params()); + assertEquals(params, net3.params()); + + assertFalse(params == net2.params()); //Different objects due to clone + assertTrue(params == net3.params()); //Same object due to clone + + + Map paramsMap = net.paramTable(); + Map paramsMap2 = net2.paramTable(); + Map paramsMap3 = net3.paramTable(); + for (String s : paramsMap.keySet()) { + assertEquals(paramsMap.get(s), paramsMap2.get(s)); + assertEquals(paramsMap.get(s), paramsMap3.get(s)); + } + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestVariableLengthTS.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestVariableLengthTS.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestVariableLengthTS.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestVariableLengthTS.java index 8cd027c56..5212865f6 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestVariableLengthTS.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/multilayer/TestVariableLengthTS.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.nn.workspace.ArrayType; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.util.TimeSeriesUtils; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.CustomOp; @@ -57,8 +54,7 @@ import java.util.Map; import java.util.Random; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestVariableLengthTS extends BaseDL4JTest { @Test @@ -128,7 +124,7 @@ public class TestVariableLengthTS extends BaseDL4JTest { for (String s : g1map.keySet()) { INDArray g1s = g1map.get(s); INDArray g2s = g2map.get(s); - assertEquals(g1s, g2s,s); + assertEquals(g1s, g2s, s); } //Finally: check that the values at the masked outputs don't actually make any differente to: @@ -146,7 +142,7 @@ public class TestVariableLengthTS extends BaseDL4JTest { for (String s : g2map.keySet()) { INDArray g2s = g2map.get(s); INDArray g2sa = g2a.getGradientFor(s); - assertEquals(g2s, g2sa,s); + assertEquals(g2s, g2sa, s); } } } @@ -235,7 +231,7 @@ public class TestVariableLengthTS extends BaseDL4JTest { // System.out.println("Variable: " + s); // System.out.println(Arrays.toString(g1s.dup().data().asFloat())); // System.out.println(Arrays.toString(g2s.dup().data().asFloat())); - assertNotEquals(g1s, g2s,s); + assertNotEquals( g1s, g2s, s); } //Modify the values at the masked time step, and check that neither the gradients, score or activations change @@ -335,7 +331,7 @@ public class TestVariableLengthTS extends BaseDL4JTest { mln.computeGradientAndScore(); double score = mln.score(); - assertEquals(expScore, score, 0.1,msg); + assertEquals(expScore, score, 0.1, msg); } } } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/rl/TestMultiModelGradientApplication.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/rl/TestMultiModelGradientApplication.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/rl/TestMultiModelGradientApplication.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/rl/TestMultiModelGradientApplication.java index edade0145..410abf970 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/rl/TestMultiModelGradientApplication.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/rl/TestMultiModelGradientApplication.java @@ -30,10 +30,7 @@ import org.deeplearning4j.nn.gradient.Gradient; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -46,8 +43,7 @@ import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestMultiModelGradientApplication extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestFrozenLayers.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestFrozenLayers.java new file mode 100644 index 000000000..e98680c51 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestFrozenLayers.java @@ -0,0 +1,202 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.transferlearning; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.layers.FrozenLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestFrozenLayers extends BaseDL4JTest { + + @Test + public void testFrozenMLN(){ + MultiLayerNetwork orig = getOriginalNet(12345); + + + for(double l1 : new double[]{0.0, 0.3}){ + for( double l2 : new double[]{0.0, 0.4}){ + String msg = "l1=" + l1 + ", l2=" + l2; + + FineTuneConfiguration ftc = new FineTuneConfiguration.Builder() + .updater(new Sgd(0.5)) + .l1(l1) + .l2(l2) + .build(); + + MultiLayerNetwork transfer = new TransferLearning.Builder(orig) + .fineTuneConfiguration(ftc) + .setFeatureExtractor(4) + .removeOutputLayer() + .addLayer(new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MEAN_ABSOLUTE_ERROR).build()) + .build(); + + assertEquals(6, transfer.getnLayers()); + for( int i=0; i<5; i++ ){ + assertTrue( transfer.getLayer(i) instanceof FrozenLayer); + } + + Map paramsBefore = new LinkedHashMap<>(); + for(Map.Entry entry : transfer.paramTable().entrySet()){ + paramsBefore.put(entry.getKey(), entry.getValue().dup()); + } + + for( int i=0; i<20; i++ ){ + INDArray f = Nd4j.rand(new int[]{16,1,28,28}); + INDArray l = Nd4j.rand(new int[]{16,10}); + transfer.fit(f,l); + } + + for(Map.Entry entry : transfer.paramTable().entrySet()){ + String s = msg + " - " + entry.getKey(); + if(entry.getKey().startsWith("5_")){ + //Non-frozen layer + assertNotEquals(paramsBefore.get(entry.getKey()), entry.getValue(),s); + } else { + assertEquals(paramsBefore.get(entry.getKey()), entry.getValue(),s); + } + } + } + } + } + + @Test + public void testFrozenCG(){ + ComputationGraph orig = getOriginalGraph(12345); + + + for(double l1 : new double[]{0.0, 0.3}){ + for( double l2 : new double[]{0.0, 0.4}){ + String msg = "l1=" + l1 + ", l2=" + l2; + + FineTuneConfiguration ftc = new FineTuneConfiguration.Builder() + .updater(new Sgd(0.5)) + .l1(l1) + .l2(l2) + .build(); + + ComputationGraph transfer = new TransferLearning.GraphBuilder(orig) + .fineTuneConfiguration(ftc) + .setFeatureExtractor("4") + .removeVertexAndConnections("5") + .addLayer("5", new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MEAN_ABSOLUTE_ERROR).build(), "4") + .setOutputs("5") + .build(); + + assertEquals(6, transfer.getNumLayers()); + for( int i=0; i<5; i++ ){ + assertTrue( transfer.getLayer(i) instanceof FrozenLayer); + } + + Map paramsBefore = new LinkedHashMap<>(); + for(Map.Entry entry : transfer.paramTable().entrySet()){ + paramsBefore.put(entry.getKey(), entry.getValue().dup()); + } + + for( int i=0; i<20; i++ ){ + INDArray f = Nd4j.rand(new int[]{16,1,28,28}); + INDArray l = Nd4j.rand(new int[]{16,10}); + transfer.fit(new INDArray[]{f},new INDArray[]{l}); + } + + for(Map.Entry entry : transfer.paramTable().entrySet()){ + String s = msg + " - " + entry.getKey(); + if(entry.getKey().startsWith("5_")){ + //Non-frozen layer + assertNotEquals(paramsBefore.get(entry.getKey()), entry.getValue(), s); + } else { + assertEquals( paramsBefore.get(entry.getKey()), entry.getValue(), s); + } + } + } + } + } + + public static MultiLayerNetwork getOriginalNet(int seed){ + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(seed) + .weightInit(WeightInit.XAVIER) + .activation(Activation.TANH) + .convolutionMode(ConvolutionMode.Same) + .updater(new Sgd(0.3)) + .list() + .layer(new ConvolutionLayer.Builder().nOut(3).kernelSize(2,2).stride(1,1).build()) + .layer(new SubsamplingLayer.Builder().kernelSize(2,2).stride(1,1).build()) + .layer(new ConvolutionLayer.Builder().nIn(3).nOut(3).kernelSize(2,2).stride(1,1).build()) + .layer(new DenseLayer.Builder().nOut(64).build()) + .layer(new DenseLayer.Builder().nIn(64).nOut(64).build()) + .layer(new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build()) + .setInputType(InputType.convolutionalFlat(28,28,1)) + .build(); + + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + return net; + } + + public static ComputationGraph getOriginalGraph(int seed){ + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(seed) + .weightInit(WeightInit.XAVIER) + .activation(Activation.TANH) + .convolutionMode(ConvolutionMode.Same) + .updater(new Sgd(0.3)) + .graphBuilder() + .addInputs("in") + .layer("0", new ConvolutionLayer.Builder().nOut(3).kernelSize(2,2).stride(1,1).build(), "in") + .layer("1", new SubsamplingLayer.Builder().kernelSize(2,2).stride(1,1).build(), "0") + .layer("2", new ConvolutionLayer.Builder().nIn(3).nOut(3).kernelSize(2,2).stride(1,1).build(), "1") + .layer("3", new DenseLayer.Builder().nOut(64).build(), "2") + .layer("4", new DenseLayer.Builder().nIn(64).nOut(64).build(), "3") + .layer("5", new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build(), "4") + .setOutputs("5") + .setInputTypes(InputType.convolutionalFlat(28,28,1)) + .build(); + + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + return net; + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestTransferLearningJson.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestTransferLearningJson.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestTransferLearningJson.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestTransferLearningJson.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestTransferLearningModelSerializer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestTransferLearningModelSerializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestTransferLearningModelSerializer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestTransferLearningModelSerializer.java diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java new file mode 100644 index 000000000..a81d96838 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java @@ -0,0 +1,669 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.transferlearning; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.constraint.UnitNormConstraint; +import org.deeplearning4j.nn.conf.distribution.ConstantDistribution; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.graph.AttentionVertex; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer; +import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.conf.weightnoise.DropConnect; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.weights.WeightInitDistribution; +import org.deeplearning4j.nn.weights.WeightInitXavier; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.RmsProp; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class TransferLearningCompGraphTest extends BaseDL4JTest { + + @Test + public void simpleFineTune() { + + long rng = 12345L; + DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); + //original conf + ComputationGraphConfiguration confToChange = new NeuralNetConfiguration.Builder().seed(rng) + .optimizationAlgo(OptimizationAlgorithm.LBFGS).updater(new Nesterovs(0.01, 0.99)) + .graphBuilder().addInputs("layer0In").setInputTypes(InputType.feedForward(4)) + .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In") + .addLayer("layer1", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "layer0") + .setOutputs("layer1").build(); + + //conf with learning parameters changed + ComputationGraphConfiguration expectedConf = new NeuralNetConfiguration.Builder().seed(rng) + .updater(new RmsProp(0.2)) + .graphBuilder().addInputs("layer0In") + .setInputTypes(InputType.feedForward(4)) + .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In") + .addLayer("layer1", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "layer0") + .setOutputs("layer1").build(); + ComputationGraph expectedModel = new ComputationGraph(expectedConf); + expectedModel.init(); + + ComputationGraph modelToFineTune = new ComputationGraph(expectedConf); + modelToFineTune.init(); + modelToFineTune.setParams(expectedModel.params()); + //model after applying changes with transfer learning + ComputationGraph modelNow = + new TransferLearning.GraphBuilder(modelToFineTune) + .fineTuneConfiguration(new FineTuneConfiguration.Builder().seed(rng) + .updater(new RmsProp(0.2)).build()) + .build(); + + //Check json + assertEquals(expectedConf.toJson(), modelNow.getConfiguration().toJson()); + + //Check params after fit + modelNow.fit(randomData); + expectedModel.fit(randomData); + assertEquals(modelNow.score(), expectedModel.score(), 1e-8); + assertEquals(modelNow.params(), expectedModel.params()); + } + + @Test + public void testNoutChanges() { + DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 2)); + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.IDENTITY); + FineTuneConfiguration fineTuneConfiguration = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.IDENTITY).build(); + + ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "layer0In") + .addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0") + .addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1") + .addLayer("layer3", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "layer2") + .setOutputs("layer3").build()); + modelToFineTune.init(); + ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune) + .fineTuneConfiguration(fineTuneConfiguration).nOutReplace("layer3", 2, WeightInit.XAVIER) + .nOutReplace("layer0", 3, new NormalDistribution(1, 1e-1), WeightInit.XAVIER) + //.setOutputs("layer3") + .build(); + + BaseLayer bl0 = ((BaseLayer) modelNow.getLayer("layer0").conf().getLayer()); + BaseLayer bl1 = ((BaseLayer) modelNow.getLayer("layer1").conf().getLayer()); + BaseLayer bl3 = ((BaseLayer) modelNow.getLayer("layer3").conf().getLayer()); + assertEquals(bl0.getWeightInitFn(), new WeightInitDistribution(new NormalDistribution(1, 1e-1))); + assertEquals(bl1.getWeightInitFn(), new WeightInitXavier()); + assertEquals(bl1.getWeightInitFn(), new WeightInitXavier()); + + ComputationGraph modelExpectedArch = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In") + .addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0") + .addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1") + .addLayer("layer3", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(2) + .build(), + "layer2") + .setOutputs("layer3").build()); + + modelExpectedArch.init(); + + //modelNow should have the same architecture as modelExpectedArch + assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); + assertArrayEquals(modelExpectedArch.getLayer("layer0").params().shape(), + modelNow.getLayer("layer0").params().shape()); + assertArrayEquals(modelExpectedArch.getLayer("layer1").params().shape(), + modelNow.getLayer("layer1").params().shape()); + assertArrayEquals(modelExpectedArch.getLayer("layer2").params().shape(), + modelNow.getLayer("layer2").params().shape()); + assertArrayEquals(modelExpectedArch.getLayer("layer3").params().shape(), + modelNow.getLayer("layer3").params().shape()); + + modelNow.setParams(modelExpectedArch.params()); + //fit should give the same results + modelExpectedArch.fit(randomData); + modelNow.fit(randomData); + assertEquals(modelExpectedArch.score(), modelNow.score(), 1e-8); + assertEquals(modelExpectedArch.params(), modelNow.params()); + } + + @Test + public void testRemoveAndAdd() { + DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.IDENTITY); + FineTuneConfiguration fineTuneConfiguration = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.IDENTITY).build(); + + ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "layer0In") + .addLayer("layer1", new DenseLayer.Builder().nIn(5).nOut(2).build(), "layer0") + .addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1") + .addLayer("layer3", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "layer2") + .setOutputs("layer3").build()); + modelToFineTune.init(); + + ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune) + .fineTuneConfiguration(fineTuneConfiguration) + .nOutReplace("layer0", 7, WeightInit.XAVIER, WeightInit.XAVIER) + .nOutReplace("layer2", 5, WeightInit.XAVIER).removeVertexKeepConnections("layer3") + .addLayer("layer3", + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(3) + .activation(Activation.SOFTMAX).build(), + "layer2") + //.setOutputs("layer3") + .build(); + + ComputationGraph modelExpectedArch = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(7).build(), "layer0In") + .addLayer("layer1", new DenseLayer.Builder().nIn(7).nOut(2).build(), "layer0") + .addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(5).build(), "layer1") + .addLayer("layer3", + new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(5).nOut(3) + .build(), + "layer2") + .setOutputs("layer3").build()); + + modelExpectedArch.init(); + + //modelNow should have the same architecture as modelExpectedArch + assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); + assertArrayEquals(modelExpectedArch.getLayer("layer0").params().shape(), + modelNow.getLayer("layer0").params().shape()); + assertArrayEquals(modelExpectedArch.getLayer("layer1").params().shape(), + modelNow.getLayer("layer1").params().shape()); + assertArrayEquals(modelExpectedArch.getLayer("layer2").params().shape(), + modelNow.getLayer("layer2").params().shape()); + assertArrayEquals(modelExpectedArch.getLayer("layer3").params().shape(), + modelNow.getLayer("layer3").params().shape()); + + modelNow.setParams(modelExpectedArch.params()); + //fit should give the same results + modelExpectedArch.fit(randomData); + modelNow.fit(randomData); + assertEquals(modelExpectedArch.score(), modelNow.score(), 1e-8); + assertEquals(modelExpectedArch.params(), modelNow.params()); + } + + @Test + public void testAllWithCNN() { + + DataSet randomData = new DataSet(Nd4j.rand(10, 28 * 28 * 3).reshape(10, 3, 28, 28), Nd4j.rand(10, 10)); + ComputationGraph modelToFineTune = + new ComputationGraph( + new NeuralNetConfiguration.Builder().seed(123) + .weightInit(WeightInit.XAVIER) + .updater(new Nesterovs(0.01, 0.9)).graphBuilder() + .addInputs("layer0In") + .setInputTypes(InputType.convolutionalFlat(28, 28, + 3)) + .addLayer("layer0", + new ConvolutionLayer.Builder(5, 5).nIn(3) + .stride(1, 1).nOut(20) + .activation(Activation.IDENTITY) + .build(), + "layer0In") + .addLayer("layer1", + new SubsamplingLayer.Builder( + SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2) + .stride(2, 2) + .build(), + "layer0") + .addLayer("layer2", + new ConvolutionLayer.Builder(5, 5).stride(1, 1) + .nOut(50) + .activation(Activation.IDENTITY) + .build(), + "layer1") + .addLayer("layer3", + new SubsamplingLayer.Builder( + SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2) + .stride(2, 2) + .build(), + "layer2") + .addLayer("layer4", + new DenseLayer.Builder() + .activation(Activation.RELU) + .nOut(500).build(), + "layer3") + .addLayer("layer5", + new DenseLayer.Builder() + .activation(Activation.RELU) + .nOut(250).build(), + "layer4") + .addLayer("layer6", + new OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(100) + .activation(Activation.SOFTMAX) + .build(), + "layer5") + .setOutputs("layer6").build()); + modelToFineTune.init(); + + //this will override the learning configuration set in the model + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().seed(456).updater(new Sgd(0.001)); + FineTuneConfiguration fineTuneConfiguration = new FineTuneConfiguration.Builder().seed(456).updater(new Sgd(0.001)) + .build(); + + ComputationGraph modelNow = + new TransferLearning.GraphBuilder(modelToFineTune).fineTuneConfiguration(fineTuneConfiguration) + .setFeatureExtractor("layer1").nOutReplace("layer4", 600, WeightInit.XAVIER) + .removeVertexAndConnections("layer5").removeVertexAndConnections("layer6") + .setInputs("layer0In").setInputTypes(InputType.convolutionalFlat(28, 28, 3)) + .addLayer("layer5", + new DenseLayer.Builder().activation(Activation.RELU).nIn(600) + .nOut(300).build(), + "layer4") + .addLayer("layer6", + new DenseLayer.Builder().activation(Activation.RELU).nIn(300) + .nOut(150).build(), + "layer5") + .addLayer("layer7", + new DenseLayer.Builder().activation(Activation.RELU).nIn(150) + .nOut(50).build(), + "layer6") + .addLayer("layer8", + new OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .activation(Activation.SOFTMAX) + .nIn(50).nOut(10).build(), + "layer7") + .setOutputs("layer8").build(); + + ComputationGraph modelExpectedArch = + new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In") + .setInputTypes(InputType.convolutionalFlat(28,28, 3)) + .addLayer("layer0", + new FrozenLayer(new ConvolutionLayer.Builder(5, 5).nIn(3) + .stride(1, 1).nOut(20) + .activation(Activation.IDENTITY).build()), + "layer0In") + .addLayer("layer1", + new FrozenLayer(new SubsamplingLayer.Builder( + SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2) + .build()), + "layer0") + .addLayer("layer2", + new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50) + .activation(Activation.IDENTITY).build(), + "layer1") + .addLayer("layer3", + new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2).build(), + "layer2") + .addLayer("layer4", + new DenseLayer.Builder().activation(Activation.RELU).nOut(600) + .build(), + "layer3") + .addLayer("layer5", + new DenseLayer.Builder().activation(Activation.RELU).nOut(300) + .build(), + "layer4") + .addLayer("layer6", + new DenseLayer.Builder().activation(Activation.RELU).nOut(150) + .build(), + "layer5") + .addLayer("layer7", + new DenseLayer.Builder().activation(Activation.RELU).nOut(50) + .build(), + "layer6") + .addLayer("layer8", + new OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(10) + .activation(Activation.SOFTMAX) + .build(), + "layer7") + .setOutputs("layer8").build()); + modelExpectedArch.init(); + modelExpectedArch.getVertex("layer0").setLayerAsFrozen(); + modelExpectedArch.getVertex("layer1").setLayerAsFrozen(); + + assertEquals(modelExpectedArch.getConfiguration().toJson(), modelNow.getConfiguration().toJson()); + + modelNow.setParams(modelExpectedArch.params()); + int i = 0; + while (i < 5) { + modelExpectedArch.fit(randomData); + modelNow.fit(randomData); + i++; + } + assertEquals(modelExpectedArch.params(), modelNow.params()); + + } + + + @Test + public void testTransferGlobalPool() { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new Adam(0.1)) + .weightInit(WeightInit.XAVIER) + .graphBuilder().addInputs("in") + .addLayer("blstm1",new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10) + .activation(Activation.TANH).build(), + "in") + .addLayer("pool", new GlobalPoolingLayer.Builder().build(), "blstm1") + .addLayer("dense", new DenseLayer.Builder().nIn(10).nOut(10).build(), "pool") + .addLayer("out", new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.IDENTITY) + .lossFunction(LossFunctions.LossFunction.MSE).build(), "dense") + .setOutputs("out").build(); + + ComputationGraph g = new ComputationGraph(conf); + g.init(); + + FineTuneConfiguration fineTuneConfiguration = + new FineTuneConfiguration.Builder().seed(12345).updater(new Sgd(0.01)).build(); + + ComputationGraph graph = new TransferLearning.GraphBuilder(g).fineTuneConfiguration(fineTuneConfiguration) + .removeVertexKeepConnections("out").setFeatureExtractor("dense") + .addLayer("out", new OutputLayer.Builder().updater(new Adam(0.1)) + .weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT) + .nIn(10).nOut(5).build(), "dense") + .build(); + + ComputationGraphConfiguration confExpected = new NeuralNetConfiguration.Builder().seed(12345) + .updater(new Sgd(0.01)) + .weightInit(WeightInit.XAVIER) + .graphBuilder().addInputs("in") + .addLayer("blstm1", + new FrozenLayer(new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10) + .activation(Activation.TANH).build()), + "in") + .addLayer("pool", new FrozenLayer(new GlobalPoolingLayer.Builder().build()), "blstm1") + .addLayer("dense", new FrozenLayer(new DenseLayer.Builder().nIn(10).nOut(10).build()), "pool") + .addLayer("out", new OutputLayer.Builder().nIn(10).nOut(5).activation(Activation.SOFTMAX) + .updater(new Adam(0.1)) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "dense") + .setOutputs("out").build(); + + ComputationGraph modelExpected = new ComputationGraph(confExpected); + modelExpected.init(); + + +// assertEquals(confExpected, graph.getConfiguration()); + assertEquals(confExpected.toJson(), graph.getConfiguration().toJson()); + } + + + @Test + public void testObjectOverrides(){ + //https://github.com/deeplearning4j/deeplearning4j/issues/4368 + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .dropOut(0.5) + .weightNoise(new DropConnect(0.5)) + .l2(0.5) + .constrainWeights(new UnitNormConstraint()) + .graphBuilder() + .addInputs("in") + .addLayer("layer", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in") + .setOutputs("layer") + .build(); + + ComputationGraph orig = new ComputationGraph(conf); + orig.init(); + + FineTuneConfiguration ftc = new FineTuneConfiguration.Builder() + .dropOut(0) + .weightNoise(null) + .constraints(null) + .l2(0.0) + .build(); + + ComputationGraph transfer = new TransferLearning.GraphBuilder(orig) + .fineTuneConfiguration(ftc) + .build(); + + DenseLayer l = (DenseLayer) transfer.getLayer(0).conf().getLayer(); + + assertNull(l.getIDropout()); + assertNull(l.getWeightNoise()); + assertNull(l.getConstraints()); + assertNull(TestUtils.getL2Reg(l)); + } + + + @Test + public void testTransferLearningSubsequent() { + String inputName = "in"; + String outputName = "out"; + + final String firstConv = "firstConv"; + final String secondConv = "secondConv"; + final INDArray input = Nd4j.create(6,6,6,6); + final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder() + .weightInit(new ConstantDistribution(666)) + .graphBuilder() + .addInputs(inputName) + .setOutputs(outputName) + .setInputTypes(InputType.inferInputTypes(input)) + .addLayer(firstConv, new Convolution2D.Builder(3, 3) + .nOut(10) + .build(), inputName) + .addLayer(secondConv, new Convolution2D.Builder(1, 1) + .nOut(3) + .build(), firstConv) + .addLayer(outputName, new OutputLayer.Builder() + .nOut(2) + .lossFunction(LossFunctions.LossFunction.MSE) + .build(), secondConv) + .build()); + graph.init(); + + final ComputationGraph newGraph = new TransferLearning + .GraphBuilder(graph) + .nOutReplace(firstConv, 7, new ConstantDistribution(333)) + .nOutReplace(secondConv, 3, new ConstantDistribution(111)) + .removeVertexAndConnections(outputName) + .addLayer(outputName, new OutputLayer.Builder() + .nIn(48).nOut(2) + .lossFunction(LossFunctions.LossFunction.MSE) + .build(), new CnnToFeedForwardPreProcessor(4,4,3), secondConv) + .setOutputs(outputName) + .build(); + newGraph.init(); + + assertEquals( 7, newGraph.layerInputSize(secondConv), "Incorrect # inputs"); + + newGraph.outputSingle(input); + } + + + + @Test + public void testChangeNOutNIn() { + final String inputName = "input"; + final String changeNoutName = "changeNout"; + final String poolName = "pool"; + final String afterPoolName = "afterPool"; + final String outputName = "output"; + final INDArray input = Nd4j.create(new long[] {1, 2, 4, 4}); + final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder() + .graphBuilder() + .addInputs(inputName) + .setOutputs(outputName) + .setInputTypes(InputType.inferInputTypes(input)) + .addLayer(changeNoutName, new Convolution2D.Builder(1, 1) + .nOut(10) + .build(), inputName) + .addLayer(poolName, new SubsamplingLayer.Builder(1,1).build(), changeNoutName) + .addLayer(afterPoolName, new Convolution2D.Builder(1, 1) + .nOut(7) + .build(), poolName) + .addLayer(outputName, new OutputLayer.Builder() + .activation(Activation.SOFTMAX) + .nOut(2) + .build(), afterPoolName) + .build()); + graph.init(); + + final ComputationGraph newGraph = new TransferLearning.GraphBuilder(graph) + .nOutReplace(changeNoutName, 5, WeightInit.XAVIER) + .nInReplace(afterPoolName, 5, WeightInit.XAVIER) + .build(); + + newGraph.init(); + + assertEquals( 5 , newGraph.layerSize(changeNoutName), "Incorrect number of outputs!"); + assertEquals( 5, newGraph.layerInputSize(afterPoolName), "Incorrect number of inputs!"); + newGraph.output(input); + } + + + + + @Test + public void testTransferLearningSameDiffLayersGraph(){ + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + + .graphBuilder() + .addInputs("in") + .layer("l0", new LSTM.Builder().nIn(5).nOut(5).build(), "in") + .layer("l1", new RecurrentAttentionLayer.Builder().nHeads(1).headSize(5).nIn(5).nOut(5).build(), "l0") + .layer("out", new RnnOutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).build(), "l1") + .setOutputs("out") + .build(); + + ComputationGraph cg = new ComputationGraph(conf); + cg.init(); + + INDArray arr = Nd4j.rand(DataType.FLOAT, 2, 5, 10); + INDArray out = cg.output(arr)[0]; + + + ComputationGraph cg2 = new TransferLearning.GraphBuilder(cg).removeVertexAndConnections("out") + .fineTuneConfiguration(FineTuneConfiguration.builder().updater(new Adam(0.01)).build()) + .removeVertexAndConnections("out") + .addLayer("newOut", new RnnOutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).build(), "l1") + .setOutputs("newOut") + .build(); + + cg2.output(arr); + + Map m = new HashMap<>(cg.paramTable()); + m.put("newOut_W", m.remove("out_W")); + m.put("newOut_b", m.remove("out_b")); + cg2.setParamTable(m); + + Map p1 = cg.paramTable(); + Map p2 = cg2.paramTable(); + for(String s : p1.keySet()){ + INDArray i1 = p1.get(s); + INDArray i2 = p2.get(s.replaceAll("out", "newOut")); + assertEquals( i1, i2, s); + } + + INDArray out2 = cg2.outputSingle(arr); + assertEquals(out, out2); + } + + @Test + public void testTransferLearningSameDiffLayersGraphVertex(){ + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + + .graphBuilder() + .addInputs("in") + .layer("l0", new LSTM.Builder().nIn(5).nOut(5).build(), "in") + .addVertex("l1", new AttentionVertex.Builder().nHeads(1).headSize(5).nInKeys(5).nInQueries(5).nInValues(5).nOut(5).build(), "l0", "l0", "l0") + .layer("out", new RnnOutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).build(), "l1") + .setOutputs("out") + .build(); + + ComputationGraph cg = new ComputationGraph(conf); + cg.init(); + + INDArray arr = Nd4j.rand(DataType.FLOAT, 2, 5, 10); + INDArray out = cg.output(arr)[0]; + + + ComputationGraph cg2 = new TransferLearning.GraphBuilder(cg).removeVertexAndConnections("out") + .fineTuneConfiguration(FineTuneConfiguration.builder().updater(new Adam(0.01)).build()) + .removeVertexAndConnections("out") + .addLayer("newOut", new RnnOutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).build(), "l1") + .setOutputs("newOut") + .build(); + + cg2.output(arr); + + Map m = new HashMap<>(cg.paramTable()); + m.put("newOut_W", m.remove("out_W")); + m.put("newOut_b", m.remove("out_b")); + cg2.setParamTable(m); + + Map p1 = cg.paramTable(); + Map p2 = cg2.paramTable(); + for(String s : p1.keySet()){ + INDArray i1 = p1.get(s); + INDArray i2 = p2.get(s.replaceAll("out", "newOut")); + assertEquals(i1, i2, s); + } + + INDArray out2 = cg2.outputSingle(arr); + assertEquals(out, out2); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java index 6e9851ab6..d30227339 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningComplex.java @@ -94,7 +94,7 @@ public class TransferLearningComplex extends BaseDL4JTest { cFound = true; assertTrue(l instanceof FrozenLayer, name); } else { - assertFalse(l instanceof FrozenLayer, name); + assertFalse( l instanceof FrozenLayer, name); } //Also check config: diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelperTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelperTest.java new file mode 100644 index 000000000..0e78a3d6c --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelperTest.java @@ -0,0 +1,252 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.transferlearning; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.graph.MergeVertex; +import org.deeplearning4j.nn.conf.graph.SubsetVertex; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +public class TransferLearningHelperTest extends BaseDL4JTest { + + @Test + public void tesUnfrozenSubset() { + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().seed(124) + .activation(Activation.IDENTITY) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)); + /* + (inCentre) (inRight) + | | + denseCentre0 | + | | + ,-------- denseCentre1 denseRight0 + / | | + subsetLeft(0-3) denseCentre2 ---- denseRight ---- mergeRight + | | | + denseLeft0 denseCentre3 denseRight1 + | | | + (outLeft) (outCentre) (outRight) + + */ + + ComputationGraphConfiguration conf = overallConf.graphBuilder().addInputs("inCentre", "inRight") + .addLayer("denseCentre0", new DenseLayer.Builder().nIn(10).nOut(9).build(), "inCentre") + .addLayer("denseCentre1", new DenseLayer.Builder().nIn(9).nOut(8).build(), "denseCentre0") + .addLayer("denseCentre2", new DenseLayer.Builder().nIn(8).nOut(7).build(), "denseCentre1") + .addLayer("denseCentre3", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2") + .addLayer("outCentre", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(7).nOut(4).build(), + "denseCentre3") + .addVertex("subsetLeft", new SubsetVertex(0, 3), "denseCentre1") + .addLayer("denseLeft0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "subsetLeft") + .addLayer("outLeft", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(6).build(), + "denseLeft0") + .addLayer("denseRight", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2") + .addLayer("denseRight0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "inRight") + .addVertex("mergeRight", new MergeVertex(), "denseRight", "denseRight0") + .addLayer("denseRight1", new DenseLayer.Builder().nIn(10).nOut(5).build(), "mergeRight") + .addLayer("outRight", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(5).build(), + "denseRight1") + .setOutputs("outLeft", "outCentre", "outRight").build(); + + ComputationGraph modelToTune = new ComputationGraph(conf); + modelToTune.init(); + + TransferLearningHelper helper = new TransferLearningHelper(modelToTune, "denseCentre2"); + + ComputationGraph modelSubset = helper.unfrozenGraph(); + + ComputationGraphConfiguration expectedConf = + overallConf.graphBuilder().addInputs("denseCentre1", "denseCentre2", "inRight") //inputs are in sorted order + .addLayer("denseCentre3", new DenseLayer.Builder().nIn(7).nOut(7).build(), + "denseCentre2") + .addLayer("outCentre", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(7) + .nOut(4).build(), + "denseCentre3") + .addVertex("subsetLeft", new SubsetVertex(0, 3), "denseCentre1") + .addLayer("denseLeft0", new DenseLayer.Builder().nIn(4).nOut(5).build(), + "subsetLeft") + .addLayer("outLeft", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5) + .nOut(6).build(), + "denseLeft0") + .addLayer("denseRight", new DenseLayer.Builder().nIn(7).nOut(7).build(), + "denseCentre2") + .addLayer("denseRight0", new DenseLayer.Builder().nIn(2).nOut(3).build(), + "inRight") + .addVertex("mergeRight", new MergeVertex(), "denseRight", "denseRight0") + .addLayer("denseRight1", new DenseLayer.Builder().nIn(10).nOut(5).build(), + "mergeRight") + .addLayer("outRight", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5) + .nOut(5).build(), + "denseRight1") + .setOutputs("outLeft", "outCentre", "outRight").build(); + ComputationGraph expectedModel = new ComputationGraph(expectedConf); + expectedModel.init(); + assertEquals(expectedConf.toJson(), modelSubset.getConfiguration().toJson()); + } + + @Test + public void testFitUnFrozen() { + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.9)).seed(124) + .activation(Activation.IDENTITY) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT); + + ComputationGraphConfiguration conf = overallConf.graphBuilder().addInputs("inCentre", "inRight") + .addLayer("denseCentre0", new DenseLayer.Builder().nIn(10).nOut(9).build(), "inCentre") + .addLayer("denseCentre1", new DenseLayer.Builder().nIn(9).nOut(8).build(), "denseCentre0") + .addLayer("denseCentre2", new DenseLayer.Builder().nIn(8).nOut(7).build(), "denseCentre1") + .addLayer("denseCentre3", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2") + .addLayer("outCentre", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(7).nOut(4).build(), + "denseCentre3") + .addVertex("subsetLeft", new SubsetVertex(0, 3), "denseCentre1") + .addLayer("denseLeft0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "subsetLeft") + .addLayer("outLeft", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(6).build(), + "denseLeft0") + .addLayer("denseRight", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2") + .addLayer("denseRight0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "inRight") + .addVertex("mergeRight", new MergeVertex(), "denseRight", "denseRight0") + .addLayer("denseRight1", new DenseLayer.Builder().nIn(10).nOut(5).build(), "mergeRight") + .addLayer("outRight", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(5).build(), + "denseRight1") + .setOutputs("outLeft", "outCentre", "outRight").build(); + + ComputationGraph modelToTune = new ComputationGraph(conf); + modelToTune.init(); + + INDArray inRight = Nd4j.rand(10, 2); + INDArray inCentre = Nd4j.rand(10, 10); + INDArray outLeft = Nd4j.rand(10, 6); + INDArray outRight = Nd4j.rand(10, 5); + INDArray outCentre = Nd4j.rand(10, 4); + MultiDataSet origData = new MultiDataSet(new INDArray[] {inCentre, inRight}, + new INDArray[] {outLeft, outCentre, outRight}); + ComputationGraph modelIdentical = modelToTune.clone(); + modelIdentical.getVertex("denseCentre0").setLayerAsFrozen(); + modelIdentical.getVertex("denseCentre1").setLayerAsFrozen(); + modelIdentical.getVertex("denseCentre2").setLayerAsFrozen(); + + TransferLearningHelper helper = new TransferLearningHelper(modelToTune, "denseCentre2"); + MultiDataSet featurizedDataSet = helper.featurize(origData); + + assertEquals(modelIdentical.getLayer("denseRight0").params(), modelToTune.getLayer("denseRight0").params()); + modelIdentical.fit(origData); + helper.fitFeaturized(featurizedDataSet); + + assertEquals(modelIdentical.getLayer("denseCentre0").params(), modelToTune.getLayer("denseCentre0").params()); + assertEquals(modelIdentical.getLayer("denseCentre1").params(), modelToTune.getLayer("denseCentre1").params()); + assertEquals(modelIdentical.getLayer("denseCentre2").params(), modelToTune.getLayer("denseCentre2").params()); + assertEquals(modelIdentical.getLayer("denseCentre3").params(), modelToTune.getLayer("denseCentre3").params()); + assertEquals(modelIdentical.getLayer("outCentre").params(), modelToTune.getLayer("outCentre").params()); + assertEquals(modelIdentical.getLayer("denseRight").conf().toJson(), + modelToTune.getLayer("denseRight").conf().toJson()); + assertEquals(modelIdentical.getLayer("denseRight").params(), modelToTune.getLayer("denseRight").params()); + assertEquals(modelIdentical.getLayer("denseRight0").conf().toJson(), + modelToTune.getLayer("denseRight0").conf().toJson()); + //assertEquals(modelIdentical.getLayer("denseRight0").params(),modelToTune.getLayer("denseRight0").params()); + assertEquals(modelIdentical.getLayer("denseRight1").params(), modelToTune.getLayer("denseRight1").params()); + assertEquals(modelIdentical.getLayer("outRight").params(), modelToTune.getLayer("outRight").params()); + assertEquals(modelIdentical.getLayer("denseLeft0").params(), modelToTune.getLayer("denseLeft0").params()); + assertEquals(modelIdentical.getLayer("outLeft").params(), modelToTune.getLayer("outLeft").params()); + +// log.info(modelIdentical.summary()); +// log.info(helper.unfrozenGraph().summary()); + modelIdentical.summary(); + helper.unfrozenGraph().summary(); + } + + @Test + public void testMLN() { + DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .activation(Activation.IDENTITY); + + MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(overallConf.clone().list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()) + .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build()); + + modelToFineTune.init(); + MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).setFeatureExtractor(1).build(); + List ff = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false); + INDArray asFrozenFeatures = ff.get(2); + + TransferLearningHelper helper = new TransferLearningHelper(modelToFineTune, 1); + + INDArray paramsLastTwoLayers = + Nd4j.hstack(modelToFineTune.getLayer(2).params(), modelToFineTune.getLayer(3).params()); + MultiLayerNetwork notFrozen = new MultiLayerNetwork(overallConf.clone().list() + .layer(0, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build(), paramsLastTwoLayers); + + assertEquals(asFrozenFeatures, helper.featurize(randomData).getFeatures()); + assertEquals(randomData.getLabels(), helper.featurize(randomData).getLabels()); + + for (int i = 0; i < 5; i++) { + notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); + helper.fitFeaturized(helper.featurize(randomData)); + modelNow.fit(randomData); + } + + INDArray expected = Nd4j.hstack(modelToFineTune.getLayer(0).params(), modelToFineTune.getLayer(1).params(), + notFrozen.params()); + INDArray act = modelNow.params(); + assertEquals(expected, act); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java new file mode 100644 index 000000000..005f2158c --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java @@ -0,0 +1,748 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.transferlearning; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.TestUtils; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.BackpropType; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.constraint.UnitNormConstraint; +import org.deeplearning4j.nn.conf.distribution.ConstantDistribution; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.RnnToCnnPreProcessor; +import org.deeplearning4j.nn.conf.serde.JsonMappers; +import org.deeplearning4j.nn.conf.weightnoise.DropConnect; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.weights.WeightInitDistribution; +import org.deeplearning4j.nn.weights.WeightInitRelu; +import org.deeplearning4j.nn.weights.WeightInitXavier; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.*; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import com.fasterxml.jackson.core.JsonProcessingException; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +public class TransferLearningMLNTest extends BaseDL4JTest { + + @Test + public void simpleFineTune() { + + long rng = 12345L; + Nd4j.getRandom().setSeed(rng); + DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 3)); + //original conf + NeuralNetConfiguration.Builder confToChange = + new NeuralNetConfiguration.Builder().seed(rng).optimizationAlgo(OptimizationAlgorithm.LBFGS) + .updater(new Nesterovs(0.01, 0.99)); + + MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(confToChange.list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build()); + modelToFineTune.init(); + + //model after applying changes with transfer learning + MultiLayerNetwork modelNow = + new TransferLearning.Builder(modelToFineTune) + .fineTuneConfiguration(new FineTuneConfiguration.Builder().seed(rng) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new RmsProp(0.5)) //Intent: override both weight and bias LR, unless bias LR is manually set also + .l2(0.4).build()) + .build(); + + for (org.deeplearning4j.nn.api.Layer l : modelNow.getLayers()) { + BaseLayer bl = ((BaseLayer) l.conf().getLayer()); + assertEquals(new RmsProp(0.5), bl.getIUpdater()); + } + + + NeuralNetConfiguration.Builder confSet = new NeuralNetConfiguration.Builder().seed(rng) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new RmsProp(0.5)).l2(0.4); + + MultiLayerNetwork expectedModel = new MultiLayerNetwork(confSet.list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build()); + expectedModel.init(); + expectedModel.setParams(modelToFineTune.params().dup()); + + assertEquals(expectedModel.params(), modelNow.params()); + + //Check json + MultiLayerConfiguration expectedConf = expectedModel.getLayerWiseConfigurations(); + assertEquals(expectedConf.toJson(), modelNow.getLayerWiseConfigurations().toJson()); + + //Check params after fit + modelNow.fit(randomData); + expectedModel.fit(randomData); + + assertEquals(modelNow.score(), expectedModel.score(), 1e-6); + INDArray pExp = expectedModel.params(); + INDArray pNow = modelNow.params(); + assertEquals(pExp, pNow); + } + + @Test + public void testNoutChanges() { + Nd4j.getRandom().setSeed(12345); + DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT,10, 2)); + + NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)); + FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)) + .build(); + + MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(equivalentConf.list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(5).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()) + .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build()); + modelToFineTune.init(); + MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf) + .nOutReplace(3, 2, WeightInit.XAVIER, WeightInit.XAVIER) + .nOutReplace(0, 3, WeightInit.XAVIER, new NormalDistribution(1, 1e-1)).build(); + + MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(equivalentConf.list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()) + .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(2) + .build()) + .build()); + modelExpectedArch.init(); + + //Will fail - expected because of dist and weight init changes + //assertEquals(modelExpectedArch.getLayerWiseConfigurations().toJson(), modelNow.getLayerWiseConfigurations().toJson()); + + BaseLayer bl0 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(0).getLayer()); + BaseLayer bl1 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(1).getLayer()); + BaseLayer bl3 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(3).getLayer()); + assertEquals(bl0.getWeightInitFn().getClass(), WeightInitXavier.class); + try { + assertEquals(JsonMappers.getMapper().writeValueAsString(bl1.getWeightInitFn()), + JsonMappers.getMapper().writeValueAsString(new WeightInitDistribution(new NormalDistribution(1, 1e-1)))); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + assertEquals(bl3.getWeightInitFn(), new WeightInitXavier()); + + //modelNow should have the same architecture as modelExpectedArch + assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape()); + + modelNow.setParams(modelExpectedArch.params()); + //fit should give the same results + modelExpectedArch.fit(randomData); + modelNow.fit(randomData); + assertEquals(modelExpectedArch.score(), modelNow.score(), 0.000001); + assertEquals(modelExpectedArch.params(), modelNow.params()); + } + + + @Test + public void testRemoveAndAdd() { + Nd4j.getRandom().setSeed(12345); + DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT,10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 3)); + + NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)); + FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build(); + + MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(//overallConf.list() + equivalentConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(5).build()) + .layer(1, new DenseLayer.Builder().nIn(5).nOut(2).build()) + .layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build()); + modelToFineTune.init(); + + MultiLayerNetwork modelNow = + new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf) + .nOutReplace(0, 7, WeightInit.XAVIER, WeightInit.XAVIER) + .nOutReplace(2, 5, WeightInit.XAVIER).removeOutputLayer() + .addLayer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5) + .nOut(3).updater(new Sgd(0.5)).activation(Activation.SOFTMAX) + .build()) + .build(); + + MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(equivalentConf.list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(7).build()) + .layer(1, new DenseLayer.Builder().nIn(7).nOut(2).build()) + .layer(2, new DenseLayer.Builder().nIn(2).nOut(5).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX) + .updater(new Sgd(0.5)).nIn(5).nOut(3).build()) + .build()); + modelExpectedArch.init(); + + //modelNow should have the same architecture as modelExpectedArch + assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape()); + + modelNow.setParams(modelExpectedArch.params()); + //fit should give the same results + modelExpectedArch.fit(randomData); + modelNow.fit(randomData); + double scoreExpected = modelExpectedArch.score(); + double scoreActual = modelNow.score(); + assertEquals(scoreExpected, scoreActual, 1e-4); + assertEquals(modelExpectedArch.params(), modelNow.params()); + } + + @Test + public void testRemoveAndProcessing() { + + int V_WIDTH = 130; + int V_HEIGHT = 130; + int V_NFRAMES = 150; + + MultiLayerConfiguration confForArchitecture = + new NeuralNetConfiguration.Builder().seed(12345).l2(0.001) //l2 regularization on all layers + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new AdaGrad(0.4)).list() + .layer(0, new ConvolutionLayer.Builder(10, 10).nIn(3) //3 channels: RGB + .nOut(30).stride(4, 4).activation(Activation.RELU).weightInit( + WeightInit.RELU).build()) //Output: (130-10+0)/4+1 = 31 -> 31*31*30 + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) + .kernelSize(3, 3).stride(2, 2).build()) //(31-3+0)/2+1 = 15 + .layer(2, new ConvolutionLayer.Builder(3, 3).nIn(30).nOut(10).stride(2, 2) + .activation(Activation.RELU).weightInit(WeightInit.RELU) + .build()) //Output: (15-3+0)/2+1 = 7 -> 7*7*10 = 490 + .layer(3, new DenseLayer.Builder().activation(Activation.RELU).nIn(490).nOut(50) + .weightInit(WeightInit.RELU).updater(new AdaGrad(0.5)) + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).build()) + .layer(4, new GravesLSTM.Builder().activation(Activation.SOFTSIGN).nIn(50) + .nOut(50).weightInit(WeightInit.XAVIER).updater(new AdaGrad(0.6)) + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).build()) + .layer(5, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(50).nOut(4) //4 possible shapes: circle, square, arc, line + .weightInit(WeightInit.XAVIER) + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).build()) + .inputPreProcessor(0, new RnnToCnnPreProcessor(V_HEIGHT, V_WIDTH, 3)) + .inputPreProcessor(3, new CnnToFeedForwardPreProcessor(7, 7, 10)) + .inputPreProcessor(4, new FeedForwardToRnnPreProcessor()) + .backpropType(BackpropType.TruncatedBPTT) + .tBPTTForwardLength(V_NFRAMES / 5).tBPTTBackwardLength(V_NFRAMES / 5).build(); + MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(confForArchitecture); + modelExpectedArch.init(); + + MultiLayerNetwork modelToTweak = + new MultiLayerNetwork( + new NeuralNetConfiguration.Builder().seed(12345) + .updater(new RmsProp(0.1)) + .list() + .layer(0, new ConvolutionLayer.Builder(10, 10) //Only keep the first layer the same + .nIn(3) //3 channels: RGB + .nOut(30).stride(4, 4) + .activation(Activation.RELU) + .weightInit(WeightInit.RELU) + .updater(new AdaGrad(0.1)).build()) //Output: (130-10+0)/4+1 = 31 -> 31*31*30 + .layer(1, new SubsamplingLayer.Builder( + SubsamplingLayer.PoolingType.MAX) //change kernel size + .kernelSize(5, 5).stride(2, 2) + .build()) //(31-5+0)/2+1 = 14 + .layer(2, new ConvolutionLayer.Builder(6, 6) //change here + .nIn(30).nOut(10).stride(2, 2) + .activation(Activation.RELU) + .weightInit(WeightInit.RELU).build()) //Output: (14-6+0)/2+1 = 5 -> 5*5*10 = 250 + .layer(3, new DenseLayer.Builder() //change here + .activation(Activation.RELU).nIn(250).nOut(50) + .weightInit(WeightInit.RELU) + .gradientNormalization( + GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10) + .updater(new RmsProp(0.01)).build()) + .layer(4, new GravesLSTM.Builder() //change here + .activation(Activation.SOFTSIGN).nIn(50) + .nOut(25).weightInit(WeightInit.XAVIER) + .build()) + .layer(5, new RnnOutputLayer.Builder( + LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .nIn(25).nOut(4) + .weightInit(WeightInit.XAVIER) + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10) + .build()) + .inputPreProcessor(0,new RnnToCnnPreProcessor(V_HEIGHT, V_WIDTH, 3)) + .inputPreProcessor(3,new CnnToFeedForwardPreProcessor(5, 5, 10)) + .inputPreProcessor(4, new FeedForwardToRnnPreProcessor()) + + .backpropType(BackpropType.TruncatedBPTT) + .tBPTTForwardLength(V_NFRAMES / 5) + .tBPTTBackwardLength(V_NFRAMES / 5).build()); + modelToTweak.init(); + + MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToTweak) + .fineTuneConfiguration( + new FineTuneConfiguration.Builder().seed(12345).l2(0.001) //l2 regularization on all layers + .updater(new AdaGrad(0.4)) + .weightInit(WeightInit.RELU).build()) + .removeLayersFromOutput(5) + .addLayer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(3, 3) + .stride(2, 2).build()) + .addLayer(new ConvolutionLayer.Builder(3, 3).nIn(30).nOut(10).stride(2, 2) + .activation(Activation.RELU).weightInit(WeightInit.RELU).build()) + .addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(490).nOut(50) + .weightInit(WeightInit.RELU).updater(new AdaGrad(0.5)) + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).build()) + .addLayer(new GravesLSTM.Builder().activation(Activation.SOFTSIGN).nIn(50).nOut(50) + .weightInit(WeightInit.XAVIER).updater(new AdaGrad(0.6)) + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).build()) + .addLayer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(50).nOut(4) //4 possible shapes: circle, square, arc, line + .weightInit(WeightInit.XAVIER) + .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) + .gradientNormalizationThreshold(10).build()) + .setInputPreProcessor(3, new CnnToFeedForwardPreProcessor(7, 7, 10)) + .setInputPreProcessor(4, new FeedForwardToRnnPreProcessor()).build(); + + //modelNow should have the same architecture as modelExpectedArch + assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(0).toJson(), + modelNow.getLayerWiseConfigurations().getConf(0).toJson()); + //some learning related info the subsampling layer will not be overwritten + //assertTrue(modelExpectedArch.getLayerWiseConfigurations().getConf(1).toJson().equals(modelNow.getLayerWiseConfigurations().getConf(1).toJson())); + assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(2).toJson(), + modelNow.getLayerWiseConfigurations().getConf(2).toJson()); + assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(3).toJson(), + modelNow.getLayerWiseConfigurations().getConf(3).toJson()); + assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(4).toJson(), + modelNow.getLayerWiseConfigurations().getConf(4).toJson()); + assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(5).toJson(), + modelNow.getLayerWiseConfigurations().getConf(5).toJson()); + + assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); + //subsampling has no params + //assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(4).params().shape(), modelNow.getLayer(4).params().shape()); + assertArrayEquals(modelExpectedArch.getLayer(5).params().shape(), modelNow.getLayer(5).params().shape()); + + } + + @Test + public void testAllWithCNN() { + Nd4j.getRandom().setSeed(12345); + + DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 28 * 28 * 3).reshape(10, 3, 28, 28), TestUtils.randomOneHot(DataType.FLOAT,10, 10)); + MultiLayerNetwork modelToFineTune = + new MultiLayerNetwork( + new NeuralNetConfiguration.Builder().seed(123) + .weightInit(WeightInit.XAVIER) + .updater(new Nesterovs(0.01, 0.9)) + .list() + .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(3).stride(1, 1) + .nOut(20).activation(Activation.IDENTITY) + .build()) + .layer(1, new SubsamplingLayer.Builder( + SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2) + .build()) + .layer(2, new ConvolutionLayer.Builder(5, 5).stride(1, 1) + .nOut(50).activation(Activation.IDENTITY) + .build()) + .layer(3, new SubsamplingLayer.Builder( + SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2) + .build()) + .layer(4, new DenseLayer.Builder().activation(Activation.RELU) + .nOut(500).build()) + .layer(5, new DenseLayer.Builder().activation(Activation.RELU) + .nOut(250).build()) + .layer(6, new OutputLayer.Builder( + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(100) + .activation(Activation.SOFTMAX) + .build()) + .setInputType(InputType.convolutionalFlat(28, 28, 3)) + .build()); + modelToFineTune.init(); + INDArray asFrozenFeatures = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false).get(2); //10x20x12x12 + + NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.2)) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT); + + FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.2)) + .build(); + + MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf) + .setFeatureExtractor(1).nOutReplace(4, 600, WeightInit.XAVIER).removeLayersFromOutput(2) + .addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(600).nOut(300).build()) + .addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(300).nOut(150).build()) + .addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(150).nOut(50).build()) + .addLayer(new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .activation(Activation.SOFTMAX).nIn(50).nOut(10).build()) + .build(); + + MultiLayerNetwork notFrozen = new MultiLayerNetwork(equivalentConf.list() + .layer(0, new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50) + .activation(Activation.IDENTITY).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2) + .stride(2, 2).build()) + .layer(2, new DenseLayer.Builder().activation(Activation.RELU).nOut(600).build()) + .layer(3, new DenseLayer.Builder().activation(Activation.RELU).nOut(300).build()) + .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(150).build()) + .layer(5, new DenseLayer.Builder().activation(Activation.RELU).nOut(50).build()) + .layer(6, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(10) + .activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(12, 12, 20)).build()); + notFrozen.init(); + + assertArrayEquals(modelToFineTune.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); + //subsampling has no params + //assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); + assertArrayEquals(notFrozen.getLayer(0).params().shape(), modelNow.getLayer(2).params().shape()); + modelNow.getLayer(2).setParams(notFrozen.getLayer(0).params()); + //subsampling has no params + //assertArrayEquals(notFrozen.getLayer(1).params().shape(), modelNow.getLayer(3).params().shape()); + assertArrayEquals(notFrozen.getLayer(2).params().shape(), modelNow.getLayer(4).params().shape()); + modelNow.getLayer(4).setParams(notFrozen.getLayer(2).params()); + assertArrayEquals(notFrozen.getLayer(3).params().shape(), modelNow.getLayer(5).params().shape()); + modelNow.getLayer(5).setParams(notFrozen.getLayer(3).params()); + assertArrayEquals(notFrozen.getLayer(4).params().shape(), modelNow.getLayer(6).params().shape()); + modelNow.getLayer(6).setParams(notFrozen.getLayer(4).params()); + assertArrayEquals(notFrozen.getLayer(5).params().shape(), modelNow.getLayer(7).params().shape()); + modelNow.getLayer(7).setParams(notFrozen.getLayer(5).params()); + assertArrayEquals(notFrozen.getLayer(6).params().shape(), modelNow.getLayer(8).params().shape()); + modelNow.getLayer(8).setParams(notFrozen.getLayer(6).params()); + + int i = 0; + while (i < 3) { + notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); + modelNow.fit(randomData); + i++; + } + + INDArray expectedParams = Nd4j.hstack(modelToFineTune.getLayer(0).params(), notFrozen.params()); + assertEquals(expectedParams, modelNow.params()); + } + + + @Test + public void testFineTuneOverride() { + //Check that fine-tune overrides are selective - i.e., if I only specify a new LR, only the LR should be modified + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().updater(new Adam(1e-4)) + .activation(Activation.TANH).weightInit(WeightInit.RELU) + .l1(0.1).l2(0.2).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(5).build()).layer(1, + new OutputLayer.Builder().nIn(5).nOut(4) + .activation(Activation.HARDSIGMOID).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + MultiLayerNetwork net2 = new TransferLearning.Builder(net) + .fineTuneConfiguration(new FineTuneConfiguration.Builder().updater(new Adam(2e-2)) + .backpropType(BackpropType.TruncatedBPTT) //Should be set on MLC + .build()) + .build(); + + + //Check original net isn't modified: + BaseLayer l0 = (BaseLayer) net.getLayer(0).conf().getLayer(); + assertEquals(new Adam(1e-4), l0.getIUpdater()); + assertEquals(Activation.TANH.getActivationFunction(), l0.getActivationFn()); + assertEquals(new WeightInitRelu(), l0.getWeightInitFn()); + assertEquals(0.1, TestUtils.getL1(l0), 1e-6); + + BaseLayer l1 = (BaseLayer) net.getLayer(1).conf().getLayer(); + assertEquals(new Adam(1e-4), l1.getIUpdater()); + assertEquals(Activation.HARDSIGMOID.getActivationFunction(), l1.getActivationFn()); + assertEquals(new WeightInitRelu(), l1.getWeightInitFn()); + assertEquals(0.2, TestUtils.getL2(l1), 1e-6); + + assertEquals(BackpropType.Standard, conf.getBackpropType()); + + //Check new net has only the appropriate things modified (i.e., LR) + l0 = (BaseLayer) net2.getLayer(0).conf().getLayer(); + assertEquals(new Adam(2e-2), l0.getIUpdater()); + assertEquals(Activation.TANH.getActivationFunction(), l0.getActivationFn()); + assertEquals(new WeightInitRelu(), l0.getWeightInitFn()); + assertEquals(0.1, TestUtils.getL1(l0), 1e-6); + + l1 = (BaseLayer) net2.getLayer(1).conf().getLayer(); + assertEquals(new Adam(2e-2), l1.getIUpdater()); + assertEquals(Activation.HARDSIGMOID.getActivationFunction(), l1.getActivationFn()); + assertEquals(new WeightInitRelu(), l1.getWeightInitFn()); + assertEquals(0.2, TestUtils.getL2(l1), 1e-6); + + assertEquals(BackpropType.TruncatedBPTT, net2.getLayerWiseConfigurations().getBackpropType()); + } + + @Test + public void testAllWithCNNNew() { + Nd4j.getRandom().setSeed(12345); + + DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT,10, 28 * 28 * 3).reshape(10, 3, 28, 28), TestUtils.randomOneHot(10, 10)); + MultiLayerNetwork modelToFineTune = + new MultiLayerNetwork( + new NeuralNetConfiguration.Builder().seed(123) + .weightInit(WeightInit.XAVIER) + .updater(new Nesterovs(0.01, 0.9)) + .list() + .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(3).stride(1, 1) + .nOut(20).activation(Activation.IDENTITY).build()) + .layer(1, new SubsamplingLayer.Builder(PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2).build()) + .layer(2, new ConvolutionLayer.Builder(5, 5).stride(1, 1) + .nOut(50).activation(Activation.IDENTITY).build()) + .layer(3, new SubsamplingLayer.Builder(PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2).build()) + .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()) + .layer(5, new DenseLayer.Builder().activation(Activation.RELU).nOut(250).build()) + .layer(6, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(100).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 3)) //See note below + .build()); + modelToFineTune.init(); + INDArray asFrozenFeatures = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false).get(2); //10x20x12x12 + + NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.2)); + FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.2)).build(); + + MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf) + .setFeatureExtractor(1).removeLayersFromOutput(5) + .addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(12 * 12 * 20).nOut(300) + .build()) + .addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(300).nOut(150).build()) + .addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(150).nOut(50).build()) + .addLayer(new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .activation(Activation.SOFTMAX).nIn(50).nOut(10).build()) + .setInputPreProcessor(2, new CnnToFeedForwardPreProcessor(12, 12, 20)).build(); + + + MultiLayerNetwork notFrozen = new MultiLayerNetwork(equivalentConf.list() + .layer(0, new DenseLayer.Builder().activation(Activation.RELU).nIn(12 * 12 * 20).nOut(300) + .build()) + .layer(1, new DenseLayer.Builder().activation(Activation.RELU).nIn(300).nOut(150).build()) + .layer(2, new DenseLayer.Builder().activation(Activation.RELU).nIn(150).nOut(50).build()) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(50) + .nOut(10).activation(Activation.SOFTMAX).build()) + .inputPreProcessor(0, new CnnToFeedForwardPreProcessor(12, 12, 20)) + .build()); + notFrozen.init(); + + assertArrayEquals(modelToFineTune.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); + //subsampling has no params + //assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); + assertArrayEquals(notFrozen.getLayer(0).params().shape(), modelNow.getLayer(2).params().shape()); + modelNow.getLayer(2).setParams(notFrozen.getLayer(0).params()); + assertArrayEquals(notFrozen.getLayer(1).params().shape(), modelNow.getLayer(3).params().shape()); + modelNow.getLayer(3).setParams(notFrozen.getLayer(1).params()); + assertArrayEquals(notFrozen.getLayer(2).params().shape(), modelNow.getLayer(4).params().shape()); + modelNow.getLayer(4).setParams(notFrozen.getLayer(2).params()); + assertArrayEquals(notFrozen.getLayer(3).params().shape(), modelNow.getLayer(5).params().shape()); + modelNow.getLayer(5).setParams(notFrozen.getLayer(3).params()); + + int i = 0; + while (i < 3) { + notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); + modelNow.fit(randomData); + i++; + } + + INDArray expectedParams = Nd4j.hstack(modelToFineTune.getLayer(0).params(), notFrozen.params()); + assertEquals(expectedParams, modelNow.params()); + } + + @Test + public void testObjectOverrides(){ + //https://github.com/deeplearning4j/deeplearning4j/issues/4368 + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dropOut(0.5) + .weightNoise(new DropConnect(0.5)) + .l2(0.5) + .constrainWeights(new UnitNormConstraint()) + .list() + .layer(new DenseLayer.Builder().nIn(10).nOut(10).build()) + .build(); + + MultiLayerNetwork orig = new MultiLayerNetwork(conf); + orig.init(); + + FineTuneConfiguration ftc = new FineTuneConfiguration.Builder() + .dropOut(0) + .weightNoise(null) + .constraints(null) + .l2(0.0) + .build(); + + MultiLayerNetwork transfer = new TransferLearning.Builder(orig) + .fineTuneConfiguration(ftc) + .build(); + + DenseLayer l = (DenseLayer) transfer.getLayer(0).conf().getLayer(); + + assertNull(l.getIDropout()); + assertNull(l.getWeightNoise()); + assertNull(l.getConstraints()); + assertNull(TestUtils.getL2Reg(l)); + } + + + @Test + public void testTransferLearningSubsequent() { + final INDArray input = Nd4j.create(6,6,6,6); + final MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder() + .weightInit(new ConstantDistribution(666)) + .list() + .setInputType(InputType.inferInputTypes(input)[0]) + .layer(new Convolution2D.Builder(3, 3).nOut(10).build()) + .layer(new Convolution2D.Builder(1, 1).nOut(3).build()) + .layer(new OutputLayer.Builder().nOut(2).lossFunction(LossFunctions.LossFunction.MSE) + .build()).build()); + net.init(); + + MultiLayerNetwork newGraph = new TransferLearning + .Builder(net) + .fineTuneConfiguration(new FineTuneConfiguration.Builder().build()) + .nOutReplace(0, 7, new ConstantDistribution(333)) + .nOutReplace(1, 3, new ConstantDistribution(111)) + .removeLayersFromOutput(1) + .addLayer(new OutputLayer.Builder() + .nIn(48).nOut(2) + .lossFunction(LossFunctions.LossFunction.MSE) + .build()) + .setInputPreProcessor(2, new CnnToFeedForwardPreProcessor(4,4,3)) + .build(); + newGraph.init(); + + assertEquals( 7, newGraph.layerInputSize(1), "Incorrect # inputs"); + + newGraph.output(input); + } + + @Test + public void testChangeNOutNIn() { + INDArray input = Nd4j.create(new long[] {1, 2, 4, 4}); + MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder() + .list() + .setInputType(InputType.inferInputTypes(input)[0]) + .layer(new Convolution2D.Builder(1, 1).nOut(10).build()) + .layer(new SubsamplingLayer.Builder(1,1).build()) + .layer(new Convolution2D.Builder(1, 1).nOut(7).build()) + .layer(new OutputLayer.Builder().activation(Activation.SOFTMAX).nOut(2).build()) + .build()); + net.init(); + + final MultiLayerNetwork newNet = new TransferLearning.Builder(net) + .nOutReplace(0, 5, WeightInit.XAVIER) + .nInReplace(2, 5, WeightInit.XAVIER) + .build(); + + newNet.init(); + + assertEquals( 5 , newNet.layerSize(0), "Incorrect number of outputs!"); + assertEquals( 5, newNet.layerInputSize(2), "Incorrect number of inputs!"); + newNet.output(input); + } + + + @Test + public void testTransferLearningSameDiffLayers(){ + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.DOUBLE) + .activation(Activation.TANH) + .updater(new Adam(0.01)) + .weightInit(WeightInit.XAVIER) + .list() + .layer(new LSTM.Builder().nOut(8).build()) + .layer( new SelfAttentionLayer.Builder().nOut(4).nHeads(2).projectInput(true).build()) + .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build()) + .layer(new OutputLayer.Builder().nOut(2).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .setInputType(InputType.recurrent(4)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + INDArray in = Nd4j.rand(DataType.FLOAT, 3, 4, 5); + INDArray out = net.output(in); + + MultiLayerNetwork net2 = new TransferLearning.Builder(net) + .fineTuneConfiguration(FineTuneConfiguration.builder().updater(new Adam(0.01)).build()) + .removeLayersFromOutput(1) + .addLayer(new OutputLayer.Builder().nIn(4).nOut(2).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + + net2.setParam("3_W", net.getParam("3_W")); + net2.setParam("3_b", net.getParam("3_b")); + + Map p1 = net.paramTable(); + Map p2 = net2.paramTable(); + for(String s : p1.keySet()){ + INDArray i1 = p1.get(s); + INDArray i2 = p2.get(s); + assertEquals(i1, i2, s); + } + + INDArray out2 = net2.output(in); + + assertEquals(out, out2); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/TestGradientNormalization.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestGradientNormalization.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/TestGradientNormalization.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestGradientNormalization.java index 0ecdeaca7..02616d66d 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/TestGradientNormalization.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestGradientNormalization.java @@ -30,10 +30,7 @@ import org.deeplearning4j.nn.conf.layers.DenseLayer; import org.deeplearning4j.nn.gradient.DefaultGradient; import org.deeplearning4j.nn.gradient.Gradient; import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.shape.Shape; import org.nd4j.linalg.factory.Nd4j; @@ -42,8 +39,7 @@ import org.nd4j.linalg.learning.config.NoOp; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestGradientNormalization extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java similarity index 99% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java index f74eb7a26..d9735fb89 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/TestUpdaters.java @@ -39,10 +39,7 @@ import org.deeplearning4j.nn.params.DefaultParamInitializer; import org.deeplearning4j.nn.params.PretrainParamInitializer; import org.deeplearning4j.nn.updater.graph.ComputationGraphUpdater; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -59,8 +56,7 @@ import java.util.*; import static org.junit.jupiter.api.Assertions.*; import static org.nd4j.linalg.indexing.NDArrayIndex.interval; import static org.nd4j.linalg.indexing.NDArrayIndex.point; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestUpdaters extends BaseDL4JTest { protected int nIn = 3; @@ -324,7 +320,7 @@ public class TestUpdaters extends BaseDL4JTest { count++; } - assertEquals(2, count,"Count should be equal to 2, one for weight gradient and one for bias gradient"); + assertEquals(2, count, "Count should be equal to 2, one for weight gradient and one for bias gradient"); /* * Check that we are not erroneously mutating moving avg gradient while calculating @@ -344,7 +340,7 @@ public class TestUpdaters extends BaseDL4JTest { actualM[i] = Math.round(actualM[i] * 1e2) / 1e2; } - assertEquals(Arrays.equals(expectedM, actualM), true, "Wrong weight gradient after first iteration's update"); + assertTrue( Arrays.equals(expectedM, actualM), "Wrong weight gradient after first iteration's update"); } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/custom/CustomGradientUpdater.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/CustomGradientUpdater.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/custom/CustomGradientUpdater.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/CustomGradientUpdater.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/custom/CustomIUpdater.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/CustomIUpdater.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/custom/CustomIUpdater.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/CustomIUpdater.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java index 97ab4e56d..703d56eb2 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/updater/custom/TestCustomUpdater.java @@ -27,10 +27,7 @@ import org.deeplearning4j.nn.conf.layers.BaseLayer; import org.deeplearning4j.nn.conf.layers.DenseLayer; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -39,9 +36,7 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) + public class TestCustomUpdater extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/util/TestDataSetConsumer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/util/TestDataSetConsumer.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/util/TestDataSetConsumer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/util/TestDataSetConsumer.java diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/LegacyWeightInitTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/LegacyWeightInitTest.java new file mode 100644 index 000000000..a2cd8d346 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/LegacyWeightInitTest.java @@ -0,0 +1,251 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.weights; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.distribution.*; +import org.deeplearning4j.nn.conf.serde.JsonMappers; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.rng.Random; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.factory.RandomFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + + +public class LegacyWeightInitTest extends BaseDL4JTest { + + private RandomFactory prevFactory; + private final static int SEED = 666; + + private final static List distributions = Arrays.asList( + new LogNormalDistribution(12.3, 4.56), + new BinomialDistribution(3, 0.3), + new NormalDistribution(0.666, 0.333), + new UniformDistribution(-1.23, 4.56), + new OrthogonalDistribution(3.45), + new TruncatedNormalDistribution(0.456, 0.123), + new ConstantDistribution(666)); + + @BeforeEach + public void setRandomFactory() { + prevFactory = Nd4j.randomFactory; + Nd4j.randomFactory = new FixedSeedRandomFactory(prevFactory); + } + + @AfterEach + public void resetRandomFactory() { + Nd4j.randomFactory = prevFactory; + } + + /** + * Test that param init is identical to legacy implementation + */ + @Test + public void initParams() { + final long[] shape = {5, 5}; // To make identity happy + final long fanIn = shape[0]; + final long fanOut = shape[1]; + + final INDArray inLegacy = Nd4j.create(fanIn * fanOut); + final INDArray inTest = inLegacy.dup(); + for (WeightInit legacyWi : WeightInit.values()) { + if (legacyWi != WeightInit.DISTRIBUTION) { + Nd4j.getRandom().setSeed(SEED); + final INDArray expected = WeightInitUtil.initWeights(fanIn, fanOut, shape, legacyWi, null, inLegacy); + + Nd4j.getRandom().setSeed(SEED); + final INDArray actual = legacyWi.getWeightInitFunction() + .init(fanIn, fanOut, shape, WeightInitUtil.DEFAULT_WEIGHT_INIT_ORDER, inTest); + assertArrayEquals(shape, actual.shape(), "Incorrect shape for " + legacyWi + "!"); + + assertEquals( expected, actual, "Incorrect weight initialization for " + legacyWi + "!"); + } + } + } + + /** + * Test that param init is identical to legacy implementation + */ + @Test + public void initParamsFromDistribution() { + final long[] shape = {3, 7}; // To make identity happy + final long fanIn = shape[0]; + final long fanOut = shape[1]; + + final INDArray inLegacy = Nd4j.create(fanIn * fanOut); + final INDArray inTest = inLegacy.dup(); + + for (Distribution dist : distributions) { + + Nd4j.getRandom().setSeed(SEED); + final INDArray expected = WeightInitUtil.initWeights( + fanIn, + fanOut, + shape, + WeightInit.DISTRIBUTION, + Distributions.createDistribution(dist), + inLegacy); + + final INDArray actual = new WeightInitDistribution(dist).init( + fanIn, + fanOut, + shape, + WeightInitUtil.DEFAULT_WEIGHT_INIT_ORDER, + inTest); + assertArrayEquals(shape, actual.shape(), "Incorrect shape for " + dist.getClass().getSimpleName() + "!"); + + assertEquals(expected, actual, "Incorrect weight initialization for " + dist.getClass().getSimpleName() + "!"); + } + } + + /** + * Test that weight inits can be serialized and de-serialized in JSON format + */ + @Test + public void serializeDeserializeJson() throws IOException { + final long[] shape = {5, 5}; // To make identity happy + final long fanIn = shape[0]; + final long fanOut = shape[1]; + + final ObjectMapper mapper = JsonMappers.getMapper(); + final INDArray inBefore = Nd4j.create(fanIn * fanOut); + final INDArray inAfter = inBefore.dup(); + + // Just use to enum to loop over all strategies + for (WeightInit legacyWi : WeightInit.values()) { + if (legacyWi != WeightInit.DISTRIBUTION) { + Nd4j.getRandom().setSeed(SEED); + final IWeightInit before = legacyWi.getWeightInitFunction(); + final INDArray expected = before.init(fanIn, fanOut, shape, inBefore.ordering(), inBefore); + + final String json = mapper.writeValueAsString(before); + final IWeightInit after = mapper.readValue(json, IWeightInit.class); + + Nd4j.getRandom().setSeed(SEED); + final INDArray actual = after.init(fanIn, fanOut, shape, inAfter.ordering(), inAfter); + + assertArrayEquals(shape, actual.shape(), "Incorrect shape for " + legacyWi + "!"); + assertEquals(expected, actual, "Incorrect weight initialization for " + legacyWi + "!"); + } + } + } + + /** + * Test that distribution can be serialized and de-serialized in JSON format + */ + @Test + public void serializeDeserializeDistributionJson() throws IOException { + final long[] shape = {3, 7}; // To make identity happy + final long fanIn = shape[0]; + final long fanOut = shape[1]; + + final ObjectMapper mapper = JsonMappers.getMapper(); + final INDArray inBefore = Nd4j.create(fanIn * fanOut); + final INDArray inAfter = inBefore.dup(); + + for (Distribution dist : distributions) { + + Nd4j.getRandom().setSeed(SEED); + final IWeightInit before = new WeightInitDistribution(dist); + final INDArray expected = before.init( + fanIn, + fanOut, + shape, + inBefore.ordering(), + inBefore); + + final String json = mapper.writeValueAsString(before); + final IWeightInit after = mapper.readValue(json, IWeightInit.class); + + Nd4j.getRandom().setSeed(SEED); + final INDArray actual = after.init(fanIn, fanOut, shape, inAfter.ordering(), inAfter); + + assertArrayEquals(shape, actual.shape(), "Incorrect shape for " + dist.getClass().getSimpleName() + "!"); + + assertEquals( expected, actual, "Incorrect weight initialization for " + dist.getClass().getSimpleName() + "!"); + } + } + + /** + * Test equals and hashcode implementation. Redundant as one can trust Lombok on this?? + */ + @Test + public void equalsAndHashCode() { + WeightInit lastInit = WeightInit.values()[WeightInit.values().length-1]; + for (WeightInit legacyWi : WeightInit.values()) { + if(legacyWi != WeightInit.DISTRIBUTION) { + assertEquals(legacyWi.getWeightInitFunction(), legacyWi.getWeightInitFunction(), "Shall be equal!"); + assertNotEquals(lastInit.getWeightInitFunction(), legacyWi.getWeightInitFunction(), "Shall be equal!"); + if (legacyWi != WeightInit.NORMAL && legacyWi != WeightInit.LECUN_NORMAL) { + lastInit = legacyWi; + } + } + } + Distribution lastDist = distributions.get(distributions.size() - 1); + for(Distribution distribution: distributions) { + assertEquals(new WeightInitDistribution(distribution), new WeightInitDistribution(distribution.clone()), "Shall be equal!"); + assertNotEquals(new WeightInitDistribution(lastDist), new WeightInitDistribution(distribution), "Shall not be equal!"); + lastDist = distribution; + } + } + + /** + * Assumes RandomFactory will only call no-args constructor while this test runs + */ + private static class FixedSeedRandomFactory extends RandomFactory { + private final RandomFactory factory; + + + private FixedSeedRandomFactory(RandomFactory factory) { + super(factory.getRandom().getClass()); + this.factory = factory; + } + + @Override + public Random getRandom() { + return getNewRandomInstance(SEED); + } + + @Override + public Random getNewRandomInstance() { + return factory.getNewRandomInstance(); + } + + @Override + public Random getNewRandomInstance(long seed) { + return factory.getNewRandomInstance(seed); + } + + @Override + public Random getNewRandomInstance(long seed, long size) { + return factory.getNewRandomInstance(seed, size); + } + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitIdentityTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitIdentityTest.java new file mode 100644 index 000000000..8b9b35e4f --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitIdentityTest.java @@ -0,0 +1,124 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.weights; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.RNNFormat; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.graph.ComputationGraph; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.impl.ActivationIdentity; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class WeightInitIdentityTest extends BaseDL4JTest { + + /** + * Test identity mapping for 1d convolution + */ + @Test + //@Ignore("Ignore for now. Underlying logic changed. Gradient checker passes so implementatin is valid.") + public void testIdConv1D() { + final INDArray input = Nd4j.randn(DataType.FLOAT, 1,5,7); + final String inputName = "input"; + final String conv = "conv"; + final String output = "output"; + final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder() + .graphBuilder() + .addInputs(inputName) + .setOutputs(output) + .layer(conv, new Convolution1DLayer.Builder(7) + .convolutionMode(ConvolutionMode.Same) + .nOut(input.size(1)) + .weightInit(new WeightInitIdentity()) + .activation(new ActivationIdentity()) + .build(), inputName) + .layer(output, new RnnLossLayer.Builder().activation(new ActivationIdentity()).build(), conv) + .setInputTypes(InputType.recurrent(5,7,RNNFormat.NCW)) + .build()); + graph.init(); + + INDArray reshape = graph.outputSingle(input).reshape(input.shape()); + assertEquals( input, reshape, "Mapping was not identity!"); + } + + /** + * Test identity mapping for 2d convolution + */ + @Test + public void testIdConv2D() { + final INDArray input = Nd4j.randn(DataType.FLOAT,1,5,7,11); + final String inputName = "input"; + final String conv = "conv"; + final String output = "output"; + final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder() + .graphBuilder() + .setInputTypes(InputType.inferInputType(input)) + .addInputs(inputName) + .setOutputs(output) + .layer(conv, new ConvolutionLayer.Builder(3,5) + .convolutionMode(ConvolutionMode.Same) + .nOut(input.size(1)) + .weightInit(new WeightInitIdentity()) + .activation(new ActivationIdentity()) + .build(), inputName) + .layer(output, new CnnLossLayer.Builder().activation(new ActivationIdentity()).build(), conv) + .build()); + graph.init(); + + assertEquals( input, graph.outputSingle(input), "Mapping was not identity!"); + } + + /** + * Test identity mapping for 3d convolution + */ + @Test + public void testIdConv3D() { + final INDArray input = Nd4j.randn(DataType.FLOAT, 1,5,7,11,13); + final String inputName = "input"; + final String conv = "conv"; + final String output = "output"; + final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder() + .graphBuilder() + .setInputTypes(InputType.inferInputType(input)) + .addInputs(inputName) + .setOutputs(output) + .layer(conv, new Convolution3D.Builder(3,7,5) + .convolutionMode(ConvolutionMode.Same) + .dataFormat(Convolution3D.DataFormat.NCDHW) + .nOut(input.size(1)) + .weightInit(new WeightInitIdentity()) + .activation(new ActivationIdentity()) + .build(), inputName) + .layer(output, new Cnn3DLossLayer.Builder(Convolution3D.DataFormat.NCDHW).activation(new ActivationIdentity()).build(), conv) + .build()); + graph.init(); + + assertEquals( input, graph.outputSingle(input), "Mapping was not identity!"); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitUtilTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitUtilTest.java similarity index 77% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitUtilTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitUtilTest.java index c1d063bb9..2bf978eda 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitUtilTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitUtilTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.weights; import org.apache.commons.math3.util.FastMath; @@ -24,127 +25,128 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.conf.distribution.Distributions; import org.deeplearning4j.nn.conf.distribution.GaussianDistribution; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.rng.distribution.Distribution; import org.nd4j.linalg.factory.Nd4j; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Weight Init Util Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class WeightInitUtilTest extends BaseDL4JTest { +public class WeightInitUtilTest extends BaseDL4JTest { protected int fanIn = 3; - protected int fanOut = 2; - - protected int[] shape = new int[] { fanIn, fanOut }; - + protected int[] shape = new int[] {fanIn, fanOut}; protected Distribution dist = Distributions.createDistribution(new GaussianDistribution(0.0, 0.1)); @BeforeEach - void doBefore() { + public void doBefore() { Nd4j.getRandom().setSeed(123); } @Test - @DisplayName("Test Distribution") - void testDistribution() { + public void testDistribution() { INDArray params = Nd4j.create(shape, 'f'); - // fan in/out not used - INDArray weightsActual = WeightInitUtil.initWeights(-1, -1, shape, WeightInit.DISTRIBUTION, dist, params); + INDArray weightsActual = WeightInitUtil.initWeights(-1, -1, shape, WeightInit.DISTRIBUTION, dist, params); //fan in/out not used + // expected calculation Nd4j.getRandom().setSeed(123); INDArray weightsExpected = dist.sample(params); + assertEquals(weightsExpected, weightsActual); } @Test - @DisplayName("Test Relu") - void testRelu() { + public void testRelu() { INDArray params = Nd4j.create(shape, 'f'); INDArray weightsActual = WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.RELU, dist, params); + // expected calculation Nd4j.getRandom().setSeed(123); INDArray weightsExpected = Nd4j.randn('f', shape).muli(FastMath.sqrt(2.0 / fanIn)); + assertEquals(weightsExpected, weightsActual); } @Test - @DisplayName("Test Sigmoid Uniform") - void testSigmoidUniform() { + public void testSigmoidUniform() { INDArray params = Nd4j.create(shape, 'f'); - INDArray weightsActual = WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.SIGMOID_UNIFORM, dist, params); + INDArray weightsActual = + WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.SIGMOID_UNIFORM, dist, params); + // expected calculation Nd4j.getRandom().setSeed(123); double min = -4.0 * Math.sqrt(6.0 / (double) (shape[0] + shape[1])); double max = 4.0 * Math.sqrt(6.0 / (double) (shape[0] + shape[1])); INDArray weightsExpected = Nd4j.getDistributions().createUniform(min, max).sample(Nd4j.createUninitialized(shape, 'f')); + assertEquals(weightsExpected, weightsActual); } @Test - @DisplayName("Test Uniform") - void testUniform() { + public void testUniform() { INDArray params = Nd4j.create(shape, 'f'); INDArray weightsActual = WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.UNIFORM, dist, params); + // expected calculation Nd4j.getRandom().setSeed(123); double a = 1.0 / Math.sqrt(fanIn); INDArray weightsExpected = Nd4j.getDistributions().createUniform(-a, a).sample(Nd4j.create(shape, 'f')); + assertEquals(weightsExpected, weightsActual); } @Test - @DisplayName("Test Xavier") - void testXavier() { + public void testXavier() { Nd4j.getRandom().setSeed(123); INDArray params = Nd4j.create(shape, 'f'); INDArray weightsActual = WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.XAVIER, dist, params); + // expected calculation Nd4j.getRandom().setSeed(123); INDArray weightsExpected = Nd4j.randn('f', shape); weightsExpected.muli(FastMath.sqrt(2.0 / (fanIn + fanOut))); + assertEquals(weightsExpected, weightsActual); } @Test - @DisplayName("Test Xavier Fan In") - void testXavierFanIn() { + public void testXavierFanIn() { INDArray params = Nd4j.create(shape, 'f'); - INDArray weightsActual = WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.XAVIER_FAN_IN, dist, params); + INDArray weightsActual = + WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.XAVIER_FAN_IN, dist, params); + // expected calculation Nd4j.getRandom().setSeed(123); INDArray weightsExpected = Nd4j.randn('f', shape); weightsExpected.divi(FastMath.sqrt(fanIn)); + assertEquals(weightsExpected, weightsActual); } @Test - @DisplayName("Test Xavier Legacy") - void testXavierLegacy() { + public void testXavierLegacy() { INDArray params = Nd4j.create(shape, 'f'); - INDArray weightsActual = WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.XAVIER_LEGACY, dist, params); + INDArray weightsActual = + WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.XAVIER_LEGACY, dist, params); + // expected calculation Nd4j.getRandom().setSeed(123); INDArray weightsExpected = Nd4j.randn('f', shape); weightsExpected.muli(FastMath.sqrt(1.0 / (fanIn + fanOut))); + assertEquals(weightsExpected, weightsActual); } @Test - @DisplayName("Test Zero") - void testZero() { + public void testZero() { INDArray params = Nd4j.create(shape, 'f'); INDArray weightsActual = WeightInitUtil.initWeights(fanIn, fanOut, shape, WeightInit.ZERO, dist, params); + // expected calculation INDArray weightsExpected = Nd4j.create(shape, 'f'); + assertEquals(weightsExpected, weightsActual); } + + } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/BackTrackLineSearchTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/BackTrackLineSearchTest.java new file mode 100644 index 000000000..8b73c10ee --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/BackTrackLineSearchTest.java @@ -0,0 +1,257 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.optimize.solver; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.optimize.solvers.BackTrackLineSearch; +import org.deeplearning4j.optimize.stepfunctions.DefaultStepFunction; +import org.deeplearning4j.optimize.stepfunctions.NegativeDefaultStepFunction; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author Adam Gibson + */ +public class BackTrackLineSearchTest extends BaseDL4JTest { + private DataSetIterator irisIter; + private DataSet irisData; + + @BeforeEach + public void before() { + if (irisIter == null) { + irisIter = new IrisDataSetIterator(5, 5); + } + if (irisData == null) { + irisData = irisIter.next(); + irisData.normalizeZeroMeanZeroUnitVariance(); + } + } + + + + @Test + public void testSingleMinLineSearch() throws Exception { + OutputLayer layer = getIrisLogisticLayerConfig(Activation.SOFTMAX, 100, + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD); + int nParams = (int)layer.numParams(); + layer.setBackpropGradientsViewArray(Nd4j.create(1, nParams)); + layer.setInput(irisData.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); + layer.setLabels(irisData.getLabels()); + layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); + + BackTrackLineSearch lineSearch = new BackTrackLineSearch(layer, layer.getOptimizer()); + double step = lineSearch.optimize(layer.params(), layer.gradient().gradient(), layer.gradient().gradient(), LayerWorkspaceMgr.noWorkspacesImmutable()); + + assertEquals(1.0, step, 1e-3); + } + + @Test + public void testSingleMaxLineSearch() throws Exception { + double score1, score2; + + OutputLayer layer = getIrisLogisticLayerConfig(Activation.SOFTMAX, 100, + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD); + int nParams = (int)layer.numParams(); + layer.setBackpropGradientsViewArray(Nd4j.create(1, nParams)); + layer.setInput(irisData.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); + layer.setLabels(irisData.getLabels()); + layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); + score1 = layer.score(); + + BackTrackLineSearch lineSearch = + new BackTrackLineSearch(layer, new NegativeDefaultStepFunction(), layer.getOptimizer()); + double step = lineSearch.optimize(layer.params(), layer.gradient().gradient(), layer.gradient().gradient(), LayerWorkspaceMgr.noWorkspacesImmutable()); + + assertEquals(1.0, step, 1e-3); + } + + + @Test + public void testMultMinLineSearch() throws Exception { + double score1, score2; + + OutputLayer layer = getIrisLogisticLayerConfig(Activation.SOFTMAX, 100, + LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD); + int nParams = (int)layer.numParams(); + layer.setBackpropGradientsViewArray(Nd4j.create(1, nParams)); + layer.setInput(irisData.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); + layer.setLabels(irisData.getLabels()); + layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); + score1 = layer.score(); + INDArray origGradient = layer.gradient().gradient().dup(); + + NegativeDefaultStepFunction sf = new NegativeDefaultStepFunction(); + BackTrackLineSearch lineSearch = new BackTrackLineSearch(layer, sf, layer.getOptimizer()); + double step = lineSearch.optimize(layer.params(), layer.gradient().gradient(), layer.gradient().gradient(), LayerWorkspaceMgr.noWorkspacesImmutable()); + INDArray currParams = layer.params(); + sf.step(currParams, origGradient, step); + layer.setParams(currParams); + layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); + + score2 = layer.score(); + + assertTrue(score1 > score2, "score1=" + score1 + ", score2=" + score2); + + } + + @Test + public void testMultMaxLineSearch() throws Exception { + double score1, score2; + + irisData.normalizeZeroMeanZeroUnitVariance(); + OutputLayer layer = getIrisLogisticLayerConfig(Activation.SOFTMAX, 100, LossFunctions.LossFunction.MCXENT); + int nParams = (int)layer.numParams(); + layer.setBackpropGradientsViewArray(Nd4j.create(1, nParams)); + layer.setInput(irisData.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); + layer.setLabels(irisData.getLabels()); + layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); + score1 = layer.score(); + INDArray origGradient = layer.gradient().gradient().dup(); + + DefaultStepFunction sf = new DefaultStepFunction(); + BackTrackLineSearch lineSearch = new BackTrackLineSearch(layer, sf, layer.getOptimizer()); + double step = lineSearch.optimize(layer.params().dup(), layer.gradient().gradient().dup(), + layer.gradient().gradient().dup(), LayerWorkspaceMgr.noWorkspacesImmutable()); + + INDArray currParams = layer.params(); + sf.step(currParams, origGradient, step); + layer.setParams(currParams); + layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); + score2 = layer.score(); + + assertTrue(score1 < score2, "score1 = " + score1 + ", score2 = " + score2); + } + + private static OutputLayer getIrisLogisticLayerConfig(Activation activationFunction, int maxIterations, + LossFunctions.LossFunction lossFunction) { + NeuralNetConfiguration conf = + new NeuralNetConfiguration.Builder().seed(12345L).miniBatch(true) + .maxNumLineSearchIterations(maxIterations) + .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(lossFunction) + .nIn(4).nOut(3).activation(activationFunction) + .weightInit(WeightInit.XAVIER).build()) + .build(); + + val numParams = conf.getLayer().initializer().numParams(conf); + INDArray params = Nd4j.create(1, numParams); + return (OutputLayer) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); + } + + /////////////////////////////////////////////////////////////////////////// + + @Test + public void testBackTrackLineGradientDescent() { + OptimizationAlgorithm optimizer = OptimizationAlgorithm.LINE_GRADIENT_DESCENT; + + DataSetIterator irisIter = new IrisDataSetIterator(1, 1); + DataSet data = irisIter.next(); + + MultiLayerNetwork network = new MultiLayerNetwork(getIrisMultiLayerConfig(Activation.SIGMOID, optimizer)); + network.init(); + TrainingListener listener = new ScoreIterationListener(10); + network.setListeners(Collections.singletonList(listener)); + double oldScore = network.score(data); + for( int i=0; i<100; i++ ) { + network.fit(data.getFeatures(), data.getLabels()); + } + double score = network.score(); + assertTrue(score < oldScore); + } + + @Test + public void testBackTrackLineCG() { + OptimizationAlgorithm optimizer = OptimizationAlgorithm.CONJUGATE_GRADIENT; + + DataSet data = irisIter.next(); + data.normalizeZeroMeanZeroUnitVariance(); + MultiLayerNetwork network = new MultiLayerNetwork(getIrisMultiLayerConfig(Activation.RELU, optimizer)); + network.init(); + TrainingListener listener = new ScoreIterationListener(10); + network.setListeners(Collections.singletonList(listener)); + double firstScore = network.score(data); + + for( int i=0; i<5; i++ ) { + network.fit(data.getFeatures(), data.getLabels()); + } + double score = network.score(); + assertTrue(score < firstScore); + + } + + @Test + public void testBackTrackLineLBFGS() { + OptimizationAlgorithm optimizer = OptimizationAlgorithm.LBFGS; + DataSet data = irisIter.next(); + data.normalizeZeroMeanZeroUnitVariance(); + MultiLayerNetwork network = new MultiLayerNetwork(getIrisMultiLayerConfig(Activation.RELU, optimizer)); + network.init(); + TrainingListener listener = new ScoreIterationListener(10); + network.setListeners(Collections.singletonList(listener)); + double oldScore = network.score(data); + + for( int i=0; i<5; i++ ) { + network.fit(data.getFeatures(), data.getLabels()); + } + double score = network.score(); + assertTrue(score < oldScore); + + } + + private static MultiLayerConfiguration getIrisMultiLayerConfig(Activation activationFunction, OptimizationAlgorithm optimizer) { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(optimizer) + .updater(new Adam(0.01)).seed(12345L).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(100).weightInit(WeightInit.XAVIER) + .activation(activationFunction).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(100).nOut(3) + .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX) + .build()) + .build(); + + + return conf; + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java index 9afb5ffc7..b17032fdd 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/TestOptimizers.java @@ -42,10 +42,7 @@ import org.deeplearning4j.optimize.solvers.LBFGS; import org.deeplearning4j.optimize.solvers.LineGradientDescent; import org.deeplearning4j.optimize.solvers.StochasticGradientDescent; import org.deeplearning4j.optimize.stepfunctions.NegativeDefaultStepFunction; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -70,9 +67,6 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.TRAINING) public class TestOptimizers extends BaseDL4JTest { //For debugging. @@ -124,13 +118,13 @@ public class TestOptimizers extends BaseDL4JTest { double[] scores = new double[nCallsToOptimizer + 1]; scores[0] = score; for (int i = 0; i < nCallsToOptimizer; i++) { - for( int j = 0; j(); for (int e = 0; e < numReaders; e++) { final int f = e; val t = new Thread(new Runnable() { - @Override public void run() { sums[f] = 0; @@ -220,37 +262,48 @@ class IndexedTailTest extends BaseDL4JTest { } } }); + t.setName("reader thread " + f); t.start(); readers.add(t); } + + int sum = 0; for (int e = 0; e < 10000; e++) { - val array = Nd4j.create(5, 5).assign(e + 1); + val array = Nd4j.create(5, 5).assign(e+1); Nd4j.getExecutioner().commit(); - sum += (e + 1); + + sum += (e+1); tail.put(array); } // just wait till everything consumed Thread.sleep(2000); tail.notifyDead(); - for (val t : readers) t.join(); - for (int e = 0; e < numReaders; e++) assertEquals(sum, sums[e],"Failed for reader [" + e + "]"); + + + for (val t:readers) + t.join(); + + + for (int e = 0; e < numReaders; e++) + assertEquals(sum, sums[e], "Failed for reader [" + e + "]"); + + assertEquals(0, tail.updatesSize()); } @Test - @DisplayName("Test Multi Threaded _ 2") - void testMultiThreaded_2() throws Exception { + public void testMultiThreaded_2() throws Exception { val numReaders = 4; val numWriters = 4; final val tail = new IndexedTail(numReaders); + final long[] sums = new long[numReaders]; val readers = new ArrayList(); for (int e = 0; e < numReaders; e++) { final int f = e; val t = new Thread(new Runnable() { - @Override public void run() { sums[f] = 0; @@ -264,51 +317,67 @@ class IndexedTailTest extends BaseDL4JTest { } } }); + t.setName("reader thread " + f); t.start(); readers.add(t); } + val writers = new ArrayList(); for (int e = 0; e < numWriters; e++) { val f = e; val t = new Thread(new Runnable() { - @Override public void run() { int sum = 0; for (int e = 0; e < 1000; e++) { - val array = Nd4j.create(5, 5).assign(e + 1); + val array = Nd4j.create(5, 5).assign(e+1); Nd4j.getExecutioner().commit(); - sum += (e + 1); + + sum += (e+1); tail.put(array); } } }); + t.setName("writer thread " + f); t.start(); writers.add(t); } - for (val t : writers) t.join(); + + + + for (val t:writers) + t.join(); + // just wait till everything consumed Thread.sleep(2000); tail.notifyDead(); - for (val t : readers) t.join(); - for (int e = 0; e < numReaders; e++) assertEquals(500500 * numWriters, sums[e],"Failed for reader [" + e + "]"); + + + + for (val t:readers) + t.join(); + + + for (int e = 0; e < numReaders; e++) + assertEquals(500500 * numWriters, sums[e], "Failed for reader [" + e + "]"); + + assertEquals(0, tail.updatesSize()); } @Test - @DisplayName("Test Multi Threaded _ 3") - void testMultiThreaded_3() throws Exception { + public void testMultiThreaded_3() throws Exception { val numReaders = 4; val numWriters = 4; - final val tail = new IndexedTail(numReaders, true, new long[] { 5, 5 }); + final val tail = new IndexedTail(numReaders, true, new long[]{5, 5}); + final long[] sums = new long[numReaders]; val readers = new ArrayList(); for (int e = 0; e < numReaders; e++) { final int f = e; val t = new Thread(new Runnable() { - @Override public void run() { sums[f] = 0; @@ -322,37 +391,52 @@ class IndexedTailTest extends BaseDL4JTest { } } }); + t.setName("reader thread " + f); t.start(); readers.add(t); } + final AtomicInteger sum = new AtomicInteger(0); val writers = new ArrayList(); for (int e = 0; e < numWriters; e++) { val f = e; val t = new Thread(new Runnable() { - @Override public void run() { for (int i = 0; i < 256; i++) { - val array = Nd4j.create(5, 5).assign(i + 1); + + val array = Nd4j.create(5, 5).assign(i+1); Nd4j.getExecutioner().commit(); - sum.addAndGet(i + 1); + + sum.addAndGet(i+1); tail.put(array); } } }); + t.setName("writer thread " + f); t.start(); writers.add(t); } - for (val t : writers) t.join(); + + + for (val t:writers) + t.join(); + // just wait till everything consumed Thread.sleep(3000); tail.notifyDead(); - for (val t : readers) t.join(); + + for (val t:readers) + t.join(); + log.info("Readers results: {}", sums); - for (int e = 0; e < numReaders; e++) assertEquals(sum.get(), sums[e],"Failed for reader [" + e + "]"); + + for (int e = 0; e < numReaders; e++) + assertEquals(sum.get(), sums[e], "Failed for reader [" + e + "]"); + + assertEquals(0, tail.updatesSize()); } -} +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java new file mode 100644 index 000000000..5d713ca59 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java @@ -0,0 +1,363 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.optimize.solver.accumulation; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.lang3.RandomUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.optimize.solvers.accumulation.SmartFancyBlockingQueue; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.common.util.ThreadUtils; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +////@Ignore("AB 2019/05/21 - Failing (stuck, causing timeouts) - Issue #7657") +@Timeout(120000L) +public class SmartFancyBlockingQueueTest extends BaseDL4JTest { + + @Test + public void testSFBQ_1() throws Exception { + val queue = new SmartFancyBlockingQueue(8, Nd4j.create(5, 5)); + + val array = Nd4j.create(5, 5); + + for (int e = 0; e < 6; e++) { + queue.put(Nd4j.create(5, 5).assign(e)); + }; + + assertEquals(6, queue.size()); + + for (int e = 6; e < 10; e++) { + queue.put(Nd4j.create(5, 5).assign(e)); + } + + assertEquals(1, queue.size()); + } + + @Test + public void testSFBQ_2() throws Exception { + final val queue = new SmartFancyBlockingQueue(1285601, Nd4j.create(5, 5)); + final val barrier = new CyclicBarrier(4); + + val threads = new ArrayList(); + for (int e = 0; e< 4; e++) { + val f = e; + val t = new Thread(new Runnable() { + @Override + public void run() { + int cnt = 0; + while (true) { + while (cnt < 1000) { + if (!queue.isEmpty()) { + if (cnt % 50 == 0) + log.info("Thread {}: [{}]", f, cnt); + + val arr = queue.poll(); + + assertNotNull(arr); + val local = arr.unsafeDuplication(true); + + assertEquals(cnt, local.meanNumber().intValue()); + cnt++; + } + + + try { + barrier.await(); + + if (f == 0) + queue.registerConsumers(4); + + barrier.await(); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } catch (BrokenBarrierException e1) { + e1.printStackTrace(); + } + } + break; + } + + + } + }); + t.setName("reader thread " + f); + t.start(); + threads.add(t); + } + + for (int e = 0; e < 1000; e++) { + queue.put(Nd4j.create(5, 5).assign(e)); + Nd4j.getExecutioner().commit(); + } + + + for (val t: threads) + t.join(); + } + + + @Test + @Tag("long-running") + public void testSFBQ_3() throws Exception { + final val queue = new SmartFancyBlockingQueue(1285601, Nd4j.create(5, 5)); + + val threads = new ArrayList(); + for (int e = 0; e< 4; e++) { + val f = e; + val t = new Thread(new Runnable() { + @Override + public void run() { + int cnt = 0; + while (true) { + while (cnt < 1000) { + if (!queue.isEmpty()) { + if (cnt % 50 == 0) + log.info("Thread {}: [{}]", f, cnt); + + val arr = queue.poll(); + + assertNotNull(arr); + val local = arr.unsafeDuplication(true); + cnt++; + } + } + break; + } + } + }); + t.start(); + threads.add(t); + } + + val b = new Thread(new Runnable() { + @Override + public void run() { + while (true) { + queue.registerConsumers(4); + ThreadUtils.uncheckedSleep(30); + } + } + }); + + b.setDaemon(true); + b.start(); + + val writers = new ArrayList(); + for (int e = 0; e < 4; e++) { + val t = new Thread(new Runnable() { + @Override + public void run() { + for (int e = 0; e <250; e++) { + try { + queue.put(Nd4j.createUninitialized(5, 5).assign(e)); + Thread.sleep(30); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + } + }); + + writers.add(t); + t.start(); + } + + for (val t: writers) + t.join(); + + for (val t: threads) + t.join(); + } + + @Test + @Tag("long-running") + public void testSFBQ_4() throws Exception { + final val queue = new SmartFancyBlockingQueue(16, Nd4j.create(5, 5)); + final val barrier = new CyclicBarrier(4); +/* + val m = new Thread(new Runnable() { + @Override + public void run() { + while (true) { + queue.registerConsumers(4); + ThreadUtils.uncheckedSleep(100); + } + } + }); + + + m.setName("master thread"); + m.setDaemon(true); + m.start(); +*/ + + val threads = new ArrayList(); + for (int e = 0; e < 4; e++) { + val f= e; + val t = new Thread(new Runnable() { + @Override + public void run() { + try { + for (int e = 0; e < 100; e++) { + + log.info("[Thread {}]: fill phase {}", f, e); + val numUpdates = RandomUtils.nextInt(8, 128); + for (int p = 0; p < numUpdates; p++) { + queue.put(Nd4j.createUninitialized(5, 5)); + } + + if (f == 0) + queue.registerConsumers(4); + + barrier.await(); + log.info("[Thread {}]: read phase {}", f, e); + while (!queue.isEmpty()) { + val arr = queue.poll(); + + assertNotNull(arr); + } + + barrier.await(); + + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (BrokenBarrierException e) { + throw new RuntimeException(e); + } + } + }); + + t.setName("worker thread " + f); + t.start(); + threads.add(t); + } + + for (val t:threads) + t.join(); + } + + + @Test + @Tag("long-running") + public void testSFBQ_5() throws Exception { + final val queue = new SmartFancyBlockingQueue(16, Nd4j.create(5, 5)); + final val barrier = new CyclicBarrier(4); + + // writers are just spamming updates every X ms + val writers = new ArrayList(); + for (int e = 0; e < 4; e++) { + val w = new Thread(new Runnable() { + @Override + public void run() { + while (true) { + try { + val n = RandomUtils.nextInt(8, 64); + for (int i = 1; i < n+1; i++) { + val arr = Nd4j.createUninitialized(5, 5).assign(i); + Nd4j.getExecutioner().commit(); + queue.put(arr); + } + + ThreadUtils.uncheckedSleep(10); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + }); + + w.setName("writer thread " + e); + w.setDaemon(true); + w.start(); + writers.add(w); + } + + // each reader will read 250 updates. supposedly equal :) + final long[] means = new long[4]; + val readers = new ArrayList(); + for (int e = 0; e < 4; e++) { + final int f = e; + means[f] = 0; + val t = new Thread(new Runnable() { + @Override + public void run() { + try { + int cnt = 0; + int fnt = 0; + while (cnt < 1000) { + + if (!queue.isEmpty()) { + while (!queue.isEmpty()) { + val m = queue.poll(); + + val arr = m.unsafeDuplication(true); + val mean = arr.meanNumber().longValue(); + assertNotEquals(0, mean, "Failed at cycle: " + cnt); + means[f] += mean; + + cnt++; + } + barrier.await(); + } + + barrier.await(); + + if (f == 0) { + log.info("Read cycle finished"); + queue.registerConsumers(4); + } + + barrier.await(); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (BrokenBarrierException e) { + throw new RuntimeException(e); + } + } + }); + + t.setName("reader thread " + f); + t.start(); + readers.add(t); + } + + + for (val t:readers) + t.join(); + + // all messages should be the same + assertEquals(means[0], means[1]); + assertEquals(means[0], means[2]); + assertEquals(means[0], means[3]); + } +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/ThresholdAlgorithmTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/ThresholdAlgorithmTests.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/ThresholdAlgorithmTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/ThresholdAlgorithmTests.java index a114ee092..390093217 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/ThresholdAlgorithmTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/ThresholdAlgorithmTests.java @@ -24,10 +24,7 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithmReducer; import org.deeplearning4j.optimize.solvers.accumulation.encoding.threshold.AdaptiveThresholdAlgorithm; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -35,8 +32,6 @@ import java.lang.reflect.Field; import static org.junit.jupiter.api.Assertions.assertEquals; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class ThresholdAlgorithmTests extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/ScoreStatTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/ScoreStatTest.java similarity index 77% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/ScoreStatTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/ScoreStatTest.java index 536697a4c..8d3b3751e 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/ScoreStatTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/ScoreStatTest.java @@ -17,101 +17,105 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.optimizer.listener; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.optimize.listeners.CollectScoresIterationListener; -import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.List; import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Score Stat Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class ScoreStatTest extends BaseDL4JTest { +public class ScoreStatTest extends BaseDL4JTest { @Test - @DisplayName("Test Score Stat Small") - void testScoreStatSmall() { + public void testScoreStatSmall() { CollectScoresIterationListener.ScoreStat statTest = new CollectScoresIterationListener.ScoreStat(); for (int i = 0; i < CollectScoresIterationListener.ScoreStat.BUCKET_LENGTH; ++i) { - double score = (double) i; + double score = (double)i; statTest.addScore(i, score); } + List indexes = statTest.getIndexes(); List scores = statTest.getScores(); + assertTrue(indexes.size() == 1); assertTrue(scores.size() == 1); + assertTrue(indexes.get(0).length == CollectScoresIterationListener.ScoreStat.BUCKET_LENGTH); assertTrue(scores.get(0).length == CollectScoresIterationListener.ScoreStat.BUCKET_LENGTH); - assertEquals(indexes.get(0)[indexes.get(0).length - 1], CollectScoresIterationListener.ScoreStat.BUCKET_LENGTH - 1); - assertEquals(scores.get(0)[scores.get(0).length - 1], CollectScoresIterationListener.ScoreStat.BUCKET_LENGTH - 1, 1e-4); + assertEquals(indexes.get(0)[indexes.get(0).length-1], CollectScoresIterationListener.ScoreStat.BUCKET_LENGTH-1); + assertEquals(scores.get(0)[scores.get(0).length-1], CollectScoresIterationListener.ScoreStat.BUCKET_LENGTH-1, 1e-4); } @Test - @DisplayName("Test Score Stat Average") - void testScoreStatAverage() { + public void testScoreStatAverage() { int dataSize = 1000000; long[] indexes = new long[dataSize]; double[] scores = new double[dataSize]; + for (int i = 0; i < dataSize; ++i) { indexes[i] = i; scores[i] = i; } + CollectScoresIterationListener.ScoreStat statTest = new CollectScoresIterationListener.ScoreStat(); for (int i = 0; i < dataSize; ++i) { statTest.addScore(indexes[i], scores[i]); } + long[] indexesStored = statTest.getIndexes().get(0); double[] scoresStored = statTest.getScores().get(0); + assertArrayEquals(indexes, indexesStored); assertArrayEquals(scores, scoresStored, 1e-4); } @Test - @DisplayName("Test Scores Clean") - void testScoresClean() { - // expected to be placed in 2 buckets of 10k elements size - int dataSize = 10256; + public void testScoresClean() { + int dataSize = 10256; // expected to be placed in 2 buckets of 10k elements size long[] indexes = new long[dataSize]; double[] scores = new double[dataSize]; + for (int i = 0; i < dataSize; ++i) { indexes[i] = i; scores[i] = i; } + CollectScoresIterationListener.ScoreStat statTest = new CollectScoresIterationListener.ScoreStat(); for (int i = 0; i < dataSize; ++i) { statTest.addScore(indexes[i], scores[i]); } + long[] indexesEffective = statTest.getEffectiveIndexes(); double[] scoresEffective = statTest.getEffectiveScores(); + assertArrayEquals(indexes, indexesEffective); assertArrayEquals(scores, scoresEffective, 1e-4); } - @Disabled @Test - @DisplayName("Test Score Stat Big") - void testScoreStatBig() { + @Tag("long-running") + public void testScoreStatBig() { CollectScoresIterationListener.ScoreStat statTest = new CollectScoresIterationListener.ScoreStat(); - long bigLength = (long) Integer.MAX_VALUE + 5; + long bigLength = (long)Integer.MAX_VALUE + 5; for (long i = 0; i < bigLength; ++i) { - double score = (double) i; + double score = (double)i; statTest.addScore(i, score); } + List indexes = statTest.getIndexes(); List scores = statTest.getScores(); + assertTrue(indexes.size() == 2); assertTrue(scores.size() == 2); + for (int i = 0; i < 5; ++i) { assertTrue(indexes.get(1)[i] == Integer.MAX_VALUE + i); assertTrue(scores.get(1)[i] == Integer.MAX_VALUE + i); + } } } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java similarity index 91% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java index 9ed25912a..9b94b1b2c 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestCheckpointListener.java @@ -29,20 +29,14 @@ import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.optimize.listeners.Checkpoint; import org.deeplearning4j.optimize.listeners.CheckpointListener; import org.deeplearning4j.util.ModelSerializer; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.lossfunctions.LossFunctions; import org.nd4j.common.primitives.Pair; import java.io.File; -import java.nio.file.Path; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -50,8 +44,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class TestCheckpointListener extends BaseDL4JTest { @Override @@ -59,6 +52,8 @@ public class TestCheckpointListener extends BaseDL4JTest { return 90000L; } + @TempDir + public File tempDir; private static Pair getNetAndData(){ MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() @@ -76,8 +71,8 @@ public class TestCheckpointListener extends BaseDL4JTest { } @Test - public void testCheckpointListenerEvery2Epochs(@TempDir Path tempDir) throws Exception { - File f = tempDir.toFile(); + public void testCheckpointListenerEvery2Epochs() throws Exception { + File f = tempDir; Pair p = getNetAndData(); MultiLayerNetwork net = p.getFirst(); DataSetIterator iter = p.getSecond(); @@ -125,8 +120,8 @@ public class TestCheckpointListener extends BaseDL4JTest { } @Test - public void testCheckpointListenerEvery5Iter(@TempDir Path tempDir) throws Exception { - File f = tempDir.toFile(); + public void testCheckpointListenerEvery5Iter() throws Exception { + File f = tempDir; Pair p = getNetAndData(); MultiLayerNetwork net = p.getFirst(); DataSetIterator iter = p.getSecond(); @@ -163,7 +158,7 @@ public class TestCheckpointListener extends BaseDL4JTest { count++; } - assertEquals( 3, ns.size(),ns.toString()); + assertEquals( 3, ns.size(), ns.toString()); assertTrue(ns.contains(25)); assertTrue(ns.contains(30)); assertTrue(ns.contains(35)); @@ -182,8 +177,8 @@ public class TestCheckpointListener extends BaseDL4JTest { } @Test - public void testCheckpointListenerEveryTimeUnit(@TempDir Path tempDir) throws Exception { - File f = tempDir.toFile(); + public void testCheckpointListenerEveryTimeUnit() throws Exception { + File f = tempDir; Pair p = getNetAndData(); MultiLayerNetwork net = p.getFirst(); DataSetIterator iter = p.getSecond(); @@ -220,14 +215,14 @@ public class TestCheckpointListener extends BaseDL4JTest { } assertEquals(2, l.availableCheckpoints().size()); - assertEquals(2, ns.size(),ns.toString()); + assertEquals(2, ns.size(), ns.toString()); System.out.println(ns); assertTrue(ns.containsAll(Arrays.asList(2,4))); } @Test - public void testCheckpointListenerKeepLast3AndEvery3(@TempDir Path tempDir) throws Exception { - File f = tempDir.toFile(); + public void testCheckpointListenerKeepLast3AndEvery3() throws Exception { + File f = tempDir; Pair p = getNetAndData(); MultiLayerNetwork net = p.getFirst(); DataSetIterator iter = p.getSecond(); @@ -265,15 +260,15 @@ public class TestCheckpointListener extends BaseDL4JTest { count++; } - assertEquals(5, ns.size(),ns.toString()); - assertTrue(ns.containsAll(Arrays.asList(5, 11, 15, 17, 19)),ns.toString()); + assertEquals(5, ns.size(), ns.toString()); + assertTrue(ns.containsAll(Arrays.asList(5, 11, 15, 17, 19)), ns.toString()); assertEquals(5, l.availableCheckpoints().size()); } @Test - public void testDeleteExisting(@TempDir Path tempDir) throws Exception { - File f = tempDir.toFile(); + public void testDeleteExisting() throws Exception { + File f = tempDir; Pair p = getNetAndData(); MultiLayerNetwork net = p.getFirst(); DataSetIterator iter = p.getSecond(); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestFailureListener.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestFailureListener.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestFailureListener.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestFailureListener.java index 8c662cec1..81786baa7 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestFailureListener.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestFailureListener.java @@ -27,11 +27,9 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.optimize.listeners.FailureTestingListener; + import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.learning.config.Adam; @@ -47,12 +45,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; * They should be run manually, not as part of standard unit test run. */ @Disabled -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.MANUAL) public class TestFailureListener extends BaseDL4JTest { - @Disabled + ////@Ignore @Test public void testFailureIter5() throws Exception { diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java new file mode 100644 index 000000000..48e610dfb --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java @@ -0,0 +1,326 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.optimizer.listener; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.listener.RoutingIterationListener; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.AutoEncoder; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.optimize.api.BaseTrainingListener; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.optimize.listeners.ComposableIterationListener; +import org.deeplearning4j.optimize.listeners.PerformanceListener; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.optimize.listeners.TimeIterationListener; +import org.deeplearning4j.optimize.listeners.CheckpointListener; +import org.deeplearning4j.optimize.solvers.BaseOptimizer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.primitives.Triple; + +import java.io.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class TestListeners extends BaseDL4JTest { + + @TempDir + public File tempDir; + + @Override + public long getTimeoutMilliseconds() { + return 90000L; + } + + @Test + public void testSettingListenersUnsupervised() { + //Pretrain layers should get copies of the listeners, in addition to the + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new AutoEncoder.Builder().nIn(10).nOut(10).build()) + .layer(1, new VariationalAutoencoder.Builder().nIn(10).nOut(10).build()).build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + net.setListeners(new ScoreIterationListener(), new TestRoutingListener()); + + for (Layer l : net.getLayers()) { + Collection layerListeners = l.getListeners(); + assertEquals(2, layerListeners.size(), l.getClass().toString()); + TrainingListener[] lArr = layerListeners.toArray(new TrainingListener[2]); + assertTrue(lArr[0] instanceof ScoreIterationListener); + assertTrue(lArr[1] instanceof TestRoutingListener); + } + + Collection netListeners = net.getListeners(); + assertEquals(2, netListeners.size()); + TrainingListener[] lArr = netListeners.toArray(new TrainingListener[2]); + assertTrue(lArr[0] instanceof ScoreIterationListener); + assertTrue(lArr[1] instanceof TestRoutingListener); + + + ComputationGraphConfiguration gConf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in") + .addLayer("0", new AutoEncoder.Builder().nIn(10).nOut(10).build(), "in") + .addLayer("1", new VariationalAutoencoder.Builder().nIn(10).nOut(10).build(), "0") + .setOutputs("1").build(); + ComputationGraph cg = new ComputationGraph(gConf); + cg.init(); + + cg.setListeners(new ScoreIterationListener(), new TestRoutingListener()); + + for (Layer l : cg.getLayers()) { + Collection layerListeners = l.getListeners(); + assertEquals(2, layerListeners.size()); + lArr = layerListeners.toArray(new TrainingListener[2]); + assertTrue(lArr[0] instanceof ScoreIterationListener); + assertTrue(lArr[1] instanceof TestRoutingListener); + } + + netListeners = cg.getListeners(); + assertEquals(2, netListeners.size()); + lArr = netListeners.toArray(new TrainingListener[2]); + assertTrue(lArr[0] instanceof ScoreIterationListener); + assertTrue(lArr[1] instanceof TestRoutingListener); + } + + private static class TestRoutingListener extends BaseTrainingListener implements RoutingIterationListener { + + @Override + public void setStorageRouter(StatsStorageRouter router) {} + + @Override + public StatsStorageRouter getStorageRouter() { + return null; + } + + @Override + public void setWorkerID(String workerID) {} + + @Override + public String getWorkerID() { + return null; + } + + @Override + public void setSessionID(String sessionID) {} + + @Override + public String getSessionID() { + return null; + } + + @Override + public RoutingIterationListener clone() { + return null; + } + + @Override + public void iterationDone(Model model, int iteration, int epoch) {} + } + + + + + + @Test + public void testListenerSerialization() throws Exception { + //Note: not all listeners are (or should be) serializable. But some should be - for Spark etc + + List listeners = new ArrayList<>(); + listeners.add(new ScoreIterationListener()); + listeners.add(new PerformanceListener(1, true, true)); + listeners.add(new TimeIterationListener(10000)); + listeners.add(new ComposableIterationListener(new ScoreIterationListener(), new PerformanceListener(1, true, true))); + listeners.add(new CheckpointListener.Builder(tempDir).keepAll().saveEveryNIterations(3).build()); //Doesn't usually need to be serialized, but no reason it can't be... + + + DataSetIterator iter = new IrisDataSetIterator(10, 150); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new OutputLayer.Builder().nIn(4).nOut(3) + .activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + net.setListeners(listeners); + + net.fit(iter); + + List listeners2 = new ArrayList<>(); + for(TrainingListener il : listeners){ + log.info("------------------"); + log.info("Testing listener: {}", il); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(il); + byte[] bytes = baos.toByteArray(); + + ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); + TrainingListener il2 = (TrainingListener) ois.readObject(); + + listeners2.add(il2); + } + + net.setListeners(listeners2); + net.fit(iter); + } + + + @Test + public void testListenerCalls(){ + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + TestListener tl = new TestListener(); + net.setListeners(tl); + + DataSetIterator irisIter = new IrisDataSetIterator(50, 150); + + net.fit(irisIter, 2); + + List> exp = new ArrayList<>(); + exp.add(new Triple<>(Call.EPOCH_START, 0, 0)); + exp.add(new Triple<>(Call.ON_FWD, 0, 0)); + exp.add(new Triple<>(Call.ON_BWD, 0, 0)); + exp.add(new Triple<>(Call.ON_GRAD, 0, 0)); + exp.add(new Triple<>(Call.ITER_DONE, 0, 0)); + exp.add(new Triple<>(Call.ON_FWD, 1, 0)); + exp.add(new Triple<>(Call.ON_BWD, 1, 0)); + exp.add(new Triple<>(Call.ON_GRAD, 1, 0)); + exp.add(new Triple<>(Call.ITER_DONE, 1, 0)); + exp.add(new Triple<>(Call.ON_FWD, 2, 0)); + exp.add(new Triple<>(Call.ON_BWD, 2, 0)); + exp.add(new Triple<>(Call.ON_GRAD, 2, 0)); + exp.add(new Triple<>(Call.ITER_DONE, 2, 0)); + exp.add(new Triple<>(Call.EPOCH_END, 3, 0)); //Post updating iter count, pre update epoch count + + exp.add(new Triple<>(Call.EPOCH_START, 3, 1)); + exp.add(new Triple<>(Call.ON_FWD, 3, 1)); + exp.add(new Triple<>(Call.ON_BWD, 3, 1)); + exp.add(new Triple<>(Call.ON_GRAD, 3, 1)); + exp.add(new Triple<>(Call.ITER_DONE, 3, 1)); + exp.add(new Triple<>(Call.ON_FWD, 4, 1)); + exp.add(new Triple<>(Call.ON_BWD, 4, 1)); + exp.add(new Triple<>(Call.ON_GRAD, 4, 1)); + exp.add(new Triple<>(Call.ITER_DONE, 4, 1)); + exp.add(new Triple<>(Call.ON_FWD, 5, 1)); + exp.add(new Triple<>(Call.ON_BWD, 5, 1)); + exp.add(new Triple<>(Call.ON_GRAD, 5, 1)); + exp.add(new Triple<>(Call.ITER_DONE, 5, 1)); + exp.add(new Triple<>(Call.EPOCH_END, 6, 1)); + + + assertEquals(exp, tl.getCalls()); + + + tl = new TestListener(); + + ComputationGraph cg = net.toComputationGraph(); + cg.setListeners(tl); + + cg.fit(irisIter, 2); + + assertEquals(exp, tl.getCalls()); + } + + private static enum Call { + ITER_DONE, + EPOCH_START, + EPOCH_END, + ON_FWD, + ON_GRAD, + ON_BWD + } + + @Data + private static class TestListener implements TrainingListener { + + private List> calls = new ArrayList<>(); + + + @Override + public void iterationDone(Model model, int iteration, int epoch) { + calls.add(new Triple<>(Call.ITER_DONE, iteration, epoch)); + } + + @Override + public void onEpochStart(Model model) { + calls.add(new Triple<>(Call.EPOCH_START, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); + } + + @Override + public void onEpochEnd(Model model) { + calls.add(new Triple<>(Call.EPOCH_END, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); + } + + @Override + public void onForwardPass(Model model, List activations) { + calls.add(new Triple<>(Call.ON_FWD, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); + } + + @Override + public void onForwardPass(Model model, Map activations) { + calls.add(new Triple<>(Call.ON_FWD, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); + } + + @Override + public void onGradientCalculation(Model model) { + calls.add(new Triple<>(Call.ON_GRAD, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); + } + + @Override + public void onBackwardPass(Model model) { + calls.add(new Triple<>(Call.ON_BWD, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); + } + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/AsyncIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/AsyncIteratorTest.java similarity index 80% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/AsyncIteratorTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/AsyncIteratorTest.java index d020dcb33..632e7f081 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/AsyncIteratorTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/AsyncIteratorTest.java @@ -17,31 +17,26 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.parallelism; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.core.parallelism.AsyncIterator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import java.util.ArrayList; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Async Iterator Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class AsyncIteratorTest extends BaseDL4JTest { +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class AsyncIteratorTest extends BaseDL4JTest { @Test - @DisplayName("Has Next") - void hasNext() throws Exception { + public void hasNext() throws Exception { ArrayList integers = new ArrayList<>(); for (int x = 0; x < 100000; x++) { integers.add(x); } + AsyncIterator iterator = new AsyncIterator<>(integers.iterator(), 512); int cnt = 0; Integer val = null; @@ -50,7 +45,10 @@ class AsyncIteratorTest extends BaseDL4JTest { assertEquals(cnt, val.intValue()); cnt++; } + System.out.println("Last val: " + val); + assertEquals(integers.size(), cnt); } + } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/FancyBlockingQueueTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/FancyBlockingQueueTests.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/FancyBlockingQueueTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/FancyBlockingQueueTests.java index a33292bb1..82dfedaeb 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/FancyBlockingQueueTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/FancyBlockingQueueTests.java @@ -24,10 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomUtils; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.optimize.solvers.accumulation.FancyBlockingQueue; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; @@ -35,8 +32,6 @@ import java.util.concurrent.atomic.AtomicLong; import static org.junit.jupiter.api.Assertions.assertEquals; @Slf4j -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class FancyBlockingQueueTests extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/MultiBooleanTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/MultiBooleanTest.java similarity index 76% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/MultiBooleanTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/MultiBooleanTest.java index 13ee9d69b..4ebd94fbc 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/MultiBooleanTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/MultiBooleanTest.java @@ -17,78 +17,89 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.parallelism; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.parallel.MultiBoolean; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -@DisplayName("Multi Boolean Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class MultiBooleanTest extends BaseDL4JTest { +public class MultiBooleanTest extends BaseDL4JTest { @Test - @DisplayName("Test Boolean 1") - void testBoolean1() throws Exception { + public void testBoolean1() throws Exception { MultiBoolean bool = new MultiBoolean(5); + assertTrue(bool.allFalse()); assertFalse(bool.allTrue()); } + @Test - @DisplayName("Test Boolean 2") - void testBoolean2() throws Exception { + public void testBoolean2() throws Exception { MultiBoolean bool = new MultiBoolean(5); + bool.set(true, 2); + assertFalse(bool.allFalse()); assertFalse(bool.allTrue()); } @Test - @DisplayName("Test Boolean 3") - void testBoolean3() throws Exception { + public void testBoolean3() throws Exception { MultiBoolean bool = new MultiBoolean(5); + bool.set(true, 0); bool.set(true, 1); bool.set(true, 2); + + bool.set(true, 3); + assertFalse(bool.allTrue()); + bool.set(true, 4); + assertFalse(bool.allFalse()); assertTrue(bool.allTrue()); + bool.set(false, 2); + assertFalse(bool.allTrue()); + bool.set(true, 2); + assertTrue(bool.allTrue()); } @Test - @DisplayName("Test Boolean 4") - void testBoolean4() throws Exception { + public void testBoolean4() throws Exception { MultiBoolean bool = new MultiBoolean(5, true); + + assertTrue(bool.get(1)); + bool.set(false, 1); + assertFalse(bool.get(1)); } + @Test - @DisplayName("Test Boolean 5") - void testBoolean5() throws Exception { + public void testBoolean5() throws Exception { MultiBoolean bool = new MultiBoolean(5, true, true); + for (int i = 0; i < 5; i++) { bool.set(false, i); } + for (int i = 0; i < 5; i++) { bool.set(true, i); } + assertTrue(bool.allFalse()); } } diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/ParallelExistingMiniBatchDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/ParallelExistingMiniBatchDataSetIteratorTest.java new file mode 100644 index 000000000..e8db44b1f --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/ParallelExistingMiniBatchDataSetIteratorTest.java @@ -0,0 +1,176 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.parallelism; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.io.ClassPathResource; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.callbacks.DataSetDeserializer; +import org.deeplearning4j.datasets.iterator.parallel.FileSplitParallelDataSetIterator; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.common.primitives.Pair; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@Slf4j +public class ParallelExistingMiniBatchDataSetIteratorTest extends BaseDL4JTest { + + @TempDir + public File tempDir; + private static File rootFolder; + + @BeforeEach + public void setUp() throws Exception { + if (rootFolder == null) { + rootFolder = new File(tempDir, "a"); + for( int i=0; i<26; i++){ + new ClassPathResource("/datasets/mnist/mnist-train-" + i + ".bin").getTempFileFromArchive(rootFolder); + } + } + } + + + @Test + @Timeout(300) + public void testNewSimpleLoop1() throws Exception { + FileSplitParallelDataSetIterator fspdsi = new FileSplitParallelDataSetIterator(rootFolder, "mnist-train-%d.bin", + new DataSetDeserializer()); + + List> pairs = new ArrayList<>(); + + + long time1 = System.nanoTime(); + int cnt = 0; + while (fspdsi.hasNext()) { + DataSet ds = fspdsi.next(); + long time2 = System.nanoTime(); + pairs.add(new Pair(time2 - time1, 0L)); + assertNotNull(ds); + + // imitating processing here + Thread.sleep(10); + + cnt++; + time1 = System.nanoTime(); + } + + assertEquals(26, cnt); + + for (Pair times : pairs) { + log.info("Parallel: {} ns; Simple: {} ns", times.getFirst(), times.getSecond()); + } + } + + + /* + @Test + public void testSimpleLoop1() throws Exception { + ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 4); + ExistingMiniBatchDataSetIterator test = new ExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin"); + + + List> pairs = new ArrayList<>(); + + int cnt = 0; + long time1 = System.nanoTime(); + while (iterator.hasNext()) { + DataSet ds = iterator.next(); + long time2 = System.nanoTime(); + assertNotNull(ds); + assertEquals(64, ds.numExamples()); + pairs.add(new Pair(time2 - time1, 0L)); + cnt++; + time1 = System.nanoTime(); + } + assertEquals(26, cnt); + + cnt = 0; + time1 = System.nanoTime(); + while (test.hasNext()) { + DataSet ds = test.next(); + long time2 = System.nanoTime(); + assertNotNull(ds); + assertEquals(64, ds.numExamples()); + pairs.get(cnt).setSecond(time2 - time1); + cnt++; + time1 = System.nanoTime(); + } + + assertEquals(26, cnt); + + for (Pair times: pairs) { + log.info("Parallel: {} ns; Simple: {} ns", times.getFirst(), times.getSecond()); + } + } + + @Test + public void testReset1() throws Exception { + ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 8); + + int cnt = 0; + long time1 = System.nanoTime(); + while (iterator.hasNext()) { + DataSet ds = iterator.next(); + long time2 = System.nanoTime(); + assertNotNull(ds); + assertEquals(64, ds.numExamples()); + cnt++; + + if (cnt == 10) + iterator.reset(); + + time1 = System.nanoTime(); + } + assertEquals(36, cnt); + } + + @Test + public void testWithAdsi1() throws Exception { + ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 8); + AsyncDataSetIterator adsi = new AsyncDataSetIterator(iterator, 8, true); + + int cnt = 0; + long time1 = System.nanoTime(); + while (adsi.hasNext()) { + DataSet ds = adsi.next(); + long time2 = System.nanoTime(); + assertNotNull(ds); + assertEquals(64, ds.numExamples()); + cnt++; + + if (cnt == 10) + adsi.reset(); + + time1 = System.nanoTime(); + } + assertEquals(36, cnt); + } + */ +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/RandomTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/RandomTests.java new file mode 100644 index 000000000..97a1cb799 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/parallelism/RandomTests.java @@ -0,0 +1,133 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.parallelism; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.WorkspaceMode; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RandomTests extends BaseDL4JTest { + + /** + * In this test we check for equality of model params after initialization in different threads + * + * @throws Exception + */ + @Test + public void testModelInitialParamsEquality1() throws Exception { + final List models = new CopyOnWriteArrayList<>(); + + for (int i = 0; i < 4; i++) { + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(119) // Training iterations as above + .l2(0.0005) + //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75) + .weightInit(WeightInit.XAVIER) + .updater(new Nesterovs(0.01, 0.9)) + .trainingWorkspaceMode(WorkspaceMode.ENABLED).list() + .layer(0, new ConvolutionLayer.Builder(5, 5) + //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied + .nIn(1).stride(1, 1).nOut(20).activation(Activation.IDENTITY) + .build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2).build()) + .layer(2, new ConvolutionLayer.Builder(5, 5) + //Note that nIn need not be specified in later layers + .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()) + .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) + .kernelSize(2, 2).stride(2, 2).build()) + .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()) + .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(10).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(28, 28, 1)) //See note below + .build(); + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + models.add(network); + } + }); + + thread.start(); + thread.join(); + } + + + // at the end of day, model params has to + for (int i = 0; i < models.size(); i++) { + assertEquals(models.get(0).params(), models.get(i).params()); + } + } + + + @Test + public void testRngInitMLN() { + Nd4j.getRandom().setSeed(12345); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).activation(Activation.TANH) + .weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2, + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(10).nOut(10).build()) + .build(); + + String json = conf.toJson(); + + MultiLayerNetwork net1 = new MultiLayerNetwork(conf); + net1.init(); + + MultiLayerNetwork net2 = new MultiLayerNetwork(conf); + net2.init(); + + assertEquals(net1.params(), net2.params()); + + MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json); + + Nd4j.getRandom().setSeed(987654321); + MultiLayerNetwork net3 = new MultiLayerNetwork(fromJson); + net3.init(); + + assertEquals(net1.params(), net3.params()); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/SystemPollingTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/SystemPollingTest.java new file mode 100644 index 000000000..bb0c0a617 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/SystemPollingTest.java @@ -0,0 +1,65 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.perf.listener; + +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.core.listener.HardwareMetric; +import org.deeplearning4j.core.listener.SystemPolling; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +//@Ignore("AB 2019/05/24 - Failing on CI - \"Could not initialize class oshi.jna.platform.linux.Libc\" - Issue #7657") +public class SystemPollingTest extends BaseDL4JTest { + + @TempDir + public File tempDir; + + @Test + public void testPolling() throws Exception { + Nd4j.create(1); + File tmpDir = tempDir; + + SystemPolling systemPolling = new SystemPolling.Builder() + .outputDirectory(tmpDir).pollEveryMillis(1000) + .build(); + systemPolling.run(); + + Thread.sleep(8000); + + systemPolling.stopPolling(); + + File[] files = tmpDir.listFiles(); + assertTrue(files != null && files.length > 0); + //System.out.println(Arrays.toString(files)); + + String yaml = FileUtils.readFileToString(files[0]); + HardwareMetric fromYaml = HardwareMetric.fromYaml(yaml); + System.out.println(fromYaml); + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/TestHardWareMetric.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/TestHardWareMetric.java similarity index 83% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/TestHardWareMetric.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/TestHardWareMetric.java index c5abceeb6..a95107943 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/TestHardWareMetric.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/TestHardWareMetric.java @@ -22,19 +22,13 @@ package org.deeplearning4j.perf.listener; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.core.listener.HardwareMetric; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import oshi.json.SystemInfo; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; -@Disabled("AB 2019/05/24 - Failing on CI - \"Could not initialize class oshi.jna.platform.linux.Libc\" - Issue #7657") -@NativeTag -@Tag(TagNames.JACKSON_SERDE) +////@Ignore("AB 2019/05/24 - Failing on CI - \"Could not initialize class oshi.jna.platform.linux.Libc\" - Issue #7657") public class TestHardWareMetric extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/TestSystemInfoPrintListener.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/TestSystemInfoPrintListener.java similarity index 88% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/TestSystemInfoPrintListener.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/TestSystemInfoPrintListener.java index 4fe2be4e0..02e089090 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/TestSystemInfoPrintListener.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/perf/listener/TestSystemInfoPrintListener.java @@ -28,32 +28,28 @@ import org.deeplearning4j.nn.conf.MultiLayerConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Disabled; - import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; -@Disabled("AB 2019/05/24 - Failing on CI - \"Could not initialize class oshi.jna.platform.linux.Libc\" - Issue #7657") +////@Ignore("AB 2019/05/24 - Failing on CI - \"Could not initialize class oshi.jna.platform.linux.Libc\" - Issue #7657") public class TestSystemInfoPrintListener extends BaseDL4JTest { - + @TempDir + public File testDir; @Test - public void testListener(@TempDir Path testDir) throws Exception { + public void testListener() throws Exception { SystemInfoPrintListener systemInfoPrintListener = SystemInfoPrintListener.builder() .printOnEpochStart(true).printOnEpochEnd(true) .build(); - File tmpFile = Files.createTempFile(testDir,"tmpfile-log","txt").toFile(); + File tmpFile = new File(testDir, "tmpfile-log.txt"); assertEquals(0, tmpFile.length() ); SystemInfoFilePrintListener systemInfoFilePrintListener = SystemInfoFilePrintListener.builder() diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/MiscRegressionTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/MiscRegressionTests.java similarity index 95% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/MiscRegressionTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/MiscRegressionTests.java index 3a3728c74..686501ff8 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/MiscRegressionTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/MiscRegressionTests.java @@ -30,11 +30,8 @@ import org.deeplearning4j.nn.conf.graph.LayerVertex; import org.deeplearning4j.nn.conf.layers.DenseLayer; import org.deeplearning4j.nn.conf.layers.Layer; import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import java.nio.charset.StandardCharsets; @@ -43,8 +40,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class MiscRegressionTests extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java similarity index 96% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java index 69038a1e7..022545685 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest050.java @@ -32,12 +32,8 @@ import org.deeplearning4j.nn.weights.WeightInitDistribution; import org.deeplearning4j.nn.weights.WeightInitRelu; import org.deeplearning4j.nn.weights.WeightInitXavier; import org.deeplearning4j.util.ModelSerializer; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.Timeout; import org.nd4j.linalg.activations.impl.ActivationLReLU; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.factory.Nd4j; @@ -52,14 +48,10 @@ import org.nd4j.common.resources.Resources; import java.io.File; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + +@Timeout(300) public class RegressionTest050 extends BaseDL4JTest { - @Override - public long getTimeoutMilliseconds() { - return 180000L; //Most tests should be fast, but slow download may cause timeout on slow connections - } @Override public DataType getDataType(){ diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java index bec283ae4..da6976b6a 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest060.java @@ -37,10 +37,7 @@ import org.deeplearning4j.nn.weights.WeightInitDistribution; import org.deeplearning4j.nn.weights.WeightInitRelu; import org.deeplearning4j.nn.weights.WeightInitXavier; import org.deeplearning4j.util.ModelSerializer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.impl.ActivationLReLU; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.factory.Nd4j; @@ -55,8 +52,7 @@ import org.nd4j.common.resources.Resources; import java.io.File; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class RegressionTest060 extends BaseDL4JTest { @Override diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java index 650d3cd09..e2ef4b233 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest071.java @@ -37,10 +37,7 @@ import org.deeplearning4j.nn.weights.WeightInitDistribution; import org.deeplearning4j.nn.weights.WeightInitRelu; import org.deeplearning4j.nn.weights.WeightInitXavier; import org.deeplearning4j.util.ModelSerializer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.impl.ActivationIdentity; import org.nd4j.linalg.activations.impl.ActivationLReLU; import org.nd4j.linalg.api.buffer.DataType; @@ -56,8 +53,7 @@ import org.nd4j.common.resources.Resources; import java.io.File; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class RegressionTest071 extends BaseDL4JTest { @Override diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java index 8045ab8cc..b2af73f06 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest080.java @@ -37,10 +37,7 @@ import org.deeplearning4j.nn.weights.WeightInitDistribution; import org.deeplearning4j.nn.weights.WeightInitRelu; import org.deeplearning4j.nn.weights.WeightInitXavier; import org.deeplearning4j.util.ModelSerializer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.impl.*; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.factory.Nd4j; @@ -55,8 +52,7 @@ import org.nd4j.common.resources.Resources; import java.io.File; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class RegressionTest080 extends BaseDL4JTest { @Override diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java similarity index 94% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java index 572669849..d2b20bea3 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100a.java @@ -35,11 +35,7 @@ import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.impl.ActivationIdentity; import org.nd4j.linalg.activations.impl.ActivationLReLU; import org.nd4j.linalg.activations.impl.ActivationSoftmax; @@ -59,9 +55,6 @@ import java.io.FileInputStream; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled -@NativeTag -@Tag(TagNames.DL4J_OLD_API) public class RegressionTest100a extends BaseDL4JTest { @Override @@ -87,7 +80,7 @@ public class RegressionTest100a extends BaseDL4JTest { fail("Expected exception"); } catch (Exception e){ String msg = e.getMessage(); - assertTrue(msg.contains("custom") && msg.contains("1.0.0-beta") && msg.contains("saved again"), msg); + assertTrue( msg.contains("custom") && msg.contains("1.0.0-beta") && msg.contains("saved again"), msg); } } @@ -174,7 +167,7 @@ public class RegressionTest100a extends BaseDL4JTest { @Test - @Disabled("AB 2019/05/23 - Failing on linux-x86_64-cuda-9.2 - see issue #7657") + //@Ignore("AB 2019/05/23 - Failing on linux-x86_64-cuda-9.2 - see issue #7657") public void testYoloHouseNumber() throws Exception { File f = Resources.asFile("regression_testing/100a/HouseNumberDetection_100a.bin"); @@ -205,7 +198,7 @@ public class RegressionTest100a extends BaseDL4JTest { //Minor bug in 1.0.0-beta and earlier: not adding epsilon value to forward pass for batch norm //Which means: the record output doesn't have this. To account for this, we'll manually set eps to 0.0 here - //https://github.com/eclipse/deeplearning4j/issues/5836#issuecomment-405526228 + //https://github.com/deeplearning4j/deeplearning4j/issues/5836#issuecomment-405526228 for(Layer l : net.getLayers()){ if(l.conf().getLayer() instanceof BatchNormalization){ BatchNormalization bn = (BatchNormalization) l.conf().getLayer(); @@ -220,12 +213,12 @@ public class RegressionTest100a extends BaseDL4JTest { log.info("Expected: {}", outExp); log.info("Actual: {}", outAct); } - assertTrue(eq, "Output not equal"); + assertTrue( eq, "Output not equal"); } @Test - @Disabled("Ignoring due to new set input types changes. Loading a network isn't a problem, but we need to set the input types yet.") + //@Ignore("Ignoring due to new set input types changes. Loading a network isn't a problem, but we need to set the input types yet.") public void testUpsampling2d() throws Exception { File f = Resources.asFile("regression_testing/100a/upsampling/net.bin"); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java index 013491845..8cca8472e 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b3.java @@ -34,11 +34,8 @@ import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInitXavier; import org.deeplearning4j.regressiontest.customlayer100a.CustomLayer; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.impl.*; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -54,9 +51,7 @@ import java.io.FileInputStream; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -@Disabled -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class RegressionTest100b3 extends BaseDL4JTest { @Override @@ -120,7 +115,7 @@ public class RegressionTest100b3 extends BaseDL4JTest { assertEquals(dt, net.getLayerWiseConfigurations().getDataType()); assertEquals(dt, net.params().dataType()); - assertEquals(outExp, outAct, dtype); + assertEquals( outExp, outAct, dtype); } } @@ -207,6 +202,7 @@ public class RegressionTest100b3 extends BaseDL4JTest { @Test + //@Ignore("AB 2019/05/23 - Failing on linux-x86_64-cuda-9.2 - see issue #7657") public void testYoloHouseNumber() throws Exception { File f = Resources.asFile("regression_testing/100b3/HouseNumberDetection_100b3.bin"); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java index 259eb6408..71c928d84 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b4.java @@ -54,11 +54,7 @@ import org.deeplearning4j.nn.graph.vertex.impl.MergeVertex; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInitXavier; import org.deeplearning4j.regressiontest.customlayer100a.CustomLayer; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.impl.ActivationIdentity; import org.nd4j.linalg.activations.impl.ActivationLReLU; import org.nd4j.linalg.activations.impl.ActivationReLU; @@ -74,9 +70,7 @@ import org.nd4j.linalg.learning.regularization.L2Regularization; import org.nd4j.linalg.lossfunctions.impl.LossMAE; import org.nd4j.linalg.lossfunctions.impl.LossMCXENT; import org.nd4j.common.resources.Resources; -@Disabled -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class RegressionTest100b4 extends BaseDL4JTest { @Override @@ -139,7 +133,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { assertEquals(dtype, net.getLayerWiseConfigurations().getDataType()); assertEquals(dtype, net.params().dataType()); boolean eq = outExp.equalsWithEps(outAct, 0.01); - assertTrue(eq,"Test for dtype: " + dtypeName + "\n" + outExp + " vs " + outAct); + assertTrue(eq, "Test for dtype: " + dtypeName + "\n" + outExp + " vs " + outAct); } } @@ -226,7 +220,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { @Test - @Disabled("Failing due to new data format changes. Sept 10,2020") + ////@Ignore("Failing due to new data format changes. Sept 10,2020") public void testYoloHouseNumber() throws Exception { File f = Resources.asFile("regression_testing/100b4/HouseNumberDetection_100b4.bin"); @@ -262,7 +256,7 @@ public class RegressionTest100b4 extends BaseDL4JTest { } @Test - @Disabled("failing due to new input data format changes.") + ////@Ignore("failing due to new input data format changes.") public void testSyntheticCNN() throws Exception { File f = Resources.asFile("regression_testing/100b4/SyntheticCNN_100b4.bin"); diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java similarity index 98% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java index ae5684734..cbf45e56d 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/RegressionTest100b6.java @@ -35,11 +35,7 @@ import org.deeplearning4j.nn.graph.vertex.impl.MergeVertex; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInitXavier; import org.deeplearning4j.regressiontest.customlayer100a.CustomLayer; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.impl.*; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -56,9 +52,7 @@ import java.io.File; import java.io.FileInputStream; import static org.junit.jupiter.api.Assertions.*; -@Disabled -@NativeTag -@Tag(TagNames.DL4J_OLD_API) + public class RegressionTest100b6 extends BaseDL4JTest { @Override @@ -121,7 +115,7 @@ public class RegressionTest100b6 extends BaseDL4JTest { assertEquals(dtype, net.getLayerWiseConfigurations().getDataType()); assertEquals(dtype, net.params().dataType()); boolean eq = outExp.equalsWithEps(outAct, 0.01); - assertTrue(eq, "Test for dtype: " + dtypeName + " - " + outExp + " vs " + outAct); + assertTrue( eq, "Test for dtype: " + dtypeName + " - " + outExp + " vs " + outAct); } } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java similarity index 94% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java index ec4ee2caf..9d66e9b5a 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/TestDistributionDeserializer.java @@ -23,18 +23,12 @@ package org.deeplearning4j.regressiontest; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.distribution.*; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.JACKSON_SERDE) public class TestDistributionDeserializer extends BaseDL4JTest { @Override diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java new file mode 100644 index 000000000..00a2b6242 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java @@ -0,0 +1,175 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.regressiontest.customlayer100a; + +import lombok.Getter; +import lombok.Setter; +import lombok.val; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.ParamInitializer; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; +import org.deeplearning4j.nn.conf.memory.LayerMemoryReport; +import org.deeplearning4j.nn.conf.memory.MemoryReport; +import org.deeplearning4j.nn.params.DefaultParamInitializer; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.util.Collection; +import java.util.Map; + +public class CustomLayer extends FeedForwardLayer { + + private IActivation secondActivationFunction; + + public CustomLayer() { + //We need a no-arg constructor so we can deserialize the configuration from JSON or YAML format + // Without this, you will likely get an exception like the following: + //com.fasterxml.jackson.databind.JsonMappingException: No suitable constructor found for type [simple type, class org.deeplearning4j.examples.misc.customlayers.layer.CustomLayer]: can not instantiate from JSON object (missing default constructor or creator, or perhaps need to add/enable type information?) + } + + private CustomLayer(Builder builder) { + super(builder); + this.secondActivationFunction = builder.secondActivationFunction; + } + + public IActivation getSecondActivationFunction() { + //We also need setter/getter methods for our layer configuration fields (if any) for JSON serialization + return secondActivationFunction; + } + + public void setSecondActivationFunction(IActivation secondActivationFunction) { + //We also need setter/getter methods for our layer configuration fields (if any) for JSON serialization + this.secondActivationFunction = secondActivationFunction; + } + + @Override + public Layer instantiate(NeuralNetConfiguration conf, Collection iterationListeners, + int layerIndex, INDArray layerParamsView, boolean initializeParams, DataType networkDataType) { + //The instantiate method is how we go from the configuration class (i.e., this class) to the implementation class + // (i.e., a CustomLayerImpl instance) + //For the most part, it's the same for each type of layer + + CustomLayerImpl myCustomLayer = new CustomLayerImpl(conf, networkDataType); + myCustomLayer.setListeners(iterationListeners); //Set the iteration listeners, if any + myCustomLayer.setIndex(layerIndex); //Integer index of the layer + + //Parameter view array: In Deeplearning4j, the network parameters for the entire network (all layers) are + // allocated in one big array. The relevant section of this parameter vector is extracted out for each layer, + // (i.e., it's a "view" array in that it's a subset of a larger array) + // This is a row vector, with length equal to the number of parameters in the layer + myCustomLayer.setParamsViewArray(layerParamsView); + + //Initialize the layer parameters. For example, + // Note that the entries in paramTable (2 entries here: a weight array of shape [nIn,nOut] and biases of shape [1,nOut] + // are in turn a view of the 'layerParamsView' array. + Map paramTable = initializer().init(conf, layerParamsView, initializeParams); + myCustomLayer.setParamTable(paramTable); + myCustomLayer.setConf(conf); + return myCustomLayer; + } + + @Override + public ParamInitializer initializer() { + //This method returns the parameter initializer for this type of layer + //In this case, we can use the DefaultParamInitializer, which is the same one used for DenseLayer + //For more complex layers, you may need to implement a custom parameter initializer + //See the various parameter initializers here: + //https://github.com/deeplearning4j/deeplearning4j/tree/master/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/params + + return DefaultParamInitializer.getInstance(); + } + + @Override + public LayerMemoryReport getMemoryReport(InputType inputType) { + //Memory report is used to estimate how much memory is required for the layer, for different configurations + //If you don't need this functionality for your custom layer, you can return a LayerMemoryReport + // with all 0s, or + + //This implementation: based on DenseLayer implementation + InputType outputType = getOutputType(-1, inputType); + + val numParams = initializer().numParams(this); + int updaterStateSize = (int) getIUpdater().stateSize(numParams); + + int trainSizeFixed = 0; + int trainSizeVariable = 0; + if (getIDropout() != null) { + //Assume we dup the input for dropout + trainSizeVariable += inputType.arrayElementsPerExample(); + } + + //Also, during backprop: we do a preOut call -> gives us activations size equal to the output size + // which is modified in-place by activation function backprop + // then we have 'epsilonNext' which is equivalent to input size + trainSizeVariable += outputType.arrayElementsPerExample(); + + return new LayerMemoryReport.Builder(layerName, CustomLayer.class, inputType, outputType) + .standardMemory(numParams, updaterStateSize) + .workingMemory(0, 0, trainSizeFixed, + trainSizeVariable) //No additional memory (beyond activations) for inference + .cacheMemory(MemoryReport.CACHE_MODE_ALL_ZEROS, + MemoryReport.CACHE_MODE_ALL_ZEROS) //No caching in DenseLayer + .build(); + } + + + //Here's an implementation of a builder pattern, to allow us to easily configure the layer + //Note that we are inheriting all of the FeedForwardLayer.Builder options: things like n + public static class Builder extends FeedForwardLayer.Builder { + + @Getter + @Setter + private IActivation secondActivationFunction; + + //This is an example of a custom property in the configuration + + /** + * A custom property used in this custom layer example. See the CustomLayerExampleReadme.md for details + * + * @param secondActivationFunction Second activation function for the layer + */ + public Builder secondActivationFunction(String secondActivationFunction) { + return secondActivationFunction(Activation.fromString(secondActivationFunction)); + } + + /** + * A custom property used in this custom layer example. See the CustomLayerExampleReadme.md for details + * + * @param secondActivationFunction Second activation function for the layer + */ + public Builder secondActivationFunction(Activation secondActivationFunction) { + this.secondActivationFunction = secondActivationFunction.getActivationFunction(); + return this; + } + + @Override + @SuppressWarnings("unchecked") //To stop warnings about unchecked cast. Not required. + public CustomLayer build() { + return new CustomLayer(this); + } + } + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayerImpl.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayerImpl.java similarity index 100% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayerImpl.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayerImpl.java diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/CompareTrainingImplementations.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/samediff/CompareTrainingImplementations.java similarity index 92% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/CompareTrainingImplementations.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/samediff/CompareTrainingImplementations.java index a97da82ed..73610f45e 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/samediff/CompareTrainingImplementations.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/samediff/CompareTrainingImplementations.java @@ -30,13 +30,10 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; import org.nd4j.autodiff.samediff.TrainingConfig; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.evaluation.classification.Evaluation; import org.nd4j.evaluation.regression.RegressionEvaluation; import org.nd4j.linalg.activations.Activation; @@ -57,11 +54,9 @@ import org.nd4j.weightinit.impl.XavierInitScheme; import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; + @Slf4j -@NativeTag -@Tag(TagNames.SAMEDIFF) public class CompareTrainingImplementations extends BaseDL4JTest { @Test @@ -186,7 +181,7 @@ public class CompareTrainingImplementations extends BaseDL4JTest { INDArray outSd = map.get(a1.name()); INDArray outDl4j = net.output(f); - assertEquals(outDl4j, outSd, testName); + assertEquals( outDl4j, outSd, testName); net.setInput(f); net.setLabels(l); @@ -198,7 +193,7 @@ public class CompareTrainingImplementations extends BaseDL4JTest { //Check score double scoreDl4j = net.score(); double scoreSd = map.get(lossMse.name()).getDouble(0) + sd.calcRegularizationScore(); - assertEquals( scoreDl4j, scoreSd, 1e-6,testName); + assertEquals(scoreDl4j, scoreSd, 1e-6, testName); double lossRegScoreSD = sd.calcRegularizationScore(); double lossRegScoreDL4J = net.calcRegularizationScore(true); @@ -212,10 +207,10 @@ public class CompareTrainingImplementations extends BaseDL4JTest { //Note that the SameDiff gradients don't include the L1/L2 terms at present just from execBackwards()... these are added in fitting only //We can check correctness though with training param checks later if(l1Val == 0 && l2Val == 0 && wdVal == 0) { - assertEquals(grads.get("1_b"), gm.get(b1.name()), testName); - assertEquals(grads.get("1_W"), gm.get(w1.name()), testName); - assertEquals(grads.get("0_b"), gm.get(b0.name()), testName); - assertEquals(grads.get("0_W"), gm.get(w0.name()), testName); + assertEquals( grads.get("1_b"), gm.get(b1.name()), testName); + assertEquals( grads.get("1_W"), gm.get(w1.name()), testName); + assertEquals( grads.get("0_b"), gm.get(b0.name()), testName); + assertEquals( grads.get("0_W"), gm.get(w0.name()), testName); } @@ -242,10 +237,10 @@ public class CompareTrainingImplementations extends BaseDL4JTest { String s = testName + " - " + j; INDArray dl4j_0W = net.getParam("0_W"); INDArray sd_0W = w0.getArr(); - assertEquals(dl4j_0W, sd_0W, s); - assertEquals(net.getParam("0_b"), b0.getArr(), s); - assertEquals(net.getParam("1_W"), w1.getArr(), s); - assertEquals(net.getParam("1_b"), b1.getArr(), s); + assertEquals( dl4j_0W, sd_0W, s); + assertEquals( net.getParam("0_b"), b0.getArr(), s); + assertEquals( net.getParam("1_W"), w1.getArr(), s); + assertEquals( net.getParam("1_b"), b1.getArr(), s); } //Compare evaluations diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/ui/UiConnectionInfoTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/ui/UiConnectionInfoTest.java new file mode 100644 index 000000000..8bbb70879 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/ui/UiConnectionInfoTest.java @@ -0,0 +1,123 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.ui; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.core.ui.UiConnectionInfo; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class UiConnectionInfoTest extends BaseDL4JTest { + + @BeforeEach + public void setUp() throws Exception { + + } + + @Test + public void testGetFirstPart1() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setPort(8080).build(); + + assertEquals("http://localhost:8080", info.getFirstPart()); + } + + @Test + public void testGetFirstPart2() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().enableHttps(true).setPort(8080).build(); + + assertEquals("https://localhost:8080", info.getFirstPart()); + } + + @Test + public void testGetFirstPart3() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082) + .build(); + + assertEquals("https://192.168.1.1:8082", info.getFirstPart()); + } + + + @Test + public void testGetSecondPart1() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082) + .setPath("www-data").build(); + + assertEquals("/www-data/", info.getSecondPart()); + } + + @Test + public void testGetSecondPart2() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082) + .setPath("/www-data/tmp/").build(); + + assertEquals("/www-data/tmp/", info.getSecondPart()); + } + + @Test + public void testGetSecondPart3() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082) + .setPath("/www-data/tmp").build(); + + assertEquals("/www-data/tmp/", info.getSecondPart()); + } + + @Test + public void testGetSecondPart4() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082) + .setPath("/www-data//tmp").build(); + + assertEquals("/www-data/tmp/", info.getSecondPart()); + } + + @Test + public void testGetSecondPart5() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082) + .setPath("/www-data//tmp").build(); + + assertEquals("/www-data/tmp/alpha/", info.getSecondPart("alpha")); + } + + @Test + public void testGetSecondPart6() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082) + .setPath("//www-data//tmp").build(); + + assertEquals("/www-data/tmp/alpha/", info.getSecondPart("/alpha/")); + } + + @Test + public void testGetSecondPart7() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082) + .setPath("//www-data//tmp").build(); + + assertEquals("/www-data/tmp/alpha/beta/", info.getSecondPart("/alpha//beta/")); + } + + @Test + public void testGetSecondPart8() throws Exception { + UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(false) + .setPort(8082).setPath("/www-data//tmp").build(); + + assertEquals("http://192.168.1.1:8082/www-data/tmp/", info.getFullAddress()); + } +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ArrayUtilTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ArrayUtilTest.java new file mode 100644 index 000000000..696faf3f9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ArrayUtilTest.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.util; + +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; +import org.nd4j.common.util.ArrayUtil; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * + */ +public class ArrayUtilTest extends BaseDL4JTest { + + @Test + public void testRange() { + int[] range = ArrayUtil.range(0, 2); + int[] test = {0, 1}; + assertEquals(true, Arrays.equals(test, range)); + + int[] test2 = {-1, 0}; + int[] range2 = ArrayUtil.range(-1, 1); + assertEquals(true, Arrays.equals(test2, range2)); + + } + + @Test + public void testStrides() { + int[] shape = {5, 4, 3}; + int[] cStyleStride = {12, 3, 1}; + int[] fortranStyleStride = {1, 5, 20}; + int[] fortranStyleTest = ArrayUtil.calcStridesFortran(shape); + int[] cStyleTest = ArrayUtil.calcStrides(shape); + assertEquals(true, Arrays.equals(cStyleStride, cStyleTest)); + assertEquals(true, Arrays.equals(fortranStyleStride, fortranStyleTest)); + + int[] shape2 = {2, 2}; + int[] cStyleStride2 = {2, 1}; + int[] fortranStyleStride2 = {1, 2}; + int[] cStyleTest2 = ArrayUtil.calcStrides(shape2); + int[] fortranStyleTest2 = ArrayUtil.calcStridesFortran(shape2); + assertEquals(true, Arrays.equals(cStyleStride2, cStyleTest2)); + assertEquals(true, Arrays.equals(fortranStyleStride2, fortranStyleTest2)); + + + + } + + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/CrashReportingUtilTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/CrashReportingUtilTest.java new file mode 100644 index 000000000..4da9883b8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/CrashReportingUtilTest.java @@ -0,0 +1,213 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.util; + +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.NormalDistribution; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.PoolingType; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.*; + +@Timeout(120) +public class CrashReportingUtilTest extends BaseDL4JTest { + + @TempDir + public File testDir; + + @Override + public DataType getDataType(){ + return DataType.FLOAT; + } + + @AfterEach + public void after(){ + //Reset dir + CrashReportingUtil.crashDumpOutputDirectory(null); + } + + @Test + public void test() throws Exception { + File dir = testDir; + CrashReportingUtil.crashDumpOutputDirectory(dir); + + int kernel = 2; + int stride = 1; + int padding = 0; + PoolingType poolingType = PoolingType.MAX; + int inputDepth = 1; + int height = 28; + int width = 28; + + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().updater(new NoOp()) + + .dist(new NormalDistribution(0, 1)) + .list().layer(0, + new ConvolutionLayer.Builder() + .kernelSize(kernel, kernel) + .stride(stride, stride) + .padding(padding, padding) + .nIn(inputDepth) + .nOut(3).build()) + .layer(1, new SubsamplingLayer.Builder(poolingType) + .kernelSize(kernel, kernel) + .stride(stride, stride) + .padding(padding, padding) + .build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX) + .nOut(10).build()) + .setInputType(InputType.convolutionalFlat(height, width, + inputDepth)) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + net.addListeners(new ScoreIterationListener(1)); + + //Test net that hasn't been trained yet + Exception e = new Exception(); + CrashReportingUtil.writeMemoryCrashDump(net, e); + + File[] list = dir.listFiles(); + assertNotNull(list); + assertEquals(1, list.length); + String str = FileUtils.readFileToString(list[0]); +// System.out.println(str); + assertTrue(str.contains("Network Information")); + assertTrue(str.contains("Layer Helpers")); + assertTrue(str.contains("JavaCPP")); + assertTrue(str.contains("ScoreIterationListener")); + + + //Train: + DataSetIterator iter = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(32, true, 12345), 5); + net.fit(iter); + dir = testDir; + CrashReportingUtil.crashDumpOutputDirectory(dir); + CrashReportingUtil.writeMemoryCrashDump(net, e); + + list = dir.listFiles(); + assertNotNull(list); + assertEquals(1, list.length); + str = FileUtils.readFileToString(list[0]); + assertTrue(str.contains("Network Information")); + assertTrue(str.contains("Layer Helpers")); + assertTrue(str.contains("JavaCPP")); + assertTrue(str.contains("ScoreIterationListener(1)")); + +// System.out.println("///////////////////////////////////////////////////////////"); +// System.out.println(str); +// System.out.println("///////////////////////////////////////////////////////////"); + + + //Also test manual memory info + String mlnMemoryInfo = net.memoryInfo(32, InputType.convolutionalFlat(28, 28, 1)); +// System.out.println("///////////////////////////////////////////////////////////"); +// System.out.println(mlnMemoryInfo); +// System.out.println("///////////////////////////////////////////////////////////"); + + assertTrue(mlnMemoryInfo.contains("Network Information")); + assertTrue(mlnMemoryInfo.contains("Layer Helpers")); + assertTrue(mlnMemoryInfo.contains("JavaCPP")); + assertTrue(mlnMemoryInfo.contains("ScoreIterationListener(1)")); + + + + //////////////////////////////////////// + //Same thing on ComputationGraph: + dir = testDir; + CrashReportingUtil.crashDumpOutputDirectory(dir); + + ComputationGraph cg = net.toComputationGraph(); + cg.setListeners(new ScoreIterationListener(1)); + + //Test net that hasn't been trained yet + CrashReportingUtil.writeMemoryCrashDump(cg, e); + + list = dir.listFiles(); + assertNotNull(list); + assertEquals(1, list.length); + str = FileUtils.readFileToString(list[0]); + assertTrue(str.contains("Network Information")); + assertTrue(str.contains("Layer Helpers")); + assertTrue(str.contains("JavaCPP")); + assertTrue(str.contains("ScoreIterationListener(1)")); + + //Train: + cg.fit(iter); + dir = testDir; + CrashReportingUtil.crashDumpOutputDirectory(dir); + CrashReportingUtil.writeMemoryCrashDump(cg, e); + + list = dir.listFiles(); + assertNotNull(list); + assertEquals(1, list.length); + str = FileUtils.readFileToString(list[0]); + assertTrue(str.contains("Network Information")); + assertTrue(str.contains("Layer Helpers")); + assertTrue(str.contains("JavaCPP")); + assertTrue(str.contains("ScoreIterationListener(1)")); + +// System.out.println("///////////////////////////////////////////////////////////"); +// System.out.println(str); +// System.out.println("///////////////////////////////////////////////////////////"); + + + //Also test manual memory info + String cgMemoryInfo = cg.memoryInfo(32, InputType.convolutionalFlat(28, 28, 1)); +// System.out.println("///////////////////////////////////////////////////////////"); +// System.out.println(cgMemoryInfo); +// System.out.println("///////////////////////////////////////////////////////////"); + + assertTrue(cgMemoryInfo.contains("Network Information")); + assertTrue(cgMemoryInfo.contains("Layer Helpers")); + assertTrue(cgMemoryInfo.contains("JavaCPP")); + assertTrue(cgMemoryInfo.contains("ScoreIterationListener(1)")); + + } + + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelGuesserTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelGuesserTest.java new file mode 100644 index 000000000..74fbd476a --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelGuesserTest.java @@ -0,0 +1,265 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.util; + +import org.apache.commons.compress.utils.IOUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.core.util.ModelGuesser; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.preprocessor.Normalizer; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.resources.Resources; + +import java.io.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Timeout(300) +public class ModelGuesserTest extends BaseDL4JTest { + + @TempDir + public File testDir; + + @Test + public void testModelGuessFile() throws Exception { + File f = Resources.asFile("modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_model.h5"); + assertTrue(f.exists()); + Model guess1 = ModelGuesser.loadModelGuess(f.getAbsolutePath()); + Assertions.assertNotNull(guess1); + f = Resources.asFile("modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_1_model.h5"); + assertTrue(f.exists()); + Model guess2 = ModelGuesser.loadModelGuess(f.getAbsolutePath()); + Assertions.assertNotNull(guess2); + + } + + @Test + public void testModelGuessInputStream() throws Exception { + File f = Resources.asFile("modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_model.h5"); + assertTrue(f.exists()); + + try (InputStream inputStream = new FileInputStream(f)) { + Model guess1 = ModelGuesser.loadModelGuess(inputStream); + Assertions.assertNotNull(guess1); + } + + f = Resources.asFile("modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_1_model.h5"); + assertTrue(f.exists()); + + try (InputStream inputStream = new FileInputStream(f)) { + Model guess1 = ModelGuesser.loadModelGuess(inputStream); + Assertions.assertNotNull(guess1); + } + } + + + + @Test + public void testLoadNormalizersFile() throws Exception { + MultiLayerNetwork net = getNetwork(); + + File tempFile = new File(testDir, "testLoadNormalizersFile.bin"); + + ModelSerializer.writeModel(net, tempFile, true); + + NormalizerMinMaxScaler normalizer = new NormalizerMinMaxScaler(0, 1); + normalizer.fit(new DataSet(Nd4j.rand(new int[] {2, 2}), Nd4j.rand(new int[] {2, 2}))); + ModelSerializer.addNormalizerToModel(tempFile, normalizer); + Model model = ModelGuesser.loadModelGuess(tempFile.getAbsolutePath()); + Normalizer normalizer1 = ModelGuesser.loadNormalizer(tempFile.getAbsolutePath()); + assertEquals(model, net); + assertEquals(normalizer, normalizer1); + + } + + + @Test + public void testNormalizerInPlace() throws Exception { + MultiLayerNetwork net = getNetwork(); + + File tempFile = new File(testDir, "testNormalizerInPlace.bin"); + + NormalizerMinMaxScaler normalizer = new NormalizerMinMaxScaler(0, 1); + normalizer.fit(new DataSet(Nd4j.rand(new int[] {2, 2}), Nd4j.rand(new int[] {2, 2}))); + ModelSerializer.writeModel(net, tempFile, true,normalizer); + + Model model = ModelGuesser.loadModelGuess(tempFile.getAbsolutePath()); + Normalizer normalizer1 = ModelGuesser.loadNormalizer(tempFile.getAbsolutePath()); + assertEquals(model, net); + assertEquals(normalizer, normalizer1); + + } + + @Test + public void testLoadNormalizersInputStream() throws Exception { + MultiLayerNetwork net = getNetwork(); + + File tempFile = new File(testDir, "testLoadNormalizersInputStream.bin"); + + ModelSerializer.writeModel(net, tempFile, true); + + NormalizerMinMaxScaler normalizer = new NormalizerMinMaxScaler(0, 1); + normalizer.fit(new DataSet(Nd4j.rand(new int[] {2, 2}), Nd4j.rand(new int[] {2, 2}))); + ModelSerializer.addNormalizerToModel(tempFile, normalizer); + Model model = ModelGuesser.loadModelGuess(tempFile.getAbsolutePath()); + try (InputStream inputStream = new FileInputStream(tempFile)) { + Normalizer normalizer1 = ModelGuesser.loadNormalizer(inputStream); + assertEquals(model, net); + assertEquals(normalizer, normalizer1); + } + + } + + + @Test + public void testModelGuesserDl4jModelFile() throws Exception { + MultiLayerNetwork net = getNetwork(); + + File tempFile = new File(testDir, "testModelGuesserDl4jModelFile.bin"); + + ModelSerializer.writeModel(net, tempFile, true); + + MultiLayerNetwork network = (MultiLayerNetwork) ModelGuesser.loadModelGuess(tempFile.getAbsolutePath()); + assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson()); + assertEquals(net.params(), network.params()); + assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); + + } + + @Test + public void testModelGuesserDl4jModelInputStream() throws Exception { + MultiLayerNetwork net = getNetwork(); + + File tempFile = new File(testDir, "testModelGuesserDl4jModelInputStream.bin"); + + ModelSerializer.writeModel(net, tempFile, true); + + try (InputStream inputStream = new FileInputStream(tempFile)) { + MultiLayerNetwork network = (MultiLayerNetwork) ModelGuesser.loadModelGuess(inputStream); + Assertions.assertNotNull(network); + assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson()); + assertEquals(net.params(), network.params()); + assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); + } + } + + + @Test + public void testModelGuessConfigFile() throws Exception { + ClassPathResource resource = new ClassPathResource("modelimport/keras/configs/cnn_tf_config.json", + ModelGuesserTest.class.getClassLoader()); + File f = getTempFile(resource); + String configFilename = f.getAbsolutePath(); + Object conf = ModelGuesser.loadConfigGuess(configFilename); + assertTrue(conf instanceof MultiLayerConfiguration); + + ClassPathResource sequenceResource = new ClassPathResource("/keras/simple/mlp_fapi_multiloss_config.json"); + File f2 = getTempFile(sequenceResource); + Object sequenceConf = ModelGuesser.loadConfigGuess(f2.getAbsolutePath()); + assertTrue(sequenceConf instanceof ComputationGraphConfiguration); + + + + ClassPathResource resourceDl4j = new ClassPathResource("model.json"); + File fDl4j = getTempFile(resourceDl4j); + String configFilenameDl4j = fDl4j.getAbsolutePath(); + Object confDl4j = ModelGuesser.loadConfigGuess(configFilenameDl4j); + assertTrue(confDl4j instanceof ComputationGraphConfiguration); + + } + + @Test + public void testModelGuessConfigInputStream() throws Exception { + ClassPathResource resource = new ClassPathResource("modelimport/keras/configs/cnn_tf_config.json", + ModelGuesserTest.class.getClassLoader()); + File f = getTempFile(resource); + + try (InputStream inputStream = new FileInputStream(f)) { + Object conf = ModelGuesser.loadConfigGuess(inputStream); + assertTrue(conf instanceof MultiLayerConfiguration); + } + + ClassPathResource sequenceResource = new ClassPathResource("/keras/simple/mlp_fapi_multiloss_config.json"); + File f2 = getTempFile(sequenceResource); + + try (InputStream inputStream = new FileInputStream(f2)) { + Object sequenceConf = ModelGuesser.loadConfigGuess(inputStream); + assertTrue(sequenceConf instanceof ComputationGraphConfiguration); + } + + + ClassPathResource resourceDl4j = new ClassPathResource("model.json"); + File fDl4j = getTempFile(resourceDl4j); + + try (InputStream inputStream = new FileInputStream(fDl4j)) { + Object confDl4j = ModelGuesser.loadConfigGuess(inputStream); + assertTrue(confDl4j instanceof ComputationGraphConfiguration); + } + + } + + + private File getTempFile(ClassPathResource classPathResource) throws Exception { + InputStream is = classPathResource.getInputStream(); + File f = new File(testDir, "file.tmp"); + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f)); + IOUtils.copy(is, bos); + bos.flush(); + bos.close(); + return f; + } + + private MultiLayerNetwork getNetwork() { + int nIn = 5; + int nOut = 6; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).l2(0.01) + .updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + return net; + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelSerializerTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelSerializerTest.java new file mode 100644 index 000000000..d1b1c3e02 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelSerializerTest.java @@ -0,0 +1,502 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.util; + +import lombok.val; +import org.apache.commons.lang3.SerializationUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.preprocessor.Normalizer; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.common.primitives.Pair; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +public class ModelSerializerTest extends BaseDL4JTest { + + @TempDir + public File tempDir; + + @Test + public void testWriteMLNModel() throws Exception { + int nIn = 5; + int nOut = 6; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + File tempFile = tempDir; + + ModelSerializer.writeModel(net, tempFile, true); + + MultiLayerNetwork network = ModelSerializer.restoreMultiLayerNetwork(tempFile); + + assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson()); + assertEquals(net.params(), network.params()); + assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); + } + + @Test + public void testWriteMlnModelInputStream() throws Exception { + int nIn = 5; + int nOut = 6; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + File tempFile = tempDir; + FileOutputStream fos = new FileOutputStream(tempFile); + + ModelSerializer.writeModel(net, fos, true); + + + // checking adding of DataNormalization to the model file + + NormalizerMinMaxScaler scaler = new NormalizerMinMaxScaler(); + DataSetIterator iter = new IrisDataSetIterator(150, 150); + scaler.fit(iter); + + ModelSerializer.addNormalizerToModel(tempFile, scaler); + + NormalizerMinMaxScaler restoredScaler = ModelSerializer.restoreNormalizerFromFile(tempFile); + + assertNotEquals(null, scaler.getMax()); + assertEquals(scaler.getMax(), restoredScaler.getMax()); + assertEquals(scaler.getMin(), restoredScaler.getMin()); + + FileInputStream fis = new FileInputStream(tempFile); + + MultiLayerNetwork network = ModelSerializer.restoreMultiLayerNetwork(fis); + + assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson()); + assertEquals(net.params(), network.params()); + assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); + } + + + @Test + public void testWriteCGModel() throws Exception { + ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)) + .graphBuilder().addInputs("in") + .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out", + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3) + .activation(Activation.SOFTMAX).build(), + "dense") + .setOutputs("out").build(); + + ComputationGraph cg = new ComputationGraph(config); + cg.init(); + + File tempFile = tempDir; + + ModelSerializer.writeModel(cg, tempFile, true); + + ComputationGraph network = ModelSerializer.restoreComputationGraph(tempFile); + + assertEquals(network.getConfiguration().toJson(), cg.getConfiguration().toJson()); + assertEquals(cg.params(), network.params()); + assertEquals(cg.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); + } + + @Test + public void testWriteCGModelInputStream() throws Exception { + ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)) + .graphBuilder().addInputs("in") + .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out", + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3) + .activation(Activation.SOFTMAX).build(), + "dense") + .setOutputs("out").build(); + + ComputationGraph cg = new ComputationGraph(config); + cg.init(); + + File tempFile = tempDir; + + ModelSerializer.writeModel(cg, tempFile, true); + FileInputStream fis = new FileInputStream(tempFile); + + ComputationGraph network = ModelSerializer.restoreComputationGraph(fis); + + assertEquals(network.getConfiguration().toJson(), cg.getConfiguration().toJson()); + assertEquals(cg.params(), network.params()); + assertEquals(cg.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); + } + + private DataSet trivialDataSet() { + INDArray inputs = Nd4j.create(new float[] {1.0f, 2.0f, 3.0f}, new int[]{1,3}); + INDArray labels = Nd4j.create(new float[] {4.0f, 5.0f, 6.0f}, new int[]{1,3}); + return new DataSet(inputs, labels); + } + + private ComputationGraph simpleComputationGraph() { + ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)) + .graphBuilder().addInputs("in") + .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out", + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3) + .activation(Activation.SOFTMAX).build(), + "dense") + .setOutputs("out").build(); + + return new ComputationGraph(config); + } + + @Test + public void testSaveRestoreNormalizerFromInputStream() throws Exception { + DataSet dataSet = trivialDataSet(); + NormalizerStandardize norm = new NormalizerStandardize(); + norm.fit(dataSet); + + ComputationGraph cg = simpleComputationGraph(); + cg.init(); + + File tempFile = tempDir; + + ModelSerializer.writeModel(cg, tempFile, true); + + ModelSerializer.addNormalizerToModel(tempFile, norm); + FileInputStream fis = new FileInputStream(tempFile); + + + NormalizerStandardize restored = ModelSerializer.restoreNormalizerFromInputStream(fis); + + assertNotEquals(null, restored); + + DataSet dataSet2 = dataSet.copy(); + + norm.preProcess(dataSet2); + assertNotEquals(dataSet.getFeatures(), dataSet2.getFeatures()); + + restored.revert(dataSet2); + assertEquals(dataSet.getFeatures(), dataSet2.getFeatures()); + } + + @Test + public void testRestoreUnsavedNormalizerFromInputStream() throws Exception { + DataSet dataSet = trivialDataSet(); + + NormalizerStandardize norm = new NormalizerStandardize(); + norm.fit(dataSet); + + ComputationGraph cg = simpleComputationGraph(); + cg.init(); + + File tempFile = tempDir; + ModelSerializer.writeModel(cg, tempFile, true); + + FileInputStream fis = new FileInputStream(tempFile); + + NormalizerStandardize restored = ModelSerializer.restoreNormalizerFromInputStream(fis); + + assertEquals(null, restored); + } + + @Test + public void testInvalidLoading1() throws Exception { + ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder() + .graphBuilder().addInputs("in") + .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in") + .addLayer("out",new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(2).nOut(3).build(), + "dense") + .setOutputs("out").build(); + + ComputationGraph cg = new ComputationGraph(config); + cg.init(); + + File tempFile = tempDir; + + ModelSerializer.writeModel(cg, tempFile, true); + + try { + ModelSerializer.restoreMultiLayerNetwork(tempFile); + fail(); + } catch (Exception e){ + String msg = e.getMessage(); + assertTrue(msg.contains("JSON") && msg.contains("restoreComputationGraph"), msg); + } + } + + @Test + public void testInvalidLoading2() throws Exception { + int nIn = 5; + int nOut = 6; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()) + .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + File tempFile = new File(tempDir, "testInvalidLoading2.bin"); + + ModelSerializer.writeModel(net, tempFile, true); + + try { + ModelSerializer.restoreComputationGraph(tempFile); + fail(); + } catch (Exception e){ + String msg = e.getMessage(); + assertTrue(msg.contains("JSON") && msg.contains("restoreMultiLayerNetwork"), msg); + } + } + + @Test + public void testInvalidStreamReuse() throws Exception { + int nIn = 5; + int nOut = 6; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .list() + .layer(new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSet dataSet = trivialDataSet(); + NormalizerStandardize norm = new NormalizerStandardize(); + norm.fit(dataSet); + + File tempFile = tempDir; + ModelSerializer.writeModel(net, tempFile, true); + ModelSerializer.addNormalizerToModel(tempFile, norm); + + InputStream is = new FileInputStream(tempFile); + ModelSerializer.restoreMultiLayerNetwork(is); + + try{ + ModelSerializer.restoreNormalizerFromInputStream(is); + fail("Expected exception"); + } catch (Exception e){ + String msg = e.getMessage(); + assertTrue(msg.contains("may have been closed"), msg); + } + + try{ + ModelSerializer.restoreMultiLayerNetwork(is); + fail("Expected exception"); + } catch (Exception e){ + String msg = e.getMessage(); + assertTrue(msg.contains("may have been closed"), msg); + } + + //Also test reading both model and normalizer from stream (correctly) + Pair pair = ModelSerializer.restoreMultiLayerNetworkAndNormalizer(new FileInputStream(tempFile), true); + assertEquals(net.params(), pair.getFirst().params()); + assertNotNull(pair.getSecond()); + } + + + @Test + public void testInvalidStreamReuseCG() throws Exception { + int nIn = 5; + int nOut = 6; + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .graphBuilder() + .addInputs("in") + .layer("0", new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build(), "in") + .setOutputs("0") + .build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + + DataSet dataSet = trivialDataSet(); + NormalizerStandardize norm = new NormalizerStandardize(); + norm.fit(dataSet); + + File tempFile = tempDir; + ModelSerializer.writeModel(net, tempFile, true); + ModelSerializer.addNormalizerToModel(tempFile, norm); + + InputStream is = new FileInputStream(tempFile); + ModelSerializer.restoreComputationGraph(is); + + try{ + ModelSerializer.restoreNormalizerFromInputStream(is); + fail("Expected exception"); + } catch (Exception e){ + String msg = e.getMessage(); + assertTrue(msg.contains("may have been closed"), msg); + } + + try{ + ModelSerializer.restoreComputationGraph(is); + fail("Expected exception"); + } catch (Exception e){ + String msg = e.getMessage(); + assertTrue(msg.contains("may have been closed"), msg); + } + + //Also test reading both model and normalizer from stream (correctly) + Pair pair = ModelSerializer.restoreComputationGraphAndNormalizer(new FileInputStream(tempFile), true); + assertEquals(net.params(), pair.getFirst().params()); + assertNotNull(pair.getSecond()); + } + + + @Test + public void testJavaSerde_1() throws Exception { + int nIn = 5; + int nOut = 6; + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .graphBuilder() + .addInputs("in") + .layer("0", new OutputLayer.Builder().nIn(nIn).nOut(nOut).build(), "in") + .setOutputs("0") + .validateOutputLayerConfig(false) + .build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + + DataSet dataSet = trivialDataSet(); + NormalizerStandardize norm = new NormalizerStandardize(); + norm.fit(dataSet); + + val b = SerializationUtils.serialize(net); + + ComputationGraph restored = SerializationUtils.deserialize(b); + + assertEquals(net, restored); + } + + @Test + public void testJavaSerde_2() throws Exception { + int nIn = 5; + int nOut = 6; + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .list() + .layer(0, new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSet dataSet = trivialDataSet(); + NormalizerStandardize norm = new NormalizerStandardize(); + norm.fit(dataSet); + + val b = SerializationUtils.serialize(net); + + MultiLayerNetwork restored = SerializationUtils.deserialize(b); + + assertEquals(net, restored); + } + + @Test + public void testPutGetObject() throws Exception { + + int nIn = 5; + int nOut = 6; + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01) + .graphBuilder() + .addInputs("in") + .layer("0", new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build(), "in") + .setOutputs("0") + .build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + + File tempFile = tempDir; + ModelSerializer.writeModel(net, tempFile, true); + + + List toWrite = Arrays.asList("zero", "one", "two"); + ModelSerializer.addObjectToFile(tempFile, "myLabels", toWrite); + List restored = ModelSerializer.getObjectFromFile(tempFile, "myLabels"); + assertEquals(toWrite, restored); + + + Map someOtherData = new HashMap<>(); + someOtherData.put("x", new float[]{0,1,2}); + someOtherData.put("y",Nd4j.linspace(1,10,10, Nd4j.dataType())); + + ModelSerializer.addObjectToFile(tempFile, "otherData.bin", someOtherData); + + Map dataRestored = ModelSerializer.getObjectFromFile(tempFile, "otherData.bin"); + assertEquals(someOtherData.keySet(), dataRestored.keySet()); + assertArrayEquals((float[])someOtherData.get("x"), (float[])dataRestored.get("x"), 0f); + assertEquals(someOtherData.get("y"), dataRestored.get("y")); + + + List entries = ModelSerializer.listObjectsInFile(tempFile); + assertEquals(2, entries.size()); + System.out.println(entries); + assertTrue(entries.contains("myLabels")); + assertTrue(entries.contains("otherData.bin")); + + ComputationGraph restoredNet = ModelSerializer.restoreComputationGraph(tempFile); + assertEquals(net.params(), restoredNet.params()); + } +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelValidatorTests.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelValidatorTests.java similarity index 97% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelValidatorTests.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelValidatorTests.java index a0aa6cdb2..9d6a27183 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelValidatorTests.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/ModelValidatorTests.java @@ -29,13 +29,8 @@ import org.deeplearning4j.nn.conf.layers.DenseLayer; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.learning.config.Adam; import org.nd4j.common.validation.ValidationResult; @@ -55,17 +50,16 @@ import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; - import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.FILE_IO) + public class ModelValidatorTests extends BaseDL4JTest { - + @TempDir + public File testDir; @Test - public void testMultiLayerNetworkValidation(@TempDir Path testDir) throws Exception { - File f = testDir.toFile(); + public void testMultiLayerNetworkValidation() throws Exception { + File f = testDir; //Test non-existent file File f0 = new File(f, "doesntExist.bin"); @@ -182,8 +176,8 @@ public class ModelValidatorTests extends BaseDL4JTest { @Test - public void testComputationGraphNetworkValidation(@TempDir Path testDir) throws Exception { - File f = testDir.toFile(); + public void testComputationGraphNetworkValidation() throws Exception { + File f = testDir; //Test non-existent file File f0 = new File(f, "doesntExist.bin"); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/MovingWindowMatrixTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/MovingWindowMatrixTest.java new file mode 100644 index 000000000..f9dc68c4c --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/MovingWindowMatrixTest.java @@ -0,0 +1,50 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.util; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.core.util.MovingWindowMatrix; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class MovingWindowMatrixTest extends BaseDL4JTest { + + @Test + public void testMovingWindow() { + INDArray ones = Nd4j.ones(4, 4); + MovingWindowMatrix m = new MovingWindowMatrix(ones, 2, 2); + List windows = m.windows(); + assertEquals(4, windows.size()); + MovingWindowMatrix m2 = new MovingWindowMatrix(ones, 2, 2, true); + List windowsRotate = m2.windows(); + assertEquals(16, windowsRotate.size()); + + + } + + + +} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/SerializationUtilsTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/SerializationUtilsTest.java old mode 100755 new mode 100644 similarity index 77% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/SerializationUtilsTest.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/SerializationUtilsTest.java index 87a8256fa..bd2fee895 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/SerializationUtilsTest.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/SerializationUtilsTest.java @@ -17,43 +17,40 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.util; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.common.util.SerializationUtils; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import java.io.File; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Serialization Utils Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class SerializationUtilsTest extends BaseDL4JTest { +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SerializationUtilsTest extends BaseDL4JTest { @TempDir - public Path testDir; + public File testDir; @Test - @DisplayName("Test Write Read") - void testWriteRead() throws Exception { + public void testWriteRead() throws Exception { DataSetIterator iter = new IrisDataSetIterator(150, 150); String irisData = "irisData.dat"; + DataSet freshDataSet = iter.next(150); - File f = testDir.resolve(irisData).toFile(); + + File f = new File(testDir, irisData); SerializationUtils.saveObject(freshDataSet, f); + DataSet readDataSet = SerializationUtils.readObject(f); + assertEquals(freshDataSet.getFeatures(), readDataSet.getFeatures()); assertEquals(freshDataSet.getLabels(), readDataSet.getLabels()); } + } diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/TestUIDProvider.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/TestUIDProvider.java similarity index 76% rename from deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/TestUIDProvider.java rename to cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/TestUIDProvider.java index b9a210a38..a23f3d513 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/TestUIDProvider.java +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/TestUIDProvider.java @@ -22,20 +22,20 @@ package org.deeplearning4j.util; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.core.util.UIDProvider; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.FILE_IO) + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + public class TestUIDProvider extends BaseDL4JTest { @Test public void testUIDProvider() { - String jvmUID = org.deeplearning4j.core.util.UIDProvider.getJVMUID(); - String hardwareUID = org.deeplearning4j.core.util.UIDProvider.getHardwareUID(); + String jvmUID = UIDProvider.getJVMUID(); + String hardwareUID = UIDProvider.getHardwareUID(); assertNotNull(jvmUID); assertNotNull(hardwareUID); @@ -43,7 +43,7 @@ public class TestUIDProvider extends BaseDL4JTest { assertTrue(!jvmUID.isEmpty()); assertTrue(!hardwareUID.isEmpty()); - assertEquals(jvmUID, org.deeplearning4j.core.util.UIDProvider.getJVMUID()); + assertEquals(jvmUID, UIDProvider.getJVMUID()); assertEquals(hardwareUID, UIDProvider.getHardwareUID()); System.out.println("JVM uid: " + jvmUID); diff --git a/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/TimeSeriesUtilsTest.java b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/TimeSeriesUtilsTest.java new file mode 100644 index 000000000..9c0b269e8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/java/org/deeplearning4j/util/TimeSeriesUtilsTest.java @@ -0,0 +1,43 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.util; + +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TimeSeriesUtilsTest extends BaseDL4JTest { + + @Test + public void testMovingAverage() { + INDArray a = Nd4j.arange(0, 20).castTo(DataType.DOUBLE); + INDArray result = Nd4j.create(new double[] {1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f, 9.5f, 10.5f, 11.5f, + 12.5f, 13.5f, 14.5f, 15.5f, 16.5f, 17.5f}); + + INDArray movingAvg = TimeSeriesUtils.movingAverage(a, 4); + assertEquals(result, movingAvg); + } + +} diff --git a/cavis-dnn/cavis-dnn-core/src/test/resources/junit-platform.properties b/cavis-dnn/cavis-dnn-core/src/test/resources/junit-platform.properties new file mode 100644 index 000000000..863ded8ac --- /dev/null +++ b/cavis-dnn/cavis-dnn-core/src/test/resources/junit-platform.properties @@ -0,0 +1,37 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + +# configuration parameter to configure when timeouts are applied. There are three modes. +#enabled, disabled, disabled_on_debug +junit.jupiter.execution.timeout.mode=enabled + +#Default timeout for all testable and lifecycle methods. +junit.jupiter.execution.timeout.default=60 s + +#junit.jupiter.execution.timeout.testable.method.default – Default timeout for all testable methods. +#junit.jupiter.execution.timeout.test.method.default – Default timeout for @Test methods. +#junit.jupiter.execution.timeout.testtemplate.method.default – Default timeout for @TestTemplate methods. +#junit.jupiter.execution.timeout.testfactory.method.default – Default timeout for @TestFactory methods. +#junit.jupiter.execution.timeout.lifecycle.method.default – Default timeout for all lifecycle methods. +#junit.jupiter.execution.timeout.beforeall.method.default – Default timeout for @BeforeAll methods. +#junit.jupiter.execution.timeout.beforeeach.method.default – Default timeout for @BeforeEach methods. +#junit.jupiter.execution.timeout.afterall.method.default – Default timeout for @AfterAll methods. +#junit.jupiter.execution.timeout.aftereach.method.default – Default timeout for @AfterEach methods. \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-core/src/test/resources/logback-test.xml b/cavis-dnn/cavis-dnn-core/src/test/resources/logback-test.xml similarity index 94% rename from deeplearning4j/deeplearning4j-core/src/test/resources/logback-test.xml rename to cavis-dnn/cavis-dnn-core/src/test/resources/logback-test.xml index 0a7ec90ee..46c82c6a2 100644 --- a/deeplearning4j/deeplearning4j-core/src/test/resources/logback-test.xml +++ b/cavis-dnn/cavis-dnn-core/src/test/resources/logback-test.xml @@ -37,9 +37,9 @@ - + - + diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/org.bytedecojavacpp1.5.4, b/cavis-dnn/cavis-dnn-data/build.gradle similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/org.bytedecojavacpp1.5.4, rename to cavis-dnn/cavis-dnn-data/build.gradle diff --git a/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle new file mode 100644 index 000000000..6eabedb4f --- /dev/null +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/build.gradle @@ -0,0 +1,30 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +dependencies { + implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators + implementation projects.cavisDnn.cavisDnnCommon + implementation "org.slf4j:slf4j-api" + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDatavec.cavisDatavecApi + implementation "commons-io:commons-io" +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/EmnistFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/base/EmnistFetcher.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/EmnistFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/base/EmnistFetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/IrisUtils.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/base/IrisUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/IrisUtils.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/base/IrisUtils.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/MnistFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/base/MnistFetcher.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/base/MnistFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/base/MnistFetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableDataSet.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableDataSet.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableDataSet.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableDataSet.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableExtractableDataSetFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableExtractableDataSetFetcher.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableExtractableDataSetFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/CacheableExtractableDataSetFetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/Cifar10Fetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/Cifar10Fetcher.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/Cifar10Fetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/Cifar10Fetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/DataSetType.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/DataSetType.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/DataSetType.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/DataSetType.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java similarity index 98% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java index e9bc68bb6..70d974e99 100644 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/EmnistDataFetcher.java @@ -49,6 +49,8 @@ public class EmnistDataFetcher extends MnistDataFetcher implements DataSetFetche String EMNIST_ROOT = DL4JResources.getDirectory(ResourceType.DATASET, "EMNIST").getAbsolutePath(); + String images; + String labels; if (train) { images = FilenameUtils.concat(EMNIST_ROOT, fetcher.getTrainingFilesFilename_unzipped()); labels = FilenameUtils.concat(EMNIST_ROOT, fetcher.getTrainingFileLabelsFilename_unzipped()); @@ -58,7 +60,7 @@ public class EmnistDataFetcher extends MnistDataFetcher implements DataSetFetche labels = FilenameUtils.concat(EMNIST_ROOT, fetcher.getTestFileLabelsFilename_unzipped()); totalExamples = EmnistDataSetIterator.numExamplesTest(dataSet); } - MnistManager man; + try { man = new MnistManager(images, labels, totalExamples); } catch (Exception e) { @@ -71,7 +73,6 @@ public class EmnistDataFetcher extends MnistDataFetcher implements DataSetFetche numOutcomes = EmnistDataSetIterator.numLabels(dataSet); this.binarize = binarize; cursor = 0; - man.setCurrent(cursor); inputColumns = man.getImages().getEntryLength(); this.train = train; this.shuffle = shuffle; @@ -91,7 +92,6 @@ public class EmnistDataFetcher extends MnistDataFetcher implements DataSetFetche oneIndexed = false; } this.fOrder = true; //MNIST is C order, EMNIST is F order - man.close(); } private boolean emnistExists(EmnistFetcher e) { diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/IrisDataFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/IrisDataFetcher.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/IrisDataFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/IrisDataFetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java old mode 100755 new mode 100644 similarity index 91% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java index a1999396d..be1dd952e --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/MnistDataFetcher.java @@ -20,7 +20,6 @@ package org.deeplearning4j.datasets.fetchers; -import lombok.SneakyThrows; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.deeplearning4j.datasets.base.MnistFetcher; @@ -55,6 +54,7 @@ public class MnistDataFetcher extends BaseDataFetcher { protected static final long[] CHECKSUMS_TRAIN = new long[]{CHECKSUM_TRAIN_FEATURES, CHECKSUM_TRAIN_LABELS}; protected static final long[] CHECKSUMS_TEST = new long[]{CHECKSUM_TEST_FEATURES, CHECKSUM_TEST_LABELS}; + protected transient MnistManager man; protected boolean binarize = true; protected boolean train; protected int[] order; @@ -65,9 +65,6 @@ public class MnistDataFetcher extends BaseDataFetcher { protected boolean firstShuffle = true; protected final int numExamples; - protected String images,labels; - //note: we default to zero here on purpose, otherwise when first initializes an error is thrown. - private long lastCursor = 0; /** @@ -85,6 +82,8 @@ public class MnistDataFetcher extends BaseDataFetcher { } String MNIST_ROOT = DL4JResources.getDirectory(ResourceType.DATASET, "MNIST").getAbsolutePath(); + String images; + String labels; long[] checksums; if (train) { images = FilenameUtils.concat(MNIST_ROOT, MnistFetcher.TRAINING_FILES_FILENAME_UNZIPPED); @@ -100,22 +99,17 @@ public class MnistDataFetcher extends BaseDataFetcher { String[] files = new String[]{images, labels}; try { - MnistManager man = new MnistManager(images, labels, train); + man = new MnistManager(images, labels, train); validateFiles(files, checksums); - man.close(); } catch (Exception e) { try { FileUtils.deleteDirectory(new File(MNIST_ROOT)); } catch (Exception e2){ } new MnistFetcher().downloadAndUntar(); - MnistManager man = new MnistManager(images, labels, train); - lastCursor = man.getCurrent(); + man = new MnistManager(images, labels, train); validateFiles(files, checksums); - man.close(); } - MnistManager man = new MnistManager(images, labels, train); - numOutcomes = 10; this.binarize = binarize; cursor = 0; @@ -133,7 +127,6 @@ public class MnistDataFetcher extends BaseDataFetcher { rng = new Random(rngSeed); this.numExamples = numExamples; reset(); //Shuffle order - man.close(); } private boolean mnistExists() { @@ -154,7 +147,7 @@ public class MnistDataFetcher extends BaseDataFetcher { return true; } - private void validateFiles(String[] files, long[] checksums) { + private void validateFiles(String[] files, long[] checksums){ //Validate files: try { for (int i = 0; i < files.length; i++) { @@ -177,19 +170,16 @@ public class MnistDataFetcher extends BaseDataFetcher { private float[][] featureData = null; - @SneakyThrows @Override public void fetch(int numExamples) { if (!hasMore()) { throw new IllegalStateException("Unable to get more; there are no more images"); } - MnistManager man = new MnistManager(images, labels, totalExamples); - man.setCurrent((int) lastCursor); INDArray labels = Nd4j.zeros(DataType.FLOAT, numExamples, numOutcomes); if(featureData == null || featureData.length < numExamples){ - featureData = new float[numExamples][28 * 28]; + featureData = new float[numExamples][28*28]; } int actualExamples = 0; @@ -198,8 +188,6 @@ public class MnistDataFetcher extends BaseDataFetcher { if (!hasMore()) break; - man.setCurrent(cursor); - lastCursor = cursor; byte[] img = man.readImageUnsafe(order[cursor]); if (fOrder) { @@ -248,7 +236,6 @@ public class MnistDataFetcher extends BaseDataFetcher { } curr = new DataSet(features, labels); - man.close(); } @Override @@ -276,7 +263,4 @@ public class MnistDataFetcher extends BaseDataFetcher { return next; } - public void close() { - } - } diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcher.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/TinyImageNetFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/TinyImageNetFetcher.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/TinyImageNetFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/TinyImageNetFetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/UciSequenceDataFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/UciSequenceDataFetcher.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/UciSequenceDataFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/fetchers/UciSequenceDataFetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/Cifar10DataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/Cifar10DataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/Cifar10DataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/Cifar10DataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/EmnistDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/EmnistDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/EmnistDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/EmnistDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/IrisDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/IrisDataSetIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/IrisDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/IrisDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/LFWDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/LFWDataSetIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/LFWDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/LFWDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java old mode 100755 new mode 100644 similarity index 92% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java index 5aa848e8c..48e3c2434 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/MnistDataSetIterator.java @@ -49,7 +49,7 @@ public class MnistDataSetIterator extends BaseDatasetIterator { */ public MnistDataSetIterator(int batchSize, boolean train, int seed) throws IOException { this(batchSize, (train ? MnistDataFetcher.NUM_EXAMPLES : MnistDataFetcher.NUM_EXAMPLES_TEST), false, train, - true, seed); + true, seed); } /**Get the specified number of MNIST examples (test or train set), with optional shuffling and binarization. @@ -61,13 +61,7 @@ public class MnistDataSetIterator extends BaseDatasetIterator { * @param rngSeed random number generator seed to use when shuffling examples */ public MnistDataSetIterator(int batch, int numExamples, boolean binarize, boolean train, boolean shuffle, - long rngSeed) throws IOException { + long rngSeed) throws IOException { super(batch, numExamples, new MnistDataFetcher(binarize, train, shuffle, rngSeed, numExamples)); } - - public void close() { - MnistDataFetcher mnistDataFetcher = (MnistDataFetcher) fetcher; - mnistDataFetcher.close(); - } - } diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/TinyImageNetDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/TinyImageNetDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/TinyImageNetDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/TinyImageNetDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/UciSequenceDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/UciSequenceDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/UciSequenceDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/iterator/impl/UciSequenceDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistDbFile.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistDbFile.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistDbFile.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistDbFile.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistImageFile.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistImageFile.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistImageFile.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistImageFile.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistLabelFile.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistLabelFile.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistLabelFile.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistLabelFile.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java old mode 100755 new mode 100644 similarity index 95% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java index b1cab7be7..4affe41b6 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datasets/src/main/java/org/deeplearning4j/datasets/mnist/MnistManager.java @@ -21,9 +21,7 @@ package org.deeplearning4j.datasets.mnist; -import lombok.SneakyThrows; import org.deeplearning4j.datasets.fetchers.MnistDataFetcher; -import org.nd4j.common.base.Preconditions; import java.io.BufferedWriter; import java.io.FileWriter; @@ -62,13 +60,6 @@ public class MnistManager { } - @SneakyThrows - public long getCurrent() { - return labels.getCurrentIndex(); - } - - - /** * Constructs an instance managing the two given data files. Supports * NULL value for one of the arguments in case reading only one @@ -86,8 +77,6 @@ public class MnistManager { this(imagesFile, labelsFile, train ? MnistDataFetcher.NUM_EXAMPLES : MnistDataFetcher.NUM_EXAMPLES_TEST); } - - public MnistManager(String imagesFile, String labelsFile, int numExamples) throws IOException { if (imagesFile != null) { images = new MnistImageFile(imagesFile, "r"); @@ -117,7 +106,6 @@ public class MnistManager { } public byte[] readImageUnsafe(int i) { - Preconditions.checkArgument(i < imagesArr.length); return imagesArr[i]; } diff --git a/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/build.gradle b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/build.gradle new file mode 100644 index 000000000..32d9a5c2f --- /dev/null +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/build.gradle @@ -0,0 +1,28 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +dependencies { + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDnn.cavisDnnApi + + implementation "org.slf4j:slf4j-api" + implementation "org.apache.commons:commons-lang3" +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/SequenceRecordReaderDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/SequenceRecordReaderDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/SequenceRecordReaderDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/SequenceRecordReaderDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/exception/ZeroLengthSequenceException.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/exception/ZeroLengthSequenceException.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/exception/ZeroLengthSequenceException.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-datavec-iterators/src/main/java/org/deeplearning4j/datasets/datavec/exception/ZeroLengthSequenceException.java diff --git a/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/build.gradle b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/build.gradle new file mode 100644 index 000000000..0ced0dbd8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/build.gradle @@ -0,0 +1,27 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +dependencies { + implementation projects.cavisDnn.cavisDnnApi + implementation "org.slf4j:slf4j-api" + implementation "commons-io:commons-io" + implementation "com.google.guava:guava" +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AbstractDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/AsyncShieldMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/BaseDatasetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/BaseDatasetIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/BaseDatasetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/BaseDatasetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedMultiDataSetPreProcessor.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedMultiDataSetPreProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedMultiDataSetPreProcessor.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedMultiDataSetPreProcessor.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessor.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessor.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/CombinedPreProcessor.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetFetcher.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetFetcher.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetFetcher.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetFetcher.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetIteratorSplitter.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetIteratorSplitter.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetIteratorSplitter.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DataSetIteratorSplitter.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DoublesDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DoublesDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DoublesDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DoublesDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyBlockMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyPreProcessor.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyPreProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyPreProcessor.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/DummyPreProcessor.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ExistingDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ExistingDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ExistingDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ExistingDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FileSplitDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FileSplitDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FileSplitDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FileSplitDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FloatsDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FloatsDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FloatsDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/FloatsDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/INDArrayDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/INDArrayDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/INDArrayDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/INDArrayDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/IteratorMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/JointMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetIteratorSplitter.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetIteratorSplitter.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetIteratorSplitter.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetIteratorSplitter.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetWrapperIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetWrapperIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetWrapperIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultiDataSetWrapperIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIterator.java old mode 100755 new mode 100644 similarity index 98% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIterator.java index 8dbf4d320..28f69c7bf --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIterator.java +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/MultipleEpochsIterator.java @@ -20,8 +20,8 @@ package org.deeplearning4j.datasets.iterator; -import org.nd4j.shade.guava.annotations.VisibleForTesting; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; import lombok.Getter; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.DataSetPreProcessor; diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/RandomMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ReconstructionDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ReconstructionDataSetIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ReconstructionDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ReconstructionDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/SamplingDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/SamplingDataSetIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/SamplingDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/SamplingDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/ScrollableMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/WorkspacesShieldDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/WorkspacesShieldDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/WorkspacesShieldDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/WorkspacesShieldDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetCallback.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetCallback.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetCallback.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetCallback.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetDeserializer.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetDeserializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetDeserializer.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DataSetDeserializer.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DefaultCallback.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DefaultCallback.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DefaultCallback.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/DefaultCallback.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/FileCallback.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/FileCallback.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/FileCallback.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/FileCallback.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/InterleavedDataSetCallback.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/InterleavedDataSetCallback.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/InterleavedDataSetCallback.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/callbacks/InterleavedDataSetCallback.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/BaseFileIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/BaseFileIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/BaseFileIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/BaseFileIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/file/FileMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/BenchmarkDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/BenchmarkDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/BenchmarkDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/BenchmarkDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/BenchmarkMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/BenchmarkMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/BenchmarkMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/BenchmarkMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/ListDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/ListDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/ListDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/ListDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/SingletonDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/SingletonDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/SingletonDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/SingletonDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/SingletonMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/SingletonMultiDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/SingletonMultiDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/impl/SingletonMultiDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/DataSetLoaderIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/DataSetLoaderIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/DataSetLoaderIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/DataSetLoaderIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/MultiDataSetLoaderIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/MultiDataSetLoaderIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/MultiDataSetLoaderIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/loader/MultiDataSetLoaderIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/BaseParallelDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/BaseParallelDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/BaseParallelDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/BaseParallelDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/FileSplitParallelDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/FileSplitParallelDataSetIterator.java similarity index 99% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/FileSplitParallelDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/FileSplitParallelDataSetIterator.java index 4f1d73560..40dd67594 100644 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/FileSplitParallelDataSetIterator.java +++ b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/FileSplitParallelDataSetIterator.java @@ -20,7 +20,7 @@ package org.deeplearning4j.datasets.iterator.parallel; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.collect.Lists; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/JointParallelDataSetIterator.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/JointParallelDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/JointParallelDataSetIterator.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/JointParallelDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/MultiBoolean.java b/cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/MultiBoolean.java similarity index 100% rename from deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/MultiBoolean.java rename to cavis-dnn/cavis-dnn-data/cavis-dnn-data-utility-iterators/src/main/java/org/deeplearning4j/datasets/iterator/parallel/MultiBoolean.java diff --git a/cavis-dnn/cavis-dnn-modelimport/build.gradle b/cavis-dnn/cavis-dnn-modelimport/build.gradle new file mode 100644 index 000000000..a0f7b3e1e --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/build.gradle @@ -0,0 +1,59 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +ext { + buildTarget = rootProject.ext.buildTarget +} + +dependencies { + implementation 'org.slf4j:slf4j-api' + implementation projects.cavisDnn.cavisDnnApi + implementation "com.google.code.gson:gson:2.8.6" + implementation projects.cavisDnn.cavisDnnNn + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" + implementation group: "org.bytedeco", name: "javacpp" + implementation group: "org.bytedeco", name: "javacpp", classifier: buildTarget + implementation group: "org.bytedeco", name: "hdf5" + implementation group: "org.bytedeco", name: "hdf5", classifier: buildTarget + implementation projects.cavisDnn.cavisDnnCommon + + implementation "org.apache.commons:commons-lang3" + implementation "commons-io:commons-io" + implementation "org.apache.commons:commons-math3" + implementation "org.apache.commons:commons-collections4:4.1" + implementation "com.google.protobuf:protobuf-java" + implementation "com.google.protobuf:protobuf-java-util" + implementation "com.github.oshi:oshi-core:3.4.2" + implementation "com.google.guava:guava" + + testImplementation projects.cavisDnn.cavisDnnCommonTests + testImplementation 'ch.qos.logback:logback-classic' + + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators + testImplementation projects.cavisNd4j.cavisNd4jTensorflow + testImplementation projects.cavisDatavec.cavisDatavecApi + testImplementation projects.cavisDnn.cavisDnnPython4j.cavisPython4jNumpy + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/Hdf5Archive.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/Hdf5Archive.java similarity index 99% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/Hdf5Archive.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/Hdf5Archive.java index 0e4ccd44f..bb8cd203f 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/Hdf5Archive.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/Hdf5Archive.java @@ -28,8 +28,8 @@ import org.bytedeco.javacpp.Loader; import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.Closeable; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasLayer.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasLayer.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasLayer.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModel.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModel.java similarity index 99% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModel.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModel.java index 4efd8eb24..1c001c1fd 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModel.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModel.java @@ -49,7 +49,7 @@ import org.nd4j.autodiff.samediff.internal.DependencyTracker; import org.nd4j.common.primitives.Counter; import org.nd4j.common.primitives.Pair; import org.nd4j.linalg.learning.config.IUpdater; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.collect.Lists; import org.tensorflow.framework.NodeDef; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModelImport.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModelImport.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModelImport.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasModelImport.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasSequentialModel.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasSequentialModel.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasSequentialModel.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/KerasSequentialModel.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/README.md b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/README.md similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/README.md rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/README.md diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/Keras1LayerConfiguration.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/Keras1LayerConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/Keras1LayerConfiguration.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/Keras1LayerConfiguration.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/Keras2LayerConfiguration.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/Keras2LayerConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/Keras2LayerConfiguration.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/Keras2LayerConfiguration.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasLayerConfiguration.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasLayerConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasLayerConfiguration.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasLayerConfiguration.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasLayerConfigurationFactory.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasLayerConfigurationFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasLayerConfigurationFactory.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasLayerConfigurationFactory.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasModelConfiguration.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasModelConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasModelConfiguration.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/config/KerasModelConfiguration.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/exceptions/InvalidKerasConfigurationException.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/exceptions/InvalidKerasConfigurationException.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/exceptions/InvalidKerasConfigurationException.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/exceptions/InvalidKerasConfigurationException.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/exceptions/UnsupportedKerasConfigurationException.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/exceptions/UnsupportedKerasConfigurationException.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/exceptions/UnsupportedKerasConfigurationException.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/exceptions/UnsupportedKerasConfigurationException.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasInput.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasInput.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasInput.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasInput.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasLoss.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasLoss.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasLoss.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasLoss.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasTFOpLayer.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasTFOpLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasTFOpLayer.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/KerasTFOpLayer.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayer.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayer.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayer.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java similarity index 96% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java index df56cc3aa..480cdf6d3 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/TFOpLayerImpl.java @@ -37,8 +37,8 @@ import org.tensorflow.framework.AttrValue; import org.tensorflow.framework.GraphDef; import org.tensorflow.framework.NodeDef; import com.google.gson.Gson; -import org.nd4j.shade.protobuf.Message; -import org.nd4j.shade.protobuf.TextFormat; +import com.google.protobuf.Message; +import com.google.protobuf.TextFormat; import java.util.*; import java.util.List; @@ -75,7 +75,7 @@ public class TFOpLayerImpl extends AbstractLayer { try{ String json = new Gson().toJson(nodeDef); NodeDef.Builder builder = NodeDef.newBuilder(); - org.nd4j.shade.protobuf.util.JsonFormat.parser().merge(json, builder); + com.google.protobuf.util.JsonFormat.parser().merge(json, builder); NodeDef nodeDef = builder.build(); List allInputNames = new ArrayList<>(); // including constants Map inputDataTypes = new HashMap<>(); @@ -112,7 +112,7 @@ public class TFOpLayerImpl extends AbstractLayer { GraphDef.Builder graphDefBuilder = GraphDef.newBuilder(); TextFormat.getParser().merge(graph, graphDefBuilder); GraphDef graphDef = graphDefBuilder.build(); - org.nd4j.shade.protobuf.ByteString serialized = graphDef.toByteString(); + com.google.protobuf.ByteString serialized = graphDef.toByteString(); byte[] graphBytes = serialized.toByteArray(); ServiceLoader sl = DL4JClassLoading.loadService(TFGraphRunnerService.class); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasELU.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasELU.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasELU.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasELU.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasLeakyReLU.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasLeakyReLU.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasLeakyReLU.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasLeakyReLU.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasPReLU.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasPReLU.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasPReLU.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasPReLU.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasReLU.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasReLU.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasReLU.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasReLU.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasSoftmax.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasSoftmax.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasSoftmax.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasSoftmax.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasThresholdedReLU.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasThresholdedReLU.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasThresholdedReLU.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activations/KerasThresholdedReLU.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution1D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution1D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution1D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution1D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution1D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution1D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution3D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution3D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution3D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolutionUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolutionUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolutionUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolutionUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping1D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping1D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping1D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping3D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping3D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasCropping3D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasDeconvolution2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasDeconvolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasDeconvolution2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasDeconvolution2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasDepthwiseConvolution2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasDepthwiseConvolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasDepthwiseConvolution2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasDepthwiseConvolution2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasSeparableConvolution2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasSeparableConvolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasSeparableConvolution2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasSeparableConvolution2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasSpaceToDepth.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasSpaceToDepth.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasSpaceToDepth.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasSpaceToDepth.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling1D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling1D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling1D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling3D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling3D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasUpsampling3D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding1D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding1D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding1D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding3D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding3D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasZeroPadding3D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivation.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivation.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivation.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivation.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDense.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDense.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDense.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDense.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropout.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropout.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropout.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropout.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasFlatten.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasFlatten.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasFlatten.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasFlatten.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasLambda.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasLambda.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasLambda.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasLambda.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMasking.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMasking.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMasking.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMasking.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMerge.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMerge.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMerge.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMerge.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermute.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermute.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermute.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermute.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVector.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVector.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVector.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVector.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshape.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshape.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshape.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshape.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/custom/KerasLRN.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/custom/KerasLRN.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/custom/KerasLRN.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/custom/KerasLRN.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/custom/KerasPoolHelper.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/custom/KerasPoolHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/custom/KerasPoolHelper.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/custom/KerasPoolHelper.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/Keras2DEmbedding.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/Keras2DEmbedding.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/Keras2DEmbedding.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/Keras2DEmbedding.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbedding.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbedding.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbedding.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbedding.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropout.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropout.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropout.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropout.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropout.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropout.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropout.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropout.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoise.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoise.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoise.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoise.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalization.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalization.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalization.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalization.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasGlobalPooling.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasGlobalPooling.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasGlobalPooling.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasGlobalPooling.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3D.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3D.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3D.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPoolingUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPoolingUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPoolingUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPoolingUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTM.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTM.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTM.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasRnnUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasRnnUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasRnnUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasRnnUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnn.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnn.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnn.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnn.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectional.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectional.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectional.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectional.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGenerator.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGenerator.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGenerator.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGenerator.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/KerasTokenizer.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/KerasTokenizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/KerasTokenizer.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/KerasTokenizer.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerMode.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerMode.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerMode.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/KerasFlattenRnnPreprocessor.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/KerasFlattenRnnPreprocessor.java similarity index 97% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/KerasFlattenRnnPreprocessor.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/KerasFlattenRnnPreprocessor.java index 5866b2f18..37df1f31b 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/KerasFlattenRnnPreprocessor.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/KerasFlattenRnnPreprocessor.java @@ -28,7 +28,7 @@ import org.deeplearning4j.nn.conf.preprocessor.BaseInputPreProcessor; import org.deeplearning4j.nn.workspace.ArrayType; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Slf4j @Data diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/PermutePreprocessor.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/PermutePreprocessor.java similarity index 97% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/PermutePreprocessor.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/PermutePreprocessor.java index 90eb74931..3f24bba92 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/PermutePreprocessor.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/PermutePreprocessor.java @@ -30,8 +30,8 @@ import org.deeplearning4j.nn.workspace.ArrayType; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @Data diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/ReshapePreprocessor.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/ReshapePreprocessor.java similarity index 98% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/ReshapePreprocessor.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/ReshapePreprocessor.java index ba41cc91d..002ce6b57 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/ReshapePreprocessor.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/ReshapePreprocessor.java @@ -34,8 +34,8 @@ import org.deeplearning4j.nn.workspace.ArrayType; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/TensorFlowCnnToFeedForwardPreProcessor.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/TensorFlowCnnToFeedForwardPreProcessor.java similarity index 96% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/TensorFlowCnnToFeedForwardPreProcessor.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/TensorFlowCnnToFeedForwardPreProcessor.java index 754850a4a..edd14e369 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/TensorFlowCnnToFeedForwardPreProcessor.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/preprocessors/TensorFlowCnnToFeedForwardPreProcessor.java @@ -27,8 +27,8 @@ import org.deeplearning4j.nn.workspace.ArrayType; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; @Slf4j @Deprecated public class TensorFlowCnnToFeedForwardPreProcessor extends CnnToFeedForwardPreProcessor { diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/DL4JKerasModelValidator.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/DL4JKerasModelValidator.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/DL4JKerasModelValidator.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/DL4JKerasModelValidator.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasActivationUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasActivationUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasActivationUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasActivationUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasConstraintUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasConstraintUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasConstraintUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasConstraintUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasInitilizationUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasInitilizationUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasInitilizationUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasInitilizationUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLayerUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLayerUtils.java similarity index 99% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLayerUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLayerUtils.java index 6c2fd66a7..536afb915 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLayerUtils.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLayerUtils.java @@ -297,7 +297,7 @@ public class KerasLayerUtils { layer = new KerasUpsampling1D(layerConfig, enforceTrainingConfig); } else if (layerClassName.equals(conf.getLAYER_CLASS_NAME_UPSAMPLING_2D())) { layer = new KerasUpsampling2D(layerConfig, enforceTrainingConfig); - }else if (layerClassName.equals(conf.getLAYER_CLASS_NAME_UPSAMPLING_2D())) { + }else if (layerClassName.equals(conf.getLAYER_CLASS_NAME_UPSAMPLING_3D())) { layer = new KerasUpsampling3D(layerConfig, enforceTrainingConfig); } else if (layerClassName.equals(conf.getLAYER_CLASS_NAME_CROPPING_3D())) { layer = new KerasCropping3D(layerConfig, enforceTrainingConfig); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLossUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLossUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLossUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasLossUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelBuilder.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelBuilder.java similarity index 99% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelBuilder.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelBuilder.java index a32219b6e..92691ddf6 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelBuilder.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelBuilder.java @@ -29,7 +29,7 @@ import org.deeplearning4j.nn.modelimport.keras.KerasSequentialModel; import org.deeplearning4j.nn.modelimport.keras.config.KerasModelConfiguration; import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.*; import java.nio.file.Files; diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelUtils.java similarity index 99% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelUtils.java index 859537f93..ad11282e5 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelUtils.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasModelUtils.java @@ -34,9 +34,9 @@ import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfig import org.deeplearning4j.nn.modelimport.keras.layers.wrappers.KerasBidirectional; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.core.type.TypeReference; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.IOException; import java.util.*; diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasOptimizerUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasOptimizerUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasOptimizerUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasOptimizerUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasRegularizerUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasRegularizerUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasRegularizerUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasRegularizerUtils.java diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/KerasTestUtils.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/KerasTestUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/KerasTestUtils.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/KerasTestUtils.java diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/MiscTests.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/MiscTests.java new file mode 100644 index 000000000..e29503c5f --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/MiscTests.java @@ -0,0 +1,240 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras; + +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.utils.DL4JKerasModelValidator; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.resources.Resources; +import org.nd4j.common.validation.ValidationResult; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; + +@Timeout(60000L) +public class MiscTests extends BaseDL4JTest { + + @TempDir + public File testDir; + +@Test + public void testMultiThreadedLoading() throws Exception { + final File f = Resources.asFile("modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_model.h5"); + + int numThreads = 4; + final CountDownLatch latch = new CountDownLatch(numThreads); + final AtomicInteger errors = new AtomicInteger(); + for( int i=0; i { + runModelConfigTest("modelimport/keras/foo/bar.json"); + }); + } + + @Test + public void notAFileTest() throws Exception { + Assertions.assertThrows(IOException.class, () -> { + runModelConfigTest("modelimport/keras/"); + }); + } + + + @Test + public void simple222ConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/model_2_2_2.json"); + } + + @Test + public void simple224ConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/model_2_2_4.json"); + } + + @Test + public void yolo9000ConfigTest() throws Exception { + KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); + runModelConfigTest("modelimport/keras/configs/keras2/yolo9000_tf_keras_2.json"); + } + + @Test + public void l1l2RegularizerDenseTfConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/l1l2_regularizer_dense_tf_keras_2_config.json"); + } + + @Test + public void dgaClassifierTfConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_dga_classifier_tf_config.json"); + } + + @Test + public void convPooling1dTfConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_conv1d_pooling1d_tf_config.json"); + } + + @Test + public void bidirectionalLstmConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/bidirectional_lstm_tf_keras_2_config.json"); + } + + @Test + public void imdbLstmTfSequentialConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/imdb_lstm_tf_keras_2_config.json"); + } + + @Test + public void imdbLstmThSequentialConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/imdb_lstm_th_keras_2_config.json"); + } + + @Test + public void simpleRnnConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/simple_rnn_tf_keras_2_config.json"); + } + + @Test + public void simplePreluConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/prelu_config_tf_keras_2.json"); + } + + @Test + public void mnistMlpTfSequentialConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/mnist_mlp_tf_keras_2_config.json"); + } + + @Test + public void mnistMlpThSequentialConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/mnist_mlp_th_keras_2_config.json"); + } + + @Test + public void mnistCnnTfSequentialConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/mnist_cnn_tf_keras_2_config.json"); + } + + @Test + public void mnistCnnThSequentialConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/mnist_cnn_th_keras_2_config.json"); + } + + @Test + public void mnistCnnNoBiasTfSequentialConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_mnist_cnn_no_bias_tf_config.json"); + } + + + @Test + public void mlpSequentialConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_mlp_config.json"); + } + + + @Test + public void mlpConstraintsConfigTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/mnist_mlp_constraint_tf_keras_2_config.json"); + } + + @Test + public void embeddingFlattenThTest() throws Exception { + runModelConfigTest("modelimport/keras/configs/keras2/embedding_flatten_graph_th_keras_2.json"); + } + + @Test + public void mlpFapiConfigTest() throws Exception { + runModelConfigTest("modelimport/keras/configs/keras2/keras2_mlp_fapi_config.json"); + } + + @Test + public void mlpFapiMultiLossConfigTest() throws Exception { + runModelConfigTest("modelimport/keras/configs/keras2/keras2_mlp_fapi_multiloss_config.json"); + } + + @Test + public void cnnTfTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_cnn_tf_config.json"); + } + + @Test + public void cnnThTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_cnn_th_config.json"); + } + + @Test + public void mnistCnnTfTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_mnist_cnn_tf_config.json"); + } + + @Test + public void mnistMlpTfTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_mnist_mlp_tf_config.json"); + } + + @Test + public void embeddingConv1DTfTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/keras2_tf_embedding_conv1d_config.json"); + } + + @Test + public void flattenConv1DTfTest() throws Exception { + runSequentialConfigTest("modelimport/keras/configs/keras2/flatten_conv1d_tf_keras_2.json"); + } + + @Test + public void embeddingLSTMMaskZeroTest() throws Exception { + String path = "modelimport/keras/configs/keras2/embedding_lstm_calculator.json"; + try(InputStream is = Resources.asStream(path)) { + ComputationGraphConfiguration config = + new KerasModel().modelBuilder().modelJsonInputStream(is) + .enforceTrainingConfig(false).buildModel().getComputationGraphConfiguration(); + ComputationGraph model = new ComputationGraph(config); + model.init(); + INDArray output = model.outputSingle(Nd4j.zeros(1,3)); + System.out.println(output.shapeInfoToString()); + } + + } + + @Test + public void permuteRetinaUnet() throws Exception { + runModelConfigTest("modelimport/keras/configs/keras2/permute_retina_unet.json"); + } + + + @Test + public void simpleAddLayerTest() throws Exception { + runModelConfigTest("modelimport/keras/configs/keras2/simple_add_tf_keras_2.json"); + } + + @Override + public long getTimeoutMilliseconds() { + return 999999999L; + } + + @Test + public void embeddingConcatTest() throws Exception { + runModelConfigTest("/modelimport/keras/configs/keras2/model_concat_embedding_sequences_tf_keras_2.json"); + } + + @Test + public void conv1dDilationTest() throws Exception { + runModelConfigTest("/modelimport/keras/configs/keras2/conv1d_dilation_tf_keras_2_config.json"); + } + + @Test + public void test5982() throws Exception { + File jsonFile = Resources.asFile("modelimport/keras/configs/bidirectional_last_timeStep.json"); + val modelGraphConf = KerasModelImport.importKerasSequentialConfiguration(jsonFile.getAbsolutePath()); + MultiLayerNetwork model = new MultiLayerNetwork(modelGraphConf); + + INDArray features = Nd4j.create(new double[]{1, 3, 1, 2, 2, 1, 82, 2, 10,1, 3, 1, 2, 1, 82, 3, 1, 10, 1, 2, 1, 3, + 1, 10, 82, 2, 1, 1, 10, 82, 2, 3, 1, 2, 1, 10, 1, 2, 3, 82, 2, 1, 10, 3, 82, 1, 2, 1, 10, 1}, new int[]{1,1,50}); + + model.init(); + INDArray out = model.output(features); + assertArrayEquals(new long[]{1,14}, out.shape()); + } + + @Test + public void oneLstmLayerTest() throws Exception { + try(InputStream is = Resources.asStream("/modelimport/keras/configs/keras2/one_lstm_no_sequences_tf_keras_2.json")) { + MultiLayerConfiguration config = + new KerasModel().modelBuilder().modelJsonInputStream(is) + .enforceTrainingConfig(false).buildSequential().getMultiLayerConfiguration(); + MultiLayerNetwork model = new MultiLayerNetwork(config); + model.init(); + INDArray input = Nd4j.create(DataType.FLOAT, 50, 1500, 500); //NWC format - [Minibatch, seqLength, channels] + INDArray out = model.output(input); + assertTrue(Arrays.equals(out.shape(), new long[]{50, 64})); + } + } + + @Test + ////@Ignore("AB 2019/11/23 - known issue - see https://github.com/eclipse/deeplearning4j/issues/8373 and https://github.com/eclipse/deeplearning4j/issues/8441") + public void ReshapeEmbeddingConcatTest() throws Exception{ + try(InputStream is = Resources.asStream("/modelimport/keras/configs/keras2/reshape_embedding_concat.json")) { + ComputationGraphConfiguration config = + new KerasModel().modelBuilder().modelJsonInputStream(is) + .enforceTrainingConfig(false).buildModel().getComputationGraphConfiguration(); + ComputationGraph model = new ComputationGraph(config); + model.init(); +// System.out.println(model.summary()); + model.outputSingle(Nd4j.zeros(1, 1), Nd4j.zeros(1, 1), Nd4j.zeros(1, 1)); + } + } + + private void runSequentialConfigTest(String path) throws Exception { + try(InputStream is = Resources.asStream(path)) { + MultiLayerConfiguration config = + new KerasModel().modelBuilder().modelJsonInputStream(is) + .enforceTrainingConfig(false).buildSequential().getMultiLayerConfiguration(); + MultiLayerNetwork model = new MultiLayerNetwork(config); + model.init(); + } + } + + private void runModelConfigTest(String path) throws Exception { + try(InputStream is = Resources.asStream(path)) { + ComputationGraphConfiguration config = + new KerasModel().modelBuilder().modelJsonInputStream(is) + .enforceTrainingConfig(false).buildModel().getComputationGraphConfiguration(); + ComputationGraph model = new ComputationGraph(config); + model.init(); + } + } +} diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/configurations/KerasInitilizationTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/configurations/KerasInitilizationTest.java new file mode 100644 index 000000000..31fb10f09 --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/configurations/KerasInitilizationTest.java @@ -0,0 +1,173 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.configurations; + +import org.deeplearning4j.nn.conf.distribution.*; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; +import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; +import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; +import org.deeplearning4j.nn.modelimport.keras.layers.core.KerasDense; +import org.deeplearning4j.nn.weights.IWeightInit; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.nn.weights.WeightInitIdentity; +import org.deeplearning4j.nn.weights.WeightInitVarScalingNormalFanIn; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class KerasInitilizationTest extends BaseDL4JTest { + + private double minValue = -0.2; + private double maxValue = 0.2; + private double mean = 0.0; + private double stdDev = 0.2; + private double value = 42.0; + private double gain = 0.2; + + private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); + private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + + @Test + public void testInitializers() throws Exception { + + Integer keras1 = 1; + Integer keras2 = 2; + + String[] keras1Inits = initializers(conf1); + String[] keras2Inits = initializers(conf2); + IWeightInit[] dl4jInits = dl4jInitializers(); + + for (int i = 0; i < dl4jInits.length - 1; i++) { + initilizationDenseLayer(conf1, keras1, keras1Inits[i], dl4jInits[i]); + initilizationDenseLayer(conf2, keras2, keras2Inits[i], dl4jInits[i]); + + initilizationDenseLayer(conf2, keras2, keras2Inits[dl4jInits.length - 1], + dl4jInits[dl4jInits.length - 1]); + } + } + + private String[] initializers(KerasLayerConfiguration conf) { + return new String[]{ + conf.getINIT_GLOROT_NORMAL(), + conf.getINIT_GLOROT_UNIFORM_ALIAS(), + conf.getINIT_LECUN_NORMAL(), + conf.getINIT_LECUN_UNIFORM(), + conf.getINIT_RANDOM_UNIFORM(), + conf.getINIT_HE_NORMAL(), + conf.getINIT_HE_UNIFORM(), + conf.getINIT_ONES(), + conf.getINIT_ZERO(), + conf.getINIT_IDENTITY(), + conf.getINIT_NORMAL(), + conf.getINIT_ORTHOGONAL(), + conf.getINIT_CONSTANT(), + conf.getINIT_VARIANCE_SCALING() + + }; + } + + private IWeightInit[] dl4jInitializers() { + return new IWeightInit[]{ + WeightInit.XAVIER.getWeightInitFunction(), + WeightInit.XAVIER_UNIFORM.getWeightInitFunction(), + WeightInit.LECUN_NORMAL.getWeightInitFunction(), + WeightInit.LECUN_UNIFORM.getWeightInitFunction(), + WeightInit.DISTRIBUTION.getWeightInitFunction(new UniformDistribution(minValue, maxValue)), + WeightInit.RELU.getWeightInitFunction(), + WeightInit.RELU_UNIFORM.getWeightInitFunction(), + WeightInit.ONES.getWeightInitFunction(), + WeightInit.ZERO.getWeightInitFunction(), + new WeightInitIdentity(0.2), + WeightInit.DISTRIBUTION.getWeightInitFunction(new NormalDistribution(mean, stdDev)), + WeightInit.DISTRIBUTION.getWeightInitFunction(new OrthogonalDistribution(gain)), + WeightInit.DISTRIBUTION.getWeightInitFunction(new ConstantDistribution(value)), + new WeightInitVarScalingNormalFanIn(0.2)}; + } + + private Distribution[] dl4jDistributions() { + return new Distribution[]{ + null, + null, + null, + null, + new UniformDistribution(minValue, maxValue), + null, + null, + null, + null, + null, + new NormalDistribution(mean, stdDev), + new OrthogonalDistribution(gain), + new ConstantDistribution(value), + null}; + } + + private void initilizationDenseLayer(KerasLayerConfiguration conf, Integer kerasVersion, + String initializer, IWeightInit dl4jInitializer) + throws Exception { + Map layerConfig = new HashMap<>(); + layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_DENSE()); + Map config = new HashMap<>(); + config.put(conf.getLAYER_FIELD_ACTIVATION(), "linear"); + config.put(conf.getLAYER_FIELD_NAME(), "init_test"); + double scale = 0.2; + if (kerasVersion == 1) { + config.put(conf.getLAYER_FIELD_INIT(), initializer); + config.put(conf.getLAYER_FIELD_INIT_MEAN(), mean); + config.put(conf.getLAYER_FIELD_INIT_STDDEV(), stdDev); + config.put(conf.getLAYER_FIELD_INIT_SCALE(), scale); + config.put(conf.getLAYER_FIELD_INIT_MINVAL(), minValue); + config.put(conf.getLAYER_FIELD_INIT_MAXVAL(), maxValue); + config.put(conf.getLAYER_FIELD_INIT_VALUE(), value); + config.put(conf.getLAYER_FIELD_INIT_GAIN(), gain); + } else { + Map init = new HashMap<>(); + init.put("class_name", initializer); + Map innerInit = new HashMap<>(); + innerInit.put(conf.getLAYER_FIELD_INIT_MEAN(), mean); + innerInit.put(conf.getLAYER_FIELD_INIT_STDDEV(), stdDev); + innerInit.put(conf.getLAYER_FIELD_INIT_SCALE(), scale); + innerInit.put(conf.getLAYER_FIELD_INIT_MINVAL(), minValue); + innerInit.put(conf.getLAYER_FIELD_INIT_MAXVAL(), maxValue); + innerInit.put(conf.getLAYER_FIELD_INIT_VALUE(), value); + innerInit.put(conf.getLAYER_FIELD_INIT_GAIN(), gain); + String mode = "fan_in"; + innerInit.put(conf.getLAYER_FIELD_INIT_MODE(), mode); + String distribution = "normal"; + innerInit.put(conf.getLAYER_FIELD_INIT_DISTRIBUTION(), distribution); + + init.put(conf.getLAYER_FIELD_CONFIG(), innerInit); + config.put(conf.getLAYER_FIELD_INIT(), init); + } + config.put(conf.getLAYER_FIELD_OUTPUT_DIM(), 1337); + layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); + layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + + DenseLayer layer = new KerasDense(layerConfig, false).getDenseLayer(); + assertEquals(dl4jInitializer, layer.getWeightInitFn()); + + } +} diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/configurations/KerasModelImportTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/configurations/KerasModelImportTest.java new file mode 100644 index 000000000..41fa9edc5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/configurations/KerasModelImportTest.java @@ -0,0 +1,122 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.configurations; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.CNN2DFormat; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; +import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; +import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; +import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.Test; +import org.nd4j.common.resources.Resources; +import org.nd4j.linalg.convolution.Convolution; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Test import of Keras models. + */ +@Slf4j +public class KerasModelImportTest extends BaseDL4JTest { + @Override + public long getTimeoutMilliseconds() { + return 9999999999999L; + } + + @Test + public void testH5WithoutTensorflowScope() throws Exception { + MultiLayerNetwork model = loadModel("modelimport/keras/tfscope/model.h5"); + assertNotNull(model); + } + + @Test + public void testNCHWNWHCChangeImport() { + MultiLayerNetwork model = loadModel("modelimport/keras/weights/conv2dnchw/simpleconv2d.hdf5"); + MultiLayerConfiguration multiLayerConfiguration = model.getLayerWiseConfigurations(); + ConvolutionLayer convolutionLayer = (ConvolutionLayer) multiLayerConfiguration.getConf(0).getLayer(); + assertEquals(CNN2DFormat.NCHW,convolutionLayer.getCnn2dDataFormat()); + SubsamplingLayer subsamplingLayer = (SubsamplingLayer) multiLayerConfiguration.getConf(1).getLayer(); + assertEquals(CNN2DFormat.NHWC,subsamplingLayer.getCnn2dDataFormat()); + ConvolutionLayer convolutionLayer1 = (ConvolutionLayer) multiLayerConfiguration.getConf(2).getLayer(); + assertEquals(CNN2DFormat.NHWC,convolutionLayer1.getCnn2dDataFormat()); + + model.output(Nd4j.zeros(1,1,28,28)); + assertNotNull(model); + } + + + @Test + public void testH5WithTensorflowScope() throws Exception { + MultiLayerNetwork model = loadModel("modelimport/keras/tfscope/model.h5.with.tensorflow.scope"); + assertNotNull(model); + } + + @Test + public void testWeightAndJsonWithoutTensorflowScope() throws Exception { + MultiLayerNetwork model = loadModel("modelimport/keras/tfscope/model.json", + "modelimport/keras/tfscope/model.weight"); + assertNotNull(model); + } + + @Test + public void testWeightAndJsonWithTensorflowScope() throws Exception { + MultiLayerNetwork model = loadModel( + "modelimport/keras/tfscope/model.json.with.tensorflow.scope", + "modelimport/keras/tfscope/model.weight.with.tensorflow.scope"); + assertNotNull(model); + } + + private MultiLayerNetwork loadModel(String modelJsonFilename, String modelWeightFilename) + throws NullPointerException { + MultiLayerNetwork network = null; + try { + network = KerasModelImport.importKerasSequentialModelAndWeights(Resources.asFile(modelJsonFilename).getAbsolutePath(), + Resources.asFile(modelWeightFilename).getAbsolutePath(), false); + } catch (IOException | InvalidKerasConfigurationException | UnsupportedKerasConfigurationException e) { + log.error("",e); + } + + return network; + } + + private MultiLayerNetwork loadModel(String modelFilename) { + MultiLayerNetwork model = null; + try { + model = KerasModelImport.importKerasSequentialModelAndWeights(Resources.asFile(modelFilename).getAbsolutePath()); + } catch (IOException | InvalidKerasConfigurationException | UnsupportedKerasConfigurationException e) { + log.error("",e); + } + + return model; + } + + +} diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasCustomLayerTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasCustomLayerTest.java new file mode 100644 index 000000000..06683cd07 --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasCustomLayerTest.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.e2e; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.common.resources.DL4JResources; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.KerasLayer; +import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; +import org.deeplearning4j.nn.modelimport.keras.layers.custom.KerasLRN; +import org.deeplearning4j.nn.modelimport.keras.layers.custom.KerasPoolHelper; +import org.deeplearning4j.util.ModelSerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.net.URL; + +@Slf4j +public class KerasCustomLayerTest extends BaseDL4JTest { + + @TempDir + public File testDir; + + // run manually, might take a long time to load (too long for unit tests) + ////@Ignore + @Test + public void testCustomLayerImport() throws Exception { + // file paths + String kerasWeightsAndConfigUrl = DL4JResources.getURLString("googlenet_keras_weightsandconfig.h5"); + File cachedKerasFile = new File(testDir.getAbsolutePath() + File.pathSeparator + "googlenet_keras_weightsandconfig.h5"); + String outputPath = new File(testDir.getAbsolutePath() + File.pathSeparator + "googlenet_dl4j_inference.zip").getAbsolutePath(); + + KerasLayer.registerCustomLayer("PoolHelper", KerasPoolHelper.class); + KerasLayer.registerCustomLayer("LRN", KerasLRN.class); + + // download file + if (!cachedKerasFile.exists()) { + log.info("Downloading model to " + cachedKerasFile.toString()); + FileUtils.copyURLToFile(new URL(kerasWeightsAndConfigUrl), cachedKerasFile); + cachedKerasFile.deleteOnExit(); + } + + org.deeplearning4j.nn.api.Model importedModel = + KerasModelImport.importKerasModelAndWeights(cachedKerasFile.getAbsolutePath()); + ModelSerializer.writeModel(importedModel, outputPath, false); + + ComputationGraph serializedModel = ModelSerializer.restoreComputationGraph(outputPath); + log.info(serializedModel.summary()); + } +} diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasCustomLossTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasCustomLossTest.java new file mode 100644 index 000000000..5d394351a --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasCustomLossTest.java @@ -0,0 +1,76 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.e2e; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.KerasSequentialModel; +import org.deeplearning4j.nn.modelimport.keras.utils.KerasLossUtils; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.resources.Resources; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.SameDiffLoss; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + + +public class KerasCustomLossTest extends BaseDL4JTest { + + @TempDir + public File testDir; + + public class LogCosh extends SameDiffLoss { + @Override + public SDVariable defineLoss(SameDiff sd, SDVariable layerInput, SDVariable labels) { + return sd.math.log(sd.math.cosh(labels.sub(layerInput))); + } + } + + @Test + public void testSequentialLambdaLayerImport() throws Exception { + KerasLossUtils.registerCustomLoss("logcosh", new LogCosh()); + + String modelPath = "modelimport/keras/examples/custom_loss.h5"; + + try(InputStream is = Resources.asStream(modelPath)) { + File modelFile = new File(testDir.getAbsolutePath() + File.pathSeparator + "tempModel" + System.currentTimeMillis() + ".h5"); + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + MultiLayerNetwork model = new KerasSequentialModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(true).buildSequential().getMultiLayerNetwork(); + + System.out.println(model.summary()); + INDArray input = Nd4j.create(new int[]{10, 3}); + + model.output(input); + } finally { + KerasLossUtils.clearCustomLoss(); + } + } + + +} diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasLambdaTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasLambdaTest.java new file mode 100644 index 000000000..726de2e1f --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasLambdaTest.java @@ -0,0 +1,116 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.e2e; + +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.samediff.SameDiffLambdaLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.KerasLayer; +import org.deeplearning4j.nn.modelimport.keras.KerasModel; +import org.deeplearning4j.nn.modelimport.keras.KerasSequentialModel; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.resources.Resources; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + + +/** + * Test importing Keras models with multiple Lamdba layers. + * + * @author Max Pumperla + */ +public class KerasLambdaTest extends BaseDL4JTest { + + @TempDir + public File testDir; + + public class ExponentialLambda extends SameDiffLambdaLayer { + @Override + public SDVariable defineLayer(SameDiff sd, SDVariable x) { return x.mul(x); } + + @Override + public InputType getOutputType(int layerIndex, InputType inputType) { return inputType; } + } + + public class TimesThreeLambda extends SameDiffLambdaLayer { + @Override + public SDVariable defineLayer(SameDiff sd, SDVariable x) { return x.mul(3); } + + @Override + public InputType getOutputType(int layerIndex, InputType inputType) { return inputType; } + } + + + @Test + public void testSequentialLambdaLayerImport() throws Exception { + KerasLayer.registerLambdaLayer("lambda_1", new ExponentialLambda()); + KerasLayer.registerLambdaLayer("lambda_2", new TimesThreeLambda()); + + String modelPath = "modelimport/keras/examples/lambda/sequential_lambda.h5"; + + try(InputStream is = Resources.asStream(modelPath)) { + File modelFile = new File(testDir.getAbsolutePath() + File.pathSeparator + "tempModel" + System.currentTimeMillis() + ".h5"); + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + MultiLayerNetwork model = new KerasSequentialModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(false).buildSequential().getMultiLayerNetwork(); + + System.out.println(model.summary()); + INDArray input = Nd4j.create(new int[]{10, 100}); + + model.output(input); + } finally { + KerasLayer.clearLambdaLayers(); + } + } + + @Test + public void testModelLambdaLayerImport() throws Exception { + KerasLayer.registerLambdaLayer("lambda_3", new ExponentialLambda()); + KerasLayer.registerLambdaLayer("lambda_4", new TimesThreeLambda()); + + String modelPath = "modelimport/keras/examples/lambda/model_lambda.h5"; + + try(InputStream is = Resources.asStream(modelPath)) { + File modelFile = new File(testDir.getAbsolutePath() + File.pathSeparator + "tempModel" + System.currentTimeMillis() + ".h5"); + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + ComputationGraph model = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(false).buildModel().getComputationGraph(); + + System.out.println(model.summary()); + INDArray input = Nd4j.create(new int[]{10, 784}); + + model.output(input); + } finally { + KerasLayer.clearLambdaLayers(); // Clear all lambdas, so other tests aren't affected. + } + } + +} diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasModelEndToEndTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasModelEndToEndTest.java new file mode 100644 index 000000000..65bac76b4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasModelEndToEndTest.java @@ -0,0 +1,1033 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.e2e; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.common.resources.DL4JResources; +import org.deeplearning4j.eval.ROCMultiClass; +import org.deeplearning4j.gradientcheck.GradientCheckUtil; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.layers.IOutputLayer; +import org.deeplearning4j.nn.conf.ConvolutionMode; +import org.deeplearning4j.nn.conf.layers.Convolution1DLayer; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; +import org.deeplearning4j.nn.conf.layers.LossLayer; +import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.Hdf5Archive; +import org.deeplearning4j.nn.modelimport.keras.KerasModel; +import org.deeplearning4j.nn.modelimport.keras.KerasSequentialModel; +import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; +import org.deeplearning4j.nn.modelimport.keras.utils.KerasModelBuilder; +import org.deeplearning4j.nn.modelimport.keras.utils.KerasModelUtils; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; +import org.deeplearning4j.nn.transferlearning.TransferLearning; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.activations.impl.*; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.function.BiFunction; +import org.nd4j.common.function.Function; +import org.nd4j.linalg.learning.config.NoOp; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.lossfunctions.impl.LossSparseMCXENT; +import org.nd4j.common.resources.Resources; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for end-to-end Keras model import. + * + * @author dave@skymind.io, Max Pumperla + */ +@Slf4j +public class KerasModelEndToEndTest extends BaseDL4JTest { + private static final String GROUP_ATTR_INPUTS = "inputs"; + private static final String GROUP_ATTR_OUTPUTS = "outputs"; + private static final String GROUP_PREDICTIONS = "predictions"; + private static final String GROUP_ACTIVATIONS = "activations"; + private static final String TEMP_OUTPUTS_FILENAME = "tempOutputs"; + private static final String TEMP_MODEL_FILENAME = "tempModel"; + private static final String H5_EXTENSION = ".h5"; + private static final double EPS = 1E-5; + + private static final boolean SKIP_GRAD_CHECKS = true; + + @TempDir + public File testDir; + + @Override + public long getTimeoutMilliseconds() { + return 900000000L; //Most benchmarks should run very quickly; large timeout is to avoid issues with unusually slow download of test resources + } + + @Test + public void fileNotFoundEndToEnd() throws Exception { + String modelPath = "modelimport/keras/examples/foo/bar.h5"; + Assertions.assertThrows(IllegalArgumentException.class, () -> { + importEndModelTest(modelPath, null, true, true, false, false); + }); + } + + /** + * MNIST MLP tests + */ + @Test + public void importMnistMlpTfKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false); + } + + @Test + public void importMnistMlpThKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_th_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_th_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, false, true, false, false); + } + + @Test + public void importMnistMlpTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false); + } + + @Test + public void importMnistMlpReshapeTfKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/mnist_mlp_reshape/mnist_mlp_reshape_tf_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/mnist_mlp_reshape/mnist_mlp_reshape_tf_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, true, false); + } + + /** + * MNIST CNN tests + */ + @Test + public void importMnistCnnTfKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, false, false, false); + } + + @Test + public void importMnistCnnThKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_th_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_th_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, false, true, true, false); + } + + @Test + public void importMnistCnnTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, true, false); + } + + /** + * IMDB Embedding and LSTM test + */ + @Test + public void importImdbLstmTfKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_tf_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_tf_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false, true, null, null); + } + + @Test + @Tag("long-running") + public void importImdbLstmThKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_th_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_th_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false, true, null, null); + } + + @Test + public void importImdbLstmTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false, true, null, null); + } + + @Test + public void importImdbLstmThKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_th_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_th_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, false, true, false, false, true, null, null); + } + + /** + * IMDB LSTM fasttext + */ + // TODO: prediction checks fail due to globalpooling for fasttext, very few grads fail as well + @Test + public void importImdbFasttextTfKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_tf_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_tf_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, false, false, false, false); + } + + @Test + public void importImdbFasttextThKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_th_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_th_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, false, false, false, false); + } + + @Test + public void importImdbFasttextTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, false, false, false); + } + + /** + * Simple LSTM (return sequences = false) into Dense layer test + */ + @Test + public void importSimpleLstmTfKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/simple_lstm/simple_lstm_tf_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/simple_lstm/simple_lstm_tf_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false); + } + + @Test + public void importSimpleLstmThKeras1() throws Exception { + String modelPath = "modelimport/keras/examples/simple_lstm/simple_lstm_th_keras_1_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/simple_lstm/simple_lstm_th_keras_1_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false); + } + + @Test + public void importSimpleLstmTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/simple_lstm/simple_lstm_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/simple_lstm/simple_lstm_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, false, false, false); + } + + + /** + * Simple LSTM (return sequences = true) into flatten into Dense layer test + */ + @Test + public void importSimpleFlattenLstmTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/simple_flatten_lstm/simple_flatten_lstm_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/simple_flatten_lstm/" + + "simple_flatten_lstm_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false); + } + + /** + * Simple RNN (return sequences = true) into flatten into Dense layer test + */ + @Test + public void importSimpleFlattenRnnTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/simple_flatten_rnn/simple_flatten_rnn_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/simple_flatten_rnn/" + + "simple_flatten_rnn_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false, true, null, null); + } + + /** + * Simple RNN (return sequences = false) into Dense layer test + */ + @Test + public void importSimpleRnnTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/simple_rnn/simple_rnn_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/simple_rnn/" + + "simple_rnn_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, false, false); + } + + /** + * CNN without bias test + */ + @Test + public void importCnnNoBiasTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/cnn_no_bias/mnist_cnn_no_bias_tf_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/cnn_no_bias/mnist_cnn_no_bias_tf_keras_2_inputs_and_outputs.h5"; + importEndModelTest(modelPath, inputsOutputPath, true, true, true, false); + } + + @Test + public void importSparseXent() throws Exception { + String modelPath = "modelimport/keras/examples/simple_sparse_xent/simple_sparse_xent_mlp_keras_2_model.h5"; + String inputsOutputPath = "modelimport/keras/examples/simple_sparse_xent/simple_sparse_xent_mlp_keras_2_inputs_and_outputs.h5"; + MultiLayerNetwork net = importEndModelTest(modelPath, inputsOutputPath, true, true, true, true); + Layer outLayer = net.getOutputLayer(); + assertTrue(outLayer instanceof org.deeplearning4j.nn.layers.LossLayer); + LossLayer llConf = (LossLayer) outLayer.getConfig(); + assertEquals(new LossSparseMCXENT(), llConf.getLossFn()); + } + + /** + * GAN import tests + */ + @Test + public void importDcganMnistDiscriminator() throws Exception { + importSequentialModelH5Test("modelimport/keras/examples/mnist_dcgan/dcgan_discriminator_epoch_50.h5"); + } + + @Test + ////@Ignore("Neither keras or tfkeras can load this.") + public void importDcganMnistGenerator() throws Exception { + importSequentialModelH5Test("modelimport/keras/examples/mnist_dcgan/dcgan_generator_epoch_50.h5"); + } + + /** + * Auxillary classifier GAN import test + */ + @Test + public void importAcganDiscriminator() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/acgan/acgan_discriminator_1_epochs.h5"); + INDArray input = Nd4j.create(10, 28, 28, 1); //NHWC + INDArray[] output = model.output(input); + } + + @Test //AB 2020/04/22 Ignored until Keras model import updated to use NHWC support + public void importAcganGenerator() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/acgan/acgan_generator_1_epochs.h5"); + //System.out.println(model.summary()) ; + INDArray latent = Nd4j.create(10, 100); + INDArray label = Nd4j.create(10, 1); + INDArray[] output = model.output(latent, label); + } + + @Test + public void importAcganCombined() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/acgan/acgan_combined_1_epochs.h5"); + // TODO: imports, but incorrectly. Has only one input, should have two. + } + + /** + * Deep convolutional GAN import test + */ + @Test + public void importDcganDiscriminator() throws Exception { + importSequentialModelH5Test("modelimport/keras/examples/gans/dcgan_discriminator.h5"); + } + + @Test + public void importDcganGenerator() throws Exception { + importSequentialModelH5Test("modelimport/keras/examples/gans/dcgan_generator.h5"); + } + + /** + * Wasserstein GAN import test + */ + @Test + public void importWganDiscriminator() throws Exception { + for (int i = 0; i < 100; i++) { + // run a few times to make sure HDF5 doesn't crash + importSequentialModelH5Test("modelimport/keras/examples/gans/wgan_discriminator.h5"); + } + } + + @Test + public void importWganGenerator() throws Exception { + importSequentialModelH5Test("modelimport/keras/examples/gans/wgan_generator.h5"); + } + + @Test + public void importCnn1d() throws Exception { + importSequentialModelH5Test("modelimport/keras/examples/cnn1d/cnn1d_flatten_tf_keras2.h5"); + } + + /** + * DGA classifier test + */ + @Test + public void importDgaClassifier() throws Exception { + importSequentialModelH5Test("modelimport/keras/examples/dga_classifier/keras2_dga_classifier_tf_model.h5"); + } + + /** + * Reshape flat input into 3D to fit into an LSTM model + */ + @Test + public void importFlatIntoLSTM() throws Exception { + importFunctionalModelH5Test("modelimport/keras/examples/reshape_to_rnn/reshape_model.h5"); + } + + + /** + * Functional LSTM test + */ + @Test + public void importFunctionalLstmTfKeras2() throws Exception { + String modelPath = "modelimport/keras/examples/functional_lstm/lstm_functional_tf_keras_2.h5"; + + // No training enabled + ComputationGraph graphNoTrain = importFunctionalModelH5Test(modelPath, null, false); + System.out.println(graphNoTrain.summary()); + + // Training enabled + ComputationGraph graph = importFunctionalModelH5Test(modelPath, null, true); + System.out.println(graph.summary()); + + // Make predictions + int miniBatch = 32; + INDArray input = Nd4j.ones(miniBatch, 10, 4); //NWC format - with nIn=4, seqLength = 10 + INDArray[] out = graph.output(input); + + // Fit model + graph.fit(new INDArray[]{input}, out); + } + + /** + * U-Net + */ + @Test + public void importUnetTfKeras2() throws Exception { + importFunctionalModelH5Test( + "modelimport/keras/examples/unet/unet_keras_2_tf.h5", null, true); + } + + /** + * ResNet50 + */ + @Test + public void importResnet50() throws Exception { + importFunctionalModelH5Test("modelimport/keras/examples/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5"); + } + + /** + * DenseNet + */ + @Test + public void importDenseNet() throws Exception { + importFunctionalModelH5Test("modelimport/keras/examples/densenet/densenet121_tf_keras_2.h5"); + } + + /** + * SqueezeNet + */ + @Test + public void importSqueezeNet() throws Exception { + importFunctionalModelH5Test("modelimport/keras/examples/squeezenet/squeezenet.h5"); + } + + + /** + * MobileNet + */ + @Test + public void importMobileNet() throws Exception { + ComputationGraph graph = importFunctionalModelH5Test("modelimport/keras/examples/mobilenet/alternative.hdf5"); + INDArray input = Nd4j.ones(10, 299, 299, 3); + graph.output(input); + } + + /** + * InceptionV3 Keras 2 no top + */ + @Test + public void importInceptionKeras2() throws Exception { + int[] inputShape = new int[]{299, 299, 3}; + ComputationGraph graph = importFunctionalModelH5Test( + "modelimport/keras/examples/inception/inception_tf_keras_2.h5", inputShape, false); + INDArray input = Nd4j.ones(10, 299, 299, 3); //TF = channels last = NHWC + graph.output(input); + System.out.println(graph.summary()); + } + + /** + * InceptionV3 + */ + @Test + @Tag("long-running") + //note this is actually keras 1 and its input dimension ordering is channels first + // Takes unreasonably long, but works + public void importInception() throws Exception { + ComputationGraph graph = importFunctionalModelH5Test( + "modelimport/keras/examples/inception/inception_v3_complete.h5"); + INDArray input = Nd4j.ones(10, 3,299, 299); //TH = channels first = NCHW + graph.output(input); + System.out.println(graph.summary()); + } + + /** + * Inception V4 + */ + @Test + @Disabled + // Model and weights have about 170mb, too large for test resources and also too excessive to enable as unit test + public void importInceptionV4() throws Exception { + String modelUrl = DL4JResources.getURLString( + "models/inceptionv4_keras_imagenet_weightsandconfig.h5"); + File kerasFile = new File(testDir.getAbsolutePath() + File.pathSeparator + "inceptionv4_keras_imagenet_weightsandconfig.h5"); + + if (!kerasFile.exists()) { + FileUtils.copyURLToFile(new URL(modelUrl), kerasFile); + kerasFile.deleteOnExit(); + } + + int[] inputShape = new int[]{299, 299, 3}; + ComputationGraph graph = importFunctionalModelH5Test( + kerasFile.getAbsolutePath(), inputShape, false); + + // System.out.println(graph.summary()); + + } + + /** + * Xception + */ + @Test + public void importXception() throws Exception { + int[] inputShape = new int[]{299, 299, 3}; + ComputationGraph graph = importFunctionalModelH5Test( + "modelimport/keras/examples/xception/xception_tf_keras_2.h5", inputShape, false); + } + + /** + * Seq2seq model + */ + @Test + // does not work yet, needs DL4J enhancements + public void importSeq2Seq() throws Exception { + importFunctionalModelH5Test("modelimport/keras/examples/seq2seq/full_model_seq2seq_5549.h5"); + + } + + + /** + * Import all AlphaGo Zero model variants, i.e. + * - Dual residual architecture + * - Dual convolutional architecture + * - Separate (policy and value) residual architecture + * - Separate (policy and value) convolutional architecture + */ + @Test //AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last + @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") + public void importSepConvPolicy() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/agz/sep_conv_policy.h5"); + INDArray input = Nd4j.create(32, 19, 19, 10); + model.output(input); + } + + @Test //AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last + @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") + public void importSepResPolicy() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/agz/sep_res_policy.h5"); + INDArray input = Nd4j.create(32, 19, 19, 10); + model.output(input); + } + + + @Test //AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last + @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") + public void importSepConvValue() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/agz/sep_conv_value.h5"); + INDArray input = Nd4j.create(32, 19, 19, 10); + model.output(input); + } + + @Test() //AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last + @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") + public void importSepResValue() throws Exception { + String filePath = "C:\\Users\\agibs\\Documents\\GitHub\\keras1-import-test\\sep_res_value.h5"; + KerasModelBuilder builder = new KerasModel().modelBuilder().modelHdf5Filename(filePath) + .enforceTrainingConfig(false); + + KerasModel model = builder.buildModel(); + ComputationGraph compGraph = model.getComputationGraph(); + //ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/agz/sep_res_value.h5"); + INDArray input = Nd4j.create(32, 19, 19, 10); + compGraph.output(input); + } + + @Test //AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last + @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") + public void importDualRes() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/agz/dual_res.h5"); + INDArray input = Nd4j.create(32, 19, 19, 10); + model.output(input); + } + + @Test() //AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last + @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") + public void importDualConv() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/agz/dual_conv.h5"); + INDArray input = Nd4j.create(32, 19, 19, 10); + model.output(input); + } + + /** + * MTCNN + */ + @Test + public void importMTCNN() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/48net_complete.h5"); + } + + @Test() + @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") + public void testNCHWNWHCChangeImportModel() throws Exception { + ComputationGraph computationGraph = importFunctionalModelH5Test("modelimport/keras/weights/simpleconv2d_model.hdf5"); + computationGraph.output(Nd4j.zeros(1,1,28,28)); + + } + + + @Test + // TODO: fails, since we can't use OldSoftMax on >2D data (here: convolution layer) + // TODO: also related to #6339, fix this together + public void importMTCNN2D() throws Exception { + ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/12net.h5", + new int[] {24, 24, 3}, false); + INDArray input = Nd4j.create(10, 24, 24,3); + model.output(input); +// System.out.println(model.summary()); + } + + /** + * Masking layers (simple Masking into LSTM) + */ + @Test + public void testMaskingZeroValue() throws Exception { + MultiLayerNetwork model = importSequentialModelH5Test( + "modelimport/keras/examples/masking/masking_zero_lstm.h5"); + model.summary(); + } + + @Test + public void testMaskingTwoValue() throws Exception { + MultiLayerNetwork model = importSequentialModelH5Test( + "modelimport/keras/examples/masking/masking_two_lstm.h5"); + model.summary(); + } + + @Test + public void testCausalConv1D() throws Exception { + String[] names = new String[]{ + "causal_conv1d_k2_s1_d1_cl_model.h5", + "causal_conv1d_k2_s1_d2_cl_model.h5", + "causal_conv1d_k2_s2_d1_cl_model.h5", + "causal_conv1d_k2_s3_d1_cl_model.h5", + "causal_conv1d_k3_s1_d1_cl_model.h5", + "causal_conv1d_k3_s1_d2_cl_model.h5", + "causal_conv1d_k3_s2_d1_cl_model.h5", + "causal_conv1d_k3_s3_d1_cl_model.h5", + "causal_conv1d_k4_s1_d1_cl_model.h5", + "causal_conv1d_k4_s1_d2_cl_model.h5", + "causal_conv1d_k4_s2_d1_cl_model.h5", + "causal_conv1d_k4_s3_d1_cl_model.h5" + }; + + for(String name : names) { + System.out.println("Starting test: " + name); + String modelPath = "modelimport/keras/examples/causal_conv1d/" + name; + String inputsOutputPath = "modelimport/keras/examples/causal_conv1d/" + (name.substring(0,name.length() - "model.h5".length()) + "inputs_and_outputs.h5"); + //TODO: + /** + * Difference in weights. Same elements, but loaded differently. Likely acceptable difference. Need to confirm though. + */ + MultiLayerNetwork net = importEndModelTest(modelPath, inputsOutputPath, true, true, + true, true, false, null, null); + Layer l = net.getLayer(0); + Convolution1DLayer c1d = (Convolution1DLayer) l.getConfig(); + assertEquals(ConvolutionMode.Causal, c1d.getConvolutionMode()); + } + } + + @Test + public void testConv1D() throws Exception { + String[] names = new String[]{ + "conv1d_k2_s1_d1_cf_same_model.h5", + "conv1d_k2_s1_d1_cf_valid_model.h5", + "conv1d_k2_s1_d1_cl_same_model.h5", + "conv1d_k2_s1_d1_cl_valid_model.h5", + "conv1d_k2_s1_d2_cf_same_model.h5", + "conv1d_k2_s1_d2_cf_valid_model.h5", + "conv1d_k2_s1_d2_cl_same_model.h5", + "conv1d_k2_s1_d2_cl_valid_model.h5", + "conv1d_k2_s2_d1_cf_same_model.h5", + "conv1d_k2_s2_d1_cf_valid_model.h5", + "conv1d_k2_s2_d1_cl_same_model.h5", + "conv1d_k2_s2_d1_cl_valid_model.h5", + "conv1d_k2_s3_d1_cf_same_model.h5", + "conv1d_k2_s3_d1_cf_valid_model.h5", + "conv1d_k2_s3_d1_cl_same_model.h5", + "conv1d_k2_s3_d1_cl_valid_model.h5", + "conv1d_k3_s1_d1_cf_same_model.h5", + "conv1d_k3_s1_d1_cf_valid_model.h5", + "conv1d_k3_s1_d1_cl_same_model.h5", + "conv1d_k3_s1_d1_cl_valid_model.h5", + "conv1d_k3_s1_d2_cf_same_model.h5", + "conv1d_k3_s1_d2_cf_valid_model.h5", + "conv1d_k3_s1_d2_cl_same_model.h5", + "conv1d_k3_s1_d2_cl_valid_model.h5", + "conv1d_k3_s2_d1_cf_same_model.h5", + "conv1d_k3_s2_d1_cf_valid_model.h5", + "conv1d_k3_s2_d1_cl_same_model.h5", + "conv1d_k3_s2_d1_cl_valid_model.h5", + "conv1d_k3_s3_d1_cf_same_model.h5", + "conv1d_k3_s3_d1_cf_valid_model.h5", + "conv1d_k3_s3_d1_cl_same_model.h5", + "conv1d_k3_s3_d1_cl_valid_model.h5", + "conv1d_k4_s1_d1_cf_same_model.h5", + "conv1d_k4_s1_d1_cf_valid_model.h5", + "conv1d_k4_s1_d1_cl_same_model.h5", + "conv1d_k4_s1_d1_cl_valid_model.h5", + "conv1d_k4_s1_d2_cf_same_model.h5", + "conv1d_k4_s1_d2_cf_valid_model.h5", + "conv1d_k4_s1_d2_cl_same_model.h5", + "conv1d_k4_s1_d2_cl_valid_model.h5", + "conv1d_k4_s2_d1_cf_same_model.h5", + "conv1d_k4_s2_d1_cf_valid_model.h5", + "conv1d_k4_s2_d1_cl_same_model.h5", + "conv1d_k4_s2_d1_cl_valid_model.h5", + "conv1d_k4_s3_d1_cf_same_model.h5", + "conv1d_k4_s3_d1_cf_valid_model.h5", + "conv1d_k4_s3_d1_cl_same_model.h5", + "conv1d_k4_s3_d1_cl_valid_model.h5", + }; + + for(String name : names) { + System.out.println("Starting test: " + name); + String modelPath = "modelimport/keras/examples/conv1d/" + name; + String inputsOutputPath = "modelimport/keras/examples/conv1d/" + (name.substring(0,name.length()-"model.h5".length()) + "inputs_and_outputs.h5"); + + importEndModelTest(modelPath, inputsOutputPath, true, true, + true, true, false, null, null); //f, f2); + } + } + + + @Test + public void testActivationLayers() throws Exception { + String[] names = new String[]{ + "ELU_0_model.h5", + "LeakyReLU_0_model.h5", + "ReLU_0_model.h5", + "ReLU_1_model.h5", + "ReLU_2_model.h5", + "ReLU_3_model.h5", + "Softmax_0_model.h5", + "ThresholdReLU_0_model.h5", + }; + + for(String name : names ){ + System.out.println("Starting test: " + name); + String modelPath = "modelimport/keras/examples/activations/" + name; + String inputsOutputPath = "modelimport/keras/examples/activations/" + (name.substring(0,name.length()-"model.h5".length()) + "inputs_and_outputs.h5"); + + importEndModelTest(modelPath, inputsOutputPath, true, true, + true, true, false, null, null); + } + } + + private ComputationGraph importFunctionalModelH5Test(String modelPath) throws Exception { + return importFunctionalModelH5Test(modelPath, null, false); + } + + + private ComputationGraph importFunctionalModelH5Test(String modelPath, int[] inputShape, boolean train) + throws Exception { + File modelFile; + try(InputStream is = Resources.asStream(modelPath)) { + modelFile = createTempFile(TEMP_MODEL_FILENAME, H5_EXTENSION); + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + KerasModelBuilder builder = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(train); + if (inputShape != null) { + builder.inputShape(inputShape); + } + KerasModel model = builder.buildModel(); + return model.getComputationGraph(); + } + + private MultiLayerNetwork importSequentialModelH5Test(String modelPath) throws Exception { + return importSequentialModelH5Test(modelPath, null); + } + + + private MultiLayerNetwork importSequentialModelH5Test(String modelPath, int[] inputShape) throws Exception { + try(InputStream is = Resources.asStream(modelPath)) { + File modelFile = createTempFile(TEMP_MODEL_FILENAME, H5_EXTENSION); + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + KerasModelBuilder builder = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(false); + if (inputShape != null) { + builder.inputShape(inputShape); + } + KerasSequentialModel model = builder.buildSequential(); + return model.getMultiLayerNetwork(); + } + } + + public MultiLayerNetwork importEndModelTest(String modelPath, String inputsOutputsPath, boolean tfOrdering, boolean checkPredictions, + boolean checkGradients, boolean enforceTrainingConfig) throws Exception { + return importEndModelTest(modelPath, inputsOutputsPath, tfOrdering, checkPredictions, checkGradients, true, enforceTrainingConfig, null, null); + } + + public MultiLayerNetwork importEndModelTest(String modelPath, String inputsOutputsPath, boolean tfOrdering, boolean checkPredictions, + boolean checkGradients, boolean enforceTrainingConfig, boolean checkAuc, Function inputPreProc, + BiFunction expectedPreProc) throws Exception { + MultiLayerNetwork model; + try(InputStream is = Resources.asStream(modelPath)) { + File modelFile = createTempFile(TEMP_MODEL_FILENAME, H5_EXTENSION); + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + KerasSequentialModel kerasModel = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(enforceTrainingConfig).buildSequential(); + + model = kerasModel.getMultiLayerNetwork(); + } + + File outputsFile = createTempFile(TEMP_OUTPUTS_FILENAME, H5_EXTENSION); + try(InputStream is = Resources.asStream(inputsOutputsPath)) { + Files.copy(is, outputsFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + try (Hdf5Archive outputsArchive = new Hdf5Archive(outputsFile.getAbsolutePath())) { + + if (checkPredictions) { + INDArray input = getInputs(outputsArchive, tfOrdering)[0]; + if(inputPreProc != null) + input = inputPreProc.apply(input); + + Map activationsKeras = getActivations(outputsArchive, tfOrdering); + for (int i = 0; i < model.getLayers().length; i++) { + String layerName = model.getLayerNames().get(i); + if (activationsKeras.containsKey(layerName)) { + INDArray activationsDl4j = model.feedForwardToLayer(i, input, false).get(i + 1); + long[] shape = activationsDl4j.shape(); + INDArray exp = activationsKeras.get(layerName); + Nd4j.getExecutioner().enableDebugMode(true); + Nd4j.getExecutioner().enableVerboseMode(true); + if(expectedPreProc != null) + exp = expectedPreProc.apply(layerName, exp); + compareINDArrays(layerName, exp, activationsDl4j, EPS); + } + } + + INDArray predictionsKeras = getPredictions(outputsArchive, tfOrdering)[0]; + INDArray predictionsDl4j = model.output(input, false); + if(expectedPreProc != null) + predictionsKeras = expectedPreProc.apply("output", predictionsKeras); + compareINDArrays("predictions", predictionsKeras, predictionsDl4j, EPS); + INDArray outputs = getOutputs(outputsArchive, true)[0]; + + if(outputs.rank() == 1) { + outputs = outputs.reshape(outputs.length(), 1); + } + val nOut = (int) outputs.size(-1); + + if(checkAuc) + compareMulticlassAUC("predictions", outputs, predictionsKeras, predictionsDl4j, nOut, EPS); + } + + if (checkGradients && ! SKIP_GRAD_CHECKS) { + Random r = new Random(12345); + INDArray input = getInputs(outputsArchive, tfOrdering)[0]; + INDArray predictionsDl4j = model.output(input, false); + + //Infer one-hot labels... this probably won't work for all + INDArray testLabels = Nd4j.create(predictionsDl4j.shape()); + if (testLabels.rank() == 2) { + for (int i = 0; i < testLabels.size(0); i++) { + testLabels.putScalar(i, r.nextInt((int) testLabels.size(1)), 1.0); + } + } else if (testLabels.rank() == 3) { + for (int i = 0; i < testLabels.size(0); i++) { + for (int j = 0; j < testLabels.size(1); j++) { + testLabels.putScalar(i, j, r.nextInt((int) testLabels.size(1)), 1.0); + } + } + } else { + throw new RuntimeException("Cannot gradient check 4d output array"); + } + checkGradients(model, input, testLabels); + } + } + + return model; + } + + private static INDArray[] getInputs(Hdf5Archive archive, boolean tensorFlowImageDimOrdering) throws Exception { + List inputNames = (List) KerasModelUtils + .parseJsonString(archive.readAttributeAsJson(GROUP_ATTR_INPUTS)).get(GROUP_ATTR_INPUTS); + INDArray[] inputs = new INDArray[inputNames.size()]; + for (int i = 0; i < inputNames.size(); i++) { + inputs[i] = archive.readDataSet(inputNames.get(i), GROUP_ATTR_INPUTS); + } + return inputs; + } + + private static Map getActivations(Hdf5Archive archive, boolean tensorFlowImageDimOrdering) + throws Exception { + Map activations = new HashMap<>(); + for (String layerName : archive.getDataSets(GROUP_ACTIVATIONS)) { + INDArray activation = archive.readDataSet(layerName, GROUP_ACTIVATIONS); + activations.put(layerName, activation); + } + return activations; + } + + private static INDArray[] getOutputs(Hdf5Archive archive, boolean tensorFlowImageDimOrdering) throws + Exception { + List outputNames = (List) KerasModelUtils + .parseJsonString(archive.readAttributeAsJson(GROUP_ATTR_OUTPUTS)).get(GROUP_ATTR_OUTPUTS); + INDArray[] outputs = new INDArray[outputNames.size()]; + for (int i = 0; i < outputNames.size(); i++) { + outputs[i] = archive.readDataSet(outputNames.get(i), GROUP_ATTR_OUTPUTS); + } + return outputs; + } + + private static INDArray[] getPredictions(Hdf5Archive archive, boolean tensorFlowImageDimOrdering) + throws Exception { + List outputNames = (List) KerasModelUtils + .parseJsonString(archive.readAttributeAsJson(GROUP_ATTR_OUTPUTS)).get(GROUP_ATTR_OUTPUTS); + INDArray[] predictions = new INDArray[outputNames.size()]; + for (int i = 0; i < outputNames.size(); i++) { + predictions[i] = archive.readDataSet(outputNames.get(i), GROUP_PREDICTIONS); + } + return predictions; + } + + private static void compareINDArrays(String label, INDArray expected, INDArray actual, double eps) { + if(!expected.equalShapes(actual)){ + throw new IllegalStateException("Shapes do not match for \"" + label + "\": got " + Arrays.toString(expected.shape()) + " vs " + Arrays.toString(actual.shape())); + } + INDArray diff = expected.sub(actual.castTo(expected.dataType())); + double min = diff.minNumber().doubleValue(); + double max = diff.maxNumber().doubleValue(); + log.info(label + ": " + expected.equalsWithEps(actual, eps) + ", " + min + ", " + max); + double threshold = 1e-7; + double aAbsMax = Math.max(Math.abs(expected.minNumber().doubleValue()), Math.abs(expected.maxNumber().doubleValue())); + double bAbsMax = Math.max(Math.abs(actual.minNumber().doubleValue()), Math.abs(actual.maxNumber().doubleValue())); + + // skip too small absolute inputs + if (Math.abs(aAbsMax) > threshold && Math.abs(bAbsMax) > threshold) { + boolean eq = expected.equalsWithEps(actual.castTo(expected.dataType()), eps); + if(!eq){ + System.out.println("Expected: " + Arrays.toString(expected.shape()) + ", actual: " + Arrays.toString(actual.shape())); + System.out.println("Expected:\n" + expected); + System.out.println("Actual: \n" + actual); + } + assertTrue(eq, "Output differs: " + label); + } + } + + private static void compareMulticlassAUC(String label, INDArray target, INDArray a, INDArray b, int nbClasses, + double eps) { + ROCMultiClass evalA = new ROCMultiClass(100); + evalA.eval(target, a); + double avgAucA = evalA.calculateAverageAUC(); + ROCMultiClass evalB = new ROCMultiClass(100); + evalB.eval(target, b); + double avgAucB = evalB.calculateAverageAUC(); + assertEquals(avgAucA, avgAucB, EPS); + + double[] aucA = new double[nbClasses]; + double[] aucB = new double[nbClasses]; + if (nbClasses > 1) { + for (int i = 0; i < nbClasses; i++) { + aucA[i] = evalA.calculateAUC(i); + aucB[i] = evalB.calculateAUC(i); + } + assertArrayEquals(aucA, aucB, EPS); + } + } + + public static void checkGradients(MultiLayerNetwork net, INDArray input, INDArray labels) { + double eps = 1e-6; + double max_rel_error = 1e-3; + double min_abs_error = 1e-8; + + MultiLayerNetwork netToTest; + if (net.getOutputLayer() instanceof IOutputLayer) { + netToTest = net; + } else { + org.deeplearning4j.nn.conf.layers.Layer l; + if (labels.rank() == 2) { + l = new LossLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE) + .activation(Activation.IDENTITY) + .build(); + } else { + //Rank 3 + l = new RnnOutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE) + .activation(Activation.IDENTITY) + .nIn(labels.size(1)) + .nOut(labels.size(1)) + .build(); + } + netToTest = new TransferLearning.Builder(net) + .fineTuneConfiguration(new FineTuneConfiguration.Builder() + .updater(new NoOp()) + .dropOut(0.0) + .build()) + .addLayer(l) + .build(); + } + + log.info("Num params: " + net.numParams()); + + for (Layer l : netToTest.getLayers()) { + // Remove any dropout manually - until this is fixed: + // https://github.com/eclipse/deeplearning4j/issues/4368 + l.conf().getLayer().setIDropout(null); + + //Also swap out activation functions... this is a bit of a hack, but should make the net gradient checkable... + if (l.conf().getLayer() instanceof FeedForwardLayer) { + FeedForwardLayer ffl = (FeedForwardLayer) l.conf().getLayer(); + IActivation activation = ffl.getActivationFn(); + if (activation instanceof ActivationReLU || activation instanceof ActivationLReLU) { + ffl.setActivationFn(new ActivationSoftPlus()); + } else if (activation instanceof ActivationHardTanH) { + ffl.setActivationFn(new ActivationTanH()); + } + } + } + + Nd4j.setDataType(DataType.DOUBLE); + boolean passed = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(netToTest).input(input) + .labels(labels).subset(true).maxPerParam(9)); + assertTrue(passed, "Gradient check failed"); + } + + private File createTempFile(String prefix, String suffix) throws IOException { + return new File(testDir.getAbsolutePath() + File.pathSeparator +prefix + "-" + System.nanoTime() + suffix); + } +} diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000PredictTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000PredictTest.java new file mode 100644 index 000000000..782923365 --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000PredictTest.java @@ -0,0 +1,82 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.e2e; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.KerasLayer; +import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; +import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasSpaceToDepth; +import org.deeplearning4j.nn.transferlearning.TransferLearning; +import org.deeplearning4j.util.ModelSerializer; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.preprocessor.ImagePreProcessingScaler; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.File; + +@Slf4j +public class KerasYolo9000PredictTest extends BaseDL4JTest { + + private static final String DL4J_MODEL_FILE_NAME = "."; + private static ImagePreProcessingScaler IMAGE_PREPROCESSING_SCALER = new ImagePreProcessingScaler(0, 1); + + @Test + ////@Ignore("Need to manually download file for ylo.") + public void testYoloPredictionImport() throws Exception { + int HEIGHT = 416; + int WIDTH = 416; + INDArray indArray = Nd4j.create(HEIGHT, WIDTH, 3); + IMAGE_PREPROCESSING_SCALER.transform(indArray); + + KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); + + String h5_FILENAME = "modelimport/keras/examples/yolo/yolo-voc.h5"; + ComputationGraph graph = KerasModelImport.importKerasModelAndWeights(h5_FILENAME, false); + + double[][] priorBoxes = {{1.3221, 1.73145}, {3.19275, 4.00944}, {5.05587, 8.09892}, {9.47112, 4.84053}, {11.2364, 10.0071}}; + INDArray priors = Nd4j.create(priorBoxes); + + ComputationGraph model = new TransferLearning.GraphBuilder(graph) + .addLayer("outputs", + new org.deeplearning4j.nn.conf.layers.objdetect.Yolo2OutputLayer.Builder() + .boundingBoxPriors(priors) + .build(), + "conv2d_23") + .setOutputs("outputs") + .build(); + + ModelSerializer.writeModel(model, DL4J_MODEL_FILE_NAME, false); + + ComputationGraph computationGraph = ModelSerializer.restoreComputationGraph(new File(DL4J_MODEL_FILE_NAME)); + + System.out.println(computationGraph.summary(InputType.convolutional(416, 416, 3))); + + INDArray results = computationGraph.outputSingle(indArray); + + + } + +} + diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000Test.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000Test.java new file mode 100644 index 000000000..2e5666241 --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000Test.java @@ -0,0 +1,67 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.e2e; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.KerasLayer; +import org.deeplearning4j.nn.modelimport.keras.KerasModel; +import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasSpaceToDepth; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.resources.Resources; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +@Slf4j +public class KerasYolo9000Test extends BaseDL4JTest { + + private static final String TEMP_MODEL_FILENAME = "tempModel"; + private static final String H5_EXTENSION = ".h5"; + + @TempDir + public File testDir; + + ////@Ignore + @Test + // TODO: yolo and yolo-voc output are too large for github, find smaller equivalents + public void testCustomLayerYoloImport() throws Exception { + KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); + + String modelPath = "modelimport/keras/examples/yolo/yolo.h5"; + + try(InputStream is = Resources.asStream(modelPath)) { + File modelFile = new File(testDir.getAbsolutePath()+File.pathSeparator+TEMP_MODEL_FILENAME + System.currentTimeMillis() + H5_EXTENSION); + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + ComputationGraph model = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(false).buildModel().getComputationGraph(); + + System.out.println(model.summary()); + } + + + } +} diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasLeakyReLUTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasLeakyReLUTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasLeakyReLUTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasLeakyReLUTest.java index 21b68d2bf..91e890cba 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasLeakyReLUTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasLeakyReLUTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.advanced.activation; import org.deeplearning4j.nn.conf.layers.ActivationLayer; @@ -25,32 +26,23 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.advanced.activations.KerasLeakyReLU; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Leaky Re LU Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasLeakyReLUTest extends BaseDL4JTest { +public class KerasLeakyReLUTest extends BaseDL4JTest { private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Leaky Re LU Layer") - void testLeakyReLULayer() throws Exception { + public void testLeakyReLULayer() throws Exception { Integer keras1 = 1; buildLeakyReLULayer(conf1, keras1); Integer keras2 = 2; @@ -59,6 +51,7 @@ class KerasLeakyReLUTest extends BaseDL4JTest { private void buildLeakyReLULayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { double alpha = 0.3; + Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_LEAKY_RELU()); Map config = new HashMap<>(); @@ -68,8 +61,9 @@ class KerasLeakyReLUTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NAME(), layerName); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + ActivationLayer layer = new KerasLeakyReLU(layerConfig).getActivationLayer(); - assertEquals(layer.getActivationFn().toString(), "leakyrelu(a=0.3)"); + assertEquals("leakyrelu(a=0.3)", layer.getActivationFn().toString()); assertEquals(layerName, layer.getLayerName()); } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasPReLUTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasPReLUTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasPReLUTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasPReLUTest.java index daed85eca..7405a6007 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasPReLUTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasPReLUTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.advanced.activation; import org.deeplearning4j.nn.conf.inputs.InputType; @@ -28,37 +29,27 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.advanced.activations.KerasPReLU; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras P Re LU Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasPReLUTest extends BaseDL4JTest { +public class KerasPReLUTest extends BaseDL4JTest { private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); @Test - @DisplayName("Test P Re LU Layer") - void testPReLULayer() throws Exception { + public void testPReLULayer() throws Exception { Integer keras1 = 1; buildPReLULayer(conf1, keras1); Integer keras2 = 2; @@ -66,6 +57,7 @@ class KerasPReLUTest extends BaseDL4JTest { } private void buildPReLULayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_LEAKY_RELU()); Map config = new HashMap<>(); @@ -80,11 +72,15 @@ class KerasPReLUTest extends BaseDL4JTest { init.put("class_name", conf.getINIT_GLOROT_NORMAL()); config.put("alpha_initializer", init); } + KerasPReLU kerasPReLU = new KerasPReLU(layerConfig); - kerasPReLU.getOutputType(InputType.convolutional(5, 4, 3)); + + kerasPReLU.getOutputType(InputType.convolutional(5,4,3)); + PReLULayer layer = kerasPReLU.getPReLULayer(); - assertArrayEquals(layer.getInputShape(), new long[] { 3, 5, 4 }); + assertArrayEquals(layer.getInputShape(), new long[] {3, 5, 4}); assertEquals(INIT_DL4J, layer.getWeightInitFn()); + assertEquals(layerName, layer.getLayerName()); } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasThresholdedReLUTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasThresholdedReLUTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasThresholdedReLUTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasThresholdedReLUTest.java index 5a1e7e324..822a140a6 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasThresholdedReLUTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/advanced/activation/KerasThresholdedReLUTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.advanced.activation; import org.deeplearning4j.nn.conf.layers.ActivationLayer; @@ -25,32 +26,23 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.advanced.activations.KerasThresholdedReLU; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Thresholded Re LU Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasThresholdedReLUTest extends BaseDL4JTest { +public class KerasThresholdedReLUTest extends BaseDL4JTest { private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Thresholded Re LU Layer") - void testThresholdedReLULayer() throws Exception { + public void testThresholdedReLULayer() throws Exception { Integer keras1 = 1; buildThresholdedReLULayer(conf1, keras1); Integer keras2 = 2; @@ -58,7 +50,9 @@ class KerasThresholdedReLUTest extends BaseDL4JTest { } private void buildThresholdedReLULayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + double theta = 0.5; + Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_THRESHOLDED_RELU()); Map config = new HashMap<>(); @@ -68,8 +62,9 @@ class KerasThresholdedReLUTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NAME(), layerName); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + ActivationLayer layer = new KerasThresholdedReLU(layerConfig).getActivationLayer(); - assertEquals(layer.getActivationFn().toString(), "thresholdedrelu(theta=0.5)"); + assertEquals("thresholdedrelu(theta=0.5)", layer.getActivationFn().toString()); assertEquals(layerName, layer.getLayerName()); } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution1DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution1DTest.java similarity index 86% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution1DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution1DTest.java index 499581219..828d1c4c2 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution1DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution1DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -29,66 +30,44 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasAtrousConvolution1D; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Atrous Convolution 1 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasAtrousConvolution1DTest extends BaseDL4JTest { +public class KerasAtrousConvolution1DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "atrous_conv_1d"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 1, 2 }; - - private final int[] DILATION = new int[] { 2 }; - - private final int[] STRIDE = new int[] { 3, 4 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2}; + private final int[] DILATION = new int[]{2}; + private final int[] STRIDE = new int[]{3, 4}; private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras1 = 1; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); @Test - @DisplayName("Test Atrous Convolution 1 D Layer") - void testAtrousConvolution1DLayer() throws Exception { + public void testAtrousConvolution1DLayer() throws Exception { buildAtrousConvolution1DLayer(conf1, keras1); } - private void buildAtrousConvolution1DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + private void buildAtrousConvolution1DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CONVOLUTION_1D()); Map config = new HashMap<>(); @@ -117,6 +96,7 @@ class KerasAtrousConvolution1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NB_FILTER(), N_OUT); config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); + Convolution1DLayer layer = new KerasAtrousConvolution1D(layerConfig).getAtrousConvolution1D(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); @@ -135,3 +115,4 @@ class KerasAtrousConvolution1DTest extends BaseDL4JTest { assertEquals(DILATION, layer.getDilation()); } } + diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution2DTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution2DTest.java index dd8adba1d..a29be581c 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasAtrousConvolution2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -29,68 +30,47 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasAtrousConvolution2D; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Atrous Convolution 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasAtrousConvolution2DTest extends BaseDL4JTest { +public class KerasAtrousConvolution2DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "atrous_conv_2d"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 1, 2 }; - - private final int[] DILATION = new int[] { 2, 2 }; - - private final int[] STRIDE = new int[] { 3, 4 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2}; + private final int[] DILATION = new int[]{2, 2}; + private final int[] STRIDE = new int[]{3, 4}; private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); @Test - @DisplayName("Test Atrous Convolution 2 D Layer") - void testAtrousConvolution2DLayer() throws Exception { + public void testAtrousConvolution2DLayer() throws Exception { Integer keras1 = 1; buildAtrousConvolution2DLayer(conf1, keras1); } - private void buildAtrousConvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + private void buildAtrousConvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CONVOLUTION_2D()); Map config = new HashMap<>(); @@ -112,20 +92,14 @@ class KerasAtrousConvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NB_ROW(), KERNEL_SIZE[0]); config.put(conf.getLAYER_FIELD_NB_COL(), KERNEL_SIZE[1]); } else { - ArrayList kernel = new ArrayList() { - - { - for (int i : KERNEL_SIZE) add(i); - } - }; + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_KERNEL_SIZE(), kernel); } - ArrayList dilation = new ArrayList() { - - { - for (int i : DILATION) add(i); - } - }; + ArrayList dilation = new ArrayList() {{ + for (int i : DILATION) add(i); + }}; config.put(conf.getLAYER_FIELD_DILATION_RATE(), dilation); List subsampleList = new ArrayList<>(); subsampleList.add(STRIDE[0]); @@ -135,6 +109,8 @@ class KerasAtrousConvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + + ConvolutionLayer layer = new KerasAtrousConvolution2D(layerConfig).getAtrousConvolution2D(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution1DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution1DTest.java similarity index 81% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution1DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution1DTest.java index 0bc384f0e..22a51b1a7 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution1DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution1DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -30,73 +31,49 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasConvolution1D; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Convolution 1 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasConvolution1DTest extends BaseDL4JTest { +public class KerasConvolution1DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "test_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 2 }; - - private final int[] DILATION = new int[] { 2 }; - - private final int[] STRIDE = new int[] { 4 }; - + private final int[] KERNEL_SIZE = new int[]{2}; + private final int[] DILATION = new int[]{2}; + private final int[] STRIDE = new int[]{4}; private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Convolution 1 D Layer") - void testConvolution1DLayer() throws Exception { + public void testConvolution1DLayer() throws Exception { buildConvolution1DLayer(conf1, keras1, false); buildConvolution1DLayer(conf2, keras2, false); buildConvolution1DLayer(conf2, keras2, true); } - private void buildConvolution1DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) throws Exception { + private void buildConvolution1DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CONVOLUTION_1D()); Map config = new HashMap<>(); @@ -111,12 +88,9 @@ class KerasConvolution1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_INIT(), init); } if (withDilation) { - ArrayList dilation = new ArrayList() { - - { - for (int i : DILATION) add(i); - } - }; + ArrayList dilation = new ArrayList() {{ + for (int i : DILATION) add(i); + }}; config.put(conf.getLAYER_FIELD_DILATION_RATE(), dilation); } Map W_reg = new HashMap(); @@ -125,23 +99,18 @@ class KerasConvolution1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_W_REGULARIZER(), W_reg); config.put(conf.getLAYER_FIELD_DROPOUT(), DROPOUT_KERAS); if (kerasVersion == 2) { - ArrayList kernel = new ArrayList() { - - { - for (int i : KERNEL_SIZE) add(i); - } - }; + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_FILTER_LENGTH(), kernel); } else { config.put(conf.getLAYER_FIELD_FILTER_LENGTH(), KERNEL_SIZE[0]); } - if (kerasVersion == 2) { - ArrayList stride = new ArrayList() { - { - for (int i : STRIDE) add(i); - } - }; + if (kerasVersion == 2) { + ArrayList stride = new ArrayList() {{ + for (int i : STRIDE) add(i); + }}; config.put(conf.getLAYER_FIELD_SUBSAMPLE_LENGTH(), stride); } else { config.put(conf.getLAYER_FIELD_SUBSAMPLE_LENGTH(), STRIDE[0]); @@ -149,6 +118,7 @@ class KerasConvolution1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NB_FILTER(), N_OUT); config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); + Convolution1DLayer layer = new KerasConvolution1D(layerConfig).getConvolution1DLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution2DTest.java similarity index 84% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution2DTest.java index ed0a162f0..f449c2cae 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -30,75 +31,53 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasConvolution2D; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Convolution 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasConvolution2DTest extends BaseDL4JTest { +public class KerasConvolution2DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "test_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 1, 2 }; - - private final int[] DILATION = new int[] { 2, 2 }; - - private final int[] STRIDE = new int[] { 3, 4 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2}; + private final int[] DILATION = new int[]{2, 2}; + private final int[] STRIDE = new int[]{3, 4}; private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Convolution 2 D Layer") - void testConvolution2DLayer() throws Exception { + public void testConvolution2DLayer() throws Exception { buildConvolution2DLayer(conf1, keras1, false); buildConvolution2DLayer(conf2, keras2, false); buildConvolution2DLayer(conf2, keras2, true); } - private void buildConvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) throws Exception { + + private void buildConvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CONVOLUTION_2D()); Map config = new HashMap<>(); @@ -120,21 +99,15 @@ class KerasConvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NB_ROW(), KERNEL_SIZE[0]); config.put(conf.getLAYER_FIELD_NB_COL(), KERNEL_SIZE[1]); } else { - ArrayList kernel = new ArrayList() { - - { - for (int i : KERNEL_SIZE) add(i); - } - }; + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_KERNEL_SIZE(), kernel); } if (withDilation) { - ArrayList dilation = new ArrayList() { - - { - for (int i : DILATION) add(i); - } - }; + ArrayList dilation = new ArrayList() {{ + for (int i : DILATION) add(i); + }}; config.put(conf.getLAYER_FIELD_DILATION_RATE(), dilation); } List subsampleList = new ArrayList<>(); @@ -145,6 +118,8 @@ class KerasConvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + + ConvolutionLayer layer = new KerasConvolution2D(layerConfig).getConvolution2DLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution3DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution3DTest.java similarity index 86% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution3DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution3DTest.java index 40f7196e4..a6c9af9c4 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution3DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasConvolution3DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -30,72 +31,51 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasConvolution3D; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Convolution 3 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasConvolution3DTest extends BaseDL4JTest { +public class KerasConvolution3DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "test_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 1, 2, 3 }; - - private final int[] STRIDE = new int[] { 3, 4, 5 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2, 3}; + private final int[] STRIDE = new int[]{3, 4, 5}; private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Convolution 3 D Layer") - void testConvolution3DLayer() throws Exception { + public void testConvolution3DLayer() throws Exception { buildConvolution3DLayer(conf1, keras1); buildConvolution3DLayer(conf2, keras2); } - private void buildConvolution3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + + private void buildConvolution3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CONVOLUTION_3D()); Map config = new HashMap<>(); @@ -117,15 +97,14 @@ class KerasConvolution3DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_3D_KERNEL_1(), KERNEL_SIZE[0]); config.put(conf.getLAYER_FIELD_3D_KERNEL_2(), KERNEL_SIZE[1]); config.put(conf.getLAYER_FIELD_3D_KERNEL_3(), KERNEL_SIZE[2]); - } else { - ArrayList kernel = new ArrayList() { - { - for (int i : KERNEL_SIZE) add(i); - } - }; + } else { + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_KERNEL_SIZE(), kernel); } + List subsampleList = new ArrayList<>(); subsampleList.add(STRIDE[0]); subsampleList.add(STRIDE[1]); @@ -135,6 +114,8 @@ class KerasConvolution3DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + + ConvolutionLayer layer = new KerasConvolution3D(layerConfig).getConvolution3DLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); @@ -147,5 +128,6 @@ class KerasConvolution3DTest extends BaseDL4JTest { assertEquals(N_OUT, layer.getNOut()); assertEquals(ConvolutionMode.Truncate, layer.getConvolutionMode()); assertArrayEquals(VALID_PADDING, layer.getPadding()); + } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping1DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping1DTest.java similarity index 84% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping1DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping1DTest.java index 125bd1182..9519aa4ac 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping1DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping1DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.convolutional.Cropping1D; @@ -25,43 +26,36 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasCropping1D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Cropping 1 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasCropping1DTest extends BaseDL4JTest { +public class KerasCropping1DTest extends BaseDL4JTest { private final String LAYER_NAME = "cropping_1D_layer"; - private final int CROPPING = 2; private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Cropping 1 D Layer") - void testCropping1DLayer() throws Exception { + public void testCropping1DLayer() throws Exception { Integer keras1 = 1; Integer keras2 = 2; buildCroppingSingleDim1DLayer(conf1, keras1); buildCroppingSingleDim1DLayer(conf2, keras2); } - private void buildCroppingSingleDim1DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + + + private void buildCroppingSingleDim1DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CROPPING_1D()); Map config = new HashMap<>(); @@ -69,6 +63,7 @@ class KerasCropping1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_CROPPING(), CROPPING); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Cropping1D layer = new KerasCropping1D(layerConfig).getCropping1DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(CROPPING, layer.getCropping()[0]); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping2DTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping2DTest.java index 347053879..966690847 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; @@ -25,37 +26,27 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasCropping2D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Cropping 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasCropping2DTest extends BaseDL4JTest { +public class KerasCropping2DTest extends BaseDL4JTest { private final String LAYER_NAME = "cropping_2D_layer"; - - private final int[] CROPPING = new int[] { 2, 3 }; + private final int[] CROPPING = new int[]{2, 3}; private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Cropping 2 D Layer") - void testCropping2DLayer() throws Exception { + public void testCropping2DLayer() throws Exception { Integer keras1 = 1; buildCropping2DLayer(conf1, keras1); Integer keras2 = 2; @@ -64,29 +55,31 @@ class KerasCropping2DTest extends BaseDL4JTest { buildCroppingSingleDim2DLayer(conf2, keras2); } - private void buildCropping2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + + private void buildCropping2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CROPPING_2D()); Map config = new HashMap<>(); config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); - ArrayList padding = new ArrayList() { - - { - for (int i : CROPPING) add(i); - } - }; + ArrayList padding = new ArrayList() {{ + for (int i : CROPPING) add(i); + }}; config.put(conf.getLAYER_FIELD_CROPPING(), padding); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Cropping2D layer = new KerasCropping2D(layerConfig).getCropping2DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(CROPPING[0], layer.getCropping()[0]); assertEquals(CROPPING[0], layer.getCropping()[1]); assertEquals(CROPPING[1], layer.getCropping()[2]); assertEquals(CROPPING[1], layer.getCropping()[3]); + } - private void buildCroppingSingleDim2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + private void buildCroppingSingleDim2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CROPPING_2D()); Map config = new HashMap<>(); @@ -94,6 +87,7 @@ class KerasCropping2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_CROPPING(), CROPPING[0]); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Cropping2D layer = new KerasCropping2D(layerConfig).getCropping2DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(CROPPING[0], layer.getCropping()[0]); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping3DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping3DTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping3DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping3DTest.java index 9120cd984..7c8f45579 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping3DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasCropping3DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.convolutional.Cropping3D; @@ -25,37 +26,27 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasCropping3D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Cropping 3 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasCropping3DTest extends BaseDL4JTest { +public class KerasCropping3DTest extends BaseDL4JTest { private final String LAYER_NAME = "cropping_3D_layer"; - - private final int[] CROPPING = new int[] { 2, 3, 5 }; + private final int[] CROPPING = new int[]{2, 3, 5}; private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Cropping 3 D Layer") - void testCropping3DLayer() throws Exception { + public void testCropping3DLayer() throws Exception { Integer keras1 = 1; buildCropping3DLayer(conf1, keras1); Integer keras2 = 2; @@ -64,20 +55,20 @@ class KerasCropping3DTest extends BaseDL4JTest { buildCroppingSingleDim3DLayer(conf2, keras2); } - private void buildCropping3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + + private void buildCropping3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CROPPING_3D()); Map config = new HashMap<>(); config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); - ArrayList padding = new ArrayList() { - - { - for (int i : CROPPING) add(i); - } - }; + ArrayList padding = new ArrayList() {{ + for (int i : CROPPING) add(i); + }}; config.put(conf.getLAYER_FIELD_CROPPING(), padding); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Cropping3D layer = new KerasCropping3D(layerConfig).getCropping3DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(CROPPING[0], layer.getCropping()[0]); @@ -86,9 +77,11 @@ class KerasCropping3DTest extends BaseDL4JTest { assertEquals(CROPPING[1], layer.getCropping()[3]); assertEquals(CROPPING[2], layer.getCropping()[4]); assertEquals(CROPPING[2], layer.getCropping()[5]); + } - private void buildCroppingSingleDim3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + private void buildCroppingSingleDim3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_CROPPING_3D()); Map config = new HashMap<>(); @@ -96,6 +89,7 @@ class KerasCropping3DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_CROPPING(), CROPPING[0]); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Cropping3D layer = new KerasCropping3D(layerConfig).getCropping3DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(CROPPING[0], layer.getCropping()[0]); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDeconvolution2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDeconvolution2DTest.java similarity index 84% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDeconvolution2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDeconvolution2DTest.java index 035055f5e..37fafc785 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDeconvolution2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDeconvolution2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -30,75 +31,53 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasDeconvolution2D; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Deconvolution 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasDeconvolution2DTest extends BaseDL4JTest { +public class KerasDeconvolution2DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "deconvolution_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 1, 2 }; - - private final int[] DILATION = new int[] { 2, 2 }; - - private final int[] STRIDE = new int[] { 3, 4 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2}; + private final int[] DILATION = new int[]{2, 2}; + private final int[] STRIDE = new int[]{3, 4}; private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Deconvolution 2 D Layer") - void testDeconvolution2DLayer() throws Exception { + public void testDeconvolution2DLayer() throws Exception { buildDeconvolution2DLayer(conf1, keras1, false); buildDeconvolution2DLayer(conf2, keras2, false); buildDeconvolution2DLayer(conf2, keras2, true); } - private void buildDeconvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) throws Exception { + + private void buildDeconvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_DECONVOLUTION_2D()); Map config = new HashMap<>(); @@ -120,21 +99,15 @@ class KerasDeconvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NB_ROW(), KERNEL_SIZE[0]); config.put(conf.getLAYER_FIELD_NB_COL(), KERNEL_SIZE[1]); } else { - ArrayList kernel = new ArrayList() { - - { - for (int i : KERNEL_SIZE) add(i); - } - }; + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_KERNEL_SIZE(), kernel); } if (withDilation) { - ArrayList dilation = new ArrayList() { - - { - for (int i : DILATION) add(i); - } - }; + ArrayList dilation = new ArrayList() {{ + for (int i : DILATION) add(i); + }}; config.put(conf.getLAYER_FIELD_DILATION_RATE(), dilation); } List subsampleList = new ArrayList<>(); @@ -145,6 +118,8 @@ class KerasDeconvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + + Deconvolution2D layer = new KerasDeconvolution2D(layerConfig).getDeconvolution2DLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDepthwiseConvolution2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDepthwiseConvolution2DTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDepthwiseConvolution2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDepthwiseConvolution2DTest.java index bd0494e51..1b4b8e7c7 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDepthwiseConvolution2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasDepthwiseConvolution2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -31,70 +32,49 @@ import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasConvolu import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasDepthwiseConvolution2D; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.base.Preconditions; + import java.util.*; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Depthwise Convolution 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasDepthwiseConvolution2DTest extends BaseDL4JTest { +public class KerasDepthwiseConvolution2DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "test_layer"; - private final String INIT_KERAS = "depthwise_conv_2d"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 1, 2 }; - - private final int[] DILATION = new int[] { 2, 2 }; - - private final int[] STRIDE = new int[] { 3, 4 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2}; + private final int[] DILATION = new int[]{2, 2}; + private final int[] STRIDE = new int[]{3, 4}; private final int DEPTH_MULTIPLIER = 4; - private final int N_IN = 3; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras2 = 2; - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Depthwise Convolution 2 D Layer") - void testDepthwiseConvolution2DLayer() throws Exception { + public void testDepthwiseConvolution2DLayer() throws Exception { buildDepthwiseConvolution2DLayer(conf2, keras2, false); buildDepthwiseConvolution2DLayer(conf2, keras2, true); } - private void buildDepthwiseConvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) throws Exception { + + private void buildDepthwiseConvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_DEPTHWISE_CONVOLUTION_2D()); Map config = new HashMap<>(); @@ -115,20 +95,16 @@ class KerasDepthwiseConvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_DEPTH_WISE_REGULARIZER(), W_reg); config.put(conf.getLAYER_FIELD_DROPOUT(), DROPOUT_KERAS); config.put(conf.getLAYER_FIELD_DEPTH_MULTIPLIER(), DEPTH_MULTIPLIER); - ArrayList kernel = new ArrayList() { - { - for (int i : KERNEL_SIZE) add(i); - } - }; + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_KERNEL_SIZE(), kernel); - if (withDilation) { - ArrayList dilation = new ArrayList() { - { - for (int i : DILATION) add(i); - } - }; + if (withDilation) { + ArrayList dilation = new ArrayList() {{ + for (int i : DILATION) add(i); + }}; config.put(conf.getLAYER_FIELD_DILATION_RATE(), dilation); } List subsampleList = new ArrayList<>(); @@ -139,12 +115,16 @@ class KerasDepthwiseConvolution2DTest extends BaseDL4JTest { layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); config.put(conf.getLAYER_FIELD_NB_FILTER(), N_IN); + KerasConvolution2D previousLayer = new KerasConvolution2D(layerConfig); Map previousLayers = new HashMap<>(); previousLayers.put("conv", previousLayer); List layerNames = Collections.singletonList("conv"); - KerasDepthwiseConvolution2D kerasLayer = new KerasDepthwiseConvolution2D(layerConfig, previousLayers, layerNames, false); + + KerasDepthwiseConvolution2D kerasLayer = new KerasDepthwiseConvolution2D( + layerConfig, previousLayers, layerNames, false); Preconditions.checkState(kerasLayer.getInboundLayerNames().get(0).equalsIgnoreCase("conv"), "Expected inbound name to be \"conv\" - was \"%s\"", kerasLayer.getInboundLayerNames().get(0)); + DepthwiseConvolution2D layer = kerasLayer.getDepthwiseConvolution2DLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasSeparableConvolution2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasSeparableConvolution2DTest.java similarity index 84% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasSeparableConvolution2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasSeparableConvolution2DTest.java index b749c625e..fb1df4525 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasSeparableConvolution2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasSeparableConvolution2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -30,77 +31,54 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasSeparableConvolution2D; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Separable Convolution 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasSeparableConvolution2DTest extends BaseDL4JTest { +public class KerasSeparableConvolution2DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "test_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 1, 2 }; - - private final int[] DILATION = new int[] { 2, 2 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2}; + private final int[] DILATION = new int[]{2, 2}; private final int DEPTH_MULTIPLIER = 4; - - private final int[] STRIDE = new int[] { 3, 4 }; - + private final int[] STRIDE = new int[]{3, 4}; private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Separable Convolution 2 D Layer") - void testSeparableConvolution2DLayer() throws Exception { + public void testSeparableConvolution2DLayer() throws Exception { buildSeparableConvolution2DLayer(conf1, keras1, false); buildSeparableConvolution2DLayer(conf2, keras2, false); buildSeparableConvolution2DLayer(conf2, keras2, true); } - private void buildSeparableConvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) throws Exception { + + private void buildSeparableConvolution2DLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean withDilation) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_SEPARABLE_CONVOLUTION_2D()); Map config = new HashMap<>(); @@ -109,11 +87,13 @@ class KerasSeparableConvolution2DTest extends BaseDL4JTest { if (kerasVersion == 1) { config.put(conf.getLAYER_FIELD_DEPTH_WISE_INIT(), INIT_KERAS); config.put(conf.getLAYER_FIELD_POINT_WISE_INIT(), INIT_KERAS); + } else { Map init = new HashMap<>(); init.put("class_name", conf.getINIT_GLOROT_NORMAL()); config.put(conf.getLAYER_FIELD_DEPTH_WISE_INIT(), init); config.put(conf.getLAYER_FIELD_POINT_WISE_INIT(), init); + } Map W_reg = new HashMap<>(); W_reg.put(conf.getREGULARIZATION_TYPE_L1(), L1_REGULARIZATION); @@ -121,25 +101,20 @@ class KerasSeparableConvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_DEPTH_WISE_REGULARIZER(), W_reg); config.put(conf.getLAYER_FIELD_DROPOUT(), DROPOUT_KERAS); config.put(conf.getLAYER_FIELD_DEPTH_MULTIPLIER(), DEPTH_MULTIPLIER); + if (kerasVersion == 1) { config.put(conf.getLAYER_FIELD_NB_ROW(), KERNEL_SIZE[0]); config.put(conf.getLAYER_FIELD_NB_COL(), KERNEL_SIZE[1]); } else { - ArrayList kernel = new ArrayList() { - - { - for (int i : KERNEL_SIZE) add(i); - } - }; + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_KERNEL_SIZE(), kernel); } if (withDilation) { - ArrayList dilation = new ArrayList() { - - { - for (int i : DILATION) add(i); - } - }; + ArrayList dilation = new ArrayList() {{ + for (int i : DILATION) add(i); + }}; config.put(conf.getLAYER_FIELD_DILATION_RATE(), dilation); } List subsampleList = new ArrayList<>(); @@ -150,6 +125,8 @@ class KerasSeparableConvolution2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + + SeparableConvolution2D layer = new KerasSeparableConvolution2D(layerConfig).getSeparableConvolution2DLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling1DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling1DTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling1DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling1DTest.java index 7b838009b..4985681cd 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling1DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling1DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.Upsampling1D; @@ -25,40 +26,28 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasUpsampling1D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Upsampling 1 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasUpsampling1DTest extends BaseDL4JTest { +public class KerasUpsampling1DTest extends BaseDL4JTest { private final String LAYER_NAME = "upsampling_1D_layer"; - private int size = 4; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Upsampling 1 D Layer") - void testUpsampling1DLayer() throws Exception { + public void testUpsampling1DLayer() throws Exception { buildUpsampling1DLayer(conf1, keras1); buildUpsampling1DLayer(conf2, keras2); } @@ -71,8 +60,10 @@ class KerasUpsampling1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Upsampling1D layer = new KerasUpsampling1D(layerConfig).getUpsampling1DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(size, layer.getSize()[0]); } + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling2DTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling2DTest.java index 9359043ff..eb38f4ec0 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.Upsampling2D; @@ -25,46 +26,35 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasUpsampling2D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Upsampling 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasUpsampling2DTest extends BaseDL4JTest { +public class KerasUpsampling2DTest extends BaseDL4JTest { private final String LAYER_NAME = "upsampling_2D_layer"; - - private int[] size = new int[] { 2, 2 }; + private int[] size = new int[]{2, 2}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Upsampling 2 D Layer") - void testUpsampling2DLayer() throws Exception { + public void testUpsampling2DLayer() throws Exception { buildUpsampling2DLayer(conf1, keras1); buildUpsampling2DLayer(conf2, keras2); } + private void buildUpsampling2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_UPSAMPLING_2D()); @@ -76,9 +66,12 @@ class KerasUpsampling2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Upsampling2D layer = new KerasUpsampling2D(layerConfig).getUpsampling2DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(size[0], layer.getSize()[0]); assertEquals(size[1], layer.getSize()[1]); + } + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling3DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling3DTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling3DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling3DTest.java index c4cc4c860..7741785d1 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling3DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasUpsampling3DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.Upsampling3D; @@ -25,46 +26,35 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasUpsampling3D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Upsampling 3 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasUpsampling3DTest extends BaseDL4JTest { +public class KerasUpsampling3DTest extends BaseDL4JTest { private final String LAYER_NAME = "upsampling_3D_layer"; - - private int[] size = new int[] { 2, 2, 2 }; + private int[] size = new int[]{2, 2, 2}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Upsampling 3 D Layer") - void testUpsampling3DLayer() throws Exception { + public void testUpsampling3DLayer() throws Exception { buildUpsampling3DLayer(conf1, keras1); buildUpsampling3DLayer(conf2, keras2); } + private void buildUpsampling3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_UPSAMPLING_3D()); @@ -77,10 +67,12 @@ class KerasUpsampling3DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Upsampling3D layer = new KerasUpsampling3D(layerConfig).getUpsampling3DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(size[0], layer.getSize()[0]); assertEquals(size[1], layer.getSize()[1]); assertEquals(size[2], layer.getSize()[2]); } + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding1DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding1DTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding1DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding1DTest.java index 4800b35a2..9cfe0bdab 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding1DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding1DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.ZeroPadding1DLayer; @@ -25,38 +26,30 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasZeroPadding1D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Zero Padding 1 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasZeroPadding1DTest extends BaseDL4JTest { +public class KerasZeroPadding1DTest extends BaseDL4JTest { private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Zero Padding 1 D Layer") - void testZeroPadding1DLayer() throws Exception { + public void testZeroPadding1DLayer() throws Exception { Integer keras1 = 1; buildZeroPadding1DLayer(conf1, keras1); Integer keras2 = 2; buildZeroPadding1DLayer(conf2, keras2); } + private void buildZeroPadding1DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_ZERO_PADDING_1D()); @@ -67,8 +60,10 @@ class KerasZeroPadding1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_ZERO_PADDING(), zeroPadding); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + ZeroPadding1DLayer layer = new KerasZeroPadding1D(layerConfig).getZeroPadding1DLayer(); assertEquals(layerName, layer.getLayerName()); assertEquals(zeroPadding, layer.getPadding()[0]); } + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding2DTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding2DTest.java index 555c02237..809cb5f0a 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.ZeroPaddingLayer; @@ -25,37 +26,27 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasZeroPadding2D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Zero Padding 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasZeroPadding2DTest extends BaseDL4JTest { +public class KerasZeroPadding2DTest extends BaseDL4JTest { private final String LAYER_NAME = "zero_padding_2D_layer"; - - private final int[] ZERO_PADDING = new int[] { 2, 3 }; + private final int[] ZERO_PADDING = new int[]{2, 3}; private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Zero Padding 2 D Layer") - void testZeroPadding2DLayer() throws Exception { + public void testZeroPadding2DLayer() throws Exception { Integer keras1 = 1; buildZeroPadding2DLayer(conf1, keras1); Integer keras2 = 2; @@ -64,29 +55,31 @@ class KerasZeroPadding2DTest extends BaseDL4JTest { buildZeroPaddingSingleDim2DLayer(conf2, keras2); } - private void buildZeroPadding2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + + private void buildZeroPadding2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_ZERO_PADDING_2D()); Map config = new HashMap<>(); config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); - ArrayList padding = new ArrayList() { - - { - for (int i : ZERO_PADDING) add(i); - } - }; + ArrayList padding = new ArrayList() {{ + for (int i : ZERO_PADDING) add(i); + }}; config.put(conf.getLAYER_FIELD_ZERO_PADDING(), padding); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + ZeroPaddingLayer layer = new KerasZeroPadding2D(layerConfig).getZeroPadding2DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(ZERO_PADDING[0], layer.getPadding()[0]); assertEquals(ZERO_PADDING[0], layer.getPadding()[1]); assertEquals(ZERO_PADDING[1], layer.getPadding()[2]); assertEquals(ZERO_PADDING[1], layer.getPadding()[3]); + } - private void buildZeroPaddingSingleDim2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + private void buildZeroPaddingSingleDim2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_ZERO_PADDING_2D()); Map config = new HashMap<>(); @@ -94,6 +87,7 @@ class KerasZeroPadding2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_ZERO_PADDING(), ZERO_PADDING[0]); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + ZeroPaddingLayer layer = new KerasZeroPadding2D(layerConfig).getZeroPadding2DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(ZERO_PADDING[0], layer.getPadding()[0]); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding3DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding3DTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding3DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding3DTest.java index 03cfc08a2..6ae93473b 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding3DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/convolution/KerasZeroPadding3DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.convolution; import org.deeplearning4j.nn.conf.layers.ZeroPadding3DLayer; @@ -25,37 +26,27 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasZeroPadding3D; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Zero Padding 3 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasZeroPadding3DTest extends BaseDL4JTest { +public class KerasZeroPadding3DTest extends BaseDL4JTest { private final String LAYER_NAME = "zero_padding_3D_layer"; - - private final int[] ZERO_PADDING = new int[] { 2, 3, 4 }; + private final int[] ZERO_PADDING = new int[]{2, 3, 4}; private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Zero Padding 3 D Layer") - void testZeroPadding3DLayer() throws Exception { + public void testZeroPadding3DLayer() throws Exception { Integer keras1 = 1; buildZeroPadding3DLayer(conf1, keras1); Integer keras2 = 2; @@ -64,20 +55,20 @@ class KerasZeroPadding3DTest extends BaseDL4JTest { buildZeroPaddingSingleDim3DLayer(conf2, keras2); } - private void buildZeroPadding3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + + private void buildZeroPadding3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_ZERO_PADDING_3D()); Map config = new HashMap<>(); config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); - ArrayList padding = new ArrayList() { - - { - for (int i : ZERO_PADDING) add(i); - } - }; + ArrayList padding = new ArrayList() {{ + for (int i : ZERO_PADDING) add(i); + }}; config.put(conf.getLAYER_FIELD_ZERO_PADDING(), padding); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + ZeroPadding3DLayer layer = new KerasZeroPadding3D(layerConfig).getZeroPadding3DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(ZERO_PADDING[0], layer.getPadding()[0]); @@ -86,9 +77,11 @@ class KerasZeroPadding3DTest extends BaseDL4JTest { assertEquals(ZERO_PADDING[1], layer.getPadding()[3]); assertEquals(ZERO_PADDING[2], layer.getPadding()[4]); assertEquals(ZERO_PADDING[2], layer.getPadding()[5]); + } - private void buildZeroPaddingSingleDim3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + private void buildZeroPaddingSingleDim3DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_ZERO_PADDING_3D()); Map config = new HashMap<>(); @@ -96,6 +89,7 @@ class KerasZeroPadding3DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_ZERO_PADDING(), ZERO_PADDING[0]); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + ZeroPadding3DLayer layer = new KerasZeroPadding3D(layerConfig).getZeroPadding3DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(ZERO_PADDING[0], layer.getPadding()[0]); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivationLayer.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivationLayer.java similarity index 94% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivationLayer.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivationLayer.java index ec716c716..ad73a4c00 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivationLayer.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasActivationLayer.java @@ -25,18 +25,13 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag + public class KerasActivationLayer extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDenseTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDenseTest.java similarity index 90% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDenseTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDenseTest.java index e196e2f9c..2d5c4f864 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDenseTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDenseTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.core; import org.deeplearning4j.nn.conf.dropout.Dropout; @@ -28,60 +29,41 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Dense Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasDenseTest extends BaseDL4JTest { +public class KerasDenseTest extends BaseDL4JTest { private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "dense"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - private final int N_OUT = 13; @Test - @DisplayName("Test Dense Layer") - void testDenseLayer() throws Exception { + public void testDenseLayer() throws Exception { buildDenseLayer(conf1, keras1); buildDenseLayer(conf2, keras2); } + private void buildDenseLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_DENSE()); @@ -103,6 +85,7 @@ class KerasDenseTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_OUTPUT_DIM(), N_OUT); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + DenseLayer layer = new KerasDense(layerConfig, false).getDenseLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropoutTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropoutTest.java similarity index 86% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropoutTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropoutTest.java index b0b416b38..322955813 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropoutTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDropoutTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.core; import org.deeplearning4j.nn.conf.dropout.Dropout; @@ -25,46 +26,35 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Dropout Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasDropoutTest extends BaseDL4JTest { +public class KerasDropoutTest extends BaseDL4JTest { String LAYER_NAME = "dropout"; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Dropout Layer") - void testDropoutLayer() throws Exception { + public void testDropoutLayer() throws Exception { buildDropoutLayer(conf1, keras1); buildDropoutLayer(conf2, keras2); } + private void buildDropoutLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_DROPOUT()); @@ -73,8 +63,11 @@ class KerasDropoutTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_DROPOUT(), DROPOUT_KERAS); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + DropoutLayer layer = new KerasDropout(layerConfig).getDropoutLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(new Dropout(DROPOUT_DL4J), layer.getIDropout()); } + + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMaskingTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMaskingTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMaskingTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMaskingTest.java index eef197b1e..f898209ce 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMaskingTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasMaskingTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.core; import org.deeplearning4j.nn.conf.layers.util.MaskZeroLayer; @@ -24,38 +25,33 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; + /** * @author Max Pumperla */ -@DisplayName("Keras Masking Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasMaskingTest extends BaseDL4JTest { +public class KerasMaskingTest extends BaseDL4JTest { + private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Masking Layer") - void testMaskingLayer() throws Exception { + public void testMaskingLayer() throws Exception { Integer keras1 = 1; buildMaskingLayer(conf1, keras1); Integer keras2 = 2; buildMaskingLayer(conf2, keras2); } + private void buildMaskingLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_MASKING()); @@ -66,7 +62,10 @@ class KerasMaskingTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_MASK_VALUE(), MASKING_VALUE); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + MaskZeroLayer layer = new KerasMasking(layerConfig).getMaskingLayer(); assertEquals(MASKING_VALUE, layer.getMaskingValue(), 0.0); } + + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermuteTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermuteTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermuteTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermuteTest.java index cd619c9d0..50efe158a 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermuteTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasPermuteTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.core; import org.deeplearning4j.nn.conf.inputs.InputType; @@ -27,44 +28,35 @@ import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException; import org.deeplearning4j.nn.modelimport.keras.preprocessors.PermutePreprocessor; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Permute Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasPermuteTest extends BaseDL4JTest { +public class KerasPermuteTest extends BaseDL4JTest { private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Permute Layer") - void testPermuteLayer() throws Exception { + public void testPermuteLayer() throws Exception { buildPermuteLayer(conf1, keras1); buildPermuteLayer(conf2, keras2); } + private void buildPermuteLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { - int[] permuteIndices = new int[] { 2, 1 }; + int[] permuteIndices = new int[]{2, 1}; List permuteList = new ArrayList<>(); permuteList.add(permuteIndices[0]); permuteList.add(permuteIndices[1]); @@ -73,7 +65,9 @@ class KerasPermuteTest extends BaseDL4JTest { assertEquals(preProcessor.getPermutationIndices()[1], permuteIndices[1]); } - private PermutePreprocessor getPermutePreProcessor(KerasLayerConfiguration conf, Integer kerasVersion, List permuteList) throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { + private PermutePreprocessor getPermutePreProcessor(KerasLayerConfiguration conf, Integer kerasVersion, + List permuteList) + throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_RESHAPE()); Map config = new HashMap<>(); @@ -83,5 +77,6 @@ class KerasPermuteTest extends BaseDL4JTest { layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); InputType inputType = InputType.InputTypeFeedForward.recurrent(20, 10); return (PermutePreprocessor) new KerasPermute(layerConfig).getInputPreprocessor(inputType); + } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVectorTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVectorTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVectorTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVectorTest.java index 413107064..7390c8bc5 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVectorTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasRepeatVectorTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.core; import org.deeplearning4j.nn.conf.layers.misc.RepeatVector; @@ -24,44 +25,34 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Repeat Vector Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasRepeatVectorTest extends BaseDL4JTest { +public class KerasRepeatVectorTest extends BaseDL4JTest { String LAYER_NAME = "repeat"; - private int REPEAT = 4; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Repeat Vector Layer") - void testRepeatVectorLayer() throws Exception { + public void testRepeatVectorLayer() throws Exception { buildRepeatVectorLayer(conf1, keras1); buildRepeatVectorLayer(conf2, keras2); } + private void buildRepeatVectorLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_REPEAT()); @@ -70,8 +61,11 @@ class KerasRepeatVectorTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_REPEAT_MULTIPLIER(), REPEAT); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + RepeatVector layer = new KerasRepeatVector(layerConfig).getRepeatVectorLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(layer.getN(), REPEAT); } + + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshapeTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshapeTest.java similarity index 82% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshapeTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshapeTest.java index fa1d3acb5..6e57fa561 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshapeTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasReshapeTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.core; import org.deeplearning4j.nn.conf.inputs.InputType; @@ -29,50 +30,39 @@ import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfig import org.deeplearning4j.nn.modelimport.keras.preprocessors.ReshapePreprocessor; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; + import java.util.*; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** * @author Max Pumperla */ -@DisplayName("Keras Reshape Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasReshapeTest extends BaseDL4JTest { +public class KerasReshapeTest extends BaseDL4JTest { private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Reshape Layer") - void testReshapeLayer() throws Exception { + public void testReshapeLayer() throws Exception { buildReshapeLayer(conf1, keras1); buildReshapeLayer(conf2, keras2); } @Test - @DisplayName("Test Reshape Dynamic Minibatch") - void testReshapeDynamicMinibatch() throws Exception { + public void testReshapeDynamicMinibatch() throws Exception { testDynamicMinibatches(conf1, keras1); testDynamicMinibatches(conf2, keras2); } private void buildReshapeLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { - int[] targetShape = new int[] { 10, 5 }; + int[] targetShape = new int[]{10, 5}; List targetShapeList = new ArrayList<>(); targetShapeList.add(targetShape[0]); targetShapeList.add(targetShape[1]); @@ -81,7 +71,9 @@ class KerasReshapeTest extends BaseDL4JTest { assertEquals(preProcessor.getTargetShape()[1], targetShape[1]); } - private ReshapePreprocessor getReshapePreProcessor(KerasLayerConfiguration conf, Integer kerasVersion, List targetShapeList) throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { + private ReshapePreprocessor getReshapePreProcessor(KerasLayerConfiguration conf, Integer kerasVersion, + List targetShapeList) + throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_RESHAPE()); Map config = new HashMap<>(); @@ -93,6 +85,7 @@ class KerasReshapeTest extends BaseDL4JTest { layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); InputType inputType = InputType.InputTypeFeedForward.feedForward(20); return (ReshapePreprocessor) new KerasReshape(layerConfig).getInputPreprocessor(inputType); + } private void testDynamicMinibatches(KerasLayerConfiguration conf, Integer kerasVersion) throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { @@ -100,7 +93,7 @@ class KerasReshapeTest extends BaseDL4JTest { ReshapePreprocessor preprocessor = getReshapePreProcessor(conf, kerasVersion, targetShape); INDArray r1 = preprocessor.preProcess(Nd4j.zeros(10, 20), 10, LayerWorkspaceMgr.noWorkspaces()); INDArray r2 = preprocessor.preProcess(Nd4j.zeros(5, 20), 5, LayerWorkspaceMgr.noWorkspaces()); - Assertions.assertArrayEquals(r2.shape(), new long[] { 5, 20 }); - Assertions.assertArrayEquals(r1.shape(), new long[] { 10, 20 }); + Assertions.assertArrayEquals(r2.shape(), new long[]{5, 20}); + Assertions.assertArrayEquals(r1.shape(), new long[]{10, 20}); } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout2DTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout2DTest.java index ad8c4931d..01d225c19 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasSpatialDropout2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.core; import org.deeplearning4j.nn.conf.dropout.SpatialDropout; @@ -25,46 +26,35 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Spatial Dropout 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasSpatialDropout2DTest extends BaseDL4JTest { +public class KerasSpatialDropout2DTest extends BaseDL4JTest { String LAYER_NAME = "spatial_dropout_2d"; - private final double RATE_KERAS = 0.3; - private final double RATE_DL4J = 1 - RATE_KERAS; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Spatial Dropout Layer") - void testSpatialDropoutLayer() throws Exception { + public void testSpatialDropoutLayer() throws Exception { buildSpatialDropoutLayer(conf1, keras1); buildSpatialDropoutLayer(conf2, keras2); } + private void buildSpatialDropoutLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_SPATIAL_DROPOUT_2D()); @@ -73,8 +63,10 @@ class KerasSpatialDropout2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_RATE(), RATE_KERAS); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + DropoutLayer layer = new KerasSpatialDropout(layerConfig).getSpatialDropoutLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(new SpatialDropout(RATE_DL4J), layer.getIDropout()); } + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbeddingTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbeddingTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbeddingTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbeddingTest.java index 9c64f9eeb..d358bd61e 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbeddingTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbeddingTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.embeddings; import org.deeplearning4j.nn.conf.layers.EmbeddingSequenceLayer; @@ -25,45 +26,30 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; + import java.util.*; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** * @author Max Pumperla */ -@DisplayName("Keras Embedding Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasEmbeddingTest extends BaseDL4JTest { +public class KerasEmbeddingTest extends BaseDL4JTest { private final String LAYER_NAME = "embedding_sequence_layer"; - private final String INIT_KERAS = "glorot_normal"; - - private final int[] INPUT_SHAPE = new int[] { 100, 20 }; - - private static final boolean[] MASK_ZERO = new boolean[] { false, true }; - + private final int[] INPUT_SHAPE = new int[]{100, 20}; + private static final boolean[] MASK_ZERO = new boolean[]{false, true}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Embedding Layer") - void testEmbeddingLayer() throws Exception { + public void testEmbeddingLayer() throws Exception { for (boolean mz : MASK_ZERO) { buildEmbeddingLayer(conf1, keras1, mz); buildEmbeddingLayer(conf2, keras2, mz); @@ -71,17 +57,17 @@ class KerasEmbeddingTest extends BaseDL4JTest { } @Test - @DisplayName("Test Embedding Layer Set Weights Mask Zero") - void testEmbeddingLayerSetWeightsMaskZero() throws Exception { - // GIVEN keras embedding with mask zero true + public void testEmbeddingLayerSetWeightsMaskZero() throws Exception { + //GIVEN keras embedding with mask zero true KerasEmbedding embedding = buildEmbeddingLayer(conf1, keras1, true); - // WHEN + //WHEN embedding.setWeights(Collections.singletonMap(conf1.getLAYER_FIELD_EMBEDDING_WEIGHTS(), Nd4j.ones(INPUT_SHAPE))); - // THEN first row is set to zeros + //THEN first row is set to zeros INDArray weights = embedding.getWeights().get(DefaultParamInitializer.WEIGHT_KEY); - assertEquals(embedding.getWeights().get(DefaultParamInitializer.WEIGHT_KEY).columns(), INPUT_SHAPE[1]); + assertEquals(embedding.getWeights().get(DefaultParamInitializer.WEIGHT_KEY).columns(),INPUT_SHAPE[1]); } + private KerasEmbedding buildEmbeddingLayer(KerasLayerConfiguration conf, Integer kerasVersion, boolean maskZero) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_EMBEDDING()); @@ -92,6 +78,7 @@ class KerasEmbeddingTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_INPUT_DIM(), inputDim); config.put(conf.getLAYER_FIELD_INPUT_LENGTH(), inputLength); config.put(conf.getLAYER_FIELD_OUTPUT_DIM(), outputDim); + List inputShape = new ArrayList<>(INPUT_SHAPE.length); for (int i : INPUT_SHAPE) { inputShape.add(i); @@ -111,6 +98,7 @@ class KerasEmbeddingTest extends BaseDL4JTest { KerasEmbedding kerasEmbedding = new KerasEmbedding(layerConfig, false); assertEquals(kerasEmbedding.getNumParams(), 1); assertEquals(kerasEmbedding.isZeroMasking(), maskZero); + EmbeddingSequenceLayer layer = kerasEmbedding.getEmbeddingLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); return kerasEmbedding; diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/flatten/KerasFlatten3dTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/flatten/KerasFlatten3dTest.java new file mode 100644 index 000000000..13664e594 --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/flatten/KerasFlatten3dTest.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.layers.flatten; + +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.preprocessor.Cnn3DToFeedForwardPreProcessor; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.graph.vertex.GraphVertex; +import org.deeplearning4j.nn.graph.vertex.impl.PreprocessorVertex; +import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; +import org.junit.jupiter.api.Test; +import org.nd4j.common.io.ClassPathResource; + +import java.io.InputStream; + +import static org.junit.jupiter.api.Assertions.*; + +public class KerasFlatten3dTest { + + + @Test + public void testFlatten3d() throws Exception { + ClassPathResource classPathResource = new ClassPathResource("modelimport/keras/weights/flatten_3d.hdf5"); + try(InputStream inputStream = classPathResource.getInputStream()) { + ComputationGraph computationGraph = KerasModelImport.importKerasModelAndWeights(inputStream); + assertNotNull(computationGraph); + assertEquals(3,computationGraph.getVertices().length); + GraphVertex[] vertices = computationGraph.getVertices(); + assertTrue(vertices[1] instanceof PreprocessorVertex); + PreprocessorVertex preprocessorVertex = (PreprocessorVertex) vertices[1]; + InputPreProcessor preProcessor = preprocessorVertex.getPreProcessor(); + assertTrue(preProcessor instanceof Cnn3DToFeedForwardPreProcessor); + Cnn3DToFeedForwardPreProcessor cnn3DToFeedForwardPreProcessor = (Cnn3DToFeedForwardPreProcessor) preProcessor; + assertTrue(cnn3DToFeedForwardPreProcessor.isNCDHW()); + assertEquals(10,cnn3DToFeedForwardPreProcessor.getInputDepth()); + assertEquals(10,cnn3DToFeedForwardPreProcessor.getInputHeight()); + assertEquals(1,cnn3DToFeedForwardPreProcessor.getNumChannels()); + assertEquals(10,cnn3DToFeedForwardPreProcessor.getInputWidth()); + System.out.println(cnn3DToFeedForwardPreProcessor); + } + } + +} diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1DTest.java similarity index 86% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1DTest.java index 868df51cb..1b2d9dfd7 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected1DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.local; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -29,70 +30,49 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Locally Connected 1 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasLocallyConnected1DTest extends BaseDL4JTest { +public class KerasLocallyConnected1DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "test_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final WeightInit INIT_DL4J = WeightInit.XAVIER; - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - private final int KERNEL_SIZE = 2; - private final int STRIDE = 3; - private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - private final int VALID_PADDING = 0; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Locally Connected 2 D Layer") - void testLocallyConnected2DLayer() throws Exception { + public void testLocallyConnected2DLayer() throws Exception { buildLocallyConnected2DLayer(conf1, keras1); buildLocallyConnected2DLayer(conf2, keras2); } - private void buildLocallyConnected2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + + private void buildLocallyConnected2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_LOCALLY_CONNECTED_2D()); Map config = new HashMap<>(); @@ -111,34 +91,34 @@ class KerasLocallyConnected1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_W_REGULARIZER(), W_reg); config.put(conf.getLAYER_FIELD_DROPOUT(), DROPOUT_KERAS); if (kerasVersion == 2) { - ArrayList kernel = new ArrayList() { - - { - add(KERNEL_SIZE); - } - }; + ArrayList kernel = new ArrayList() {{ + add(KERNEL_SIZE); + }}; config.put(conf.getLAYER_FIELD_FILTER_LENGTH(), kernel); } else { config.put(conf.getLAYER_FIELD_FILTER_LENGTH(), KERNEL_SIZE); } - if (kerasVersion == 2) { - ArrayList stride = new ArrayList() { - { - add(STRIDE); - } - }; + if (kerasVersion == 2) { + ArrayList stride = new ArrayList() {{ + add(STRIDE); + }}; config.put(conf.getLAYER_FIELD_SUBSAMPLE_LENGTH(), stride); } else { config.put(conf.getLAYER_FIELD_SUBSAMPLE_LENGTH(), STRIDE); } + config.put(conf.getLAYER_FIELD_NB_FILTER(), N_OUT); config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + + KerasLocallyConnected1D kerasLocal = new KerasLocallyConnected1D(layerConfig); + // once get output type is triggered, inputshape, output shape and input depth are updated - kerasLocal.getOutputType(InputType.recurrent(3, 4)); + kerasLocal.getOutputType(InputType.recurrent(3, 4)); + LocallyConnected1D layer = kerasLocal.getLocallyConnected1DLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivation().toString().toLowerCase()); assertEquals(LAYER_NAME, layer.getLayerName()); @@ -151,7 +131,9 @@ class KerasLocallyConnected1DTest extends BaseDL4JTest { assertEquals(N_OUT, layer.getNOut()); assertEquals(ConvolutionMode.Truncate, layer.getCm()); assertEquals(VALID_PADDING, layer.getPadding()); + assertEquals(layer.getInputSize(), 4); assertEquals(layer.getNIn(), 3); } } + diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2DTest.java similarity index 83% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2DTest.java index 92aadddfa..b703a482b 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/local/KerasLocallyConnected2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.local; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -29,74 +30,52 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Locally Connected 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasLocallyConnected2DTest extends BaseDL4JTest { +public class KerasLocallyConnected2DTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "test_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final WeightInit INIT_DL4J = WeightInit.XAVIER; - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - - private final int[] KERNEL_SIZE = new int[] { 1, 2 }; - - private final int[] DILATION = new int[] { 2, 2 }; - - private final int[] STRIDE = new int[] { 3, 4 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2}; + private final int[] DILATION = new int[]{2, 2}; + private final int[] STRIDE = new int[]{3, 4}; private final int N_OUT = 13; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Locally Connected 2 D Layer") - void testLocallyConnected2DLayer() throws Exception { + public void testLocallyConnected2DLayer() throws Exception { buildLocallyConnected2DLayer(conf1, keras1); buildLocallyConnected2DLayer(conf2, keras2); } - private void buildLocallyConnected2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { + + private void buildLocallyConnected2DLayer(KerasLayerConfiguration conf, Integer kerasVersion) + throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_LOCALLY_CONNECTED_2D()); Map config = new HashMap<>(); @@ -118,14 +97,12 @@ class KerasLocallyConnected2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_NB_ROW(), KERNEL_SIZE[0]); config.put(conf.getLAYER_FIELD_NB_COL(), KERNEL_SIZE[1]); } else { - ArrayList kernel = new ArrayList() { - - { - for (int i : KERNEL_SIZE) add(i); - } - }; + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_KERNEL_SIZE(), kernel); } + List subsampleList = new ArrayList<>(); subsampleList.add(STRIDE[0]); subsampleList.add(STRIDE[1]); @@ -134,9 +111,13 @@ class KerasLocallyConnected2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + + KerasLocallyConnected2D kerasLocal = new KerasLocallyConnected2D(layerConfig); + // once get output type is triggered, inputshape, output shape and input depth are updated - kerasLocal.getOutputType(InputType.convolutional(4, 4, 3)); + kerasLocal.getOutputType(InputType.convolutional(4,4,3)); + LocallyConnected2D layer = kerasLocal.getLocallyConnected2DLayer(); assertEquals(ACTIVATION_DL4J, layer.getActivation().toString().toLowerCase()); assertEquals(LAYER_NAME, layer.getLayerName()); @@ -149,7 +130,9 @@ class KerasLocallyConnected2DTest extends BaseDL4JTest { assertEquals(N_OUT, layer.getNOut()); assertEquals(ConvolutionMode.Truncate, layer.getCm()); assertArrayEquals(VALID_PADDING, layer.getPadding()); - assertArrayEquals(layer.getInputSize(), new int[] { 4, 4 }); + + assertArrayEquals(layer.getInputSize(), new int[] {4, 4}); assertEquals(layer.getNIn(), 3); } } + diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropoutTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropoutTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropoutTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropoutTest.java index 68616bf9c..fa3a2feae 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropoutTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasAlphaDropoutTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.noise; import org.deeplearning4j.nn.conf.dropout.AlphaDropout; @@ -25,46 +26,35 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Alpha Dropout Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasAlphaDropoutTest extends BaseDL4JTest { +public class KerasAlphaDropoutTest extends BaseDL4JTest { String LAYER_NAME = "alpha_dropout"; - private final double RATE_KERAS = 0.3; - private final double RATE_DL4J = 1 - RATE_KERAS; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Alpha Dropout Layer") - void testAlphaDropoutLayer() throws Exception { + public void testAlphaDropoutLayer() throws Exception { buildAlphaDropoutLayer(conf1, keras1); buildAlphaDropoutLayer(conf2, keras2); } + private void buildAlphaDropoutLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_DROPOUT()); @@ -73,8 +63,10 @@ class KerasAlphaDropoutTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_RATE(), RATE_KERAS); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + DropoutLayer layer = new KerasAlphaDropout(layerConfig).getAlphaDropoutLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(new AlphaDropout(RATE_DL4J), layer.getIDropout()); } + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropoutTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropoutTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropoutTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropoutTest.java index e8b19dddc..e23356da2 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropoutTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianDropoutTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.noise; import org.deeplearning4j.nn.conf.dropout.GaussianDropout; @@ -25,46 +26,35 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Gaussian Dropout Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasGaussianDropoutTest extends BaseDL4JTest { +public class KerasGaussianDropoutTest extends BaseDL4JTest { String LAYER_NAME = "gaussian_dropout"; - private final double RATE_KERAS = 0.3; - private final double RATE_DL4J = 1 - RATE_KERAS; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Gaussian Dropout Layer") - void testGaussianDropoutLayer() throws Exception { + public void testGaussianDropoutLayer() throws Exception { buildGaussianDropoutLayer(conf1, keras1); buildGaussianDropoutLayer(conf2, keras2); } + private void buildGaussianDropoutLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_DROPOUT()); @@ -73,8 +63,10 @@ class KerasGaussianDropoutTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_RATE(), RATE_KERAS); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + DropoutLayer layer = new KerasGaussianDropout(layerConfig).getGaussianDropoutLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(new GaussianDropout(RATE_DL4J), layer.getIDropout()); } + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoiseTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoiseTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoiseTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoiseTest.java index 838a9fb2e..4eb0042b5 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoiseTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/noise/KerasGaussianNoiseTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.noise; import org.deeplearning4j.nn.conf.dropout.GaussianNoise; @@ -25,44 +26,34 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Gaussian Noise Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasGaussianNoiseTest extends BaseDL4JTest { +public class KerasGaussianNoiseTest extends BaseDL4JTest { String LAYER_NAME = "gaussian_noise"; - private final double STDDEV = 0.3; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Gaussian Noise Layer") - void testGaussianNoiseLayer() throws Exception { + public void testGaussianNoiseLayer() throws Exception { buildGaussianNoiseLayer(conf1, keras1); buildGaussianNoiseLayer(conf2, keras2); } + private void buildGaussianNoiseLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_DROPOUT()); @@ -71,8 +62,10 @@ class KerasGaussianNoiseTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_GAUSSIAN_VARIANCE(), STDDEV); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + DropoutLayer layer = new KerasGaussianNoise(layerConfig).getGaussianNoiseLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(new GaussianNoise(STDDEV), layer.getIDropout()); } + } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalizationTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalizationTest.java similarity index 88% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalizationTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalizationTest.java index 9f887c58e..d8341de8f 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalizationTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/normalization/KerasBatchNormalizationTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.normalization; import org.deeplearning4j.nn.conf.layers.BatchNormalization; @@ -24,50 +25,41 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** * @author Max Pumperla */ -@DisplayName("Keras Batch Normalization Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasBatchNormalizationTest extends BaseDL4JTest { - +public class KerasBatchNormalizationTest extends BaseDL4JTest { public static final String PARAM_NAME_BETA = "beta"; - private final String LAYER_NAME = "batch_norm_layer"; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); + @Test - @DisplayName("Test Batchnorm Layer") - void testBatchnormLayer() throws Exception { + public void testBatchnormLayer() throws Exception { buildBatchNormalizationLayer(conf1, keras1); buildBatchNormalizationLayer(conf2, keras2); } + private void buildBatchNormalizationLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { double epsilon = 1E-5; double momentum = 0.99; + KerasBatchNormalization batchNormalization = new KerasBatchNormalization(kerasVersion); + Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_BATCHNORMALIZATION()); Map config = new HashMap<>(); @@ -80,21 +72,25 @@ class KerasBatchNormalizationTest extends BaseDL4JTest { config.put(batchNormalization.getLAYER_FIELD_AXIS(), 3); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + BatchNormalization layer = new KerasBatchNormalization(layerConfig).getBatchNormalizationLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(epsilon, layer.getEps(), 0.0); assertEquals(momentum, layer.getDecay(), 0.0); + } @Test - @DisplayName("Test Set Weights") - void testSetWeights() throws Exception { + public void testSetWeights() throws Exception { Map weights = weightsWithoutGamma(); KerasBatchNormalization batchNormalization = new KerasBatchNormalization(keras2); + batchNormalization.setScale(false); batchNormalization.setWeights(weights); + int size = batchNormalization.getWeights().size(); assertEquals(4, size); + } private Map weightsWithoutGamma() { diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1DTest.java similarity index 79% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1DTest.java index a7163d1cf..25557e595 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling1DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.pooling; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -26,76 +27,56 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Pooling 1 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasPooling1DTest extends BaseDL4JTest { +public class KerasPooling1DTest extends BaseDL4JTest { private final String LAYER_NAME = "test_layer"; - - private final int[] KERNEL_SIZE = new int[] { 2 }; - - private final int[] STRIDE = new int[] { 4 }; - + private final int[] KERNEL_SIZE = new int[]{2}; + private final int[] STRIDE = new int[]{4}; private final PoolingType POOLING_TYPE = PoolingType.MAX; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Pooling 1 D Layer") - void testPooling1DLayer() throws Exception { + public void testPooling1DLayer() throws Exception { buildPooling1DLayer(conf1, keras1); buildPooling1DLayer(conf2, keras2); } + private void buildPooling1DLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_MAX_POOLING_1D()); Map config = new HashMap<>(); config.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); if (kerasVersion == 2) { - ArrayList kernel = new ArrayList() { - - { - for (int i : KERNEL_SIZE) add(i); - } - }; + ArrayList kernel = new ArrayList() {{ + for (int i : KERNEL_SIZE) add(i); + }}; config.put(conf.getLAYER_FIELD_POOL_1D_SIZE(), kernel); } else { config.put(conf.getLAYER_FIELD_POOL_1D_SIZE(), KERNEL_SIZE[0]); } - if (kerasVersion == 2) { - ArrayList stride = new ArrayList() { - { - for (int i : STRIDE) add(i); - } - }; + if (kerasVersion == 2) { + ArrayList stride = new ArrayList() {{ + for (int i : STRIDE) add(i); + }}; config.put(conf.getLAYER_FIELD_POOL_1D_STRIDES(), stride); } else { config.put(conf.getLAYER_FIELD_POOL_1D_STRIDES(), STRIDE[0]); @@ -103,6 +84,7 @@ class KerasPooling1DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Subsampling1DLayer layer = new KerasPooling1D(layerConfig).getSubsampling1DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(KERNEL_SIZE[0], layer.getKernelSize()[0]); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2DTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2DTest.java index 1b1f2ed6f..189cea1da 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling2DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.pooling; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -26,51 +27,35 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Pooling 2 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasPooling2DTest extends BaseDL4JTest { +public class KerasPooling2DTest extends BaseDL4JTest { private final String LAYER_NAME = "test_layer"; - - private final int[] KERNEL_SIZE = new int[] { 1, 2 }; - - private final int[] STRIDE = new int[] { 3, 4 }; - + private final int[] KERNEL_SIZE = new int[]{1, 2}; + private final int[] STRIDE = new int[]{3, 4}; private final PoolingType POOLING_TYPE = PoolingType.MAX; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Pooling 2 D Layer") - void testPooling2DLayer() throws Exception { + public void testPooling2DLayer() throws Exception { buildPooling2DLayer(conf1, keras1); buildPooling2DLayer(conf2, keras2); } @@ -91,6 +76,7 @@ class KerasPooling2DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + SubsamplingLayer layer = new KerasPooling2D(layerConfig).getSubsampling2DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertArrayEquals(KERNEL_SIZE, layer.getKernelSize()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3DTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3DTest.java similarity index 85% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3DTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3DTest.java index e877387ae..eefba12b4 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3DTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/pooling/KerasPooling3DTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.pooling; import org.deeplearning4j.nn.conf.ConvolutionMode; @@ -26,51 +27,35 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Pooling 3 D Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasPooling3DTest extends BaseDL4JTest { +public class KerasPooling3DTest extends BaseDL4JTest { private final String LAYER_NAME = "pooling_3d"; - - private final int[] KERNEL_SIZE = new int[] { 2, 2, 2 }; - - private final int[] STRIDE = new int[] { 1, 1, 1 }; - + private final int[] KERNEL_SIZE = new int[]{2, 2, 2}; + private final int[] STRIDE = new int[]{1, 1, 1}; private final PoolingType POOLING_TYPE = PoolingType.MAX; - private final String BORDER_MODE_VALID = "valid"; - - private final int[] VALID_PADDING = new int[] { 0, 0, 0 }; + private final int[] VALID_PADDING = new int[]{0, 0, 0}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Pooling 3 D Layer") - void testPooling3DLayer() throws Exception { + public void testPooling3DLayer() throws Exception { buildPooling3DLayer(conf1, keras1); buildPooling3DLayer(conf2, keras2); } @@ -93,6 +78,7 @@ class KerasPooling3DTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_BORDER_MODE(), BORDER_MODE_VALID); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + Subsampling3DLayer layer = new KerasPooling3D(layerConfig).getSubsampling3DLayer(); assertEquals(LAYER_NAME, layer.getLayerName()); assertArrayEquals(KERNEL_SIZE, layer.getKernelSize()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTMTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTMTest.java similarity index 90% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTMTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTMTest.java index 7f41eff19..b88f3c94c 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTMTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLSTMTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.recurrent; import org.deeplearning4j.nn.conf.dropout.Dropout; @@ -35,62 +36,40 @@ import org.deeplearning4j.nn.modelimport.keras.layers.embeddings.KerasEmbedding; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras LSTM Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasLSTMTest extends BaseDL4JTest { +public class KerasLSTMTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "lstm_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - private final int N_OUT = 13; - private Boolean[] returnSequences = new Boolean[] { true, false }; - - private Boolean[] maskZero = new Boolean[] { true, false }; - + private Boolean[] returnSequences = new Boolean[]{true, false}; + private Boolean[] maskZero = new Boolean[]{true, false}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Lstm Layer") - void testLstmLayer() throws Exception { + public void testLstmLayer() throws Exception { for (Boolean rs : returnSequences) { buildLstmLayer(conf1, keras1, rs); buildLstmLayer(conf2, keras2, rs); @@ -106,6 +85,7 @@ class KerasLSTMTest extends BaseDL4JTest { double lstmForgetBiasDouble = 1.0; String lstmForgetBiasString = "one"; boolean lstmUnroll = true; + Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_LSTM()); Map config = new HashMap<>(); @@ -115,6 +95,7 @@ class KerasLSTMTest extends BaseDL4JTest { if (kerasVersion == 1) { config.put(conf.getLAYER_FIELD_INNER_INIT(), INIT_KERAS); config.put(conf.getLAYER_FIELD_INIT(), INIT_KERAS); + } else { Map init = new HashMap<>(); init.put("class_name", conf.getINIT_GLOROT_NORMAL()); @@ -126,6 +107,7 @@ class KerasLSTMTest extends BaseDL4JTest { W_reg.put(conf.getREGULARIZATION_TYPE_L2(), L2_REGULARIZATION); config.put(conf.getLAYER_FIELD_W_REGULARIZER(), W_reg); config.put(conf.getLAYER_FIELD_RETURN_SEQUENCES(), rs); + config.put(conf.getLAYER_FIELD_DROPOUT_W(), DROPOUT_KERAS); config.put(conf.getLAYER_FIELD_DROPOUT_U(), 0.0); config.put(conf.getLAYER_FIELD_FORGET_BIAS_INIT(), lstmForgetBiasString); @@ -133,6 +115,7 @@ class KerasLSTMTest extends BaseDL4JTest { config.put(conf.getLAYER_FIELD_UNROLL(), lstmUnroll); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + LSTM layer; LastTimeStep lts; KerasLSTM kerasLstm = new KerasLSTM(layerConfig); @@ -154,12 +137,15 @@ class KerasLSTMTest extends BaseDL4JTest { assertEquals(new Dropout(DROPOUT_DL4J), layer.getIDropout()); assertEquals(lstmForgetBiasDouble, layer.getForgetGateBiasInit(), 0.0); assertEquals(N_OUT, layer.getNOut()); + } - private void buildMaskZeroLstmLayer(KerasLayerConfiguration conf, Integer kerasVersion, Boolean maskZero) throws Exception { + private void buildMaskZeroLstmLayer(KerasLayerConfiguration conf, Integer kerasVersion, Boolean maskZero) + throws Exception { String innerActivation = "hard_sigmoid"; String lstmForgetBiasString = "one"; boolean lstmUnroll = true; + Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_LSTM()); Map config = new HashMap<>(); @@ -169,6 +155,7 @@ class KerasLSTMTest extends BaseDL4JTest { if (kerasVersion == 1) { config.put(conf.getLAYER_FIELD_INNER_INIT(), INIT_KERAS); config.put(conf.getLAYER_FIELD_INIT(), INIT_KERAS); + } else { Map init = new HashMap<>(); init.put("class_name", conf.getINIT_GLOROT_NORMAL()); @@ -179,22 +166,28 @@ class KerasLSTMTest extends BaseDL4JTest { W_reg.put(conf.getREGULARIZATION_TYPE_L1(), L1_REGULARIZATION); W_reg.put(conf.getREGULARIZATION_TYPE_L2(), L2_REGULARIZATION); config.put(conf.getLAYER_FIELD_W_REGULARIZER(), W_reg); + config.put(conf.getLAYER_FIELD_DROPOUT_W(), DROPOUT_KERAS); config.put(conf.getLAYER_FIELD_DROPOUT_U(), 0.0); config.put(conf.getLAYER_FIELD_FORGET_BIAS_INIT(), lstmForgetBiasString); config.put(conf.getLAYER_FIELD_OUTPUT_DIM(), N_OUT); config.put(conf.getLAYER_FIELD_UNROLL(), lstmUnroll); config.put(conf.getLAYER_FIELD_RETURN_SEQUENCES(), true); + layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); - layerConfig.put(conf.getLAYER_FIELD_INBOUND_NODES(), Arrays.asList(Arrays.asList(Arrays.asList("embedding")))); + layerConfig.put(conf.getLAYER_FIELD_INBOUND_NODES(), + Arrays.asList(Arrays.asList( + Arrays.asList("embedding")))); KerasEmbedding embedding = getEmbedding(maskZero); Map previousLayers = Collections.singletonMap("embedding", embedding); + KerasLSTM kerasLstm = new KerasLSTM(layerConfig, previousLayers); Assertions.assertEquals(kerasLstm.getLayer() instanceof MaskZeroLayer, maskZero); } - private KerasEmbedding getEmbedding(boolean maskZero) throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { + private KerasEmbedding getEmbedding(boolean maskZero) + throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { KerasEmbedding embedding = new KerasEmbedding(); embedding.setZeroMasking(maskZero); return embedding; diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnnTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnnTest.java similarity index 87% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnnTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnnTest.java index f5a00b62e..e68627e3e 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnnTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasSimpleRnnTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.recurrent; import org.deeplearning4j.nn.conf.dropout.Dropout; @@ -29,56 +30,36 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.weights.IWeightInit; import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; /** * @author Max Pumperla */ -@DisplayName("Keras Simple Rnn Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasSimpleRnnTest extends BaseDL4JTest { +public class KerasSimpleRnnTest extends BaseDL4JTest { private final String ACTIVATION = "sigmoid"; - private final String LAYER_NAME = "simple_rnn_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final IWeightInit INIT_DL4J = new WeightInitXavier(); - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - private final int N_OUT = 13; - private Boolean[] returnSequences = new Boolean[] { true, false }; - + private Boolean[] returnSequences = new Boolean[]{true, false}; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Simple Rnn Layer") - void testSimpleRnnLayer() throws Exception { + public void testSimpleRnnLayer() throws Exception { for (Boolean rs : returnSequences) { buildSimpleRnnLayer(conf1, keras1, rs); buildSimpleRnnLayer(conf2, keras2, rs); @@ -86,6 +67,7 @@ class KerasSimpleRnnTest extends BaseDL4JTest { } private void buildSimpleRnnLayer(KerasLayerConfiguration conf, Integer kerasVersion, Boolean rs) throws Exception { + Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_LSTM()); Map config = new HashMap<>(); @@ -94,6 +76,7 @@ class KerasSimpleRnnTest extends BaseDL4JTest { if (kerasVersion == 1) { config.put(conf.getLAYER_FIELD_INNER_INIT(), INIT_KERAS); config.put(conf.getLAYER_FIELD_INIT(), INIT_KERAS); + } else { Map init = new HashMap<>(); init.put("class_name", conf.getINIT_GLOROT_NORMAL()); @@ -105,13 +88,17 @@ class KerasSimpleRnnTest extends BaseDL4JTest { W_reg.put(conf.getREGULARIZATION_TYPE_L2(), L2_REGULARIZATION); config.put(conf.getLAYER_FIELD_W_REGULARIZER(), W_reg); config.put(conf.getLAYER_FIELD_RETURN_SEQUENCES(), rs); + config.put(conf.getLAYER_FIELD_DROPOUT_W(), DROPOUT_KERAS); config.put(conf.getLAYER_FIELD_DROPOUT_U(), 0.0); config.put(conf.getLAYER_FIELD_OUTPUT_DIM(), N_OUT); config.put(conf.getLAYER_FIELD_UNROLL(), true); layerConfig.put(conf.getLAYER_FIELD_CONFIG(), config); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); - SimpleRnn layer = rs ? (SimpleRnn) new KerasSimpleRnn(layerConfig).getSimpleRnnLayer() : (SimpleRnn) ((LastTimeStep) new KerasSimpleRnn(layerConfig).getSimpleRnnLayer()).getUnderlying(); + + + SimpleRnn layer = rs ? (SimpleRnn) new KerasSimpleRnn(layerConfig).getSimpleRnnLayer() : + (SimpleRnn) ((LastTimeStep) new KerasSimpleRnn(layerConfig).getSimpleRnnLayer()).getUnderlying(); assertEquals(ACTIVATION, layer.getActivationFn().toString()); assertEquals(LAYER_NAME, layer.getLayerName()); assertEquals(INIT_DL4J, layer.getWeightInitFn()); diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectionalTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectionalTest.java similarity index 88% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectionalTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectionalTest.java index a3db5fc1f..7073a6cba 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectionalTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/wrappers/KerasBidirectionalTest.java @@ -17,6 +17,7 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelimport.keras.layers.wrappers; import org.deeplearning4j.nn.conf.layers.LSTM; @@ -26,59 +27,38 @@ import org.deeplearning4j.nn.modelimport.keras.config.Keras1LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.Keras2LayerConfiguration; import org.deeplearning4j.nn.modelimport.keras.config.KerasLayerConfiguration; import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; + import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; /** * @author Max Pumperla */ -@DisplayName("Keras Bidirectional Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasBidirectionalTest extends BaseDL4JTest { +public class KerasBidirectionalTest extends BaseDL4JTest { private final String ACTIVATION_KERAS = "linear"; - private final String ACTIVATION_DL4J = "identity"; - private final String LAYER_NAME = "bidirectional_layer"; - private final String INIT_KERAS = "glorot_normal"; - private final WeightInit INIT_DL4J = WeightInit.XAVIER; - private final double L1_REGULARIZATION = 0.01; - private final double L2_REGULARIZATION = 0.02; - private final double DROPOUT_KERAS = 0.3; - private final double DROPOUT_DL4J = 1 - DROPOUT_KERAS; - private final int N_OUT = 13; - private final String mode = "sum"; private Integer keras1 = 1; - private Integer keras2 = 2; - private Keras1LayerConfiguration conf1 = new Keras1LayerConfiguration(); - private Keras2LayerConfiguration conf2 = new Keras2LayerConfiguration(); @Test - @DisplayName("Test Lstm Layer") - void testLstmLayer() throws Exception { + public void testLstmLayer() throws Exception { buildLstmLayer(conf1, keras1); buildLstmLayer(conf2, keras2); } @@ -86,17 +66,17 @@ class KerasBidirectionalTest extends BaseDL4JTest { private void buildLstmLayer(KerasLayerConfiguration conf, Integer kerasVersion) throws Exception { String innerActivation = "hard_sigmoid"; String lstmForgetBiasString = "one"; + Map layerConfig = new HashMap<>(); layerConfig.put(conf.getLAYER_FIELD_CLASS_NAME(), conf.getLAYER_CLASS_NAME_LSTM()); Map lstmConfig = new HashMap<>(); - // keras linear -> dl4j identity - lstmConfig.put(conf.getLAYER_FIELD_ACTIVATION(), ACTIVATION_KERAS); - // keras linear -> dl4j identity - lstmConfig.put(conf.getLAYER_FIELD_INNER_ACTIVATION(), innerActivation); + lstmConfig.put(conf.getLAYER_FIELD_ACTIVATION(), ACTIVATION_KERAS); // keras linear -> dl4j identity + lstmConfig.put(conf.getLAYER_FIELD_INNER_ACTIVATION(), innerActivation); // keras linear -> dl4j identity lstmConfig.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); if (kerasVersion == 1) { lstmConfig.put(conf.getLAYER_FIELD_INNER_INIT(), INIT_KERAS); lstmConfig.put(conf.getLAYER_FIELD_INIT(), INIT_KERAS); + } else { Map init = new HashMap<>(); init.put("class_name", conf.getINIT_GLOROT_NORMAL()); @@ -108,23 +88,31 @@ class KerasBidirectionalTest extends BaseDL4JTest { W_reg.put(conf.getREGULARIZATION_TYPE_L2(), L2_REGULARIZATION); lstmConfig.put(conf.getLAYER_FIELD_W_REGULARIZER(), W_reg); lstmConfig.put(conf.getLAYER_FIELD_RETURN_SEQUENCES(), true); + lstmConfig.put(conf.getLAYER_FIELD_DROPOUT_W(), DROPOUT_KERAS); lstmConfig.put(conf.getLAYER_FIELD_DROPOUT_U(), 0.0); lstmConfig.put(conf.getLAYER_FIELD_FORGET_BIAS_INIT(), lstmForgetBiasString); lstmConfig.put(conf.getLAYER_FIELD_OUTPUT_DIM(), N_OUT); lstmConfig.put(conf.getLAYER_FIELD_UNROLL(), true); + Map innerRnnConfig = new HashMap<>(); innerRnnConfig.put("class_name", "LSTM"); innerRnnConfig.put("config", lstmConfig); + Map innerConfig = new HashMap<>(); innerConfig.put("merge_mode", mode); innerConfig.put("layer", innerRnnConfig); innerConfig.put(conf.getLAYER_FIELD_NAME(), LAYER_NAME); + layerConfig.put("config", innerConfig); layerConfig.put(conf.getLAYER_FIELD_KERAS_VERSION(), kerasVersion); + KerasBidirectional kerasBidirectional = new KerasBidirectional(layerConfig); Bidirectional layer = kerasBidirectional.getBidirectionalLayer(); + assertEquals(Bidirectional.Mode.ADD, layer.getMode()); - assertEquals(Activation.HARDSIGMOID.toString().toLowerCase(), ((LSTM) kerasBidirectional.getUnderlyingRecurrentLayer()).getGateActivationFn().toString()); + assertEquals(Activation.HARDSIGMOID.toString().toLowerCase(), + ((LSTM) kerasBidirectional.getUnderlyingRecurrentLayer()).getGateActivationFn().toString()); + } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/optimizers/OptimizerImport.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/optimizers/OptimizerImport.java similarity index 94% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/optimizers/OptimizerImport.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/optimizers/OptimizerImport.java index 298df7ca2..380e93a52 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/optimizers/OptimizerImport.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/optimizers/OptimizerImport.java @@ -25,19 +25,14 @@ import org.deeplearning4j.nn.modelimport.keras.KerasModel; import org.deeplearning4j.nn.modelimport.keras.KerasSequentialModel; import org.deeplearning4j.nn.modelimport.keras.utils.KerasModelBuilder; import org.deeplearning4j.common.util.DL4JFileUtils; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.StandardCopyOption; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag + public class OptimizerImport extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorImportTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorImportTest.java similarity index 88% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorImportTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorImportTest.java index df8ffb1fc..bfe75b3c9 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorImportTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorImportTest.java @@ -22,12 +22,9 @@ package org.deeplearning4j.nn.modelimport.keras.preprocessing.sequence; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.IOException; @@ -36,16 +33,12 @@ import java.io.IOException; * * @author Max Pumperla */ -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag +@Timeout(300000) public class TimeSeriesGeneratorImportTest extends BaseDL4JTest { - @Test() - @Timeout(300000) + @Test public void importTimeSeriesTest() throws IOException, InvalidKerasConfigurationException { String path = "modelimport/keras/preprocessing/timeseries_generator.json"; - TimeSeriesGenerator gen = TimeSeriesGenerator.fromJson(Resources.asFile(path).getAbsolutePath()); } } diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorTest.java similarity index 93% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorTest.java index 5a2de5dfb..91c86ba81 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/sequence/TimeSeriesGeneratorTest.java @@ -22,18 +22,13 @@ package org.deeplearning4j.nn.modelimport.keras.preprocessing.sequence; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Pair; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag + public class TimeSeriesGeneratorTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerImportTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerImportTest.java similarity index 91% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerImportTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerImportTest.java index b2de46143..b40a3511c 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerImportTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerImportTest.java @@ -22,12 +22,9 @@ package org.deeplearning4j.nn.modelimport.keras.preprocessing.text; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.IOException; @@ -38,16 +35,13 @@ import static org.junit.jupiter.api.Assertions.*; * * @author Max Pumperla */ -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag +@Timeout(300000) public class TokenizerImportTest extends BaseDL4JTest { ClassLoader classLoader = getClass().getClassLoader(); - @Test() - @Timeout(300000) + @Test public void importTest() throws IOException, InvalidKerasConfigurationException { String path = "modelimport/keras/preprocessing/tokenizer.json"; @@ -63,9 +57,7 @@ public class TokenizerImportTest extends BaseDL4JTest { } - - @Test() - @Timeout(300000) + @Test public void importNumWordsNullTest() throws IOException, InvalidKerasConfigurationException { String path = "modelimport/keras/preprocessing/tokenizer_num_words_null.json"; diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerTest.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerTest.java similarity index 95% rename from deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerTest.java rename to cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerTest.java index 0e7699c2d..6e421b404 100644 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerTest.java +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/preprocessing/text/TokenizerTest.java @@ -21,10 +21,7 @@ package org.deeplearning4j.nn.modelimport.keras.preprocessing.text; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import java.util.HashMap; @@ -38,9 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; * * @author Max Pumperla */ -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag public class TokenizerTest extends BaseDL4JTest { @Test diff --git a/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/weights/KerasWeightSettingTests.java b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/weights/KerasWeightSettingTests.java new file mode 100644 index 000000000..9de2cb73a --- /dev/null +++ b/cavis-dnn/cavis-dnn-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/weights/KerasWeightSettingTests.java @@ -0,0 +1,365 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.modelimport.keras.weights; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.modelimport.keras.KerasLayer; +import org.deeplearning4j.nn.modelimport.keras.KerasModel; +import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasSpaceToDepth; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.resources.Resources; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +public class KerasWeightSettingTests extends BaseDL4JTest { + + @TempDir + private File testDir; + + @Override + public long getTimeoutMilliseconds() { + return 9999999L; + } + + @Test + public void testSimpleLayersWithWeights() throws Exception { + int[] kerasVersions = new int[]{1, 2}; + String[] backends = new String[]{"tensorflow", "theano"}; + + for (int version : kerasVersions) { + for (String backend : backends) { + String densePath = "modelimport/keras/weights/dense_" + backend + "_" + version + ".h5"; + importDense(densePath); + + String conv2dPath = "modelimport/keras/weights/conv2d_" + backend + "_" + version + ".h5"; + importConv2D(conv2dPath); + + if (version == 2 && backend.equals("tensorflow")) { // TODO should work for theano + String conv2dReshapePath = "modelimport/keras/weights/conv2d_reshape_" + + backend + "_" + version + ".h5"; + System.out.println(backend + "_" + version); + importConv2DReshape(conv2dReshapePath); + } + + if (version == 2) { + String conv1dFlattenPath = "modelimport/keras/weights/embedding_conv1d_flatten_" + + backend + "_" + version + ".h5"; + importConv1DFlatten(conv1dFlattenPath); + } + + String lstmPath = "modelimport/keras/weights/lstm_" + backend + "_" + version + ".h5"; + importLstm(lstmPath); + + String embeddingLstmPath = "modelimport/keras/weights/embedding_lstm_" + + backend + "_" + version + ".h5"; + importEmbeddingLstm(embeddingLstmPath); + + + if (version == 2) { + String embeddingConv1dExtendedPath = "modelimport/keras/weights/embedding_conv1d_extended_" + + backend + "_" + version + ".h5"; + importEmbeddingConv1DExtended(embeddingConv1dExtendedPath); + } + + if (version == 2) { + String embeddingConv1dPath = "modelimport/keras/weights/embedding_conv1d_" + + backend + "_" + version + ".h5"; + importEmbeddingConv1D(embeddingConv1dPath); + } + + String simpleRnnPath = "modelimport/keras/weights/simple_rnn_" + backend + "_" + version + ".h5"; + importSimpleRnn(simpleRnnPath); + + String bidirectionalLstmPath = "modelimport/keras/weights/bidirectional_lstm_" + + backend + "_" + version + ".h5"; + importBidirectionalLstm(bidirectionalLstmPath); + + String bidirectionalLstmNoSequencesPath = + "modelimport/keras/weights/bidirectional_lstm_no_return_sequences_" + + backend + "_" + version + ".h5"; + importBidirectionalLstm(bidirectionalLstmNoSequencesPath); + + if (version == 2 && backend.equals("tensorflow")) { + String batchToConv2dPath = "modelimport/keras/weights/batch_to_conv2d_" + + backend + "_" + version + ".h5"; + importBatchNormToConv2D(batchToConv2dPath); + } + + if (backend.equals("tensorflow") && version == 2) { // TODO should work for theano + String simpleSpaceToBatchPath = "modelimport/keras/weights/space_to_depth_simple_" + + backend + "_" + version + ".h5"; + importSimpleSpaceToDepth(simpleSpaceToBatchPath); + } + + if (backend.equals("tensorflow") && version == 2) { + String graphSpaceToBatchPath = "modelimport/keras/weights/space_to_depth_graph_" + + backend + "_" + version + ".h5"; + importGraphSpaceToDepth(graphSpaceToBatchPath); + } + + if (backend.equals("tensorflow") && version == 2) { + String sepConvPath = "modelimport/keras/weights/sepconv2d_" + backend + "_" + version + ".h5"; + importSepConv2D(sepConvPath); + } + } + } + } + + private void logSuccess(String modelPath) { + log.info("***** Successfully imported " + modelPath); + } + + private void importDense(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, true); + + INDArray weights = model.getLayer(0).getParam("W"); + val weightShape = weights.shape(); + assertEquals(4, weightShape[0]); + assertEquals(6, weightShape[1]); + + INDArray bias = model.getLayer(0).getParam("b"); + assertEquals(6, bias.length()); + logSuccess(modelPath); + } + + private void importSepConv2D(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + + INDArray depthWeights = model.getLayer(0).getParam("W"); + val depthWeightShape = depthWeights.shape(); + + long depthMult = 2; + long kernel = 3; + long nIn = 5; + long nOut = 6; + + assertEquals(depthMult, depthWeightShape[0]); + assertEquals(nIn, depthWeightShape[1]); + assertEquals(kernel, depthWeightShape[2]); + assertEquals(kernel, depthWeightShape[3]); + + INDArray weights = model.getLayer(0).getParam("pW"); + val weightShape = weights.shape(); + + + assertEquals(nOut, weightShape[0]); + assertEquals(nIn * depthMult, weightShape[1]); + assertEquals(1, weightShape[2]); + assertEquals(1, weightShape[3]); + + INDArray bias = model.getLayer(0).getParam("b"); + assertEquals(6, bias.length()); + + INDArray input = Nd4j.ones(1, 3, 4, 5); //NHWC + INDArray output = model.output(input); + + assertArrayEquals(new long[] {1, 1, 2, 6}, output.shape()); //NHWC + + logSuccess(modelPath); + } + + private void importConv2D(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + + INDArray weights = model.getLayer(0).getParam("W"); + val weightShape = weights.shape(); + assertEquals(6, weightShape[0]); + assertEquals(5, weightShape[1]); + assertEquals(3, weightShape[2]); + assertEquals(3, weightShape[3]); + + INDArray bias = model.getLayer(0).getParam("b"); + assertEquals(6,bias.length()); + logSuccess(modelPath); + } + + + private void importConv2DReshape(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + + + int nOut = 12; + int mb = 10; + ; + int[] inShape = new int[]{5, 5, 5}; + INDArray input = Nd4j.zeros(mb, inShape[0], inShape[1], inShape[2]); + INDArray output = model.output(input); + assertArrayEquals(new long[]{mb, nOut}, output.shape()); + logSuccess(modelPath); + } + + private void importConv1DFlatten(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + + int nOut = 6; + int inputLength = 10; + int mb = 42; + int kernel = 3; + + INDArray input = Nd4j.zeros(mb, inputLength); + INDArray output = model.output(input); + if(modelPath.contains("tensorflow")) + assertArrayEquals(new long[]{mb, inputLength - kernel + 1, nOut}, output.shape()); //NWC + else if(modelPath.contains("theano")) { + assertArrayEquals(new long[]{mb, nOut,inputLength - kernel + 1}, output.shape()); //NCW + + } + logSuccess(modelPath); + } + + private void importBatchNormToConv2D(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + model.summary(); + logSuccess(modelPath); + } + + private void importSimpleSpaceToDepth(String modelPath) throws Exception { + KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + + INDArray input = Nd4j.zeros(10, 6, 6, 4); + INDArray output = model.output(input); + assertArrayEquals(new long[]{10, 3, 3, 16}, output.shape()); + logSuccess(modelPath); + } + + private void importGraphSpaceToDepth(String modelPath) throws Exception { + KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); + ComputationGraph model = loadComputationalGraph(modelPath, false); + +// INDArray input[] = new INDArray[]{Nd4j.zeros(10, 4, 6, 6), Nd4j.zeros(10, 16, 3, 3)}; + INDArray input[] = new INDArray[]{Nd4j.zeros(10, 6, 6, 4), Nd4j.zeros(10, 3, 3, 16)}; + INDArray[] output = model.output(input); + log.info(Arrays.toString(output[0].shape())); + assertArrayEquals(new long[]{10, 3, 3, 32}, output[0].shape()); + logSuccess(modelPath); + } + + private void importLstm(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + model.summary(); + // TODO: check weights + logSuccess(modelPath); + } + + private void importEmbeddingLstm(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + + int nIn = 4; + int nOut = 6; + int outputDim = 5; + int inputLength = 10; + int mb = 42; + + INDArray embeddingWeight = model.getLayer(0).getParam("W"); + val embeddingWeightShape = embeddingWeight.shape(); + assertEquals(nIn, embeddingWeightShape[0]); + assertEquals(outputDim, embeddingWeightShape[1]); + + INDArray inEmbedding = Nd4j.zeros(mb, inputLength); + INDArray output = model.output(inEmbedding); + assertArrayEquals(new long[]{mb, inputLength, nOut}, output.shape()); //NWC format + logSuccess(modelPath); + } + + private void importEmbeddingConv1DExtended(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + logSuccess(modelPath); + } + + private void importEmbeddingConv1D(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + + int nIn = 4; + int nOut = 6; + int outputDim = 5; + int inputLength = 10; + int kernel = 3; + int mb = 42; + + INDArray embeddingWeight = model.getLayer(0).getParam("W"); + val embeddingWeightShape = embeddingWeight.shape(); + assertEquals(nIn, embeddingWeightShape[0]); + assertEquals(outputDim, embeddingWeightShape[1]); + + INDArray inEmbedding = Nd4j.zeros(mb, inputLength); + INDArray output = model.output(inEmbedding); + if(modelPath.contains("tensorflow")) + assertArrayEquals(new long[]{mb, inputLength - kernel + 1, nOut}, output.shape()); //NWC + else if(modelPath.contains("theano")) + assertArrayEquals(new long[]{mb, nOut,inputLength - kernel + 1}, output.shape()); //NCC + + logSuccess(modelPath); + } + + private void importSimpleRnn(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + model.summary(); + logSuccess(modelPath); + // TODO: check weights + } + + private void importBidirectionalLstm(String modelPath) throws Exception { + MultiLayerNetwork model = loadMultiLayerNetwork(modelPath, false); + model.summary(); + logSuccess(modelPath); + // TODO: check weights + } + + private MultiLayerNetwork loadMultiLayerNetwork(String modelPath, boolean training) throws Exception { + File modelFile = createTempFile("temp", ".h5"); + try(InputStream is = Resources.asStream(modelPath)) { + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + return new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(training).buildSequential().getMultiLayerNetwork(); + } + } + + private ComputationGraph loadComputationalGraph(String modelPath, boolean training) throws Exception { + File modelFile = createTempFile("temp", ".h5"); + try(InputStream is = Resources.asStream(modelPath)) { + Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + return new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) + .enforceTrainingConfig(training).buildModel().getComputationGraph(); + } + } + + private File createTempFile(String prefix, String suffix) throws IOException { + return new File(testDir.getAbsolutePath() + File.pathSeparator + prefix + "-" + System.nanoTime() + suffix); + } + +} diff --git a/cavis-dnn/cavis-dnn-nlp/build.gradle b/cavis-dnn/cavis-dnn-nlp/build.gradle new file mode 100644 index 000000000..7e4aecfe3 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nlp/build.gradle @@ -0,0 +1,56 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisDnn.cavisDnnApi + implementation "commons-lang:commons-lang:2.6" + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisNative.cavisNativeCommon + implementation "org.threadly:threadly:4.10.0" + testImplementation "org.mockito:mockito-core:3.3.3" + testImplementation 'ch.qos.logback:logback-classic' + implementation "org.apache.commons:commons-lang3" + implementation "commons-codec:commons-codec" + implementation "org.apache.commons:commons-compress" + implementation "com.github.vinhkhuc:jfasttext:0.4" + testImplementation 'org.hamcrest:hamcrest:2.2' + testImplementation projects.cavisDnn.cavisDnnCommonTests + implementation "com.google.guava:guava" + implementation "com.google.code.gson:gson:2.8.6" + implementation "org.slf4j:slf4j-api" + implementation "org.apache.commons:commons-math3" + implementation "commons-io:commons-io" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "it.unimi.dsi:fastutil:8.1.1" + testImplementation projects.cavisDnn.cavisDnnCommonTests + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testImplementation "com.sun.jna:jna:3.0.9" + + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BaseTextVectorizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BaseTextVectorizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BaseTextVectorizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/BaseTextVectorizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/DefaultInputStreamCreator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/DefaultInputStreamCreator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/DefaultInputStreamCreator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/DefaultInputStreamCreator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TextVectorizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TextVectorizer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TextVectorizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TextVectorizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/BertIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/BertIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/BertIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/BertIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/CnnSentenceDataSetIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/CnnSentenceDataSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/CnnSentenceDataSetIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/CnnSentenceDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/LabeledPairSentenceProvider.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/LabeledPairSentenceProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/LabeledPairSentenceProvider.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/LabeledPairSentenceProvider.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/LabeledSentenceProvider.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/LabeledSentenceProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/LabeledSentenceProvider.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/LabeledSentenceProvider.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertMaskedLMMasker.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertMaskedLMMasker.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertMaskedLMMasker.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertMaskedLMMasker.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertSequenceMasker.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertSequenceMasker.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertSequenceMasker.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/bert/BertSequenceMasker.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledPairSentenceProvider.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledPairSentenceProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledPairSentenceProvider.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledPairSentenceProvider.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledSentenceProvider.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledSentenceProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledSentenceProvider.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/provider/CollectionLabeledSentenceProvider.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/FileLabeledSentenceProvider.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/provider/FileLabeledSentenceProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/FileLabeledSentenceProvider.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/provider/FileLabeledSentenceProvider.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/LabelAwareConverter.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/provider/LabelAwareConverter.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/iterator/provider/LabelAwareConverter.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/iterator/provider/LabelAwareConverter.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/WeightLookupTable.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/WeightLookupTable.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/WeightLookupTable.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/WeightLookupTable.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTable.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTable.java similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTable.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTable.java index 6672f5756..c58995ddd 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTable.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTable.java @@ -20,7 +20,7 @@ package org.deeplearning4j.models.embeddings.inmemory; -import org.nd4j.shade.guava.util.concurrent.AtomicDouble; +import com.google.common.util.concurrent.AtomicDouble; import lombok.Getter; import lombok.NonNull; import lombok.Setter; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/ElementsLearningAlgorithm.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/ElementsLearningAlgorithm.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/ElementsLearningAlgorithm.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/ElementsLearningAlgorithm.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/SequenceLearningAlgorithm.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/SequenceLearningAlgorithm.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/SequenceLearningAlgorithm.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/SequenceLearningAlgorithm.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchItem.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchItem.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchItem.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchItem.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchSequences.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchSequences.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchSequences.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/BatchSequences.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/CBOW.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/CBOW.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/CBOW.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/CBOW.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/SkipGram.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/SkipGram.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/SkipGram.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/elements/SkipGram.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DBOW.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DBOW.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DBOW.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DBOW.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DM.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DM.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DM.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/learning/impl/sequence/DM.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/VectorsConfiguration.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/VectorsConfiguration.java similarity index 93% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/VectorsConfiguration.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/VectorsConfiguration.java index 0dd6c28dc..c613671db 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/VectorsConfiguration.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/VectorsConfiguration.java @@ -22,10 +22,10 @@ package org.deeplearning4j.models.embeddings.loader; import lombok.Data; import org.apache.commons.codec.binary.Base64; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import java.io.IOException; import java.io.Serializable; @@ -110,7 +110,7 @@ public class VectorsConfiguration implements Serializable { That's ugly method, but its way more memory-friendly then loading whole 10GB json file just to create another 10GB memory array. */ return mapper.writeValueAsString(this); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java old mode 100755 new mode 100644 similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java index 861bd79a6..a2e3fb8c6 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java @@ -67,10 +67,10 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.ops.transforms.Transforms; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import org.nd4j.storage.CompressedRamStorage; import java.io.BufferedInputStream; @@ -2609,14 +2609,8 @@ public class WordVectorSerializer { String tokenPreProcessorClassName = configuration.getTokenPreProcessor(); if (StringUtils.isNotEmpty(tokenPreProcessorClassName)) { - Object preProcessor = DL4JClassLoading.createNewInstance(tokenizerFactoryClassName); - if(preProcessor instanceof TokenPreProcess) { - TokenPreProcess tokenPreProcess = (TokenPreProcess) preProcessor; - factory.setTokenPreProcessor(tokenPreProcess); - } - else { - log.warn("Found instance of {}, was not actually a pre processor. Ignoring.",tokenPreProcessorClassName); - } + TokenPreProcess preProcessor = DL4JClassLoading.createNewInstance(tokenizerFactoryClassName); + factory.setTokenPreProcessor(preProcessor); } return factory; @@ -2674,7 +2668,7 @@ public class WordVectorSerializer { Nd4j.getMemoryManager().setOccasionalGcFrequency(50000); CompressedRamStorage storage = new CompressedRamStorage.Builder().useInplaceCompression(false) - .setCompressor(new NoOp()).emulateIsAbsent(false).build(); + .setCompressor(new NoOp()).emulateIsAbsent(false).build(); VocabCache vocabCache = new AbstractCache.Builder().build(); @@ -2950,7 +2944,7 @@ public class WordVectorSerializer { public static void writeLookupTable(WeightLookupTable weightLookupTable, @NonNull File file) throws IOException { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), - StandardCharsets.UTF_8))) { + StandardCharsets.UTF_8))) { int numWords = weightLookupTable.getVocabCache().numWords(); int layersSize = weightLookupTable.layerSize(); long totalNumberOfDocs = weightLookupTable.getVocabCache().totalNumberOfDocs(); @@ -3065,8 +3059,8 @@ public class WordVectorSerializer { * @return Word2Vec */ public static Word2Vec readWord2Vec( - @NonNull InputStream stream, - boolean readExtendedTable) throws IOException { + @NonNull InputStream stream, + boolean readExtendedTable) throws IOException { SequenceVectors vectors = readSequenceVectors(stream, readExtendedTable); Word2Vec word2Vec = new Word2Vec @@ -3109,7 +3103,7 @@ public class WordVectorSerializer { * * @param path File */ - public static FastText readWordVectors(File path) { + public static FastText readWordVectors(File path) { FastText result = null; try { FileInputStream fileIn = new FileInputStream(path); @@ -3118,7 +3112,7 @@ public class WordVectorSerializer { result = (FastText) in.readObject(); } catch (ClassNotFoundException ex) { - } + } } catch (FileNotFoundException ex) { ex.printStackTrace(); } catch (IOException ex) { @@ -3156,8 +3150,8 @@ public class WordVectorSerializer { } /** - * Helper static methods to read data from input stream. - */ + * Helper static methods to read data from input stream. + */ public static class ReadHelper { /** * Read a float from a data input stream Credit to: diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/ModelUtils.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/ModelUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/ModelUtils.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/ModelUtils.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/BasicModelUtils.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/BasicModelUtils.java similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/BasicModelUtils.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/BasicModelUtils.java index d4a05b242..39f8a3df0 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/BasicModelUtils.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/BasicModelUtils.java @@ -20,7 +20,7 @@ package org.deeplearning4j.models.embeddings.reader.impl; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.collect.Lists; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NonNull; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/FlatModelUtils.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/FlatModelUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/FlatModelUtils.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/FlatModelUtils.java diff --git a/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/TreeModelUtils.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/TreeModelUtils.java new file mode 100644 index 000000000..e2707ff0b --- /dev/null +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/reader/impl/TreeModelUtils.java @@ -0,0 +1,120 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.models.embeddings.reader.impl; + +import lombok.NonNull; +import org.deeplearning4j.clustering.sptree.DataPoint; +import org.deeplearning4j.clustering.vptree.VPTree; +import org.deeplearning4j.models.embeddings.WeightLookupTable; +import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; +import org.nd4j.common.util.SetUtils; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.*; + +public class TreeModelUtils extends BasicModelUtils { + protected VPTree vpTree; + + @Override + public void init(@NonNull WeightLookupTable lookupTable) { + super.init(lookupTable); + vpTree = null; + } + + protected synchronized void checkTree() { + // build new tree if it wasn't created before + if (vpTree == null) { + List points = new ArrayList<>(); + for (String word : vocabCache.words()) { + points.add(new DataPoint(vocabCache.indexOf(word), lookupTable.vector(word))); + } + vpTree = new VPTree(points); + } + } + + + /** + * This method returns nearest words for target word, based on tree structure. + * This method is recommended to use if you're going to call for nearest words multiple times. + * VPTree will be built upon firt call to this method + * + * @param label label of element we're looking nearest words to + * @param n number of nearest elements to return + * @return + */ + @Override + public Collection wordsNearest(String label, int n) { + if (!vocabCache.hasToken(label)) + return new ArrayList<>(); + + Collection collection = wordsNearest(Arrays.asList(label), new ArrayList(), n + 1); + if (collection.contains(label)) + collection.remove(label); + + return collection; + } + + @Override + public Collection wordsNearest(Collection positive, Collection negative, int top) { + + // Check every word is in the model + for (String p : SetUtils.union(new HashSet<>(positive), new HashSet<>(negative))) { + if (!vocabCache.containsWord(p)) { + return new ArrayList<>(); + } + } + + INDArray words = Nd4j.create(positive.size() + negative.size(), lookupTable.layerSize()); + int row = 0; + for (String s : positive) { + words.putRow(row++, lookupTable.vector(s)); + } + + for (String s : negative) { + words.putRow(row++, lookupTable.vector(s).mul(-1)); + } + + INDArray mean = words.isMatrix() ? words.mean(0) : words; + + return wordsNearest(mean, top); + } + + @Override + public Collection wordsNearest(INDArray words, int top) { + checkTree(); + words = adjustRank(words); + + List add = new ArrayList<>(); + List distances = new ArrayList<>(); + + // we need n+1 to address original datapoint removal + vpTree.search(words, top, add, distances); + + Collection ret = new ArrayList<>(); + for (DataPoint e : add) { + String word = vocabCache.wordAtIndex(e.getIndex()); + ret.add(word); + } + + return super.wordsNearest(words, top); + } +} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectors.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectors.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectors.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectors.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImpl.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImpl.java similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImpl.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImpl.java index 9af125e54..fb5156441 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImpl.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImpl.java @@ -20,7 +20,7 @@ package org.deeplearning4j.models.embeddings.wordvectors; -import org.nd4j.shade.guava.util.concurrent.AtomicDouble; +import com.google.common.util.concurrent.AtomicDouble; import lombok.Getter; import lombok.NonNull; import lombok.Setter; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTLossFunctions.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTLossFunctions.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTLossFunctions.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTLossFunctions.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTModels.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTModels.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTModels.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTModels.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTOptions.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTOptions.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTOptions.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/fasttext/FTOptions.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FastText.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/fasttext/FastText.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/fasttext/FastText.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/fasttext/FastText.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/node2vec/Node2Vec.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/node2vec/Node2Vec.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/node2vec/Node2Vec.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/node2vec/Node2Vec.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java index 8ba2b12f2..c4558c331 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectors.java @@ -20,7 +20,7 @@ package org.deeplearning4j.models.paragraphvectors; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.collect.Lists; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import lombok.Getter; @@ -58,10 +58,10 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Counter; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import org.threadly.concurrent.PriorityScheduler; import org.threadly.concurrent.TaskPriority; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/Consumer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/Consumer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/Consumer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/Consumer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/PCService.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/PCService.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/PCService.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/PCService.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java index ccd4f376c..c6752ae75 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/SequenceVectors.java @@ -22,8 +22,8 @@ package org.deeplearning4j.models.sequencevectors; import org.apache.commons.lang3.StringUtils; import org.deeplearning4j.common.config.DL4JClassLoading; -import org.nd4j.shade.guava.primitives.Ints; -import org.nd4j.shade.guava.util.concurrent.AtomicDouble; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.AtomicDouble; import lombok.Getter; import lombok.NonNull; import lombok.Setter; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/enums/ListenerEvent.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/enums/ListenerEvent.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/enums/ListenerEvent.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/enums/ListenerEvent.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/NoEdgeHandling.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/NoEdgeHandling.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/NoEdgeHandling.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/NoEdgeHandling.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/PopularityMode.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/PopularityMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/PopularityMode.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/PopularityMode.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SamplingMode.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SamplingMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SamplingMode.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SamplingMode.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SpreadSpectrum.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SpreadSpectrum.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SpreadSpectrum.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/SpreadSpectrum.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkDirection.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkDirection.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkDirection.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkDirection.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkMode.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkMode.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/enums/WalkMode.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/NoEdgesException.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/NoEdgesException.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/NoEdgesException.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/NoEdgesException.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/ParseException.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/ParseException.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/ParseException.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/exception/ParseException.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/BinaryTree.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/BinaryTree.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/BinaryTree.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/BinaryTree.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/GraphHuffman.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/GraphHuffman.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/GraphHuffman.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/huffman/GraphHuffman.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Edge.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Edge.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Edge.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Edge.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Graph.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Graph.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Graph.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Graph.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/IGraph.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/IGraph.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/IGraph.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/IGraph.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Vertex.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Vertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Vertex.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/primitives/Vertex.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/AbstractVertexFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/AbstractVertexFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/AbstractVertexFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/AbstractVertexFactory.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/VertexFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/VertexFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/VertexFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/vertex/VertexFactory.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/GraphWalker.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/GraphWalker.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/GraphWalker.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/GraphWalker.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/NearestVertexWalker.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/NearestVertexWalker.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/NearestVertexWalker.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/NearestVertexWalker.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/PopularityWalker.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/PopularityWalker.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/PopularityWalker.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/PopularityWalker.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalker.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalker.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalker.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalker.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalker.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalker.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalker.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalker.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceElementFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceElementFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceElementFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceElementFactory.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/SequenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/VectorsListener.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/VectorsListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/VectorsListener.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/interfaces/VectorsListener.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/AbstractSequenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/AbstractSequenceIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/AbstractSequenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/AbstractSequenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/FilteredSequenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/FilteredSequenceIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/FilteredSequenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/FilteredSequenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/SynchronizedSequenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/SynchronizedSequenceIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/SynchronizedSequenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/iterators/SynchronizedSequenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/ScoreListener.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/ScoreListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/ScoreListener.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/ScoreListener.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SerializingListener.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SerializingListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SerializingListener.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SerializingListener.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SimilarityListener.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SimilarityListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SimilarityListener.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/listeners/SimilarityListener.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/Sequence.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/Sequence.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/Sequence.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/Sequence.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/SequenceElement.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/SequenceElement.java similarity index 96% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/SequenceElement.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/SequenceElement.java index 46ae3fbc3..d62f84099 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/SequenceElement.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/SequenceElement.java @@ -20,18 +20,18 @@ package org.deeplearning4j.models.sequencevectors.sequence; -import org.nd4j.shade.guava.util.concurrent.AtomicDouble; +import com.google.common.util.concurrent.AtomicDouble; import lombok.Getter; import lombok.NonNull; import lombok.Setter; import org.deeplearning4j.models.word2vec.VocabWord; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.util.HashUtil; -import org.nd4j.shade.jackson.annotation.*; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import java.io.Serializable; import java.util.ArrayList; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/ShallowSequenceElement.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/ShallowSequenceElement.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/ShallowSequenceElement.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/sequence/ShallowSequenceElement.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactory.java similarity index 95% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactory.java index 497b712b0..15a434749 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactory.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactory.java @@ -23,7 +23,7 @@ package org.deeplearning4j.models.sequencevectors.serialization; import lombok.NonNull; import org.deeplearning4j.models.sequencevectors.interfaces.SequenceElementFactory; import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +78,7 @@ public class AbstractElementFactory implements Sequen ObjectMapper mapper = SequenceElement.mapper(); try { json = mapper.writeValueAsString(element); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactory.java similarity index 97% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactory.java index ca816be84..0c10b3f84 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactory.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactory.java @@ -23,7 +23,7 @@ package org.deeplearning4j.models.sequencevectors.serialization; import org.deeplearning4j.models.sequencevectors.interfaces.SequenceElementFactory; import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; import org.deeplearning4j.models.word2vec.VocabWord; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/SequenceTransformer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/SequenceTransformer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/SequenceTransformer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/SequenceTransformer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/SentenceTransformer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/SentenceTransformer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/SentenceTransformer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/SentenceTransformer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/BasicTransformerIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/BasicTransformerIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/BasicTransformerIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/BasicTransformerIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Huffman.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/Huffman.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Huffman.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/Huffman.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/InputStreamCreator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/InputStreamCreator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/InputStreamCreator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/InputStreamCreator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/StaticWord2Vec.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/StaticWord2Vec.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/StaticWord2Vec.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/StaticWord2Vec.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/StreamWork.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/StreamWork.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/StreamWork.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/StreamWork.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWord.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWord.java old mode 100755 new mode 100644 similarity index 95% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWord.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWord.java index 921318d21..b4c815c38 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWord.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWord.java @@ -24,9 +24,9 @@ import lombok.Getter; import lombok.NonNull; import lombok.Setter; import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.Serializable; @@ -156,7 +156,7 @@ public class VocabWord extends SequenceElement implements Serializable { we need JSON as single line to save it at first line of the CSV model file */ return mapper.writeValueAsString(this); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWork.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWork.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWork.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/VocabWork.java diff --git a/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java new file mode 100644 index 000000000..62354d69e --- /dev/null +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java @@ -0,0 +1,717 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.models.word2vec; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import lombok.Getter; +import lombok.NonNull; +import org.deeplearning4j.models.embeddings.WeightLookupTable; +import org.deeplearning4j.models.embeddings.learning.ElementsLearningAlgorithm; +import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration; +import org.deeplearning4j.models.embeddings.reader.ModelUtils; +import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; +import org.deeplearning4j.models.sequencevectors.SequenceVectors; +import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator; +import org.deeplearning4j.models.sequencevectors.interfaces.VectorsListener; +import org.deeplearning4j.models.sequencevectors.iterators.AbstractSequenceIterator; +import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; +import org.deeplearning4j.text.documentiterator.DocumentIterator; +import org.deeplearning4j.text.documentiterator.LabelAwareIterator; +import org.deeplearning4j.text.sentenceiterator.SentenceIterator; +import org.deeplearning4j.text.sentenceiterator.StreamLineIterator; +import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; +import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.type.CollectionType; + +import java.io.IOException; +import java.util.*; + +public class Word2Vec extends SequenceVectors { + private static final long serialVersionUID = 78249242142L; + + protected transient SentenceIterator sentenceIter; + @Getter + protected transient TokenizerFactory tokenizerFactory; + + /** + * This method defines TokenizerFactory instance to be using during model building + * + * @param tokenizerFactory TokenizerFactory instance + */ + public void setTokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) { + this.tokenizerFactory = tokenizerFactory; + + if (sentenceIter != null) { + SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(sentenceIter) + .tokenizerFactory(this.tokenizerFactory).build(); + this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build(); + } + } + + /** + * This method defines SentenceIterator instance, that will be used as training corpus source + * + * @param iterator SentenceIterator instance + */ + public void setSentenceIterator(@NonNull SentenceIterator iterator) { + //if (tokenizerFactory == null) throw new IllegalStateException("Please call setTokenizerFactory() prior to setSentenceIter() call."); + + if (tokenizerFactory != null) { + SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(iterator) + .tokenizerFactory(tokenizerFactory) + .allowMultithreading(configuration == null || configuration.isAllowParallelTokenization()) + .build(); + this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build(); + } else + log.error("Please call setTokenizerFactory() prior to setSentenceIter() call."); + } + + /** + * This method defines SequenceIterator instance, that will be used as training corpus source. + * Main difference with other iterators here: it allows you to pass already tokenized Sequence for training + * + * @param iterator + */ + public void setSequenceIterator(@NonNull SequenceIterator iterator) { + this.iterator = iterator; + } + + private static ObjectMapper mapper = null; + private static final Object lock = new Object(); + + private static ObjectMapper mapper() { + if (mapper == null) { + synchronized (lock) { + if (mapper == null) { + mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + return mapper; + } + } + } + return mapper; + } + + private static final String CLASS_FIELD = "@class"; + private static final String VOCAB_LIST_FIELD = "VocabCache"; + + public String toJson() throws JsonProcessingException { + + JsonObject retVal = new JsonObject(); + ObjectMapper mapper = mapper(); + + retVal.addProperty(CLASS_FIELD, mapper.writeValueAsString(this.getClass().getName())); + + if (this.vocab instanceof AbstractCache) { + retVal.addProperty(VOCAB_LIST_FIELD, ((AbstractCache) this.vocab).toJson()); + } + + return retVal.toString(); + } + + public static Word2Vec fromJson(String jsonString) throws IOException { + + Word2Vec ret = new Word2Vec(); + + JsonParser parser = new JsonParser(); + JsonObject json = parser.parse(jsonString).getAsJsonObject(); + + VocabCache cache = AbstractCache.fromJson(json.get(VOCAB_LIST_FIELD).getAsString()); + + ret.setVocab(cache); + return ret; + } + + public static class Builder extends SequenceVectors.Builder { + protected SentenceIterator sentenceIterator; + protected LabelAwareIterator labelAwareIterator; + protected TokenizerFactory tokenizerFactory; + protected boolean allowParallelTokenization = true; + + + public Builder() { + + } + + /** + * This method has no effect for Word2Vec + * + * @param vec existing WordVectors model + * @return + */ + @Override + protected Builder useExistingWordVectors(@NonNull WordVectors vec) { + return this; + } + + public Builder(@NonNull VectorsConfiguration configuration) { + super(configuration); + this.allowParallelTokenization = configuration.isAllowParallelTokenization(); + } + + public Builder iterate(@NonNull DocumentIterator iterator) { + this.sentenceIterator = new StreamLineIterator.Builder(iterator).setFetchSize(100).build(); + return this; + } + + /** + * This method used to feed SentenceIterator, that contains training corpus, into ParagraphVectors + * + * @param iterator + * @return + */ + public Builder iterate(@NonNull SentenceIterator iterator) { + this.sentenceIterator = iterator; + return this; + } + + /** + * This method defines TokenizerFactory to be used for strings tokenization during training + * PLEASE NOTE: If external VocabCache is used, the same TokenizerFactory should be used to keep derived tokens equal. + * + * @param tokenizerFactory + * @return + */ + public Builder tokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) { + this.tokenizerFactory = tokenizerFactory; + return this; + } + + /** + * This method used to feed SequenceIterator, that contains training corpus, into ParagraphVectors + * + * @param iterator + * @return + */ + @Override + public Builder iterate(@NonNull SequenceIterator iterator) { + super.iterate(iterator); + return this; + } + + /** + * This method used to feed LabelAwareIterator, that is usually used + * + * @param iterator + * @return + */ + public Builder iterate(@NonNull LabelAwareIterator iterator) { + this.labelAwareIterator = iterator; + return this; + } + + /** + * This method defines mini-batch size + * @param batchSize + * @return + */ + @Override + public Builder batchSize(int batchSize) { + super.batchSize(batchSize); + return this; + } + + /** + * This method defines number of iterations done for each mini-batch during training + * @param iterations + * @return + */ + @Override + public Builder iterations(int iterations) { + super.iterations(iterations); + return this; + } + + /** + * This method defines number of epochs (iterations over whole training corpus) for training + * @param numEpochs + * @return + */ + @Override + public Builder epochs(int numEpochs) { + super.epochs(numEpochs); + return this; + } + + /** + * This method defines number of dimensions for output vectors + * @param layerSize + * @return + */ + @Override + public Builder layerSize(int layerSize) { + super.layerSize(layerSize); + return this; + } + + /** + * This method defines initial learning rate for model training + * + * @param learningRate + * @return + */ + @Override + public Builder learningRate(double learningRate) { + super.learningRate(learningRate); + return this; + } + + /** + * This method defines minimal word frequency in training corpus. All words below this threshold will be removed prior model training + * + * @param minWordFrequency + * @return + */ + @Override + public Builder minWordFrequency(int minWordFrequency) { + super.minWordFrequency(minWordFrequency); + return this; + } + + /** + * This method defines minimal learning rate value for training + * + * @param minLearningRate + * @return + */ + @Override + public Builder minLearningRate(double minLearningRate) { + super.minLearningRate(minLearningRate); + return this; + } + + /** + * This method defines whether model should be totally wiped out prior building, or not + * + * @param reallyReset + * @return + */ + @Override + public Builder resetModel(boolean reallyReset) { + super.resetModel(reallyReset); + return this; + } + + /** + * This method sets vocabulary limit during construction. + * + * Default value: 0. Means no limit + * + * @param limit + * @return + */ + @Override + public Builder limitVocabularySize(int limit) { + super.limitVocabularySize(limit); + return this; + } + + /** + * This method allows to define external VocabCache to be used + * + * @param vocabCache + * @return + */ + @Override + public Builder vocabCache(@NonNull VocabCache vocabCache) { + super.vocabCache(vocabCache); + return this; + } + + /** + * This method allows to define external WeightLookupTable to be used + * + * @param lookupTable + * @return + */ + @Override + public Builder lookupTable(@NonNull WeightLookupTable lookupTable) { + super.lookupTable(lookupTable); + return this; + } + + /** + * This method defines whether subsampling should be used or not + * + * @param sampling set > 0 to subsampling argument, or 0 to disable + * @return + */ + @Override + public Builder sampling(double sampling) { + super.sampling(sampling); + return this; + } + + /** + * This method defines whether adaptive gradients should be used or not + * + * @param reallyUse + * @return + */ + @Override + public Builder useAdaGrad(boolean reallyUse) { + super.useAdaGrad(reallyUse); + return this; + } + + /** + * This method defines whether negative sampling should be used or not + * + * PLEASE NOTE: If you're going to use negative sampling, you might want to disable HierarchicSoftmax, which is enabled by default + * + * Default value: 0 + * + * @param negative set > 0 as negative sampling argument, or 0 to disable + * @return + */ + @Override + public Builder negativeSample(double negative) { + super.negativeSample(negative); + return this; + } + + /** + * This method defines stop words that should be ignored during training + * + * @param stopList + * @return + */ + @Override + public Builder stopWords(@NonNull List stopList) { + super.stopWords(stopList); + return this; + } + + /** + * This method is hardcoded to TRUE, since that's whole point of Word2Vec + * + * @param trainElements + * @return + */ + @Override + public Builder trainElementsRepresentation(boolean trainElements) { + throw new IllegalStateException("You can't change this option for Word2Vec"); + } + + /** + * This method is hardcoded to FALSE, since that's whole point of Word2Vec + * + * @param trainSequences + * @return + */ + @Override + public Builder trainSequencesRepresentation(boolean trainSequences) { + throw new IllegalStateException("You can't change this option for Word2Vec"); + } + + /** + * This method defines stop words that should be ignored during training + * + * @param stopList + * @return + */ + @Override + public Builder stopWords(@NonNull Collection stopList) { + super.stopWords(stopList); + return this; + } + + /** + * This method defines context window size + * + * @param windowSize + * @return + */ + @Override + public Builder windowSize(int windowSize) { + super.windowSize(windowSize); + return this; + } + + /** + * This method defines random seed for random numbers generator + * @param randomSeed + * @return + */ + @Override + public Builder seed(long randomSeed) { + super.seed(randomSeed); + return this; + } + + /** + * This method defines maximum number of concurrent threads available for training + * + * @param numWorkers + * @return + */ + @Override + public Builder workers(int numWorkers) { + super.workers(numWorkers); + return this; + } + + /** + * Sets ModelUtils that gonna be used as provider for utility methods: similarity(), wordsNearest(), accuracy(), etc + * + * @param modelUtils model utils to be used + * @return + */ + @Override + public Builder modelUtils(@NonNull ModelUtils modelUtils) { + super.modelUtils(modelUtils); + return this; + } + + /** + * This method allows to use variable window size. In this case, every batch gets processed using one of predefined window sizes + * + * @param windows + * @return + */ + @Override + public Builder useVariableWindow(int... windows) { + super.useVariableWindow(windows); + return this; + } + + /** + * This method allows you to specify SequenceElement that will be used as UNK element, if UNK is used + * + * @param element + * @return + */ + @Override + public Builder unknownElement(VocabWord element) { + super.unknownElement(element); + return this; + } + + /** + * This method allows you to specify, if UNK word should be used internally + * + * @param reallyUse + * @return + */ + @Override + public Builder useUnknown(boolean reallyUse) { + super.useUnknown(reallyUse); + if (this.unknownElement == null) { + this.unknownElement(new VocabWord(1.0, Word2Vec.DEFAULT_UNK)); + } + return this; + } + + /** + * This method sets VectorsListeners for this SequenceVectors model + * + * @param vectorsListeners + * @return + */ + @Override + public Builder setVectorsListeners(@NonNull Collection> vectorsListeners) { + super.setVectorsListeners(vectorsListeners); + return this; + } + + @Override + public Builder elementsLearningAlgorithm(@NonNull String algorithm) { + super.elementsLearningAlgorithm(algorithm); + return this; + } + + @Override + public Builder elementsLearningAlgorithm(@NonNull ElementsLearningAlgorithm algorithm) { + super.elementsLearningAlgorithm(algorithm); + return this; + } + + /** + * This method enables/disables parallel tokenization. + * + * Default value: TRUE + * @param allow + * @return + */ + public Builder allowParallelTokenization(boolean allow) { + this.allowParallelTokenization = allow; + return this; + } + + /** + * This method ebables/disables periodical vocab truncation during construction + * + * Default value: disabled + * + * @param reallyEnable + * @return + */ + @Override + public Builder enableScavenger(boolean reallyEnable) { + super.enableScavenger(reallyEnable); + return this; + } + + /** + * This method enables/disables Hierarchic softmax + * + * Default value: enabled + * + * @param reallyUse + * @return + */ + @Override + public Builder useHierarchicSoftmax(boolean reallyUse) { + super.useHierarchicSoftmax(reallyUse); + return this; + } + + @Override + public Builder usePreciseWeightInit(boolean reallyUse) { + super.usePreciseWeightInit(reallyUse); + return this; + } + + @Override + public Builder usePreciseMode(boolean reallyUse) { + super.usePreciseMode(reallyUse); + return this; + } + + @Override + public Builder intersectModel(@NonNull SequenceVectors vectors, boolean isLocked) { + super.intersectModel(vectors, isLocked); + return this; + } + + public Word2Vec build() { + presetTables(); + + Word2Vec ret = new Word2Vec(); + + if (sentenceIterator != null) { + if (tokenizerFactory == null) + tokenizerFactory = new DefaultTokenizerFactory(); + + SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(sentenceIterator) + .tokenizerFactory(tokenizerFactory).allowMultithreading(allowParallelTokenization) + .build(); + this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build(); + } + + if (this.labelAwareIterator != null) { + if (tokenizerFactory == null) + tokenizerFactory = new DefaultTokenizerFactory(); + + SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(labelAwareIterator) + .tokenizerFactory(tokenizerFactory).allowMultithreading(allowParallelTokenization) + .build(); + this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build(); + } + + ret.numEpochs = this.numEpochs; + ret.numIterations = this.iterations; + ret.vocab = this.vocabCache; + ret.minWordFrequency = this.minWordFrequency; + ret.learningRate.set(this.learningRate); + ret.minLearningRate = this.minLearningRate; + ret.sampling = this.sampling; + ret.negative = this.negative; + ret.layerSize = this.layerSize; + ret.batchSize = this.batchSize; + ret.learningRateDecayWords = this.learningRateDecayWords; + ret.window = this.window; + ret.resetModel = this.resetModel; + ret.useAdeGrad = this.useAdaGrad; + ret.stopWords = this.stopWords; + ret.workers = this.workers; + ret.useUnknown = this.useUnknown; + ret.unknownElement = this.unknownElement; + ret.variableWindows = this.variableWindows; + ret.seed = this.seed; + ret.enableScavenger = this.enableScavenger; + ret.vocabLimit = this.vocabLimit; + + if (ret.unknownElement == null) + ret.unknownElement = new VocabWord(1.0,SequenceVectors.DEFAULT_UNK); + + + ret.iterator = this.iterator; + ret.lookupTable = this.lookupTable; + ret.tokenizerFactory = this.tokenizerFactory; + ret.modelUtils = this.modelUtils; + + ret.elementsLearningAlgorithm = this.elementsLearningAlgorithm; + ret.sequenceLearningAlgorithm = this.sequenceLearningAlgorithm; + + ret.intersectModel = this.intersectVectors; + ret.lockFactor = this.lockFactor; + + this.configuration.setLearningRate(this.learningRate); + this.configuration.setLayersSize(layerSize); + this.configuration.setHugeModelExpected(hugeModelExpected); + this.configuration.setWindow(window); + this.configuration.setMinWordFrequency(minWordFrequency); + this.configuration.setIterations(iterations); + this.configuration.setSeed(seed); + this.configuration.setBatchSize(batchSize); + this.configuration.setLearningRateDecayWords(learningRateDecayWords); + this.configuration.setMinLearningRate(minLearningRate); + this.configuration.setSampling(this.sampling); + this.configuration.setUseAdaGrad(useAdaGrad); + this.configuration.setNegative(negative); + this.configuration.setEpochs(this.numEpochs); + this.configuration.setStopList(this.stopWords); + this.configuration.setVariableWindows(variableWindows); + this.configuration.setUseHierarchicSoftmax(this.useHierarchicSoftmax); + this.configuration.setPreciseWeightInit(this.preciseWeightInit); + this.configuration.setModelUtils(this.modelUtils.getClass().getCanonicalName()); + this.configuration.setAllowParallelTokenization(this.allowParallelTokenization); + this.configuration.setPreciseMode(this.preciseMode); + + if (tokenizerFactory != null) { + this.configuration.setTokenizerFactory(tokenizerFactory.getClass().getCanonicalName()); + if (tokenizerFactory.getTokenPreProcessor() != null) + this.configuration.setTokenPreProcessor( + tokenizerFactory.getTokenPreProcessor().getClass().getCanonicalName()); + } + + ret.configuration = this.configuration; + + // we hardcode + ret.trainSequenceVectors = false; + ret.trainElementsVectors = true; + + ret.eventListeners = this.vectorsListeners; + + + return ret; + } + } +} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataFetcher.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataFetcher.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataFetcher.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataFetcher.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/HuffmanNode.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/HuffmanNode.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/HuffmanNode.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/HuffmanNode.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabCache.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabCache.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabCache.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabCache.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolder.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolder.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolder.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolder.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyWord.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyWord.java similarity index 92% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyWord.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyWord.java index 70019f125..46c2103af 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyWord.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyWord.java @@ -22,10 +22,10 @@ package org.deeplearning4j.models.word2vec.wordstore; import lombok.Data; import lombok.NonNull; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import java.io.IOException; import java.io.Serializable; @@ -115,7 +115,7 @@ public class VocabularyWord implements Serializable { we need JSON as single line to save it at first line of the CSV model file */ return mapper.writeValueAsString(this); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCache.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCache.java similarity index 98% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCache.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCache.java index 0e8709e93..7c96a3ae6 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCache.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCache.java @@ -28,11 +28,11 @@ import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; import org.deeplearning4j.models.word2vec.VocabWord; import org.deeplearning4j.models.word2vec.wordstore.VocabCache; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.*; -import org.nd4j.shade.jackson.databind.type.CollectionType; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.type.CollectionType; import java.io.IOException; import java.util.*; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/InMemoryLookupCache.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/InMemoryLookupCache.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/InMemoryLookupCache.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/InMemoryLookupCache.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/DocumentIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/DocumentIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/DocumentIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/DocumentIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FileDocumentIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FileDocumentIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FileDocumentIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FileDocumentIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareDocumentIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareDocumentIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareDocumentIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareDocumentIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareIteratorWrapper.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareIteratorWrapper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareIteratorWrapper.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelAwareIteratorWrapper.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelledDocument.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelledDocument.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelledDocument.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelledDocument.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelsSource.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelsSource.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelsSource.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/LabelsSource.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/SimpleLabelAwareIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/SimpleLabelAwareIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/SimpleLabelAwareIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/SimpleLabelAwareIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/interoperability/DocumentIteratorConverter.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/interoperability/DocumentIteratorConverter.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/documentiterator/interoperability/DocumentIteratorConverter.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/documentiterator/interoperability/DocumentIteratorConverter.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/inputsanitation/InputHomogenization.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/inputsanitation/InputHomogenization.java old mode 100755 new mode 100644 similarity index 98% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/inputsanitation/InputHomogenization.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/inputsanitation/InputHomogenization.java index edbf9d0d6..049f0c047 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/inputsanitation/InputHomogenization.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/inputsanitation/InputHomogenization.java @@ -87,7 +87,7 @@ public class InputHomogenization { normalized = normalized.replace("(", ""); normalized = normalized.replace(")", ""); normalized = normalized.replace("“", ""); - normalized = normalized.replace("”", ""); + //normalized = normalized.replace(" quote", ""); normalized = normalized.replace("…", ""); normalized = normalized.replace("|", ""); normalized = normalized.replace("/", ""); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/invertedindex/InvertedIndex.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/invertedindex/InvertedIndex.java similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/invertedindex/InvertedIndex.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/invertedindex/InvertedIndex.java index f89acaa95..1fab7c6db 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/invertedindex/InvertedIndex.java +++ b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/invertedindex/InvertedIndex.java @@ -20,7 +20,7 @@ package org.deeplearning4j.text.invertedindex; -import org.nd4j.shade.guava.base.Function; +import com.google.common.base.Function; import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; import org.nd4j.common.primitives.Pair; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/labels/LabelsProvider.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/labels/LabelsProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/labels/LabelsProvider.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/labels/LabelsProvider.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/ContextLabelRetriever.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/ContextLabelRetriever.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/ContextLabelRetriever.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/ContextLabelRetriever.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Util.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Util.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Util.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Util.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Window.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Window.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Window.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Window.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/WindowConverter.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/WindowConverter.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/WindowConverter.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/WindowConverter.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Windows.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Windows.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Windows.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/Windows.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/WordConverter.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/WordConverter.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/movingwindow/WordConverter.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/movingwindow/WordConverter.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BaseSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BaseSentenceIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BaseSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BaseSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BasicLineIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BasicLineIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BasicLineIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BasicLineIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BasicResultSetIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BasicResultSetIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BasicResultSetIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/BasicResultSetIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/CollectionSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/CollectionSentenceIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/CollectionSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/CollectionSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/FileSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/FileSentenceIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/FileSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/FileSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/LineSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/LineSentenceIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/LineSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/LineSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SentenceIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SentencePreProcessor.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SentencePreProcessor.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SentencePreProcessor.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SentencePreProcessor.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/StreamLineIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/StreamLineIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/StreamLineIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/StreamLineIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SynchronizedSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SynchronizedSentenceIterator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SynchronizedSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/SynchronizedSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/interoperability/SentenceIteratorConverter.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/interoperability/SentenceIteratorConverter.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/interoperability/SentenceIteratorConverter.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/interoperability/SentenceIteratorConverter.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/labelaware/LabelAwareFileSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/labelaware/LabelAwareFileSentenceIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/labelaware/LabelAwareFileSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/labelaware/LabelAwareFileSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/labelaware/LabelAwareSentenceIterator.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/labelaware/LabelAwareSentenceIterator.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/labelaware/LabelAwareSentenceIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/sentenceiterator/labelaware/LabelAwareSentenceIterator.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/stopwords/StopWords.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/stopwords/StopWords.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/stopwords/StopWords.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/stopwords/StopWords.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceStreamTokenizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceStreamTokenizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceStreamTokenizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceStreamTokenizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/DefaultStreamTokenizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/DefaultStreamTokenizer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/DefaultStreamTokenizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/DefaultStreamTokenizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/DefaultTokenizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/DefaultTokenizer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/DefaultTokenizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/DefaultTokenizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/TokenPreProcess.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/TokenPreProcess.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/TokenPreProcess.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/TokenPreProcess.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/Tokenizer.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/Tokenizer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/Tokenizer.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/Tokenizer.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/BertWordPiecePreProcessor.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/BertWordPiecePreProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/BertWordPiecePreProcessor.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/BertWordPiecePreProcessor.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/CommonPreprocessor.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/CommonPreprocessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/CommonPreprocessor.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/CommonPreprocessor.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/CompositePreProcessor.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/CompositePreProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/CompositePreProcessor.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/CompositePreProcessor.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/EndingPreProcessor.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/EndingPreProcessor.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/EndingPreProcessor.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/EndingPreProcessor.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/LowCasePreProcessor.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/LowCasePreProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/LowCasePreProcessor.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/LowCasePreProcessor.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/StringCleaning.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/StringCleaning.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/StringCleaning.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizer/preprocessor/StringCleaning.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/BertWordPieceTokenizerFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/BertWordPieceTokenizerFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/BertWordPieceTokenizerFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/BertWordPieceTokenizerFactory.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/DefaultTokenizerFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/DefaultTokenizerFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/DefaultTokenizerFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/DefaultTokenizerFactory.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactory.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/TokenizerFactory.java b/cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/TokenizerFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/TokenizerFactory.java rename to cavis-dnn/cavis-dnn-nlp/src/main/java/org/deeplearning4j/text/tokenization/tokenizerfactory/TokenizerFactory.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/adjectives b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/adjectives old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/adjectives rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/adjectives diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/affirmative.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/affirmative.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/affirmative.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/affirmative.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/classscores b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/classscores old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/classscores rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/classscores diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/doubt.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/doubt.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/doubt.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/doubt.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/intense.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/intense.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/intense.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/intense.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/negative.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/negative.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/negative.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/negative.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/negativedoc b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/negativedoc old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/negativedoc rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/negativedoc diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/positivedoc b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/positivedoc old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/positivedoc rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/positivedoc diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/weakintense.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/weakintense.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/adverbs/weakintense.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/adverbs/weakintense.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/assets/d3.min.js b/cavis-dnn/cavis-dnn-nlp/src/main/resources/assets/d3.min.js old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/assets/d3.min.js rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/assets/d3.min.js diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/assets/jquery.rest.min.js b/cavis-dnn/cavis-dnn-nlp/src/main/resources/assets/jquery.rest.min.js old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/assets/jquery.rest.min.js rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/assets/jquery.rest.min.js diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/assets/render.js b/cavis-dnn/cavis-dnn-nlp/src/main/resources/assets/render.js old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/assets/render.js rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/assets/render.js diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/org/deeplearning4j/ehcache.xml b/cavis-dnn/cavis-dnn-nlp/src/main/resources/org/deeplearning4j/ehcache.xml old mode 100755 new mode 100644 similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/org/deeplearning4j/ehcache.xml rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/org/deeplearning4j/ehcache.xml index 269b87dbf..7fa866cce --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/org/deeplearning4j/ehcache.xml +++ b/cavis-dnn/cavis-dnn-nlp/src/main/resources/org/deeplearning4j/ehcache.xml @@ -248,7 +248,7 @@ This is the size to allocate the DiskStore for a spool buffer. Writes are made to this area and then asynchronously written to disk. The default size is 30MB. Each spool buffer is used only by its cache. If you get OutOfMemory errors consider - lowering this value. To improve DiskStore performance consider increasing it. INFO level + lowering this value. To improve DiskStore performance consider increasing it. Trace level logging in the DiskStore will show if put back ups are occurring. clearOnFlush: diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/org/deeplearning4j/plot/dropwizard/nearestneighbors/render.ftl b/cavis-dnn/cavis-dnn-nlp/src/main/resources/org/deeplearning4j/plot/dropwizard/nearestneighbors/render.ftl similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/org/deeplearning4j/plot/dropwizard/nearestneighbors/render.ftl rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/org/deeplearning4j/plot/dropwizard/nearestneighbors/render.ftl diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/org/deeplearning4j/plot/dropwizard/render.ftl b/cavis-dnn/cavis-dnn-nlp/src/main/resources/org/deeplearning4j/plot/dropwizard/render.ftl old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/org/deeplearning4j/plot/dropwizard/render.ftl rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/org/deeplearning4j/plot/dropwizard/render.ftl diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/render/dropwizard.yml b/cavis-dnn/cavis-dnn-nlp/src/main/resources/render/dropwizard.yml old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/render/dropwizard.yml rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/render/dropwizard.yml diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/adjectives b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/adjectives old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/adjectives rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/adjectives diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/affirmative.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/affirmative.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/affirmative.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/affirmative.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/classscores b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/classscores old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/classscores rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/classscores diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/doubt.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/doubt.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/doubt.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/doubt.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/intense.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/intense.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/intense.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/intense.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/negative.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/negative.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/negative.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/negative.csv diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/negativedoc b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/negativedoc old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/negativedoc rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/negativedoc diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/positivedoc b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/positivedoc old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/positivedoc rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/positivedoc diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/sentiwordnet.txt b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/sentiwordnet.txt old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/sentiwordnet.txt rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/sentiwordnet.txt diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/weakintense.csv b/cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/weakintense.csv old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/resources/sentiment/weakintense.csv rename to cavis-dnn/cavis-dnn-nlp/src/main/resources/sentiment/weakintense.csv diff --git a/cavis-dnn/cavis-dnn-nlp/src/main/resources/stopwords.txt b/cavis-dnn/cavis-dnn-nlp/src/main/resources/stopwords.txt new file mode 100644 index 000000000..f64dfcc52 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nlp/src/main/resources/stopwords.txt @@ -0,0 +1,194 @@ +a +----s +act +"the +"The +about +above +after +again +against +all +am +an +and +any +are +aren't +as +at +be +because +been +before +being +below +between +both +but +by +can't +cannot +could +couldn't +did +didn't +do +does +doesn't +doing +don't +down +during +each +few +for +from +further +had +hadn't +has +hasn't +have +haven't +having +he +he'd +he'll +he's +her +here +here's +hers +herself +him +himself +his +how +how's +i +i'd +i'll +i'm +i've +if +in +into +is +isn't +it +it's +its +itself +let's +me +more +most +mustn't +my +myself +no +nor +not +of +off +on +once +only +or +other +ought +our +ours +ourselves +out +over +own +put +same +shan't +she +she'd +she'll +she's +should +somebody +something +shouldn't +so +some +such +take +than +that +that's +the +their +theirs +them +themselves +then +there +there's +these +they +they'd +they'll +they're +they've +this +those +through +to +too +under +until +up +very +was +wasn't +we +we'd +we'll +we're +we've +were +weren't +what +what's +when +when's +where +where's +which +while +who +who's +whom +why +why's +will +with +without +won't +would +wouldn't +you +you'd +you'll +you're +you've +your +yours +yourself +yourselves +. +? +! +, ++ += +also +- +; +: diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/AssertTestsExtendBaseClass.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/AssertTestsExtendBaseClass.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/AssertTestsExtendBaseClass.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/AssertTestsExtendBaseClass.java diff --git a/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/TsneTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/TsneTest.java new file mode 100644 index 000000000..7edb8ea3e --- /dev/null +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/TsneTest.java @@ -0,0 +1,65 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.time.StopWatch; +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.deeplearning4j.nn.conf.WorkspaceMode; + + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.common.primitives.Pair; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class TsneTest extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return 180000L; + } + + @TempDir + public File testDir; + + @Override + public DataType getDataType() { + return DataType.FLOAT; + } + + @Override + public DataType getDefaultFPDataType() { + return DataType.FLOAT; + } + +} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizerTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizerTest.java old mode 100755 new mode 100644 similarity index 85% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizerTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizerTest.java index 8e7be9518..fa0f1c6dc --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizerTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/BagOfWordsVectorizerTest.java @@ -26,12 +26,9 @@ import lombok.val; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMax; import org.deeplearning4j.models.word2vec.VocabWord; import org.deeplearning4j.models.word2vec.wordstore.VocabCache; @@ -47,26 +44,29 @@ import org.nd4j.common.util.SerializationUtils; import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** *@author Adam Gibson */ @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag +@Timeout(60) public class BagOfWordsVectorizerTest extends BaseDL4JTest { - @Test() - @Timeout(60000L) - public void testBagOfWordsVectorizer(@TempDir Path testDir) throws Exception { - val rootDir = testDir.toFile(); + @TempDir + public File testDir; + + + + @Test + public void testBagOfWordsVectorizer() throws Exception { + val rootDir = testDir; ClassPathResource resource = new ClassPathResource("rootdir/"); resource.copyDirectory(rootDir); @@ -75,15 +75,15 @@ public class BagOfWordsVectorizerTest extends BaseDL4JTest { TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory(); BagOfWordsVectorizer vectorizer = new BagOfWordsVectorizer.Builder().setMinWordFrequency(1) - .setStopWords(new ArrayList<>()).setTokenizerFactory(tokenizerFactory).setIterator(iter) - .allowParallelTokenization(false) - // .labels(labels) - // .cleanup(true) - .build(); + .setStopWords(new ArrayList()).setTokenizerFactory(tokenizerFactory).setIterator(iter) + .allowParallelTokenization(false) + // .labels(labels) + // .cleanup(true) + .build(); vectorizer.fit(); VocabWord word = vectorizer.getVocabCache().wordFor("file."); - assertNotNull(word); + assumeTrue(word != null); assertEquals(word, vectorizer.getVocabCache().tokenFor("file.")); assertEquals(2, vectorizer.getVocabCache().totalNumberOfDocs()); @@ -141,7 +141,7 @@ public class BagOfWordsVectorizerTest extends BaseDL4JTest { assertNotEquals(idx2, idx1); // Serialization check - File tempFile = createTempFile(testDir,"fdsf", "fdfsdf"); + File tempFile = createTempFile("fdsf", "fdfsdf"); tempFile.deleteOnExit(); SerializationUtils.saveObject(vectorizer, tempFile); @@ -153,9 +153,8 @@ public class BagOfWordsVectorizerTest extends BaseDL4JTest { assertEquals(array, dataSet.getFeatures()); } - private File createTempFile(Path tempDir,String prefix, String suffix) throws IOException { - File newFile = Files.createTempFile(tempDir,prefix + "-" + System.nanoTime(),suffix).toFile(); - return newFile; + private File createTempFile(String prefix, String suffix) throws IOException { + return new File(testDir,prefix + "-" + System.nanoTime() + suffix); } } diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizerTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizerTest.java similarity index 87% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizerTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizerTest.java index b8874a563..e9140507e 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizerTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/bagofwords/vectorizer/TfidfVectorizerTest.java @@ -25,7 +25,6 @@ import lombok.val; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; @@ -43,56 +42,52 @@ import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.common.util.SerializationUtils; import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * @author Adam Gibson */ @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag +@Timeout(60) public class TfidfVectorizerTest extends BaseDL4JTest { + @TempDir + public File testDir; - - @Test() - @Timeout(60000L) - public void testTfIdfVectorizer(@TempDir Path testDir) throws Exception { - val rootDir = testDir.toFile(); + @Test + public void testTfIdfVectorizer() throws Exception { + val rootDir = testDir; ClassPathResource resource = new ClassPathResource("tripledir/"); resource.copyDirectory(rootDir); - + assertTrue(rootDir.isDirectory()); LabelAwareSentenceIterator iter = new LabelAwareFileSentenceIterator(rootDir); TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory(); TfidfVectorizer vectorizer = new TfidfVectorizer.Builder().setMinWordFrequency(1) - .setStopWords(new ArrayList()).setTokenizerFactory(tokenizerFactory).setIterator(iter) - .allowParallelTokenization(false) - // .labels(labels) - // .cleanup(true) - .build(); + .setStopWords(new ArrayList()).setTokenizerFactory(tokenizerFactory).setIterator(iter) + .allowParallelTokenization(false) + // .labels(labels) + // .cleanup(true) + .build(); vectorizer.fit(); VocabWord word = vectorizer.getVocabCache().wordFor("file."); - assertNotNull(word); + assumeTrue(word != null); assertEquals(word, vectorizer.getVocabCache().tokenFor("file.")); assertEquals(3, vectorizer.getVocabCache().totalNumberOfDocs()); @@ -136,8 +131,7 @@ public class TfidfVectorizerTest extends BaseDL4JTest { assertEquals(1, cnt); - - File tempFile = Files.createTempFile(testDir,"somefile","bin").toFile(); + File tempFile = new File(testDir,"somefile.bin"); tempFile.delete(); SerializationUtils.saveObject(vectorizer, tempFile); @@ -161,24 +155,24 @@ public class TfidfVectorizerTest extends BaseDL4JTest { List docs = new ArrayList<>(2); docs.add(doc1); docs.add(doc2); - + LabelAwareIterator iterator = new SimpleLabelAwareIterator(docs); TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory(); TfidfVectorizer vectorizer = new TfidfVectorizer - .Builder() - .setMinWordFrequency(1) - .setStopWords(new ArrayList()) - .setTokenizerFactory(tokenizerFactory) - .setIterator(iterator) - .allowParallelTokenization(false) - .build(); + .Builder() + .setMinWordFrequency(1) + .setStopWords(new ArrayList()) + .setTokenizerFactory(tokenizerFactory) + .setIterator(iterator) + .allowParallelTokenization(false) + .build(); vectorizer.fit(); DataSet dataset = vectorizer.vectorize("it meows like a cat", "cat"); assertNotNull(dataset); - + LabelsSource source = vectorizer.getLabelsSource(); assertEquals(2, source.getNumberOfLabelsUsed()); List labels = source.getLabels(); @@ -186,8 +180,7 @@ public class TfidfVectorizerTest extends BaseDL4JTest { assertEquals("cat", labels.get(1)); } - @Test() - @Timeout(10000L) + @Test public void testParallelFlag1() throws Exception { val vectorizer = new TfidfVectorizer.Builder() .allowParallelTokenization(false) @@ -197,10 +190,9 @@ public class TfidfVectorizerTest extends BaseDL4JTest { } - @Test() - @Timeout(20000L) + @Test public void testParallelFlag2() throws Exception { - assertThrows(ND4JIllegalStateException.class,() -> { + assertThrows(ND4JIllegalStateException.class, () -> { val collection = new ArrayList(); collection.add("First string"); collection.add("Second string"); @@ -222,14 +214,13 @@ public class TfidfVectorizerTest extends BaseDL4JTest { vectorizer.fit(); }); - } - @Test() - @Timeout(20000L) + @Test public void testParallelFlag3() throws Exception { - assertThrows(ND4JIllegalStateException.class,() -> { + assertThrows(ND4JIllegalStateException.class, () -> { val collection = new ArrayList(); + collection.add("First string"); collection.add("Second string"); collection.add("Third string"); @@ -245,13 +236,9 @@ public class TfidfVectorizerTest extends BaseDL4JTest { .build(); vectorizer.buildVocab(); - - log.info("Fitting vectorizer..."); - vectorizer.fit(); }); - } @@ -270,8 +257,6 @@ public class TfidfVectorizerTest extends BaseDL4JTest { if (triggerSentence >= 0 && cnt.incrementAndGet() >= triggerSentence) throw new ND4JIllegalStateException("TokenizerFactory exploded"); - - val tkn = new ExplodingTokenizer(toTokenize, triggerWord); return tkn; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/iterator/TestBertIterator.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/iterator/TestBertIterator.java similarity index 98% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/iterator/TestBertIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/iterator/TestBertIterator.java index c78e3c463..f4303f28e 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/iterator/TestBertIterator.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/iterator/TestBertIterator.java @@ -27,12 +27,8 @@ import org.deeplearning4j.iterator.bert.BertMaskedLMMasker; import org.deeplearning4j.iterator.provider.CollectionLabeledPairSentenceProvider; import org.deeplearning4j.iterator.provider.CollectionLabeledSentenceProvider; import org.deeplearning4j.text.tokenization.tokenizerfactory.BertWordPieceTokenizerFactory; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.api.MultiDataSet; @@ -50,11 +46,7 @@ import java.util.*; import static org.junit.jupiter.api.Assertions.*; - -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) +@Timeout(200) public class TestBertIterator extends BaseDL4JTest { private static File pathToVocab = Resources.asFile("other/vocab.txt"); @@ -64,6 +56,8 @@ public class TestBertIterator extends BaseDL4JTest { private static String sentenceA = "Goodnight noises everywhere"; private static String sentenceB = "Goodnight moon"; + public TestBertIterator() throws IOException { + } @Test() public void testBertSequenceClassification() throws Exception { @@ -139,8 +133,7 @@ public class TestBertIterator extends BaseDL4JTest { assertEquals(segmentId, b.featurizeSentences(testHelper.getSentences()).getFirst()[1]); } - @Test() - @Timeout(20000) + @Test public void testBertUnsupervised() throws Exception { int minibatchSize = 2; TestSentenceHelper testHelper = new TestSentenceHelper(); @@ -171,8 +164,7 @@ public class TestBertIterator extends BaseDL4JTest { assertTrue(b.hasNext()); } - @Test() - @Timeout(20000) + @Test public void testLengthHandling() throws Exception { int minibatchSize = 2; TestSentenceHelper testHelper = new TestSentenceHelper(); @@ -241,8 +233,7 @@ public class TestBertIterator extends BaseDL4JTest { assertArrayEquals(expShape, mds.getFeaturesMaskArray(0).shape()); } - @Test() - @Timeout(20000) + @Test public void testMinibatchPadding() throws Exception { Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); int minibatchSize = 3; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/iterator/TestCnnSentenceDataSetIterator.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/iterator/TestCnnSentenceDataSetIterator.java similarity index 99% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/iterator/TestCnnSentenceDataSetIterator.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/iterator/TestCnnSentenceDataSetIterator.java index a57f671a5..1a274766a 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/iterator/TestCnnSentenceDataSetIterator.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/iterator/TestCnnSentenceDataSetIterator.java @@ -22,9 +22,6 @@ package org.deeplearning4j.iterator; import org.deeplearning4j.BaseDL4JTest; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.common.io.ClassPathResource; import org.deeplearning4j.iterator.provider.CollectionLabeledSentenceProvider; @@ -41,8 +38,7 @@ import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag + public class TestCnnSentenceDataSetIterator extends BaseDL4JTest { @BeforeEach diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTableTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTableTest.java similarity index 86% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTableTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTableTest.java index 0563ef907..5faad62ad 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTableTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/inmemory/InMemoryLookupTableTest.java @@ -24,7 +24,7 @@ import lombok.val; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; import org.deeplearning4j.models.sequencevectors.iterators.AbstractSequenceIterator; @@ -37,27 +37,26 @@ import org.deeplearning4j.text.sentenceiterator.BasicLineIterator; import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; -import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag +@Timeout(300) public class InMemoryLookupTableTest extends BaseDL4JTest { + @TempDir + public File testDir; @BeforeEach public void setUp() throws Exception { } - @Test() - @Timeout(300000) + @Test public void testConsumeOnEqualVocabs() throws Exception { TokenizerFactory t = new DefaultTokenizerFactory(); t.setTokenPreProcessor(new CommonPreprocessor()); @@ -84,14 +83,14 @@ public class InMemoryLookupTableTest extends BaseDL4JTest { assertEquals(244, cacheSource.numWords()); InMemoryLookupTable mem1 = - new InMemoryLookupTable.Builder().vectorLength(100) - .cache(cacheSource).seed(17).build(); + (InMemoryLookupTable) new InMemoryLookupTable.Builder().vectorLength(100) + .cache(cacheSource).seed(17).build(); mem1.resetWeights(true); InMemoryLookupTable mem2 = - new InMemoryLookupTable.Builder().vectorLength(100) - .cache(cacheSource).seed(15).build(); + (InMemoryLookupTable) new InMemoryLookupTable.Builder().vectorLength(100) + .cache(cacheSource).seed(15).build(); mem2.resetWeights(true); @@ -104,11 +103,8 @@ public class InMemoryLookupTableTest extends BaseDL4JTest { } - @Test() - @Timeout(300000) - @Disabled("d file hash does not match expected hash: https://dl4jtest.blob.core.windows.net/resources/big/raw_sentences.txt.gzx.v1 ") - @Tag(TagNames.NEEDS_VERIFY) - public void testConsumeOnNonEqualVocabs(@TempDir Path testDir) throws Exception { + @Test + public void testConsumeOnNonEqualVocabs() throws Exception { TokenizerFactory t = new DefaultTokenizerFactory(); t.setTokenPreProcessor(new CommonPreprocessor()); @@ -134,8 +130,8 @@ public class InMemoryLookupTableTest extends BaseDL4JTest { assertEquals(244, cacheSource.numWords()); InMemoryLookupTable mem1 = - new InMemoryLookupTable.Builder().vectorLength(100) - .cache(cacheSource).build(); + (InMemoryLookupTable) new InMemoryLookupTable.Builder().vectorLength(100) + .cache(cacheSource).build(); mem1.resetWeights(true); @@ -144,7 +140,7 @@ public class InMemoryLookupTableTest extends BaseDL4JTest { AbstractCache cacheTarget = new AbstractCache.Builder().build(); - val dir = testDir.toFile(); + val dir = testDir; new ClassPathResource("/paravec/labeled/").copyDirectory(dir); FileLabelAwareIterator labelAwareIterator = new FileLabelAwareIterator.Builder() diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java similarity index 96% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java index d4b93b3ff..2e3e26b96 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializerTest.java @@ -36,12 +36,9 @@ import org.deeplearning4j.models.word2vec.Word2Vec; import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -50,7 +47,6 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -60,12 +56,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag public class WordVectorSerializerTest extends BaseDL4JTest { private AbstractCache cache; - + @TempDir + public File testDir; @BeforeEach public void setUp() throws Exception { @@ -259,7 +254,7 @@ public class WordVectorSerializerTest extends BaseDL4JTest { } @Test - public void weightLookupTable_Correct_WhenDeserialized(@TempDir Path testDir) throws Exception { + public void weightLookupTable_Correct_WhenDeserialized() throws Exception { INDArray syn0 = Nd4j.rand(DataType.FLOAT, 10, 2), syn1 = Nd4j.rand(DataType.FLOAT, 10, 2), @@ -275,7 +270,7 @@ public class WordVectorSerializerTest extends BaseDL4JTest { lookupTable.setSyn1(syn1); lookupTable.setSyn1Neg(syn1Neg); - File dir = testDir.toFile(); + File dir = testDir; File file = new File(dir, "lookupTable.txt"); WeightLookupTable deser = null; @@ -308,12 +303,12 @@ public class WordVectorSerializerTest extends BaseDL4JTest { } @Test - public void FastText_Correct_WhenDeserialized(@TempDir Path testDir) throws IOException { + public void FastText_Correct_WhenDeserialized() throws IOException { FastText fastText = FastText.builder().cbow(true).build(); - File dir = testDir.toFile(); + File dir = testDir; WordVectorSerializer.writeWordVectors(fastText, new File(dir, "some.data")); FastText deser = null; diff --git a/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/reader/impl/FlatModelUtilsTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/reader/impl/FlatModelUtilsTest.java new file mode 100644 index 000000000..9db75e32e --- /dev/null +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/reader/impl/FlatModelUtilsTest.java @@ -0,0 +1,112 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.models.embeddings.reader.impl; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; +import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.Word2Vec; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.ops.transforms.Transforms; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +//@Ignore +public class FlatModelUtilsTest extends BaseDL4JTest { + private Word2Vec vec; + private static final Logger log = LoggerFactory.getLogger(FlatModelUtilsTest.class); + + @BeforeEach + public void setUp() throws Exception { + if (vec == null) { + //vec = WordVectorSerializer.loadFullModel("/Users/raver119/develop/model.dat"); + vec = WordVectorSerializer.loadFullModel("/ext/Temp/Models/model.dat"); + //vec = WordVectorSerializer.loadFullModel("/ext/Temp/Models/raw_sentences.dat"); + } + } + + @Test + public void testWordsNearestFlat1() throws Exception { + vec.setModelUtils(new FlatModelUtils()); + + Collection list = vec.wordsNearest("energy", 10); + log.info("Flat model results:"); + printWords("energy", list, vec); + } + + @Test + public void testWordsNearestBasic1() throws Exception { + + //WordVectors vec = WordVectorSerializer.loadTxtVectors(new File("/ext/Temp/Models/model.dat_trans")); + vec.setModelUtils(new BasicModelUtils()); + + String target = "energy"; + + INDArray arr1 = vec.getWordVectorMatrix(target).dup(); + + System.out.println("[-]: " + arr1); + System.out.println("[+]: " + Transforms.unitVec(arr1)); + + Collection list = vec.wordsNearest(target, 10); + log.info("Transpose model results:"); + printWords(target, list, vec); + + list = vec.wordsNearest(target, 10); + log.info("Transpose model results 2:"); + printWords(target, list, vec); + + list = vec.wordsNearest(target, 10); + log.info("Transpose model results 3:"); + printWords(target, list, vec); + + + INDArray arr2 = vec.getWordVectorMatrix(target).dup(); + + assertEquals(arr1, arr2); + } + + @Test + //@Ignore + public void testWordsNearestTree1() throws Exception { + vec.setModelUtils(new TreeModelUtils()); + + Collection list = vec.wordsNearest("energy", 10); + log.info("Tree model results:"); + printWords("energy", list, vec); + } + + private static void printWords(String target, Collection list, WordVectors vec) { + System.out.println("Words close to [" + target + "]:"); + for (String word : list) { + double sim = vec.similarity(target, word); + System.out.print("'" + word + "': [" + sim + "]"); + } + System.out.print("\n"); + } +} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImplTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImplTest.java similarity index 92% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImplTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImplTest.java index dc4864c10..3b7cff88f 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImplTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/embeddings/wordvectors/WordVectorsImplTest.java @@ -20,10 +20,7 @@ package org.deeplearning4j.models.embeddings.wordvectors; -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.guava.collect.Lists; +import com.google.common.collect.Lists; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.models.embeddings.WeightLookupTable; import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; @@ -36,8 +33,7 @@ import org.nd4j.linalg.factory.Nd4j; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; -@Tag(TagNames.FILE_IO) -@NativeTag + public class WordVectorsImplTest extends BaseDL4JTest { private VocabCache vocabCache; private WeightLookupTable weightLookupTable; diff --git a/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java new file mode 100644 index 000000000..2879fdffe --- /dev/null +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java @@ -0,0 +1,271 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.models.fasttext; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; +import org.deeplearning4j.models.word2vec.Word2Vec; +import org.deeplearning4j.text.sentenceiterator.BasicLineIterator; +import org.deeplearning4j.text.sentenceiterator.SentenceIterator; + + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.resources.Resources; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +//@Ignore +@Timeout(30) +public class FastTextTest extends BaseDL4JTest { + + + + private File inputFile = Resources.asFile("models/fasttext/data/labeled_data.txt"); + private File supModelFile = Resources.asFile("models/fasttext/supervised.model.bin"); + private File cbowModelFile = Resources.asFile("models/fasttext/cbow.model.bin"); + private File supervisedVectors = Resources.asFile("models/fasttext/supervised.model.vec"); + + @TempDir + public File testDir; + + @Test + public void testTrainSupervised() throws IOException { + + File output = new File(testDir, UUID.randomUUID().toString()); + + FastText fastText = + FastText.builder().supervised(true). + inputFile(inputFile.getAbsolutePath()). + outputFile(output.getAbsolutePath()).build(); + log.info("\nTraining supervised model ...\n"); + fastText.fit(); + } + + @Test + public void testTrainSkipgram() throws IOException { + + File output = new File(testDir, UUID.randomUUID().toString()); + + FastText fastText = + FastText.builder().skipgram(true). + inputFile(inputFile.getAbsolutePath()). + outputFile(output.getAbsolutePath()).build(); + log.info("\nTraining supervised model ...\n"); + fastText.fit(); + } + + @Test + public void testTrainSkipgramWithBuckets() throws IOException { + + File output = new File(testDir, UUID.randomUUID().toString()); + + FastText fastText = + FastText.builder().skipgram(true). + bucket(150). + inputFile(inputFile.getAbsolutePath()). + outputFile(output.getAbsolutePath()).build(); + log.info("\nTraining supervised model ...\n"); + fastText.fit(); + } + + @Test + public void testTrainCBOW() throws IOException { + + File output = new File(testDir, UUID.randomUUID().toString()); + + FastText fastText = + FastText.builder().cbow(true). + inputFile(inputFile.getAbsolutePath()). + outputFile(output.getAbsolutePath()).build(); + log.info("\nTraining supervised model ...\n"); + fastText.fit(); + } + + @Test + public void tesLoadCBOWModel() { + + FastText fastText = new FastText(cbowModelFile); + fastText.test(cbowModelFile); + + assertEquals(19, fastText.vocab().numWords()); + assertEquals("enjoy", fastText.vocab().wordAtIndex(fastText.vocab().numWords() - 1)); + + double[] expected = {5.040466203354299E-4, 0.001005030469968915, 2.8882650076411664E-4, -6.413314840756357E-4, -1.78931062691845E-4, -0.0023157168179750443, -0.002215880434960127, 0.00274421414360404, -1.5344757412094623E-4, 4.6274057240225375E-4, -1.4383681991603225E-4, 3.7832374800927937E-4, 2.523412986192852E-4, 0.0018913350068032742, -0.0024741862434893847, -4.976555937901139E-4, 0.0039220210164785385, -0.001781729981303215, -6.010578363202512E-4, -0.00244093406945467, -7.98621098510921E-4, -0.0010007203090935946, -0.001640203408896923, 7.897148607298732E-4, 9.131592814810574E-4, -0.0013367272913455963, -0.0014030139427632093, -7.755287806503475E-4, -4.2878396925516427E-4, 6.912827957421541E-4, -0.0011824817629531026, -0.0036014916840940714, 0.004353308118879795, -7.073904271237552E-5, -9.646290563978255E-4, -0.0031849315855652094, 2.3360115301329643E-4, -2.9103990527801216E-4, -0.0022990566212683916, -0.002393763978034258, -0.001034979010000825, -0.0010725988540798426, 0.0018285386031493545, -0.0013178540393710136, -1.6632364713586867E-4, -1.4665909475297667E-5, 5.445032729767263E-4, 2.999933494720608E-4, -0.0014367225812748075, -0.002345481887459755, 0.001117417006753385, -8.688368834555149E-4, -0.001830018823966384, 0.0013242220738902688, -8.880519890226424E-4, -6.888324278406799E-4, -0.0036394784692674875, 0.002179111586883664, -1.7201311129610986E-4, 0.002365073887631297, 0.002688770182430744, 0.0023955567739903927, 0.001469283364713192, 0.0011803617235273123, 5.871498142369092E-4, -7.099180947989225E-4, 7.518937345594168E-4, -8.599072461947799E-4, -6.600041524507105E-4, -0.002724145073443651, -8.365285466425121E-4, 0.0013173354091122746, 0.001083166105672717, 0.0014539906987920403, -3.1698777456767857E-4, -2.387022686889395E-4, 1.9560157670639455E-4, 0.0020277926232665777, -0.0012741144746541977, -0.0013026101514697075, -1.5212174912448972E-4, 0.0014194383984431624, 0.0012500399025157094, 0.0013362085446715355, 3.692879108712077E-4, 4.319801155361347E-5, 0.0011261265026405454, 0.0017244465416297317, 5.564604725805111E-5, 0.002170475199818611, 0.0014707016525790095, 0.001303741242736578, 0.005553730763494968, -0.0011097051901742816, -0.0013661726843565702, 0.0014100460102781653, 0.0011811562580987811, -6.622733199037611E-4, 7.860265322960913E-4, -9.811905911192298E-4}; + assertArrayEquals(expected, fastText.getWordVector("enjoy"), 2e-3); + } + + @Test + public void testPredict() { + String text = "I like soccer"; + + FastText fastText = new FastText(supModelFile); + assertEquals(48, fastText.vocab().numWords()); + assertEquals("association", fastText.vocab().wordAtIndex(fastText.vocab().numWords() - 1)); + + double[] expected = {-0.006423053797334433, 0.007660661358386278, 0.006068876478821039, -0.004772625397890806, -0.007143457420170307, -0.007735592778772116, -0.005607823841273785, -0.00836215727031231, 0.0011235733982175589, 2.599214785732329E-4, 0.004131870809942484, 0.007203693501651287, 0.0016768622444942594, 0.008694255724549294, -0.0012487826170399785, -0.00393667770549655, -0.006292815785855055, 0.0049359360709786415, -3.356488887220621E-4, -0.009407570585608482, -0.0026168026961386204, -0.00978928804397583, 0.0032913016621023417, -0.0029464277904480696, -0.008649969473481178, 8.056449587456882E-4, 0.0043088337406516075, -0.008980576880276203, 0.008716211654245853, 0.0073893265798687935, -0.007388216909021139, 0.003814412746578455, -0.005518500227481127, 0.004668557550758123, 0.006603693123906851, 0.003820829326286912, 0.007174000144004822, -0.006393063813447952, -0.0019381389720365405, -0.0046371882781386375, -0.006193376146256924, -0.0036685809027403593, 7.58899434003979E-4, -0.003185075242072344, -0.008330358192324638, 3.3206873922608793E-4, -0.005389622412621975, 0.009706716984510422, 0.0037855932023376226, -0.008665262721478939, -0.0032511046156287193, 4.4134497875347733E-4, -0.008377416990697384, -0.009110655635595322, 0.0019723298028111458, 0.007486093323677778, 0.006400121841579676, 0.00902814231812954, 0.00975200068205595, 0.0060582347214221954, -0.0075621469877660275, 1.0270809434587136E-4, -0.00673140911385417, -0.007316927425563335, 0.009916870854794979, -0.0011407854035496712, -4.502215306274593E-4, -0.007612560410052538, 0.008726916275918484, -3.0280642022262327E-5, 0.005529289599508047, -0.007944817654788494, 0.005593308713287115, 0.003423960180953145, 4.1348213562741876E-4, 0.009524818509817123, -0.0025129399728029966, -0.0030074280221015215, -0.007503866218030453, -0.0028124507516622543, -0.006841592025011778, -2.9375351732596755E-4, 0.007195258513092995, -0.007775942329317331, 3.951996040996164E-4, -0.006887971889227629, 0.0032655203249305487, -0.007975360378623009, -4.840183464693837E-6, 0.004651934839785099, 0.0031739831902086735, 0.004644941072911024, -0.007461248897016048, 0.003057275665923953, 0.008903342299163342, 0.006857945583760738, 0.007567950990051031, 0.001506582135334611, 0.0063307867385447025, 0.005645462777465582}; + assertArrayEquals(expected, fastText.getWordVector("association"), 2e-3); + + String label = fastText.predict(text); + assertEquals("__label__soccer", label); + } + + @Test + public void testIllegalState() { + assertThrows(IllegalStateException.class, () -> { + String text = "I like soccer"; + + FastText fastText = new FastText(supModelFile); + assertEquals(48, fastText.vocab().numWords()); + assertEquals("association", fastText.vocab().wordAtIndex(fastText.vocab().numWords() - 1)); + + double[] expected = {-0.006423053797334433, 0.007660661358386278, 0.006068876478821039, -0.004772625397890806, -0.007143457420170307, -0.007735592778772116, -0.005607823841273785, -0.00836215727031231, 0.0011235733982175589, 2.599214785732329E-4, 0.004131870809942484, 0.007203693501651287, 0.0016768622444942594, 0.008694255724549294, -0.0012487826170399785, -0.00393667770549655, -0.006292815785855055, 0.0049359360709786415, -3.356488887220621E-4, -0.009407570585608482, -0.0026168026961386204, -0.00978928804397583, 0.0032913016621023417, -0.0029464277904480696, -0.008649969473481178, 8.056449587456882E-4, 0.0043088337406516075, -0.008980576880276203, 0.008716211654245853, 0.0073893265798687935, -0.007388216909021139, 0.003814412746578455, -0.005518500227481127, 0.004668557550758123, 0.006603693123906851, 0.003820829326286912, 0.007174000144004822, -0.006393063813447952, -0.0019381389720365405, -0.0046371882781386375, -0.006193376146256924, -0.0036685809027403593, 7.58899434003979E-4, -0.003185075242072344, -0.008330358192324638, 3.3206873922608793E-4, -0.005389622412621975, 0.009706716984510422, 0.0037855932023376226, -0.008665262721478939, -0.0032511046156287193, 4.4134497875347733E-4, -0.008377416990697384, -0.009110655635595322, 0.0019723298028111458, 0.007486093323677778, 0.006400121841579676, 0.00902814231812954, 0.00975200068205595, 0.0060582347214221954, -0.0075621469877660275, 1.0270809434587136E-4, -0.00673140911385417, -0.007316927425563335, 0.009916870854794979, -0.0011407854035496712, -4.502215306274593E-4, -0.007612560410052538, 0.008726916275918484, -3.0280642022262327E-5, 0.005529289599508047, -0.007944817654788494, 0.005593308713287115, 0.003423960180953145, 4.1348213562741876E-4, 0.009524818509817123, -0.0025129399728029966, -0.0030074280221015215, -0.007503866218030453, -0.0028124507516622543, -0.006841592025011778, -2.9375351732596755E-4, 0.007195258513092995, -0.007775942329317331, 3.951996040996164E-4, -0.006887971889227629, 0.0032655203249305487, -0.007975360378623009, -4.840183464693837E-6, 0.004651934839785099, 0.0031739831902086735, 0.004644941072911024, -0.007461248897016048, 0.003057275665923953, 0.008903342299163342, 0.006857945583760738, 0.007567950990051031, 0.001506582135334611, 0.0063307867385447025, 0.005645462777465582}; + assertArrayEquals(expected, fastText.getWordVector("association"), 2e-3); + + String label = fastText.predict(text); + fastText.wordsNearest("test", 1); + }); + } + + @Test + public void testPredictProbability() { + String text = "I like soccer"; + + FastText fastText = new FastText(supModelFile); + + Pair result = fastText.predictProbability(text); + assertEquals("__label__soccer", result.getFirst()); + assertEquals(-0.6930, result.getSecond(), 2e-3); + + assertEquals(48, fastText.vocabSize()); + assertEquals(0.0500, fastText.getLearningRate(), 2e-3); + assertEquals(100, fastText.getDimension()); + assertEquals(5, fastText.getContextWindowSize()); + assertEquals(5, fastText.getEpoch()); + assertEquals(5, fastText.getNegativesNumber()); + assertEquals(1, fastText.getWordNgrams()); + assertEquals("softmax", fastText.getLossName()); + assertEquals("sup", fastText.getModelName()); + assertEquals(0, fastText.getNumberOfBuckets()); + } + + @Test + public void testVocabulary() { + FastText fastText = new FastText(supModelFile); + assertEquals(48, fastText.vocab().numWords()); + assertEquals(48, fastText.vocabSize()); + + String[] expected = {"", ".", "is", "game", "the", "soccer", "?", "football", "3", "12", "takes", "usually", "A", "US", + "in", "popular", "most", "hours", "and", "clubs", "minutes", "Do", "you", "like", "Is", "your", "favorite", "games", + "Premier", "Soccer", "a", "played", "by", "two", "teams", "of", "eleven", "players", "The", "Football", "League", "an", + "English", "professional", "league", "for", "men's", "association"}; + + for (int i = 0; i < fastText.vocabSize(); ++i) { + assertEquals(expected[i], fastText.vocab().wordAtIndex(i)); + } + } + + @Test + public void testLoadIterator() throws FileNotFoundException { + SentenceIterator iter = new BasicLineIterator(inputFile.getAbsolutePath()); + FastText + .builder() + .supervised(true) + .iterator(iter) + .build() + .loadIterator(); + } + + @Test + public void testState() { + Assertions.assertThrows(IllegalStateException.class, () -> { + FastText fastText = new FastText(); + fastText.predict("something"); + }); + } + + @Test + public void testPretrainedVectors() throws IOException { + File output = new File(testDir, UUID.randomUUID().toString()); + + FastText fastText = FastText + .builder() + .supervised(true) + .inputFile(inputFile.getAbsolutePath()) + .pretrainedVectorsFile(supervisedVectors.getAbsolutePath()) + .outputFile(output.getAbsolutePath()) + .build(); + + log.info("\nTraining supervised model ...\n"); + fastText.fit(); + } + + @Test + public void testWordsStatistics() throws IOException { + File output = new File(testDir, UUID.randomUUID().toString()); + + FastText fastText = FastText + .builder() + .supervised(true) + .inputFile(inputFile.getAbsolutePath()) + .outputFile(output.getAbsolutePath()) + .build(); + + log.info("\nTraining supervised model ...\n"); + fastText.fit(); + + File file = new File(output.getAbsolutePath() + ".vec"); + Word2Vec word2Vec = WordVectorSerializer.readAsCsv(file); + + assertEquals(48, word2Vec.getVocab().numWords()); + assertEquals(0.1667751520872116, word2Vec.similarity("Football", "teams"), 2e-3); + assertEquals(0.10083991289138794, word2Vec.similarity("professional", "minutes"), 2e-3); + assertEquals(Double.NaN, word2Vec.similarity("java","cpp"), 0.0); + assertThat(word2Vec.wordsNearest("association", 3), hasItems("Football", "Soccer", "men's")); + } + + @Test + public void testWordsNativeStatistics() { + FastText fastText = new FastText(); + fastText.loadPretrainedVectors(supervisedVectors); + + log.info("\nTraining supervised model ...\n"); + + assertEquals(48, fastText.vocab().numWords()); + assertThat(fastText.wordsNearest("association", 3), hasItems("most","eleven","hours")); + assertEquals(0.1657, fastText.similarity("Football", "teams"), 2e-3); + assertEquals(0.3661, fastText.similarity("professional", "minutes"), 2e-3); + assertEquals(Double.NaN, fastText.similarity("java","cpp"), 0.0); + } +} diff --git a/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectorsTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectorsTest.java new file mode 100644 index 000000000..cac34901f --- /dev/null +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectorsTest.java @@ -0,0 +1,1239 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.models.paragraphvectors; + + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.LineIterator; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.models.sequencevectors.sequence.Sequence; +import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer; +import org.deeplearning4j.models.sequencevectors.transformers.impl.iterables.BasicTransformerIterator; +import org.deeplearning4j.text.sentenceiterator.*; + + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.common.io.ClassPathResource; +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram; +import org.deeplearning4j.models.embeddings.learning.impl.sequence.DBOW; +import org.deeplearning4j.models.embeddings.learning.impl.sequence.DM; +import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration; +import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; +import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.Word2Vec; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; +import org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache; +import org.deeplearning4j.text.documentiterator.FileLabelAwareIterator; +import org.deeplearning4j.text.documentiterator.LabelAwareIterator; +import org.deeplearning4j.text.documentiterator.LabelledDocument; +import org.deeplearning4j.text.documentiterator.LabelsSource; +import org.deeplearning4j.text.sentenceiterator.interoperability.SentenceIteratorConverter; +import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; +import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; +import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.io.CollectionUtils; +import org.nd4j.linalg.ops.transforms.Transforms; +import org.nd4j.common.util.SerializationUtils; +import org.nd4j.common.resources.Resources; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +@Timeout(240) +public class ParagraphVectorsTest extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return isIntegrationTests() ? 600_000 : 240_000; + } + + @TempDir + public File testDir; + + @Override + public DataType getDataType() { + return DataType.FLOAT; + } + + @Override + public DataType getDefaultFPDataType() { + return DataType.FLOAT; + } + + /* + @Test + public void testWord2VecRunThroughVectors() throws Exception { + ClassPathResource resource = new ClassPathResource("/big/raw_sentences.txt"); + File file = resource.getFile().getParentFile(); + LabelAwareSentenceIterator iter = LabelAwareUimaSentenceIterator.createWithPath(file.getAbsolutePath()); + + + TokenizerFactory t = new UimaTokenizerFactory(); + + + ParagraphVectors vec = new ParagraphVectors.Builder() + .minWordFrequency(1).iterations(5).labels(Arrays.asList("label1", "deeple")) + .layerSize(100) + .stopWords(new ArrayList()) + .windowSize(5).iterate(iter).tokenizerFactory(t).build(); + + assertEquals(new ArrayList(), vec.getStopWords()); + + + vec.fit(); + double sim = vec.similarity("day","night"); + log.info("day/night similarity: " + sim); + new File("cache.ser").delete(); + + } + */ + + /** + * This test checks, how vocab is built using SentenceIterator provided, without labels. + * + * @throws Exception + */ + @Test + public void testParagraphVectorsVocabBuilding1() throws Exception { + File file = Resources.asFile("/big/raw_sentences.txt"); + SentenceIterator iter = new BasicLineIterator(file); //UimaSentenceIterator.createWithPath(file.getAbsolutePath()); + + int numberOfLines = 0; + while (iter.hasNext()) { + iter.nextSentence(); + numberOfLines++; + } + + iter.reset(); + + InMemoryLookupCache cache = new InMemoryLookupCache(false); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + // LabelsSource source = new LabelsSource("DOC_"); + + ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(5).layerSize(100) + // .labelsGenerator(source) + .windowSize(5).iterate(iter).vocabCache(cache).tokenizerFactory(t).build(); + + vec.buildVocab(); + + LabelsSource source = vec.getLabelsSource(); + + + //VocabCache cache = vec.getVocab(); + log.info("Number of lines in corpus: " + numberOfLines); + assertEquals(numberOfLines, source.getLabels().size()); + assertEquals(97162, source.getLabels().size()); + + assertNotEquals(null, cache); + assertEquals(97406, cache.numWords()); + + // proper number of words for minWordsFrequency = 1 is 244 + assertEquals(244, cache.numWords() - source.getLabels().size()); + } + + /** + * This test doesn't really cares about actual results. We only care about equality between live model & restored models + * + * @throws Exception + */ + @Test + //@Ignore("AB 2019/05/21 - Failing on linux-x86_64-cuda-9.2 - Issue #7657") + public void testParagraphVectorsModelling1() throws Exception { + File file = Resources.asFile("/big/raw_sentences.txt"); + SentenceIterator iter = new BasicLineIterator(file); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + LabelsSource source = new LabelsSource("DOC_"); + + ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(5).seed(119).epochs(1) + .layerSize(150).learningRate(0.025).labelsSource(source).windowSize(5) + .sequenceLearningAlgorithm(new DM()).iterate(iter).trainWordVectors(true) + .usePreciseWeightInit(true) + .batchSize(8192) + .tokenizerFactory(t).workers(4).sampling(0).build(); + + vec.fit(); + + VocabCache cache = vec.getVocab(); + + File fullFile = File.createTempFile("paravec", "tests"); + fullFile.deleteOnExit(); + + INDArray originalSyn1_17 = ((InMemoryLookupTable) vec.getLookupTable()).getSyn1().getRow(17, true).dup(); + + WordVectorSerializer.writeParagraphVectors(vec, fullFile); + + int cnt1 = cache.wordFrequency("day"); + int cnt2 = cache.wordFrequency("me"); + + assertNotEquals(1, cnt1); + assertNotEquals(1, cnt2); + assertNotEquals(cnt1, cnt2); + + assertEquals(97406, cache.numWords()); + + assertTrue(vec.hasWord("DOC_16392")); + assertTrue(vec.hasWord("DOC_3720")); + + List result = new ArrayList<>(vec.nearestLabels(vec.getWordVectorMatrix("DOC_16392"), 10)); + System.out.println("nearest labels: " + result); + for (String label : result) { + System.out.println(label + "/DOC_16392: " + vec.similarity(label, "DOC_16392")); + } + assertTrue(result.contains("DOC_16392")); + //assertTrue(result.contains("DOC_21383")); + + + + /* + We have few lines that contain pretty close words invloved. + These sentences should be pretty close to each other in vector space + */ + // line 3721: This is my way . + // line 6348: This is my case . + // line 9836: This is my house . + // line 12493: This is my world . + // line 16393: This is my work . + + // this is special sentence, that has nothing common with previous sentences + // line 9853: We now have one . + + double similarityD = vec.similarity("day", "night"); + log.info("day/night similarity: " + similarityD); + + if (similarityD < 0.0) { + log.info("Day: " + Arrays.toString(vec.getWordVectorMatrix("day").dup().data().asDouble())); + log.info("Night: " + Arrays.toString(vec.getWordVectorMatrix("night").dup().data().asDouble())); + } + + + List labelsOriginal = vec.labelsSource.getLabels(); + + double similarityW = vec.similarity("way", "work"); + log.info("way/work similarity: " + similarityW); + + double similarityH = vec.similarity("house", "world"); + log.info("house/world similarity: " + similarityH); + + double similarityC = vec.similarity("case", "way"); + log.info("case/way similarity: " + similarityC); + + double similarity1 = vec.similarity("DOC_9835", "DOC_12492"); + log.info("9835/12492 similarity: " + similarity1); + // assertTrue(similarity1 > 0.7d); + + double similarity2 = vec.similarity("DOC_3720", "DOC_16392"); + log.info("3720/16392 similarity: " + similarity2); + // assertTrue(similarity2 > 0.7d); + + double similarity3 = vec.similarity("DOC_6347", "DOC_3720"); + log.info("6347/3720 similarity: " + similarity3); + // assertTrue(similarity2 > 0.7d); + + // likelihood in this case should be significantly lower + double similarityX = vec.similarity("DOC_3720", "DOC_9852"); + log.info("3720/9852 similarity: " + similarityX); + assertTrue(similarityX < 0.5d); + + File tempFile = File.createTempFile("paravec", "ser"); + tempFile.deleteOnExit(); + + INDArray day = vec.getWordVectorMatrix("day").dup(); + + /* + Testing txt serialization + */ + File tempFile2 = File.createTempFile("paravec", "ser"); + tempFile2.deleteOnExit(); + + WordVectorSerializer.writeWordVectors(vec, tempFile2); + + ParagraphVectors vec3 = WordVectorSerializer.readParagraphVectorsFromText(tempFile2); + + INDArray day3 = vec3.getWordVectorMatrix("day").dup(); + + List labelsRestored = vec3.labelsSource.getLabels(); + + assertEquals(day, day3); + + assertEquals(labelsOriginal.size(), labelsRestored.size()); + + /* + Testing binary serialization + */ + SerializationUtils.saveObject(vec, tempFile); + + + ParagraphVectors vec2 = (ParagraphVectors) SerializationUtils.readObject(tempFile); + INDArray day2 = vec2.getWordVectorMatrix("day").dup(); + + List labelsBinary = vec2.labelsSource.getLabels(); + + assertEquals(day, day2); + + tempFile.delete(); + + + assertEquals(labelsOriginal.size(), labelsBinary.size()); + + INDArray original = vec.getWordVectorMatrix("DOC_16392").dup(); + INDArray originalPreserved = original.dup(); + INDArray inferredA1 = vec.inferVector("This is my work ."); + INDArray inferredB1 = vec.inferVector("This is my work ."); + + double cosAO1 = Transforms.cosineSim(inferredA1.dup(), original.dup()); + double cosAB1 = Transforms.cosineSim(inferredA1.dup(), inferredB1.dup()); + + log.info("Cos O/A: {}", cosAO1); + log.info("Cos A/B: {}", cosAB1); + log.info("Inferred: {}", inferredA1); + // assertTrue(cosAO1 > 0.45); + assertTrue(cosAB1 > 0.95); + + //assertArrayEquals(inferredA.data().asDouble(), inferredB.data().asDouble(), 0.01); + + ParagraphVectors restoredVectors = WordVectorSerializer.readParagraphVectors(fullFile); + restoredVectors.setTokenizerFactory(t); + + INDArray restoredSyn1_17 = ((InMemoryLookupTable) restoredVectors.getLookupTable()).getSyn1().getRow(17, true).dup(); + + assertEquals(originalSyn1_17, restoredSyn1_17); + + INDArray originalRestored = vec.getWordVectorMatrix("DOC_16392").dup(); + + assertEquals(originalPreserved, originalRestored); + + INDArray inferredA2 = restoredVectors.inferVector("This is my work ."); + INDArray inferredB2 = restoredVectors.inferVector("This is my work ."); + INDArray inferredC2 = restoredVectors.inferVector("world way case ."); + + double cosAO2 = Transforms.cosineSim(inferredA2.dup(), original.dup()); + double cosAB2 = Transforms.cosineSim(inferredA2.dup(), inferredB2.dup()); + double cosAAX = Transforms.cosineSim(inferredA1.dup(), inferredA2.dup()); + double cosAC2 = Transforms.cosineSim(inferredC2.dup(), inferredA2.dup()); + + log.info("Cos A2/B2: {}", cosAB2); + log.info("Cos A1/A2: {}", cosAAX); + log.info("Cos O/A2: {}", cosAO2); + log.info("Cos C2/A2: {}", cosAC2); + + log.info("Vector: {}", Arrays.toString(inferredA1.data().asFloat())); + + log.info("cosAO2: {}", cosAO2); + + // assertTrue(cosAO2 > 0.45); + assertTrue(cosAB2 > 0.95); + assertTrue(cosAAX > 0.95); + } + + + @Test + public void testParagraphVectorsDM() throws Exception { + String backend = Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"); + if(!isIntegrationTests() && "CUDA".equalsIgnoreCase(backend)) { + skipUnlessIntegrationTests(); //Skip CUDA except for integration tests due to very slow test speed + } + + File file = Resources.asFile("/big/raw_sentences.txt"); + SentenceIterator iter = new BasicLineIterator(file); + + AbstractCache cache = new AbstractCache.Builder().build(); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + LabelsSource source = new LabelsSource("DOC_"); + + ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(2).seed(119).epochs(1) + .layerSize(100).learningRate(0.025).labelsSource(source).windowSize(5).iterate(iter) + .trainWordVectors(true).vocabCache(cache).tokenizerFactory(t).negativeSample(0) + .useHierarchicSoftmax(true).sampling(0).workers(1).usePreciseWeightInit(true) + .sequenceLearningAlgorithm(new DM()).build(); + + vec.fit(); + + + int cnt1 = cache.wordFrequency("day"); + int cnt2 = cache.wordFrequency("me"); + + assertNotEquals(1, cnt1); + assertNotEquals(1, cnt2); + assertNotEquals(cnt1, cnt2); + + double simDN = vec.similarity("day", "night"); + log.info("day/night similariry: {}", simDN); + + double similarity1 = vec.similarity("DOC_9835", "DOC_12492"); + log.info("9835/12492 similarity: " + similarity1); + // assertTrue(similarity1 > 0.2d); + + double similarity2 = vec.similarity("DOC_3720", "DOC_16392"); + log.info("3720/16392 similarity: " + similarity2); + // assertTrue(similarity2 > 0.2d); + + double similarity3 = vec.similarity("DOC_6347", "DOC_3720"); + log.info("6347/3720 similarity: " + similarity3); + // assertTrue(similarity3 > 0.6d); + + double similarityX = vec.similarity("DOC_3720", "DOC_9852"); + log.info("3720/9852 similarity: " + similarityX); + if(isIntegrationTests()) { + assertTrue(similarityX < 0.5d); + } + + + // testing DM inference now + + INDArray original = vec.getWordVectorMatrix("DOC_16392").dup(); + INDArray inferredA1 = vec.inferVector("This is my work"); + INDArray inferredB1 = vec.inferVector("This is my work ."); + + double cosAO1 = Transforms.cosineSim(inferredA1.dup(), original.dup()); + double cosAB1 = Transforms.cosineSim(inferredA1.dup(), inferredB1.dup()); + + log.info("Cos O/A: {}", cosAO1); + log.info("Cos A/B: {}", cosAB1); + } + + + @Test + public void testParagraphVectorsDBOW() throws Exception { + skipUnlessIntegrationTests(); + + File file = Resources.asFile("/big/raw_sentences.txt"); + SentenceIterator iter = new BasicLineIterator(file); + + AbstractCache cache = new AbstractCache.Builder().build(); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + LabelsSource source = new LabelsSource("DOC_"); + + ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(5).seed(119).epochs(1) + .layerSize(100).learningRate(0.025).labelsSource(source).windowSize(5).iterate(iter) + .trainWordVectors(true).vocabCache(cache).tokenizerFactory(t).negativeSample(0) + .allowParallelTokenization(true).useHierarchicSoftmax(true).sampling(0).workers(4) + .usePreciseWeightInit(true).sequenceLearningAlgorithm(new DBOW()).build(); + + vec.fit(); + + assertFalse(((InMemoryLookupTable)vec.getLookupTable()).getSyn0().isAttached()); + assertFalse(((InMemoryLookupTable)vec.getLookupTable()).getSyn1().isAttached()); + + int cnt1 = cache.wordFrequency("day"); + int cnt2 = cache.wordFrequency("me"); + + assertNotEquals(1, cnt1); + assertNotEquals(1, cnt2); + assertNotEquals(cnt1, cnt2); + + double simDN = vec.similarity("day", "night"); + log.info("day/night similariry: {}", simDN); + + double similarity1 = vec.similarity("DOC_9835", "DOC_12492"); + log.info("9835/12492 similarity: " + similarity1); + // assertTrue(similarity1 > 0.2d); + + double similarity2 = vec.similarity("DOC_3720", "DOC_16392"); + log.info("3720/16392 similarity: " + similarity2); + // assertTrue(similarity2 > 0.2d); + + double similarity3 = vec.similarity("DOC_6347", "DOC_3720"); + log.info("6347/3720 similarity: " + similarity3); + // assertTrue(similarity3 > 0.6d); + + double similarityX = vec.similarity("DOC_3720", "DOC_9852"); + log.info("3720/9852 similarity: " + similarityX); + assertTrue(similarityX < 0.5d); + + + // testing DM inference now + + INDArray original = vec.getWordVectorMatrix("DOC_16392").dup(); + INDArray inferredA1 = vec.inferVector("This is my work"); + INDArray inferredB1 = vec.inferVector("This is my work ."); + INDArray inferredC1 = vec.inferVector("This is my day"); + INDArray inferredD1 = vec.inferVector("This is my night"); + + log.info("A: {}", Arrays.toString(inferredA1.data().asFloat())); + log.info("C: {}", Arrays.toString(inferredC1.data().asFloat())); + + assertNotEquals(inferredA1, inferredC1); + + double cosAO1 = Transforms.cosineSim(inferredA1.dup(), original.dup()); + double cosAB1 = Transforms.cosineSim(inferredA1.dup(), inferredB1.dup()); + double cosAC1 = Transforms.cosineSim(inferredA1.dup(), inferredC1.dup()); + double cosCD1 = Transforms.cosineSim(inferredD1.dup(), inferredC1.dup()); + + log.info("Cos O/A: {}", cosAO1); + log.info("Cos A/B: {}", cosAB1); + log.info("Cos A/C: {}", cosAC1); + log.info("Cos C/D: {}", cosCD1); + + } + + @Test + public void testParagraphVectorsWithWordVectorsModelling1() throws Exception { + String backend = Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"); + if(!isIntegrationTests() && "CUDA".equalsIgnoreCase(backend)) { + skipUnlessIntegrationTests(); //Skip CUDA except for integration tests due to very slow test speed + } + + File file = Resources.asFile("/big/raw_sentences.txt"); + SentenceIterator iter = new BasicLineIterator(file); + + // InMemoryLookupCache cache = new InMemoryLookupCache(false); + AbstractCache cache = new AbstractCache.Builder().build(); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + LabelsSource source = new LabelsSource("DOC_"); + + ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(3).epochs(1).layerSize(100) + .learningRate(0.025).labelsSource(source).windowSize(5).iterate(iter).trainWordVectors(true) + .vocabCache(cache).tokenizerFactory(t).sampling(0).build(); + + vec.fit(); + + + int cnt1 = cache.wordFrequency("day"); + int cnt2 = cache.wordFrequency("me"); + + assertNotEquals(1, cnt1); + assertNotEquals(1, cnt2); + assertNotEquals(cnt1, cnt2); + + /* + We have few lines that contain pretty close words invloved. + These sentences should be pretty close to each other in vector space + */ + // line 3721: This is my way . + // line 6348: This is my case . + // line 9836: This is my house . + // line 12493: This is my world . + // line 16393: This is my work . + + // this is special sentence, that has nothing common with previous sentences + // line 9853: We now have one . + + assertTrue(vec.hasWord("DOC_3720")); + + double similarityD = vec.similarity("day", "night"); + log.info("day/night similarity: " + similarityD); + + double similarityW = vec.similarity("way", "work"); + log.info("way/work similarity: " + similarityW); + + double similarityH = vec.similarity("house", "world"); + log.info("house/world similarity: " + similarityH); + + double similarityC = vec.similarity("case", "way"); + log.info("case/way similarity: " + similarityC); + + double similarity1 = vec.similarity("DOC_9835", "DOC_12492"); + log.info("9835/12492 similarity: " + similarity1); + // assertTrue(similarity1 > 0.7d); + + double similarity2 = vec.similarity("DOC_3720", "DOC_16392"); + log.info("3720/16392 similarity: " + similarity2); + // assertTrue(similarity2 > 0.7d); + + double similarity3 = vec.similarity("DOC_6347", "DOC_3720"); + log.info("6347/3720 similarity: " + similarity3); + // assertTrue(similarity2 > 0.7d); + + // likelihood in this case should be significantly lower + // however, since corpus is small, and weight initialization is random-based, sometimes this test CAN fail + double similarityX = vec.similarity("DOC_3720", "DOC_9852"); + log.info("3720/9852 similarity: " + similarityX); + assertTrue(similarityX < 0.5d); + + + double sim119 = vec.similarityToLabel("This is my case .", "DOC_6347"); + double sim120 = vec.similarityToLabel("This is my case .", "DOC_3720"); + log.info("1/2: " + sim119 + "/" + sim120); + //assertEquals(similarity3, sim119, 0.001); + } + + + /** + * This test is not indicative. + * there's no need in this test within travis, use it manually only for problems detection + * + * @throws Exception + */ + @Test + //@Ignore + public void testParagraphVectorsReducedLabels1() throws Exception { + val tempDir = testDir; + ClassPathResource resource = new ClassPathResource("/labeled"); + resource.copyDirectory(tempDir); + + LabelAwareIterator iter = new FileLabelAwareIterator.Builder().addSourceFolder(tempDir).build(); + + TokenizerFactory t = new DefaultTokenizerFactory(); + + /** + * Please note: text corpus is REALLY small, and some kind of "results" could be received with HIGH epochs number, like 30. + * But there's no reason to keep at that high + */ + + ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).epochs(3).layerSize(100) + .stopWords(new ArrayList()).windowSize(5).iterate(iter).tokenizerFactory(t).build(); + + vec.fit(); + + //WordVectorSerializer.writeWordVectors(vec, "vectors.txt"); + + INDArray w1 = vec.lookupTable().vector("I"); + INDArray w2 = vec.lookupTable().vector("am"); + INDArray w3 = vec.lookupTable().vector("sad."); + + INDArray words = Nd4j.create(3, vec.lookupTable().layerSize()); + + words.putRow(0, w1); + words.putRow(1, w2); + words.putRow(2, w3); + + + INDArray mean = words.isMatrix() ? words.mean(0) : words; + + log.info("Mean" + Arrays.toString(mean.dup().data().asDouble())); + log.info("Array" + Arrays.toString(vec.lookupTable().vector("negative").dup().data().asDouble())); + + double simN = Transforms.cosineSim(mean, vec.lookupTable().vector("negative")); + log.info("Similarity negative: " + simN); + + + double simP = Transforms.cosineSim(mean, vec.lookupTable().vector("neutral")); + log.info("Similarity neutral: " + simP); + + double simV = Transforms.cosineSim(mean, vec.lookupTable().vector("positive")); + log.info("Similarity positive: " + simV); + } + + @Test + public void testParallelIterator() throws IOException { + TokenizerFactory factory = new DefaultTokenizerFactory(); + SentenceIterator iterator = new BasicLineIterator(Resources.asFile("big/raw_sentences.txt")); + + SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(iterator).allowMultithreading(true) + .tokenizerFactory(factory).build(); + + BasicTransformerIterator iter = (BasicTransformerIterator)transformer.iterator(); + for (int i = 0; i < 100; ++i) { + int cnt = 0; + long counter = 0; + Sequence sequence = null; + while (iter.hasNext()) { + sequence = iter.next(); + counter += sequence.size(); + cnt++; + } + iter.reset(); + assertEquals(757172, counter); + } + } + + @Test + public void testIterator() throws IOException { + val folder_labeled = testDir; + val folder_unlabeled = testDir; + new ClassPathResource("/paravec/labeled/").copyDirectory(folder_labeled); + new ClassPathResource("/paravec/unlabeled/").copyDirectory(folder_unlabeled); + + + FileLabelAwareIterator labelAwareIterator = new FileLabelAwareIterator.Builder() + .addSourceFolder(folder_labeled).build(); + + File resource_sentences = Resources.asFile("/big/raw_sentences.txt"); + SentenceIterator iter = new BasicLineIterator(resource_sentences); + + int i = 0; + for (; i < 10; ++i) { + int j = 0; + int labels = 0; + int words = 0; + while (labelAwareIterator.hasNextDocument()) { + ++j; + LabelledDocument document = labelAwareIterator.nextDocument(); + labels += document.getLabels().size(); + List lst = document.getReferencedContent(); + if (!CollectionUtils.isEmpty(lst)) + words += lst.size(); + } + labelAwareIterator.reset(); + //System.out.println(words + " " + labels + " " + j); + assertEquals(0, words); + assertEquals(30, labels); + assertEquals(30, j); + j = 0; + while (iter.hasNext()) { + ++j; + iter.nextSentence(); + } + assertEquals(97162, j); + iter.reset(); + } + + } + + /* + In this test we'll build w2v model, and will use it's vocab and weights for ParagraphVectors. + there's no need in this test within travis, use it manually only for problems detection + */ + @Test + public void testParagraphVectorsOverExistingWordVectorsModel() throws Exception { + String backend = Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"); + if(!isIntegrationTests() && "CUDA".equalsIgnoreCase(backend)) { + skipUnlessIntegrationTests(); //Skip CUDA except for integration tests due to very slow test speed + } + + // we build w2v from multiple sources, to cover everything + File resource_sentences = Resources.asFile("/big/raw_sentences.txt"); + + val folder_mixed = testDir; + ClassPathResource resource_mixed = new ClassPathResource("paravec/"); + resource_mixed.copyDirectory(folder_mixed); + + SentenceIterator iter = new AggregatingSentenceIterator.Builder() + .addSentenceIterator(new BasicLineIterator(resource_sentences)) + .addSentenceIterator(new FileSentenceIterator(folder_mixed)).build(); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + Word2Vec wordVectors = new Word2Vec.Builder().seed(119).minWordFrequency(1).batchSize(250).iterations(1).epochs(1) + .learningRate(0.025).layerSize(150).minLearningRate(0.001) + .elementsLearningAlgorithm(new SkipGram()).useHierarchicSoftmax(true).windowSize(5) + .allowParallelTokenization(true) + .workers(1) + .iterate(iter).tokenizerFactory(t).build(); + + wordVectors.fit(); + + VocabWord day_A = wordVectors.getVocab().tokenFor("day"); + + INDArray vector_day1 = wordVectors.getWordVectorMatrix("day").dup(); + + // At this moment we have ready w2v model. It's time to use it for ParagraphVectors + + val folder_labeled = testDir; + val folder_unlabeled = testDir; + new ClassPathResource("/paravec/labeled/").copyDirectory(folder_labeled); + new ClassPathResource("/paravec/unlabeled/").copyDirectory(folder_unlabeled); + + + FileLabelAwareIterator labelAwareIterator = new FileLabelAwareIterator.Builder() + .addSourceFolder(folder_labeled).build(); + + + // documents from this iterator will be used for classification + FileLabelAwareIterator unlabeledIterator = new FileLabelAwareIterator.Builder() + .addSourceFolder(folder_unlabeled).build(); + + + // we're building classifier now, with pre-built w2v model passed in + ParagraphVectors paragraphVectors = new ParagraphVectors.Builder().seed(119).iterate(labelAwareIterator) + .learningRate(0.025).minLearningRate(0.001).iterations(10).epochs(1).layerSize(150) + .tokenizerFactory(t).sequenceLearningAlgorithm(new DBOW()).useHierarchicSoftmax(true) + .allowParallelTokenization(true) + .workers(1) + .trainWordVectors(false).useExistingWordVectors(wordVectors).build(); + + paragraphVectors.fit(); + + VocabWord day_B = paragraphVectors.getVocab().tokenFor("day"); + + assertEquals(day_A.getIndex(), day_B.getIndex()); + + /* + double similarityD = wordVectors.similarity("day", "night"); + log.info("day/night similarity: " + similarityD); + assertTrue(similarityD > 0.5d); + */ + + INDArray vector_day2 = paragraphVectors.getWordVectorMatrix("day").dup(); + double crossDay = arraysSimilarity(vector_day1, vector_day2); + + log.info("Day1: " + vector_day1); + log.info("Day2: " + vector_day2); + log.info("Cross-Day similarity: " + crossDay); + log.info("Cross-Day similiarity 2: " + Transforms.cosineSim(Transforms.unitVec(vector_day1), Transforms.unitVec(vector_day2))); + + assertTrue(crossDay > 0.9d); + + /** + * + * Here we're checking cross-vocabulary equality + * + */ + /* + Random rnd = new Random(); + VocabCache cacheP = paragraphVectors.getVocab(); + VocabCache cacheW = wordVectors.getVocab(); + for (int x = 0; x < 1000; x++) { + int idx = rnd.nextInt(cacheW.numWords()); + + String wordW = cacheW.wordAtIndex(idx); + String wordP = cacheP.wordAtIndex(idx); + + assertEquals(wordW, wordP); + + INDArray arrayW = wordVectors.getWordVectorMatrix(wordW); + INDArray arrayP = paragraphVectors.getWordVectorMatrix(wordP); + + double simWP = Transforms.cosineSim(arrayW, arrayP); + assertTrue(simWP >= 0.9); + } + */ + + log.info("Zfinance: " + paragraphVectors.getWordVectorMatrix("Zfinance")); + log.info("Zhealth: " + paragraphVectors.getWordVectorMatrix("Zhealth")); + log.info("Zscience: " + paragraphVectors.getWordVectorMatrix("Zscience")); + + assertTrue(unlabeledIterator.hasNext()); + LabelledDocument document = unlabeledIterator.nextDocument(); + + log.info("Results for document '" + document.getLabel() + "'"); + + List results = new ArrayList<>(paragraphVectors.predictSeveral(document, 3)); + for (String result : results) { + double sim = paragraphVectors.similarityToLabel(document, result); + log.info("Similarity to [" + result + "] is [" + sim + "]"); + } + + String topPrediction = paragraphVectors.predict(document); + assertEquals("Z"+document.getLabel(), topPrediction); + } + + /* + Left as reference implementation, before stuff was changed in w2v + */ + @Deprecated + private double arraysSimilarity(@NonNull INDArray array1, @NonNull INDArray array2) { + if (array1.equals(array2)) + return 1.0; + + INDArray vector = Transforms.unitVec(array1); + INDArray vector2 = Transforms.unitVec(array2); + + if (vector == null || vector2 == null) + return -1; + + return Transforms.cosineSim(vector, vector2); + + } + + /** + * Special test to check d2v inference against pre-trained gensim model and + */ + //@Ignore + @Test + public void testGensimEquality() throws Exception { + + INDArray expA = Nd4j.create(new double[] {-0.02461922, -0.00801059, -0.01821643, 0.0167951, 0.02240154, + -0.00414107, -0.0022868, 0.00278438, -0.00651088, -0.02066556, -0.01045411, -0.02853066, + 0.00153375, 0.02707097, -0.00754221, -0.02795872, -0.00275301, -0.01455731, -0.00981289, + 0.01557207, -0.005259, 0.00355505, 0.01503531, -0.02185878, 0.0339283, -0.05049067, 0.02849454, + -0.01242505, 0.00438659, -0.03037345, 0.01866657, -0.00740161, -0.01850279, 0.00851284, + -0.01774663, -0.01976997, -0.03317627, 0.00372983, 0.01313218, -0.00041131, 0.00089357, + -0.0156924, 0.01278253, -0.01596088, -0.01415407, -0.01795845, 0.00558284, -0.00529536, + -0.03508032, 0.00725479, -0.01910841, -0.0008098, 0.00614283, -0.00926585, 0.01761538, + -0.00272953, -0.01483113, 0.02062481, -0.03134528, 0.03416841, -0.0156226, -0.01418961, + -0.00817538, 0.01848741, 0.00444605, 0.01090323, 0.00746163, -0.02490317, 0.00835013, + 0.01091823, -0.0177979, 0.0207753, -0.00854185, 0.04269911, 0.02786852, 0.00179449, 0.00303065, + -0.00127148, -0.01589409, -0.01110292, 0.01736244, -0.01177608, 0.00110929, 0.01790557, + -0.01800732, 0.00903072, 0.00210271, 0.0103053, -0.01508116, 0.00336775, 0.00319031, + -0.00982859, 0.02409827, -0.0079536, 0.01347831, -0.02555985, 0.00282605, 0.00350526, + -0.00471707, -0.00592073, -0.01009063, -0.02396305, 0.02643895, -0.05487461, -0.01710705, + -0.0082839, 0.01322765, 0.00098093, 0.01707118, 0.00290805, 0.03256396, 0.00277155, 0.00350602, + 0.0096487, -0.0062662, 0.0331796, -0.01758772, 0.0295204, 0.00295053, -0.00670782, 0.02172252, + 0.00172433, 0.0122977, -0.02401575, 0.01179839, -0.01646545, -0.0242724, 0.01318037, + -0.00745518, -0.00400624, -0.01735787, 0.01627645, 0.04445697, -0.0189355, 0.01315041, + 0.0131585, 0.01770667, -0.00114554, 0.00581599, 0.00745188, -0.01318868, -0.00801476, + -0.00884938, 0.00084786, 0.02578231, -0.01312729, -0.02047793, 0.00485749, -0.00342519, + -0.00744475, 0.01180929, 0.02871456, 0.01483848, -0.00696516, 0.02003011, -0.01721076, + -0.0124568, -0.0114492, -0.00970469, 0.01971609, 0.01599673, -0.01426137, 0.00808409, + -0.01431519, 0.01187332, 0.00144421, -0.00459554, 0.00384032, 0.00866845, 0.00265177, + -0.01003456, 0.0289338, 0.00353483, -0.01664903, -0.03050662, 0.01305057, -0.0084294, + -0.01615093, -0.00897918, 0.00768479, 0.02155688, 0.01594496, 0.00034328, -0.00557031, + -0.00256555, 0.03939554, 0.00274235, 0.001288, 0.02933025, 0.0070212, -0.00573742, 0.00883708, + 0.00829396, -0.01100356, -0.02653269, -0.01023274, 0.03079773, -0.00765917, 0.00949703, + 0.01212146, -0.01362515, -0.0076843, -0.00290596, -0.01707907, 0.02899382, -0.00089925, + 0.01510732, 0.02378234, -0.00947305, 0.0010998, -0.00558241, 0.00057873, 0.01098226, + -0.02019168, -0.013942, -0.01639287, -0.00675588, -0.00400709, -0.02914054, -0.00433462, + 0.01551765, -0.03552055, 0.01681101, -0.00629782, -0.01698086, 0.01891401, 0.03597684, + 0.00888052, -0.01587857, 0.00935822, 0.00931327, -0.0128156, 0.05170929, -0.01811879, + 0.02096679, 0.00897546, 0.00132624, -0.01796336, 0.01888563, -0.01142226, -0.00805926, + 0.00049782, -0.02151541, 0.00747257, 0.023373, -0.00198183, 0.02968843, 0.00443042, -0.00328569, + -0.04200815, 0.01306543, -0.01608924, -0.01604842, 0.03137267, 0.0266054, 0.00172526, + -0.01205696, 0.00047532, 0.00321026, 0.00671424, 0.01710422, -0.01129941, 0.00268044, + -0.01065434, -0.01107133, 0.00036135, -0.02991677, 0.02351665, -0.00343891, -0.01736755, + -0.00100577, -0.00312481, -0.01083809, 0.00387084, 0.01136449, 0.01675043, -0.01978249, + -0.00765182, 0.02746241, -0.01082247, -0.01587164, 0.01104732, -0.00878782, -0.00497555, + -0.00186257, -0.02281011, 0.00141792, 0.00432851, -0.01290263, -0.00387155, 0.00802639, + -0.00761913, 0.01508144, 0.02226428, 0.0107248, 0.01003709, 0.01587571, 0.00083492, -0.01632052, + -0.00435973}); + INDArray expB = Nd4j.create(new double[] {-0.02465764, 0.00756337, -0.0268607, 0.01588023, 0.01580242, + -0.00150542, 0.00116652, 0.0021577, -0.00754891, -0.02441176, -0.01271976, -0.02015191, + 0.00220599, 0.03722657, -0.01629612, -0.02779619, -0.01157856, -0.01937938, -0.00744667, + 0.01990043, -0.00505888, 0.00573646, 0.00385467, -0.0282531, 0.03484593, -0.05528606, + 0.02428633, -0.01510474, 0.00153177, -0.03637344, 0.01747423, -0.00090738, -0.02199888, + 0.01410434, -0.01710641, -0.01446697, -0.04225266, 0.00262217, 0.00871943, 0.00471594, + 0.0101348, -0.01991908, 0.00874325, -0.00606416, -0.01035323, -0.01376545, 0.00451507, + -0.01220307, -0.04361237, 0.00026028, -0.02401881, 0.00580314, 0.00238946, -0.01325974, + 0.01879044, -0.00335623, -0.01631887, 0.02222102, -0.02998703, 0.03190075, -0.01675236, + -0.01799807, -0.01314015, 0.01950069, 0.0011723, 0.01013178, 0.01093296, -0.034143, 0.00420227, + 0.01449351, -0.00629987, 0.01652851, -0.01286825, 0.03314656, 0.03485073, 0.01120341, + 0.01298241, 0.0019494, -0.02420256, -0.0063762, 0.01527091, -0.00732881, 0.0060427, 0.019327, + -0.02068196, 0.00876712, 0.00292274, 0.01312969, -0.01529114, 0.0021757, -0.00565621, + -0.01093122, 0.02758765, -0.01342688, 0.01606117, -0.02666447, 0.00541112, 0.00375426, + -0.00761796, 0.00136015, -0.01169962, -0.03012749, 0.03012953, -0.05491332, -0.01137303, + -0.01392103, 0.01370098, -0.00794501, 0.0248435, 0.00319645, 0.04261713, -0.00364211, + 0.00780485, 0.01182583, -0.00647098, 0.03291231, -0.02515565, 0.03480943, 0.00119836, + -0.00490694, 0.02615346, -0.00152456, 0.00196142, -0.02326461, 0.00603225, -0.02414703, + -0.02540966, 0.0072112, -0.01090273, -0.00505061, -0.02196866, 0.00515245, 0.04981546, + -0.02237269, -0.00189305, 0.0169786, 0.01782372, -0.00430022, 0.00551226, 0.00293861, + -0.01337168, -0.00302476, -0.01869966, 0.00270757, 0.03199976, -0.01614617, -0.02716484, + 0.01560035, -0.01312686, -0.01604082, 0.01347521, 0.03229654, 0.00707219, -0.00588392, + 0.02444809, -0.01068742, -0.0190814, -0.00556385, -0.00462766, 0.01283929, 0.02001247, + -0.00837629, -0.00041943, -0.02298774, 0.00874839, 0.00434907, -0.00963332, 0.00476905, + 0.00793049, -0.00212557, -0.01839353, 0.03345517, 0.00838255, -0.0157447, -0.0376134, + 0.01059611, -0.02323246, -0.01326356, -0.01116734, 0.00598869, 0.0211626, 0.01872963, + -0.0038276, -0.01208279, -0.00989125, 0.04147648, 0.00181867, -0.00369355, 0.02312465, + 0.0048396, 0.00564515, 0.01317832, -0.0057621, -0.01882041, -0.02869064, -0.00670661, + 0.02585443, -0.01108428, 0.01411031, 0.01204507, -0.01244726, -0.00962342, -0.00205239, + -0.01653971, 0.02871559, -0.00772978, 0.0214524, 0.02035478, -0.01324312, 0.00169302, + -0.00064739, 0.00531795, 0.01059279, -0.02455794, -0.00002782, -0.0068906, -0.0160858, + -0.0031842, -0.02295724, 0.01481094, 0.01769004, -0.02925742, 0.02050495, -0.00029003, + -0.02815636, 0.02467367, 0.03419458, 0.00654938, -0.01847546, 0.00999932, 0.00059222, + -0.01722176, 0.05172159, -0.01548486, 0.01746444, 0.007871, 0.0078471, -0.02414417, 0.01898077, + -0.01470176, -0.00299465, 0.00368212, -0.02474656, 0.01317451, 0.03706085, -0.00032923, + 0.02655881, 0.0013586, -0.0120303, -0.05030316, 0.0222294, -0.0070967, -0.02150935, 0.03254268, + 0.01369857, 0.00246183, -0.02253576, -0.00551247, 0.00787363, 0.01215617, 0.02439827, + -0.01104699, -0.00774596, -0.01898127, -0.01407653, 0.00195514, -0.03466602, 0.01560903, + -0.01239944, -0.02474852, 0.00155114, 0.00089324, -0.01725949, -0.00011816, 0.00742845, + 0.01247074, -0.02467943, -0.00679623, 0.01988366, -0.00626181, -0.02396477, 0.01052101, + -0.01123178, -0.00386291, -0.00349261, -0.02714747, -0.00563315, 0.00228767, -0.01303677, + -0.01971108, 0.00014759, -0.00346399, 0.02220698, 0.01979946, -0.00526076, 0.00647453, + 0.01428513, 0.00223467, -0.01690172, -0.0081715}); + + VectorsConfiguration configuration = new VectorsConfiguration(); + + configuration.setIterations(5); + configuration.setLearningRate(0.01); + configuration.setUseHierarchicSoftmax(true); + configuration.setNegative(0); + + Word2Vec w2v = WordVectorSerializer.readWord2VecFromText( + new File("/home/raver119/Downloads/gensim_models_for_dl4j/word"), + new File("/home/raver119/Downloads/gensim_models_for_dl4j/hs"), + new File("/home/raver119/Downloads/gensim_models_for_dl4j/hs_code"), + new File("/home/raver119/Downloads/gensim_models_for_dl4j/hs_mapping"), configuration); + + TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory(); + tokenizerFactory.setTokenPreProcessor(new CommonPreprocessor()); + + + assertNotEquals(null, w2v.getLookupTable()); + assertNotEquals(null, w2v.getVocab()); + + ParagraphVectors d2v = new ParagraphVectors.Builder(configuration).useExistingWordVectors(w2v) + .sequenceLearningAlgorithm(new DM()).tokenizerFactory(tokenizerFactory) + .resetModel(false).build(); + + + assertNotEquals(null, d2v.getLookupTable()); + assertNotEquals(null, d2v.getVocab()); + + assertTrue(d2v.getVocab() == w2v.getVocab()); + assertTrue(d2v.getLookupTable() == w2v.getLookupTable()); + + String textA = "Donald Trump referred to President Obama as \"your president\" during the first presidential debate on Monday, much to many people’s chagrin on social media. Trump, made the reference after saying that the greatest threat facing the world is nuclear weapons. He then turned to Hillary Clinton and said, \"Not global warming like you think and your President thinks,\" referring to Obama."; + + String textB = "The comment followed Trump doubling down on his false claims about the so-called birther conspiracy theory about Obama. People following the debate were immediately angered that Trump implied Obama is not his president."; + + String textC = "practice of trust owned Trump for example indeed and conspiracy between provoke"; + + INDArray arrayA = d2v.inferVector(textA); + INDArray arrayB = d2v.inferVector(textB); + INDArray arrayC = d2v.inferVector(textC); + + assertNotEquals(null, arrayA); + assertNotEquals(null, arrayB); + + Transforms.unitVec(arrayA); + Transforms.unitVec(arrayB); + + Transforms.unitVec(expA); + Transforms.unitVec(expB); + + double simX = Transforms.cosineSim(arrayA, arrayB); + double simC = Transforms.cosineSim(arrayA, arrayC); + double simB = Transforms.cosineSim(arrayB, expB); + + log.info("SimilarityX: {}", simX); + log.info("SimilarityC: {}", simC); + log.info("SimilarityB: {}", simB); + } + + @Test + //@Ignore //AB 2020/02/06 - https://github.com/eclipse/deeplearning4j/issues/8677 + @Disabled + public void testDirectInference() throws Exception { + boolean isIntegration = isIntegrationTests(); + File resource = Resources.asFile("/big/raw_sentences.txt"); + SentenceIterator sentencesIter = getIterator(isIntegration, resource); + + ClassPathResource resource_mixed = new ClassPathResource("paravec/"); + File local_resource_mixed = testDir; + resource_mixed.copyDirectory(local_resource_mixed); + SentenceIterator iter = new AggregatingSentenceIterator.Builder() + .addSentenceIterator(sentencesIter) + .addSentenceIterator(new FileSentenceIterator(local_resource_mixed)).build(); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + Word2Vec wordVectors = new Word2Vec.Builder().minWordFrequency(1).batchSize(250).iterations(1).epochs(1) + .learningRate(0.025).layerSize(150).minLearningRate(0.001) + .elementsLearningAlgorithm(new SkipGram()).useHierarchicSoftmax(true).windowSize(5) + .iterate(iter).tokenizerFactory(t).build(); + + wordVectors.fit(); + + ParagraphVectors pv = new ParagraphVectors.Builder().tokenizerFactory(t).iterations(10) + .useHierarchicSoftmax(true).trainWordVectors(true).useExistingWordVectors(wordVectors) + .negativeSample(0).sequenceLearningAlgorithm(new DM()).build(); + + INDArray vec1 = pv.inferVector("This text is pretty awesome"); + INDArray vec2 = pv.inferVector("Fantastic process of crazy things happening inside just for history purposes"); + + log.info("vec1/vec2: {}", Transforms.cosineSim(vec1, vec2)); + } + + + @Test + @Disabled + public void testGoogleModelForInference() throws Exception { + WordVectors googleVectors = WordVectorSerializer.readWord2VecModel(new File("/ext/GoogleNews-vectors-negative300.bin.gz")); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + ParagraphVectors pv = + new ParagraphVectors.Builder().tokenizerFactory(t).iterations(10).useHierarchicSoftmax(false) + .trainWordVectors(false).iterations(10).useExistingWordVectors(googleVectors) + .negativeSample(10).sequenceLearningAlgorithm(new DM()).build(); + + INDArray vec1 = pv.inferVector("This text is pretty awesome"); + INDArray vec2 = pv.inferVector("Fantastic process of crazy things happening inside just for history purposes"); + + log.info("vec1/vec2: {}", Transforms.cosineSim(vec1, vec2)); + } + + @Test + public void testHash() { + VocabWord w1 = new VocabWord(1.0, "D1"); + VocabWord w2 = new VocabWord(1.0, "Bo"); + + + + log.info("W1 > Short hash: {}; Long hash: {}", w1.getLabel().hashCode(), w1.getStorageId()); + log.info("W2 > Short hash: {}; Long hash: {}", w2.getLabel().hashCode(), w2.getStorageId()); + + assertNotEquals(w1.getStorageId(), w2.getStorageId()); + } + + + /** + * This is very long test, to track memory consumption over time + * + * @throws Exception + */ + @Test + @Tag("long-running") + public void testsParallelFit1() throws Exception { + final File file = Resources.asFile("big/raw_sentences.txt"); + + for (int i = 0; i < 1000; i++) { + List threads = new ArrayList<>(); + for (int t = 0; t < 3; t++) { + threads.add(new Thread(new Runnable() { + @Override + public void run() { + try { + TokenizerFactory t = new DefaultTokenizerFactory(); + + LabelsSource source = new LabelsSource("DOC_"); + + SentenceIteratorConverter sic = + new SentenceIteratorConverter(new BasicLineIterator(file), source); + + ParagraphVectors vec = new ParagraphVectors.Builder().seed(42) + //.batchSize(10) + .minWordFrequency(1).iterations(1).epochs(5).layerSize(100) + .learningRate(0.05) + //.labelsSource(source) + .windowSize(5).trainWordVectors(true).allowParallelTokenization(false) + //.vocabCache(cache) + .tokenizerFactory(t).workers(1).iterate(sic).build(); + + vec.fit(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + })); + } + + for (Thread t : threads) { + t.start(); + } + + for (Thread t : threads) { + t.join(); + } + } + } + + @Test + public void testJSONSerialization() { + ParagraphVectors paragraphVectors = new ParagraphVectors.Builder().build(); + AbstractCache cache = new AbstractCache.Builder().build(); + + val words = new VocabWord[3]; + words[0] = new VocabWord(1.0, "word"); + words[1] = new VocabWord(2.0, "test"); + words[2] = new VocabWord(3.0, "tester"); + + for (int i = 0; i < words.length; ++i) { + cache.addToken(words[i]); + cache.addWordToIndex(i, words[i].getLabel()); + } + paragraphVectors.setVocab(cache); + + String json = null; + Word2Vec unserialized = null; + try { + json = paragraphVectors.toJson(); + log.info("{}", json.toString()); + + unserialized = ParagraphVectors.fromJson(json); + } catch (Exception e) { + log.error("",e); + fail(); + } + + assertEquals(cache.totalWordOccurrences(), ((ParagraphVectors) unserialized).getVocab().totalWordOccurrences()); + assertEquals(cache.totalNumberOfDocs(), ((ParagraphVectors) unserialized).getVocab().totalNumberOfDocs()); + + for (int i = 0; i < words.length; ++i) { + val cached = cache.wordAtIndex(i); + val restored = ((ParagraphVectors) unserialized).getVocab().wordAtIndex(i); + assertNotNull(cached); + assertEquals(cached, restored); + } + } + + @Test + public void testDoubleFit() throws Exception { + boolean isIntegration = isIntegrationTests(); + File resource = Resources.asFile("/big/raw_sentences.txt"); + SentenceIterator iter = getIterator(isIntegration, resource); + + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + LabelsSource source = new LabelsSource("DOC_"); + + val builder = new ParagraphVectors.Builder(); + ParagraphVectors vec = builder.minWordFrequency(1).iterations(5).seed(119).epochs(1) + .layerSize(150).learningRate(0.025).labelsSource(source).windowSize(5) + .sequenceLearningAlgorithm(new DM()).iterate(iter).trainWordVectors(true) + .usePreciseWeightInit(true) + .batchSize(8192) + .allowParallelTokenization(false) + .tokenizerFactory(t).workers(1).sampling(0).build(); + + vec.fit(); + long num1 = vec.vocab().totalNumberOfDocs(); + + vec.fit(); + System.out.println(vec.vocab().totalNumberOfDocs()); + long num2 = vec.vocab().totalNumberOfDocs(); + + assertEquals(num1, num2); + } + + public static SentenceIterator getIterator(boolean isIntegration, File file) throws IOException { + return getIterator(isIntegration, file, 500); + } + + public static SentenceIterator getIterator(boolean isIntegration, File file, int linesForUnitTest) throws IOException { + if(isIntegration){ + return new BasicLineIterator(file); + } else { + List lines = new ArrayList<>(); + try(InputStream is = new BufferedInputStream(new FileInputStream(file))){ + LineIterator lineIter = IOUtils.lineIterator(is, StandardCharsets.UTF_8); + try{ + for( int i=0; i graph; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalkerTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalkerTest.java similarity index 98% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalkerTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalkerTest.java index 441de3d2c..7c150a610 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalkerTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/RandomWalkerTest.java @@ -32,16 +32,12 @@ import org.deeplearning4j.models.sequencevectors.graph.walkers.GraphWalker; import org.deeplearning4j.models.sequencevectors.sequence.Sequence; import org.deeplearning4j.models.word2vec.VocabWord; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag + public class RandomWalkerTest extends BaseDL4JTest { private static IGraph graph; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalkerTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalkerTest.java similarity index 95% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalkerTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalkerTest.java index 57bfd674a..7cf36eb9e 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalkerTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/graph/walkers/impl/WeightedWalkerTest.java @@ -29,15 +29,11 @@ import org.deeplearning4j.models.sequencevectors.graph.walkers.GraphWalker; import org.deeplearning4j.models.sequencevectors.sequence.Sequence; import org.deeplearning4j.models.word2vec.VocabWord; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + public class WeightedWalkerTest extends BaseDL4JTest { private static Graph basicGraph; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactoryTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactoryTest.java similarity index 93% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactoryTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactoryTest.java index 3b6aedb11..ee7d33022 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactoryTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/AbstractElementFactoryTest.java @@ -23,14 +23,10 @@ package org.deeplearning4j.models.sequencevectors.serialization; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.models.word2vec.VocabWord; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + public class AbstractElementFactoryTest extends BaseDL4JTest { @BeforeEach diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/ExtVocabWord.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/ExtVocabWord.java similarity index 93% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/ExtVocabWord.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/ExtVocabWord.java index f193bd1ec..aa2ab2d10 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/ExtVocabWord.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/ExtVocabWord.java @@ -24,8 +24,8 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.NonNull; import org.deeplearning4j.models.word2vec.VocabWord; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY, getterVisibility = JsonAutoDetect.Visibility.NONE, diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactoryTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactoryTest.java similarity index 93% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactoryTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactoryTest.java index d4ca621f0..6ad958e79 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactoryTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/serialization/VocabWordFactoryTest.java @@ -23,14 +23,10 @@ package org.deeplearning4j.models.sequencevectors.serialization; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.models.word2vec.VocabWord; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + public class VocabWordFactoryTest extends BaseDL4JTest { @BeforeEach diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformerTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformerTest.java similarity index 95% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformerTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformerTest.java index d0051f7bd..e0cb1c5cc 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformerTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/GraphTransformerTest.java @@ -30,16 +30,12 @@ import org.deeplearning4j.models.sequencevectors.graph.walkers.impl.RandomWalker import org.deeplearning4j.models.sequencevectors.sequence.Sequence; import org.deeplearning4j.models.word2vec.VocabWord; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.Iterator; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + public class GraphTransformerTest extends BaseDL4JTest { private static IGraph graph; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIteratorTest.java similarity index 88% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIteratorTest.java index 1e7c4c746..ceef572e1 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/sequencevectors/transformers/impl/iterables/ParallelTransformerIteratorTest.java @@ -33,10 +33,10 @@ import org.deeplearning4j.text.sentenceiterator.MutipleEpochsSentenceIterator; import org.deeplearning4j.text.sentenceiterator.SentenceIterator; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.InputStream; import java.util.Iterator; @@ -46,8 +46,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag +@Timeout(300) public class ParallelTransformerIteratorTest extends BaseDL4JTest { private TokenizerFactory factory = new DefaultTokenizerFactory(); @@ -56,8 +55,7 @@ public class ParallelTransformerIteratorTest extends BaseDL4JTest { } - @Test() - @Timeout(30000) + @Test public void hasNext() throws Exception { SentenceIterator iterator = new BasicLineIterator(Resources.asFile("big/raw_sentences.txt")); @@ -69,8 +67,8 @@ public class ParallelTransformerIteratorTest extends BaseDL4JTest { Sequence sequence = null; while (iter.hasNext()) { sequence = iter.next(); - assertNotEquals( null, sequence,"Failed on [" + cnt + "] iteration"); - assertNotEquals(0, sequence.size(),"Failed on [" + cnt + "] iteration"); + assertNotEquals( null, sequence, "Failed on [" + cnt + "] iteration"); + assertNotEquals( 0, sequence.size(), "Failed on [" + cnt + "] iteration"); cnt++; } @@ -79,8 +77,7 @@ public class ParallelTransformerIteratorTest extends BaseDL4JTest { assertEquals(97162, cnt); } - @Test() - @Timeout(30000) + @Test public void testSpeedComparison1() throws Exception { SentenceIterator iterator = new MutipleEpochsSentenceIterator( new BasicLineIterator(Resources.asFile("big/raw_sentences.txt")), 25); @@ -93,8 +90,8 @@ public class ParallelTransformerIteratorTest extends BaseDL4JTest { long time1 = System.currentTimeMillis(); while (iter.hasNext()) { Sequence sequence = iter.next(); - assertNotEquals(null, sequence,"Failed on [" + cnt + "] iteration"); - assertNotEquals( 0, sequence.size(),"Failed on [" + cnt + "] iteration"); + assertNotEquals( null, sequence, "Failed on [" + cnt + "] iteration"); + assertNotEquals( 0, sequence.size(), "Failed on [" + cnt + "] iteration"); cnt++; } long time2 = System.currentTimeMillis(); @@ -110,8 +107,8 @@ public class ParallelTransformerIteratorTest extends BaseDL4JTest { time1 = System.currentTimeMillis(); while (iter.hasNext()) { Sequence sequence = iter.next(); - assertNotEquals(null, sequence,"Failed on [" + cnt + "] iteration"); - assertNotEquals(0, sequence.size(),"Failed on [" + cnt + "] iteration"); + assertNotEquals( null, sequence, "Failed on [" + cnt + "] iteration"); + assertNotEquals( 0, sequence.size(), "Failed on [" + cnt + "] iteration"); cnt++; } time2 = System.currentTimeMillis(); @@ -135,7 +132,7 @@ public class ParallelTransformerIteratorTest extends BaseDL4JTest { while (iter.hasNext()) { Sequence sequence = iter.next(); assertNotEquals(null, sequence, "Failed on [" + cnt + "] iteration"); - assertNotEquals(0, sequence.size(),"Failed on [" + cnt + "] iteration"); + assertNotEquals( 0, sequence.size(), "Failed on [" + cnt + "] iteration"); cnt++; } time2 = System.currentTimeMillis(); @@ -152,8 +149,8 @@ public class ParallelTransformerIteratorTest extends BaseDL4JTest { time1 = System.currentTimeMillis(); while (iter.hasNext()) { Sequence sequence = iter.next(); - assertNotEquals(null, sequence, "Failed on [" + cnt + "] iteration"); - assertNotEquals(0, sequence.size(),"Failed on [" + cnt + "] iteration"); + assertNotEquals( null, sequence, "Failed on [" + cnt + "] iteration"); + assertNotEquals( 0, sequence.size(), "Failed on [" + cnt + "] iteration"); cnt++; } time2 = System.currentTimeMillis(); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecTestsSmall.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecTestsSmall.java similarity index 97% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecTestsSmall.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecTestsSmall.java index a81377fc2..6d7bfaf63 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecTestsSmall.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecTestsSmall.java @@ -37,8 +37,6 @@ import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFac import org.deeplearning4j.util.ModelSerializer; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Timeout; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -62,8 +60,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag public class Word2VecTestsSmall extends BaseDL4JTest { WordVectors word2vec; @@ -98,8 +94,8 @@ public class Word2VecTestsSmall extends BaseDL4JTest { assertEquals(neighbours, nearestWords.size()); } - @Test() - @Timeout(300000) + @Test + @Timeout(300) public void testUnkSerialization_1() throws Exception { val inputFile = Resources.asFile("big/raw_sentences.txt"); // val iter = new BasicLineIterator(inputFile); @@ -159,8 +155,9 @@ public class Word2VecTestsSmall extends BaseDL4JTest { } - @Test() - @Timeout(300000) + @Test + @Timeout(300) + @Tag("long-running") public void testW2VEmbeddingLayerInit() throws Exception { Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecVisualizationTests.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecVisualizationTests.java similarity index 88% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecVisualizationTests.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecVisualizationTests.java index e80325c7c..5529b59ac 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecVisualizationTests.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/Word2VecVisualizationTests.java @@ -24,15 +24,10 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -@Disabled -@Tag(TagNames.FILE_IO) -@NativeTag +import org.junit.jupiter.api.Test; + +//@Ignore public class Word2VecVisualizationTests extends BaseDL4JTest { private static WordVectors vectors; diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIteratorTest.java similarity index 96% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIteratorTest.java index 23b9530f5..a72f92211 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/iterator/Word2VecDataSetIteratorTest.java @@ -32,11 +32,8 @@ import org.deeplearning4j.text.sentenceiterator.labelaware.LabelAwareSentenceIte import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.common.resources.Resources; @@ -48,8 +45,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertArrayEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + public class Word2VecDataSetIteratorTest extends BaseDL4JTest { @Override @@ -61,6 +57,7 @@ public class Word2VecDataSetIteratorTest extends BaseDL4JTest { * Basically all we want from this test - being able to finish without exceptions. */ @Test + //@Ignore public void testIterator1() throws Exception { File inputFile = Resources.asFile("big/raw_sentences.txt"); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructorTest.java similarity index 97% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructorTest.java index 5827fa4d7..33b2715bf 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructorTest.java @@ -24,7 +24,7 @@ import lombok.val; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator; @@ -40,30 +40,27 @@ import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer; import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.nio.file.Path; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag +@Timeout(300) public class VocabConstructorTest extends BaseDL4JTest { - - protected static final Logger log = LoggerFactory.getLogger(VocabConstructorTest.class); TokenizerFactory t = new DefaultTokenizerFactory(); - + @TempDir + public File testDir; @BeforeEach @@ -292,7 +289,7 @@ public class VocabConstructorTest extends BaseDL4JTest { } @Test - public void testMergedVocabWithLabels1(@TempDir Path testDir) throws Exception { + public void testMergedVocabWithLabels1() throws Exception { AbstractCache cacheSource = new AbstractCache.Builder().build(); AbstractCache cacheTarget = new AbstractCache.Builder().build(); @@ -316,7 +313,7 @@ public class VocabConstructorTest extends BaseDL4JTest { int sourceSize = cacheSource.numWords(); log.info("Source Vocab size: " + sourceSize); - val dir = testDir.toFile(); + val dir = testDir; new ClassPathResource("/paravec/labeled/").copyDirectory(dir); @@ -437,8 +434,8 @@ public class VocabConstructorTest extends BaseDL4JTest { } - @Test() // 5s timeout - @Timeout(5000) + @Test + @Timeout(5) // 5s timeout public void testParallelTokenizationDisabled_Completes() throws Exception { File inputFile = Resources.asFile("big/raw_sentences.txt"); SentenceIterator iter = new BasicLineIterator(inputFile); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolderTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolderTest.java similarity index 98% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolderTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolderTest.java index da6333c4e..41c1fc0ba 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolderTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/VocabularyHolderTest.java @@ -22,14 +22,10 @@ package org.deeplearning4j.models.word2vec.wordstore; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + public class VocabularyHolderTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCacheTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCacheTest.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCacheTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/models/word2vec/wordstore/inmemory/AbstractCacheTest.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIteratorTest.java similarity index 90% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIteratorTest.java index f1e27ca5a..27025b35e 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/AsyncLabelAwareIteratorTest.java @@ -23,21 +23,15 @@ package org.deeplearning4j.text.documentiterator; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.text.sentenceiterator.BasicLineIterator; import org.deeplearning4j.text.sentenceiterator.SentenceIterator; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag +@Timeout(300) public class AsyncLabelAwareIteratorTest extends BaseDL4JTest { - @Test() - @Timeout(30000) + @Test public void nextDocument() throws Exception { SentenceIterator sentence = new BasicLineIterator(Resources.asFile("big/raw_sentences.txt")); BasicLabelAwareIterator backed = new BasicLabelAwareIterator.Builder(sentence).build(); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIteratorTest.java similarity index 92% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIteratorTest.java index 97134c790..85ab90cdd 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/BasicLabelAwareIteratorTest.java @@ -22,31 +22,28 @@ package org.deeplearning4j.text.documentiterator; import org.deeplearning4j.BaseDL4JTest; - import org.deeplearning4j.text.sentenceiterator.BasicLineIterator; import org.deeplearning4j.text.sentenceiterator.SentenceIterator; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + +@Timeout(300) public class BasicLabelAwareIteratorTest extends BaseDL4JTest { - - @BeforeEach - public void setUp() throws Exception {} + public void setUp() throws Exception { + + } @Test public void testHasNextDocument1() throws Exception { + File inputFile = Resources.asFile("big/raw_sentences.txt"); SentenceIterator iter = new BasicLineIterator(inputFile.getAbsolutePath()); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/DefaultDocumentIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/DefaultDocumentIteratorTest.java old mode 100755 new mode 100644 similarity index 93% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/DefaultDocumentIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/DefaultDocumentIteratorTest.java index 6a123ca78..06cdb5fcb --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/DefaultDocumentIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/DefaultDocumentIteratorTest.java @@ -24,18 +24,14 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import java.io.InputStream; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + public class DefaultDocumentIteratorTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileDocumentIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileDocumentIteratorTest.java similarity index 88% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileDocumentIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileDocumentIteratorTest.java index a16de22db..62cafc39d 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileDocumentIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileDocumentIteratorTest.java @@ -27,30 +27,31 @@ import org.apache.commons.io.IOUtils; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; import java.io.File; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashSet; import java.util.Set; +import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag +//@Ignore +@Timeout(20) public class FileDocumentIteratorTest extends BaseDL4JTest { - + @TempDir + public File testDir; @BeforeEach public void setUp() throws Exception { @@ -110,10 +111,9 @@ public class FileDocumentIteratorTest extends BaseDL4JTest { assertEquals(48, cnt); } - @Test() - @Timeout(5000) - public void testEmptyDocument(@TempDir Path testDir) throws Exception { - File f = Files.createTempFile(testDir,"newfile","bin").toFile(); + @Test + public void testEmptyDocument() throws Exception { + File f = new File(testDir, UUID.randomUUID().toString()); assertTrue(f.exists()); assertEquals(0, f.length()); @@ -125,10 +125,9 @@ public class FileDocumentIteratorTest extends BaseDL4JTest { } } - @Test() - @Timeout(5000) - public void testEmptyDocument2(@TempDir Path testDir) throws Exception { - File dir = testDir.toFile(); + @Test + public void testEmptyDocument2() throws Exception { + File dir = testDir; File f1 = new File(dir, "1.txt"); FileUtils.writeStringToFile(f1, "line 1\nline2", StandardCharsets.UTF_8); File f2 = new File(dir, "2.txt"); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIteratorTest.java similarity index 82% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIteratorTest.java index 24ad3b162..4c15e0f0b 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FileLabelAwareIteratorTest.java @@ -24,24 +24,19 @@ import lombok.val; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; -import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag public class FileLabelAwareIteratorTest extends BaseDL4JTest { + @ TempDir + public File testDir; @BeforeEach public void setUp() throws Exception { @@ -49,9 +44,8 @@ public class FileLabelAwareIteratorTest extends BaseDL4JTest { } @Test - public void testExtractLabelFromPath1(@TempDir Path testDir) throws Exception { - val dir = testDir.resolve("new-folder").toFile(); - dir.mkdirs(); + public void testExtractLabelFromPath1() throws Exception { + val dir = testDir; val resource = new ClassPathResource("/labeled/"); resource.copyDirectory(dir); @@ -78,13 +72,9 @@ public class FileLabelAwareIteratorTest extends BaseDL4JTest { @Test - public void testExtractLabelFromPath2(@TempDir Path testDir) throws Exception { - testDir = testDir.resolve("new-folder"); - testDir.toFile().mkdirs(); - val dir0 = new File(testDir.toFile(),"dir-0"); - val dir1 = new File(testDir.toFile(),"dir-1"); - dir0.mkdirs(); - dir1.mkdirs(); + public void testExtractLabelFromPath2() throws Exception { + val dir0 = testDir; + val dir1 = testDir; val resource = new ClassPathResource("/labeled/"); val resource2 = new ClassPathResource("/rootdir/"); resource.copyDirectory(dir0); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIteratorTest.java similarity index 87% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIteratorTest.java index 3bf3567b7..44a9ebd5a 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/FilenamesLabelAwareIteratorTest.java @@ -25,27 +25,21 @@ import org.deeplearning4j.BaseDL4JTest; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import java.nio.file.Path; +import java.io.File; import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.NEEDS_VERIFY) -@Disabled("Permissions issues on CI") public class FilenamesLabelAwareIteratorTest extends BaseDL4JTest { + @TempDir + public File testDir; @BeforeEach public void setUp() throws Exception { @@ -53,8 +47,8 @@ public class FilenamesLabelAwareIteratorTest extends BaseDL4JTest { } @Test - public void testNextDocument(@TempDir Path testDir) throws Exception { - val tempDir = testDir.toFile(); + public void testNextDocument() throws Exception { + val tempDir = testDir; Resources.copyDirectory("/big/", tempDir); FilenamesLabelAwareIterator iterator = new FilenamesLabelAwareIterator.Builder() diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/LabelsSourceTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/LabelsSourceTest.java similarity index 95% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/LabelsSourceTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/LabelsSourceTest.java index a8ecaf8a8..673b38485 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/documentiterator/LabelsSourceTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/documentiterator/LabelsSourceTest.java @@ -22,17 +22,13 @@ package org.deeplearning4j.text.documentiterator; import org.deeplearning4j.BaseDL4JTest; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) + public class LabelsSourceTest extends BaseDL4JTest { @BeforeEach diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIteratorTest.java similarity index 86% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIteratorTest.java index 176899cfb..67090ca46 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/AggregatingSentenceIteratorTest.java @@ -21,23 +21,18 @@ package org.deeplearning4j.text.sentenceiterator; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import static org.junit.jupiter.api.Assertions.assertEquals; +@Timeout(300) public class AggregatingSentenceIteratorTest extends BaseDL4JTest { - @Test() - @Timeout(30000) - @Disabled("Needs verification, could be permissions issues: g.opentest4j.AssertionFailedError: expected: <388648> but was: <262782> at line 60") - @Tag(TagNames.NEEDS_VERIFY) + @Test public void testHasNext() throws Exception { File file = Resources.asFile("/big/raw_sentences.txt"); BasicLineIterator iterator = new BasicLineIterator(file); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicLineIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicLineIteratorTest.java similarity index 91% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicLineIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicLineIteratorTest.java index 97235d5bc..5402ddad7 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicLineIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicLineIteratorTest.java @@ -22,31 +22,25 @@ package org.deeplearning4j.text.sentenceiterator; import org.deeplearning4j.BaseDL4JTest; - import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; import java.io.FileInputStream; import static org.junit.jupiter.api.Assertions.assertEquals; +@Timeout(30) public class BasicLineIteratorTest extends BaseDL4JTest { - - @BeforeEach public void setUp() throws Exception { } @Test - @Disabled(".opentest4j.AssertionFailedError: expected: <97162> but was: <16889> Line 66") - @Tag(TagNames.NEEDS_VERIFY) public void testHasMoreLinesFile() throws Exception { File file = Resources.asFile("/big/raw_sentences.txt"); BasicLineIterator iterator = new BasicLineIterator(file); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicResultSetIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicResultSetIteratorTest.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicResultSetIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/BasicResultSetIteratorTest.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIteratorTest.java similarity index 82% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIteratorTest.java index e4ce784bd..d2db2f6fd 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/MutipleEpochsSentenceIteratorTest.java @@ -21,20 +21,15 @@ package org.deeplearning4j.text.sentenceiterator; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; +@Timeout(300) public class MutipleEpochsSentenceIteratorTest extends BaseDL4JTest { - @Test() - @Timeout(30000) - @Disabled("Downloads need verification ile hash does not match expected hash: https://dl4jtest.blob.core.windows.net/resources/big/raw_sentences.txt.gzx.v1") - @Tag(TagNames.NEEDS_VERIFY) + @Test public void hasNext() throws Exception { SentenceIterator iterator = new MutipleEpochsSentenceIterator( new BasicLineIterator(Resources.asFile("big/raw_sentences.txt")), 100); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIteratorTest.java similarity index 98% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIteratorTest.java index 12524e3c7..e6241529a 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIteratorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/PrefetchingSentenceIteratorTest.java @@ -22,9 +22,8 @@ package org.deeplearning4j.text.sentenceiterator; import org.deeplearning4j.BaseDL4JTest; - -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.resources.Resources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,11 +33,10 @@ import java.io.File; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Disabled("Deprecated module") +@Timeout(30) public class PrefetchingSentenceIteratorTest extends BaseDL4JTest { - protected static final Logger log = LoggerFactory.getLogger(PrefetchingSentenceIteratorTest.class); @Test diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/StreamLineIteratorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/StreamLineIteratorTest.java similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/StreamLineIteratorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/sentenceiterator/StreamLineIteratorTest.java diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizerTests.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizerTests.java similarity index 96% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizerTests.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizerTests.java index cce78f317..d36ab414c 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizerTests.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/BertWordPieceTokenizerTests.java @@ -26,14 +26,11 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.BertWordPiecePreProcessor; import org.deeplearning4j.text.tokenization.tokenizerfactory.BertWordPieceTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.io.ClassPathResource; import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.ByteArrayInputStream; import java.io.File; @@ -46,9 +43,8 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled -@Tag(TagNames.FILE_IO) -@NativeTag +//@Ignore +@Timeout(300) public class BertWordPieceTokenizerTests extends BaseDL4JTest { private File pathToVocab = Resources.asFile("other/vocab.txt"); @@ -118,7 +114,7 @@ public class BertWordPieceTokenizerTests extends BaseDL4JTest { } @Test - @Disabled("AB 2019/05/24 - Disabled until dev branch merged - see issue #7657") + //@Ignore("AB 2019/05/24 - Disabled until dev branch merged - see issue #7657") public void testBertWordPieceTokenizer5() throws Exception { // Longest Token in Vocab is 22 chars long, so make sure splits on the edge are properly handled String toTokenize = "Donaudampfschifffahrts Kapitänsmützeninnenfuttersaum"; @@ -200,7 +196,7 @@ public class BertWordPieceTokenizerTests extends BaseDL4JTest { String m = e.getMessage(); assertNotNull(m); m = m.toLowerCase(); - assertTrue(m.contains("invalid") && m.contains("token") && m.contains("preprocessor"), m); + assertTrue( m.contains("invalid") && m.contains("token") && m.contains("preprocessor"), m); } try { @@ -216,8 +212,7 @@ public class BertWordPieceTokenizerTests extends BaseDL4JTest { } - @Test() - @Timeout(300000) + @Test public void testBertWordPieceTokenizer10() throws Exception { File f = Resources.asFile("deeplearning4j-nlp/bert/uncased_L-12_H-768_A-12/vocab.txt"); BertWordPieceTokenizerFactory t = new BertWordPieceTokenizerFactory(f, true, true, StandardCharsets.UTF_8); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/DefaulTokenizerTests.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/DefaulTokenizerTests.java old mode 100755 new mode 100644 similarity index 96% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/DefaulTokenizerTests.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/DefaulTokenizerTests.java index c4b873748..d3ab0bfc8 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/DefaulTokenizerTests.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/DefaulTokenizerTests.java @@ -22,13 +22,10 @@ package org.deeplearning4j.text.tokenization.tokenizer; import org.apache.commons.io.FileUtils; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; import org.nd4j.common.io.ClassPathResource; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,8 +33,7 @@ import java.io.ByteArrayInputStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@NativeTag + public class DefaulTokenizerTests extends BaseDL4JTest { protected static final Logger log = LoggerFactory.getLogger(DefaulTokenizerTests.class); diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizerTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizerTest.java similarity index 94% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizerTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizerTest.java index 59f08a359..6d36889cf 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizerTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/NGramTokenizerTest.java @@ -24,10 +24,7 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.NGramTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.List; @@ -38,8 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author sonali */ -@Tag(TagNames.FILE_IO) -@NativeTag public class NGramTokenizerTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/tokenprepreprocessor/EndingPreProcessorTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/tokenprepreprocessor/EndingPreProcessorTest.java old mode 100755 new mode 100644 similarity index 90% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/tokenprepreprocessor/EndingPreProcessorTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/tokenprepreprocessor/EndingPreProcessorTest.java index 8cc0855cc..03db99995 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/tokenprepreprocessor/EndingPreProcessorTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizer/tokenprepreprocessor/EndingPreProcessorTest.java @@ -23,14 +23,10 @@ package org.deeplearning4j.text.tokenization.tokenizer.tokenprepreprocessor; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess; import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.EndingPreProcessor; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag + public class EndingPreProcessorTest extends BaseDL4JTest { @Test public void testPreProcessor() { diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactoryTest.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactoryTest.java similarity index 93% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactoryTest.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactoryTest.java index 591b65d21..32ccee306 100644 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactoryTest.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/text/tokenization/tokenizerfactory/NGramTokenizerFactoryTest.java @@ -24,16 +24,11 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag public class NGramTokenizerFactoryTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/wordstore/InMemoryVocabStoreTests.java b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/wordstore/InMemoryVocabStoreTests.java old mode 100755 new mode 100644 similarity index 91% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/wordstore/InMemoryVocabStoreTests.java rename to cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/wordstore/InMemoryVocabStoreTests.java index 236e4aaa2..fab3d2e89 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/wordstore/InMemoryVocabStoreTests.java +++ b/cavis-dnn/cavis-dnn-nlp/src/test/java/org/deeplearning4j/wordstore/InMemoryVocabStoreTests.java @@ -24,16 +24,12 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.models.word2vec.VocabWord; import org.deeplearning4j.models.word2vec.wordstore.VocabCache; import org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag + public class InMemoryVocabStoreTests extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/resources/logback-test.xml b/cavis-dnn/cavis-dnn-nlp/src/test/resources/logback-test.xml similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/resources/logback-test.xml rename to cavis-dnn/cavis-dnn-nlp/src/test/resources/logback-test.xml diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/resources/models/fasttext/cbow.model.vec b/cavis-dnn/cavis-dnn-nlp/src/test/resources/models/fasttext/cbow.model.vec similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/resources/models/fasttext/cbow.model.vec rename to cavis-dnn/cavis-dnn-nlp/src/test/resources/models/fasttext/cbow.model.vec diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/resources/models/fasttext/skipgram.model.vec b/cavis-dnn/cavis-dnn-nlp/src/test/resources/models/fasttext/skipgram.model.vec similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/resources/models/fasttext/skipgram.model.vec rename to cavis-dnn/cavis-dnn-nlp/src/test/resources/models/fasttext/skipgram.model.vec diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/resources/models/fasttext/supervised.model.vec b/cavis-dnn/cavis-dnn-nlp/src/test/resources/models/fasttext/supervised.model.vec similarity index 100% rename from deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/resources/models/fasttext/supervised.model.vec rename to cavis-dnn/cavis-dnn-nlp/src/test/resources/models/fasttext/supervised.model.vec diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-client/build.gradle b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-client/build.gradle new file mode 100644 index 000000000..3695fe3d8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-client/build.gradle @@ -0,0 +1,32 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation "com.mashape.unirest:unirest-java:1.4.9" + //implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnClient + testImplementation projects.cavisDnn.cavisDnnCommonTests + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnModel +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-client/src/main/java/org/deeplearning4j/nearestneighbor/client/NearestNeighborsClient.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-client/src/main/java/org/deeplearning4j/nearestneighbor/client/NearestNeighborsClient.java new file mode 100644 index 000000000..4e185df0e --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-client/src/main/java/org/deeplearning4j/nearestneighbor/client/NearestNeighborsClient.java @@ -0,0 +1,142 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.nearestneighbor.client; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.mashape.unirest.http.ObjectMapper; +import com.mashape.unirest.http.Unirest; +import com.mashape.unirest.request.HttpRequest; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.val; +import org.deeplearning4j.nearestneighbor.model.Base64NDArrayBody; +import org.deeplearning4j.nearestneighbor.model.NearestNeighborRequest; +import org.deeplearning4j.nearestneighbor.model.NearestNeighborsResults; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.serde.base64.Nd4jBase64; + +import java.io.IOException; + +/** + * Client for the nearest neighbors server. + * To create a client, pass in a host port combination with the following format: + * http://host:port + * + * @author Adam Gibson + */ +@AllArgsConstructor +public class NearestNeighborsClient { + + private String url; + @Setter + @Getter + protected String authToken; + + public NearestNeighborsClient(String url){ + this(url, null); + } + + static { + // Only one time + + Unirest.setObjectMapper(new ObjectMapper() { + private com.fasterxml.jackson.databind.ObjectMapper jacksonObjectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public T readValue(String value, Class valueType) { + try { + return jacksonObjectMapper.readValue(value, valueType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String writeValue(Object value) { + try { + return jacksonObjectMapper.writeValueAsString(value); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + }); + } + + + /** + * Runs knn on the given index + * with the given k (note that this is for data + * already within the existing dataset not new data) + * @param index the index of the + * EXISTING ndarray + * to run a search on + * @param k the number of results + * @return + * @throws Exception + */ + public NearestNeighborsResults knn(int index, int k) throws Exception { + NearestNeighborRequest request = new NearestNeighborRequest(); + request.setInputIndex(index); + request.setK(k); + val req = Unirest.post(url + "/knn"); + req.header("accept", "application/json") + .header("Content-Type", "application/json").body(request); + addAuthHeader(req); + + NearestNeighborsResults ret = req.asObject(NearestNeighborsResults.class).getBody(); + return ret; + } + + /** + * Run a k nearest neighbors search + * on a NEW data point + * @param k the number of results + * to retrieve + * @param arr the array to run the search on. + * Note that this must be a row vector + * @return + * @throws Exception + */ + public NearestNeighborsResults knnNew(int k, INDArray arr) throws Exception { + Base64NDArrayBody base64NDArrayBody = + Base64NDArrayBody.builder().k(k).ndarray(Nd4jBase64.base64String(arr)).build(); + + val req = Unirest.post(url + "/knnnew"); + req.header("accept", "application/json") + .header("Content-Type", "application/json").body(base64NDArrayBody); + addAuthHeader(req); + + NearestNeighborsResults ret = req.asObject(NearestNeighborsResults.class).getBody(); + + return ret; + } + + + /** + * Add the specified authentication header to the specified HttpRequest + * + * @param request HTTP Request to add the authentication header to + */ + protected HttpRequest addAuthHeader(HttpRequest request) { + if (authToken != null) { + request.header("authorization", "Bearer " + authToken); + } + + return request; + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/build.gradle b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/build.gradle new file mode 100644 index 000000000..3eb796e6e --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/build.gradle @@ -0,0 +1,35 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisDnn.cavisDnnApi + testImplementation 'ch.qos.logback:logback-classic' + implementation projects.cavisDnn.cavisDnnNn + + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets + testImplementation 'joda-time:joda-time:2.10.3' + testImplementation projects.cavisDnn.cavisDnnCommonTests + implementation "org.apache.commons:commons-lang3" + implementation "org.apache.commons:commons-math3" + implementation "org.slf4j:slf4j-api" + implementation "com.google.guava:guava" +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/BaseClusteringAlgorithm.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/BaseClusteringAlgorithm.java new file mode 100644 index 000000000..9ff125ad9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/BaseClusteringAlgorithm.java @@ -0,0 +1,222 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.algorithm; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.lang3.ArrayUtils; +import org.deeplearning4j.clustering.cluster.Cluster; +import org.deeplearning4j.clustering.cluster.ClusterSet; +import org.deeplearning4j.clustering.cluster.ClusterUtils; +import org.deeplearning4j.clustering.cluster.Point; +import org.deeplearning4j.clustering.info.ClusterSetInfo; +import org.deeplearning4j.clustering.iteration.IterationHistory; +import org.deeplearning4j.clustering.iteration.IterationInfo; +import org.deeplearning4j.clustering.strategy.ClusteringStrategy; +import org.deeplearning4j.clustering.strategy.ClusteringStrategyType; +import org.deeplearning4j.clustering.strategy.OptimisationStrategy; +import org.deeplearning4j.clustering.util.MultiThreadUtils; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; + +/** + * + * adapted to ndarray matrices + * + * @author Adam Gibson + * @author Julien Roch + * + */ +@Slf4j +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class BaseClusteringAlgorithm implements ClusteringAlgorithm, Serializable { + + private static final long serialVersionUID = 338231277453149972L; + + private ClusteringStrategy clusteringStrategy; + private IterationHistory iterationHistory; + private int currentIteration = 0; + private ClusterSet clusterSet; + private List initialPoints; + private transient ExecutorService exec; + private boolean useKmeansPlusPlus; + + + protected BaseClusteringAlgorithm(ClusteringStrategy clusteringStrategy, boolean useKmeansPlusPlus) { + this.clusteringStrategy = clusteringStrategy; + this.exec = MultiThreadUtils.newExecutorService(); + this.useKmeansPlusPlus = useKmeansPlusPlus; + } + + /** + * + * @param clusteringStrategy + * @return + */ + public static BaseClusteringAlgorithm setup(ClusteringStrategy clusteringStrategy, boolean useKmeansPlusPlus) { + return new BaseClusteringAlgorithm(clusteringStrategy, useKmeansPlusPlus); + } + + /** + * + * @param points + * @return + */ + public ClusterSet applyTo(List points) { + resetState(points); + initClusters(useKmeansPlusPlus); + iterations(); + return clusterSet; + } + + private void resetState(List points) { + this.iterationHistory = new IterationHistory(); + this.currentIteration = 0; + this.clusterSet = null; + this.initialPoints = points; + } + + /** Run clustering iterations until a + * termination condition is hit. + * This is done by first classifying all points, + * and then updating cluster centers based on + * those classified points + */ + private void iterations() { + int iterationCount = 0; + while ((clusteringStrategy.getTerminationCondition() != null + && !clusteringStrategy.getTerminationCondition().isSatisfied(iterationHistory)) + || iterationHistory.getMostRecentIterationInfo().isStrategyApplied()) { + currentIteration++; + removePoints(); + classifyPoints(); + applyClusteringStrategy(); + log.trace("Completed clustering iteration {}", ++iterationCount); + } + } + + protected void classifyPoints() { + //Classify points. This also adds each point to the ClusterSet + ClusterSetInfo clusterSetInfo = ClusterUtils.classifyPoints(clusterSet, initialPoints, exec); + //Update the cluster centers, based on the points within each cluster + ClusterUtils.refreshClustersCenters(clusterSet, clusterSetInfo, exec); + iterationHistory.getIterationsInfos().put(currentIteration, + new IterationInfo(currentIteration, clusterSetInfo)); + } + + /** + * Initialize the + * cluster centers at random + */ + protected void initClusters(boolean kMeansPlusPlus) { + log.info("Generating initial clusters"); + List points = new ArrayList<>(initialPoints); + + //Initialize the ClusterSet with a single cluster center (based on position of one of the points chosen randomly) + val random = Nd4j.getRandom(); + Distance distanceFn = clusteringStrategy.getDistanceFunction(); + int initialClusterCount = clusteringStrategy.getInitialClusterCount(); + clusterSet = new ClusterSet(distanceFn, + clusteringStrategy.inverseDistanceCalculation(), new long[]{initialClusterCount, points.get(0).getArray().length()}); + clusterSet.addNewClusterWithCenter(points.remove(random.nextInt(points.size()))); + + + //dxs: distances between + // each point and nearest cluster to that point + INDArray dxs = Nd4j.create(points.size()); + dxs.addi(clusteringStrategy.inverseDistanceCalculation() ? -Double.MAX_VALUE : Double.MAX_VALUE); + + //Generate the initial cluster centers, by randomly selecting a point between 0 and max distance + //Thus, we are more likely to select (as a new cluster center) a point that is far from an existing cluster + while (clusterSet.getClusterCount() < initialClusterCount && !points.isEmpty()) { + dxs = ClusterUtils.computeSquareDistancesFromNearestCluster(clusterSet, points, dxs, exec); + double summed = Nd4j.sum(dxs).getDouble(0); + double r = kMeansPlusPlus ? random.nextDouble() * summed: + random.nextFloat() * dxs.maxNumber().doubleValue(); + + for (int i = 0; i < dxs.length(); i++) { + double distance = dxs.getDouble(i); + Preconditions.checkState(distance >= 0, "Encountered negative distance: distance function is not valid? Distance " + + "function must return values >= 0, got distance %s for function s", distance, distanceFn); + if (dxs.getDouble(i) >= r) { + clusterSet.addNewClusterWithCenter(points.remove(i)); + dxs = Nd4j.create(ArrayUtils.remove(dxs.data().asDouble(), i)); + break; + } + } + } + + ClusterSetInfo initialClusterSetInfo = ClusterUtils.computeClusterSetInfo(clusterSet); + iterationHistory.getIterationsInfos().put(currentIteration, + new IterationInfo(currentIteration, initialClusterSetInfo)); + } + + + protected void applyClusteringStrategy() { + if (!isStrategyApplicableNow()) + return; + + ClusterSetInfo clusterSetInfo = iterationHistory.getMostRecentClusterSetInfo(); + if (!clusteringStrategy.isAllowEmptyClusters()) { + int removedCount = removeEmptyClusters(clusterSetInfo); + if (removedCount > 0) { + iterationHistory.getMostRecentIterationInfo().setStrategyApplied(true); + + if (clusteringStrategy.isStrategyOfType(ClusteringStrategyType.FIXED_CLUSTER_COUNT) + && clusterSet.getClusterCount() < clusteringStrategy.getInitialClusterCount()) { + int splitCount = ClusterUtils.splitMostSpreadOutClusters(clusterSet, clusterSetInfo, + clusteringStrategy.getInitialClusterCount() - clusterSet.getClusterCount(), exec); + if (splitCount > 0) + iterationHistory.getMostRecentIterationInfo().setStrategyApplied(true); + } + } + } + if (clusteringStrategy.isStrategyOfType(ClusteringStrategyType.OPTIMIZATION)) + optimize(); + } + + protected void optimize() { + ClusterSetInfo clusterSetInfo = iterationHistory.getMostRecentClusterSetInfo(); + OptimisationStrategy optimization = (OptimisationStrategy) clusteringStrategy; + boolean applied = ClusterUtils.applyOptimization(optimization, clusterSet, clusterSetInfo, exec); + iterationHistory.getMostRecentIterationInfo().setStrategyApplied(applied); + } + + private boolean isStrategyApplicableNow() { + return clusteringStrategy.isOptimizationDefined() && iterationHistory.getIterationCount() != 0 + && clusteringStrategy.isOptimizationApplicableNow(iterationHistory); + } + + protected int removeEmptyClusters(ClusterSetInfo clusterSetInfo) { + List removedClusters = clusterSet.removeEmptyClusters(); + clusterSetInfo.removeClusterInfos(removedClusters); + return removedClusters.size(); + } + + protected void removePoints() { + clusterSet.removePoints(); + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/ClusteringAlgorithm.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/ClusteringAlgorithm.java new file mode 100644 index 000000000..b5fa5c1c5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/ClusteringAlgorithm.java @@ -0,0 +1,40 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.algorithm; + +import org.deeplearning4j.clustering.cluster.ClusterSet; +import org.deeplearning4j.clustering.cluster.Point; + +import java.util.List; + +/** + * An interface for a clustering + * algorithm. + * This is for applying a clustering + * algorithm to a list of points. + */ +public interface ClusteringAlgorithm { + + /** + * Apply a clustering + * algorithm for a given result + * @param points + * @return + */ + ClusterSet applyTo(List points); + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/Distance.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/Distance.java new file mode 100644 index 000000000..6aff39dbb --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/algorithm/Distance.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.algorithm; + +public enum Distance { + EUCLIDEAN("euclidean"), + COSINE_DISTANCE("cosinedistance"), + COSINE_SIMILARITY("cosinesimilarity"), + MANHATTAN("manhattan"), + DOT("dot"), + JACCARD("jaccard"), + HAMMING("hamming"); + + private String functionName; + private Distance(String name) { + functionName = name; + } + + @Override + public String toString() { + return functionName; + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/CentersHolder.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/CentersHolder.java new file mode 100644 index 000000000..25542dc8f --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/CentersHolder.java @@ -0,0 +1,101 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.cluster; + +import org.deeplearning4j.clustering.algorithm.Distance; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.ReduceOp; +import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMin; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +public class CentersHolder { + private INDArray centers; + private long index = 0; + + protected transient ReduceOp op; + protected ArgMin imin; + protected transient INDArray distances; + protected transient INDArray argMin; + + private long rows, cols; + + public CentersHolder(long rows, long cols) { + this.rows = rows; + this.cols = cols; + } + + public INDArray getCenters() { + return this.centers; + } + + public synchronized void addCenter(INDArray pointView) { + if (centers == null) + this.centers = Nd4j.create(pointView.dataType(), new long[] {rows, cols}); + + centers.putRow(index++, pointView); + } + + public synchronized Pair getCenterByMinDistance(Point point, Distance distanceFunction) { + if (distances == null) + distances = Nd4j.create(centers.dataType(), centers.rows()); + + if (argMin == null) + argMin = Nd4j.createUninitialized(DataType.LONG, new long[0]); + + if (op == null) { + op = ClusterUtils.createDistanceFunctionOp(distanceFunction, centers, point.getArray(), 1); + imin = new ArgMin(distances, argMin); + op.setZ(distances); + } + + op.setY(point.getArray()); + + Nd4j.getExecutioner().exec(op); + Nd4j.getExecutioner().exec(imin); + + Pair result = new Pair<>(); + result.setFirst(distances.getDouble(argMin.getLong(0))); + result.setSecond(argMin.getLong(0)); + return result; + } + + public synchronized INDArray getMinDistances(Point point, Distance distanceFunction) { + if (distances == null) + distances = Nd4j.create(centers.dataType(), centers.rows()); + + if (argMin == null) + argMin = Nd4j.createUninitialized(DataType.LONG, new long[0]); + + if (op == null) { + op = ClusterUtils.createDistanceFunctionOp(distanceFunction, centers, point.getArray(), 1); + imin = new ArgMin(distances, argMin); + op.setZ(distances); + } + + op.setY(point.getArray()); + + Nd4j.getExecutioner().exec(op); + Nd4j.getExecutioner().exec(imin); + + System.out.println(distances); + return distances; + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/Cluster.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/Cluster.java new file mode 100644 index 000000000..c9b1d806c --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/Cluster.java @@ -0,0 +1,151 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.cluster; + +import lombok.Data; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +/** + * A cluster. + * + * + */ +@Data +public class Cluster implements Serializable { + + private String id = UUID.randomUUID().toString(); + private String label; + + private Point center; + private List points = Collections.synchronizedList(new ArrayList()); + private boolean inverse = false; + private Distance distanceFunction; + + public Cluster() { + super(); + } + + /** + * + * @param center + * @param distanceFunction + */ + public Cluster(Point center, Distance distanceFunction) { + this(center, false, distanceFunction); + } + + /** + * + * @param center + * @param distanceFunction + */ + public Cluster(Point center, boolean inverse, Distance distanceFunction) { + this.distanceFunction = distanceFunction; + this.inverse = inverse; + setCenter(center); + } + + /** + * Get the distance to the given + * point from the cluster + * @param point the point to get the distance for + * @return + */ + public double getDistanceToCenter(Point point) { + return Nd4j.getExecutioner().execAndReturn( + ClusterUtils.createDistanceFunctionOp(distanceFunction, center.getArray(), point.getArray())) + .getFinalResult().doubleValue(); + } + + /** + * Add a point to the cluster + * @param point + */ + public void addPoint(Point point) { + addPoint(point, true); + } + + /** + * Add a point to the cluster + * @param point the point to add + * @param moveClusterCenter whether to update + * the cluster centroid or not + */ + public void addPoint(Point point, boolean moveClusterCenter) { + if (moveClusterCenter) { + if (isInverse()) { + center.getArray().muli(points.size()).subi(point.getArray()).divi(points.size() + 1); + } else { + center.getArray().muli(points.size()).addi(point.getArray()).divi(points.size() + 1); + } + } + + getPoints().add(point); + } + + /** + * Clear out the ponits + */ + public void removePoints() { + if (getPoints() != null) + getPoints().clear(); + } + + /** + * Whether the cluster is empty or not + * @return + */ + public boolean isEmpty() { + return points == null || points.isEmpty(); + } + + /** + * Return the point with the given id + * @param id + * @return + */ + public Point getPoint(String id) { + for (Point point : points) + if (id.equals(point.getId())) + return point; + return null; + } + + /** + * Remove the point and return it + * @param id + * @return + */ + public Point removePoint(String id) { + Point removePoint = null; + for (Point point : points) + if (id.equals(point.getId())) + removePoint = point; + if (removePoint != null) + points.remove(removePoint); + return removePoint; + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/ClusterSet.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/ClusterSet.java new file mode 100644 index 000000000..d7241eed9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/ClusterSet.java @@ -0,0 +1,255 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.cluster; + +import lombok.Data; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.io.Serializable; +import java.util.*; + +@Data +public class ClusterSet implements Serializable { + + private Distance distanceFunction; + private List clusters; + private CentersHolder centersHolder; + private Map pointDistribution; + private boolean inverse; + + public ClusterSet(boolean inverse) { + this(null, inverse, null); + } + + public ClusterSet(Distance distanceFunction, boolean inverse, long[] shape) { + this.distanceFunction = distanceFunction; + this.inverse = inverse; + this.clusters = Collections.synchronizedList(new ArrayList()); + this.pointDistribution = Collections.synchronizedMap(new HashMap()); + if (shape != null) + this.centersHolder = new CentersHolder(shape[0], shape[1]); + } + + + public boolean isInverse() { + return inverse; + } + + /** + * + * @param center + * @return + */ + public Cluster addNewClusterWithCenter(Point center) { + Cluster newCluster = new Cluster(center, distanceFunction); + getClusters().add(newCluster); + setPointLocation(center, newCluster); + centersHolder.addCenter(center.getArray()); + return newCluster; + } + + /** + * + * @param point + * @return + */ + public PointClassification classifyPoint(Point point) { + return classifyPoint(point, true); + } + + /** + * + * @param points + */ + public void classifyPoints(List points) { + classifyPoints(points, true); + } + + /** + * + * @param points + * @param moveClusterCenter + */ + public void classifyPoints(List points, boolean moveClusterCenter) { + for (Point point : points) + classifyPoint(point, moveClusterCenter); + } + + /** + * + * @param point + * @param moveClusterCenter + * @return + */ + public PointClassification classifyPoint(Point point, boolean moveClusterCenter) { + Pair nearestCluster = nearestCluster(point); + Cluster newCluster = nearestCluster.getKey(); + boolean locationChange = isPointLocationChange(point, newCluster); + addPointToCluster(point, newCluster, moveClusterCenter); + return new PointClassification(nearestCluster.getKey(), nearestCluster.getValue(), locationChange); + } + + private boolean isPointLocationChange(Point point, Cluster newCluster) { + if (!getPointDistribution().containsKey(point.getId())) + return true; + return !getPointDistribution().get(point.getId()).equals(newCluster.getId()); + } + + private void addPointToCluster(Point point, Cluster cluster, boolean moveClusterCenter) { + cluster.addPoint(point, moveClusterCenter); + setPointLocation(point, cluster); + } + + private void setPointLocation(Point point, Cluster cluster) { + pointDistribution.put(point.getId(), cluster.getId()); + } + + + /** + * + * @param point + * @return + */ + public Pair nearestCluster(Point point) { + + /*double minDistance = isInverse() ? Float.MIN_VALUE : Float.MAX_VALUE; + + double currentDistance; + for (Cluster cluster : getClusters()) { + currentDistance = cluster.getDistanceToCenter(point); + if (isInverse()) { + if (currentDistance > minDistance) { + minDistance = currentDistance; + nearestCluster = cluster; + } + } else { + if (currentDistance < minDistance) { + minDistance = currentDistance; + nearestCluster = cluster; + } + } + + }*/ + + Pair nearestCenterData = centersHolder. + getCenterByMinDistance(point, distanceFunction); + Cluster nearestCluster = getClusters().get(nearestCenterData.getSecond().intValue()); + double minDistance = nearestCenterData.getFirst(); + return Pair.of(nearestCluster, minDistance); + } + + /** + * + * @param m1 + * @param m2 + * @return + */ + public double getDistance(Point m1, Point m2) { + return Nd4j.getExecutioner() + .execAndReturn(ClusterUtils.createDistanceFunctionOp(distanceFunction, m1.getArray(), m2.getArray())) + .getFinalResult().doubleValue(); + } + + /** + * + * @param point + * @return + */ + /*public double getDistanceFromNearestCluster(Point point) { + return nearestCluster(point).getValue(); + }*/ + + + /** + * + * @param clusterId + * @return + */ + public String getClusterCenterId(String clusterId) { + Point clusterCenter = getClusterCenter(clusterId); + return clusterCenter == null ? null : clusterCenter.getId(); + } + + /** + * + * @param clusterId + * @return + */ + public Point getClusterCenter(String clusterId) { + Cluster cluster = getCluster(clusterId); + return cluster == null ? null : cluster.getCenter(); + } + + /** + * + * @param id + * @return + */ + public Cluster getCluster(String id) { + for (int i = 0, j = clusters.size(); i < j; i++) + if (id.equals(clusters.get(i).getId())) + return clusters.get(i); + return null; + } + + /** + * + * @return + */ + public int getClusterCount() { + return getClusters() == null ? 0 : getClusters().size(); + } + + /** + * + */ + public void removePoints() { + for (Cluster cluster : getClusters()) + cluster.removePoints(); + } + + /** + * + * @param count + * @return + */ + public List getMostPopulatedClusters(int count) { + List mostPopulated = new ArrayList<>(clusters); + Collections.sort(mostPopulated, new Comparator() { + public int compare(Cluster o1, Cluster o2) { + return Integer.compare(o2.getPoints().size(), o1.getPoints().size()); + } + }); + return mostPopulated.subList(0, count); + } + + /** + * + * @return + */ + public List removeEmptyClusters() { + List emptyClusters = new ArrayList<>(); + for (Cluster cluster : clusters) + if (cluster.isEmpty()) + emptyClusters.add(cluster); + clusters.removeAll(emptyClusters); + return emptyClusters; + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/ClusterUtils.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/ClusterUtils.java new file mode 100644 index 000000000..54f355b67 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/ClusterUtils.java @@ -0,0 +1,531 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.cluster; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.lang3.ArrayUtils; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.info.ClusterInfo; +import org.deeplearning4j.clustering.info.ClusterSetInfo; +import org.deeplearning4j.clustering.optimisation.ClusteringOptimizationType; +import org.deeplearning4j.clustering.strategy.OptimisationStrategy; +import org.deeplearning4j.clustering.util.MathUtils; +import org.deeplearning4j.clustering.util.MultiThreadUtils; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.ReduceOp; +import org.nd4j.linalg.api.ops.impl.reduce3.*; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.*; +import java.util.concurrent.ExecutorService; + +/** + * + * Basic cluster utilities + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@Slf4j +public class ClusterUtils { + + /** Classify the set of points base on cluster centers. This also adds each point to the ClusterSet */ + public static ClusterSetInfo classifyPoints(final ClusterSet clusterSet, List points, + ExecutorService executorService) { + final ClusterSetInfo clusterSetInfo = ClusterSetInfo.initialize(clusterSet, true); + + List tasks = new ArrayList<>(); + for (final Point point : points) { + //tasks.add(new Runnable() { + // public void run() { + try { + PointClassification result = classifyPoint(clusterSet, point); + if (result.isNewLocation()) + clusterSetInfo.getPointLocationChange().incrementAndGet(); + clusterSetInfo.getClusterInfo(result.getCluster().getId()).getPointDistancesFromCenter() + .put(point.getId(), result.getDistanceFromCenter()); + } catch (Throwable t) { + log.warn("Error classifying point", t); + } + // } + } + + //MultiThreadUtils.parallelTasks(tasks, executorService); + return clusterSetInfo; + } + + public static PointClassification classifyPoint(ClusterSet clusterSet, Point point) { + return clusterSet.classifyPoint(point, false); + } + + public static void refreshClustersCenters(final ClusterSet clusterSet, final ClusterSetInfo clusterSetInfo, + ExecutorService executorService) { + List tasks = new ArrayList<>(); + int nClusters = clusterSet.getClusterCount(); + for (int i = 0; i < nClusters; i++) { + final Cluster cluster = clusterSet.getClusters().get(i); + //tasks.add(new Runnable() { + // public void run() { + try { + final ClusterInfo clusterInfo = clusterSetInfo.getClusterInfo(cluster.getId()); + refreshClusterCenter(cluster, clusterInfo); + deriveClusterInfoDistanceStatistics(clusterInfo); + } catch (Throwable t) { + log.warn("Error refreshing cluster centers", t); + } + // } + //}); + } + //MultiThreadUtils.parallelTasks(tasks, executorService); + } + + public static void refreshClusterCenter(Cluster cluster, ClusterInfo clusterInfo) { + int pointsCount = cluster.getPoints().size(); + if (pointsCount == 0) + return; + Point center = new Point(Nd4j.create(cluster.getPoints().get(0).getArray().length())); + for (Point point : cluster.getPoints()) { + INDArray arr = point.getArray(); + if (cluster.isInverse()) + center.getArray().subi(arr); + else + center.getArray().addi(arr); + } + center.getArray().divi(pointsCount); + cluster.setCenter(center); + } + + /** + * + * @param info + */ + public static void deriveClusterInfoDistanceStatistics(ClusterInfo info) { + int pointCount = info.getPointDistancesFromCenter().size(); + if (pointCount == 0) + return; + + double[] distances = + ArrayUtils.toPrimitive(info.getPointDistancesFromCenter().values().toArray(new Double[] {})); + double max = info.isInverse() ? MathUtils.min(distances) : MathUtils.max(distances); + double total = MathUtils.sum(distances); + info.setMaxPointDistanceFromCenter(max); + info.setTotalPointDistanceFromCenter(total); + info.setAveragePointDistanceFromCenter(total / pointCount); + info.setPointDistanceFromCenterVariance(MathUtils.variance(distances)); + } + + /** + * + * @param clusterSet + * @param points + * @param previousDxs + * @param executorService + * @return + */ + public static INDArray computeSquareDistancesFromNearestCluster(final ClusterSet clusterSet, + final List points, INDArray previousDxs, ExecutorService executorService) { + final int pointsCount = points.size(); + final INDArray dxs = Nd4j.create(pointsCount); + final Cluster newCluster = clusterSet.getClusters().get(clusterSet.getClusters().size() - 1); + + List tasks = new ArrayList<>(); + for (int i = 0; i < pointsCount; i++) { + final int i2 = i; + //tasks.add(new Runnable() { + // public void run() { + try { + Point point = points.get(i2); + double dist = clusterSet.isInverse() ? newCluster.getDistanceToCenter(point) + : Math.pow(newCluster.getDistanceToCenter(point), 2); + dxs.putScalar(i2, /*clusterSet.isInverse() ? dist :*/ dist); + } catch (Throwable t) { + log.warn("Error computing squared distance from nearest cluster", t); + } + // } + //}); + + } + + //MultiThreadUtils.parallelTasks(tasks, executorService); + for (int i = 0; i < pointsCount; i++) { + double previousMinDistance = previousDxs.getDouble(i); + if (clusterSet.isInverse()) { + if (dxs.getDouble(i) < previousMinDistance) { + + dxs.putScalar(i, previousMinDistance); + } + } else if (dxs.getDouble(i) > previousMinDistance) + dxs.putScalar(i, previousMinDistance); + } + + return dxs; + } + + public static INDArray computeWeightedProbaDistancesFromNearestCluster(final ClusterSet clusterSet, + final List points, INDArray previousDxs) { + final int pointsCount = points.size(); + final INDArray dxs = Nd4j.create(pointsCount); + final Cluster newCluster = clusterSet.getClusters().get(clusterSet.getClusters().size() - 1); + + Double sum = new Double(0); + for (int i = 0; i < pointsCount; i++) { + + Point point = points.get(i); + double dist = Math.pow(newCluster.getDistanceToCenter(point), 2); + sum += dist; + dxs.putScalar(i, sum); + } + + return dxs; + } + /** + * + * @param clusterSet + * @return + */ + public static ClusterSetInfo computeClusterSetInfo(ClusterSet clusterSet) { + ExecutorService executor = MultiThreadUtils.newExecutorService(); + ClusterSetInfo info = computeClusterSetInfo(clusterSet, executor); + executor.shutdownNow(); + return info; + } + + public static ClusterSetInfo computeClusterSetInfo(final ClusterSet clusterSet, ExecutorService executorService) { + final ClusterSetInfo info = new ClusterSetInfo(clusterSet.isInverse(), true); + int clusterCount = clusterSet.getClusterCount(); + + List tasks = new ArrayList<>(); + for (int i = 0; i < clusterCount; i++) { + final Cluster cluster = clusterSet.getClusters().get(i); + //tasks.add(new Runnable() { + // public void run() { + try { + info.getClustersInfos().put(cluster.getId(), + computeClusterInfos(cluster, clusterSet.getDistanceFunction())); + } catch (Throwable t) { + log.warn("Error computing cluster set info", t); + } + //} + //}); + } + + + //MultiThreadUtils.parallelTasks(tasks, executorService); + + //tasks = new ArrayList<>(); + for (int i = 0; i < clusterCount; i++) { + final int clusterIdx = i; + final Cluster fromCluster = clusterSet.getClusters().get(i); + //tasks.add(new Runnable() { + //public void run() { + try { + for (int k = clusterIdx + 1, l = clusterSet.getClusterCount(); k < l; k++) { + Cluster toCluster = clusterSet.getClusters().get(k); + double distance = Nd4j.getExecutioner() + .execAndReturn(ClusterUtils.createDistanceFunctionOp( + clusterSet.getDistanceFunction(), + fromCluster.getCenter().getArray(), + toCluster.getCenter().getArray())) + .getFinalResult().doubleValue(); + info.getDistancesBetweenClustersCenters().put(fromCluster.getId(), toCluster.getId(), + distance); + } + } catch (Throwable t) { + log.warn("Error computing distances", t); + } + // } + //}); + + } + + //MultiThreadUtils.parallelTasks(tasks, executorService); + + return info; + } + + /** + * + * @param cluster + * @param distanceFunction + * @return + */ + public static ClusterInfo computeClusterInfos(Cluster cluster, Distance distanceFunction) { + ClusterInfo info = new ClusterInfo(cluster.isInverse(), true); + for (int i = 0, j = cluster.getPoints().size(); i < j; i++) { + Point point = cluster.getPoints().get(i); + //shouldn't need to inverse here. other parts of + //the code should interpret the "distance" or score here + double distance = Nd4j.getExecutioner() + .execAndReturn(ClusterUtils.createDistanceFunctionOp(distanceFunction, + cluster.getCenter().getArray(), point.getArray())) + .getFinalResult().doubleValue(); + info.getPointDistancesFromCenter().put(point.getId(), distance); + double diff = info.getTotalPointDistanceFromCenter() + distance; + info.setTotalPointDistanceFromCenter(diff); + } + + if (!cluster.getPoints().isEmpty()) + info.setAveragePointDistanceFromCenter(info.getTotalPointDistanceFromCenter() / cluster.getPoints().size()); + return info; + } + + /** + * + * @param optimization + * @param clusterSet + * @param clusterSetInfo + * @param executor + * @return + */ + public static boolean applyOptimization(OptimisationStrategy optimization, ClusterSet clusterSet, + ClusterSetInfo clusterSetInfo, ExecutorService executor) { + + if (optimization.isClusteringOptimizationType( + ClusteringOptimizationType.MINIMIZE_AVERAGE_POINT_TO_CENTER_DISTANCE)) { + int splitCount = ClusterUtils.splitClustersWhereAverageDistanceFromCenterGreaterThan(clusterSet, + clusterSetInfo, optimization.getClusteringOptimizationValue(), executor); + return splitCount > 0; + } + + if (optimization.isClusteringOptimizationType( + ClusteringOptimizationType.MINIMIZE_MAXIMUM_POINT_TO_CENTER_DISTANCE)) { + int splitCount = ClusterUtils.splitClustersWhereMaximumDistanceFromCenterGreaterThan(clusterSet, + clusterSetInfo, optimization.getClusteringOptimizationValue(), executor); + return splitCount > 0; + } + + return false; + } + + /** + * + * @param clusterSet + * @param info + * @param count + * @return + */ + public static List getMostSpreadOutClusters(final ClusterSet clusterSet, final ClusterSetInfo info, + int count) { + List clusters = new ArrayList<>(clusterSet.getClusters()); + Collections.sort(clusters, new Comparator() { + public int compare(Cluster o1, Cluster o2) { + Double o1TotalDistance = info.getClusterInfo(o1.getId()).getTotalPointDistanceFromCenter(); + Double o2TotalDistance = info.getClusterInfo(o2.getId()).getTotalPointDistanceFromCenter(); + int comp = o1TotalDistance.compareTo(o2TotalDistance); + return !clusterSet.getClusters().get(0).isInverse() ? -comp : comp; + } + }); + + return clusters.subList(0, count); + } + + /** + * + * @param clusterSet + * @param info + * @param maximumAverageDistance + * @return + */ + public static List getClustersWhereAverageDistanceFromCenterGreaterThan(final ClusterSet clusterSet, + final ClusterSetInfo info, double maximumAverageDistance) { + List clusters = new ArrayList<>(); + for (Cluster cluster : clusterSet.getClusters()) { + ClusterInfo clusterInfo = info.getClusterInfo(cluster.getId()); + if (clusterInfo != null) { + //distances + if (clusterInfo.isInverse()) { + if (clusterInfo.getAveragePointDistanceFromCenter() < maximumAverageDistance) + clusters.add(cluster); + } else { + if (clusterInfo.getAveragePointDistanceFromCenter() > maximumAverageDistance) + clusters.add(cluster); + } + + } + + } + return clusters; + } + + /** + * + * @param clusterSet + * @param info + * @param maximumDistance + * @return + */ + public static List getClustersWhereMaximumDistanceFromCenterGreaterThan(final ClusterSet clusterSet, + final ClusterSetInfo info, double maximumDistance) { + List clusters = new ArrayList<>(); + for (Cluster cluster : clusterSet.getClusters()) { + ClusterInfo clusterInfo = info.getClusterInfo(cluster.getId()); + if (clusterInfo != null) { + if (clusterInfo.isInverse() && clusterInfo.getMaxPointDistanceFromCenter() < maximumDistance) { + clusters.add(cluster); + } else if (clusterInfo.getMaxPointDistanceFromCenter() > maximumDistance) { + clusters.add(cluster); + + } + } + } + return clusters; + } + + /** + * + * @param clusterSet + * @param clusterSetInfo + * @param count + * @param executorService + * @return + */ + public static int splitMostSpreadOutClusters(ClusterSet clusterSet, ClusterSetInfo clusterSetInfo, int count, + ExecutorService executorService) { + List clustersToSplit = getMostSpreadOutClusters(clusterSet, clusterSetInfo, count); + splitClusters(clusterSet, clusterSetInfo, clustersToSplit, executorService); + return clustersToSplit.size(); + } + + /** + * + * @param clusterSet + * @param clusterSetInfo + * @param maxWithinClusterDistance + * @param executorService + * @return + */ + public static int splitClustersWhereAverageDistanceFromCenterGreaterThan(ClusterSet clusterSet, + ClusterSetInfo clusterSetInfo, double maxWithinClusterDistance, ExecutorService executorService) { + List clustersToSplit = getClustersWhereAverageDistanceFromCenterGreaterThan(clusterSet, clusterSetInfo, + maxWithinClusterDistance); + splitClusters(clusterSet, clusterSetInfo, clustersToSplit, maxWithinClusterDistance, executorService); + return clustersToSplit.size(); + } + + /** + * + * @param clusterSet + * @param clusterSetInfo + * @param maxWithinClusterDistance + * @param executorService + * @return + */ + public static int splitClustersWhereMaximumDistanceFromCenterGreaterThan(ClusterSet clusterSet, + ClusterSetInfo clusterSetInfo, double maxWithinClusterDistance, ExecutorService executorService) { + List clustersToSplit = getClustersWhereMaximumDistanceFromCenterGreaterThan(clusterSet, clusterSetInfo, + maxWithinClusterDistance); + splitClusters(clusterSet, clusterSetInfo, clustersToSplit, maxWithinClusterDistance, executorService); + return clustersToSplit.size(); + } + + /** + * + * @param clusterSet + * @param clusterSetInfo + * @param count + * @param executorService + */ + public static void splitMostPopulatedClusters(ClusterSet clusterSet, ClusterSetInfo clusterSetInfo, int count, + ExecutorService executorService) { + List clustersToSplit = clusterSet.getMostPopulatedClusters(count); + splitClusters(clusterSet, clusterSetInfo, clustersToSplit, executorService); + } + + /** + * + * @param clusterSet + * @param clusterSetInfo + * @param clusters + * @param maxDistance + * @param executorService + */ + public static void splitClusters(final ClusterSet clusterSet, final ClusterSetInfo clusterSetInfo, + List clusters, final double maxDistance, ExecutorService executorService) { + final Random random = new Random(); + List tasks = new ArrayList<>(); + for (final Cluster cluster : clusters) { + tasks.add(new Runnable() { + public void run() { + try { + ClusterInfo clusterInfo = clusterSetInfo.getClusterInfo(cluster.getId()); + List fartherPoints = clusterInfo.getPointsFartherFromCenterThan(maxDistance); + int rank = Math.min(fartherPoints.size(), 3); + String pointId = fartherPoints.get(random.nextInt(rank)); + Point point = cluster.removePoint(pointId); + clusterSet.addNewClusterWithCenter(point); + } catch (Throwable t) { + log.warn("Error splitting clusters", t); + } + } + }); + } + MultiThreadUtils.parallelTasks(tasks, executorService); + } + + /** + * + * @param clusterSet + * @param clusterSetInfo + * @param clusters + * @param executorService + */ + public static void splitClusters(final ClusterSet clusterSet, final ClusterSetInfo clusterSetInfo, + List clusters, ExecutorService executorService) { + final Random random = new Random(); + List tasks = new ArrayList<>(); + for (final Cluster cluster : clusters) { + tasks.add(new Runnable() { + public void run() { + try { + Point point = cluster.getPoints().remove(random.nextInt(cluster.getPoints().size())); + clusterSet.addNewClusterWithCenter(point); + } catch (Throwable t) { + log.warn("Error Splitting clusters (2)", t); + } + } + }); + } + + MultiThreadUtils.parallelTasks(tasks, executorService); + } + + public static ReduceOp createDistanceFunctionOp(Distance distanceFunction, INDArray x, INDArray y, int...dimensions){ + val op = createDistanceFunctionOp(distanceFunction, x, y); + op.setDimensions(dimensions); + return op; + } + + public static ReduceOp createDistanceFunctionOp(Distance distanceFunction, INDArray x, INDArray y){ + switch (distanceFunction){ + case COSINE_DISTANCE: + return new CosineDistance(x,y); + case COSINE_SIMILARITY: + return new CosineSimilarity(x,y); + case DOT: + return new Dot(x,y); + case EUCLIDEAN: + return new EuclideanDistance(x,y); + case JACCARD: + return new JaccardDistance(x,y); + case MANHATTAN: + return new ManhattanDistance(x,y); + default: + throw new IllegalStateException("Unknown distance function: " + distanceFunction); + } + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/Point.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/Point.java new file mode 100644 index 000000000..86bbe439a --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/Point.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.cluster; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * + */ +@Data +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class Point implements Serializable { + + private static final long serialVersionUID = -6658028541426027226L; + + private String id = UUID.randomUUID().toString(); + private String label; + private INDArray array; + + + /** + * + * @param array + */ + public Point(INDArray array) { + super(); + this.array = array; + } + + /** + * + * @param id + * @param array + */ + public Point(String id, INDArray array) { + super(); + this.id = id; + this.array = array; + } + + public Point(String id, String label, double[] data) { + this(id, label, Nd4j.create(data)); + } + + public Point(String id, String label, INDArray array) { + super(); + this.id = id; + this.label = label; + this.array = array; + } + + + /** + * + * @param matrix + * @return + */ + public static List toPoints(INDArray matrix) { + List arr = new ArrayList<>(matrix.rows()); + for (int i = 0; i < matrix.rows(); i++) { + arr.add(new Point(matrix.getRow(i))); + } + + return arr; + } + + /** + * + * @param vectors + * @return + */ + public static List toPoints(List vectors) { + List points = new ArrayList<>(); + for (INDArray vector : vectors) + points.add(new Point(vector)); + return points; + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/PointClassification.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/PointClassification.java new file mode 100644 index 000000000..9f99287d0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/cluster/PointClassification.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.cluster; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@Data +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@AllArgsConstructor +public class PointClassification implements Serializable { + + private Cluster cluster; + private double distanceFromCenter; + private boolean newLocation; + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/ClusteringAlgorithmCondition.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/ClusteringAlgorithmCondition.java new file mode 100644 index 000000000..f5ee6b195 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/ClusteringAlgorithmCondition.java @@ -0,0 +1,33 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.condition; + +import org.deeplearning4j.clustering.iteration.IterationHistory; + +/** + * + */ +public interface ClusteringAlgorithmCondition { + + /** + * + * @param iterationHistory + * @return + */ + boolean isSatisfied(IterationHistory iterationHistory); + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/ConvergenceCondition.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/ConvergenceCondition.java new file mode 100644 index 000000000..93fc2b8bc --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/ConvergenceCondition.java @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.condition; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import org.deeplearning4j.clustering.iteration.IterationHistory; +import org.nd4j.linalg.indexing.conditions.Condition; +import org.nd4j.linalg.indexing.conditions.LessThan; + +import java.io.Serializable; + +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@AllArgsConstructor(access = AccessLevel.PROTECTED) +public class ConvergenceCondition implements ClusteringAlgorithmCondition, Serializable { + + private Condition convergenceCondition; + private double pointsDistributionChangeRate; + + + /** + * + * @param pointsDistributionChangeRate + * @return + */ + public static ConvergenceCondition distributionVariationRateLessThan(double pointsDistributionChangeRate) { + Condition condition = new LessThan(pointsDistributionChangeRate); + return new ConvergenceCondition(condition, pointsDistributionChangeRate); + } + + + /** + * + * @param iterationHistory + * @return + */ + public boolean isSatisfied(IterationHistory iterationHistory) { + int iterationCount = iterationHistory.getIterationCount(); + if (iterationCount <= 1) + return false; + + double variation = iterationHistory.getMostRecentClusterSetInfo().getPointLocationChange().get(); + variation /= iterationHistory.getMostRecentClusterSetInfo().getPointsCount(); + + return convergenceCondition.apply(variation); + } + + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/FixedIterationCountCondition.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/FixedIterationCountCondition.java new file mode 100644 index 000000000..cf2df786b --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/FixedIterationCountCondition.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.condition; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.deeplearning4j.clustering.iteration.IterationHistory; +import org.nd4j.linalg.indexing.conditions.Condition; +import org.nd4j.linalg.indexing.conditions.GreaterThanOrEqual; + +import java.io.Serializable; + +/** + * + */ +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class FixedIterationCountCondition implements ClusteringAlgorithmCondition, Serializable { + + private Condition iterationCountCondition; + + protected FixedIterationCountCondition(int initialClusterCount) { + iterationCountCondition = new GreaterThanOrEqual(initialClusterCount); + } + + /** + * + * @param iterationCount + * @return + */ + public static FixedIterationCountCondition iterationCountGreaterThan(int iterationCount) { + return new FixedIterationCountCondition(iterationCount); + } + + /** + * + * @param iterationHistory + * @return + */ + public boolean isSatisfied(IterationHistory iterationHistory) { + return iterationCountCondition.apply(iterationHistory == null ? 0 : iterationHistory.getIterationCount()); + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/VarianceVariationCondition.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/VarianceVariationCondition.java new file mode 100644 index 000000000..783b67fde --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/condition/VarianceVariationCondition.java @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.condition; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; +import org.deeplearning4j.clustering.iteration.IterationHistory; +import org.nd4j.linalg.indexing.conditions.Condition; +import org.nd4j.linalg.indexing.conditions.LessThan; + +import java.io.Serializable; + +/** + * + */ +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@AllArgsConstructor +public class VarianceVariationCondition implements ClusteringAlgorithmCondition, Serializable { + + private Condition varianceVariationCondition; + private int period; + + + + /** + * + * @param varianceVariation + * @param period + * @return + */ + public static VarianceVariationCondition varianceVariationLessThan(double varianceVariation, int period) { + Condition condition = new LessThan(varianceVariation); + return new VarianceVariationCondition(condition, period); + } + + + /** + * + * @param iterationHistory + * @return + */ + public boolean isSatisfied(IterationHistory iterationHistory) { + if (iterationHistory.getIterationCount() <= period) + return false; + + for (int i = 0, j = iterationHistory.getIterationCount(); i < period; i++) { + double variation = iterationHistory.getIterationInfo(j - i).getClusterSetInfo() + .getPointDistanceFromClusterVariance(); + variation -= iterationHistory.getIterationInfo(j - i - 1).getClusterSetInfo() + .getPointDistanceFromClusterVariance(); + variation /= iterationHistory.getIterationInfo(j - i - 1).getClusterSetInfo() + .getPointDistanceFromClusterVariance(); + + if (!varianceVariationCondition.apply(variation)) + return false; + } + + return true; + } + + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/info/ClusterInfo.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/info/ClusterInfo.java new file mode 100644 index 000000000..dd78e30f0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/info/ClusterInfo.java @@ -0,0 +1,110 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.info; + +import lombok.Data; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + */ +@Data +public class ClusterInfo implements Serializable { + + private double averagePointDistanceFromCenter; + private double maxPointDistanceFromCenter; + private double pointDistanceFromCenterVariance; + private double totalPointDistanceFromCenter; + private boolean inverse; + private Map pointDistancesFromCenter = new ConcurrentHashMap<>(); + + public ClusterInfo(boolean inverse) { + this(false, inverse); + } + + /** + * + * @param threadSafe + */ + public ClusterInfo(boolean threadSafe, boolean inverse) { + super(); + this.inverse = inverse; + if (threadSafe) { + pointDistancesFromCenter = Collections.synchronizedMap(pointDistancesFromCenter); + } + } + + /** + * + * @return + */ + public Set> getSortedPointDistancesFromCenter() { + SortedSet> sortedEntries = new TreeSet<>(new Comparator>() { + @Override + public int compare(Map.Entry e1, Map.Entry e2) { + int res = e1.getValue().compareTo(e2.getValue()); + return res != 0 ? res : 1; + } + }); + sortedEntries.addAll(pointDistancesFromCenter.entrySet()); + return sortedEntries; + } + + /** + * + * @return + */ + public Set> getReverseSortedPointDistancesFromCenter() { + SortedSet> sortedEntries = new TreeSet<>(new Comparator>() { + @Override + public int compare(Map.Entry e1, Map.Entry e2) { + int res = e1.getValue().compareTo(e2.getValue()); + return -(res != 0 ? res : 1); + } + }); + sortedEntries.addAll(pointDistancesFromCenter.entrySet()); + return sortedEntries; + } + + /** + * + * @param maxDistance + * @return + */ + public List getPointsFartherFromCenterThan(double maxDistance) { + Set> sorted = getReverseSortedPointDistancesFromCenter(); + List ids = new ArrayList<>(); + for (Map.Entry entry : sorted) { + if (inverse && entry.getValue() < -maxDistance) { + if (entry.getValue() < -maxDistance) + break; + } + + else if (entry.getValue() > maxDistance) + break; + + ids.add(entry.getKey()); + } + return ids; + } + + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/info/ClusterSetInfo.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/info/ClusterSetInfo.java new file mode 100644 index 000000000..1c57bc38a --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/info/ClusterSetInfo.java @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.info; + +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; +import org.deeplearning4j.clustering.cluster.Cluster; +import org.deeplearning4j.clustering.cluster.ClusterSet; + +import java.io.Serializable; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class ClusterSetInfo implements Serializable { + + private Map clustersInfos = new HashMap<>(); + private Table distancesBetweenClustersCenters = HashBasedTable.create(); + private AtomicInteger pointLocationChange; + private boolean threadSafe; + private boolean inverse; + + public ClusterSetInfo(boolean inverse) { + this(inverse, false); + } + + /** + * + * @param inverse + * @param threadSafe + */ + public ClusterSetInfo(boolean inverse, boolean threadSafe) { + this.pointLocationChange = new AtomicInteger(0); + this.threadSafe = threadSafe; + this.inverse = inverse; + if (threadSafe) { + clustersInfos = Collections.synchronizedMap(clustersInfos); + } + } + + + /** + * + * @param clusterSet + * @param threadSafe + * @return + */ + public static ClusterSetInfo initialize(ClusterSet clusterSet, boolean threadSafe) { + ClusterSetInfo info = new ClusterSetInfo(clusterSet.isInverse(), threadSafe); + for (int i = 0, j = clusterSet.getClusterCount(); i < j; i++) + info.addClusterInfo(clusterSet.getClusters().get(i).getId()); + return info; + } + + public void removeClusterInfos(List clusters) { + for (Cluster cluster : clusters) { + clustersInfos.remove(cluster.getId()); + } + } + + public ClusterInfo addClusterInfo(String clusterId) { + ClusterInfo clusterInfo = new ClusterInfo(this.threadSafe); + clustersInfos.put(clusterId, clusterInfo); + return clusterInfo; + } + + public ClusterInfo getClusterInfo(String clusterId) { + return clustersInfos.get(clusterId); + } + + public double getAveragePointDistanceFromClusterCenter() { + if (clustersInfos == null || clustersInfos.isEmpty()) + return 0; + + double average = 0; + for (ClusterInfo info : clustersInfos.values()) + average += info.getAveragePointDistanceFromCenter(); + return average / clustersInfos.size(); + } + + public double getPointDistanceFromClusterVariance() { + if (clustersInfos == null || clustersInfos.isEmpty()) + return 0; + + double average = 0; + for (ClusterInfo info : clustersInfos.values()) + average += info.getPointDistanceFromCenterVariance(); + return average / clustersInfos.size(); + } + + public int getPointsCount() { + int count = 0; + for (ClusterInfo clusterInfo : clustersInfos.values()) + count += clusterInfo.getPointDistancesFromCenter().size(); + return count; + } + + public Map getClustersInfos() { + return clustersInfos; + } + + public void setClustersInfos(Map clustersInfos) { + this.clustersInfos = clustersInfos; + } + + public Table getDistancesBetweenClustersCenters() { + return distancesBetweenClustersCenters; + } + + public void setDistancesBetweenClustersCenters(Table interClusterDistances) { + this.distancesBetweenClustersCenters = interClusterDistances; + } + + public AtomicInteger getPointLocationChange() { + return pointLocationChange; + } + + public void setPointLocationChange(AtomicInteger pointLocationChange) { + this.pointLocationChange = pointLocationChange; + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/iteration/IterationHistory.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/iteration/IterationHistory.java new file mode 100644 index 000000000..82dea11ca --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/iteration/IterationHistory.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.iteration; + +import lombok.Getter; +import lombok.Setter; +import org.deeplearning4j.clustering.info.ClusterSetInfo; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +public class IterationHistory implements Serializable { + @Getter + @Setter + private Map iterationsInfos = new HashMap<>(); + + /** + * + * @return + */ + public ClusterSetInfo getMostRecentClusterSetInfo() { + IterationInfo iterationInfo = getMostRecentIterationInfo(); + return iterationInfo == null ? null : iterationInfo.getClusterSetInfo(); + } + + /** + * + * @return + */ + public IterationInfo getMostRecentIterationInfo() { + return getIterationInfo(getIterationCount() - 1); + } + + /** + * + * @return + */ + public int getIterationCount() { + return getIterationsInfos().size(); + } + + /** + * + * @param iterationIdx + * @return + */ + public IterationInfo getIterationInfo(int iterationIdx) { + return getIterationsInfos().get(iterationIdx); + } + + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/iteration/IterationInfo.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/iteration/IterationInfo.java new file mode 100644 index 000000000..d9f32f58e --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/iteration/IterationInfo.java @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.iteration; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.deeplearning4j.clustering.info.ClusterSetInfo; + +import java.io.Serializable; + +@Data +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class IterationInfo implements Serializable { + + private int index; + private ClusterSetInfo clusterSetInfo; + private boolean strategyApplied; + + public IterationInfo(int index) { + super(); + this.index = index; + } + + public IterationInfo(int index, ClusterSetInfo clusterSetInfo) { + super(); + this.index = index; + this.clusterSetInfo = clusterSetInfo; + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kdtree/HyperRect.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kdtree/HyperRect.java new file mode 100644 index 000000000..013263629 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kdtree/HyperRect.java @@ -0,0 +1,141 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.kdtree; + +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.custom.KnnMinDistance; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.io.Serializable; + +/** + * Created by agibsonccc on 12/29/14. + */ +public class HyperRect implements Serializable { + + //private List points; + private float[] lowerEnds; + private float[] higherEnds; + private INDArray lowerEndsIND; + private INDArray higherEndsIND; + + public HyperRect(float[] lowerEndsIn, float[] higherEndsIn) { + this.lowerEnds = new float[lowerEndsIn.length]; + this.higherEnds = new float[lowerEndsIn.length]; + System.arraycopy(lowerEndsIn, 0 , this.lowerEnds, 0, lowerEndsIn.length); + System.arraycopy(higherEndsIn, 0 , this.higherEnds, 0, higherEndsIn.length); + lowerEndsIND = Nd4j.createFromArray(lowerEnds); + higherEndsIND = Nd4j.createFromArray(higherEnds); + } + + public HyperRect(float[] point) { + this(point, point); + } + + public HyperRect(Pair ends) { + this(ends.getFirst(), ends.getSecond()); + } + + + public void enlargeTo(INDArray point) { + float[] pointAsArray = point.toFloatVector(); + for (int i = 0; i < lowerEnds.length; i++) { + float p = pointAsArray[i]; + if (lowerEnds[i] > p) + lowerEnds[i] = p; + else if (higherEnds[i] < p) + higherEnds[i] = p; + } + } + + public static Pair point(INDArray vector) { + Pair ret = new Pair<>(); + float[] curr = new float[(int)vector.length()]; + for (int i = 0; i < vector.length(); i++) { + curr[i] = vector.getFloat(i); + } + ret.setFirst(curr); + ret.setSecond(curr); + return ret; + } + + + /*public List contains(INDArray hPoint) { + List ret = new ArrayList<>(); + for (int i = 0; i < hPoint.length(); i++) { + ret.add(lowerEnds[i] <= hPoint.getDouble(i) && + higherEnds[i] >= hPoint.getDouble(i)); + } + return ret; + }*/ + + public double minDistance(INDArray hPoint, INDArray output) { + Nd4j.exec(new KnnMinDistance(hPoint, lowerEndsIND, higherEndsIND, output)); + return output.getFloat(0); + + /*double ret = 0.0; + double[] pointAsArray = hPoint.toDoubleVector(); + for (int i = 0; i < pointAsArray.length; i++) { + double p = pointAsArray[i]; + if (!(lowerEnds[i] <= p || higherEnds[i] <= p)) { + if (p < lowerEnds[i]) + ret += Math.pow((p - lowerEnds[i]), 2); + else + ret += Math.pow((p - higherEnds[i]), 2); + } + } + ret = Math.pow(ret, 0.5); + return ret;*/ + } + + public HyperRect getUpper(INDArray hPoint, int desc) { + //Interval interval = points.get(desc); + float higher = higherEnds[desc]; + float d = hPoint.getFloat(desc); + if (higher < d) + return null; + HyperRect ret = new HyperRect(lowerEnds,higherEnds); + if (ret.lowerEnds[desc] < d) + ret.lowerEnds[desc] = d; + return ret; + } + + public HyperRect getLower(INDArray hPoint, int desc) { + //Interval interval = points.get(desc); + float lower = lowerEnds[desc]; + float d = hPoint.getFloat(desc); + if (lower > d) + return null; + HyperRect ret = new HyperRect(lowerEnds,higherEnds); + //Interval i2 = ret.points.get(desc); + if (ret.higherEnds[desc] > d) + ret.higherEnds[desc] = d; + return ret; + } + + @Override + public String toString() { + String retVal = ""; + retVal += "["; + for (int i = 0; i < lowerEnds.length; ++i) { + retVal += "(" + lowerEnds[i] + " - " + higherEnds[i] + ") "; + } + retVal += "]"; + return retVal; + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kdtree/KDTree.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kdtree/KDTree.java new file mode 100644 index 000000000..68ccf6281 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kdtree/KDTree.java @@ -0,0 +1,371 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.kdtree; + +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.reduce.bool.Any; +import org.nd4j.linalg.api.ops.impl.reduce3.EuclideanDistance; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * KDTree based on: https://github.com/nicky-zs/kdtree-python/blob/master/kdtree.py + * + * @author Adam Gibson + */ +public class KDTree implements Serializable { + + private KDNode root; + private int dims = 100; + public final static int GREATER = 1; + public final static int LESS = 0; + private int size = 0; + private HyperRect rect; + + public KDTree(int dims) { + this.dims = dims; + } + + /** + * Insert a point in to the tree + * @param point the point to insert + */ + public void insert(INDArray point) { + if (!point.isVector() || point.length() != dims) + throw new IllegalArgumentException("Point must be a vector of length " + dims); + + if (root == null) { + root = new KDNode(point); + rect = new HyperRect(/*HyperRect.point(point)*/ point.toFloatVector()); + } else { + int disc = 0; + KDNode node = root; + KDNode insert = new KDNode(point); + int successor; + while (true) { + //exactly equal + INDArray pt = node.getPoint(); + INDArray countEq = Nd4j.getExecutioner().execAndReturn(new Any(pt.neq(point))).z(); + if (countEq.getInt(0) == 0) { + return; + } else { + successor = successor(node, point, disc); + KDNode child; + if (successor < 1) + child = node.getLeft(); + else + child = node.getRight(); + if (child == null) + break; + disc = (disc + 1) % dims; + node = child; + } + } + + if (successor < 1) + node.setLeft(insert); + + else + node.setRight(insert); + + rect.enlargeTo(point); + insert.setParent(node); + } + size++; + + } + + + public INDArray delete(INDArray point) { + KDNode node = root; + int _disc = 0; + while (node != null) { + if (node.point == point) + break; + int successor = successor(node, point, _disc); + if (successor < 1) + node = node.getLeft(); + else + node = node.getRight(); + _disc = (_disc + 1) % dims; + } + + if (node != null) { + if (node == root) { + root = delete(root, _disc); + } else + node = delete(node, _disc); + size--; + if (size == 1) { + rect = new HyperRect(HyperRect.point(point)); + } else if (size == 0) + rect = null; + + } + return node.getPoint(); + } + + // Share this data for recursive calls of "knn" + private float currentDistance; + private INDArray currentPoint; + private INDArray minDistance = Nd4j.scalar(0.f); + + + public List> knn(INDArray point, float distance) { + List> best = new ArrayList<>(); + currentDistance = distance; + currentPoint = point; + knn(root, rect, best, 0); + Collections.sort(best, new Comparator>() { + @Override + public int compare(Pair o1, Pair o2) { + return Float.compare(o1.getKey(), o2.getKey()); + } + }); + + return best; + } + + + private void knn(KDNode node, HyperRect rect, List> best, int _disc) { + if (node == null || rect == null || rect.minDistance(currentPoint, minDistance) > currentDistance) + return; + int _discNext = (_disc + 1) % dims; + float distance = Nd4j.getExecutioner().execAndReturn(new EuclideanDistance(currentPoint,node.point, minDistance)).getFinalResult() + .floatValue(); + + if (distance <= currentDistance) { + best.add(Pair.of(distance, node.getPoint())); + } + + HyperRect lower = rect.getLower(node.point, _disc); + HyperRect upper = rect.getUpper(node.point, _disc); + knn(node.getLeft(), lower, best, _discNext); + knn(node.getRight(), upper, best, _discNext); + } + + /** + * Query for nearest neighbor. Returns the distance and point + * @param point the point to query for + * @return + */ + public Pair nn(INDArray point) { + return nn(root, point, rect, Double.POSITIVE_INFINITY, null, 0); + } + + + private Pair nn(KDNode node, INDArray point, HyperRect rect, double dist, INDArray best, + int _disc) { + if (node == null || rect.minDistance(point, minDistance) > dist) + return Pair.of(Double.POSITIVE_INFINITY, null); + + int _discNext = (_disc + 1) % dims; + double dist2 = Nd4j.getExecutioner().execAndReturn(new EuclideanDistance(point, Nd4j.zeros(point.dataType(), point.shape()))).getFinalResult().doubleValue(); + if (dist2 < dist) { + best = node.getPoint(); + dist = dist2; + } + + HyperRect lower = rect.getLower(node.point, _disc); + HyperRect upper = rect.getUpper(node.point, _disc); + + if (point.getDouble(_disc) < node.point.getDouble(_disc)) { + Pair left = nn(node.getLeft(), point, lower, dist, best, _discNext); + Pair right = nn(node.getRight(), point, upper, dist, best, _discNext); + if (left.getKey() < dist) + return left; + else if (right.getKey() < dist) + return right; + + } else { + Pair left = nn(node.getRight(), point, upper, dist, best, _discNext); + Pair right = nn(node.getLeft(), point, lower, dist, best, _discNext); + if (left.getKey() < dist) + return left; + else if (right.getKey() < dist) + return right; + } + + return Pair.of(dist, best); + + } + + private KDNode delete(KDNode delete, int _disc) { + if (delete.getLeft() != null && delete.getRight() != null) { + if (delete.getParent() != null) { + if (delete.getParent().getLeft() == delete) + delete.getParent().setLeft(null); + else + delete.getParent().setRight(null); + + } + return null; + } + + int disc = _disc; + _disc = (_disc + 1) % dims; + Pair qd = null; + if (delete.getRight() != null) { + qd = min(delete.getRight(), disc, _disc); + } else if (delete.getLeft() != null) + qd = max(delete.getLeft(), disc, _disc); + if (qd == null) {// is leaf + return null; + } + delete.point = qd.getKey().point; + KDNode qFather = qd.getKey().getParent(); + if (qFather.getLeft() == qd.getKey()) { + qFather.setLeft(delete(qd.getKey(), disc)); + } else if (qFather.getRight() == qd.getKey()) { + qFather.setRight(delete(qd.getKey(), disc)); + + } + + return delete; + + + } + + + private Pair max(KDNode node, int disc, int _disc) { + int discNext = (_disc + 1) % dims; + if (_disc == disc) { + KDNode child = node.getLeft(); + if (child != null) { + return max(child, disc, discNext); + } + } else if (node.getLeft() != null || node.getRight() != null) { + Pair left = null, right = null; + if (node.getLeft() != null) + left = max(node.getLeft(), disc, discNext); + if (node.getRight() != null) + right = max(node.getRight(), disc, discNext); + if (left != null && right != null) { + double pointLeft = left.getKey().getPoint().getDouble(disc); + double pointRight = right.getKey().getPoint().getDouble(disc); + if (pointLeft > pointRight) + return left; + else + return right; + } else if (left != null) + return left; + else + return right; + } + + return Pair.of(node, _disc); + } + + + + private Pair min(KDNode node, int disc, int _disc) { + int discNext = (_disc + 1) % dims; + if (_disc == disc) { + KDNode child = node.getLeft(); + if (child != null) { + return min(child, disc, discNext); + } + } else if (node.getLeft() != null || node.getRight() != null) { + Pair left = null, right = null; + if (node.getLeft() != null) + left = min(node.getLeft(), disc, discNext); + if (node.getRight() != null) + right = min(node.getRight(), disc, discNext); + if (left != null && right != null) { + double pointLeft = left.getKey().getPoint().getDouble(disc); + double pointRight = right.getKey().getPoint().getDouble(disc); + if (pointLeft < pointRight) + return left; + else + return right; + } else if (left != null) + return left; + else + return right; + } + + return Pair.of(node, _disc); + } + + /** + * The number of elements in the tree + * @return the number of elements in the tree + */ + public int size() { + return size; + } + + private int successor(KDNode node, INDArray point, int disc) { + for (int i = disc; i < dims; i++) { + double pointI = point.getDouble(i); + double nodePointI = node.getPoint().getDouble(i); + if (pointI < nodePointI) + return LESS; + else if (pointI > nodePointI) + return GREATER; + + } + + throw new IllegalStateException("Point is equal!"); + } + + + private static class KDNode { + private INDArray point; + private KDNode left, right, parent; + + public KDNode(INDArray point) { + this.point = point; + } + + public INDArray getPoint() { + return point; + } + + public KDNode getLeft() { + return left; + } + + public void setLeft(KDNode left) { + this.left = left; + } + + public KDNode getRight() { + return right; + } + + public void setRight(KDNode right) { + this.right = right; + } + + public KDNode getParent() { + return parent; + } + + public void setParent(KDNode parent) { + this.parent = parent; + } + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kmeans/KMeansClustering.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kmeans/KMeansClustering.java new file mode 100644 index 000000000..e95cd5c9e --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/kmeans/KMeansClustering.java @@ -0,0 +1,110 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.kmeans; + +import org.deeplearning4j.clustering.algorithm.BaseClusteringAlgorithm; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.strategy.ClusteringStrategy; +import org.deeplearning4j.clustering.strategy.FixedClusterCountStrategy; + + +/** + * + * @author Julien Roch + * + */ +public class KMeansClustering extends BaseClusteringAlgorithm { + + private static final long serialVersionUID = 8476951388145944776L; + private static final double VARIATION_TOLERANCE= 1e-4; + + + /** + * + * @param clusteringStrategy + */ + protected KMeansClustering(ClusteringStrategy clusteringStrategy, boolean useKMeansPlusPlus) { + super(clusteringStrategy, useKMeansPlusPlus); + } + + /** + * Setup a kmeans instance + * @param clusterCount the number of clusters + * @param maxIterationCount the max number of iterations + * to run kmeans + * @param distanceFunction the distance function to use for grouping + * @return + */ + public static KMeansClustering setup(int clusterCount, int maxIterationCount, Distance distanceFunction, + boolean inverse, boolean useKMeansPlusPlus) { + ClusteringStrategy clusteringStrategy = + FixedClusterCountStrategy.setup(clusterCount, distanceFunction, inverse); + clusteringStrategy.endWhenIterationCountEquals(maxIterationCount); + return new KMeansClustering(clusteringStrategy, useKMeansPlusPlus); + } + + /** + * + * @param clusterCount + * @param minDistributionVariationRate + * @param distanceFunction + * @param allowEmptyClusters + * @return + */ + public static KMeansClustering setup(int clusterCount, double minDistributionVariationRate, Distance distanceFunction, + boolean inverse, boolean allowEmptyClusters, boolean useKMeansPlusPlus) { + ClusteringStrategy clusteringStrategy = FixedClusterCountStrategy.setup(clusterCount, distanceFunction, inverse) + .endWhenDistributionVariationRateLessThan(minDistributionVariationRate); + return new KMeansClustering(clusteringStrategy, useKMeansPlusPlus); + } + + + /** + * Setup a kmeans instance + * @param clusterCount the number of clusters + * @param maxIterationCount the max number of iterations + * to run kmeans + * @param distanceFunction the distance function to use for grouping + * @return + */ + public static KMeansClustering setup(int clusterCount, int maxIterationCount, Distance distanceFunction, boolean useKMeansPlusPlus) { + return setup(clusterCount, maxIterationCount, distanceFunction, false, useKMeansPlusPlus); + } + + /** + * + * @param clusterCount + * @param minDistributionVariationRate + * @param distanceFunction + * @param allowEmptyClusters + * @return + */ + public static KMeansClustering setup(int clusterCount, double minDistributionVariationRate, Distance distanceFunction, + boolean allowEmptyClusters, boolean useKMeansPlusPlus) { + ClusteringStrategy clusteringStrategy = FixedClusterCountStrategy.setup(clusterCount, distanceFunction, false); + clusteringStrategy.endWhenDistributionVariationRateLessThan(minDistributionVariationRate); + return new KMeansClustering(clusteringStrategy, useKMeansPlusPlus); + } + + public static KMeansClustering setup(int clusterCount, Distance distanceFunction, + boolean allowEmptyClusters, boolean useKMeansPlusPlus) { + ClusteringStrategy clusteringStrategy = FixedClusterCountStrategy.setup(clusterCount, distanceFunction, false); + clusteringStrategy.endWhenDistributionVariationRateLessThan(VARIATION_TOLERANCE); + return new KMeansClustering(clusteringStrategy, useKMeansPlusPlus); + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/lsh/LSH.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/lsh/LSH.java new file mode 100644 index 000000000..e4e6e99fd --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/lsh/LSH.java @@ -0,0 +1,92 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.lsh; + +import org.nd4j.linalg.api.ndarray.INDArray; + +/** + * This interface gathers the minimal elements for an LSH implementation + * + * See chapter 3 of : + * _Mining Massive Datasets_, Anand Rajaraman and Jeffrey Ullman + * http://www.mmds.org/ + * + */ +public interface LSH { + + /** + * Returns an instance of the distance measure associated to the LSH family of this implementation. + * Beware, hashing families and their amplification constructs are distance-specific. + */ + String getDistanceMeasure(); + + /** + * Returns the size of a hash compared against in one hashing bucket, corresponding to an AND construction + * + * denoting hashLength by h, + * amplifies a (d1, d2, p1, p2) hash family into a + * (d1, d2, p1^h, p2^h)-sensitive one (match probability is decreasing with h) + * + * @return the length of the hash in the AND construction used by this index + */ + int getHashLength(); + + /** + * + * denoting numTables by n, + * amplifies a (d1, d2, p1, p2) hash family into a + * (d1, d2, (1-p1^n), (1-p2^n))-sensitive one (match probability is increasing with n) + * + * @return the # of hash tables in the OR construction used by this index + */ + int getNumTables(); + + /** + * @return The dimension of the index vectors and queries + */ + int getInDimension(); + + /** + * Populates the index with data vectors. + * @param data the vectors to index + */ + void makeIndex(INDArray data); + + /** + * Returns the set of all vectors that could approximately be considered negihbors of the query, + * without selection on the basis of distance or number of neighbors. + * @param query a vector to find neighbors for + * @return its approximate neighbors, unfiltered + */ + INDArray bucket(INDArray query); + + /** + * Returns the approximate neighbors within a distance bound. + * @param query a vector to find neighbors for + * @param maxRange the maximum distance between results and the query + * @return approximate neighbors within the distance bounds + */ + INDArray search(INDArray query, double maxRange); + + /** + * Returns the approximate neighbors within a k-closest bound + * @param query a vector to find neighbors for + * @param k the maximum number of closest neighbors to return + * @return at most k neighbors of the query, ordered by increasing distance + */ + INDArray search(INDArray query, int k); +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/lsh/RandomProjectionLSH.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/lsh/RandomProjectionLSH.java new file mode 100644 index 000000000..75e342e78 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/lsh/RandomProjectionLSH.java @@ -0,0 +1,245 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.lsh; + +import lombok.Getter; +import lombok.val; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.broadcast.bool.BroadcastEqualTo; +import org.nd4j.linalg.api.ops.impl.transforms.same.Sign; +import org.nd4j.linalg.api.ops.random.impl.GaussianDistribution; +import org.nd4j.linalg.api.rng.Random; +import org.nd4j.linalg.exception.ND4JIllegalStateException; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.BooleanIndexing; +import org.nd4j.linalg.indexing.conditions.Conditions; +import org.nd4j.linalg.ops.transforms.Transforms; + +import java.util.Arrays; + + +/** + * This class implements Entropy LSH for the cosine distance, in order to preserve memory for large datasets. + * + * Entropy SLH is the LSH scheme of + * + * _Entropy based nearest neighbor search in high dimensions_ + * R Panigrahy - SIAM 2006 + * https://arxiv.org/pdf/cs/0510019.pdf + * + * To read more about LSH, in particular for the Cosine distance, see + * chapter 3 of : + * _Mining Massive Datasets_, Anand Rajaraman and Jeffrey Ullman + * http://www.mmds.org/ + * + * The original development of LSH for the cosine distance is from + * Similarity estimation techniques from rounding algorithms + * MS Charikar - STOCS, 2002 + * + * Note for high-precision or distributed settings, you should not + * use this and rather extend this to layered LSH ( https://arxiv.org/abs/1210.7057 ) + * + */ +public class RandomProjectionLSH implements LSH { + + @Override + public String getDistanceMeasure(){ + return "cosinedistance"; + } + + @Getter private int hashLength; + + @Getter private int numTables; + + @Getter private int inDimension; + + + @Getter private double radius; + + INDArray randomProjection; + + INDArray index; + + INDArray indexData; + + + private INDArray gaussianRandomMatrix(int[] shape, Random rng){ + INDArray res = Nd4j.create(shape); + + GaussianDistribution op1 = new GaussianDistribution(res, 0.0, 1.0 / Math.sqrt(shape[0])); + + Nd4j.getExecutioner().exec(op1, rng); + return res; + } + + public RandomProjectionLSH(int hashLength, int numTables, int inDimension, double radius){ + this(hashLength, numTables, inDimension, radius, Nd4j.getRandom()); + } + + /** + * Creates a locality-sensitive hashing index for the cosine distance, + * a (d1, d2, (180 − d1)/180,(180 − d2)/180)-sensitive hash family before amplification + * + * @param hashLength the length of the compared hash in an AND construction, + * @param numTables the entropy-equivalent of a nb of hash tables in an OR construction, implemented here with the multiple + * probes of Panigraphi (op. cit). + * @param inDimension the dimendionality of the points being indexed + * @param radius the radius of points to generate probes for. Instead of using multiple physical hash tables in an OR construction + * @param rng a Random object to draw samples from + */ + public RandomProjectionLSH(int hashLength, int numTables, int inDimension, double radius, Random rng){ + this.hashLength = hashLength; + this.numTables = numTables; + this.inDimension = inDimension; + this.radius = radius; + randomProjection = gaussianRandomMatrix(new int[]{inDimension, hashLength}, rng); + } + + /** + * This picks uniformaly distributed random points on the unit of a sphere using the method of: + * + * An efficient method for generating uniformly distributed points on the surface of an n-dimensional sphere + * JS Hicks, RF Wheeling - Communications of the ACM, 1959 + * @param data a query to generate multiple probes for + * @return `numTables` + */ + public INDArray entropy(INDArray data){ + + INDArray data2 = + Nd4j.getExecutioner().exec(new GaussianDistribution(Nd4j.create(numTables, inDimension), radius)); + + INDArray norms = Nd4j.norm2(data2.dup(), -1); + + Preconditions.checkState(norms.rank() == 1 && norms.size(0) == numTables, "Expected norm2 to have shape [%s], is %ndShape", norms.size(0), norms); + + data2.diviColumnVector(norms); + data2.addiRowVector(data); + return data2; + } + + /** + * Returns hash values for a particular query + * @param data a query vector + * @return its hashed value + */ + public INDArray hash(INDArray data) { + if (data.shape()[1] != inDimension){ + throw new ND4JIllegalStateException( + String.format("Invalid shape: Requested INDArray shape %s, this table expects dimension %d", + Arrays.toString(data.shape()), inDimension)); + } + INDArray projected = data.mmul(randomProjection); + INDArray res = Nd4j.getExecutioner().exec(new Sign(projected)); + return res; + } + + /** + * Populates the index. Beware, not incremental, any further call replaces the index instead of adding to it. + * @param data the vectors to index + */ + @Override + public void makeIndex(INDArray data) { + index = hash(data); + indexData = data; + } + + // data elements in the same bucket as the query, without entropy + INDArray rawBucketOf(INDArray query){ + INDArray pattern = hash(query); + + INDArray res = Nd4j.zeros(DataType.BOOL, index.shape()); + Nd4j.getExecutioner().exec(new BroadcastEqualTo(index, pattern, res, -1)); + return res.castTo(Nd4j.defaultFloatingPointType()).min(-1); + } + + @Override + public INDArray bucket(INDArray query) { + INDArray queryRes = rawBucketOf(query); + + if(numTables > 1) { + INDArray entropyQueries = entropy(query); + + // loop, addi + conditionalreplace -> poor man's OR function + for (int i = 0; i < numTables; i++) { + INDArray row = entropyQueries.getRow(i, true); + queryRes.addi(rawBucketOf(row)); + } + BooleanIndexing.replaceWhere(queryRes, 1.0, Conditions.greaterThan(0.0)); + } + + return queryRes; + } + + // data elements in the same entropy bucket as the query, + INDArray bucketData(INDArray query){ + INDArray mask = bucket(query); + int nRes = mask.sum(0).getInt(0); + INDArray res = Nd4j.create(new int[] {nRes, inDimension}); + int j = 0; + for (int i = 0; i < nRes; i++){ + while (mask.getInt(j) == 0 && j < mask.length() - 1) { + j += 1; + } + if (mask.getInt(j) == 1) res.putRow(i, indexData.getRow(j)); + j += 1; + } + return res; + } + + @Override + public INDArray search(INDArray query, double maxRange) { + if (maxRange < 0) + throw new IllegalArgumentException("ANN search should have a positive maximum search radius"); + + INDArray bucketData = bucketData(query); + INDArray distances = Transforms.allCosineDistances(bucketData, query, -1); + INDArray[] idxs = Nd4j.sortWithIndices(distances, -1, true); + + INDArray shuffleIndexes = idxs[0]; + INDArray sortedDistances = idxs[1]; + int accepted = 0; + while (accepted < sortedDistances.length() && sortedDistances.getInt(accepted) <= maxRange) accepted +=1; + + INDArray res = Nd4j.create(new int[] {accepted, inDimension}); + for(int i = 0; i < accepted; i++){ + res.putRow(i, bucketData.getRow(shuffleIndexes.getInt(i))); + } + return res; + } + + @Override + public INDArray search(INDArray query, int k) { + if (k < 1) + throw new IllegalArgumentException("An ANN search for k neighbors should at least seek one neighbor"); + + INDArray bucketData = bucketData(query); + INDArray distances = Transforms.allCosineDistances(bucketData, query, -1); + INDArray[] idxs = Nd4j.sortWithIndices(distances, -1, true); + + INDArray shuffleIndexes = idxs[0]; + INDArray sortedDistances = idxs[1]; + val accepted = Math.min(k, sortedDistances.shape()[1]); + + INDArray res = Nd4j.create(accepted, inDimension); + for(int i = 0; i < accepted; i++){ + res.putRow(i, bucketData.getRow(shuffleIndexes.getInt(i))); + } + return res; + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/optimisation/ClusteringOptimization.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/optimisation/ClusteringOptimization.java new file mode 100644 index 000000000..231939bf3 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/optimisation/ClusteringOptimization.java @@ -0,0 +1,34 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.optimisation; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@Data +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@AllArgsConstructor +public class ClusteringOptimization implements Serializable { + + private ClusteringOptimizationType type; + private double value; + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/optimisation/ClusteringOptimizationType.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/optimisation/ClusteringOptimizationType.java new file mode 100644 index 000000000..e2d09b94f --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/optimisation/ClusteringOptimizationType.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.optimisation; + +/** + * + */ +public enum ClusteringOptimizationType { + MINIMIZE_AVERAGE_POINT_TO_CENTER_DISTANCE, MINIMIZE_MAXIMUM_POINT_TO_CENTER_DISTANCE, MINIMIZE_AVERAGE_POINT_TO_POINT_DISTANCE, MINIMIZE_MAXIMUM_POINT_TO_POINT_DISTANCE, MINIMIZE_PER_CLUSTER_POINT_COUNT +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/quadtree/Cell.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/quadtree/Cell.java new file mode 100644 index 000000000..3db3aff91 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/quadtree/Cell.java @@ -0,0 +1,115 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.quadtree; + +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.Serializable; + +/** + * A cell representing a bounding box forthe quad tree + * @author Adam Gibson + */ +public class Cell implements Serializable { + private double x, y, hw, hh; + + public Cell(double x, double y, double hw, double hh) { + this.x = x; + this.y = y; + this.hw = hw; + this.hh = hh; + } + + /** + * Whether the given point is contained + * within this cell + * @param point the point to check + * @return true if the point is contained, false otherwise + */ + public boolean containsPoint(INDArray point) { + double first = point.getDouble(0), second = point.getDouble(1); + return x - hw <= first && x + hw >= first && y - hh <= second && y + hh >= second; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof Cell)) + return false; + + Cell cell = (Cell) o; + + if (Double.compare(cell.hh, hh) != 0) + return false; + if (Double.compare(cell.hw, hw) != 0) + return false; + if (Double.compare(cell.x, x) != 0) + return false; + return Double.compare(cell.y, y) == 0; + + } + + @Override + public int hashCode() { + int result; + long temp; + temp = Double.doubleToLongBits(x); + result = (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(y); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(hw); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(hh); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + public double getX() { + return x; + } + + public void setX(double x) { + this.x = x; + } + + public double getY() { + return y; + } + + public void setY(double y) { + this.y = y; + } + + public double getHw() { + return hw; + } + + public void setHw(double hw) { + this.hw = hw; + } + + public double getHh() { + return hh; + } + + public void setHh(double hh) { + this.hh = hh; + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/quadtree/QuadTree.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/quadtree/QuadTree.java new file mode 100644 index 000000000..b26ffc636 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/quadtree/QuadTree.java @@ -0,0 +1,389 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.quadtree; + +import com.google.common.util.concurrent.AtomicDouble; +import org.apache.commons.math3.util.FastMath; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.Serializable; + +import static java.lang.Math.max; + +/** + * QuadTree: http://en.wikipedia.org/wiki/Quadtree + * + * Reference impl based on the paper by: + * https://arxiv.org/pdf/1301.3342v2.pdf + * + * Primarily focused on 2 dimensions, may expand later if there's a reason. + * + * @author Adam Gibson + */ +public class QuadTree implements Serializable { + private QuadTree parent, northWest, northEast, southWest, southEast; + private boolean isLeaf = true; + private int size, cumSize; + private Cell boundary; + static final int QT_NO_DIMS = 2; + static final int QT_NODE_CAPACITY = 1; + private INDArray buf = Nd4j.create(QT_NO_DIMS); + private INDArray data, centerOfMass = Nd4j.create(QT_NO_DIMS); + private int[] index = new int[QT_NODE_CAPACITY]; + + + /** + * Pass in a matrix + * @param data + */ + public QuadTree(INDArray data) { + INDArray meanY = data.mean(0); + INDArray minY = data.min(0); + INDArray maxY = data.max(0); + init(data, meanY.getDouble(0), meanY.getDouble(1), + max(maxY.getDouble(0) - meanY.getDouble(0), meanY.getDouble(0) - minY.getDouble(0)) + + Nd4j.EPS_THRESHOLD, + max(maxY.getDouble(1) - meanY.getDouble(1), meanY.getDouble(1) - minY.getDouble(1)) + + Nd4j.EPS_THRESHOLD); + fill(); + } + + public QuadTree(QuadTree parent, INDArray data, Cell boundary) { + this.parent = parent; + this.boundary = boundary; + this.data = data; + + } + + public QuadTree(Cell boundary) { + this.boundary = boundary; + } + + private void init(INDArray data, double x, double y, double hw, double hh) { + boundary = new Cell(x, y, hw, hh); + this.data = data; + } + + private void fill() { + for (int i = 0; i < data.rows(); i++) + insert(i); + } + + + + /** + * Returns the cell of this element + * + * @param coordinates + * @return + */ + protected QuadTree findIndex(INDArray coordinates) { + + // Compute the sector for the coordinates + boolean left = (coordinates.getDouble(0) <= (boundary.getX() + boundary.getHw() / 2)); + boolean top = (coordinates.getDouble(1) <= (boundary.getY() + boundary.getHh() / 2)); + + // top left + QuadTree index = getNorthWest(); + if (left) { + // left side + if (!top) { + // bottom left + index = getSouthWest(); + } + } else { + // right side + if (top) { + // top right + index = getNorthEast(); + } else { + // bottom right + index = getSouthEast(); + + } + } + + return index; + } + + + /** + * Insert an index of the data in to the tree + * @param newIndex the index to insert in to the tree + * @return whether the index was inserted or not + */ + public boolean insert(int newIndex) { + // Ignore objects which do not belong in this quad tree + INDArray point = data.slice(newIndex); + if (!boundary.containsPoint(point)) + return false; + + cumSize++; + double mult1 = (double) (cumSize - 1) / (double) cumSize; + double mult2 = 1.0 / (double) cumSize; + + centerOfMass.muli(mult1); + centerOfMass.addi(point.mul(mult2)); + + // If there is space in this quad tree and it is a leaf, add the object here + if (isLeaf() && size < QT_NODE_CAPACITY) { + index[size] = newIndex; + size++; + return true; + } + + //duplicate point + if (size > 0) { + for (int i = 0; i < size; i++) { + INDArray compPoint = data.slice(index[i]); + if (point.getDouble(0) == compPoint.getDouble(0) && point.getDouble(1) == compPoint.getDouble(1)) + return true; + } + } + + + + // If this Node has already been subdivided just add the elements to the + // appropriate cell + if (!isLeaf()) { + QuadTree index = findIndex(point); + index.insert(newIndex); + return true; + } + + if (isLeaf()) + subDivide(); + + boolean ret = insertIntoOneOf(newIndex); + + + + return ret; + } + + private boolean insertIntoOneOf(int index) { + boolean success = false; + success = northWest.insert(index); + if (!success) + success = northEast.insert(index); + if (!success) + success = southWest.insert(index); + if (!success) + success = southEast.insert(index); + return success; + } + + + /** + * Returns whether the tree is consistent or not + * @return whether the tree is consistent or not + */ + public boolean isCorrect() { + + for (int n = 0; n < size; n++) { + INDArray point = data.slice(index[n]); + if (!boundary.containsPoint(point)) + return false; + } + + return isLeaf() || northWest.isCorrect() && northEast.isCorrect() && southWest.isCorrect() + && southEast.isCorrect(); + + } + + + + /** + * Create four children + * which fully divide this cell + * into four quads of equal area + */ + public void subDivide() { + northWest = new QuadTree(this, data, new Cell(boundary.getX() - .5 * boundary.getHw(), + boundary.getY() - .5 * boundary.getHh(), .5 * boundary.getHw(), .5 * boundary.getHh())); + northEast = new QuadTree(this, data, new Cell(boundary.getX() + .5 * boundary.getHw(), + boundary.getY() - .5 * boundary.getHh(), .5 * boundary.getHw(), .5 * boundary.getHh())); + southWest = new QuadTree(this, data, new Cell(boundary.getX() - .5 * boundary.getHw(), + boundary.getY() + .5 * boundary.getHh(), .5 * boundary.getHw(), .5 * boundary.getHh())); + southEast = new QuadTree(this, data, new Cell(boundary.getX() + .5 * boundary.getHw(), + boundary.getY() + .5 * boundary.getHh(), .5 * boundary.getHw(), .5 * boundary.getHh())); + + } + + + /** + * Compute non edge forces using barnes hut + * @param pointIndex + * @param theta + * @param negativeForce + * @param sumQ + */ + public void computeNonEdgeForces(int pointIndex, double theta, INDArray negativeForce, AtomicDouble sumQ) { + // Make sure that we spend no time on empty nodes or self-interactions + if (cumSize == 0 || (isLeaf() && size == 1 && index[0] == pointIndex)) + return; + + + // Compute distance between point and center-of-mass + buf.assign(data.slice(pointIndex)).subi(centerOfMass); + + double D = Nd4j.getBlasWrapper().dot(buf, buf); + + // Check whether we can use this node as a "summary" + if (isLeaf || FastMath.max(boundary.getHh(), boundary.getHw()) / FastMath.sqrt(D) < theta) { + + // Compute and add t-SNE force between point and current node + double Q = 1.0 / (1.0 + D); + double mult = cumSize * Q; + sumQ.addAndGet(mult); + mult *= Q; + negativeForce.addi(buf.mul(mult)); + + } else { + + // Recursively apply Barnes-Hut to children + northWest.computeNonEdgeForces(pointIndex, theta, negativeForce, sumQ); + northEast.computeNonEdgeForces(pointIndex, theta, negativeForce, sumQ); + southWest.computeNonEdgeForces(pointIndex, theta, negativeForce, sumQ); + southEast.computeNonEdgeForces(pointIndex, theta, negativeForce, sumQ); + } + } + + + + /** + * + * @param rowP a vector + * @param colP + * @param valP + * @param N + * @param posF + */ + public void computeEdgeForces(INDArray rowP, INDArray colP, INDArray valP, int N, INDArray posF) { + if (!rowP.isVector()) + throw new IllegalArgumentException("RowP must be a vector"); + + // Loop over all edges in the graph + double D; + for (int n = 0; n < N; n++) { + for (int i = rowP.getInt(n); i < rowP.getInt(n + 1); i++) { + + // Compute pairwise distance and Q-value + buf.assign(data.slice(n)).subi(data.slice(colP.getInt(i))); + + D = Nd4j.getBlasWrapper().dot(buf, buf); + D = valP.getDouble(i) / D; + + // Sum positive force + posF.slice(n).addi(buf.mul(D)); + + } + } + } + + + /** + * The depth of the node + * @return the depth of the node + */ + public int depth() { + if (isLeaf()) + return 1; + return 1 + max(max(northWest.depth(), northEast.depth()), max(southWest.depth(), southEast.depth())); + } + + public INDArray getCenterOfMass() { + return centerOfMass; + } + + public void setCenterOfMass(INDArray centerOfMass) { + this.centerOfMass = centerOfMass; + } + + public QuadTree getParent() { + return parent; + } + + public void setParent(QuadTree parent) { + this.parent = parent; + } + + public QuadTree getNorthWest() { + return northWest; + } + + public void setNorthWest(QuadTree northWest) { + this.northWest = northWest; + } + + public QuadTree getNorthEast() { + return northEast; + } + + public void setNorthEast(QuadTree northEast) { + this.northEast = northEast; + } + + public QuadTree getSouthWest() { + return southWest; + } + + public void setSouthWest(QuadTree southWest) { + this.southWest = southWest; + } + + public QuadTree getSouthEast() { + return southEast; + } + + public void setSouthEast(QuadTree southEast) { + this.southEast = southEast; + } + + public boolean isLeaf() { + return isLeaf; + } + + public void setLeaf(boolean isLeaf) { + this.isLeaf = isLeaf; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public int getCumSize() { + return cumSize; + } + + public void setCumSize(int cumSize) { + this.cumSize = cumSize; + } + + public Cell getBoundary() { + return boundary; + } + + public void setBoundary(Cell boundary) { + this.boundary = boundary; + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPForest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPForest.java new file mode 100644 index 000000000..a5966af22 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPForest.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.randomprojection; + +import lombok.Data; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.primitives.Pair; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +@Data +public class RPForest { + + private int numTrees; + private List trees; + private INDArray data; + private int maxSize = 1000; + private String similarityFunction; + + /** + * Create the rp forest with the specified number of trees + * @param numTrees the number of trees in the forest + * @param maxSize the max size of each tree + * @param similarityFunction the distance function to use + */ + public RPForest(int numTrees,int maxSize,String similarityFunction) { + this.numTrees = numTrees; + this.maxSize = maxSize; + this.similarityFunction = similarityFunction; + trees = new ArrayList<>(numTrees); + + } + + + /** + * Build the trees from the given dataset + * @param x the input dataset (should be a 2d matrix) + */ + public void fit(INDArray x) { + this.data = x; + for(int i = 0; i < numTrees; i++) { + RPTree tree = new RPTree(data.columns(),maxSize,similarityFunction); + tree.buildTree(x); + trees.add(tree); + } + } + + /** + * Get all candidates relative to a specific datapoint. + * @param input + * @return + */ + public INDArray getAllCandidates(INDArray input) { + return RPUtils.getAllCandidates(input,trees,similarityFunction); + } + + /** + * Query results up to length n + * nearest neighbors + * @param toQuery the query item + * @param n the number of nearest neighbors for the given data point + * @return the indices for the nearest neighbors + */ + public INDArray queryAll(INDArray toQuery,int n) { + return RPUtils.queryAll(toQuery,data,trees,n,similarityFunction); + } + + + /** + * Query all with the distances + * sorted by index + * @param query the query vector + * @param numResults the number of results to return + * @return a list of samples + */ + public List> queryWithDistances(INDArray query, int numResults) { + return RPUtils.queryAllWithDistances(query,this.data, trees,numResults,similarityFunction); + } + + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPHyperPlanes.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPHyperPlanes.java new file mode 100644 index 000000000..0d4b856f2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPHyperPlanes.java @@ -0,0 +1,53 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.randomprojection; + +import lombok.Data; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +@Data +public class RPHyperPlanes { + private int dim; + private INDArray wholeHyperPlane; + + public RPHyperPlanes(int dim) { + this.dim = dim; + } + + public INDArray getHyperPlaneAt(int depth) { + if(wholeHyperPlane.isVector()) + return wholeHyperPlane; + return wholeHyperPlane.slice(depth); + } + + + /** + * Add a new random element to the hyper plane. + */ + public void addRandomHyperPlane() { + INDArray newPlane = Nd4j.randn(new int[] {1,dim}); + newPlane.divi(newPlane.normmaxNumber()); + if(wholeHyperPlane == null) + wholeHyperPlane = newPlane; + else { + wholeHyperPlane = Nd4j.concat(0,wholeHyperPlane,newPlane); + } + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPNode.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPNode.java new file mode 100644 index 000000000..faf054569 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPNode.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.randomprojection; + + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; + +@Data +public class RPNode { + private int depth; + private RPNode left,right; + private Future leftFuture,rightFuture; + private List indices; + private double median; + private RPTree tree; + + + public RPNode(RPTree tree,int depth) { + this.depth = depth; + this.tree = tree; + indices = new ArrayList<>(); + } + + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPTree.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPTree.java new file mode 100644 index 000000000..1360a5c92 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPTree.java @@ -0,0 +1,126 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.randomprojection; + +import lombok.Builder; +import lombok.Data; +import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; +import org.nd4j.linalg.api.memory.enums.*; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.primitives.Pair; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; + +@Data +public class RPTree { + private RPNode root; + private RPHyperPlanes rpHyperPlanes; + private int dim; + //also knows as leave size + private int maxSize; + private INDArray X; + private String similarityFunction = "euclidean"; + private WorkspaceConfiguration workspaceConfiguration; + private ExecutorService searchExecutor; + private int searchWorkers; + + /** + * + * @param dim the dimension of the vectors + * @param maxSize the max size of the leaves + * + */ + @Builder + public RPTree(int dim, int maxSize,String similarityFunction) { + this.dim = dim; + this.maxSize = maxSize; + rpHyperPlanes = new RPHyperPlanes(dim); + root = new RPNode(this,0); + this.similarityFunction = similarityFunction; + workspaceConfiguration = WorkspaceConfiguration.builder().cyclesBeforeInitialization(1) + .policyAllocation(AllocationPolicy.STRICT).policyLearning(LearningPolicy.FIRST_LOOP) + .policyMirroring(MirroringPolicy.FULL).policyReset(ResetPolicy.BLOCK_LEFT) + .policySpill(SpillPolicy.REALLOCATE).build(); + + } + + /** + * + * @param dim the dimension of the vectors + * @param maxSize the max size of the leaves + * + */ + public RPTree(int dim, int maxSize) { + this(dim,maxSize,"euclidean"); + } + + /** + * Build the tree with the given input data + * @param x + */ + + public void buildTree(INDArray x) { + this.X = x; + for(int i = 0; i < x.rows(); i++) { + root.getIndices().add(i); + } + + + + RPUtils.buildTree(this,root,rpHyperPlanes, + x,maxSize,0,similarityFunction); + } + + + + public void addNodeAtIndex(int idx,INDArray toAdd) { + RPNode query = RPUtils.query(root,rpHyperPlanes,toAdd,similarityFunction); + query.getIndices().add(idx); + } + + + public List getLeaves() { + List nodes = new ArrayList<>(); + RPUtils.scanForLeaves(nodes,getRoot()); + return nodes; + } + + + /** + * Query all with the distances + * sorted by index + * @param query the query vector + * @param numResults the number of results to return + * @return a list of samples + */ + public List> queryWithDistances(INDArray query, int numResults) { + return RPUtils.queryAllWithDistances(query,X,Arrays.asList(this),numResults,similarityFunction); + } + + public INDArray query(INDArray query,int numResults) { + return RPUtils.queryAll(query,X,Arrays.asList(this),numResults,similarityFunction); + } + + public List getCandidates(INDArray target) { + return RPUtils.getCandidates(target,Arrays.asList(this),similarityFunction); + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPUtils.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPUtils.java new file mode 100644 index 000000000..aecdae476 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/randomprojection/RPUtils.java @@ -0,0 +1,482 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.randomprojection; + +import com.google.common.primitives.Doubles; +import lombok.val; +import org.nd4j.autodiff.functions.DifferentialFunction; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.ReduceOp; +import org.nd4j.linalg.api.ops.impl.reduce3.*; +import org.nd4j.linalg.exception.ND4JIllegalArgumentException; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.util.*; + +/** + * A port of: https://github.com/lyst/rpforest to nd4j + * + * @author + */ +public class RPUtils { + + + private static ThreadLocal> functionInstances = new ThreadLocal<>(); + + public static DifferentialFunction getOp(String name, + INDArray x, + INDArray y, + INDArray result) { + Map ops = functionInstances.get(); + if(ops == null) { + ops = new HashMap<>(); + functionInstances.set(ops); + } + + boolean allDistances = x.length() != y.length(); + + switch(name) { + case "cosinedistance": + if(!ops.containsKey(name) || ((CosineDistance)ops.get(name)).isComplexAccumulation() != allDistances) { + CosineDistance cosineDistance = new CosineDistance(x,y,result,allDistances); + ops.put(name,cosineDistance); + return cosineDistance; + } + else { + CosineDistance cosineDistance = (CosineDistance) ops.get(name); + return cosineDistance; + } + case "cosinesimilarity": + if(!ops.containsKey(name) || ((CosineSimilarity)ops.get(name)).isComplexAccumulation() != allDistances) { + CosineSimilarity cosineSimilarity = new CosineSimilarity(x,y,result,allDistances); + ops.put(name,cosineSimilarity); + return cosineSimilarity; + } + else { + CosineSimilarity cosineSimilarity = (CosineSimilarity) ops.get(name); + cosineSimilarity.setX(x); + cosineSimilarity.setY(y); + cosineSimilarity.setZ(result); + return cosineSimilarity; + + } + case "manhattan": + if(!ops.containsKey(name) || ((ManhattanDistance)ops.get(name)).isComplexAccumulation() != allDistances) { + ManhattanDistance manhattanDistance = new ManhattanDistance(x,y,result,allDistances); + ops.put(name,manhattanDistance); + return manhattanDistance; + } + else { + ManhattanDistance manhattanDistance = (ManhattanDistance) ops.get(name); + manhattanDistance.setX(x); + manhattanDistance.setY(y); + manhattanDistance.setZ(result); + return manhattanDistance; + } + case "jaccard": + if(!ops.containsKey(name) || ((JaccardDistance)ops.get(name)).isComplexAccumulation() != allDistances) { + JaccardDistance jaccardDistance = new JaccardDistance(x,y,result,allDistances); + ops.put(name,jaccardDistance); + return jaccardDistance; + } + else { + JaccardDistance jaccardDistance = (JaccardDistance) ops.get(name); + jaccardDistance.setX(x); + jaccardDistance.setY(y); + jaccardDistance.setZ(result); + return jaccardDistance; + } + case "hamming": + if(!ops.containsKey(name) || ((HammingDistance)ops.get(name)).isComplexAccumulation() != allDistances) { + HammingDistance hammingDistance = new HammingDistance(x,y,result,allDistances); + ops.put(name,hammingDistance); + return hammingDistance; + } + else { + HammingDistance hammingDistance = (HammingDistance) ops.get(name); + hammingDistance.setX(x); + hammingDistance.setY(y); + hammingDistance.setZ(result); + return hammingDistance; + } + //euclidean + default: + if(!ops.containsKey(name) || ((EuclideanDistance)ops.get(name)).isComplexAccumulation() != allDistances) { + EuclideanDistance euclideanDistance = new EuclideanDistance(x,y,result,allDistances); + ops.put(name,euclideanDistance); + return euclideanDistance; + } + else { + EuclideanDistance euclideanDistance = (EuclideanDistance) ops.get(name); + euclideanDistance.setX(x); + euclideanDistance.setY(y); + euclideanDistance.setZ(result); + return euclideanDistance; + } + } + } + + + /** + * Query all trees using the given input and data + * @param toQuery the query vector + * @param X the input data to query + * @param trees the trees to query + * @param n the number of results to search for + * @param similarityFunction the similarity function to use + * @return the indices (in order) in the ndarray + */ + public static List> queryAllWithDistances(INDArray toQuery,INDArray X,List trees,int n,String similarityFunction) { + if(trees.isEmpty()) { + throw new ND4JIllegalArgumentException("Trees is empty!"); + } + + List candidates = getCandidates(toQuery, trees,similarityFunction); + val sortedCandidates = sortCandidates(toQuery,X,candidates,similarityFunction); + int numReturns = Math.min(n,sortedCandidates.size()); + List> ret = new ArrayList<>(numReturns); + for(int i = 0; i < numReturns; i++) { + ret.add(sortedCandidates.get(i)); + } + + return ret; + } + + /** + * Query all trees using the given input and data + * @param toQuery the query vector + * @param X the input data to query + * @param trees the trees to query + * @param n the number of results to search for + * @param similarityFunction the similarity function to use + * @return the indices (in order) in the ndarray + */ + public static INDArray queryAll(INDArray toQuery,INDArray X,List trees,int n,String similarityFunction) { + if(trees.isEmpty()) { + throw new ND4JIllegalArgumentException("Trees is empty!"); + } + + List candidates = getCandidates(toQuery, trees,similarityFunction); + val sortedCandidates = sortCandidates(toQuery,X,candidates,similarityFunction); + int numReturns = Math.min(n,sortedCandidates.size()); + + INDArray result = Nd4j.create(numReturns); + for(int i = 0; i < numReturns; i++) { + result.putScalar(i,sortedCandidates.get(i).getSecond()); + } + + + return result; + } + + /** + * Get the sorted distances given the + * query vector, input data, given the list of possible search candidates + * @param x the query vector + * @param X the input data to use + * @param candidates the possible search candidates + * @param similarityFunction the similarity function to use + * @return the sorted distances + */ + public static List> sortCandidates(INDArray x,INDArray X, + List candidates, + String similarityFunction) { + int prevIdx = -1; + List> ret = new ArrayList<>(); + for(int i = 0; i < candidates.size(); i++) { + if(candidates.get(i) != prevIdx) { + ret.add(Pair.of(computeDistance(similarityFunction,X.slice(candidates.get(i)),x),candidates.get(i))); + } + + prevIdx = i; + } + + + Collections.sort(ret, new Comparator>() { + @Override + public int compare(Pair doubleIntegerPair, Pair t1) { + return Doubles.compare(doubleIntegerPair.getFirst(),t1.getFirst()); + } + }); + + return ret; + } + + + + /** + * Get the search candidates as indices given the input + * and similarity function + * @param x the input data to search with + * @param trees the trees to search + * @param similarityFunction the function to use for similarity + * @return the list of indices as the search results + */ + public static INDArray getAllCandidates(INDArray x,List trees,String similarityFunction) { + List candidates = getCandidates(x,trees,similarityFunction); + Collections.sort(candidates); + + int prevIdx = -1; + int idxCount = 0; + List> scores = new ArrayList<>(); + for(int i = 0; i < candidates.size(); i++) { + if(candidates.get(i) == prevIdx) { + idxCount++; + } + else if(prevIdx != -1) { + scores.add(Pair.of(idxCount,prevIdx)); + idxCount = 1; + } + + prevIdx = i; + } + + + scores.add(Pair.of(idxCount,prevIdx)); + + INDArray arr = Nd4j.create(scores.size()); + for(int i = 0; i < scores.size(); i++) { + arr.putScalar(i,scores.get(i).getSecond()); + } + + return arr; + } + + + /** + * Get the search candidates as indices given the input + * and similarity function + * @param x the input data to search with + * @param roots the trees to search + * @param similarityFunction the function to use for similarity + * @return the list of indices as the search results + */ + public static List getCandidates(INDArray x,List roots,String similarityFunction) { + Set ret = new LinkedHashSet<>(); + for(RPTree tree : roots) { + RPNode root = tree.getRoot(); + RPNode query = query(root,tree.getRpHyperPlanes(),x,similarityFunction); + ret.addAll(query.getIndices()); + } + + return new ArrayList<>(ret); + } + + + /** + * Query the tree starting from the given node + * using the given hyper plane and similarity function + * @param from the node to start from + * @param planes the hyper plane to query + * @param x the input data + * @param similarityFunction the similarity function to use + * @return the leaf node representing the given query from a + * search in the tree + */ + public static RPNode query(RPNode from,RPHyperPlanes planes,INDArray x,String similarityFunction) { + if(from.getLeft() == null && from.getRight() == null) { + return from; + } + + INDArray hyperPlane = planes.getHyperPlaneAt(from.getDepth()); + double dist = computeDistance(similarityFunction,x,hyperPlane); + if(dist <= from.getMedian()) { + return query(from.getLeft(),planes,x,similarityFunction); + } + + else { + return query(from.getRight(),planes,x,similarityFunction); + } + + } + + + /** + * Compute the distance between 2 vectors + * given a function name. Valid function names: + * euclidean: euclidean distance + * cosinedistance: cosine distance + * cosine similarity: cosine similarity + * manhattan: manhattan distance + * jaccard: jaccard distance + * hamming: hamming distance + * @param function the function to use (default euclidean distance) + * @param x the first vector + * @param y the second vector + * @return the distance between the 2 vectors given the inputs + */ + public static INDArray computeDistanceMulti(String function,INDArray x,INDArray y,INDArray result) { + ReduceOp op = (ReduceOp) getOp(function, x, y, result); + op.setDimensions(1); + Nd4j.getExecutioner().exec(op); + return op.z(); + } + + /** + + /** + * Compute the distance between 2 vectors + * given a function name. Valid function names: + * euclidean: euclidean distance + * cosinedistance: cosine distance + * cosine similarity: cosine similarity + * manhattan: manhattan distance + * jaccard: jaccard distance + * hamming: hamming distance + * @param function the function to use (default euclidean distance) + * @param x the first vector + * @param y the second vector + * @return the distance between the 2 vectors given the inputs + */ + public static double computeDistance(String function,INDArray x,INDArray y,INDArray result) { + ReduceOp op = (ReduceOp) getOp(function, x, y, result); + Nd4j.getExecutioner().exec(op); + return op.z().getDouble(0); + } + + /** + * Compute the distance between 2 vectors + * given a function name. Valid function names: + * euclidean: euclidean distance + * cosinedistance: cosine distance + * cosine similarity: cosine similarity + * manhattan: manhattan distance + * jaccard: jaccard distance + * hamming: hamming distance + * @param function the function to use (default euclidean distance) + * @param x the first vector + * @param y the second vector + * @return the distance between the 2 vectors given the inputs + */ + public static double computeDistance(String function,INDArray x,INDArray y) { + return computeDistance(function,x,y,Nd4j.scalar(0.0)); + } + + /** + * Initialize the tree given the input parameters + * @param tree the tree to initialize + * @param from the starting node + * @param planes the hyper planes to use (vector space for similarity) + * @param X the input data + * @param maxSize the max number of indices on a given leaf node + * @param depth the current depth of the tree + * @param similarityFunction the similarity function to use + */ + public static void buildTree(RPTree tree, + RPNode from, + RPHyperPlanes planes, + INDArray X, + int maxSize, + int depth, + String similarityFunction) { + if(from.getIndices().size() <= maxSize) { + //slimNode + slimNode(from); + return; + } + + + List distances = new ArrayList<>(); + RPNode left = new RPNode(tree,depth + 1); + RPNode right = new RPNode(tree,depth + 1); + + if(planes.getWholeHyperPlane() == null || depth >= planes.getWholeHyperPlane().rows()) { + planes.addRandomHyperPlane(); + } + + + INDArray hyperPlane = planes.getHyperPlaneAt(depth); + + + + for(int i = 0; i < from.getIndices().size(); i++) { + double cosineSim = computeDistance(similarityFunction,hyperPlane,X.slice(from.getIndices().get(i))); + distances.add(cosineSim); + } + + Collections.sort(distances); + from.setMedian(distances.get(distances.size() / 2)); + + + for(int i = 0; i < from.getIndices().size(); i++) { + double cosineSim = computeDistance(similarityFunction,hyperPlane,X.slice(from.getIndices().get(i))); + if(cosineSim <= from.getMedian()) { + left.getIndices().add(from.getIndices().get(i)); + } + else { + right.getIndices().add(from.getIndices().get(i)); + } + } + + //failed split + if(left.getIndices().isEmpty() || right.getIndices().isEmpty()) { + slimNode(from); + return; + } + + + from.setLeft(left); + from.setRight(right); + slimNode(from); + + + buildTree(tree,left,planes,X,maxSize,depth + 1,similarityFunction); + buildTree(tree,right,planes,X,maxSize,depth + 1,similarityFunction); + + } + + + /** + * Scan for leaves accumulating + * the nodes in the passed in list + * @param nodes the nodes so far + * @param scan the tree to scan + */ + public static void scanForLeaves(List nodes,RPTree scan) { + scanForLeaves(nodes,scan.getRoot()); + } + + /** + * Scan for leaves accumulating + * the nodes in the passed in list + * @param nodes the nodes so far + */ + public static void scanForLeaves(List nodes,RPNode current) { + if(current.getLeft() == null && current.getRight() == null) + nodes.add(current); + if(current.getLeft() != null) + scanForLeaves(nodes,current.getLeft()); + if(current.getRight() != null) + scanForLeaves(nodes,current.getRight()); + } + + + /** + * Prune indices from the given node + * when it's a leaf + * @param node the node to prune + */ + public static void slimNode(RPNode node) { + if(node.getRight() != null && node.getLeft() != null) { + node.getIndices().clear(); + } + + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/Cell.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/Cell.java new file mode 100644 index 000000000..2781f2ce4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/Cell.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.sptree; + +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.Serializable; + +/** + * @author Adam Gibson + */ +public class Cell implements Serializable { + private int dimension; + private INDArray corner, width; + + public Cell(int dimension) { + this.dimension = dimension; + } + + public double corner(int d) { + return corner.getDouble(d); + } + + public double width(int d) { + return width.getDouble(d); + } + + public void setCorner(int d, double corner) { + this.corner.putScalar(d, corner); + } + + public void setWidth(int d, double width) { + this.width.putScalar(d, width); + } + + public void setWidth(INDArray width) { + this.width = width; + } + + public void setCorner(INDArray corner) { + this.corner = corner; + } + + + public boolean contains(INDArray point) { + INDArray cornerMinusWidth = corner.sub(width); + INDArray cornerPlusWidth = corner.add(width); + for (int d = 0; d < dimension; d++) { + double pointD = point.getDouble(d); + if (cornerMinusWidth.getDouble(d) > pointD) + return false; + if (cornerPlusWidth.getDouble(d) < pointD) + return false; + } + return true; + + } + + public INDArray width() { + return width; + } + + public INDArray corner() { + return corner; + } + + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/DataPoint.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/DataPoint.java new file mode 100644 index 000000000..ae9902de0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/DataPoint.java @@ -0,0 +1,96 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.sptree; + +import lombok.Data; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.reduce3.CosineSimilarity; +import org.nd4j.linalg.api.ops.impl.reduce3.EuclideanDistance; +import org.nd4j.linalg.api.ops.impl.reduce3.ManhattanDistance; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.Serializable; + +/** + * + * A vector with an index and function for distance + * @author Adam Gibson + */ +@Data +public class DataPoint implements Serializable { + private int index; + private INDArray point; + private long d; + private String functionName; + private boolean invert = false; + + + public DataPoint(int index, INDArray point, boolean invert) { + this(index, point, "euclidean"); + this.invert = invert; + } + + public DataPoint(int index, INDArray point, String functionName, boolean invert) { + this.index = index; + this.point = point; + this.functionName = functionName; + this.d = point.length(); + this.invert = invert; + } + + + public DataPoint(int index, INDArray point) { + this(index, point, false); + } + + public DataPoint(int index, INDArray point, String functionName) { + this(index, point, functionName, false); + } + + /** + * Euclidean distance + * @param point the distance from this point to the given point + * @return the distance between the two points + */ + public float distance(DataPoint point) { + switch (functionName) { + case "euclidean": + float ret = Nd4j.getExecutioner().execAndReturn(new EuclideanDistance(this.point, point.point)) + .getFinalResult().floatValue(); + return invert ? -ret : ret; + + case "cosinesimilarity": + float ret2 = Nd4j.getExecutioner().execAndReturn(new CosineSimilarity(this.point, point.point)) + .getFinalResult().floatValue(); + return invert ? -ret2 : ret2; + + case "manhattan": + float ret3 = Nd4j.getExecutioner().execAndReturn(new ManhattanDistance(this.point, point.point)) + .getFinalResult().floatValue(); + return invert ? -ret3 : ret3; + case "dot": + float dotRet = (float) Nd4j.getBlasWrapper().dot(this.point, point.point); + return invert ? -dotRet : dotRet; + default: + float ret4 = Nd4j.getExecutioner().execAndReturn(new EuclideanDistance(this.point, point.point)) + .getFinalResult().floatValue(); + return invert ? -ret4 : ret4; + + } + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/HeapItem.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/HeapItem.java new file mode 100644 index 000000000..6b2f55481 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/HeapItem.java @@ -0,0 +1,79 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.sptree; + +import java.io.Serializable; + +/** + * @author Adam Gibson + */ +public class HeapItem implements Serializable, Comparable { + private int index; + private double distance; + + + public HeapItem(int index, double distance) { + this.index = index; + this.distance = distance; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + public double getDistance() { + return distance; + } + + public void setDistance(double distance) { + this.distance = distance; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + HeapItem heapItem = (HeapItem) o; + + if (index != heapItem.index) + return false; + return Double.compare(heapItem.distance, distance) == 0; + + } + + @Override + public int hashCode() { + int result; + long temp; + result = index; + temp = Double.doubleToLongBits(distance); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public int compareTo(HeapItem o) { + return distance < o.distance ? 1 : 0; + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/HeapObject.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/HeapObject.java new file mode 100644 index 000000000..e154a75f1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/HeapObject.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.sptree; + +import lombok.Data; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.Serializable; + +/** + * @author raver119@gmail.com + */ +@Data +public class HeapObject implements Serializable, Comparable { + private int index; + private INDArray point; + private double distance; + + + public HeapObject(int index, INDArray point, double distance) { + this.index = index; + this.point = point; + this.distance = distance; + } + + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + HeapObject heapObject = (HeapObject) o; + + if (!point.equals(heapObject.point)) + return false; + + return Double.compare(heapObject.distance, distance) == 0; + + } + + @Override + public int hashCode() { + int result; + long temp; + result = index; + temp = Double.doubleToLongBits(distance); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public int compareTo(HeapObject o) { + return distance < o.distance ? 1 : 0; + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/SpTree.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/SpTree.java new file mode 100644 index 000000000..1ef6dcaf6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/sptree/SpTree.java @@ -0,0 +1,421 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.sptree; + +import com.google.common.util.concurrent.AtomicDouble; +import lombok.val; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.nn.conf.WorkspaceMode; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.custom.BarnesEdgeForces; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.api.memory.abstracts.DummyWorkspace; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; + + +/** + * @author Adam Gibson + */ +public class SpTree implements Serializable { + + + public final static String workspaceExternal = "SPTREE_LOOP_EXTERNAL"; + + + private int D; + private INDArray data; + public final static int NODE_RATIO = 8000; + private int N; + private int size; + private int cumSize; + private Cell boundary; + private INDArray centerOfMass; + private SpTree parent; + private int[] index; + private int nodeCapacity; + private int numChildren = 2; + private boolean isLeaf = true; + private Collection indices; + private SpTree[] children; + private static Logger log = LoggerFactory.getLogger(SpTree.class); + private String similarityFunction = Distance.EUCLIDEAN.toString(); + + + + public SpTree(SpTree parent, INDArray data, INDArray corner, INDArray width, Collection indices, + String similarityFunction) { + init(parent, data, corner, width, indices, similarityFunction); + } + + + public SpTree(INDArray data, Collection indices, String similarityFunction) { + this.indices = indices; + this.N = data.rows(); + this.D = data.columns(); + this.similarityFunction = similarityFunction; + data = data.dup(); + INDArray meanY = data.mean(0); + INDArray minY = data.min(0); + INDArray maxY = data.max(0); + INDArray width = Nd4j.create(data.dataType(), meanY.shape()); + for (int i = 0; i < width.length(); i++) { + width.putScalar(i, Math.max(maxY.getDouble(i) - meanY.getDouble(i), + meanY.getDouble(i) - minY.getDouble(i)) + Nd4j.EPS_THRESHOLD); + } + + try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) { + init(null, data, meanY, width, indices, similarityFunction); + fill(N); + } + } + + + public SpTree(SpTree parent, INDArray data, INDArray corner, INDArray width, Collection indices) { + this(parent, data, corner, width, indices, "euclidean"); + } + + + public SpTree(INDArray data, Collection indices) { + this(data, indices, "euclidean"); + } + + + + public SpTree(INDArray data) { + this(data, new ArrayList()); + } + + public MemoryWorkspace workspace() { + return null; + } + + private void init(SpTree parent, INDArray data, INDArray corner, INDArray width, Collection indices, + String similarityFunction) { + + this.parent = parent; + D = data.columns(); + N = data.rows(); + this.similarityFunction = similarityFunction; + nodeCapacity = N % NODE_RATIO; + index = new int[nodeCapacity]; + for (int d = 1; d < this.D; d++) + numChildren *= 2; + this.indices = indices; + isLeaf = true; + size = 0; + cumSize = 0; + children = new SpTree[numChildren]; + this.data = data; + boundary = new Cell(D); + boundary.setCorner(corner.dup()); + boundary.setWidth(width.dup()); + centerOfMass = Nd4j.create(data.dataType(), D); + } + + + + private boolean insert(int index) { + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + INDArray point = data.slice(index); + /*boolean contains = false; + SpTreeCell op = new SpTreeCell(boundary.corner(), boundary.width(), point, N, contains); + Nd4j.getExecutioner().exec(op); + op.getOutputArgument(0).getScalar(0); + if (!contains) return false;*/ + if (!boundary.contains(point)) + return false; + + + cumSize++; + double mult1 = (double) (cumSize - 1) / (double) cumSize; + double mult2 = 1.0 / (double) cumSize; + centerOfMass.muli(mult1); + centerOfMass.addi(point.mul(mult2)); + // If there is space in this quad tree and it is a leaf, add the object here + if (isLeaf() && size < nodeCapacity) { + this.index[size] = index; + indices.add(point); + size++; + return true; + } + + + for (int i = 0; i < size; i++) { + INDArray compPoint = data.slice(this.index[i]); + if (compPoint.equals(point)) + return true; + } + + + if (isLeaf()) + subDivide(); + + + // Find out where the point can be inserted + for (int i = 0; i < numChildren; i++) { + if (children[i].insert(index)) + return true; + } + + throw new IllegalStateException("Shouldn't reach this state"); + } + } + + + /** + * Subdivide the node in to + * 4 children + */ + public void subDivide() { + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + try (MemoryWorkspace ws = workspace.notifyScopeEntered()) */{ + + INDArray newCorner = Nd4j.create(data.dataType(), D); + INDArray newWidth = Nd4j.create(data.dataType(), D); + for (int i = 0; i < numChildren; i++) { + int div = 1; + for (int d = 0; d < D; d++) { + newWidth.putScalar(d, .5 * boundary.width(d)); + if ((i / div) % 2 == 1) + newCorner.putScalar(d, boundary.corner(d) - .5 * boundary.width(d)); + else + newCorner.putScalar(d, boundary.corner(d) + .5 * boundary.width(d)); + div *= 2; + } + + children[i] = new SpTree(this, data, newCorner, newWidth, indices); + + } + + // Move existing points to correct children + for (int i = 0; i < size; i++) { + boolean success = false; + for (int j = 0; j < this.numChildren; j++) + if (!success) + success = children[j].insert(index[i]); + + index[i] = -1; + } + + // Empty parent node + size = 0; + isLeaf = false; + } + } + + + + /** + * Compute non edge forces using barnes hut + * @param pointIndex + * @param theta + * @param negativeForce + * @param sumQ + */ + public void computeNonEdgeForces(int pointIndex, double theta, INDArray negativeForce, AtomicDouble sumQ) { + // Make sure that we spend no time on empty nodes or self-interactions + INDArray buf = Nd4j.create(data.dataType(), this.D); + + if (cumSize == 0 || (isLeaf() && size == 1 && index[0] == pointIndex)) + return; + /* MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + // Compute distance between point and center-of-mass + data.slice(pointIndex).subi(centerOfMass, buf); + + double D = Nd4j.getBlasWrapper().dot(buf, buf); + // Check whether we can use this node as a "summary" + double maxWidth = boundary.width().maxNumber().doubleValue(); + // Check whether we can use this node as a "summary" + if (isLeaf() || maxWidth / Math.sqrt(D) < theta) { + + // Compute and add t-SNE force between point and current node + double Q = 1.0 / (1.0 + D); + double mult = cumSize * Q; + sumQ.addAndGet(mult); + mult *= Q; + negativeForce.addi(buf.mul(mult)); + } else { + + // Recursively apply Barnes-Hut to children + for (int i = 0; i < numChildren; i++) { + children[i].computeNonEdgeForces(pointIndex, theta, negativeForce, sumQ); + } + + } + } + } + + + /** + * + * Compute edge forces using barnes hut + * @param rowP a vector + * @param colP + * @param valP + * @param N the number of elements + * @param posF the positive force + */ + public void computeEdgeForces(INDArray rowP, INDArray colP, INDArray valP, int N, INDArray posF) { + if (!rowP.isVector()) + throw new IllegalArgumentException("RowP must be a vector"); + + // Loop over all edges in the graph + // just execute native op + Nd4j.exec(new BarnesEdgeForces(rowP, colP, valP, data, N, posF)); + + /* + INDArray buf = Nd4j.create(data.dataType(), this.D); + double D; + for (int n = 0; n < N; n++) { + INDArray slice = data.slice(n); + for (int i = rowP.getInt(n); i < rowP.getInt(n + 1); i++) { + + // Compute pairwise distance and Q-value + slice.subi(data.slice(colP.getInt(i)), buf); + + D = 1.0 + Nd4j.getBlasWrapper().dot(buf, buf); + D = valP.getDouble(i) / D; + + // Sum positive force + posF.slice(n).addi(buf.muli(D)); + } + } + */ + } + + + + public boolean isLeaf() { + return isLeaf; + } + + /** + * Verifies the structure of the tree (does bounds checking on each node) + * @return true if the structure of the tree + * is correct. + */ + public boolean isCorrect() { + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + for (int n = 0; n < size; n++) { + INDArray point = data.slice(index[n]); + if (!boundary.contains(point)) + return false; + } + if (!isLeaf()) { + boolean correct = true; + for (int i = 0; i < numChildren; i++) + correct = correct && children[i].isCorrect(); + return correct; + } + + return true; + } + } + + /** + * The depth of the node + * @return the depth of the node + */ + public int depth() { + if (isLeaf()) + return 1; + int depth = 1; + int maxChildDepth = 0; + for (int i = 0; i < numChildren; i++) { + maxChildDepth = Math.max(maxChildDepth, children[0].depth()); + } + + return depth + maxChildDepth; + } + + private void fill(int n) { + if (indices.isEmpty() && parent == null) + for (int i = 0; i < n; i++) { + log.trace("Inserted " + i); + insert(i); + } + else + log.warn("Called fill already"); + } + + + public SpTree[] getChildren() { + return children; + } + + public int getD() { + return D; + } + + public INDArray getCenterOfMass() { + return centerOfMass; + } + + public Cell getBoundary() { + return boundary; + } + + public int[] getIndex() { + return index; + } + + public int getCumSize() { + return cumSize; + } + + public void setCumSize(int cumSize) { + this.cumSize = cumSize; + } + + public int getNumChildren() { + return numChildren; + } + + public void setNumChildren(int numChildren) { + this.numChildren = numChildren; + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/BaseClusteringStrategy.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/BaseClusteringStrategy.java new file mode 100644 index 000000000..42592335b --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/BaseClusteringStrategy.java @@ -0,0 +1,113 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.strategy; + +import lombok.*; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.condition.ClusteringAlgorithmCondition; +import org.deeplearning4j.clustering.condition.ConvergenceCondition; +import org.deeplearning4j.clustering.condition.FixedIterationCountCondition; + +import java.io.Serializable; + +@AllArgsConstructor(access = AccessLevel.PROTECTED) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public abstract class BaseClusteringStrategy implements ClusteringStrategy, Serializable { + @Getter(AccessLevel.PUBLIC) + @Setter(AccessLevel.PROTECTED) + protected ClusteringStrategyType type; + @Getter(AccessLevel.PUBLIC) + @Setter(AccessLevel.PROTECTED) + protected Integer initialClusterCount; + @Getter(AccessLevel.PUBLIC) + @Setter(AccessLevel.PROTECTED) + protected ClusteringAlgorithmCondition optimizationPhaseCondition; + @Getter(AccessLevel.PUBLIC) + @Setter(AccessLevel.PROTECTED) + protected ClusteringAlgorithmCondition terminationCondition; + @Getter(AccessLevel.PUBLIC) + @Setter(AccessLevel.PROTECTED) + protected boolean inverse; + @Getter(AccessLevel.PUBLIC) + @Setter(AccessLevel.PROTECTED) + protected Distance distanceFunction; + @Getter(AccessLevel.PUBLIC) + @Setter(AccessLevel.PROTECTED) + protected boolean allowEmptyClusters; + + public BaseClusteringStrategy(ClusteringStrategyType type, Integer initialClusterCount, Distance distanceFunction, + boolean allowEmptyClusters, boolean inverse) { + this.type = type; + this.initialClusterCount = initialClusterCount; + this.distanceFunction = distanceFunction; + this.allowEmptyClusters = allowEmptyClusters; + this.inverse = inverse; + } + + public BaseClusteringStrategy(ClusteringStrategyType clusteringStrategyType, int initialClusterCount, + Distance distanceFunction, boolean inverse) { + this(clusteringStrategyType, initialClusterCount, distanceFunction, false, inverse); + } + + + /** + * + * @param maxIterationCount + * @return + */ + public BaseClusteringStrategy endWhenIterationCountEquals(int maxIterationCount) { + setTerminationCondition(FixedIterationCountCondition.iterationCountGreaterThan(maxIterationCount)); + return this; + } + + /** + * + * @param rate + * @return + */ + public BaseClusteringStrategy endWhenDistributionVariationRateLessThan(double rate) { + setTerminationCondition(ConvergenceCondition.distributionVariationRateLessThan(rate)); + return this; + } + + /** + * @return + */ + @Override + public boolean inverseDistanceCalculation() { + return inverse; + } + + /** + * + * @param type + * @return + */ + public boolean isStrategyOfType(ClusteringStrategyType type) { + return type.equals(this.type); + } + + /** + * + * @return + */ + public Integer getInitialClusterCount() { + return initialClusterCount; + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/ClusteringStrategy.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/ClusteringStrategy.java new file mode 100644 index 000000000..2ac66ef14 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/ClusteringStrategy.java @@ -0,0 +1,98 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.strategy; + +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.condition.ClusteringAlgorithmCondition; +import org.deeplearning4j.clustering.iteration.IterationHistory; + +/** + * + */ +public interface ClusteringStrategy { + + /** + * + * @return + */ + boolean inverseDistanceCalculation(); + + /** + * + * @return + */ + ClusteringStrategyType getType(); + + /** + * + * @param type + * @return + */ + boolean isStrategyOfType(ClusteringStrategyType type); + + /** + * + * @return + */ + Integer getInitialClusterCount(); + + /** + * + * @return + */ + Distance getDistanceFunction(); + + /** + * + * @return + */ + boolean isAllowEmptyClusters(); + + /** + * + * @return + */ + ClusteringAlgorithmCondition getTerminationCondition(); + + /** + * + * @return + */ + boolean isOptimizationDefined(); + + /** + * + * @param iterationHistory + * @return + */ + boolean isOptimizationApplicableNow(IterationHistory iterationHistory); + + /** + * + * @param maxIterationCount + * @return + */ + BaseClusteringStrategy endWhenIterationCountEquals(int maxIterationCount); + + /** + * + * @param rate + * @return + */ + BaseClusteringStrategy endWhenDistributionVariationRateLessThan(double rate); + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/ClusteringStrategyType.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/ClusteringStrategyType.java new file mode 100644 index 000000000..5d67974cf --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/ClusteringStrategyType.java @@ -0,0 +1,21 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.strategy; + +public enum ClusteringStrategyType { + FIXED_CLUSTER_COUNT, OPTIMIZATION +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/FixedClusterCountStrategy.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/FixedClusterCountStrategy.java new file mode 100644 index 000000000..a6bdd285d --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/FixedClusterCountStrategy.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.strategy; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.iteration.IterationHistory; + +/** + * + */ +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class FixedClusterCountStrategy extends BaseClusteringStrategy { + + + protected FixedClusterCountStrategy(Integer initialClusterCount, Distance distanceFunction, + boolean allowEmptyClusters, boolean inverse) { + super(ClusteringStrategyType.FIXED_CLUSTER_COUNT, initialClusterCount, distanceFunction, allowEmptyClusters, + inverse); + } + + /** + * + * @param clusterCount + * @param distanceFunction + * @param inverse + * @return + */ + public static FixedClusterCountStrategy setup(int clusterCount, Distance distanceFunction, boolean inverse) { + return new FixedClusterCountStrategy(clusterCount, distanceFunction, false, inverse); + } + + /** + * @return + */ + @Override + public boolean inverseDistanceCalculation() { + return inverse; + } + + public boolean isOptimizationDefined() { + return false; + } + + public boolean isOptimizationApplicableNow(IterationHistory iterationHistory) { + return false; + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/OptimisationStrategy.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/OptimisationStrategy.java new file mode 100644 index 000000000..ac697cb2b --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/strategy/OptimisationStrategy.java @@ -0,0 +1,78 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.strategy; + +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.condition.ClusteringAlgorithmCondition; +import org.deeplearning4j.clustering.condition.ConvergenceCondition; +import org.deeplearning4j.clustering.condition.FixedIterationCountCondition; +import org.deeplearning4j.clustering.iteration.IterationHistory; +import org.deeplearning4j.clustering.optimisation.ClusteringOptimization; +import org.deeplearning4j.clustering.optimisation.ClusteringOptimizationType; + +public class OptimisationStrategy extends BaseClusteringStrategy { + public static int defaultIterationCount = 100; + + private ClusteringOptimization clusteringOptimisation; + private ClusteringAlgorithmCondition clusteringOptimisationApplicationCondition; + + protected OptimisationStrategy() { + super(); + } + + protected OptimisationStrategy(int initialClusterCount, Distance distanceFunction) { + super(ClusteringStrategyType.OPTIMIZATION, initialClusterCount, distanceFunction, false); + } + + public static OptimisationStrategy setup(int initialClusterCount, Distance distanceFunction) { + return new OptimisationStrategy(initialClusterCount, distanceFunction); + } + + public OptimisationStrategy optimize(ClusteringOptimizationType type, double value) { + clusteringOptimisation = new ClusteringOptimization(type, value); + return this; + } + + public OptimisationStrategy optimizeWhenIterationCountMultipleOf(int value) { + clusteringOptimisationApplicationCondition = FixedIterationCountCondition.iterationCountGreaterThan(value); + return this; + } + + public OptimisationStrategy optimizeWhenPointDistributionVariationRateLessThan(double rate) { + clusteringOptimisationApplicationCondition = ConvergenceCondition.distributionVariationRateLessThan(rate); + return this; + } + + + public double getClusteringOptimizationValue() { + return clusteringOptimisation.getValue(); + } + + public boolean isClusteringOptimizationType(ClusteringOptimizationType type) { + return clusteringOptimisation != null && clusteringOptimisation.getType().equals(type); + } + + public boolean isOptimizationDefined() { + return clusteringOptimisation != null; + } + + public boolean isOptimizationApplicableNow(IterationHistory iterationHistory) { + return clusteringOptimisationApplicationCondition != null + && clusteringOptimisationApplicationCondition.isSatisfied(iterationHistory); + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/MathUtils.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/MathUtils.java new file mode 100644 index 000000000..0f657569e --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/MathUtils.java @@ -0,0 +1,1329 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.util; + + +import org.apache.commons.math3.linear.CholeskyDecomposition; +import org.apache.commons.math3.linear.NonSquareMatrixException; +import org.apache.commons.math3.linear.RealMatrix; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.util.FastMath; +import org.nd4j.common.primitives.Counter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.Set; + + +/** + * This is a math utils class. + * + * @author Adam Gibson + * + */ +public class MathUtils { + + /** The natural logarithm of 2. */ + public static double log2 = Math.log(2); + + /** + * Normalize a value + * (val - min) / (max - min) + * @param val value to normalize + * @param max max value + * @param min min value + * @return the normalized value + */ + public static double normalize(double val, double min, double max) { + if (max < min) + throw new IllegalArgumentException("Max must be greater than min"); + + return (val - min) / (max - min); + } + + /** + * Clamps the value to a discrete value + * @param value the value to clamp + * @param min min for the probability distribution + * @param max max for the probability distribution + * @return the discrete value + */ + public static int clamp(int value, int min, int max) { + if (value < min) + value = min; + if (value > max) + value = max; + return value; + } + + /** + * Discretize the given value + * @param value the value to discretize + * @param min the min of the distribution + * @param max the max of the distribution + * @param binCount the number of bins + * @return the discretized value + */ + public static int discretize(double value, double min, double max, int binCount) { + int discreteValue = (int) (binCount * normalize(value, min, max)); + return clamp(discreteValue, 0, binCount - 1); + } + + + /** + * See: https://stackoverflow.com/questions/466204/rounding-off-to-nearest-power-of-2 + * @param v the number to getFromOrigin the next power of 2 for + * @return the next power of 2 for the passed in value + */ + public static long nextPowOf2(long v) { + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v++; + return v; + + } + + + + /** + * Generates a binomial distributed number using + * the given rng + * @param rng + * @param n + * @param p + * @return + */ + public static int binomial(RandomGenerator rng, int n, double p) { + if ((p < 0) || (p > 1)) { + return 0; + } + int c = 0; + for (int i = 0; i < n; i++) { + if (rng.nextDouble() < p) { + c++; + } + } + return c; + } + + /** + * Generate a uniform random number from the given rng + * @param rng the rng to use + * @param min the min num + * @param max the max num + * @return a number uniformly distributed between min and max + */ + public static double uniform(Random rng, double min, double max) { + return rng.nextDouble() * (max - min) + min; + } + + /** + * Returns the correlation coefficient of two double vectors. + * + * @param residuals residuals + * @param targetAttribute target attribute vector + * + * @return the correlation coefficient or r + */ + public static double correlation(double[] residuals, double targetAttribute[]) { + double[] predictedValues = new double[residuals.length]; + for (int i = 0; i < predictedValues.length; i++) { + predictedValues[i] = targetAttribute[i] - residuals[i]; + } + double ssErr = ssError(predictedValues, targetAttribute); + double total = ssTotal(residuals, targetAttribute); + return 1 - (ssErr / total); + }//end correlation + + /** + * 1 / 1 + exp(-x) + * @param x + * @return + */ + public static double sigmoid(double x) { + return 1.0 / (1.0 + FastMath.exp(-x)); + } + + + /** + * How much of the variance is explained by the regression + * @param residuals error + * @param targetAttribute data for target attribute + * @return the sum squares of regression + */ + public static double ssReg(double[] residuals, double[] targetAttribute) { + double mean = sum(targetAttribute) / targetAttribute.length; + double ret = 0; + for (int i = 0; i < residuals.length; i++) { + ret += Math.pow(residuals[i] - mean, 2); + } + return ret; + } + + /** + * How much of the variance is NOT explained by the regression + * @param predictedValues predicted values + * @param targetAttribute data for target attribute + * @return the sum squares of regression + */ + public static double ssError(double[] predictedValues, double[] targetAttribute) { + double ret = 0; + for (int i = 0; i < predictedValues.length; i++) { + ret += Math.pow(targetAttribute[i] - predictedValues[i], 2); + } + return ret; + + } + + + /** + * Calculate string similarity with tfidf weights relative to each character + * frequency and how many times a character appears in a given string + * @param strings the strings to calculate similarity for + * @return the cosine similarity between the strings + */ + public static double stringSimilarity(String... strings) { + if (strings == null) + return 0; + Counter counter = new Counter<>(); + Counter counter2 = new Counter<>(); + + for (int i = 0; i < strings[0].length(); i++) + counter.incrementCount(String.valueOf(strings[0].charAt(i)), 1.0f); + + for (int i = 0; i < strings[1].length(); i++) + counter2.incrementCount(String.valueOf(strings[1].charAt(i)), 1.0f); + Set v1 = counter.keySet(); + Set v2 = counter2.keySet(); + + + Set both = SetUtils.intersection(v1, v2); + + double sclar = 0, norm1 = 0, norm2 = 0; + for (String k : both) + sclar += counter.getCount(k) * counter2.getCount(k); + for (String k : v1) + norm1 += counter.getCount(k) * counter.getCount(k); + for (String k : v2) + norm2 += counter2.getCount(k) * counter2.getCount(k); + return sclar / Math.sqrt(norm1 * norm2); + } + + /** + * Returns the vector length (sqrt(sum(x_i)) + * @param vector the vector to return the vector length for + * @return the vector length of the passed in array + */ + public static double vectorLength(double[] vector) { + double ret = 0; + if (vector == null) + return ret; + else { + for (int i = 0; i < vector.length; i++) { + ret += Math.pow(vector[i], 2); + } + + } + return ret; + } + + /** + * Inverse document frequency: the total docs divided by the number of times the word + * appeared in a document + * @param totalDocs the total documents for the data applyTransformToDestination + * @param numTimesWordAppearedInADocument the number of times the word occurred in a document + * @return log(10) (totalDocs/numTImesWordAppearedInADocument) + */ + public static double idf(double totalDocs, double numTimesWordAppearedInADocument) { + //return totalDocs > 0 ? Math.log10(totalDocs/numTimesWordAppearedInADocument) : 0; + if (totalDocs == 0) + return 0; + double idf = Math.log10(totalDocs / numTimesWordAppearedInADocument); + return idf; + } + + /** + * Term frequency: 1+ log10(count) + * @param count the count of a word or character in a given string or document + * @return 1+ log(10) count + */ + public static double tf(int count, int documentLength) { + //return count > 0 ? 1 + Math.log10(count) : 0 + double tf = ((double) count / documentLength); + return tf; + } + + /** + * Return td * idf + * @param tf the term frequency (assumed calculated) + * @param idf inverse document frequency (assumed calculated) + * @return td * idf + */ + public static double tfidf(double tf, double idf) { + // System.out.println("TF-IDF Value: " + (tf * idf)); + return tf * idf; + } + + private static int charForLetter(char c) { + char[] chars = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + for (int i = 0; i < chars.length; i++) + if (chars[i] == c) + return i; + return -1; + + } + + + + /** + * Total variance in target attribute + * @param residuals error + * @param targetAttribute data for target attribute + * @return Total variance in target attribute + */ + public static double ssTotal(double[] residuals, double[] targetAttribute) { + return ssReg(residuals, targetAttribute) + ssError(residuals, targetAttribute); + } + + /** + * This returns the sum of the given array. + * @param nums the array of numbers to sum + * @return the sum of the given array + */ + public static double sum(double[] nums) { + + double ret = 0; + for (double d : nums) + ret += d; + + return ret; + }//end sum + + /** + * This will merge the coordinates of the given coordinate system. + * @param x the x coordinates + * @param y the y coordinates + * @return a vector such that each (x,y) pair is at ret[i],ret[i+1] + */ + public static double[] mergeCoords(double[] x, double[] y) { + if (x.length != y.length) + throw new IllegalArgumentException( + "Sample sizes must be the same for each data applyTransformToDestination."); + double[] ret = new double[x.length + y.length]; + + for (int i = 0; i < x.length; i++) { + ret[i] = x[i]; + ret[i + 1] = y[i]; + } + return ret; + }//end mergeCoords + + /** + * This will merge the coordinates of the given coordinate system. + * @param x the x coordinates + * @param y the y coordinates + * @return a vector such that each (x,y) pair is at ret[i],ret[i+1] + */ + public static List mergeCoords(List x, List y) { + if (x.size() != y.size()) + throw new IllegalArgumentException( + "Sample sizes must be the same for each data applyTransformToDestination."); + + List ret = new ArrayList<>(); + + for (int i = 0; i < x.size(); i++) { + ret.add(x.get(i)); + ret.add(y.get(i)); + } + return ret; + }//end mergeCoords + + /** + * This returns the minimized loss values for a given vector. + * It is assumed that the x, y pairs are at + * vector[i], vector[i+1] + * @param vector the vector of numbers to getFromOrigin the weights for + * @return a double array with w_0 and w_1 are the associated indices. + */ + public static double[] weightsFor(List vector) { + /* split coordinate system */ + List coords = coordSplit(vector); + /* x vals */ + double[] x = coords.get(0); + /* y vals */ + double[] y = coords.get(1); + + + double meanX = sum(x) / x.length; + double meanY = sum(y) / y.length; + + double sumOfMeanDifferences = sumOfMeanDifferences(x, y); + double xDifferenceOfMean = sumOfMeanDifferencesOnePoint(x); + + double w_1 = sumOfMeanDifferences / xDifferenceOfMean; + + double w_0 = meanY - (w_1) * meanX; + + //double w_1=(n*sumOfProducts(x,y) - sum(x) * sum(y))/(n*sumOfSquares(x) - Math.pow(sum(x),2)); + + // double w_0=(sum(y) - (w_1 * sum(x)))/n; + + double[] ret = new double[vector.size()]; + ret[0] = w_0; + ret[1] = w_1; + + return ret; + }//end weightsFor + + /** + * This will return the squared loss of the given + * points + * @param x the x coordinates to use + * @param y the y coordinates to use + * @param w_0 the first weight + * + * @param w_1 the second weight + * @return the squared loss of the given points + */ + public static double squaredLoss(double[] x, double[] y, double w_0, double w_1) { + double sum = 0; + for (int j = 0; j < x.length; j++) { + sum += Math.pow((y[j] - (w_1 * x[j] + w_0)), 2); + } + return sum; + }//end squaredLoss + + + public static double w_1(double[] x, double[] y, int n) { + return (n * sumOfProducts(x, y) - sum(x) * sum(y)) / (n * sumOfSquares(x) - Math.pow(sum(x), 2)); + } + + public static double w_0(double[] x, double[] y, int n) { + double weight1 = w_1(x, y, n); + + return (sum(y) - (weight1 * sum(x))) / n; + } + + /** + * This returns the minimized loss values for a given vector. + * It is assumed that the x, y pairs are at + * vector[i], vector[i+1] + * @param vector the vector of numbers to getFromOrigin the weights for + * @return a double array with w_0 and w_1 are the associated indices. + */ + public static double[] weightsFor(double[] vector) { + + /* split coordinate system */ + List coords = coordSplit(vector); + /* x vals */ + double[] x = coords.get(0); + /* y vals */ + double[] y = coords.get(1); + + + double meanX = sum(x) / x.length; + double meanY = sum(y) / y.length; + + double sumOfMeanDifferences = sumOfMeanDifferences(x, y); + double xDifferenceOfMean = sumOfMeanDifferencesOnePoint(x); + + double w_1 = sumOfMeanDifferences / xDifferenceOfMean; + + double w_0 = meanY - (w_1) * meanX; + + + + double[] ret = new double[vector.length]; + ret[0] = w_0; + ret[1] = w_1; + + return ret; + }//end weightsFor + + public static double errorFor(double actual, double prediction) { + return actual - prediction; + } + + /** + * Used for calculating top part of simple regression for + * beta 1 + * @param vector the x coordinates + * @param vector2 the y coordinates + * @return the sum of mean differences for the input vectors + */ + public static double sumOfMeanDifferences(double[] vector, double[] vector2) { + double mean = sum(vector) / vector.length; + double mean2 = sum(vector2) / vector2.length; + double ret = 0; + for (int i = 0; i < vector.length; i++) { + double vec1Diff = vector[i] - mean; + double vec2Diff = vector2[i] - mean2; + ret += vec1Diff * vec2Diff; + } + return ret; + }//end sumOfMeanDifferences + + /** + * Used for calculating top part of simple regression for + * beta 1 + * @param vector the x coordinates + * @return the sum of mean differences for the input vectors + */ + public static double sumOfMeanDifferencesOnePoint(double[] vector) { + double mean = sum(vector) / vector.length; + double ret = 0; + for (int i = 0; i < vector.length; i++) { + double vec1Diff = Math.pow(vector[i] - mean, 2); + ret += vec1Diff; + } + return ret; + }//end sumOfMeanDifferences + + public static double variance(double[] vector) { + return sumOfMeanDifferencesOnePoint(vector) / vector.length; + } + + /** + * This returns the product of all numbers in the given array. + * @param nums the numbers to multiply over + * @return the product of all numbers in the array, or 0 + * if the length is or nums i null + */ + public static double times(double[] nums) { + if (nums == null || nums.length == 0) + return 0; + double ret = 1; + for (int i = 0; i < nums.length; i++) + ret *= nums[i]; + return ret; + }//end times + + + /** + * This returns the sum of products for the given + * numbers. + * @param nums the sum of products for the give numbers + * @return the sum of products for the given numbers + */ + public static double sumOfProducts(double[]... nums) { + if (nums == null || nums.length < 1) + return 0; + double sum = 0; + + for (int i = 0; i < nums.length; i++) { + /* The ith column for all of the rows */ + double[] column = column(i, nums); + sum += times(column); + + } + return sum; + }//end sumOfProducts + + + /** + * This returns the given column over an n arrays + * @param column the column to getFromOrigin values for + * @param nums the arrays to extract values from + * @return a double array containing all of the numbers in that column + * for all of the arrays. + * @throws IllegalArgumentException if the index is < 0 + */ + private static double[] column(int column, double[]... nums) throws IllegalArgumentException { + + double[] ret = new double[nums.length]; + + for (int i = 0; i < nums.length; i++) { + double[] curr = nums[i]; + ret[i] = curr[column]; + } + return ret; + }//end column + + /** + * This returns the coordinate split in a list of coordinates + * such that the values for ret[0] are the x values + * and ret[1] are the y values + * @param vector the vector to split with x and y values/ + * @return a coordinate split for the given vector of values. + * if null, is passed in null is returned + */ + public static List coordSplit(double[] vector) { + + if (vector == null) + return null; + List ret = new ArrayList<>(); + /* x coordinates */ + double[] xVals = new double[vector.length / 2]; + /* y coordinates */ + double[] yVals = new double[vector.length / 2]; + /* current points */ + int xTracker = 0; + int yTracker = 0; + for (int i = 0; i < vector.length; i++) { + //even value, x coordinate + if (i % 2 == 0) + xVals[xTracker++] = vector[i]; + //y coordinate + else + yVals[yTracker++] = vector[i]; + } + ret.add(xVals); + ret.add(yVals); + + return ret; + }//end coordSplit + + + /** + * This returns the coordinate split in a list of coordinates + * such that the values for ret[0] are the x values + * and ret[1] are the y values + * @param vector the vector to split with x and y values + * Note that the list will be more stable due to the size operator. + * The array version will have extraneous values if not monitored + * properly. + * @return a coordinate split for the given vector of values. + * if null, is passed in null is returned + */ + public static List coordSplit(List vector) { + + if (vector == null) + return null; + List ret = new ArrayList<>(); + /* x coordinates */ + double[] xVals = new double[vector.size() / 2]; + /* y coordinates */ + double[] yVals = new double[vector.size() / 2]; + /* current points */ + int xTracker = 0; + int yTracker = 0; + for (int i = 0; i < vector.size(); i++) { + //even value, x coordinate + if (i % 2 == 0) + xVals[xTracker++] = vector.get(i); + //y coordinate + else + yVals[yTracker++] = vector.get(i); + } + ret.add(xVals); + ret.add(yVals); + + return ret; + }//end coordSplit + + + + /** + * This returns the x values of the given vector. + * These are assumed to be the even values of the vector. + * @param vector the vector to getFromOrigin the values for + * @return the x values of the given vector + */ + public static double[] xVals(double[] vector) { + + + if (vector == null) + return null; + double[] x = new double[vector.length / 2]; + int count = 0; + for (int i = 0; i < vector.length; i++) { + if (i % 2 != 0) + x[count++] = vector[i]; + } + return x; + }//end xVals + + /** + * This returns the odd indexed values for the given vector + * @param vector the odd indexed values of rht egiven vector + * @return the y values of the given vector + */ + public static double[] yVals(double[] vector) { + double[] y = new double[vector.length / 2]; + int count = 0; + for (int i = 0; i < vector.length; i++) { + if (i % 2 == 0) + y[count++] = vector[i]; + } + return y; + }//end yVals + + + /** + * This returns the sum of squares for the given vector. + * + * @param vector the vector to obtain the sum of squares for + * @return the sum of squares for this vector + */ + public static double sumOfSquares(double[] vector) { + double ret = 0; + for (double d : vector) + ret += Math.pow(d, 2); + return ret; + } + + /** + * This returns the determination coefficient of two vectors given a length + * @param y1 the first vector + * @param y2 the second vector + * @param n the length of both vectors + * @return the determination coefficient or r^2 + */ + public static double determinationCoefficient(double[] y1, double[] y2, int n) { + return Math.pow(correlation(y1, y2), 2); + } + + + + /** + * Returns the logarithm of a for base 2. + * + * @param a a double + * @return the logarithm for base 2 + */ + public static double log2(double a) { + if (a == 0) + return 0.0; + return Math.log(a) / log2; + } + + /** + * This returns the slope of the given points. + * @param x1 the first x to use + * @param x2 the end x to use + * @param y1 the begin y to use + * @param y2 the end y to use + * @return the slope of the given points + */ + public double slope(double x1, double x2, double y1, double y2) { + return (y2 - y1) / (x2 - x1); + }//end slope + + /** + * This returns the root mean squared error of two data sets + * @param real the real values + * @param predicted the predicted values + * @return the root means squared error for two data sets + */ + public static double rootMeansSquaredError(double[] real, double[] predicted) { + double ret = 0.0; + for (int i = 0; i < real.length; i++) { + ret += Math.pow((real[i] - predicted[i]), 2); + } + return Math.sqrt(ret / real.length); + }//end rootMeansSquaredError + + /** + * This returns the entropy (information gain, or uncertainty of a random variable). + * @param vector the vector of values to getFromOrigin the entropy for + * @return the entropy of the given vector + */ + public static double entropy(double[] vector) { + if (vector == null || vector.length < 1) + return 0; + else { + double ret = 0; + for (double d : vector) + ret += d * Math.log(d); + return ret; + + } + }//end entropy + + /** + * This returns the kronecker delta of two doubles. + * @param i the first number to compare + * @param j the second number to compare + * @return 1 if they are equal, 0 otherwise + */ + public static int kroneckerDelta(double i, double j) { + return (i == j) ? 1 : 0; + } + + /** + * This calculates the adjusted r^2 including degrees of freedom. + * Also known as calculating "strength" of a regression + * @param rSquared the r squared value to calculate + * @param numRegressors number of variables + * @param numDataPoints size of the data applyTransformToDestination + * @return an adjusted r^2 for degrees of freedom + */ + public static double adjustedrSquared(double rSquared, int numRegressors, int numDataPoints) { + double divide = (numDataPoints - 1.0) / (numDataPoints - numRegressors - 1.0); + double rSquaredDiff = 1 - rSquared; + return 1 - (rSquaredDiff * divide); + } + + + public static double[] normalizeToOne(double[] doubles) { + normalize(doubles, sum(doubles)); + return doubles; + } + + public static double min(double[] doubles) { + double ret = doubles[0]; + for (double d : doubles) + if (d < ret) + ret = d; + return ret; + } + + public static double max(double[] doubles) { + double ret = doubles[0]; + for (double d : doubles) + if (d > ret) + ret = d; + return ret; + } + + /** + * Normalizes the doubles in the array using the given value. + * + * @param doubles the array of double + * @param sum the value by which the doubles are to be normalized + * @exception IllegalArgumentException if sum is zero or NaN + */ + public static void normalize(double[] doubles, double sum) { + + if (Double.isNaN(sum)) { + throw new IllegalArgumentException("Can't normalize array. Sum is NaN."); + } + if (sum == 0) { + // Maybe this should just be a return. + throw new IllegalArgumentException("Can't normalize array. Sum is zero."); + } + for (int i = 0; i < doubles.length; i++) { + doubles[i] /= sum; + } + }//end normalize + + /** + * Converts an array containing the natural logarithms of + * probabilities stored in a vector back into probabilities. + * The probabilities are assumed to sum to one. + * + * @param a an array holding the natural logarithms of the probabilities + * @return the converted array + */ + public static double[] logs2probs(double[] a) { + + double max = a[maxIndex(a)]; + double sum = 0.0; + + double[] result = new double[a.length]; + for (int i = 0; i < a.length; i++) { + result[i] = Math.exp(a[i] - max); + sum += result[i]; + } + + normalize(result, sum); + + return result; + }//end logs2probs + + /** + * This returns the entropy for a given vector of probabilities. + * @param probabilities the probabilities to getFromOrigin the entropy for + * @return the entropy of the given probabilities. + */ + public static double information(double[] probabilities) { + double total = 0.0; + for (double d : probabilities) { + total += (-1.0 * log2(d) * d); + } + return total; + }//end information + + /** + * + * + * Returns index of maximum element in a given + * array of doubles. First maximum is returned. + * + * @param doubles the array of doubles + * @return the index of the maximum element + */ + public static /*@pure@*/ int maxIndex(double[] doubles) { + + double maximum = 0; + int maxIndex = 0; + + for (int i = 0; i < doubles.length; i++) { + if ((i == 0) || (doubles[i] > maximum)) { + maxIndex = i; + maximum = doubles[i]; + } + } + + return maxIndex; + }//end maxIndex + + /** + * This will return the factorial of the given number n. + * @param n the number to getFromOrigin the factorial for + * @return the factorial for this number + */ + public static double factorial(double n) { + if (n == 1 || n == 0) + return 1; + for (double i = n; i > 0; i--, n *= (i > 0 ? i : 1)) { + } + return n; + }//end factorial + + + + /** The small deviation allowed in double comparisons. */ + public static double SMALL = 1e-6; + + /** + * Returns the log-odds for a given probability. + * + * @param prob the probability + * + * @return the log-odds after the probability has been mapped to + * [Utils.SMALL, 1-Utils.SMALL] + */ + public static /*@pure@*/ double probToLogOdds(double prob) { + + if (gr(prob, 1) || (sm(prob, 0))) { + throw new IllegalArgumentException("probToLogOdds: probability must " + "be in [0,1] " + prob); + } + double p = SMALL + (1.0 - 2 * SMALL) * prob; + return Math.log(p / (1 - p)); + } + + /** + * Rounds a double to the next nearest integer value. The JDK version + * of it doesn't work properly. + * + * @param value the double value + * @return the resulting integer value + */ + public static /*@pure@*/ int round(double value) { + + return value > 0 ? (int) (value + 0.5) : -(int) (Math.abs(value) + 0.5); + }//end round + + /** + * This returns the permutation of n choose r. + * @param n the n to choose + * @param r the number of elements to choose + * @return the permutation of these numbers + */ + public static double permutation(double n, double r) { + double nFac = MathUtils.factorial(n); + double nMinusRFac = MathUtils.factorial((n - r)); + return nFac / nMinusRFac; + }//end permutation + + + /** + * This returns the combination of n choose r + * @param n the number of elements overall + * @param r the number of elements to choose + * @return the amount of possible combinations for this applyTransformToDestination of elements + */ + public static double combination(double n, double r) { + double nFac = MathUtils.factorial(n); + double rFac = MathUtils.factorial(r); + double nMinusRFac = MathUtils.factorial((n - r)); + + return nFac / (rFac * nMinusRFac); + }//end combination + + + /** + * sqrt(a^2 + b^2) without under/overflow. + */ + public static double hypotenuse(double a, double b) { + double r; + if (Math.abs(a) > Math.abs(b)) { + r = b / a; + r = Math.abs(a) * Math.sqrt(1 + r * r); + } else if (b != 0) { + r = a / b; + r = Math.abs(b) * Math.sqrt(1 + r * r); + } else { + r = 0.0; + } + return r; + }//end hypotenuse + + /** + * Rounds a double to the next nearest integer value in a probabilistic + * fashion (e.g. 0.8 has a 20% chance of being rounded down to 0 and a + * 80% chance of being rounded up to 1). In the limit, the average of + * the rounded numbers generated by this procedure should converge to + * the original double. + * + * @param value the double value + * @param rand the random number generator + * @return the resulting integer value + */ + public static int probRound(double value, Random rand) { + + if (value >= 0) { + double lower = Math.floor(value); + double prob = value - lower; + if (rand.nextDouble() < prob) { + return (int) lower + 1; + } else { + return (int) lower; + } + } else { + double lower = Math.floor(Math.abs(value)); + double prob = Math.abs(value) - lower; + if (rand.nextDouble() < prob) { + return -((int) lower + 1); + } else { + return -(int) lower; + } + } + }//end probRound + + /** + * Rounds a double to the given number of decimal places. + * + * @param value the double value + * @param afterDecimalPoint the number of digits after the decimal point + * @return the double rounded to the given precision + */ + public static /*@pure@*/ double roundDouble(double value, int afterDecimalPoint) { + + double mask = Math.pow(10.0, (double) afterDecimalPoint); + + return (double) (Math.round(value * mask)) / mask; + }//end roundDouble + + + + /** + * Rounds a double to the given number of decimal places. + * + * @param value the double value + * @param afterDecimalPoint the number of digits after the decimal point + * @return the double rounded to the given precision + */ + public static /*@pure@*/ float roundFloat(float value, int afterDecimalPoint) { + + float mask = (float) Math.pow(10, (float) afterDecimalPoint); + + return (float) (Math.round(value * mask)) / mask; + }//end roundDouble + + /** + * This will return the bernoulli trial for the given event. + * A bernoulli trial is a mechanism for detecting the probability + * of a given event occurring k times in n independent trials + * @param n the number of trials + * @param k the number of times the target event occurs + * @param successProb the probability of the event happening + * @return the probability of the given event occurring k times. + */ + public static double bernoullis(double n, double k, double successProb) { + + double combo = MathUtils.combination(n, k); + double q = 1 - successProb; + return combo * Math.pow(successProb, k) * Math.pow(q, n - k); + }//end bernoullis + + /** + * Tests if a is smaller than b. + * + * @param a a double + * @param b a double + */ + public static /*@pure@*/ boolean sm(double a, double b) { + + return (b - a > SMALL); + } + + /** + * Tests if a is greater than b. + * + * @param a a double + * @param b a double + */ + public static /*@pure@*/ boolean gr(double a, double b) { + + return (a - b > SMALL); + } + + /** + * This will take a given string and separator and convert it to an equivalent + * double array. + * @param data the data to separate + * @param separator the separator to use + * @return the new double array based on the given data + */ + public static double[] fromString(String data, String separator) { + String[] split = data.split(separator); + double[] ret = new double[split.length]; + for (int i = 0; i < split.length; i++) { + ret[i] = Double.parseDouble(split[i]); + } + return ret; + }//end fromString + + /** + * Computes the mean for an array of doubles. + * + * @param vector the array + * @return the mean + */ + public static /*@pure@*/ double mean(double[] vector) { + + double sum = 0; + + if (vector.length == 0) { + return 0; + } + for (int i = 0; i < vector.length; i++) { + sum += vector[i]; + } + return sum / (double) vector.length; + }//end mean + + /** + * This will return the cholesky decomposition of + * the given matrix + * @param m the matrix to convert + * @return the cholesky decomposition of the given + * matrix. + * See: + * http://en.wikipedia.org/wiki/Cholesky_decomposition + * @throws NonSquareMatrixException + */ + public CholeskyDecomposition choleskyFromMatrix(RealMatrix m) throws Exception { + return new CholeskyDecomposition(m); + }//end choleskyFromMatrix + + + + /** + * This will convert the given binary string to a decimal based + * integer + * @param binary the binary string to convert + * @return an equivalent base 10 number + */ + public static int toDecimal(String binary) { + long num = Long.parseLong(binary); + long rem; + /* Use the remainder method to ensure validity */ + while (num > 0) { + rem = num % 10; + num = num / 10; + if (rem != 0 && rem != 1) { + System.out.println("This is not a binary number."); + System.out.println("Please try once again."); + return -1; + } + } + return Integer.parseInt(binary, 2); + }//end toDecimal + + + /** + * This will translate a vector in to an equivalent integer + * @param vector the vector to translate + * @return a z value such that the value is the interleaved lsd to msd for each + * double in the vector + */ + public static int distanceFinderZValue(double[] vector) { + StringBuilder binaryBuffer = new StringBuilder(); + List binaryReps = new ArrayList<>(vector.length); + for (int i = 0; i < vector.length; i++) { + double d = vector[i]; + int j = (int) d; + String binary = Integer.toBinaryString(j); + binaryReps.add(binary); + } + //append from left to right, the least to the most significant bit + //till all strings are empty + while (!binaryReps.isEmpty()) { + for (int j = 0; j < binaryReps.size(); j++) { + String curr = binaryReps.get(j); + if (!curr.isEmpty()) { + char first = curr.charAt(0); + binaryBuffer.append(first); + curr = curr.substring(1); + binaryReps.set(j, curr); + } else + binaryReps.remove(j); + } + } + return Integer.parseInt(binaryBuffer.toString(), 2); + + }//end distanceFinderZValue + + /** + * This returns the distance of two vectors + * sum(i=1,n) (q_i - p_i)^2 + * @param p the first vector + * @param q the second vector + * @return the distance between two vectors + */ + public static double euclideanDistance(double[] p, double[] q) { + + double ret = 0; + for (int i = 0; i < p.length; i++) { + double diff = (q[i] - p[i]); + double sq = Math.pow(diff, 2); + ret += sq; + } + return ret; + + }//end euclideanDistance + + /** + * This returns the distance of two vectors + * sum(i=1,n) (q_i - p_i)^2 + * @param p the first vector + * @param q the second vector + * @return the distance between two vectors + */ + public static double euclideanDistance(float[] p, float[] q) { + + double ret = 0; + for (int i = 0; i < p.length; i++) { + double diff = (q[i] - p[i]); + double sq = Math.pow(diff, 2); + ret += sq; + } + return ret; + + }//end euclideanDistance + + /** + * This will generate a series of uniformally distributed + * numbers between l times + * @param l the number of numbers to generate + * @return l uniformally generated numbers + */ + public static double[] generateUniform(int l) { + double[] ret = new double[l]; + Random rgen = new Random(); + for (int i = 0; i < l; i++) { + ret[i] = rgen.nextDouble(); + } + return ret; + }//end generateUniform + + + /** + * This will calculate the Manhattan distance between two sets of points. + * The Manhattan distance is equivalent to: + * 1_sum_n |p_i - q_i| + * @param p the first point vector + * @param q the second point vector + * @return the Manhattan distance between two object + */ + public static double manhattanDistance(double[] p, double[] q) { + + double ret = 0; + for (int i = 0; i < p.length; i++) { + double difference = p[i] - q[i]; + ret += Math.abs(difference); + } + return ret; + }//end manhattanDistance + + + + public static double[] sampleDoublesInInterval(double[][] doubles, int l) { + double[] sample = new double[l]; + for (int i = 0; i < l; i++) { + int rand1 = randomNumberBetween(0, doubles.length - 1); + int rand2 = randomNumberBetween(0, doubles[i].length); + sample[i] = doubles[rand1][rand2]; + } + + return sample; + } + + /** + * Generates a random integer between the specified numbers + * @param begin the begin of the interval + * @param end the end of the interval + * @return an int between begin and end + */ + public static int randomNumberBetween(double begin, double end) { + if (begin > end) + throw new IllegalArgumentException("Begin must not be less than end"); + return (int) begin + (int) (Math.random() * ((end - begin) + 1)); + } + + /** + * Generates a random integer between the specified numbers + * @param begin the begin of the interval + * @param end the end of the interval + * @return an int between begin and end + */ + public static int randomNumberBetween(double begin, double end, RandomGenerator rng) { + if (begin > end) + throw new IllegalArgumentException("Begin must not be less than end"); + return (int) begin + (int) (rng.nextDouble() * ((end - begin) + 1)); + } + + /** + * Generates a random integer between the specified numbers + * @param begin the begin of the interval + * @param end the end of the interval + * @return an int between begin and end + */ + public static int randomNumberBetween(double begin, double end, org.nd4j.linalg.api.rng.Random rng) { + if (begin > end) + throw new IllegalArgumentException("Begin must not be less than end"); + return (int) begin + (int) (rng.nextDouble() * ((end - begin) + 1)); + } + + /** + * + * @param begin + * @param end + * @return + */ + public static float randomFloatBetween(float begin, float end) { + float rand = (float) Math.random(); + return begin + (rand * ((end - begin))); + } + + public static double randomDoubleBetween(double begin, double end) { + return begin + (Math.random() * ((end - begin))); + } + + public static void shuffleArray(int[] array, long rngSeed) { + shuffleArray(array, new Random(rngSeed)); + } + + public static void shuffleArray(int[] array, Random rng) { + //https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm + for (int i = array.length - 1; i > 0; i--) { + int j = rng.nextInt(i + 1); + int temp = array[j]; + array[j] = array[i]; + array[i] = temp; + } + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/MultiThreadUtils.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/MultiThreadUtils.java new file mode 100644 index 000000000..5ca73a1ac --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/MultiThreadUtils.java @@ -0,0 +1,70 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.*; + +public class MultiThreadUtils { + + private static Logger log = LoggerFactory.getLogger(MultiThreadUtils.class); + + private static ExecutorService instance; + + private MultiThreadUtils() {} + + public static synchronized ExecutorService newExecutorService() { + int nThreads = Runtime.getRuntime().availableProcessors(); + return new ThreadPoolExecutor(nThreads, nThreads, 60L, TimeUnit.SECONDS, new LinkedTransferQueue(), + new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; + } + }); + } + + public static void parallelTasks(final List tasks, ExecutorService executorService) { + int tasksCount = tasks.size(); + final CountDownLatch latch = new CountDownLatch(tasksCount); + for (int i = 0; i < tasksCount; i++) { + final int taskIdx = i; + executorService.execute(new Runnable() { + public void run() { + try { + tasks.get(taskIdx).run(); + } catch (Throwable e) { + log.info("Unchecked exception thrown by task", e); + } finally { + latch.countDown(); + } + } + }); + } + + try { + latch.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/SetUtils.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/SetUtils.java new file mode 100644 index 000000000..00f5be11a --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/util/SetUtils.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.util; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +public class SetUtils { + private SetUtils() {} + + // Set specific operations + + public static Set intersection(Collection parentCollection, Collection removeFromCollection) { + Set results = new HashSet<>(parentCollection); + results.retainAll(removeFromCollection); + return results; + } + + public static boolean intersectionP(Set s1, Set s2) { + for (T elt : s1) { + if (s2.contains(elt)) + return true; + } + return false; + } + + public static Set union(Set s1, Set s2) { + Set s3 = new HashSet<>(s1); + s3.addAll(s2); + return s3; + } + + /** Return is s1 \ s2 */ + + public static Set difference(Collection s1, Collection s2) { + Set s3 = new HashSet<>(s1); + s3.removeAll(s2); + return s3; + } +} + + diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTree.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTree.java new file mode 100644 index 000000000..417154cf2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTree.java @@ -0,0 +1,635 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.vptree; + +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.clustering.sptree.DataPoint; +import org.deeplearning4j.clustering.sptree.HeapObject; +import org.deeplearning4j.clustering.util.MathUtils; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; +import org.nd4j.linalg.api.memory.enums.*; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.reduce3.*; +import org.nd4j.linalg.exception.ND4JIllegalStateException; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Vantage point tree implementation + * + * @author Adam Gibson + * @author raver119@gmail.com + */ +@Slf4j +@Builder +@AllArgsConstructor +public class VPTree implements Serializable { + private static final long serialVersionUID = 1L; + + public static final String EUCLIDEAN = "euclidean"; + private double tau; + @Getter + @Setter + private INDArray items; + private List itemsList; + private Node root; + private String similarityFunction; + @Getter + private boolean invert = false; + private transient ExecutorService executorService; + @Getter + private int workers = 1; + private AtomicInteger size = new AtomicInteger(0); + + private transient ThreadLocal scalars = new ThreadLocal<>(); + + private WorkspaceConfiguration workspaceConfiguration; + + protected VPTree() { + // method for serialization only + scalars = new ThreadLocal<>(); + } + + /** + * + * @param points + * @param invert + */ + public VPTree(INDArray points, boolean invert) { + this(points, "euclidean", 1, invert); + } + + /** + * + * @param points + * @param invert + * @param workers number of parallel workers for tree building (increases memory requirements!) + */ + public VPTree(INDArray points, boolean invert, int workers) { + this(points, "euclidean", workers, invert); + } + + /** + * + * @param items the items to use + * @param similarityFunction the similarity function to use + * @param invert whether to invert the distance (similarity functions have different min/max objectives) + */ + public VPTree(INDArray items, String similarityFunction, boolean invert) { + this.similarityFunction = similarityFunction; + this.invert = invert; + this.items = items; + root = buildFromPoints(items); + workers = 1; + } + + /** + * + * @param items the items to use + * @param similarityFunction the similarity function to use + * @param workers number of parallel workers for tree building (increases memory requirements!) + * @param invert whether to invert the metric (different optimization objective) + */ + public VPTree(List items, String similarityFunction, int workers, boolean invert) { + this.workers = workers; + + val list = new INDArray[items.size()]; + + // build list of INDArrays first + for (int i = 0; i < items.size(); i++) + list[i] = items.get(i).getPoint(); + //this.items.putRow(i, items.get(i).getPoint()); + + // just stack them out with concat :) + this.items = Nd4j.pile(list); + + this.invert = invert; + this.similarityFunction = similarityFunction; + root = buildFromPoints(this.items); + } + + + + /** + * + * @param items + * @param similarityFunction + */ + public VPTree(INDArray items, String similarityFunction) { + this(items, similarityFunction, 1, false); + } + + /** + * + * @param items + * @param similarityFunction + * @param workers number of parallel workers for tree building (increases memory requirements!) + * @param invert + */ + public VPTree(INDArray items, String similarityFunction, int workers, boolean invert) { + this.similarityFunction = similarityFunction; + this.invert = invert; + this.items = items; + + this.workers = workers; + root = buildFromPoints(items); + } + + + /** + * + * @param items + * @param similarityFunction + */ + public VPTree(List items, String similarityFunction) { + this(items, similarityFunction, 1, false); + } + + + /** + * + * @param items + */ + public VPTree(INDArray items) { + this(items, EUCLIDEAN); + } + + + /** + * + * @param items + */ + public VPTree(List items) { + this(items, EUCLIDEAN); + } + + /** + * Create an ndarray + * from the datapoints + * @param data + * @return + */ + public static INDArray buildFromData(List data) { + INDArray ret = Nd4j.create(data.size(), data.get(0).getD()); + for (int i = 0; i < ret.slices(); i++) + ret.putSlice(i, data.get(i).getPoint()); + return ret; + } + + + + /** + * + * @param basePoint + * @param distancesArr + */ + public void calcDistancesRelativeTo(INDArray items, INDArray basePoint, INDArray distancesArr) { + switch (similarityFunction) { + case "euclidean": + Nd4j.getExecutioner().exec(new EuclideanDistance(items, basePoint, distancesArr, true,-1)); + break; + case "cosinedistance": + Nd4j.getExecutioner().exec(new CosineDistance(items, basePoint, distancesArr, true, -1)); + break; + case "cosinesimilarity": + Nd4j.getExecutioner().exec(new CosineSimilarity(items, basePoint, distancesArr, true, -1)); + break; + case "manhattan": + Nd4j.getExecutioner().exec(new ManhattanDistance(items, basePoint, distancesArr, true, -1)); + break; + case "dot": + Nd4j.getExecutioner().exec(new Dot(items, basePoint, distancesArr, -1)); + break; + case "jaccard": + Nd4j.getExecutioner().exec(new JaccardDistance(items, basePoint, distancesArr, true, -1)); + break; + case "hamming": + Nd4j.getExecutioner().exec(new HammingDistance(items, basePoint, distancesArr, true, -1)); + break; + default: + Nd4j.getExecutioner().exec(new EuclideanDistance(items, basePoint, distancesArr, true, -1)); + break; + + } + + if (invert) + distancesArr.negi(); + + } + + public void calcDistancesRelativeTo(INDArray basePoint, INDArray distancesArr) { + calcDistancesRelativeTo(items, basePoint, distancesArr); + } + + + /** + * Euclidean distance + * @return the distance between the two points + */ + public double distance(INDArray arr1, INDArray arr2) { + if (scalars == null) + scalars = new ThreadLocal<>(); + + if (scalars.get() == null) + scalars.set(Nd4j.scalar(arr1.dataType(), 0.0)); + + switch (similarityFunction) { + case "jaccard": + double ret7 = Nd4j.getExecutioner() + .execAndReturn(new JaccardDistance(arr1, arr2, scalars.get())) + .getFinalResult().doubleValue(); + return invert ? -ret7 : ret7; + case "hamming": + double ret8 = Nd4j.getExecutioner() + .execAndReturn(new HammingDistance(arr1, arr2, scalars.get())) + .getFinalResult().doubleValue(); + return invert ? -ret8 : ret8; + case "euclidean": + double ret = Nd4j.getExecutioner() + .execAndReturn(new EuclideanDistance(arr1, arr2, scalars.get())) + .getFinalResult().doubleValue(); + return invert ? -ret : ret; + case "cosinesimilarity": + double ret2 = Nd4j.getExecutioner() + .execAndReturn(new CosineSimilarity(arr1, arr2, scalars.get())) + .getFinalResult().doubleValue(); + return invert ? -ret2 : ret2; + case "cosinedistance": + double ret6 = Nd4j.getExecutioner() + .execAndReturn(new CosineDistance(arr1, arr2, scalars.get())) + .getFinalResult().doubleValue(); + return invert ? -ret6 : ret6; + case "manhattan": + double ret3 = Nd4j.getExecutioner() + .execAndReturn(new ManhattanDistance(arr1, arr2, scalars.get())) + .getFinalResult().doubleValue(); + return invert ? -ret3 : ret3; + case "dot": + double dotRet = Nd4j.getBlasWrapper().dot(arr1, arr2); + return invert ? -dotRet : dotRet; + default: + double ret4 = Nd4j.getExecutioner() + .execAndReturn(new EuclideanDistance(arr1, arr2, scalars.get())) + .getFinalResult().doubleValue(); + return invert ? -ret4 : ret4; + + } + } + + protected class NodeBuilder implements Callable { + protected List list; + protected List indices; + + public NodeBuilder(List list, List indices) { + this.list = list; + this.indices = indices; + } + + @Override + public Node call() throws Exception { + return buildFromPoints(list, indices); + } + } + + private Node buildFromPoints(List points, List indices) { + Node ret = new Node(0, 0); + + + // nothing to sort here + if (points.size() == 1) { + ret.point = points.get(0); + ret.index = indices.get(0); + return ret; + } + + // opening workspace, and creating it if that's the first call + /* MemoryWorkspace workspace = + Nd4j.getWorkspaceManager().getAndActivateWorkspace(workspaceConfiguration, "VPTREE_WORSKPACE");*/ + + INDArray items = Nd4j.vstack(points); + int randomPoint = MathUtils.randomNumberBetween(0, items.rows() - 1, Nd4j.getRandom()); + INDArray basePoint = points.get(randomPoint);//items.getRow(randomPoint); + ret.point = basePoint; + ret.index = indices.get(randomPoint); + INDArray distancesArr = Nd4j.create(items.rows(), 1); + + calcDistancesRelativeTo(items, basePoint, distancesArr); + + double medianDistance = distancesArr.medianNumber().doubleValue(); + + ret.threshold = (float) medianDistance; + + List leftPoints = new ArrayList<>(); + List leftIndices = new ArrayList<>(); + List rightPoints = new ArrayList<>(); + List rightIndices = new ArrayList<>(); + + for (int i = 0; i < distancesArr.length(); i++) { + if (i == randomPoint) + continue; + + if (distancesArr.getDouble(i) < medianDistance) { + leftPoints.add(points.get(i)); + leftIndices.add(indices.get(i)); + } else { + rightPoints.add(points.get(i)); + rightIndices.add(indices.get(i)); + } + } + + // closing workspace + //workspace.notifyScopeLeft(); + //log.info("Thread: {}; Workspace size: {} MB; ConstantCache: {}; ShapeCache: {}; TADCache: {}", Thread.currentThread().getId(), (int) (workspace.getCurrentSize() / 1024 / 1024 ), Nd4j.getConstantHandler().getCachedBytes(), Nd4j.getShapeInfoProvider().getCachedBytes(), Nd4j.getExecutioner().getTADManager().getCachedBytes()); + + if (workers > 1) { + if (!leftPoints.isEmpty()) + ret.futureLeft = executorService.submit(new NodeBuilder(leftPoints, leftIndices)); // = buildFromPoints(leftPoints); + + if (!rightPoints.isEmpty()) + ret.futureRight = executorService.submit(new NodeBuilder(rightPoints, rightIndices)); + } else { + if (!leftPoints.isEmpty()) + ret.left = buildFromPoints(leftPoints, leftIndices); + + if (!rightPoints.isEmpty()) + ret.right = buildFromPoints(rightPoints, rightIndices); + } + + return ret; + } + + private Node buildFromPoints(INDArray items) { + if (executorService == null && items == this.items && workers > 1) { + final val deviceId = Nd4j.getAffinityManager().getDeviceForCurrentThread(); + + executorService = Executors.newFixedThreadPool(workers, new ThreadFactory() { + @Override + public Thread newThread(final Runnable r) { + Thread t = new Thread(new Runnable() { + + @Override + public void run() { + Nd4j.getAffinityManager().unsafeSetDevice(deviceId); + r.run(); + } + }); + + t.setDaemon(true); + t.setName("VPTree thread"); + + return t; + } + }); + } + + + final Node ret = new Node(0, 0); + size.incrementAndGet(); + + /*workspaceConfiguration = WorkspaceConfiguration.builder().cyclesBeforeInitialization(1) + .policyAllocation(AllocationPolicy.STRICT).policyLearning(LearningPolicy.FIRST_LOOP) + .policyMirroring(MirroringPolicy.FULL).policyReset(ResetPolicy.BLOCK_LEFT) + .policySpill(SpillPolicy.REALLOCATE).build(); + + // opening workspace + MemoryWorkspace workspace = + Nd4j.getWorkspaceManager().getAndActivateWorkspace(workspaceConfiguration, "VPTREE_WORSKPACE");*/ + + int randomPoint = MathUtils.randomNumberBetween(0, items.rows() - 1, Nd4j.getRandom()); + INDArray basePoint = items.getRow(randomPoint, true); + INDArray distancesArr = Nd4j.create(items.rows(), 1); + ret.point = basePoint; + ret.index = randomPoint; + + calcDistancesRelativeTo(items, basePoint, distancesArr); + + double medianDistance = distancesArr.medianNumber().doubleValue(); + + ret.threshold = (float) medianDistance; + + List leftPoints = new ArrayList<>(); + List leftIndices = new ArrayList<>(); + List rightPoints = new ArrayList<>(); + List rightIndices = new ArrayList<>(); + + for (int i = 0; i < distancesArr.length(); i++) { + if (i == randomPoint) + continue; + + if (distancesArr.getDouble(i) < medianDistance) { + leftPoints.add(items.getRow(i, true)); + leftIndices.add(i); + } else { + rightPoints.add(items.getRow(i, true)); + rightIndices.add(i); + } + } + + // closing workspace + //workspace.notifyScopeLeft(); + //workspace.destroyWorkspace(true); + + if (!leftPoints.isEmpty()) + ret.left = buildFromPoints(leftPoints, leftIndices); + + if (!rightPoints.isEmpty()) + ret.right = buildFromPoints(rightPoints, rightIndices); + + // destroy once again + //workspace.destroyWorkspace(true); + + if (ret.left != null) + ret.left.fetchFutures(); + + if (ret.right != null) + ret.right.fetchFutures(); + + if (executorService != null) + executorService.shutdown(); + + return ret; + } + + public void search(@NonNull INDArray target, int k, List results, List distances) { + search(target, k, results, distances, true); + } + + public void search(@NonNull INDArray target, int k, List results, List distances, + boolean filterEqual) { + search(target, k, results, distances, filterEqual, false); + } + /** + * + * @param target + * @param k + * @param results + * @param distances + */ + public void search(@NonNull INDArray target, int k, List results, List distances, + boolean filterEqual, boolean dropEdge) { + if (items != null) + if (!target.isVectorOrScalar() || target.columns() != items.columns() || target.rows() > 1) + throw new ND4JIllegalStateException("Target for search should have shape of [" + 1 + ", " + + items.columns() + "] but got " + Arrays.toString(target.shape()) + " instead"); + + k = Math.min(k, items.rows()); + results.clear(); + distances.clear(); + + PriorityQueue pq = new PriorityQueue<>(items.rows(), new HeapObjectComparator()); + + search(root, target, k + (filterEqual ? 2 : 1), pq, Double.MAX_VALUE); + + while (!pq.isEmpty()) { + HeapObject ho = pq.peek(); + results.add(new DataPoint(ho.getIndex(), ho.getPoint())); + distances.add(ho.getDistance()); + pq.poll(); + } + + Collections.reverse(results); + Collections.reverse(distances); + + if (dropEdge || results.size() > k) { + if (filterEqual && distances.get(0) == 0.0) { + results.remove(0); + distances.remove(0); + } + + while (results.size() > k) { + results.remove(results.size() - 1); + distances.remove(distances.size() - 1); + } + } + } + + /** + * + * @param node + * @param target + * @param k + * @param pq + */ + public void search(Node node, INDArray target, int k, PriorityQueue pq, double cTau) { + + if (node == null) + return; + + double tau = cTau; + + INDArray get = node.getPoint(); //items.getRow(node.getIndex()); + double distance = distance(get, target); + if (distance < tau) { + if (pq.size() == k) + pq.poll(); + + pq.add(new HeapObject(node.getIndex(), node.getPoint(), distance)); + if (pq.size() == k) + tau = pq.peek().getDistance(); + } + + Node left = node.getLeft(); + Node right = node.getRight(); + + if (left == null && right == null) + return; + + if (distance < node.getThreshold()) { + if (distance - tau < node.getThreshold()) { // if there can still be neighbors inside the ball, recursively search left child first + search(left, target, k, pq, tau); + } + + if (distance + tau >= node.getThreshold()) { // if there can still be neighbors outside the ball, recursively search right child + search(right, target, k, pq, tau); + } + + } else { + if (distance + tau >= node.getThreshold()) { // if there can still be neighbors outside the ball, recursively search right child first + search(right, target, k, pq, tau); + } + + if (distance - tau < node.getThreshold()) { // if there can still be neighbors inside the ball, recursively search left child + search(left, target, k, pq, tau); + } + } + + } + + + protected class HeapObjectComparator implements Comparator { + + @Override + public int compare(HeapObject o1, HeapObject o2) { + return Double.compare(o2.getDistance(), o1.getDistance()); + } + } + + @Data + public static class Node implements Serializable { + private static final long serialVersionUID = 2L; + + private int index; + private float threshold; + private Node left, right; + private INDArray point; + protected transient Future futureLeft; + protected transient Future futureRight; + + public Node(int index, float threshold) { + this.index = index; + this.threshold = threshold; + } + + + public void fetchFutures() { + try { + if (futureLeft != null) { + /*while (!futureLeft.isDone()) + Thread.sleep(100);*/ + + + left = futureLeft.get(); + } + + if (futureRight != null) { + /*while (!futureRight.isDone()) + Thread.sleep(100);*/ + + right = futureRight.get(); + } + + + if (left != null) + left.fetchFutures(); + + if (right != null) + right.fetchFutures(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + + } + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTreeFillSearch.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTreeFillSearch.java new file mode 100644 index 000000000..9dbc75416 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/VPTreeFillSearch.java @@ -0,0 +1,85 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.vptree; + +import lombok.Getter; +import org.deeplearning4j.clustering.sptree.DataPoint; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.List; + +/** + * Brute force search + * for running search + * relative to a target + * but forced to fill the result list + * until the desired k is matched. + * + * The algorithm does this by searching + * nearby points by k in a greedy fashion + */ +public class VPTreeFillSearch { + private VPTree vpTree; + private int k; + @Getter + private List results; + @Getter + private List distances; + private INDArray target; + + public VPTreeFillSearch(VPTree vpTree, int k, INDArray target) { + this.vpTree = vpTree; + this.k = k; + this.target = target; + } + + public void search() { + results = new ArrayList<>(); + distances = new ArrayList<>(); + //initial search + //vpTree.search(target,k,results,distances); + + //fill till there is k results + //by going down the list + // if(results.size() < k) { + INDArray distancesArr = Nd4j.create(vpTree.getItems().rows(), 1); + vpTree.calcDistancesRelativeTo(target, distancesArr); + INDArray[] sortWithIndices = Nd4j.sortWithIndices(distancesArr, 0, !vpTree.isInvert()); + results.clear(); + distances.clear(); + if (vpTree.getItems().isVector()) { + for (int i = 0; i < k; i++) { + int idx = sortWithIndices[0].getInt(i); + results.add(new DataPoint(idx, Nd4j.scalar(vpTree.getItems().getDouble(idx)))); + distances.add(sortWithIndices[1].getDouble(idx)); + } + } else { + for (int i = 0; i < k; i++) { + int idx = sortWithIndices[0].getInt(i); + results.add(new DataPoint(idx, vpTree.getItems().getRow(idx))); + //distances.add(sortWithIndices[1].getDouble(idx)); + distances.add(sortWithIndices[1].getDouble(i)); + } + } + + + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/package-info.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/package-info.java new file mode 100644 index 000000000..487753a00 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/main/java/org/deeplearning4j/clustering/vptree/package-info.java @@ -0,0 +1,22 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +/** + * Created by agibsonccc on 1/3/15. + * Work adapted from: + * https://code.google.com/p/vptree/ + */ +package org.deeplearning4j.clustering.vptree; diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/cluster/ClusterSetTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/cluster/ClusterSetTest.java new file mode 100644 index 000000000..1f0e98194 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/cluster/ClusterSetTest.java @@ -0,0 +1,26 @@ +package org.deeplearning4j.clustering.cluster; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.List; + +public class ClusterSetTest { + @Test + public void testGetMostPopulatedClusters() { + ClusterSet clusterSet = new ClusterSet(false); + List clusters = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Cluster cluster = new Cluster(); + cluster.setPoints(Point.toPoints(Nd4j.randn(i + 1, 5))); + clusters.add(cluster); + } + clusterSet.setClusters(clusters); + List mostPopulatedClusters = clusterSet.getMostPopulatedClusters(5); + for (int i = 0; i < 5; i++) { + Assertions.assertEquals(5 - i, mostPopulatedClusters.get(i).getPoints().size()); + } + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/kdtree/KDTreeTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/kdtree/KDTreeTest.java new file mode 100644 index 000000000..cb6b05d89 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/kdtree/KDTreeTest.java @@ -0,0 +1,422 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.kdtree; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.joda.time.Duration; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; +import com.google.common.base.Stopwatch; +import com.google.common.primitives.Doubles; +import com.google.common.primitives.Floats; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Created by agibsonccc on 1/1/15. + */ +public class KDTreeTest extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return 120000L; + } + + private KDTree kdTree; + + @BeforeAll + public static void beforeClass(){ + Nd4j.setDataType(DataType.FLOAT); + } + + @BeforeEach + public void setUp() { + kdTree = new KDTree(2); + float[] data = new float[]{7,2}; + kdTree.insert(Nd4j.createFromArray(data)); + data = new float[]{5,4}; + kdTree.insert(Nd4j.createFromArray(data)); + data = new float[]{2,3}; + kdTree.insert(Nd4j.createFromArray(data)); + data = new float[]{4,7}; + kdTree.insert(Nd4j.createFromArray(data)); + data = new float[]{9,6}; + kdTree.insert(Nd4j.createFromArray(data)); + data = new float[]{8,1}; + kdTree.insert(Nd4j.createFromArray(data)); + } + + @Test + public void testTree() { + KDTree tree = new KDTree(2); + INDArray half = Nd4j.create(new double[] {0.5, 0.5}, new long[]{1,2}).castTo(DataType.FLOAT); + INDArray one = Nd4j.create(new double[] {1, 1}, new long[]{1,2}).castTo(DataType.FLOAT); + tree.insert(half); + tree.insert(one); + Pair pair = tree.nn(Nd4j.create(new double[] {0.5, 0.5}, new long[]{1,2}).castTo(DataType.FLOAT)); + assertEquals(half, pair.getValue()); + } + + @Test + public void testInsert() { + int elements = 10; + List digits = Arrays.asList(1.0, 0.0, 2.0, 3.0); + + KDTree kdTree = new KDTree(digits.size()); + List> lists = new ArrayList<>(); + for (int i = 0; i < elements; i++) { + List thisList = new ArrayList<>(digits.size()); + for (int k = 0; k < digits.size(); k++) { + thisList.add(digits.get(k) + i); + } + lists.add(thisList); + } + + for (int i = 0; i < elements; i++) { + double[] features = Doubles.toArray(lists.get(i)); + INDArray ind = Nd4j.create(features, new long[]{1, features.length}, DataType.FLOAT); + kdTree.insert(ind); + assertEquals(i + 1, kdTree.size()); + } + } + + @Test + public void testDelete() { + int elements = 10; + List digits = Arrays.asList(1.0, 0.0, 2.0, 3.0); + + KDTree kdTree = new KDTree(digits.size()); + List> lists = new ArrayList<>(); + for (int i = 0; i < elements; i++) { + List thisList = new ArrayList<>(digits.size()); + for (int k = 0; k < digits.size(); k++) { + thisList.add(digits.get(k) + i); + } + lists.add(thisList); + } + + INDArray toDelete = Nd4j.empty(DataType.DOUBLE), + leafToDelete = Nd4j.empty(DataType.DOUBLE); + for (int i = 0; i < elements; i++) { + double[] features = Doubles.toArray(lists.get(i)); + INDArray ind = Nd4j.create(features, new long[]{1, features.length}, DataType.FLOAT); + if (i == 1) + toDelete = ind; + if (i == elements - 1) { + leafToDelete = ind; + } + kdTree.insert(ind); + assertEquals(i + 1, kdTree.size()); + } + + kdTree.delete(toDelete); + assertEquals(9, kdTree.size()); + kdTree.delete(leafToDelete); + assertEquals(8, kdTree.size()); + } + + @Test + public void testNN() { + int n = 10; + + // make a KD-tree of dimension {#n} + KDTree kdTree = new KDTree(n); + for (int i = -1; i < n; i++) { + // Insert a unit vector along each dimension + List vec = new ArrayList<>(n); + // i = -1 ensures the origin is in the Tree + for (int k = 0; k < n; k++) { + vec.add((k == i) ? 1.0 : 0.0); + } + INDArray indVec = Nd4j.create(Doubles.toArray(vec), new long[]{1, vec.size()}, DataType.FLOAT); + kdTree.insert(indVec); + } + Random rand = new Random(); + + // random point in the Hypercube + List pt = new ArrayList(n); + for (int k = 0; k < n; k++) { + pt.add(rand.nextDouble()); + } + Pair result = kdTree.nn(Nd4j.create(Doubles.toArray(pt), new long[]{1, pt.size()}, DataType.FLOAT)); + + // Always true for points in the unitary hypercube + assertTrue(result.getKey() < Double.MAX_VALUE); + + } + + @Test + public void testKNN() { + int dimensions = 512; + int vectorsNo = isIntegrationTests() ? 50000 : 1000; + // make a KD-tree of dimension {#dimensions} + Stopwatch stopwatch = Stopwatch.createStarted(); + KDTree kdTree = new KDTree(dimensions); + for (int i = -1; i < vectorsNo; i++) { + // Insert a unit vector along each dimension + INDArray indVec = Nd4j.rand(DataType.FLOAT, 1,dimensions); + kdTree.insert(indVec); + } + stopwatch.stop(); + System.out.println("Time elapsed for " + kdTree.size() + " nodes construction is "+ stopwatch.elapsed(SECONDS)); + + Random rand = new Random(); + // random point in the Hypercube + List pt = new ArrayList(dimensions); + for (int k = 0; k < dimensions; k++) { + pt.add(rand.nextFloat() * 10.0); + } + stopwatch.reset(); + stopwatch.start(); + List> list = kdTree.knn(Nd4j.create(Nd4j.createBuffer(Floats.toArray(pt))), 20.0f); + stopwatch.stop(); + System.out.println("Time elapsed for Search is "+ stopwatch.elapsed(MILLISECONDS)); + } + + @Test + public void testKNN_Simple() { + int n = 2; + KDTree kdTree = new KDTree(n); + + float[] data = new float[]{3,3}; + kdTree.insert(Nd4j.createFromArray(data)); + data = new float[]{1,1}; + kdTree.insert(Nd4j.createFromArray(data)); + data = new float[]{2,2}; + kdTree.insert(Nd4j.createFromArray(data)); + + data = new float[]{0,0}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 4.5f); + + assertEquals(1.0, result.get(0).getSecond().getDouble(0), 1e-5); + assertEquals(1.0, result.get(0).getSecond().getDouble(1), 1e-5); + + assertEquals(2.0, result.get(1).getSecond().getDouble(0), 1e-5); + assertEquals(2.0, result.get(1).getSecond().getDouble(1), 1e-5); + + assertEquals(3.0, result.get(2).getSecond().getDouble(0), 1e-5); + assertEquals(3.0, result.get(2).getSecond().getDouble(1), 1e-5); + } + + @Test + public void testKNN_1() { + + assertEquals(6, kdTree.size()); + + float[] data = new float[]{8,1}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 10.0f); + assertEquals(8.0, result.get(0).getSecond().getFloat(0), 1e-5); + assertEquals(1.0, result.get(0).getSecond().getFloat(1), 1e-5); + assertEquals(7.0, result.get(1).getSecond().getFloat(0), 1e-5); + assertEquals(2.0, result.get(1).getSecond().getFloat(1), 1e-5); + assertEquals(5.0, result.get(2).getSecond().getFloat(0), 1e-5); + assertEquals(4.0, result.get(2).getSecond().getFloat(1), 1e-5); + assertEquals(9.0, result.get(3).getSecond().getFloat(0), 1e-5); + assertEquals(6.0, result.get(3).getSecond().getFloat(1), 1e-5); + assertEquals(2.0, result.get(4).getSecond().getFloat(0), 1e-5); + assertEquals(3.0, result.get(4).getSecond().getFloat(1), 1e-5); + assertEquals(4.0, result.get(5).getSecond().getFloat(0), 1e-5); + assertEquals(7.0, result.get(5).getSecond().getFloat(1), 1e-5); + } + + @Test + public void testKNN_2() { + float[] data = new float[]{8, 1}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 5.0f); + assertEquals(8.0, result.get(0).getSecond().getFloat(0), 1e-5); + assertEquals(1.0, result.get(0).getSecond().getFloat(1), 1e-5); + assertEquals(7.0, result.get(1).getSecond().getFloat(0), 1e-5); + assertEquals(2.0, result.get(1).getSecond().getFloat(1), 1e-5); + assertEquals(5.0, result.get(2).getSecond().getFloat(0), 1e-5); + assertEquals(4.0, result.get(2).getSecond().getFloat(1), 1e-5); + } + + @Test + public void testKNN_3() { + + float[] data = new float[]{2, 3}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 10.0f); + assertEquals(2.0, result.get(0).getSecond().getFloat(0), 1e-5); + assertEquals(3.0, result.get(0).getSecond().getFloat(1), 1e-5); + assertEquals(5.0, result.get(1).getSecond().getFloat(0), 1e-5); + assertEquals(4.0, result.get(1).getSecond().getFloat(1), 1e-5); + assertEquals(4.0, result.get(2).getSecond().getFloat(0), 1e-5); + assertEquals(7.0, result.get(2).getSecond().getFloat(1), 1e-5); + assertEquals(7.0, result.get(3).getSecond().getFloat(0), 1e-5); + assertEquals(2.0, result.get(3).getSecond().getFloat(1), 1e-5); + assertEquals(8.0, result.get(4).getSecond().getFloat(0), 1e-5); + assertEquals(1.0, result.get(4).getSecond().getFloat(1), 1e-5); + assertEquals(9.0, result.get(5).getSecond().getFloat(0), 1e-5); + assertEquals(6.0, result.get(5).getSecond().getFloat(1), 1e-5); + } + + + @Test + public void testKNN_4() { + float[] data = new float[]{2, 3}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 5.0f); + assertEquals(2.0, result.get(0).getSecond().getFloat(0), 1e-5); + assertEquals(3.0, result.get(0).getSecond().getFloat(1), 1e-5); + assertEquals(5.0, result.get(1).getSecond().getFloat(0), 1e-5); + assertEquals(4.0, result.get(1).getSecond().getFloat(1), 1e-5); + assertEquals(4.0, result.get(2).getSecond().getFloat(0), 1e-5); + assertEquals(7.0, result.get(2).getSecond().getFloat(1), 1e-5); + } + + @Test + public void testKNN_5() { + float[] data = new float[]{2, 3}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 20.0f); + assertEquals(2.0, result.get(0).getSecond().getFloat(0), 1e-5); + assertEquals(3.0, result.get(0).getSecond().getFloat(1), 1e-5); + assertEquals(5.0, result.get(1).getSecond().getFloat(0), 1e-5); + assertEquals(4.0, result.get(1).getSecond().getFloat(1), 1e-5); + assertEquals(4.0, result.get(2).getSecond().getFloat(0), 1e-5); + assertEquals(7.0, result.get(2).getSecond().getFloat(1), 1e-5); + assertEquals(7.0, result.get(3).getSecond().getFloat(0), 1e-5); + assertEquals(2.0, result.get(3).getSecond().getFloat(1), 1e-5); + assertEquals(8.0, result.get(4).getSecond().getFloat(0), 1e-5); + assertEquals(1.0, result.get(4).getSecond().getFloat(1), 1e-5); + assertEquals(9.0, result.get(5).getSecond().getFloat(0), 1e-5); + assertEquals(6.0, result.get(5).getSecond().getFloat(1), 1e-5); + } + + @Test + public void test_KNN_6() { + float[] data = new float[]{4, 6}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 10.0f); + assertEquals(4.0, result.get(0).getSecond().getDouble(0), 1e-5); + assertEquals(7.0, result.get(0).getSecond().getDouble(1), 1e-5); + assertEquals(5.0, result.get(1).getSecond().getDouble(0), 1e-5); + assertEquals(4.0, result.get(1).getSecond().getDouble(1), 1e-5); + assertEquals(2.0, result.get(2).getSecond().getDouble(0), 1e-5); + assertEquals(3.0, result.get(2).getSecond().getDouble(1), 1e-5); + assertEquals(7.0, result.get(3).getSecond().getDouble(0), 1e-5); + assertEquals(2.0, result.get(3).getSecond().getDouble(1), 1e-5); + assertEquals(9.0, result.get(4).getSecond().getDouble(0), 1e-5); + assertEquals(6.0, result.get(4).getSecond().getDouble(1), 1e-5); + assertEquals(8.0, result.get(5).getSecond().getDouble(0), 1e-5); + assertEquals(1.0, result.get(5).getSecond().getDouble(1), 1e-5); + } + + @Test + public void test_KNN_7() { + float[] data = new float[]{4, 6}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 5.0f); + assertEquals(4.0, result.get(0).getSecond().getDouble(0), 1e-5); + assertEquals(7.0, result.get(0).getSecond().getDouble(1), 1e-5); + assertEquals(5.0, result.get(1).getSecond().getDouble(0), 1e-5); + assertEquals(4.0, result.get(1).getSecond().getDouble(1), 1e-5); + assertEquals(2.0, result.get(2).getSecond().getDouble(0), 1e-5); + assertEquals(3.0, result.get(2).getSecond().getDouble(1), 1e-5); + assertEquals(7.0, result.get(3).getSecond().getDouble(0), 1e-5); + assertEquals(2.0, result.get(3).getSecond().getDouble(1), 1e-5); + assertEquals(9.0, result.get(4).getSecond().getDouble(0), 1e-5); + assertEquals(6.0, result.get(4).getSecond().getDouble(1), 1e-5); + } + + @Test + public void test_KNN_8() { + float[] data = new float[]{4, 6}; + List> result = kdTree.knn(Nd4j.createFromArray(data), 20.0f); + assertEquals(4.0, result.get(0).getSecond().getDouble(0), 1e-5); + assertEquals(7.0, result.get(0).getSecond().getDouble(1), 1e-5); + assertEquals(5.0, result.get(1).getSecond().getDouble(0), 1e-5); + assertEquals(4.0, result.get(1).getSecond().getDouble(1), 1e-5); + assertEquals(2.0, result.get(2).getSecond().getDouble(0), 1e-5); + assertEquals(3.0, result.get(2).getSecond().getDouble(1), 1e-5); + assertEquals(7.0, result.get(3).getSecond().getDouble(0), 1e-5); + assertEquals(2.0, result.get(3).getSecond().getDouble(1), 1e-5); + assertEquals(9.0, result.get(4).getSecond().getDouble(0), 1e-5); + assertEquals(6.0, result.get(4).getSecond().getDouble(1), 1e-5); + assertEquals(8.0, result.get(5).getSecond().getDouble(0), 1e-5); + assertEquals(1.0, result.get(5).getSecond().getDouble(1), 1e-5); + } + + @Test + public void testNoDuplicates() { + int N = 100; + KDTree bigTree = new KDTree(2); + + List points = new ArrayList<>(); + for (int i = 0; i < N; ++i) { + double[] data = new double[]{i, i}; + points.add(Nd4j.createFromArray(data)); + } + + for (int i = 0; i < N; ++i) { + bigTree.insert(points.get(i)); + } + + assertEquals(N, bigTree.size()); + + INDArray node = Nd4j.empty(DataType.DOUBLE); + for (int i = 0; i < N; ++i) { + node = bigTree.delete(node.isEmpty() ? points.get(i) : node); + } + + assertEquals(0, bigTree.size()); + } + + ////@Ignore + @Test + @Tag("performance") + public void performanceTest() { + int n = 2; + int num = 100000; + // make a KD-tree of dimension {#n} + long start = System.currentTimeMillis(); + KDTree kdTree = new KDTree(n); + INDArray inputArrray = Nd4j.randn(DataType.DOUBLE, num, n); + for (int i = 0 ; i < num; ++i) { + kdTree.insert(inputArrray.getRow(i)); + } + + long end = System.currentTimeMillis(); + Duration duration = new Duration(start, end); + System.out.println("Elapsed time for tree construction " + duration.getStandardSeconds() + " " + duration.getMillis()); + + List pt = new ArrayList(num); + for (int k = 0; k < n; k++) { + pt.add((float)(num / 2)); + } + start = System.currentTimeMillis(); + List> list = kdTree.knn(Nd4j.create(Nd4j.createBuffer(Doubles.toArray(pt))), 20.0f); + end = System.currentTimeMillis(); + duration = new Duration(start, end); + long elapsed = end - start; + System.out.println("Elapsed time for tree search " + duration.getStandardSeconds() + " " + duration.getMillis()); + for (val pair : list) { + System.out.println(pair.getFirst() + " " + pair.getSecond()) ; + } + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/kmeans/KMeansTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/kmeans/KMeansTest.java new file mode 100644 index 000000000..4b35b9f6a --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/kmeans/KMeansTest.java @@ -0,0 +1,289 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.kmeans; + +import org.apache.commons.lang3.time.StopWatch; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.Performance; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.cluster.*; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Created by agibsonccc on 7/2/17. + */ +public class KMeansTest extends BaseDL4JTest { + + private boolean[] useKMeansPlusPlus = {true, false}; + + @Override + public long getTimeoutMilliseconds() { + return 60000L; + } + + @Test + public void testKMeans() { + Nd4j.getRandom().setSeed(7); + for (boolean mode : useKMeansPlusPlus) { + KMeansClustering kMeansClustering = KMeansClustering.setup(5, 5, Distance.EUCLIDEAN, mode); + List points = Point.toPoints(Nd4j.randn(5, 5)); + ClusterSet clusterSet = kMeansClustering.applyTo(points); + PointClassification pointClassification = clusterSet.classifyPoint(points.get(0)); + System.out.println(pointClassification); + } + } + + @Test + public void testKmeansCosine() { + + Nd4j.getRandom().setSeed(7); + int numClusters = 5; + for (boolean mode : useKMeansPlusPlus) { + KMeansClustering kMeansClustering = KMeansClustering.setup(numClusters, 1000, Distance.COSINE_DISTANCE, mode); + List points = Point.toPoints(Nd4j.rand(5, 300)); + ClusterSet clusterSet = kMeansClustering.applyTo(points); + PointClassification pointClassification = clusterSet.classifyPoint(points.get(0)); + + + KMeansClustering kMeansClusteringEuclidean = KMeansClustering.setup(numClusters, 1000, Distance.EUCLIDEAN, mode); + ClusterSet clusterSetEuclidean = kMeansClusteringEuclidean.applyTo(points); + PointClassification pointClassificationEuclidean = clusterSetEuclidean.classifyPoint(points.get(0)); + System.out.println("Cosine " + pointClassification); + System.out.println("Euclidean " + pointClassificationEuclidean); + + assertEquals(pointClassification.getCluster().getPoints().get(0), + pointClassificationEuclidean.getCluster().getPoints().get(0)); + } + } + + ////@Ignore + @Test + @Performance + public void testPerformanceAllIterations() { + Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); + Nd4j.getRandom().setSeed(7); + int numClusters = 20; + for (boolean mode : useKMeansPlusPlus) { + StopWatch watch = new StopWatch(); + watch.start(); + KMeansClustering kMeansClustering = KMeansClustering.setup(numClusters, 1000, Distance.COSINE_DISTANCE, mode); + List points = Point.toPoints(Nd4j.linspace(0, 5000 * 300, 5000 * 300).reshape(5000, 300)); + + ClusterSet clusterSet = kMeansClustering.applyTo(points); + watch.stop(); + System.out.println("Elapsed for clustering : " + watch); + + watch.reset(); + watch.start(); + for (Point p : points) { + PointClassification pointClassification = clusterSet.classifyPoint(p); + } + watch.stop(); + System.out.println("Elapsed for search: " + watch); + } + } + + @Test + @Performance + public void testPerformanceWithConvergence() { + Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); + Nd4j.getRandom().setSeed(7); + int numClusters = 20; + for (boolean mode : useKMeansPlusPlus) { + StopWatch watch = new StopWatch(); + watch.start(); + KMeansClustering kMeansClustering = KMeansClustering.setup(numClusters, Distance.COSINE_DISTANCE, false, mode); + + List points = Point.toPoints(Nd4j.linspace(0, 10000 * 300, 10000 * 300).reshape(10000, 300)); + + ClusterSet clusterSet = kMeansClustering.applyTo(points); + watch.stop(); + System.out.println("Elapsed for clustering : " + watch); + + watch.reset(); + watch.start(); + for (Point p : points) { + PointClassification pointClassification = clusterSet.classifyPoint(p); + } + watch.stop(); + System.out.println("Elapsed for search: " + watch); + + watch.reset(); + watch.start(); + kMeansClustering = KMeansClustering.setup(numClusters, 0.05, Distance.COSINE_DISTANCE, false, mode); + + points = Point.toPoints(Nd4j.linspace(0, 10000 * 300, 10000 * 300).reshape(10000, 300)); + + clusterSet = kMeansClustering.applyTo(points); + watch.stop(); + System.out.println("Elapsed for clustering : " + watch); + + watch.reset(); + watch.start(); + for (Point p : points) { + PointClassification pointClassification = clusterSet.classifyPoint(p); + } + watch.stop(); + System.out.println("Elapsed for search: " + watch); + } + } + + @Test + public void testCorrectness() { + + /*for (int c = 0; c < 10; ++c)*/ { + Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); + Nd4j.getRandom().setSeed(7); + int numClusters = 3; + for (boolean mode : useKMeansPlusPlus) { + KMeansClustering kMeansClustering = KMeansClustering.setup(numClusters, 1000, Distance.EUCLIDEAN, mode); + double[] data = new double[]{ + 15, 16, + 16, 18.5, + 17, 20.2, + 16.4, 17.12, + 17.23, 18.12, + 43, 43, + 44.43, 45.212, + 45.8, 54.23, + 46.313, 43.123, + 50.21, 46.3, + 99, 99.22, + 100.32, 98.123, + 100.32, 97.423, + 102, 93.23, + 102.23, 94.23 + }; + List points = Point.toPoints(Nd4j.createFromArray(data).reshape(15, 2)); + + ClusterSet clusterSet = kMeansClustering.applyTo(points); + + + INDArray row0 = Nd4j.createFromArray(new double[]{16.6575, 18.4850}); + INDArray row1 = Nd4j.createFromArray(new double[]{32.6050, 31.1500}); + INDArray row2 = Nd4j.createFromArray(new double[]{75.9348, 74.1990}); + + /*List clusters = clusterSet.getClusters(); + assertEquals(row0, clusters.get(0).getCenter().getArray()); + assertEquals(row1, clusters.get(1).getCenter().getArray()); + assertEquals(row2, clusters.get(2).getCenter().getArray());*/ + + PointClassification pointClassification = null; + for (Point p : points) { + pointClassification = clusterSet.classifyPoint(p); + System.out.println("Point: " + p.getArray() + " " + " assigned to cluster: " + pointClassification.getCluster().getCenter().getArray()); + List clusters = clusterSet.getClusters(); + for (int i = 0; i < clusters.size(); ++i) + System.out.println("Choice: " + clusters.get(i).getCenter().getArray()); + } + } + /*assertEquals(Nd4j.createFromArray(new double[]{75.9348, 74.1990}), + pointClassification.getCluster().getCenter().getArray());*/ + + /*clusters = clusterSet.getClusters(); + assertEquals(row0, clusters.get(0).getCenter().getArray()); + assertEquals(row1, clusters.get(1).getCenter().getArray()); + assertEquals(row2, clusters.get(2).getCenter().getArray());*/ + } + } + + @Test + public void testCentersHolder() { + int rows = 3, cols = 2; + CentersHolder ch = new CentersHolder(rows, cols); + + INDArray row0 = Nd4j.createFromArray(new double[]{16.4000, 17.1200}); + INDArray row1 = Nd4j.createFromArray(new double[]{45.8000, 54.2300}); + INDArray row2 = Nd4j.createFromArray(new double[]{95.9348, 94.1990}); + + ch.addCenter(row0); + ch.addCenter(row1); + ch.addCenter(row2); + + double[] data = new double[]{ + 15, 16, + 16, 18.5, + 17, 20.2, + 16.4, 17.12, + 17.23, 18.12, + 43, 43, + 44.43, 45.212, + 45.8, 54.23, + 46.313, 43.123, + 50.21, 46.3, + 99, 99.22, + 100.32, 98.123, + 100.32, 97.423, + 102, 93.23, + 102.23, 94.23 + }; + + INDArray pointData = Nd4j.createFromArray(data); + List points = Point.toPoints(pointData.reshape(15,2)); + + for (int i = 0 ; i < points.size(); ++i) { + INDArray dist = ch.getMinDistances(points.get(i), Distance.EUCLIDEAN); + System.out.println("Point: " + points.get(i).getArray()); + System.out.println("Centers: " + ch.getCenters()); + System.out.println("Distance: " + dist); + System.out.println(); + } + } + + @Test + public void testInitClusters() { + Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); + Nd4j.getRandom().setSeed(7); + { + KMeansClustering kMeansClustering = KMeansClustering.setup(5, 1, Distance.EUCLIDEAN, true); + + double[][] dataArray = {{1000000.0, 2.8E7, 5.5E7, 8.2E7}, {2.8E7, 5.5E7, 8.2E7, 1.09E8}, {5.5E7, 8.2E7, 1.09E8, 1.36E8}, + {8.2E7, 1.09E8, 1.36E8, 1.63E8}, {1.09E8, 1.36E8, 1.63E8, 1.9E8}, {1.36E8, 1.63E8, 1.9E8, 2.17E8}, + {1.63E8, 1.9E8, 2.17E8, 2.44E8}, {1.9E8, 2.17E8, 2.44E8, 2.71E8}, {2.17E8, 2.44E8, 2.71E8, 2.98E8}, + {2.44E8, 2.71E8, 2.98E8, 3.25E8}, {2.71E8, 2.98E8, 3.25E8, 3.52E8}, {2.98E8, 3.25E8, 3.52E8, 3.79E8}, + {3.25E8, 3.52E8, 3.79E8, 4.06E8}, {3.52E8, 3.79E8, 4.06E8, 4.33E8}, {3.79E8, 4.06E8, 4.33E8, 4.6E8}, + {4.06E8, 4.33E8, 4.6E8, 4.87E8}, {4.33E8, 4.6E8, 4.87E8, 5.14E8}, {4.6E8, 4.87E8, 5.14E8, 5.41E8}, + {4.87E8, 5.14E8, 5.41E8, 5.68E8}, {5.14E8, 5.41E8, 5.68E8, 5.95E8}, {5.41E8, 5.68E8, 5.95E8, 6.22E8}, + {5.68E8, 5.95E8, 6.22E8, 6.49E8}, {5.95E8, 6.22E8, 6.49E8, 6.76E8}, {6.22E8, 6.49E8, 6.76E8, 7.03E8}, + {6.49E8, 6.76E8, 7.03E8, 7.3E8}, {6.76E8, 7.03E8, 7.3E8, 7.57E8}, {7.03E8, 7.3E8, 7.57E8, 7.84E8}}; + INDArray data = Nd4j.createFromArray(dataArray); + List points = Point.toPoints(data); + + ClusterSet clusterSet = kMeansClustering.applyTo(points); + + double[] centroid1 = {2.44e8, 2.71e8, 2.98e8, 3.25e8}; + double[] centroid2 = {5.14e8, 5.41e8, 5.68e8, 5.95e8}; + double[] centroid3 = {1000000.0, 2.8E7, 5.5E7, 8.2E7}; + double[] centroid4 = {7.03E8, 7.3E8, 7.57E8, 7.84E8}; + double[] centroid5 = {3.79E8, 4.06E8, 4.33E8, 4.6E8}; + + assertArrayEquals(centroid1, clusterSet.getClusters().get(0).getCenter().getArray().toDoubleVector(), 1e-4); + assertArrayEquals(centroid2, clusterSet.getClusters().get(1).getCenter().getArray().toDoubleVector(), 1e-4); + assertArrayEquals(centroid3, clusterSet.getClusters().get(2).getCenter().getArray().toDoubleVector(), 1e-4); + assertArrayEquals(centroid4, clusterSet.getClusters().get(3).getCenter().getArray().toDoubleVector(), 1e-4); + assertArrayEquals(centroid5, clusterSet.getClusters().get(4).getCenter().getArray().toDoubleVector(), 1e-4); + } + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/lsh/RandomProjectionLSHTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/lsh/RandomProjectionLSHTest.java new file mode 100644 index 000000000..f65589d62 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/lsh/RandomProjectionLSHTest.java @@ -0,0 +1,207 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.lsh; + +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.impl.broadcast.bool.BroadcastEqualTo; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RandomProjectionLSHTest extends BaseDL4JTest { + + int hashLength = 31; + int numTables = 2; + int intDimensions = 13; + + RandomProjectionLSH rpLSH; + INDArray e1; + INDArray inputs; + + @BeforeEach + public void setUp() { + Nd4j.getRandom().setSeed(12345); + Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); + rpLSH = new RandomProjectionLSH(hashLength, numTables, intDimensions, 0.1f); + inputs = Nd4j.rand(DataType.DOUBLE, 100, intDimensions); + e1 = Nd4j.ones(DataType.DOUBLE, 1, intDimensions); + } + + + @AfterEach + public void tearDown() { inputs = null; } + + @Test + public void testEntropyDims(){ + assertArrayEquals(new long[]{numTables, intDimensions}, rpLSH.entropy(e1).shape()); + } + + @Test + public void testHashDims(){ + assertArrayEquals(new long[]{1, hashLength}, rpLSH.hash(e1).shape()); + } + + @Test + public void testHashDimsMultiple(){ + INDArray data = Nd4j.ones(1, intDimensions); + assertArrayEquals(new long[]{1, hashLength}, rpLSH.hash(data).shape()); + + data = Nd4j.ones(100, intDimensions); + assertArrayEquals(new long[]{100, hashLength}, rpLSH.hash(data).shape()); + } + + @Test + public void testSigNums(){ + assertEquals(1.0f, rpLSH.hash(e1).aminNumber().floatValue(),1e-3f); + } + + + @Test + public void testIndexDims(){ + rpLSH.makeIndex(Nd4j.rand(100, intDimensions)); + assertArrayEquals(new long[]{100, hashLength}, rpLSH.index.shape()); + } + + + @Test + public void testGetRawBucketOfDims(){ + rpLSH.makeIndex(inputs); + assertArrayEquals(new long[]{100}, rpLSH.rawBucketOf(e1).shape()); + } + + @Test + public void testRawBucketOfReflexive(){ + rpLSH.makeIndex(inputs); + int idx = (new Random(12345)).nextInt(100); + INDArray row = inputs.getRow(idx, true); + assertEquals(1.0f, rpLSH.rawBucketOf(row).maxNumber().floatValue(), 1e-3f); + } + + @Test + public void testBucketDims(){ + rpLSH.makeIndex(inputs); + assertArrayEquals(new long[]{100}, rpLSH.bucket(e1).shape()); + } + + @Test + public void testBucketReflexive(){ + rpLSH.makeIndex(inputs); + int idx = (new Random(12345)).nextInt(100); + INDArray row = inputs.getRow(idx, true); + assertEquals(1.0f, rpLSH.bucket(row).maxNumber().floatValue(), 1e-3f); + } + + + @Test + public void testBucketDataReflexiveDimensions() { + rpLSH.makeIndex(inputs); + int idx = (new Random(12345)).nextInt(100); + INDArray row = inputs.getRow(idx, true); + INDArray bucketData = rpLSH.bucketData(row); + + assertEquals(intDimensions, bucketData.shape()[1]); + assertTrue(1 <= bucketData.shape()[0]); + } + + @Test + public void testBucketDataReflexive(){ + rpLSH.makeIndex(inputs); + int idx = (new Random(12345)).nextInt(100); + INDArray row = inputs.getRow(idx, true); + INDArray bucketData = rpLSH.bucketData(row); + + INDArray res = Nd4j.zeros(DataType.BOOL, bucketData.shape()); + Nd4j.getExecutioner().exec(new BroadcastEqualTo(bucketData, row, res, -1)); + res = res.castTo(DataType.FLOAT); + + assertEquals( 1.0f, res.min(-1).maxNumber().floatValue(), 1e-3f, + String.format("Expected one bucket content to be the query %s, but found %s", row, rpLSH.bucket(row))); + } + + + @Test + public void testSearchReflexiveDimensions() { + rpLSH.makeIndex(inputs); + int idx = (new Random(12345)).nextInt(100); + INDArray row = inputs.getRow(idx, true); + INDArray searchResults = rpLSH.search(row, 10.0f); + + assertTrue(searchResults.shape()[0] >= 1, + String.format("Expected the search to return at least one result, the query %s but found %s yielding %d results", row, searchResults, searchResults.shape()[0])); + } + + + @Test + public void testSearchReflexive() { + rpLSH.makeIndex(inputs); + int idx = (new Random(12345)).nextInt(100); + INDArray row = inputs.getRow(idx, true); + + INDArray searchResults = rpLSH.search(row, 10.0f); + + + INDArray res = Nd4j.zeros(DataType.BOOL, searchResults.shape()); + Nd4j.getExecutioner().exec(new BroadcastEqualTo(searchResults, row, res, -1)); + res = res.castTo(DataType.FLOAT); + + assertEquals( + + 1.0f, res.min(-1).maxNumber().floatValue(), 1e-3f, + String.format("Expected one search result to be the query %s, but found %s", row, searchResults)); + } + + + + @Test + public void testANNSearchReflexiveDimensions() { + rpLSH.makeIndex(inputs); + int idx = (new Random(12345)).nextInt(100); + INDArray row = inputs.getRow(idx, true); + INDArray searchResults = rpLSH.search(row, 100); + + assertTrue(searchResults.shape()[0] >= 1, + String.format("Expected the search to return at least one result, the query %s but found %s yielding %d results", row, searchResults, searchResults.shape()[0])); + } + + + @Test + public void testANNSearchReflexive() { + rpLSH.makeIndex(inputs); + int idx = (new Random(12345)).nextInt(100); + INDArray row = inputs.getRow(idx).reshape(1, intDimensions); + + INDArray searchResults = rpLSH.search(row, 100); + + + INDArray res = Nd4j.zeros(DataType.BOOL, searchResults.shape()); + Nd4j.getExecutioner().exec(new BroadcastEqualTo(searchResults, row, res, -1)); + res = res.castTo(DataType.FLOAT); + + assertEquals(1.0f, res.min(-1).maxNumber().floatValue(), 1e-3f, + String.format("Expected one search result to be the query %s, but found %s", row, searchResults)); + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/quadtree/QuadTreeTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/quadtree/QuadTreeTest.java new file mode 100644 index 000000000..4bdca3c78 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/quadtree/QuadTreeTest.java @@ -0,0 +1,45 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.quadtree; + +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Created by agibsonccc on 1/2/15. + */ +public class QuadTreeTest extends BaseDL4JTest { + + @Test + public void testQuadTree() { + INDArray n = Nd4j.ones(3, 2); + n.slice(1).addi(1); + n.slice(2).addi(2); + QuadTree quadTree = new QuadTree(n); + assertEquals(n.rows(), quadTree.getCumSize()); + assertTrue(quadTree.isCorrect()); + + + + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/randomprojection/RPTreeTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/randomprojection/RPTreeTest.java new file mode 100644 index 000000000..fa8a0f564 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/randomprojection/RPTreeTest.java @@ -0,0 +1,97 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.randomprojection; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler; +import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class RPTreeTest extends BaseDL4JTest { + + @BeforeEach + public void setUp() { + Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); + } + + + @Test + public void testRPTree() throws Exception { + DataSetIterator mnist = new MnistDataSetIterator(150,150); + RPTree rpTree = new RPTree(784,50); + DataSet d = mnist.next(); + NormalizerStandardize normalizerStandardize = new NormalizerStandardize(); + normalizerStandardize.fit(d); + normalizerStandardize.transform(d.getFeatures()); + INDArray data = d.getFeatures(); + rpTree.buildTree(data); + assertEquals(4,rpTree.getLeaves().size()); + assertEquals(0,rpTree.getRoot().getDepth()); + + List candidates = rpTree.getCandidates(data.getRow(0)); + assertFalse(candidates.isEmpty()); + assertEquals(10,rpTree.query(data.slice(0),10).length()); + System.out.println(candidates.size()); + + rpTree.addNodeAtIndex(150,data.getRow(0)); + + } + + @Test + public void testFindSelf() throws Exception { + DataSetIterator mnist = new MnistDataSetIterator(100, 6000); + NormalizerMinMaxScaler minMaxNormalizer = new NormalizerMinMaxScaler(0, 1); + minMaxNormalizer.fit(mnist); + DataSet d = mnist.next(); + minMaxNormalizer.transform(d.getFeatures()); + RPForest rpForest = new RPForest(100, 100, "euclidean"); + rpForest.fit(d.getFeatures()); + for (int i = 0; i < 10; i++) { + INDArray indexes = rpForest.queryAll(d.getFeatures().slice(i), 10); + assertEquals(i,indexes.getInt(0)); + } + } + + @Test + public void testRpTreeMaxNodes() throws Exception { + DataSetIterator mnist = new MnistDataSetIterator(150,150); + RPForest rpTree = new RPForest(4,4,"euclidean"); + DataSet d = mnist.next(); + NormalizerStandardize normalizerStandardize = new NormalizerStandardize(); + normalizerStandardize.fit(d); + rpTree.fit(d.getFeatures()); + for(RPTree tree : rpTree.getTrees()) { + for(RPNode node : tree.getLeaves()) { + assertTrue(node.getIndices().size() <= rpTree.getMaxSize()); + } + } + + } + + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/randomprojection/RPUtilsTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/randomprojection/RPUtilsTest.java new file mode 100644 index 000000000..6d75aff78 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/randomprojection/RPUtilsTest.java @@ -0,0 +1,41 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.randomprojection; + +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RPUtilsTest extends BaseDL4JTest { + + @Test + public void testDistanceComputeBatch() { + INDArray x = Nd4j.linspace(1,4,4, Nd4j.dataType()).reshape(1, 4); + INDArray y = Nd4j.linspace(1,16,16, Nd4j.dataType()).reshape(4,4); + INDArray result = Nd4j.create(1, 4); + INDArray distances = RPUtils.computeDistanceMulti("euclidean",x,y,result); + INDArray scalarResult = Nd4j.scalar(1.0); + for(int i = 0; i < result.length(); i++) { + double dist = RPUtils.computeDistance("euclidean",x,y.slice(i),scalarResult); + assertEquals(dist,distances.getDouble(i),1e-3); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/sptree/SPTreeTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/sptree/SPTreeTest.java new file mode 100644 index 000000000..17af2afd4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/sptree/SPTreeTest.java @@ -0,0 +1,104 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.sptree; + +import org.apache.commons.lang3.time.StopWatch; +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.buffer.util.DataTypeUtil; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import com.google.common.util.concurrent.AtomicDouble; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Adam Gibson + */ +public class SPTreeTest extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return 120000L; + } + + @BeforeEach + public void setUp() { + DataTypeUtil.setDTypeForContext(DataType.DOUBLE); + } + + @Test + public void testStructure() { + INDArray data = Nd4j.create(new double[][] {{1, 2, 3}, {4, 5, 6}}); + SpTree tree = new SpTree(data); + /*try (MemoryWorkspace ws = tree.workspace().notifyScopeEntered())*/ { + assertEquals(Nd4j.create(new double[]{2.5f, 3.5f, 4.5f}), tree.getCenterOfMass()); + assertEquals(2, tree.getCumSize()); + assertEquals(8, tree.getNumChildren()); + assertTrue(tree.isCorrect()); + } + } + + @Test + public void testComputeEdgeForces() { + Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); + double[] aData = new double[]{ + 0.2999816948164936, 0.26252049735806526, 0.2673853427498767, 0.8604464129156685, 0.4802652829902563, 0.10959096539488711, 0.7950242948008909, 0.5917848948003486, + 0.2738285999345498, 0.9519684328285567, 0.9690024759209738, 0.8585615547624705, 0.8087760944312002, 0.5337951589543348, 0.5960876109129123, 0.7187130179825856, + 0.4629777327445964, 0.08665909175584818, 0.7748005397731237, 0.48020186965468536, 0.24927351841378798, 0.32272599988270445, 0.306414968984427, 0.6980212149215657, + 0.7977183964212472, 0.7673513094629704, 0.1679681724796478, 0.3107359484804584, 0.021701726051792103, 0.13797462786662518, 0.8618953518813538, 0.841333838365635, + 0.5284957375170422, 0.9703367685039823, 0.677388096913733, 0.2624474979832243, 0.43740966353106536, 0.15685545957858893, 0.11072929134449871, 0.06007395961283357, + 0.4093918718557811, 0.9563909195720572, 0.5994144944480242, 0.8278927844215804, 0.38586830957105667, 0.6201844716257464, 0.7603829079070265, 0.07875691596842949, + 0.08651136699915507, 0.7445210640026082, 0.6547649514127559, 0.3384719042666908, 0.05816723105860,0.6248951423054205, 0.7431868493349041}; + INDArray data = Nd4j.createFromArray(aData).reshape(11,5); + INDArray rows = Nd4j.createFromArray(new int[]{ + 0, 9, 18, 27, 36, 45, 54, 63, 72, 81, 90, 99}); + INDArray cols = Nd4j.createFromArray(new int[]{ + 4, 3, 10, 8, 6, 7, 1, 5, 9, 4, 9, 8, 10, 2, 0, 6, 7, 3, 6, 8, 3, 9, 10, 1, 4, 0, 5, 10, 0, 4, 6, 8, 9, 2, 5, 7, 0, 10, 3, 1, 8, 9, 6, 7, 2, 7, 9, 3, 10, 0, 4, 2, 8, 1, 2, 8, 3, 10, 0, 4, 9, 1, 5, 5, 9, 0, 3, 10, 4, 8, 1, 2, 6, 2, 0, 3, 4, 1, 10, 9, 7, 10, 1, 3, 7, 4, 5, 2, 8, 6, 3, 4, 0, 9, 6, 5, 8, 7, 1}); + INDArray vals = Nd4j.createFromArray(new double[] + { 0.6806, 0.1978, 0.1349, 0.0403, 0.0087, 0.0369, 0.0081, 0.0172, 0.0014, 0.0046, 0.0081, 0.3375, 0.2274, 0.0556, 0.0098, 0.0175, 0.0027, 0.0077, 0.0014, 0.0023, 0.0175, 0.6569, 0.1762, 0.0254, 0.0200, 0.0118, 0.0074, 0.0046, 0.0124, 0.0012, 0.1978, 0.0014, 0.0254, 0.7198, 0.0712, 0.0850, 0.0389, 0.0555, 0.0418, 0.0286, 0.6806, 0.3375, 0.0074, 0.0712, 0.2290, 0.0224, 0.0189, 0.0080, 0.0187, 0.0097, 0.0172, 0.0124, 0.0418, 0.7799, 0.0521, 0.0395, 0.0097, 0.0030, 0.0023, 1.706e-5, 0.0087, 0.0027, 0.6569, 0.0850, 0.0080, 0.5562, 0.0173, 0.0015, 1.706e-5, 0.0369, 0.0077, 0.0286, 0.0187, 0.7799, 0.0711, 0.0200, 0.0084, 0.0012, 0.0403, 0.0556, 0.1762, 0.0389, 0.0224, 0.0030, 0.5562, 0.0084, 0.0060, 0.0028, 0.0014, 0.2274, 0.0200, 0.0555, 0.0189, 0.0521, 0.0015, 0.0711, 0.0028, 0.3911, 0.1349, 0.0098, 0.0118, 0.7198, 0.2290, 0.0395, 0.0173, 0.0200, 0.0060, 0.3911}); + SpTree tree = new SpTree(data); + INDArray posF = Nd4j.create(11, 5); + /*try (MemoryWorkspace ws = tree.workspace().notifyScopeEntered())*/ { + tree.computeEdgeForces(rows, cols, vals, 11, posF); + } + INDArray expected = Nd4j.createFromArray(new double[]{ -0.08045664291717945, -0.1010737980370276, 0.01793326162563703, 0.16108447776416351, -0.20679423033936287, -0.15788549368713395, 0.02546624825966788, 0.062309466206907055, -0.165806093080134, 0.15266225270841186, 0.17508365896345726, 0.09588570563583201, 0.34124767300538084, 0.14606666020839956, -0.06786563815470595, -0.09326646571247202, -0.19896040730569928, -0.3618837364446506, 0.13946315445146712, -0.04570186310149667, -0.2473462951783839, -0.41362278505023914, -0.1094083777758208, 0.10705807646770374, 0.24462088260113946, 0.21722270026621748, -0.21799892431326567, -0.08205544003080587, -0.11170161709042685, -0.2674768703060442, 0.03617747284043274, 0.16430316252598698, 0.04552845070022399, 0.2593696744801452, 0.1439989190892037, -0.059339471967457376, 0.05460893792863096, -0.0595168036583193, -0.2527693197519917, -0.15850951859835274, -0.2945536856938165, 0.15434659331638875, -0.022910846947667776, 0.23598009757792854, -0.11149279745674007, 0.09670616593772939, 0.11125703954547914, -0.08519984596392606, -0.12779827002328714, 0.23025192887225998, 0.13741473964038722, -0.06193553503816597, -0.08349781586292176, 0.1622156410642145, 0.155975447743472}).reshape(11,5); + for (int i = 0; i < 11; ++i) + assertArrayEquals(expected.getRow(i).toDoubleVector(), posF.getRow(i).toDoubleVector(), 1e-2); + + AtomicDouble sumQ = new AtomicDouble(0.0); + /*try (MemoryWorkspace ws = tree.workspace().notifyScopeEntered())*/ { + tree.computeNonEdgeForces(0, 0.5, Nd4j.zeros(5), sumQ); + } + assertEquals(8.65, sumQ.get(), 1e-2); + } + + @Test + ////@Ignore + public void testLargeTree() { + int num = isIntegrationTests() ? 100000 : 1000; + StopWatch watch = new StopWatch(); + watch.start(); + INDArray arr = Nd4j.linspace(1, num, num, Nd4j.dataType()).reshape(num, 1); + SpTree tree = new SpTree(arr); + watch.stop(); + System.out.println("Tree of size " + num + " created in " + watch); + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/vptree/VPTreeSerializationTests.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/vptree/VPTreeSerializationTests.java new file mode 100644 index 000000000..c4146ebe2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/vptree/VPTreeSerializationTests.java @@ -0,0 +1,115 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.vptree; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.lang3.SerializationUtils; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.clustering.sptree.DataPoint; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * VPTree java serialization tests + * @author raver119@gmail.com + */ +@Slf4j +public class VPTreeSerializationTests extends BaseDL4JTest { + + @Test + public void testSerialization_1() throws Exception { + val points = Nd4j.rand(new int[] {10, 15}); + val treeA = new VPTree(points, true, 2); + + try (val bos = new ByteArrayOutputStream()) { + SerializationUtils.serialize(treeA, bos); + + try (val bis = new ByteArrayInputStream(bos.toByteArray())) { + VPTree treeB = SerializationUtils.deserialize(bis); + + assertEquals(points, treeA.getItems()); + assertEquals(points, treeB.getItems()); + + assertEquals(treeA.getWorkers(), treeB.getWorkers()); + + val row = points.getRow(1).dup('c'); + + val dpListA = new ArrayList(); + val dListA = new ArrayList(); + + val dpListB = new ArrayList(); + val dListB = new ArrayList(); + + treeA.search(row, 3, dpListA, dListA); + treeB.search(row, 3, dpListB, dListB); + + assertTrue(dpListA.size() != 0); + assertTrue(dListA.size() != 0); + + assertEquals(dpListA.size(), dpListB.size()); + assertEquals(dListA.size(), dListB.size()); + + for (int e = 0; e < dpListA.size(); e++) { + val rA = dpListA.get(e).getPoint(); + val rB = dpListB.get(e).getPoint(); + + assertEquals(rA, rB); + } + } + } + } + + + @Test + public void testNewConstructor_1() { + val points = Nd4j.rand(new int[] {10, 15}); + val treeA = new VPTree(points, true, 2); + + val rows = Nd4j.tear(points, 1); + + val list = new ArrayList(); + + int idx = 0; + for (val r: rows) + list.add(new DataPoint(idx++, r)); + + val treeB = new VPTree(list); + + assertEquals(points, treeA.getItems()); + assertEquals(points, treeB.getItems()); + } + + @Test + ////@Ignore + public void testBigTrees_1() throws Exception { + val list = new ArrayList(); + + for (int e = 0; e < 3200000; e++) { + val dp = new DataPoint(e, Nd4j.rand(new long[] {1, 300})); + } + + log.info("DataPoints created"); + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/vptree/VpTreeNodeTest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/vptree/VpTreeNodeTest.java new file mode 100644 index 000000000..99acc67d7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-core/src/test/java/org/deeplearning4j/clustering/vptree/VpTreeNodeTest.java @@ -0,0 +1,414 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.clustering.vptree; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.clustering.sptree.DataPoint; +import org.joda.time.Duration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.exception.ND4JIllegalStateException; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Counter; +import org.nd4j.common.primitives.Pair; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Anatoly Borisov + */ +@Slf4j +public class VpTreeNodeTest extends BaseDL4JTest { + + + private static class DistIndex implements Comparable { + public double dist; + public int index; + + public int compareTo(DistIndex r) { + return Double.compare(dist, r.dist); + } + } + + @BeforeAll + public static void beforeClass(){ + Nd4j.setDataType(DataType.FLOAT); + } + + @Test + public void testKnnK() { + INDArray arr = Nd4j.randn(10, 5); + VPTree t = new VPTree(arr, false); + List resultList = new ArrayList<>(); + List distances = new ArrayList<>(); + t.search(arr.getRow(0), 5, resultList, distances); + assertEquals(5, resultList.size()); + } + + + @Test + public void testParallel_1() { + int k = 5; + + for (int e = 0; e < 5; e++) { + Nd4j.getRandom().setSeed(7); + INDArray randn = Nd4j.rand(100, 3); + VPTree vpTree = new VPTree(randn, false, 4); + Nd4j.getRandom().setSeed(7); + VPTree vpTreeNoParallel = new VPTree(randn, false, 1); + List results = new ArrayList<>(); + List distances = new ArrayList<>(); + List noParallelResults = new ArrayList<>(); + List noDistances = new ArrayList<>(); + vpTree.search(randn.getRow(0), k, results, distances, true); + vpTreeNoParallel.search(randn.getRow(0), k, noParallelResults, noDistances, true); + + assertEquals( k, results.size(), "Failed at iteration " + e); + assertEquals(noParallelResults.size(), results.size(), "Failed at iteration " + e); + assertNotEquals(randn.getRow(0, true), results.get(0).getPoint()); + assertEquals(noParallelResults, results, "Failed at iteration " + e); + assertEquals(noDistances, distances, "Failed at iteration " + e); + } + } + + @Test + public void testParallel_2() { + int k = 5; + + for (int e = 0; e < 5; e++) { + Nd4j.getRandom().setSeed(7); + INDArray randn = Nd4j.rand(100, 3); + VPTree vpTree = new VPTree(randn, false, 4); + Nd4j.getRandom().setSeed(7); + VPTree vpTreeNoParallel = new VPTree(randn, false, 1); + List results = new ArrayList<>(); + List distances = new ArrayList<>(); + List noParallelResults = new ArrayList<>(); + List noDistances = new ArrayList<>(); + vpTree.search(randn.getRow(0), k, results, distances, false); + vpTreeNoParallel.search(randn.getRow(0), k, noParallelResults, noDistances, false); + + assertEquals(k, results.size(), "Failed at iteration " + e); + assertEquals(noParallelResults.size(), results.size(), "Failed at iteration " + e); + assertEquals(randn.getRow(0, true), results.get(0).getPoint()); + assertEquals(noParallelResults, results, "Failed at iteration " + e); + assertEquals(noDistances, distances, "Failed at iteration " + e); + } + } + + @Test + public void testReproducibility() { + val results = new ArrayList(); + val distances = new ArrayList(); + Nd4j.getRandom().setSeed(7); + val randn = Nd4j.rand(1000, 100); + + for (int e = 0; e < 10; e++) { + Nd4j.getRandom().setSeed(7); + val vpTree = new VPTree(randn, false, 1); + + val cresults = new ArrayList(); + val cdistances = new ArrayList(); + vpTree.search(randn.getRow(0), 5, cresults, cdistances); + + if (e == 0) { + results.addAll(cresults); + distances.addAll(cdistances); + } else { + assertEquals(results, cresults, "Failed at iteration " + e); + assertEquals(distances, cdistances, "Failed at iteration " + e); + } + } + } + + @Test + public void knnManualRandom() { + knnManual(Nd4j.randn(3, 5)); + } + + @Test + public void knnManualNaturals() { + knnManual(generateNaturalsMatrix(20, 2)); + } + + public static void knnManual(INDArray arr) { + Nd4j.getRandom().setSeed(7); + VPTree t = new VPTree(arr, false); + int k = 1; + int m = arr.rows(); + for (int targetIndex = 0; targetIndex < m; targetIndex++) { + // Do an exhaustive search + TreeSet s = new TreeSet<>(); + INDArray query = arr.getRow(targetIndex, true); + + Counter counter = new Counter<>(); + for (int j = 0; j < m; j++) { + double d = t.distance(query, (arr.getRow(j, true))); + counter.setCount(j, (float) d); + + } + + PriorityQueue> pq = counter.asReversedPriorityQueue(); + // keep closest k + for (int i = 0; i < k; i++) { + Pair di = pq.poll(); + System.out.println("exhaustive d=" + di.getFirst()); + s.add(di.getFirst()); + } + + // Check what VPTree gives for results + List results = new ArrayList<>(); + VPTreeFillSearch fillSearch = new VPTreeFillSearch(t, k, query); + fillSearch.search(); + results = fillSearch.getResults(); + + //List items = t.getItems(); + TreeSet resultSet = new TreeSet<>(); + + // keep k in a set + for (int i = 0; i < k; ++i) { + DataPoint result = results.get(i); + int r = result.getIndex(); + resultSet.add(r); + } + + + + // check + for (int r : resultSet) { + INDArray expectedResult = arr.getRow(r, true); + if (!s.contains(r)) { + fillSearch = new VPTreeFillSearch(t, k, query); + fillSearch.search(); + results = fillSearch.getResults(); + } + assertTrue(s.contains(r), String.format( + "VPTree result" + " %d is not in the " + "closest %d " + " " + "from the exhaustive" + + " search with query point %s and " + + "result %s and target not found %s", + r, k, query.toString(), results.toString(), expectedResult.toString())); + } + + } + } + + @Test + public void vpTreeTest() { + List points = new ArrayList<>(); + points.add(new DataPoint(0, Nd4j.create(new double[] {55, 55}))); + points.add(new DataPoint(1, Nd4j.create(new double[] {60, 60}))); + points.add(new DataPoint(2, Nd4j.create(new double[] {65, 65}))); + VPTree tree = new VPTree(points, "euclidean"); + List add = new ArrayList<>(); + List distances = new ArrayList<>(); + tree.search(Nd4j.create(new double[] {50, 50}), 1, add, distances); + DataPoint assertion = add.get(0); + assertEquals(new DataPoint(0, Nd4j.create(new double[] {55, 55}).reshape(1,2)), assertion); + + tree.search(Nd4j.create(new double[] {61, 61}), 2, add, distances, false); + assertion = add.get(0); + assertEquals(Nd4j.create(new double[] {60, 60}).reshape(1,2), assertion.getPoint()); + } + + @Test + public void vpTreeTest2() { + List points = new ArrayList<>(); + points.add(new DataPoint(0, Nd4j.create(new double[] {55, 55}))); + points.add(new DataPoint(1, Nd4j.create(new double[] {60, 60}))); + points.add(new DataPoint(2, Nd4j.create(new double[] {65, 65}))); + VPTree tree = new VPTree(points, "euclidean"); + Assertions.assertThrows(ND4JIllegalStateException.class, () -> { + tree.search(Nd4j.create(1, 10), 2, new ArrayList(), new ArrayList()); + }); + } + + @Test + public void vpTreeTest3() { + List points = new ArrayList<>(); + points.add(new DataPoint(0, Nd4j.create(new double[] {55, 55}))); + points.add(new DataPoint(1, Nd4j.create(new double[] {60, 60}))); + points.add(new DataPoint(2, Nd4j.create(new double[] {65, 65}))); + VPTree tree = new VPTree(points, "euclidean"); + Assertions.assertThrows(ND4JIllegalStateException.class, () -> { + tree.search(Nd4j.create(2, 10), 2, new ArrayList(), new ArrayList()); + }); + } + + @Test + public void vpTreeTest4() { + List points = new ArrayList<>(); + points.add(new DataPoint(0, Nd4j.create(new double[] {55, 55}))); + points.add(new DataPoint(1, Nd4j.create(new double[] {60, 60}))); + points.add(new DataPoint(2, Nd4j.create(new double[] {65, 65}))); + VPTree tree = new VPTree(points, "euclidean"); + Assertions.assertThrows(ND4JIllegalStateException.class, () -> { + tree.search(Nd4j.create(2, 10, 10), 2, new ArrayList(), new ArrayList()); + }); + } + + public static INDArray generateNaturalsMatrix(int nrows, int ncols) { + INDArray col = Nd4j.arange(0, nrows).reshape(nrows, 1).castTo(DataType.DOUBLE); + INDArray points = Nd4j.create(DataType.DOUBLE, nrows, ncols); + if (points.isColumnVectorOrScalar()) + points = col.dup(); + else { + for (int i = 0; i < ncols; i++) + points.putColumn(i, col); + } + return points; + } + + @Test + public void testVPSearchOverNaturals1D() throws Exception { + testVPSearchOverNaturalsPD(20, 1, 5); + } + + @Test + public void testVPSearchOverNaturals2D() throws Exception { + testVPSearchOverNaturalsPD(20, 2, 5); + } + + @Test + public void testTreeOrder() { + + int N = 10, dim = 1; + INDArray dataset = Nd4j.randn(N, dim); + double[] rawData = dataset.toDoubleVector(); + Arrays.sort(dataset.toDoubleVector()); + dataset = Nd4j.createFromArray(rawData).reshape(1,N); + + List points = new ArrayList<>(); + + for (int i = 0; i < rawData.length; ++i) { + points.add(new DataPoint(i, Nd4j.create(new double[]{rawData[i]}))); + } + + VPTree tree = new VPTree(points, "euclidean"); + INDArray points1 = tree.getItems(); + assertEquals(dataset, points1); + } + + @Test + public void testNearestNeighbors() { + + List points = new ArrayList<>(); + + points.add(new DataPoint(0, Nd4j.create(new double[] {0.83494041, 1.70294823, -1.34172191, 0.02350972, + -0.87519361, 0.64401935, -0.5634212, -1.1274308, + 0.19245948, -0.11349026}))); + points.add(new DataPoint(1, Nd4j.create(new double[] {-0.41115537, -0.7686138, -0.67923172, 1.01638281, + 0.04390801, 0.29753166, 0.78915771, -0.13564866, + -1.06053692, -0.15953041}))); + + VPTree tree = new VPTree(points, "euclidean"); + + List results = new ArrayList<>(); + List distances = new ArrayList<>(); + + final int k = 1; + double[] input = new double[]{0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5}; + tree.search(Nd4j.createFromArray(input), k, results, distances); + assertEquals(k, distances.size()); + assertEquals(2.7755637844503016, distances.get(0), 1e-5); + + double[] results_pattern = new double[]{-0.41115537, -0.7686138 , -0.67923172, 1.01638281, 0.04390801, + 0.29753166, 0.78915771, -0.13564866, -1.06053692, -0.15953041}; + for (int i = 0; i < results_pattern.length; ++i) { + assertEquals(results_pattern[i], results.get(0).getPoint().getDouble(i), 1e-5); + } + } + + @Test + public void performanceTest() { + final int dim = 300; + final int rows = 8000; + final int k = 5; + + INDArray inputArrray = Nd4j.linspace(DataType.DOUBLE, 0.0, 1.0, rows * dim).reshape(rows, dim); + + //INDArray inputArrray = Nd4j.randn(DataType.DOUBLE, 200000, dim); + long start = System.currentTimeMillis(); + VPTree tree = new VPTree(inputArrray, "euclidean"); + long end = System.currentTimeMillis(); + Duration duration = new Duration(start, end); + System.out.println("Elapsed time for tree construction " + duration.getStandardSeconds()); + + double[] input = new double[dim]; + for (int i = 0; i < dim; ++i) { + input[i] = 119; + } + List results = new ArrayList<>(); + List distances = new ArrayList<>(); + start = System.currentTimeMillis(); + tree.search(Nd4j.createFromArray(input), k, results, distances); + end = System.currentTimeMillis(); + duration = new Duration(start, end); + System.out.println("Elapsed time for tree search " + duration.getStandardSeconds()); + assertEquals(1590.2987519949422, distances.get(0), 1e-4); + } + + public static void testVPSearchOverNaturalsPD(int nrows, int ncols, int K) throws Exception { + final int queryPoint = 12; + + INDArray points = generateNaturalsMatrix(nrows, ncols); + INDArray query = Nd4j.zeros(DataType.DOUBLE, 1, ncols); + for (int i = 0; i < ncols; i++) + query.putScalar(0, i, queryPoint); + + INDArray trueResults = Nd4j.zeros(DataType.DOUBLE, K, ncols); + for (int j = 0; j < K; j++) { + int pt = queryPoint - K / 2 + j; + for (int i = 0; i < ncols; i++) + trueResults.putScalar(j, i, pt); + } + + VPTree tree = new VPTree(points, "euclidean", 1, false); + + List results = new ArrayList<>(); + List distances = new ArrayList<>(); + tree.search(query, K, results, distances, false); + int dimensionToSort = 0; + + INDArray sortedResults = Nd4j.zeros(DataType.DOUBLE, K, ncols); + int i = 0; + for (DataPoint p : results) { + sortedResults.putRow(i++, p.getPoint()); + } + + sortedResults = Nd4j.sort(sortedResults, dimensionToSort, true); + assertTrue(trueResults.equalsWithEps(sortedResults, 1e-5)); + + VPTreeFillSearch fillSearch = new VPTreeFillSearch(tree, K, query); + fillSearch.search(); + results = fillSearch.getResults(); + sortedResults = Nd4j.zeros(DataType.FLOAT, K, ncols); + i = 0; + for (DataPoint p : results) + sortedResults.putRow(i++, p.getPoint()); + INDArray[] sortedWithIndices = Nd4j.sortWithIndices(sortedResults, dimensionToSort, true);; + sortedResults = sortedWithIndices[1]; + assertEquals(trueResults.sumNumber().doubleValue(), sortedResults.sumNumber().doubleValue(), 1e-5); + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/build.gradle b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/build.gradle new file mode 100644 index 000000000..f0567afbd --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/build.gradle @@ -0,0 +1,27 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + + implementation projects.cavisDnn.cavisDnnApi + testImplementation projects.cavisDnn.cavisDnnCommonTests +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/Base64NDArrayBody.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/Base64NDArrayBody.java new file mode 100644 index 000000000..83813e3d0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/Base64NDArrayBody.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.nearestneighbor.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +/** + * Created by agibsonccc on 12/24/16. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class Base64NDArrayBody implements Serializable { + private String ndarray; + private int k; + private boolean forceFillK; +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/BatchRecord.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/BatchRecord.java new file mode 100644 index 000000000..a315d0402 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/BatchRecord.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.nearestneighbor.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Created by agibsonccc on 1/21/17. + */ +@Data +@AllArgsConstructor +@Builder +@NoArgsConstructor +public class BatchRecord implements Serializable { + private List records; + + /** + * Add a record + * @param record + */ + public void add(CSVRecord record) { + if (records == null) + records = new ArrayList<>(); + records.add(record); + } + + + /** + * Return a batch record based on a dataset + * @param dataSet the dataset to get the batch record for + * @return the batch record + */ + public static BatchRecord fromDataSet(DataSet dataSet) { + BatchRecord batchRecord = new BatchRecord(); + for (int i = 0; i < dataSet.numExamples(); i++) { + batchRecord.add(CSVRecord.fromRow(dataSet.get(i))); + } + + return batchRecord; + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/CSVRecord.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/CSVRecord.java new file mode 100644 index 000000000..b40a4e091 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/CSVRecord.java @@ -0,0 +1,84 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.nearestneighbor.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.Serializable; + +/** + * Created by agibsonccc on 12/24/16. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class CSVRecord implements Serializable { + private String[] values; + + /** + * Instantiate a csv record from a vector + * given either an input dataset and a + * one hot matrix, the index will be appended to + * the end of the record, or for regression + * it will append all values in the labels + * @param row the input vectors + * @return the record from this {@link DataSet} + */ + public static CSVRecord fromRow(DataSet row) { + if (!row.getFeatures().isVector() && !row.getFeatures().isScalar()) + throw new IllegalArgumentException("Passed in dataset must represent a scalar or vector"); + if (!row.getLabels().isVector() && !row.getLabels().isScalar()) + throw new IllegalArgumentException("Passed in dataset labels must be a scalar or vector"); + //classification + CSVRecord record; + int idx = 0; + if (row.getLabels().sumNumber().doubleValue() == 1.0) { + String[] values = new String[row.getFeatures().columns() + 1]; + for (int i = 0; i < row.getFeatures().length(); i++) { + values[idx++] = String.valueOf(row.getFeatures().getDouble(i)); + } + int maxIdx = 0; + for (int i = 0; i < row.getLabels().length(); i++) { + if (row.getLabels().getDouble(maxIdx) < row.getLabels().getDouble(i)) { + maxIdx = i; + } + } + + values[idx++] = String.valueOf(maxIdx); + record = new CSVRecord(values); + } + //regression (any number of values) + else { + String[] values = new String[row.getFeatures().columns() + row.getLabels().columns()]; + for (int i = 0; i < row.getFeatures().length(); i++) { + values[idx++] = String.valueOf(row.getFeatures().getDouble(i)); + } + for (int i = 0; i < row.getLabels().length(); i++) { + values[idx++] = String.valueOf(row.getLabels().getDouble(i)); + } + + + record = new CSVRecord(values); + + } + return record; + } + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborRequest.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborRequest.java new file mode 100644 index 000000000..06d4006c1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborRequest.java @@ -0,0 +1,31 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.nearestneighbor.model; + +import lombok.Data; + +import java.io.Serializable; + +/** + * Created by agibsonccc on 4/26/17. + */ +@Data +public class NearestNeighborRequest implements Serializable { + private int k; + private int inputIndex; + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborsResult.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborsResult.java new file mode 100644 index 000000000..e3134bbf7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborsResult.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.nearestneighbor.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +/** + * Created by agibsonccc on 4/26/17. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class NearestNeighborsResult { + public NearestNeighborsResult(int index, double distance) { + this(index, distance, null); + } + + private int index; + private double distance; + private String label; +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborsResults.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborsResults.java new file mode 100644 index 000000000..0075c620b --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-model/src/main/java/org/deeplearning4j/nearestneighbor/model/NearestNeighborsResults.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.nearestneighbor.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.List; + +/** + * Created by agibsonccc on 4/27/17. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class NearestNeighborsResults implements Serializable { + private List results; + +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/build.gradle b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/build.gradle new file mode 100644 index 000000000..9279620f2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/build.gradle @@ -0,0 +1,41 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnModel + implementation projects.cavisDnn.cavisDnnCore + implementation "io.vertx:vertx-core:4.0.2" + implementation "io.vertx:vertx-web:4.0.2" + testImplementation "com.mashape.unirest:unirest-java:1.4.9" + testImplementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnClient + implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore + implementation "com.beust:jcommander:1.27" + testImplementation 'ch.qos.logback:logback-classic' + testImplementation projects.cavisDnn.cavisDnnCommonTests + implementation projects.cavisDnn.cavisDnnApi + implementation "commons-io:commons-io" + implementation projects.cavisDnn.cavisDnnNn + implementation "org.slf4j:slf4j-api" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-core" +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/src/main/java/org/deeplearning4j/nearestneighbor/server/NearestNeighbor.java b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/src/main/java/org/deeplearning4j/nearestneighbor/server/NearestNeighbor.java new file mode 100644 index 000000000..d0cf6cca9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/src/main/java/org/deeplearning4j/nearestneighbor/server/NearestNeighbor.java @@ -0,0 +1,71 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.deeplearning4j.nearestneighbor.server; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import org.deeplearning4j.clustering.sptree.DataPoint; +import org.deeplearning4j.clustering.vptree.VPTree; +import org.deeplearning4j.nearestneighbor.model.NearestNeighborRequest; +import org.deeplearning4j.nearestneighbor.model.NearestNeighborsResult; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by agibsonccc on 4/27/17. + */ +@AllArgsConstructor +@Builder +public class NearestNeighbor { + private NearestNeighborRequest record; + private VPTree tree; + private INDArray points; + + public List search() { + INDArray input = points.slice(record.getInputIndex()); + List results = new ArrayList<>(); + if (input.isVector()) { + List add = new ArrayList<>(); + List distances = new ArrayList<>(); + tree.search(input, record.getK(), add, distances); + + if (add.size() != distances.size()) { + throw new IllegalStateException( + String.format("add.size == %d != %d == distances.size", + add.size(), distances.size())); + } + + for (int i=0; i print the usage info + jcmdr.usage(); + if (r.ndarrayPath == null) + log.error("Json path parameter is missing (null)"); + try { + Thread.sleep(500); + } catch (Exception e2) { + } + System.exit(1); + } + + instanceArgs = r; + try { + Vertx vertx = Vertx.vertx(); + vertx.deployVerticle(NearestNeighborsServer.class.getName()); + } catch (Throwable t){ + log.error("Error in NearestNeighboursServer run method",t); + } + } + + @Override + public void start() throws Exception { + instance = this; + + String[] pathArr = instanceArgs.ndarrayPath.split(","); + //INDArray[] pointsArr = new INDArray[pathArr.length]; + // first of all we reading shapes of saved eariler files + int rows = 0; + int cols = 0; + for (int i = 0; i < pathArr.length; i++) { + DataBuffer shape = BinarySerde.readShapeFromDisk(new File(pathArr[i])); + + log.info("Loading shape {} of {}; Shape: [{} x {}]", i + 1, pathArr.length, Shape.size(shape, 0), + Shape.size(shape, 1)); + + if (Shape.rank(shape) != 2) + throw new DL4JInvalidInputException("NearestNeighborsServer assumes 2D chunks"); + + rows += Shape.size(shape, 0); + + if (cols == 0) + cols = Shape.size(shape, 1); + else if (cols != Shape.size(shape, 1)) + throw new DL4JInvalidInputException( + "NearestNeighborsServer requires equal 2D chunks. Got columns mismatch."); + } + + final List labels = new ArrayList<>(); + if (instanceArgs.labelsPath != null) { + String[] labelsPathArr = instanceArgs.labelsPath.split(","); + for (int i = 0; i < labelsPathArr.length; i++) { + labels.addAll(FileUtils.readLines(new File(labelsPathArr[i]), "utf-8")); + } + } + if (!labels.isEmpty() && labels.size() != rows) + throw new DL4JInvalidInputException(String.format("Number of labels must match number of rows in points matrix (expected %d, found %d)", rows, labels.size())); + + final INDArray points = Nd4j.createUninitialized(rows, cols); + + int lastPosition = 0; + for (int i = 0; i < pathArr.length; i++) { + log.info("Loading chunk {} of {}", i + 1, pathArr.length); + INDArray pointsArr = BinarySerde.readFromDisk(new File(pathArr[i])); + + points.get(NDArrayIndex.interval(lastPosition, lastPosition + pointsArr.rows())).assign(pointsArr); + lastPosition += pointsArr.rows(); + + // let's ensure we don't bring too much stuff in next loop + System.gc(); + } + + VPTree tree = new VPTree(points, instanceArgs.similarityFunction, instanceArgs.invert); + + //Set play secret key, if required + //http://www.playframework.com/documentation/latest/ApplicationSecret + String crypto = System.getProperty("play.crypto.secret"); + if (crypto == null || "changeme".equals(crypto) || "".equals(crypto)) { + byte[] newCrypto = new byte[1024]; + + new Random().nextBytes(newCrypto); + + String base64 = Base64.getEncoder().encodeToString(newCrypto); + System.setProperty("play.crypto.secret", base64); + } + + Router r = Router.router(vertx); + r.route().handler(BodyHandler.create()); //NOTE: Setting this is required to receive request body content at all + createRoutes(r, labels, tree, points); + + vertx.createHttpServer() + .requestHandler(r) + .listen(instanceArgs.port); + } + + private void createRoutes(Router r, List labels, VPTree tree, INDArray points){ + + r.post("/knn").handler(rc -> { + try { + String json = rc.getBodyAsJson().encode(); + NearestNeighborRequest record = JsonMappers.getMapper().readValue(json, NearestNeighborRequest.class); + + NearestNeighbor nearestNeighbor = + NearestNeighbor.builder().points(points).record(record).tree(tree).build(); + + if (record == null) { + rc.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) + .putHeader("content-type", "application/json") + .end(JsonMappers.getMapper().writeValueAsString(Collections.singletonMap("status", "invalid json passed."))); + return; + } + + NearestNeighborsResults results = NearestNeighborsResults.builder().results(nearestNeighbor.search()).build(); + + rc.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) + .putHeader("content-type", "application/json") + .end(JsonMappers.getMapper().writeValueAsString(results)); + return; + } catch (Throwable e) { + log.error("Error in POST /knn",e); + rc.response().setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()) + .end("Error parsing request - " + e.getMessage()); + return; + } + }); + + r.post("/knnnew").handler(rc -> { + try { + String json = rc.getBodyAsJson().encode(); + Base64NDArrayBody record = JsonMappers.getMapper().readValue(json, Base64NDArrayBody.class); + if (record == null) { + rc.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) + .putHeader("content-type", "application/json") + .end(JsonMappers.getMapper().writeValueAsString(Collections.singletonMap("status", "invalid json passed."))); + return; + } + + INDArray arr = Nd4jBase64.fromBase64(record.getNdarray()); + List results; + List distances; + + if (record.isForceFillK()) { + VPTreeFillSearch vpTreeFillSearch = new VPTreeFillSearch(tree, record.getK(), arr); + vpTreeFillSearch.search(); + results = vpTreeFillSearch.getResults(); + distances = vpTreeFillSearch.getDistances(); + } else { + results = new ArrayList<>(); + distances = new ArrayList<>(); + tree.search(arr, record.getK(), results, distances); + } + + if (results.size() != distances.size()) { + rc.response() + .setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()) + .end(String.format("results.size == %d != %d == distances.size", results.size(), distances.size())); + return; + } + + List nnResult = new ArrayList<>(); + for (int i=0; i results = nearestNeighbor.search(); + assertEquals(1, results.get(0).getIndex()); + assertEquals(2, results.size()); + + assertEquals(1.0, results.get(0).getDistance(), 1e-4); + assertEquals(4.0, results.get(1).getDistance(), 1e-4); + } + + @Test + public void testNearestNeighborInverted() { + double[][] data = new double[][] {{1, 2, 3, 4}, {1, 2, 3, 5}, {3, 4, 5, 6}}; + INDArray arr = Nd4j.create(data); + + VPTree vpTree = new VPTree(arr, true); + NearestNeighborRequest request = new NearestNeighborRequest(); + request.setK(2); + request.setInputIndex(0); + NearestNeighbor nearestNeighbor = NearestNeighbor.builder().tree(vpTree).points(arr).record(request).build(); + List results = nearestNeighbor.search(); + assertEquals(2, results.get(0).getIndex()); + assertEquals(2, results.size()); + + assertEquals(-4.0, results.get(0).getDistance(), 1e-4); + assertEquals(-1.0, results.get(1).getDistance(), 1e-4); + } + + @Test + public void vpTreeTest() throws Exception { + INDArray matrix = Nd4j.rand(new int[] {400,10}); + INDArray rowVector = matrix.getRow(70); + INDArray resultArr = Nd4j.zeros(400,1); + Executor executor = Executors.newSingleThreadExecutor(); + VPTree vpTree = new VPTree(matrix); + System.out.println("Ran!"); + } + + + + public static int getAvailablePort() { + try { + ServerSocket socket = new ServerSocket(0); + try { + return socket.getLocalPort(); + } finally { + socket.close(); + } + } catch (IOException e) { + throw new IllegalStateException("Cannot find available port: " + e.getMessage(), e); + } + } + + @Test + public void testServer() throws Exception { + int localPort = getAvailablePort(); + Nd4j.getRandom().setSeed(7); + INDArray rand = Nd4j.randn(10, 5); + File writeToTmp = new File(testDir, UUID.randomUUID().toString()); + writeToTmp.deleteOnExit(); + BinarySerde.writeArrayToDisk(rand, writeToTmp); + NearestNeighborsServer.runMain("--ndarrayPath", writeToTmp.getAbsolutePath(), "--nearestNeighborsPort", + String.valueOf(localPort)); + + Thread.sleep(3000); + + NearestNeighborsClient client = new NearestNeighborsClient("http://localhost:" + localPort); + NearestNeighborsResults result = client.knnNew(5, rand.getRow(0)); + assertEquals(5, result.getResults().size()); + NearestNeighborsServer.getInstance().stop(); + } + + + + @Test + public void testFullSearch() throws Exception { + int numRows = 1000; + int numCols = 100; + int numNeighbors = 42; + INDArray points = Nd4j.rand(numRows, numCols); + VPTree tree = new VPTree(points); + INDArray query = Nd4j.rand(new int[] {1, numCols}); + VPTreeFillSearch fillSearch = new VPTreeFillSearch(tree, numNeighbors, query); + fillSearch.search(); + List results = fillSearch.getResults(); + List distances = fillSearch.getDistances(); + assertEquals(numNeighbors, distances.size()); + assertEquals(numNeighbors, results.size()); + } + + @Test + public void testDistances() { + + INDArray indArray = Nd4j.create(new float[][]{{3, 4}, {1, 2}, {5, 6}}); + INDArray record = Nd4j.create(new float[][]{{7, 6}}); + VPTree vpTree = new VPTree(indArray, "euclidean", false); + VPTreeFillSearch vpTreeFillSearch = new VPTreeFillSearch(vpTree, 3, record); + vpTreeFillSearch.search(); + //System.out.println(vpTreeFillSearch.getResults()); + System.out.println(vpTreeFillSearch.getDistances()); + } +} diff --git a/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/src/test/resources/logback.xml b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/src/test/resources/logback.xml new file mode 100644 index 000000000..1c1a984f6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn-parent/cavis-dnn-nn-server/src/test/resources/logback.xml @@ -0,0 +1,47 @@ + + + + + + logs/application.log + + %logger{15} - %message%n%xException{5} + + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-nn/build.gradle b/cavis-dnn/cavis-dnn-nn/build.gradle new file mode 100644 index 000000000..d9792730a --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/build.gradle @@ -0,0 +1,52 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +dependencies { + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators + implementation 'org.lucee:oswego-concurrent:1.3.4' + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisNative.cavisNativeBlas + implementation "commons-io:commons-io" + + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCommon + + implementation "com.google.code.gson:gson:2.8.6" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.google.protobuf:protobuf-java" + implementation "com.google.protobuf:protobuf-java-util" + implementation "com.github.oshi:oshi-core:3.4.2" + + implementation "it.unimi.dsi:fastutil:8.1.1" + + testImplementation 'ch.qos.logback:logback-classic' + testImplementation projects.cavisDnn.cavisDnnCommonTests + + implementation "org.bytedeco:javacpp" + implementation "org.apache.commons:commons-lang3" + implementation "org.apache.commons:commons-math3" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" + implementation "com.jakewharton.byteunits:byteunits:0.9.1" +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingConfiguration.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingConfiguration.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingModelSaver.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingModelSaver.java similarity index 93% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingModelSaver.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingModelSaver.java index cea0f9276..a9793175a 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingModelSaver.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingModelSaver.java @@ -24,9 +24,9 @@ import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; import org.deeplearning4j.earlystopping.saver.LocalFileGraphSaver; import org.deeplearning4j.earlystopping.saver.LocalFileModelSaver; import org.deeplearning4j.nn.api.Model; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.IOException; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingResult.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingResult.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingResult.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/EarlyStoppingResult.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/listener/EarlyStoppingListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/listener/EarlyStoppingListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/listener/EarlyStoppingListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/listener/EarlyStoppingListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/saver/InMemoryModelSaver.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/saver/InMemoryModelSaver.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/saver/InMemoryModelSaver.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/saver/InMemoryModelSaver.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/saver/LocalFileGraphSaver.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/saver/LocalFileGraphSaver.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/saver/LocalFileGraphSaver.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/saver/LocalFileGraphSaver.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/saver/LocalFileModelSaver.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/saver/LocalFileModelSaver.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/saver/LocalFileModelSaver.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/saver/LocalFileModelSaver.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/AutoencoderScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/AutoencoderScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/AutoencoderScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/AutoencoderScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ClassificationScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ClassificationScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ClassificationScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ClassificationScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculator.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculator.java index 28b44593e..e8d403a7f 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculator.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculator.java @@ -29,7 +29,7 @@ import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.MultiDataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; public class DataSetLossCalculator extends BaseScoreCalculator { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculatorCG.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculatorCG.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculatorCG.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculatorCG.java index 370ef1072..1d6b9fb7a 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculatorCG.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/DataSetLossCalculatorCG.java @@ -27,8 +27,8 @@ import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.MultiDataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; -import org.nd4j.shade.jackson.annotation.JsonIgnore; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; @NoArgsConstructor @Deprecated diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ROCScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ROCScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ROCScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ROCScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/RegressionScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/RegressionScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/RegressionScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/RegressionScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ScoreCalculator.java similarity index 91% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ScoreCalculator.java index d85c238a8..8e994a678 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ScoreCalculator.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/ScoreCalculator.java @@ -21,9 +21,9 @@ package org.deeplearning4j.earlystopping.scorecalc; import org.deeplearning4j.nn.api.Model; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/VAEReconErrorScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/VAEReconErrorScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/VAEReconErrorScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/VAEReconErrorScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/VAEReconProbScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/VAEReconProbScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/VAEReconProbScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/VAEReconProbScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseIEvaluationScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseIEvaluationScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseIEvaluationScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseIEvaluationScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseMLNScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseMLNScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseMLNScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseMLNScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseScoreCalculator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseScoreCalculator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseScoreCalculator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/scorecalc/base/BaseScoreCalculator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/BestScoreEpochTerminationCondition.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/BestScoreEpochTerminationCondition.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/BestScoreEpochTerminationCondition.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/BestScoreEpochTerminationCondition.java index 51458505f..3aeea5d96 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/BestScoreEpochTerminationCondition.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/BestScoreEpochTerminationCondition.java @@ -21,7 +21,7 @@ package org.deeplearning4j.earlystopping.termination; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class BestScoreEpochTerminationCondition implements EpochTerminationCondition { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/EpochTerminationCondition.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/EpochTerminationCondition.java similarity index 91% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/EpochTerminationCondition.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/EpochTerminationCondition.java index 8c3a026d2..2aed68348 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/EpochTerminationCondition.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/EpochTerminationCondition.java @@ -21,9 +21,9 @@ package org.deeplearning4j.earlystopping.termination; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/InvalidScoreIterationTerminationCondition.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/InvalidScoreIterationTerminationCondition.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/InvalidScoreIterationTerminationCondition.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/InvalidScoreIterationTerminationCondition.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/IterationTerminationCondition.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/IterationTerminationCondition.java similarity index 93% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/IterationTerminationCondition.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/IterationTerminationCondition.java index 87b52ae07..b45a63f8c 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/IterationTerminationCondition.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/IterationTerminationCondition.java @@ -20,8 +20,8 @@ package org.deeplearning4j.earlystopping.termination; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxEpochsTerminationCondition.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxEpochsTerminationCondition.java similarity index 94% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxEpochsTerminationCondition.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxEpochsTerminationCondition.java index 8fa02a212..aa0da9d68 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxEpochsTerminationCondition.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxEpochsTerminationCondition.java @@ -22,8 +22,8 @@ package org.deeplearning4j.earlystopping.termination; import lombok.Data; import lombok.NoArgsConstructor; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; @NoArgsConstructor @Data diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxScoreIterationTerminationCondition.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxScoreIterationTerminationCondition.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxScoreIterationTerminationCondition.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxScoreIterationTerminationCondition.java index e47f42a26..32929a157 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxScoreIterationTerminationCondition.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxScoreIterationTerminationCondition.java @@ -21,7 +21,7 @@ package org.deeplearning4j.earlystopping.termination; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class MaxScoreIterationTerminationCondition implements IterationTerminationCondition { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxTimeIterationTerminationCondition.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxTimeIterationTerminationCondition.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxTimeIterationTerminationCondition.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxTimeIterationTerminationCondition.java index 81827f4cf..0f48f2d50 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxTimeIterationTerminationCondition.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/MaxTimeIterationTerminationCondition.java @@ -21,7 +21,7 @@ package org.deeplearning4j.earlystopping.termination; import lombok.Data; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.concurrent.TimeUnit; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/ScoreImprovementEpochTerminationCondition.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/ScoreImprovementEpochTerminationCondition.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/ScoreImprovementEpochTerminationCondition.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/ScoreImprovementEpochTerminationCondition.java index 567bd88b8..fe84514fc 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/termination/ScoreImprovementEpochTerminationCondition.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/termination/ScoreImprovementEpochTerminationCondition.java @@ -22,7 +22,7 @@ package org.deeplearning4j.earlystopping.termination; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Slf4j @Data diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/BaseEarlyStoppingTrainer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/BaseEarlyStoppingTrainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/BaseEarlyStoppingTrainer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/BaseEarlyStoppingTrainer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/EarlyStoppingGraphTrainer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/EarlyStoppingGraphTrainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/EarlyStoppingGraphTrainer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/EarlyStoppingGraphTrainer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/EarlyStoppingTrainer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/EarlyStoppingTrainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/EarlyStoppingTrainer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/EarlyStoppingTrainer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/IEarlyStoppingTrainer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/IEarlyStoppingTrainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/IEarlyStoppingTrainer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/earlystopping/trainer/IEarlyStoppingTrainer.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/BaseEvaluation.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/BaseEvaluation.java new file mode 100644 index 000000000..0a872ef72 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/BaseEvaluation.java @@ -0,0 +1,68 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.nd4j.common.primitives.AtomicBoolean; +import org.nd4j.common.primitives.AtomicDouble; +import org.nd4j.common.primitives.serde.JsonDeserializerAtomicBoolean; +import org.nd4j.common.primitives.serde.JsonDeserializerAtomicDouble; +import org.nd4j.common.primitives.serde.JsonSerializerAtomicBoolean; +import org.nd4j.common.primitives.serde.JsonSerializerAtomicDouble; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +@Deprecated +@EqualsAndHashCode +public abstract class BaseEvaluation extends org.nd4j.evaluation.BaseEvaluation { + + @Getter + private static ObjectMapper objectMapper = configureMapper(new ObjectMapper()); + @Getter + private static ObjectMapper yamlMapper = configureMapper(new ObjectMapper(new YAMLFactory())); + + private static ObjectMapper configureMapper(ObjectMapper ret) { + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, false); + ret.enable(SerializationFeature.INDENT_OUTPUT); + SimpleModule atomicModule = new SimpleModule(); + atomicModule.addSerializer(AtomicDouble.class,new JsonSerializerAtomicDouble()); + atomicModule.addSerializer(AtomicBoolean.class,new JsonSerializerAtomicBoolean()); + atomicModule.addDeserializer(AtomicDouble.class,new JsonDeserializerAtomicDouble()); + atomicModule.addDeserializer(AtomicBoolean.class,new JsonDeserializerAtomicBoolean()); + ret.registerModule(atomicModule); + //Serialize fields only, not using getters + ret.setVisibilityChecker(ret.getSerializationConfig().getDefaultVisibilityChecker() + .withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withSetterVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)); + return ret; + } +} diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ConfusionMatrix.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ConfusionMatrix.java new file mode 100644 index 000000000..b45172e35 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ConfusionMatrix.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; +import lombok.Getter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Deprecated +public class ConfusionMatrix> extends org.nd4j.evaluation.classification.ConfusionMatrix { + + /** + * @deprecated Use {@link org.nd4j.evaluation.classification.ConfusionMatrix} + */ + @Deprecated + public ConfusionMatrix(List classes) { + super(classes); + } + + /** + * @deprecated Use {@link org.nd4j.evaluation.classification.ConfusionMatrix} + */ + @Deprecated + public ConfusionMatrix() { + super(); + } + + /** + * @deprecated Use {@link org.nd4j.evaluation.classification.ConfusionMatrix} + */ + @Deprecated + public ConfusionMatrix(ConfusionMatrix other) { + super(other); + } +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/Evaluation.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/Evaluation.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/Evaluation.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/Evaluation.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationAveraging.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationAveraging.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationAveraging.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationAveraging.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationBinary.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationBinary.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationBinary.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationBinary.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationCalibration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationCalibration.java new file mode 100644 index 000000000..9b699a401 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationCalibration.java @@ -0,0 +1,57 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Deprecated +@Getter +@EqualsAndHashCode(callSuper = true) +public class EvaluationCalibration extends org.nd4j.evaluation.classification.EvaluationCalibration implements org.deeplearning4j.eval.IEvaluation { + + /** + * @deprecated Use {@link org.nd4j.evaluation.classification.EvaluationCalibration} + */ + @Deprecated + public EvaluationCalibration() { + super(); + } + + /** + * @deprecated Use {@link org.nd4j.evaluation.classification.EvaluationCalibration} + */ + @Deprecated + public EvaluationCalibration(int reliabilityDiagNumBins, int histogramNumBins) { + super(reliabilityDiagNumBins, histogramNumBins); + } + + /** + * @deprecated Use {@link org.nd4j.evaluation.classification.EvaluationCalibration} + */ + @Deprecated + public EvaluationCalibration(@JsonProperty("reliabilityDiagNumBins") int reliabilityDiagNumBins, + @JsonProperty("histogramNumBins") int histogramNumBins, + @JsonProperty("excludeEmptyBins") boolean excludeEmptyBins) { + super(reliabilityDiagNumBins, histogramNumBins, excludeEmptyBins); + } +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/EvaluationUtils.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/IEvaluation.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/IEvaluation.java new file mode 100644 index 000000000..5bccf65ea --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/IEvaluation.java @@ -0,0 +1,29 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@Deprecated +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY) +public interface IEvaluation extends org.nd4j.evaluation.IEvaluation { + +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ROC.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ROC.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ROC.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ROC.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ROCBinary.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ROCBinary.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ROCBinary.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ROCBinary.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ROCMultiClass.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ROCMultiClass.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ROCMultiClass.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/ROCMultiClass.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/RegressionEvaluation.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/RegressionEvaluation.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/RegressionEvaluation.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/RegressionEvaluation.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java new file mode 100644 index 000000000..b5f4a5107 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java @@ -0,0 +1,52 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval.curves; + +import lombok.Data; +import org.nd4j.evaluation.curves.BaseHistogram; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Deprecated +@Data +public class Histogram extends org.nd4j.evaluation.curves.Histogram { + + /** + * @deprecated Use {@link org.nd4j.evaluation.curves.Histogram} + */ + public Histogram(@JsonProperty("title") String title, @JsonProperty("lower") double lower, + @JsonProperty("upper") double upper, @JsonProperty("binCounts") int[] binCounts) { + super(title, lower, upper, binCounts); + } + + /** + * @deprecated Use {@link org.nd4j.evaluation.curves.Histogram} + */ + public static Histogram fromJson(String json) { + return BaseHistogram.fromJson(json, Histogram.class); + } + + /** + * @deprecated Use {@link org.nd4j.evaluation.curves.Histogram} + */ + public static Histogram fromYaml(String yaml) { + return BaseHistogram.fromYaml(yaml, Histogram.class); + } +} diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/PrecisionRecallCurve.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/PrecisionRecallCurve.java new file mode 100644 index 000000000..0f3e4a4cf --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/PrecisionRecallCurve.java @@ -0,0 +1,58 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval.curves; + +import com.google.common.base.Preconditions; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Arrays; + +@Deprecated +@Data +@EqualsAndHashCode(callSuper = true) +public class PrecisionRecallCurve extends org.nd4j.evaluation.curves.PrecisionRecallCurve{ + + /** + * @deprecated Use {@link org.nd4j.evaluation.curves.ReliabilityDiagram} + */ + @Deprecated + public PrecisionRecallCurve(@JsonProperty("threshold") double[] threshold, + @JsonProperty("precision") double[] precision, @JsonProperty("recall") double[] recall, + @JsonProperty("tpCount") int[] tpCount, @JsonProperty("fpCount") int[] fpCount, + @JsonProperty("fnCount") int[] fnCount, @JsonProperty("totalCount") int totalCount) { + super(threshold, precision, recall, tpCount, fpCount, fnCount, totalCount); + } + + public static class Point extends org.nd4j.evaluation.curves.PrecisionRecallCurve.Point{ + public Point(int idx, double threshold, double precision, double recall) { + super(idx, threshold, precision, recall); + } + } + + public static class Confusion extends org.nd4j.evaluation.curves.PrecisionRecallCurve.Confusion{ + public Confusion(org.nd4j.evaluation.curves.PrecisionRecallCurve.Point point, int tpCount, int fpCount, int fnCount, int tnCount) { + super(point, tpCount, fpCount, fnCount, tnCount); + } + } +} diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/ReliabilityDiagram.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/ReliabilityDiagram.java new file mode 100644 index 000000000..66ffe2e11 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/ReliabilityDiagram.java @@ -0,0 +1,38 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval.curves; + +import lombok.NonNull; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Deprecated +public class ReliabilityDiagram extends org.nd4j.evaluation.curves.ReliabilityDiagram { + + /** + * @deprecated Use {@link org.nd4j.evaluation.curves.ReliabilityDiagram} + */ + @Deprecated + public ReliabilityDiagram(@JsonProperty("title") String title, + @NonNull @JsonProperty("meanPredictedValueX") double[] meanPredictedValueX, + @NonNull @JsonProperty("fractionPositivesY") double[] fractionPositivesY) { + super(title, meanPredictedValueX, fractionPositivesY); + } +} diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java new file mode 100644 index 000000000..17b176b9d --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.eval.curves; + +import com.google.common.base.Preconditions; +import lombok.Data; +import lombok.EqualsAndHashCode; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Deprecated +@Data +@EqualsAndHashCode(exclude = {"auc"}, callSuper = false) +public class RocCurve extends org.nd4j.evaluation.curves.RocCurve { + + /** + * @deprecated Use {@link org.nd4j.evaluation.curves.RocCurve} + */ + @Deprecated + public RocCurve(@JsonProperty("threshold") double[] threshold, @JsonProperty("fpr") double[] fpr, + @JsonProperty("tpr") double[] tpr) { + super(threshold, fpr, tpr); + } + + + /** + * @deprecated Use {@link org.nd4j.evaluation.curves.RocCurve} + */ + @Deprecated + public static RocCurve fromJson(String json) { + return fromJson(json, RocCurve.class); + } + + /** + * @deprecated Use {@link org.nd4j.evaluation.curves.RocCurve} + */ + @Deprecated + public static RocCurve fromYaml(String yaml) { + return fromYaml(yaml, RocCurve.class); + } + +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/meta/Prediction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/meta/Prediction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/meta/Prediction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/eval/meta/Prediction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/DL4JException.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/DL4JException.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/DL4JException.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/DL4JException.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/DL4JInvalidConfigException.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/DL4JInvalidConfigException.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/DL4JInvalidConfigException.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/DL4JInvalidConfigException.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/DL4JInvalidInputException.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/DL4JInvalidInputException.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/DL4JInvalidInputException.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/DL4JInvalidInputException.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/DeepLearningException.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/DeepLearningException.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/DeepLearningException.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/DeepLearningException.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/InvalidStepException.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/InvalidStepException.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/exception/InvalidStepException.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/exception/InvalidStepException.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java index 3c763d8b2..121102214 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java @@ -424,7 +424,12 @@ public class GradientCheckUtil { throw new IllegalArgumentException( "Invalid labels arrays: expect " + c.net.getNumOutputArrays() + " outputs"); - + DataType dataType = DataTypeUtil.getDtypeFromContext(); + if (dataType != DataType.DOUBLE) { + throw new IllegalStateException("Cannot perform gradient check: Datatype is not set to double precision (" + + "is: " + dataType + "). Double precision must be used for gradient checks. Set " + + "DataTypeUtil.setDTypeForContext(DataType.DOUBLE); before using GradientCheckUtil"); + } DataType netDataType = c.net.getConfiguration().getDataType(); if (netDataType != DataType.DOUBLE) { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/adapters/ArgmaxAdapter.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/adapters/ArgmaxAdapter.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/adapters/ArgmaxAdapter.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/adapters/ArgmaxAdapter.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/adapters/Regression2dAdapter.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/adapters/Regression2dAdapter.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/adapters/Regression2dAdapter.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/adapters/Regression2dAdapter.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/adapters/YoloModelAdapter.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/adapters/YoloModelAdapter.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/adapters/YoloModelAdapter.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/adapters/YoloModelAdapter.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Classifier.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Classifier.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Classifier.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Classifier.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/FwdPassType.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/FwdPassType.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/FwdPassType.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/FwdPassType.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Layer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Layer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Layer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Layer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/MaskState.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/MaskState.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/MaskState.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/MaskState.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Model.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Model.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Model.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Model.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/ModelAdapter.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/ModelAdapter.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/ModelAdapter.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/ModelAdapter.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/NeuralNetwork.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/NeuralNetwork.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/NeuralNetwork.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/NeuralNetwork.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/OptimizationAlgorithm.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/OptimizationAlgorithm.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/OptimizationAlgorithm.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/OptimizationAlgorithm.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/ParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/ParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/ParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/ParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Trainable.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Trainable.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Trainable.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Trainable.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/TrainingConfig.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/TrainingConfig.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/TrainingConfig.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/TrainingConfig.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Updater.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Updater.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/Updater.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/Updater.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/layers/IOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/layers/IOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/layers/IOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/layers/IOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/layers/LayerConstraint.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/layers/LayerConstraint.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/layers/LayerConstraint.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/layers/LayerConstraint.java index ea3924b69..fff8bd77d 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/layers/LayerConstraint.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/layers/LayerConstraint.java @@ -21,7 +21,7 @@ package org.deeplearning4j.nn.api.layers; import org.deeplearning4j.nn.api.Layer; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.Set; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/layers/RecurrentLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/layers/RecurrentLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/api/layers/RecurrentLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/api/layers/RecurrentLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/BackpropType.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/BackpropType.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/BackpropType.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/BackpropType.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/CNN2DFormat.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/CNN2DFormat.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/CNN2DFormat.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/CNN2DFormat.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/CacheMode.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/CacheMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/CacheMode.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/CacheMode.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ComputationGraphConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/ComputationGraphConfiguration.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ComputationGraphConfiguration.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/ComputationGraphConfiguration.java index 1831aa19c..efe5b0f60 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ComputationGraphConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/ComputationGraphConfiguration.java @@ -43,9 +43,9 @@ import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.exc.InvalidTypeIdException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,7 +115,7 @@ public class ComputationGraphConfiguration implements Serializable, Cloneable { synchronized (mapper) { try { return mapper.writeValueAsString(this); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } @@ -147,7 +147,7 @@ public class ComputationGraphConfiguration implements Serializable, Cloneable { //when writeValueAsString is used by multiple threads. This results in invalid JSON. See issue #3243 try { return mapper.writeValueAsString(this); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ConvolutionMode.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/ConvolutionMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ConvolutionMode.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/ConvolutionMode.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/DataFormat.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/DataFormat.java new file mode 100644 index 000000000..e8bd06860 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/DataFormat.java @@ -0,0 +1,30 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ +package org.deeplearning4j.nn.conf; + +import org.deeplearning4j.nn.conf.serde.format.DataFormatDeserializer; +import org.deeplearning4j.nn.conf.serde.format.DataFormatSerializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize(using = DataFormatSerializer.class) +@JsonDeserialize(using = DataFormatDeserializer.class) +public interface DataFormat { +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/GradientNormalization.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/GradientNormalization.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/GradientNormalization.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/GradientNormalization.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/InputPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/InputPreProcessor.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/InputPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/InputPreProcessor.java index 5ec1a4d63..9667f4909 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/InputPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/InputPreProcessor.java @@ -26,7 +26,7 @@ import org.deeplearning4j.nn.conf.inputs.InputType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/MultiLayerConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/MultiLayerConfiguration.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/MultiLayerConfiguration.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/MultiLayerConfiguration.java index fe44c26ec..a48dc85ba 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/MultiLayerConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/MultiLayerConfiguration.java @@ -43,10 +43,10 @@ import org.nd4j.linalg.lossfunctions.impl.LossBinaryXENT; import org.nd4j.linalg.lossfunctions.impl.LossMCXENT; import org.nd4j.linalg.lossfunctions.impl.LossMSE; import org.nd4j.linalg.lossfunctions.impl.LossNegativeLogLikelihood; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.exc.InvalidTypeIdException; -import org.nd4j.shade.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.io.IOException; import java.io.Serializable; @@ -108,7 +108,7 @@ public class MultiLayerConfiguration implements Serializable, Cloneable { synchronized (mapper) { try { return mapper.writeValueAsString(this); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } @@ -140,7 +140,7 @@ public class MultiLayerConfiguration implements Serializable, Cloneable { //when writeValueAsString is used by multiple threads. This results in invalid JSON. See issue #3243 try { return mapper.writeValueAsString(this); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java old mode 100755 new mode 100644 similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java index 3d4add356..5ceb3ea63 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java @@ -57,7 +57,7 @@ import org.nd4j.linalg.learning.regularization.L1Regularization; import org.nd4j.linalg.learning.regularization.L2Regularization; import org.nd4j.linalg.learning.regularization.Regularization; import org.nd4j.linalg.learning.regularization.WeightDecay; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.Serializable; @@ -300,7 +300,7 @@ public class NeuralNetConfiguration implements Serializable, Cloneable { String ret = mapper.writeValueAsString(this); return ret; - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } @@ -331,7 +331,7 @@ public class NeuralNetConfiguration implements Serializable, Cloneable { try { return mapper.writeValueAsString(this); - } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) { + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { throw new RuntimeException(e); } } diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/RNNFormat.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/RNNFormat.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/RNNFormat.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/RNNFormat.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/Updater.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/Updater.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/Updater.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/Updater.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/WorkspaceMode.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/WorkspaceMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/WorkspaceMode.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/WorkspaceMode.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/BaseConstraint.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/BaseConstraint.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/BaseConstraint.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/BaseConstraint.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/MaxNormConstraint.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/MaxNormConstraint.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/MaxNormConstraint.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/MaxNormConstraint.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/MinMaxNormConstraint.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/MinMaxNormConstraint.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/MinMaxNormConstraint.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/MinMaxNormConstraint.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/NonNegativeConstraint.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/NonNegativeConstraint.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/NonNegativeConstraint.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/NonNegativeConstraint.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/UnitNormConstraint.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/UnitNormConstraint.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/UnitNormConstraint.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/constraint/UnitNormConstraint.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/BinomialDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/BinomialDistribution.java new file mode 100644 index 000000000..883b027eb --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/BinomialDistribution.java @@ -0,0 +1,89 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.distribution; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class BinomialDistribution extends Distribution { + + private static final long serialVersionUID = 7407024251874318749L; + + private final int numberOfTrials; + private double probabilityOfSuccess; + + /** + * Create a distribution + * + * @param numberOfTrials the number of trials + * @param probabilityOfSuccess the probability of success + */ + @JsonCreator + public BinomialDistribution(@JsonProperty("numberOfTrials") int numberOfTrials, + @JsonProperty("probabilityOfSuccess") double probabilityOfSuccess) { + this.numberOfTrials = numberOfTrials; + this.probabilityOfSuccess = probabilityOfSuccess; + } + + public double getProbabilityOfSuccess() { + return probabilityOfSuccess; + } + + public void setProbabilityOfSuccess(double probabilityOfSuccess) { + this.probabilityOfSuccess = probabilityOfSuccess; + } + + public int getNumberOfTrials() { + return numberOfTrials; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + numberOfTrials; + long temp; + temp = Double.doubleToLongBits(probabilityOfSuccess); + result = prime * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + BinomialDistribution other = (BinomialDistribution) obj; + if (numberOfTrials != other.numberOfTrials) + return false; + if (Double.doubleToLongBits(probabilityOfSuccess) != Double.doubleToLongBits(other.probabilityOfSuccess)) + return false; + return true; + } + + public String toString() { + return "BinomialDistribution(" + "numberOfTrials=" + numberOfTrials + ", probabilityOfSuccess=" + + probabilityOfSuccess + ')'; + } +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/ConstantDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/ConstantDistribution.java similarity index 93% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/ConstantDistribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/ConstantDistribution.java index 848d26a23..9d50bcc6b 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/ConstantDistribution.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/ConstantDistribution.java @@ -22,8 +22,8 @@ package org.deeplearning4j.nn.conf.distribution; import lombok.Data; import lombok.EqualsAndHashCode; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = false) diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distribution.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distribution.java index 712427aaf..f9cfb41cd 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distribution.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distribution.java @@ -21,7 +21,7 @@ package org.deeplearning4j.nn.conf.distribution; import org.deeplearning4j.nn.conf.distribution.serde.LegacyDistributionHelper; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distributions.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distributions.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distributions.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/Distributions.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/GaussianDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/GaussianDistribution.java new file mode 100644 index 000000000..59300a7e2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/GaussianDistribution.java @@ -0,0 +1,40 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.distribution; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +@Deprecated +public class GaussianDistribution extends NormalDistribution { + + /** + * Create a gaussian distribution (equivalent to normal) + * with the given mean and std + * + * @param mean the mean + * @param std the standard deviation + */ + @JsonCreator + public GaussianDistribution(@JsonProperty("mean") double mean, @JsonProperty("std") double std) { + super(mean, std); + } +} diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/LogNormalDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/LogNormalDistribution.java new file mode 100644 index 000000000..5a2d12035 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/LogNormalDistribution.java @@ -0,0 +1,56 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.distribution; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * A log-normal distribution, with two parameters: mean and standard deviation. + * Note: the mean and standard deviation are for the logarithm of the values. + * Put another way: if X~LogN(M,S), then mean(log(X))=M, and stdev(log(X))=S + * + */ +@EqualsAndHashCode(callSuper = false) +@Data +public class LogNormalDistribution extends Distribution { + + private double mean, std; + + /** + * Create a log-normal distribution + * with the given mean and std + * + * @param mean the mean + * @param std the standard deviation + */ + @JsonCreator + public LogNormalDistribution(@JsonProperty("mean") double mean, @JsonProperty("std") double std) { + this.mean = mean; + this.std = std; + } + + public String toString() { + return "LogNormalDistribution(" + "mean=" + mean + ", std=" + std + ')'; + } +} diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/NormalDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/NormalDistribution.java new file mode 100644 index 000000000..566c58f66 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/NormalDistribution.java @@ -0,0 +1,98 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.distribution; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * A normal (Gaussian) distribution, with two parameters: mean and standard deviation + * + */ +@EqualsAndHashCode(callSuper = false) +@Data +public class NormalDistribution extends Distribution { + + private double mean, std; + + /** + * Create a normal distribution + * with the given mean and std + * + * @param mean the mean + * @param std the standard deviation + */ + @JsonCreator + public NormalDistribution(@JsonProperty("mean") double mean, @JsonProperty("std") double std) { + this.mean = mean; + this.std = std; + } + + public double getMean() { + return mean; + } + + public void setMean(double mean) { + this.mean = mean; + } + + public double getStd() { + return std; + } + + public void setStd(double std) { + this.std = std; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + long temp; + temp = Double.doubleToLongBits(mean); + result = prime * result + (int) (temp ^ (temp >>> 32)); + temp = Double.doubleToLongBits(std); + result = prime * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + NormalDistribution other = (NormalDistribution) obj; + if (Double.doubleToLongBits(mean) != Double.doubleToLongBits(other.mean)) + return false; + if (Double.doubleToLongBits(std) != Double.doubleToLongBits(other.std)) + return false; + return true; + } + + public String toString() { + return "NormalDistribution(" + "mean=" + mean + ", std=" + std + ')'; + } +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/OrthogonalDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/OrthogonalDistribution.java similarity index 93% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/OrthogonalDistribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/OrthogonalDistribution.java index 7e4efa84d..f34fbc93f 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/OrthogonalDistribution.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/OrthogonalDistribution.java @@ -22,8 +22,8 @@ package org.deeplearning4j.nn.conf.distribution; import lombok.Data; import lombok.EqualsAndHashCode; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Orthogonal distribution, with gain parameter.
diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/TruncatedNormalDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/TruncatedNormalDistribution.java new file mode 100644 index 000000000..027471534 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/TruncatedNormalDistribution.java @@ -0,0 +1,50 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.distribution; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +@EqualsAndHashCode(callSuper = false) +@Data +public class TruncatedNormalDistribution extends Distribution { + + private double mean, std; + + /** + * Create a truncated normal distribution + * with the given mean and std + * + * @param mean the mean + * @param std the standard deviation + */ + @JsonCreator + public TruncatedNormalDistribution(@JsonProperty("mean") double mean, @JsonProperty("std") double std) { + this.mean = mean; + this.std = std; + } + + public String toString() { + return "TruncatedNormalDistribution(" + "mean=" + mean + ", std=" + std + ')'; + } +} diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/UniformDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/UniformDistribution.java new file mode 100644 index 000000000..0c3e29de5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/UniformDistribution.java @@ -0,0 +1,62 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.distribution; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.apache.commons.math3.exception.NumberIsTooLargeException; +import org.apache.commons.math3.exception.util.LocalizedFormats; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * A uniform distribution, with two parameters: lower and upper - i.e., U(lower,upper) + * + */ +@EqualsAndHashCode(callSuper = false) +@Data +public class UniformDistribution extends Distribution { + + private double upper, lower; + + /** + * Create a uniform real distribution using the given lower and upper + * bounds. + * + * @param lower Lower bound of this distribution (inclusive). + * @param upper Upper bound of this distribution (exclusive). + * @throws NumberIsTooLargeException if {@code lower >= upper}. + */ + @JsonCreator + public UniformDistribution(@JsonProperty("lower") double lower, @JsonProperty("upper") double upper) + throws NumberIsTooLargeException { + if (lower >= upper) { + throw new NumberIsTooLargeException(LocalizedFormats.LOWER_BOUND_NOT_BELOW_UPPER_BOUND, lower, upper, + false); + } + this.lower = lower; + this.upper = upper; + } + + public String toString() { + return "UniformDistribution(lower=" + lower + ", upper=" + upper + ")"; + } +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionDeserializer.java similarity index 92% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionDeserializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionDeserializer.java index 1d7d645d5..88415f1cf 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionDeserializer.java @@ -21,12 +21,12 @@ package org.deeplearning4j.nn.conf.distribution.serde; import org.deeplearning4j.nn.conf.distribution.*; -import org.nd4j.shade.jackson.core.JsonParseException; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionHelper.java similarity index 94% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionHelper.java index 580f35db1..8f1168ef6 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionHelper.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/serde/LegacyDistributionHelper.java @@ -21,7 +21,7 @@ package org.deeplearning4j.nn.conf.distribution.serde; import org.deeplearning4j.nn.conf.distribution.Distribution; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @JsonDeserialize(using = LegacyDistributionDeserializer.class) public class LegacyDistributionHelper extends Distribution { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/AlphaDropout.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/AlphaDropout.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/AlphaDropout.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/AlphaDropout.java index fbffa1ef7..e5234e13e 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/AlphaDropout.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/AlphaDropout.java @@ -32,8 +32,8 @@ import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.MulOp; import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(exclude = {"lastPValue","alphaPrime","a","b", "mask"}) @@ -98,7 +98,7 @@ public class AlphaDropout implements IDropout { @Override public INDArray applyDropout(INDArray inputActivations, INDArray output, int iteration, int epoch, LayerWorkspaceMgr workspaceMgr) { //https://arxiv.org/pdf/1706.02515.pdf pg6 - // "...we propose “alpha dropout”, that randomly sets inputs to α'" + // "...we propose "alpha dropout", that randomly sets inputs to α'" // "The affine transformation a(xd + α'(1−d))+b allows to determine parameters a and b such that mean and // variance are kept to their values" diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/Dropout.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/Dropout.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/Dropout.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/Dropout.java index a8ff440d3..0078801bc 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/Dropout.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/Dropout.java @@ -36,8 +36,8 @@ import org.nd4j.linalg.api.ops.random.impl.DropOutInverted; import org.nd4j.linalg.exception.ND4JOpProfilerException; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @JsonIgnoreProperties({"mask", "helper", "helperCountFail", "initializedHelper"}) diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/DropoutHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/DropoutHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/DropoutHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/DropoutHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianDropout.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianDropout.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianDropout.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianDropout.java index a22d096d3..6157ef078 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianDropout.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianDropout.java @@ -30,8 +30,8 @@ import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.MulOp; import org.nd4j.linalg.api.ops.random.impl.GaussianDistribution; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @JsonIgnoreProperties({"noise"}) diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianNoise.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianNoise.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianNoise.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianNoise.java index 242abc8be..f33a99150 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianNoise.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/GaussianNoise.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.AddOp; import org.nd4j.linalg.api.ops.random.impl.GaussianDistribution; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class GaussianNoise implements IDropout { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/IDropout.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/IDropout.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/IDropout.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/IDropout.java index 08d2d6869..a5aa67288 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/IDropout.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/IDropout.java @@ -22,7 +22,7 @@ package org.deeplearning4j.nn.conf.dropout; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/SpatialDropout.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/SpatialDropout.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/SpatialDropout.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/SpatialDropout.java index 97cdadd8f..d0bb26529 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/SpatialDropout.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/dropout/SpatialDropout.java @@ -31,8 +31,8 @@ import org.nd4j.linalg.api.ops.random.impl.DropOutInverted; import org.nd4j.linalg.factory.Broadcast; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.schedule.ISchedule; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @JsonIgnoreProperties({"mask"}) diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/AttentionVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/AttentionVertex.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/AttentionVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/AttentionVertex.java index 9b73502d6..1a5a30e72 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/AttentionVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/AttentionVertex.java @@ -19,7 +19,7 @@ */ package org.deeplearning4j.nn.conf.graph; -import org.nd4j.shade.guava.base.Preconditions; +import com.google.common.base.Preconditions; import lombok.*; import org.deeplearning4j.nn.api.MaskState; import org.deeplearning4j.nn.conf.inputs.InputType; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertex.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertex.java index 6afea271f..5ea6e64b2 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertex.java @@ -30,7 +30,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class ElementWiseVertex extends GraphVertex { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/FrozenVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/FrozenVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/FrozenVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/FrozenVertex.java index 2b994f13b..73557c2c1 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/FrozenVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/FrozenVertex.java @@ -28,7 +28,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = false) diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/GraphVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/GraphVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/GraphVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/GraphVertex.java index 2c2d1365d..00f0f7f52 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/GraphVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/GraphVertex.java @@ -26,7 +26,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2NormalizeVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2NormalizeVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2NormalizeVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2NormalizeVertex.java index 4bed61427..502021120 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2NormalizeVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2NormalizeVertex.java @@ -30,7 +30,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = false) diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2Vertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2Vertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2Vertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/L2Vertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/LayerVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/MergeVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/MergeVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/MergeVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/MergeVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/PoolHelperVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/PoolHelperVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/PoolHelperVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/PoolHelperVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/PreprocessorVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/PreprocessorVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/PreprocessorVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/PreprocessorVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ReshapeVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ReshapeVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ReshapeVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ReshapeVertex.java index 9a774aa47..7ae4a374b 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ReshapeVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ReshapeVertex.java @@ -29,7 +29,7 @@ import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ScaleVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ScaleVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ScaleVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ScaleVertex.java index d79f549ab..a51fd312b 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ScaleVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ScaleVertex.java @@ -28,7 +28,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class ScaleVertex extends GraphVertex { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ShiftVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ShiftVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ShiftVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ShiftVertex.java index 391f14972..c1e878c60 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ShiftVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/ShiftVertex.java @@ -31,7 +31,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @NoArgsConstructor diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/StackVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/StackVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/StackVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/StackVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/SubsetVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/SubsetVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/SubsetVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/SubsetVertex.java index 4ad09de82..9c98c8b3b 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/SubsetVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/SubsetVertex.java @@ -29,7 +29,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/UnstackVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/UnstackVertex.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/UnstackVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/UnstackVertex.java index cc154ebf5..091194c1a 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/UnstackVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/UnstackVertex.java @@ -30,7 +30,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Getter public class UnstackVertex extends GraphVertex { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/DuplicateToTimeSeriesVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/DuplicateToTimeSeriesVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/DuplicateToTimeSeriesVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/DuplicateToTimeSeriesVertex.java index 90b0cc337..892562a4a 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/DuplicateToTimeSeriesVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/DuplicateToTimeSeriesVertex.java @@ -30,7 +30,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = false) diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/LastTimeStepVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/LastTimeStepVertex.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/LastTimeStepVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/LastTimeStepVertex.java index d25855787..dceb70a12 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/LastTimeStepVertex.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/LastTimeStepVertex.java @@ -29,7 +29,7 @@ import org.deeplearning4j.nn.conf.memory.MemoryReport; import org.deeplearning4j.nn.graph.ComputationGraph; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class LastTimeStepVertex extends GraphVertex { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/ReverseTimeSeriesVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/ReverseTimeSeriesVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/ReverseTimeSeriesVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/graph/rnn/ReverseTimeSeriesVertex.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InputType.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InputType.java new file mode 100644 index 000000000..db98572c0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InputType.java @@ -0,0 +1,522 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.inputs; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.nn.conf.DataFormat; +import org.deeplearning4j.nn.conf.RNNFormat; +import org.deeplearning4j.nn.conf.CNN2DFormat; +import org.deeplearning4j.nn.conf.layers.Convolution3D; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.util.OneTimeLogger; +import org.nd4j.linalg.api.ndarray.INDArray; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.io.Serializable; +import java.util.Arrays; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") +@Slf4j +public abstract class InputType implements Serializable { + + /** + * The type of activations in/out of a given GraphVertex
+ * FF: Standard feed-foward (2d minibatch, 1d per example) data
+ * RNN: Recurrent neural network (3d minibatch) time series data
+ * CNN: 2D Convolutional neural network (4d minibatch, [miniBatchSize, channels, height, width]) + * CNNFlat: Flattened 2D conv net data (2d minibatch, [miniBatchSize, height * width * channels]) + * CNN3D: 3D convolutional neural network (5d minibatch, [miniBatchSize, channels, height, width, channels]) + */ + public enum Type { + FF, RNN, CNN, CNNFlat, CNN3D + } + + public static CNN2DFormat getDefaultCNN2DFormat() { + return defaultCNN2DFormat; + } + + public static void setDefaultCNN2DFormat(CNN2DFormat defaultCNN2DFormat) { + InputType.defaultCNN2DFormat = defaultCNN2DFormat; + } + + private static CNN2DFormat defaultCNN2DFormat = CNN2DFormat.NCHW; + + @JsonIgnore + public abstract Type getType(); + + @Override + public abstract String toString(); + + @JsonIgnore + public abstract long arrayElementsPerExample(); + + /** + * Returns the shape of this InputType + * + * @param includeBatchDim Whether to include minibatch in the return shape array + * @return int[] + */ + @JsonIgnore + public abstract long[] getShape(boolean includeBatchDim); + + /** + * Returns the shape of this InputType without minibatch dimension in the returned array + * + * @return int[] + */ + public long[] getShape() { + return getShape(false); + } + + /** + * InputType for feed forward network data + * + * @param size The size of the activations + * @return InputTypeFeedForward + */ + public static InputType feedForward(long size) { + return new InputTypeFeedForward(size, null); + } + + public static InputType feedForward(long size, DataFormat timeDistributedFormat) { + return new InputTypeFeedForward(size,timeDistributedFormat); + } + + /** + * InputType for recurrent neural network (time series) data + * + * @param size The size of the activations + * @return InputTypeRecurrent + */ + public static InputType recurrent(long size) { + return new InputTypeRecurrent(size); + } + + /** + * InputType for recurrent neural network (time series) data + * + * @param size The size of the activations + * @param timeSeriesLength Length of the input time series + * @return InputTypeRecurrent + */ + public static InputType recurrent(long size, long timeSeriesLength) { + return new InputTypeRecurrent(size, timeSeriesLength, RNNFormat.NCW); + } + + public static InputType recurrent(long size, RNNFormat format){ + return new InputTypeRecurrent(size, format); + } + + public static InputType recurrent(long size, long timeSeriesLength, RNNFormat format){ + return new InputTypeRecurrent(size, timeSeriesLength, format); + } + /** + * Input type for convolutional (CNN) data, that is 4d with shape [miniBatchSize, channels, height, width]. + * For CNN data that has been flattened, use {@link #convolutionalFlat(long, long, long)} + * + * @param height height of the input + * @param width Width of the input + * @param depth Depth, or number of channels + * @return InputTypeConvolutional + */ + public static InputType convolutional(long height, long width, long depth) { + return convolutional(height, width, depth, getDefaultCNN2DFormat()); + } + + public static InputType convolutional(long height, long width, long depth, CNN2DFormat format){ + return new InputTypeConvolutional(height, width, depth, format); + } + + /** + * Input type for 3D convolutional (CNN3D) data in NDHWC format, that is 5d with shape + * [miniBatchSize, depth, height, width, channels]. + * + * @param height height of the input + * @param width Width of the input + * @param depth Depth of the input + * @param channels Number of channels of the input + * @return InputTypeConvolutional3D + * @deprecated Use {@link #convolutional3D(Convolution3D.DataFormat, long, long, long, long)} + */ + @Deprecated + public static InputType convolutional3D(long depth, long height, long width, long channels) { + return convolutional3D(Convolution3D.DataFormat.NDHWC, depth, height, width, channels); + } + + /** + * Input type for 3D convolutional (CNN3D) 5d data:
+ * If NDHWC format [miniBatchSize, depth, height, width, channels]
+ * If NDCWH + * + * @param height height of the input + * @param width Width of the input + * @param depth Depth of the input + * @param channels Number of channels of the input + * @return InputTypeConvolutional3D + */ + public static InputType convolutional3D(Convolution3D.DataFormat dataFormat, long depth, long height, long width, long channels) { + return new InputTypeConvolutional3D(dataFormat, depth, height, width, channels); + } + + /** + * Input type for convolutional (CNN) data, where the data is in flattened (row vector) format. + * Expect data with shape [miniBatchSize, height * width * channels]. For CNN data in 4d format, + * use {@link #convolutional(long, long, long)} + * + * @param height Height of the (unflattened) data represented by this input type + * @param width Width of the (unflattened) data represented by this input type + * @param depth Depth of the (unflattened) data represented by this input type + * @return InputTypeConvolutionalFlat + */ + public static InputType convolutionalFlat(long height, long width, long depth) { + return new InputTypeConvolutionalFlat(height, width, depth); + } + + + @NoArgsConstructor + @Getter + @EqualsAndHashCode(callSuper = false) + public static class InputTypeFeedForward extends InputType { + private long size; + private DataFormat timeDistributedFormat; + + public InputTypeFeedForward(@JsonProperty("size") long size, @JsonProperty("timeDistributedFormat") DataFormat timeDistributedFormat) { + if(size <= 0) { + OneTimeLogger.warn(log,"Assigning a size of zero. This is normally only valid in model import cases with unknown dimensions."); + } + this.size = size; + this.timeDistributedFormat = timeDistributedFormat; + } + + @Override + public Type getType() { + return Type.FF; + } + + @Override + public String toString() { + return "InputTypeFeedForward(" + size + (timeDistributedFormat != null ? "," + timeDistributedFormat : "") + ")"; + } + + @Override + public long arrayElementsPerExample() { + return size; + } + + @Override + public long[] getShape(boolean includeBatchDim) { + if(includeBatchDim) return new long[]{-1, size}; + else return new long[]{size}; + } + } + + @NoArgsConstructor + @Getter + @EqualsAndHashCode(callSuper = false) + public static class InputTypeRecurrent extends InputType { + private long size; + private long timeSeriesLength; + private RNNFormat format = RNNFormat.NCW; + public InputTypeRecurrent(long size) { + this(size, -1); + } + public InputTypeRecurrent(long size, long timeSeriesLength){ + this(size, timeSeriesLength, RNNFormat.NCW); + } + + public InputTypeRecurrent(long size, RNNFormat format){ + this(size, -1, format); + } + public InputTypeRecurrent(@JsonProperty("size") long size, + @JsonProperty("timeSeriesLength") long timeSeriesLength, + @JsonProperty("format") RNNFormat format) { + this.size = size; + this.timeSeriesLength = timeSeriesLength; + this.format = format; + } + + @Override + public Type getType() { + return Type.RNN; + } + + @Override + public String toString() { + if (timeSeriesLength > 0) { + return "InputTypeRecurrent(" + size + ",timeSeriesLength=" + timeSeriesLength + ",format=" + format + ")"; + } else { + return "InputTypeRecurrent(" + size + ",format=" + format + ")"; + } + } + + @Override + public long arrayElementsPerExample() { + if (timeSeriesLength <= 0) { + throw new IllegalStateException("Cannot calculate number of array elements per example: " + + "time series length is not set. Use InputType.recurrent(int size, int timeSeriesLength) instead?"); + } + return timeSeriesLength * size; + } + + @Override + public long[] getShape(boolean includeBatchDim) { + if (includeBatchDim){ + if (format == RNNFormat.NCW) { + return new long[]{-1, size, timeSeriesLength}; + } + else{ + return new long[]{-1, timeSeriesLength, size}; + } + + } + else{ + if (format == RNNFormat.NCW) { + return new long[]{size, timeSeriesLength}; + } + else{ + return new long[]{timeSeriesLength, size}; + } + } + } + } + + @NoArgsConstructor + @Data + @EqualsAndHashCode(callSuper = false) + public static class InputTypeConvolutional extends InputType { + private long height; + private long width; + private long channels; + private CNN2DFormat format = CNN2DFormat.NCHW; //Default for JSON deserialization of older configurations + + public InputTypeConvolutional(@JsonProperty("height") long height, @JsonProperty("width") long width, + @JsonProperty("channels") long channels, @JsonProperty("format") CNN2DFormat format) { + if(height <= 0) { + OneTimeLogger.warn(log,"Assigning height of 0. Normally this is not valid. Exceptions for this are generally related" + + "to model import and unknown dimensions"); + } + + if(width <= 0) { + OneTimeLogger.warn(log,"Assigning height of 0. Normally this is not valid. Exceptions for this are generally related" + + "to model import and unknown dimensions"); + } + + if(width <= 0) { + OneTimeLogger.warn(log,"Assigning width of 0. Normally this is not valid. Exceptions for this are generally related" + + "to model import and unknown dimensions"); + } + + if(channels <= 0) { + OneTimeLogger.warn(log,"Assigning width of 0. Normally this is not valid. Exceptions for this are generally related" + + "to model import and unknown dimensions"); + } + + + this.height = height; + this.width = width; + this.channels = channels; + if(format != null) + this.format = format; + } + + public InputTypeConvolutional(long height, long width, long channels) { + this(height, width, channels, CNN2DFormat.NCHW); + } + + /** + * Return the number of channels / depth for this 2D convolution. This method has been deprecated, + * for consistency purposes, use getChannels() instead. + * + * @return number of channels, i.e. depth for 2D convolutions + */ + @Deprecated + public long getDepth() { + return channels; + } + + /** + * Set the number of channels / depth for this 2D convolution. This method has been deprecated, + * for consistency purposes, use setChannels(channels) instead. + * + **/ + @Deprecated + public void setDepth(long depth) { + this.channels = depth; + } + + @Override + public Type getType() { + return Type.CNN; + } + + @Override + public String toString() { + return "InputTypeConvolutional(h=" + height + ",w=" + width + ",c=" + channels + "," + format + ")"; + } + + @Override + public long arrayElementsPerExample() { + return height * width * channels; + } + + @Override + public long[] getShape(boolean includeBatchDim) { + if(format == CNN2DFormat.NCHW){ + if(includeBatchDim) return new long[]{-1, channels, height, width}; + else return new long[]{channels, height, width}; + } else { + if(includeBatchDim) return new long[]{-1, height, width, channels}; + else return new long[]{height, width, channels}; + } + } + } + + @NoArgsConstructor + @Data + @EqualsAndHashCode(callSuper = false) + public static class InputTypeConvolutional3D extends InputType { + private Convolution3D.DataFormat dataFormat; + private long depth; + private long height; + private long width; + private long channels; + + public InputTypeConvolutional3D(@JsonProperty("dataFormat") Convolution3D.DataFormat dataFormat, + @JsonProperty("depth") long depth, @JsonProperty("height") long height, @JsonProperty("width") long width, @JsonProperty("channels") long channels) { + this.dataFormat = dataFormat; + this.depth = depth; + this.height = height; + this.width = width; + this.channels = channels; + } + + @Override + public Type getType() { + return Type.CNN3D; + } + + @Override + public String toString() { + return "InputTypeConvolutional3D(format=" + dataFormat + ",d=" + depth + ",h=" + height + ",w=" + width + ",c=" + channels + ")"; + } + + @Override + public long arrayElementsPerExample() { + return height * width * depth * channels; + } + + @Override + public long[] getShape(boolean includeBatchDim) { + if(dataFormat == Convolution3D.DataFormat.NDHWC){ + if(includeBatchDim) return new long[]{-1, depth, height, width, channels}; + else return new long[]{depth, height, width, channels}; + } else { + if(includeBatchDim) return new long[]{-1, channels, depth, height, width}; + else return new long[]{channels, depth, height, width}; + } + } + } + + @NoArgsConstructor + @Data + @EqualsAndHashCode(callSuper = false) + public static class InputTypeConvolutionalFlat extends InputType { + private long height; + private long width; + private long depth; + + public InputTypeConvolutionalFlat(@JsonProperty("height") long height, @JsonProperty("width") long width, @JsonProperty("depth") long depth) { + this.height = height; + this.width = width; + this.depth = depth; + } + + @Override + public Type getType() { + return Type.CNNFlat; + } + + public long getFlattenedSize() { + return height * width * depth; + } + + public InputType getUnflattenedType() { + return InputType.convolutional(height, width, depth); + } + + @Override + public String toString() { + return "InputTypeConvolutionalFlat(h=" + height + ",w=" + width + ",d=" + depth + ")"; + } + + @Override + public long arrayElementsPerExample() { + return height * width * depth; + } + + @Override + public long[] getShape(boolean includeBatchDim) { + if(includeBatchDim) return new long[]{-1, depth, height, width}; + else return new long[]{depth, height, width}; + } + } + + + public static InputType inferInputType(INDArray inputArray) { + //Note: ConvolutionalFlat and FeedForward look identical... but either should work OK if using something + // like FeedForwardToCnnPreProcessor + + switch (inputArray.rank()) { + case 2: + return InputType.feedForward(inputArray.size(1)); + case 3: + return InputType.recurrent(inputArray.size(1), (int) inputArray.size(2)); + case 4: + //Order: [minibatch, channels, height, width] -> [h, w, c] + return InputType.convolutional(inputArray.size(2), (int) inputArray.size(3), (int) inputArray.size(1)); + case 5: + //Order: [minibatch, channels, depth, height, width] -> [d, h, w, c] + return InputType.convolutional3D(inputArray.size(2), (int) inputArray.size(3), + (int) inputArray.size(4), (int) inputArray.size(1)); + default: + throw new IllegalArgumentException( + "Cannot infer input type for array with shape: " + Arrays.toString(inputArray.shape())); + } + } + + public static InputType[] inferInputTypes(INDArray... inputArrays) { + InputType[] out = new InputType[inputArrays.length]; + for (int i = 0; i < inputArrays.length; i++) { + out[i] = inferInputType(inputArrays[i]); + } + + return out; + } + +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InvalidInputTypeException.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InvalidInputTypeException.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InvalidInputTypeException.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InvalidInputTypeException.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AbstractLSTM.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AbstractLSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AbstractLSTM.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AbstractLSTM.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ActivationLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ActivationLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ActivationLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ActivationLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AutoEncoder.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AutoEncoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AutoEncoder.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/AutoEncoder.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BasePretrainNetwork.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BasePretrainNetwork.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BasePretrainNetwork.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BasePretrainNetwork.java index 5834f7a0a..8d958c3ec 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BasePretrainNetwork.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BasePretrainNetwork.java @@ -23,7 +23,7 @@ package org.deeplearning4j.nn.conf.layers; import lombok.*; import org.deeplearning4j.nn.params.PretrainParamInitializer; import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @Data @NoArgsConstructor diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseRecurrentLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseRecurrentLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseRecurrentLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseRecurrentLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseUpsamplingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseUpsamplingLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseUpsamplingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BaseUpsamplingLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BatchNormalization.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BatchNormalization.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BatchNormalization.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/BatchNormalization.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CapsuleLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CapsuleLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CapsuleLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CapsuleLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CapsuleStrengthLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CapsuleStrengthLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CapsuleStrengthLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CapsuleStrengthLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CenterLossOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CenterLossOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CenterLossOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CenterLossOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Cnn3DLossLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Cnn3DLossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Cnn3DLossLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Cnn3DLossLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CnnLossLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CnnLossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CnnLossLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/CnnLossLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution1D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution1D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution1D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution1DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution1DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution1DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution1DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution2D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution3D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution3D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Convolution3D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java index 016f5e7aa..9276408a9 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ConvolutionLayer.java @@ -34,7 +34,7 @@ import org.deeplearning4j.util.ValidationUtils; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.Arrays; import java.util.Collection; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Deconvolution2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Deconvolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Deconvolution2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Deconvolution2D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Deconvolution3D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Deconvolution3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Deconvolution3D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Deconvolution3D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DenseLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DenseLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DenseLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DenseLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DepthwiseConvolution2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DepthwiseConvolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DepthwiseConvolution2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DepthwiseConvolution2D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DropoutLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DropoutLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DropoutLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/DropoutLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingSequenceLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingSequenceLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingSequenceLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/EmbeddingSequenceLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/FeedForwardLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/FeedForwardLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/FeedForwardLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/FeedForwardLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GlobalPoolingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GlobalPoolingLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GlobalPoolingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GlobalPoolingLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GravesBidirectionalLSTM.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GravesBidirectionalLSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GravesBidirectionalLSTM.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GravesBidirectionalLSTM.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GravesLSTM.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GravesLSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GravesLSTM.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/GravesLSTM.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/InputTypeUtil.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/InputTypeUtil.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/InputTypeUtil.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/InputTypeUtil.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LSTM.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LSTM.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LSTM.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Layer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Layer.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Layer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Layer.java index 5b677ff26..cccc3cb1b 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Layer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Layer.java @@ -38,7 +38,7 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.config.IUpdater; import org.nd4j.linalg.learning.regularization.Regularization; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.lang.reflect.Field; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerValidation.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerValidation.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerValidation.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LayerValidation.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LearnedSelfAttentionLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LearnedSelfAttentionLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LearnedSelfAttentionLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LearnedSelfAttentionLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocalResponseNormalization.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocalResponseNormalization.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocalResponseNormalization.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocalResponseNormalization.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java index 1d0dcf90b..921d0f9ea 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected1D.java @@ -41,7 +41,7 @@ import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.*; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java index 630226231..724a0c22d 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LocallyConnected2D.java @@ -41,7 +41,7 @@ import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.*; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LossLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LossLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/LossLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/NoParamLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/NoParamLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/NoParamLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/NoParamLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/OutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/OutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/OutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/OutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PReLULayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PReLULayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PReLULayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PReLULayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Pooling1D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Pooling1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Pooling1D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Pooling1D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Pooling2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Pooling2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Pooling2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Pooling2D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PoolingType.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PoolingType.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PoolingType.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PoolingType.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PrimaryCapsules.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PrimaryCapsules.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PrimaryCapsules.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/PrimaryCapsules.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RecurrentAttentionLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RecurrentAttentionLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RecurrentAttentionLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RecurrentAttentionLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RnnLossLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RnnLossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RnnLossLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RnnLossLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RnnOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RnnOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RnnOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/RnnOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SelfAttentionLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SelfAttentionLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SelfAttentionLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SelfAttentionLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SeparableConvolution2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SeparableConvolution2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SeparableConvolution2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SeparableConvolution2D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SpaceToBatchLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SpaceToBatchLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SpaceToBatchLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SpaceToBatchLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SpaceToDepthLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SpaceToDepthLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SpaceToDepthLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SpaceToDepthLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Subsampling1DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Subsampling1DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Subsampling1DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Subsampling1DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Subsampling3DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Subsampling3DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Subsampling3DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Subsampling3DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SubsamplingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SubsamplingLayer.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SubsamplingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SubsamplingLayer.java index f0ebd4e7a..f1d546234 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SubsamplingLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/SubsamplingLayer.java @@ -36,7 +36,7 @@ import org.deeplearning4j.util.ValidationUtils; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.Collection; import java.util.Map; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling1D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling1D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling1D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling2D.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling2D.java index 1e07cf836..bdbbb0c73 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling2D.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling2D.java @@ -32,7 +32,7 @@ import org.deeplearning4j.optimize.api.TrainingListener; import org.deeplearning4j.util.ValidationUtils; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.util.Collection; import java.util.Map; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling3D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling3D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling3D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPadding1DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPadding1DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPadding1DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPadding1DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPadding3DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPadding3DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPadding3DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPadding3DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPaddingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPaddingLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPaddingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/ZeroPaddingLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping1D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping1D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping1D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping2D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping3D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping3D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/convolutional/Cropping3D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/ElementWiseMultiplicationLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/ElementWiseMultiplicationLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/ElementWiseMultiplicationLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/ElementWiseMultiplicationLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayer.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayer.java index 89c543ba2..ba5674bbb 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayer.java @@ -37,8 +37,8 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.config.IUpdater; import org.nd4j.linalg.learning.regularization.Regularization; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.util.Collection; import java.util.List; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java index a48f41b98..f782673cb 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/FrozenLayerWithBackprop.java @@ -32,7 +32,7 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.config.IUpdater; import org.nd4j.linalg.learning.regularization.Regularization; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Collection; import java.util.List; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/RepeatVector.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/RepeatVector.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/RepeatVector.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/misc/RepeatVector.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/BoundingBoxesDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/BoundingBoxesDeserializer.java similarity index 90% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/BoundingBoxesDeserializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/BoundingBoxesDeserializer.java index 004621bf1..747a95320 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/BoundingBoxesDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/BoundingBoxesDeserializer.java @@ -23,11 +23,11 @@ import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.serde.jackson.shaded.NDArrayTextDeSerializer; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/Yolo2OutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/Yolo2OutputLayer.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/Yolo2OutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/Yolo2OutputLayer.java index 2df05c32b..ec93eee55 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/Yolo2OutputLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/objdetect/Yolo2OutputLayer.java @@ -40,8 +40,8 @@ import org.nd4j.linalg.learning.regularization.Regularization; import org.nd4j.linalg.lossfunctions.ILossFunction; import org.nd4j.linalg.lossfunctions.impl.LossL2; import org.nd4j.serde.jackson.shaded.NDArrayTextSerializer; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.Arrays; import java.util.Collection; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/Bidirectional.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/Bidirectional.java new file mode 100644 index 000000000..388e131cd --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/Bidirectional.java @@ -0,0 +1,271 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.layers.recurrent; + +import lombok.*; +import org.deeplearning4j.nn.api.ParamInitializer; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.RNNFormat; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.BaseRecurrentLayer; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; +import org.deeplearning4j.nn.conf.layers.Layer; +import org.deeplearning4j.nn.conf.layers.wrapper.BaseWrapperLayer; +import org.deeplearning4j.nn.conf.memory.LayerMemoryReport; +import org.deeplearning4j.nn.layers.recurrent.BidirectionalLayer; +import org.deeplearning4j.nn.params.BidirectionalParamInitializer; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.util.TimeSeriesUtils; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.learning.regularization.Regularization; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static org.nd4j.linalg.indexing.NDArrayIndex.interval; +import static org.nd4j.linalg.indexing.NDArrayIndex.point; + +@NoArgsConstructor +@Data +@EqualsAndHashCode(callSuper = true, exclude = {"initializer"}) +@JsonIgnoreProperties({"initializer"}) +public class Bidirectional extends Layer { + + /** + * This Mode enumeration defines how the activations for the forward and backward networks should be combined.
+ * ADD: out = forward + backward (elementwise addition)
MUL: out = forward * backward (elementwise + * multiplication)
AVERAGE: out = 0.5 * (forward + backward)
CONCAT: Concatenate the activations.
Where + * 'forward' is the activations for the forward RNN, and 'backward' is the activations for the backward RNN. In all + * cases except CONCAT, the output activations size is the same size as the standard RNN that is being wrapped by + * this layer. In the CONCAT case, the output activations size (dimension 1) is 2x larger than the standard RNN's + * activations array. + */ + public enum Mode { + ADD, MUL, AVERAGE, CONCAT + } + + private Layer fwd; + private Layer bwd; + private Mode mode; + private transient BidirectionalParamInitializer initializer; + + private Bidirectional(Bidirectional.Builder builder) { + super(builder); + } + + /** + * Create a Bidirectional wrapper, with the default Mode (CONCAT) for the specified layer + * + * @param layer layer to wrap + */ + public Bidirectional(@NonNull Layer layer) { + this(Mode.CONCAT, layer); + } + + /** + * Create a Bidirectional wrapper for the specified layer + * + * @param mode Mode to use to combine activations. See {@link Mode} for details + * @param layer layer to wrap + */ + public Bidirectional(@NonNull Mode mode, @NonNull Layer layer) { + if (!(layer instanceof BaseRecurrentLayer || layer instanceof LastTimeStep + || layer instanceof BaseWrapperLayer)) { + throw new IllegalArgumentException("Cannot wrap a non-recurrent layer: " + + "config must extend BaseRecurrentLayer or LastTimeStep " + "Got class: " + + layer.getClass()); + } + this.fwd = layer; + this.bwd = layer.clone(); + this.mode = mode; + } + + public long getNOut() { + if (this.fwd instanceof LastTimeStep) { + return ((FeedForwardLayer) ((LastTimeStep) this.fwd).getUnderlying()).getNOut(); + } else { + return ((FeedForwardLayer) this.fwd).getNOut(); + } + } + + public long getNIn() { + if (this.fwd instanceof LastTimeStep) { + return ((FeedForwardLayer) ((LastTimeStep) this.fwd).getUnderlying()).getNIn(); + } else { + return ((FeedForwardLayer) this.fwd).getNIn(); + } + } + + public RNNFormat getRNNDataFormat(){ + return TimeSeriesUtils.getFormatFromRnnLayer(fwd); + } + + @Override + public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf, + Collection trainingListeners, int layerIndex, INDArray layerParamsView, + boolean initializeParams, DataType networkDataType) { + NeuralNetConfiguration c1 = conf.clone(); + NeuralNetConfiguration c2 = conf.clone(); + c1.setLayer(fwd); + c2.setLayer(bwd); + + long n = layerParamsView.length() / 2; + INDArray fp = layerParamsView.get(interval(0,0,true), interval(0, n)); + INDArray bp = layerParamsView.get(interval(0,0,true), interval(n, 2 * n)); + org.deeplearning4j.nn.api.Layer f = fwd.instantiate(c1, trainingListeners, layerIndex, fp, initializeParams, networkDataType); + + org.deeplearning4j.nn.api.Layer b = bwd.instantiate(c2, trainingListeners, layerIndex, bp, initializeParams, networkDataType); + + BidirectionalLayer ret = new BidirectionalLayer(conf, f, b, layerParamsView); + Map paramTable = initializer().init(conf, layerParamsView, initializeParams); + ret.setParamTable(paramTable); + ret.setConf(conf); + + return ret; + } + + @Override + public ParamInitializer initializer() { + if (initializer == null) { + initializer = new BidirectionalParamInitializer(this); + } + return initializer; + } + + @Override + public InputType getOutputType(int layerIndex, InputType inputType) { + InputType outOrig = fwd.getOutputType(layerIndex, inputType); + + if (fwd instanceof LastTimeStep) { + InputType.InputTypeFeedForward ff = (InputType.InputTypeFeedForward) outOrig; + if (mode == Mode.CONCAT) { + return InputType.feedForward(2 * ff.getSize()); + } else { + return ff; + } + } else { + InputType.InputTypeRecurrent r = (InputType.InputTypeRecurrent) outOrig; + if (mode == Mode.CONCAT) { + return InputType.recurrent(2 * r.getSize(), getRNNDataFormat()); + } else { + return r; + } + } + } + + @Override + public void setNIn(InputType inputType, boolean override) { + fwd.setNIn(inputType, override); + bwd.setNIn(inputType, override); + } + + @Override + public InputPreProcessor getPreProcessorForInputType(InputType inputType) { + return fwd.getPreProcessorForInputType(inputType); + } + + @Override + public List getRegularizationByParam(String paramName){ + //Strip forward/backward prefix from param name + return fwd.getRegularizationByParam(paramName.substring(1)); + } + + @Override + public boolean isPretrainParam(String paramName) { + return fwd.isPretrainParam(paramName.substring(1)); + } + + /** + * Get the updater for the given parameter. Typically the same updater will be used for all updaters, but this is + * not necessarily the case + * + * @param paramName Parameter name + * @return IUpdater for the parameter + */ + public IUpdater getUpdaterByParam(String paramName) { + String sub = paramName.substring(1); + return fwd.getUpdaterByParam(sub); + } + + @Override + public GradientNormalization getGradientNormalization() { + return fwd.getGradientNormalization(); + } + + @Override + public double getGradientNormalizationThreshold() { + return fwd.getGradientNormalizationThreshold(); + } + + @Override + public void setLayerName(String layerName) { + this.layerName = layerName; + fwd.setLayerName(layerName); + bwd.setLayerName(layerName); + } + + @Override + public LayerMemoryReport getMemoryReport(InputType inputType) { + LayerMemoryReport lmr = fwd.getMemoryReport(inputType); + lmr.scale(2); //Double all memory use + return lmr; + } + + @AllArgsConstructor + @Getter + @Setter + public static class Builder extends Layer.Builder { + + private Mode mode; + private Layer layer; + + public void setLayer(Layer layer) { + rnnLayer(layer); + } + + public Builder mode(Mode mode) { + this.setMode(mode); + return this; + } + + public Builder rnnLayer(Layer layer) { + if (!(layer instanceof BaseRecurrentLayer || layer instanceof LastTimeStep + || layer instanceof BaseWrapperLayer)) { + throw new IllegalArgumentException("Cannot wrap a non-recurrent layer: " + + "config must extend BaseRecurrentLayer or LastTimeStep " + "Got class: " + + layer.getClass()); + } + this.setLayer(layer); + return this; + } + + @SuppressWarnings("unchecked") + public Bidirectional build() { + return new Bidirectional(this); + } + } +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/LastTimeStep.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/LastTimeStep.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/LastTimeStep.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/LastTimeStep.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/SimpleRnn.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/SimpleRnn.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/SimpleRnn.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/SimpleRnn.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/TimeDistributed.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/TimeDistributed.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/TimeDistributed.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/TimeDistributed.java index 81e553509..d6004f6bb 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/TimeDistributed.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/TimeDistributed.java @@ -33,7 +33,7 @@ import org.deeplearning4j.nn.layers.recurrent.TimeDistributedLayer; import org.deeplearning4j.optimize.api.TrainingListener; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Collection; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/AbstractSameDiffLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/AbstractSameDiffLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/AbstractSameDiffLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/AbstractSameDiffLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDLayerParams.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDLayerParams.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDLayerParams.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDLayerParams.java index 562a4e8f4..99b3f85f8 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDLayerParams.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDLayerParams.java @@ -24,10 +24,10 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.NonNull; import org.nd4j.common.base.Preconditions; -import org.nd4j.shade.jackson.annotation.JsonIgnore; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; import java.util.*; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDVertexParams.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDVertexParams.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDVertexParams.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SDVertexParams.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLambdaVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLayerUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLayerUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLayerUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffLayerUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/samediff/SameDiffVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java index 028ea2ccb..410ea08c5 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/util/MaskZeroLayer.java @@ -33,7 +33,7 @@ import org.deeplearning4j.nn.conf.memory.LayerMemoryReport; import org.deeplearning4j.optimize.api.TrainingListener; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Collection; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/BernoulliReconstructionDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/BernoulliReconstructionDistribution.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/BernoulliReconstructionDistribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/BernoulliReconstructionDistribution.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/CompositeReconstructionDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/CompositeReconstructionDistribution.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/CompositeReconstructionDistribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/CompositeReconstructionDistribution.java index 4cfc64d53..47cfffd43 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/CompositeReconstructionDistribution.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/CompositeReconstructionDistribution.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.indexing.INDArrayIndex; import org.nd4j.linalg.indexing.NDArrayIndex; import org.nd4j.linalg.lossfunctions.ILossFunction; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ExponentialReconstructionDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ExponentialReconstructionDistribution.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ExponentialReconstructionDistribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ExponentialReconstructionDistribution.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/GaussianReconstructionDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/GaussianReconstructionDistribution.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/GaussianReconstructionDistribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/GaussianReconstructionDistribution.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/LossFunctionWrapper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/LossFunctionWrapper.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/LossFunctionWrapper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/LossFunctionWrapper.java index 51b4fe6af..3622018b6 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/LossFunctionWrapper.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/LossFunctionWrapper.java @@ -25,7 +25,7 @@ import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.activations.IActivation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.lossfunctions.ILossFunction; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class LossFunctionWrapper implements ReconstructionDistribution { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ReconstructionDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ReconstructionDistribution.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ReconstructionDistribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ReconstructionDistribution.java index 065967cf0..94b9a77d8 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ReconstructionDistribution.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/ReconstructionDistribution.java @@ -21,8 +21,8 @@ package org.deeplearning4j.nn.conf.layers.variational; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/VariationalAutoencoder.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/VariationalAutoencoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/VariationalAutoencoder.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/variational/VariationalAutoencoder.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/layers/wrapper/BaseWrapperLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/LayerMemoryReport.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/LayerMemoryReport.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/LayerMemoryReport.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/LayerMemoryReport.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryReport.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryReport.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryReport.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryReport.java index d62798e77..a0a6103d8 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryReport.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryReport.java @@ -27,8 +27,8 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.buffer.util.DataTypeUtil; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; import java.util.Collections; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryType.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryType.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryType.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryType.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryUseMode.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryUseMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryUseMode.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/MemoryUseMode.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/NetworkMemoryReport.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/NetworkMemoryReport.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/NetworkMemoryReport.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/NetworkMemoryReport.java index a4e9b97c4..9182ccfb9 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/memory/NetworkMemoryReport.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/memory/NetworkMemoryReport.java @@ -27,7 +27,7 @@ import org.deeplearning4j.nn.conf.CacheMode; import org.deeplearning4j.nn.conf.inputs.InputType; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.text.DecimalFormat; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/misc/DummyConfig.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/misc/DummyConfig.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/misc/DummyConfig.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/misc/DummyConfig.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/module/GraphBuilderModule.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/module/GraphBuilderModule.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/module/GraphBuilderModule.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/module/GraphBuilderModule.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ocnn/OCNNOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/ocnn/OCNNOutputLayer.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ocnn/OCNNOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/ocnn/OCNNOutputLayer.java index 71d177141..696d63f5d 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/ocnn/OCNNOutputLayer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/ocnn/OCNNOutputLayer.java @@ -34,9 +34,9 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.learning.regularization.Regularization; import org.nd4j.linalg.lossfunctions.ILossFunction; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Collection; import java.util.List; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/BaseInputPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/BaseInputPreProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/BaseInputPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/BaseInputPreProcessor.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/Cnn3DToFeedForwardPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/Cnn3DToFeedForwardPreProcessor.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/Cnn3DToFeedForwardPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/Cnn3DToFeedForwardPreProcessor.java index 8e660532d..ea57d875a 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/Cnn3DToFeedForwardPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/Cnn3DToFeedForwardPreProcessor.java @@ -30,8 +30,8 @@ import org.deeplearning4j.nn.workspace.ArrayType; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToFeedForwardPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToFeedForwardPreProcessor.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToFeedForwardPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToFeedForwardPreProcessor.java index 702767ef3..8f7be97d5 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToFeedForwardPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToFeedForwardPreProcessor.java @@ -31,8 +31,8 @@ import org.nd4j.linalg.api.shape.Shape; import org.nd4j.common.primitives.Pair; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToRnnPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToRnnPreProcessor.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToRnnPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToRnnPreProcessor.java index be44758f0..42a38e786 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToRnnPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToRnnPreProcessor.java @@ -32,8 +32,8 @@ import org.nd4j.common.primitives.Pair; import org.nd4j.common.util.ArrayUtil; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/ComposableInputPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/ComposableInputPreProcessor.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/ComposableInputPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/ComposableInputPreProcessor.java index 7636acbb7..be8b50316 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/ComposableInputPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/ComposableInputPreProcessor.java @@ -29,8 +29,8 @@ import org.deeplearning4j.nn.workspace.ArrayType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @EqualsAndHashCode(callSuper = false) diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnn3DPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnn3DPreProcessor.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnn3DPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnn3DPreProcessor.java index 2e55871ba..02e8a1544 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnn3DPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnn3DPreProcessor.java @@ -29,8 +29,8 @@ import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Pair; import org.nd4j.common.util.ArrayUtil; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnnPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnnPreProcessor.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnnPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnnPreProcessor.java index f485df6d0..111e253c5 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnnPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToCnnPreProcessor.java @@ -30,8 +30,8 @@ import org.nd4j.common.primitives.Pair; import org.nd4j.common.util.ArrayUtil; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToRnnPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToRnnPreProcessor.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToRnnPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToRnnPreProcessor.java index d0b6a0f79..cdc0bfab8 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToRnnPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/FeedForwardToRnnPreProcessor.java @@ -33,7 +33,7 @@ import org.nd4j.linalg.api.shape.Shape; import org.nd4j.common.primitives.Pair; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; @Data diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToCnnPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToCnnPreProcessor.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToCnnPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToCnnPreProcessor.java index 16307d87d..af53377e3 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToCnnPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToCnnPreProcessor.java @@ -32,7 +32,7 @@ import org.nd4j.common.primitives.Pair; import org.nd4j.common.util.ArrayUtil; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToFeedForwardPreProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToFeedForwardPreProcessor.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToFeedForwardPreProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToFeedForwardPreProcessor.java index 287722dfe..0afc0d82f 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToFeedForwardPreProcessor.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/preprocessor/RnnToFeedForwardPreProcessor.java @@ -34,7 +34,7 @@ import org.nd4j.linalg.api.shape.Shape; import org.nd4j.common.primitives.Pair; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java index cacfb49de..513b42aa8 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/BaseNetConfigDeserializer.java @@ -37,14 +37,14 @@ import org.nd4j.linalg.learning.regularization.Regularization; import org.nd4j.linalg.learning.regularization.WeightDecay; import org.nd4j.linalg.lossfunctions.ILossFunction; import org.nd4j.linalg.lossfunctions.impl.*; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonMappingException; -import org.nd4j.shade.jackson.databind.deser.ResolvableDeserializer; -import org.nd4j.shade.jackson.databind.deser.std.StdDeserializer; -import org.nd4j.shade.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.deser.ResolvableDeserializer; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.lang.reflect.InvocationTargetException; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java index ab5238768..edd9cbef8 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/ComputationGraphConfigurationDeserializer.java @@ -32,13 +32,13 @@ import org.deeplearning4j.nn.conf.layers.BatchNormalization; import org.deeplearning4j.nn.conf.layers.Layer; import org.deeplearning4j.nn.conf.weightnoise.DropConnect; import org.deeplearning4j.nn.params.BatchNormalizationParamInitializer; -import org.nd4j.shade.jackson.core.JsonLocation; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.core.JsonLocation; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.io.StringReader; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/JsonMappers.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/JsonMappers.java new file mode 100644 index 000000000..c5a8fe912 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/JsonMappers.java @@ -0,0 +1,90 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.serde; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.serde.legacy.LegacyJsonFormat; +import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +@Slf4j +public class JsonMappers { + + private static ObjectMapper jsonMapper = new ObjectMapper(); + private static ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); + + private static ObjectMapper legacyMapper; + + static { + configureMapper(jsonMapper); + configureMapper(yamlMapper); + } + + /** + * @return The default/primary ObjectMapper for deserializing JSON network configurations in DL4J + */ + public static ObjectMapper getMapper(){ + return jsonMapper; + } + + public static synchronized ObjectMapper getLegacyMapper(){ + if(legacyMapper == null){ + legacyMapper = LegacyJsonFormat.getMapper100alpha(); + configureMapper(legacyMapper); + } + return legacyMapper; + } + + /** + * @return The default/primary ObjectMapper for deserializing network configurations in DL4J (YAML format) + */ + public static ObjectMapper getMapperYaml() { + return yamlMapper; + } + + private static void configureMapper(ObjectMapper ret) { + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); + ret.enable(SerializationFeature.INDENT_OUTPUT); + + SimpleModule customDeserializerModule = new SimpleModule(); + customDeserializerModule.setDeserializerModifier(new BeanDeserializerModifier() { + @Override + public JsonDeserializer modifyDeserializer(DeserializationConfig config, BeanDescription beanDesc, + JsonDeserializer deserializer) { + //Use our custom deserializers to handle backward compatibility for updaters -> IUpdater + if (beanDesc.getBeanClass() == MultiLayerConfiguration.class) { + return new MultiLayerConfigurationDeserializer(deserializer); + } else if (beanDesc.getBeanClass() == ComputationGraphConfiguration.class) { + return new ComputationGraphConfigurationDeserializer(deserializer); + } + return deserializer; + } + }); + + ret.registerModule(customDeserializerModule); + } +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/MultiLayerConfigurationDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/MultiLayerConfigurationDeserializer.java similarity index 95% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/MultiLayerConfigurationDeserializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/MultiLayerConfigurationDeserializer.java index b3e9d9600..36f4a9b45 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/MultiLayerConfigurationDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/MultiLayerConfigurationDeserializer.java @@ -30,14 +30,14 @@ import org.deeplearning4j.nn.conf.layers.BatchNormalization; import org.deeplearning4j.nn.conf.layers.Layer; import org.deeplearning4j.nn.conf.weightnoise.DropConnect; import org.deeplearning4j.nn.params.BatchNormalizationParamInitializer; -import org.nd4j.shade.jackson.core.JsonLocation; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.node.ArrayNode; -import org.nd4j.shade.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.core.JsonLocation; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.io.StringReader; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatDeserializer.java similarity index 86% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatDeserializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatDeserializer.java index f4893b8f6..d7c3d636a 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatDeserializer.java @@ -22,11 +22,11 @@ package org.deeplearning4j.nn.conf.serde.format; import org.deeplearning4j.nn.conf.CNN2DFormat; import org.deeplearning4j.nn.conf.DataFormat; import org.deeplearning4j.nn.conf.RNNFormat; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatSerializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatSerializer.java similarity index 89% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatSerializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatSerializer.java index 76c641d63..835f15120 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatSerializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/format/DataFormatSerializer.java @@ -22,9 +22,9 @@ package org.deeplearning4j.nn.conf.serde.format; import org.deeplearning4j.nn.conf.CNN2DFormat; import org.deeplearning4j.nn.conf.DataFormat; import org.deeplearning4j.nn.conf.RNNFormat; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyIntArrayDeserializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyIntArrayDeserializer.java similarity index 83% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyIntArrayDeserializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyIntArrayDeserializer.java index 2de46c80f..3bbb5f8f6 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyIntArrayDeserializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyIntArrayDeserializer.java @@ -20,12 +20,12 @@ package org.deeplearning4j.nn.conf.serde.legacy; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; -import org.nd4j.shade.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.io.IOException; diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyJsonFormat.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyJsonFormat.java new file mode 100644 index 000000000..c654b2698 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyJsonFormat.java @@ -0,0 +1,187 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.nn.conf.serde.legacy; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.graph.*; +import org.deeplearning4j.nn.conf.graph.rnn.DuplicateToTimeSeriesVertex; +import org.deeplearning4j.nn.conf.graph.rnn.LastTimeStepVertex; +import org.deeplearning4j.nn.conf.graph.rnn.ReverseTimeSeriesVertex; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.convolutional.Cropping1D; +import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; +import org.deeplearning4j.nn.conf.layers.misc.ElementWiseMultiplicationLayer; +import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer; +import org.deeplearning4j.nn.conf.layers.objdetect.Yolo2OutputLayer; +import org.deeplearning4j.nn.conf.layers.recurrent.Bidirectional; +import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; +import org.deeplearning4j.nn.conf.layers.util.MaskLayer; +import org.deeplearning4j.nn.conf.layers.util.MaskZeroLayer; +import org.deeplearning4j.nn.conf.layers.variational.*; +import org.deeplearning4j.nn.conf.preprocessor.*; +import org.nd4j.linalg.activations.IActivation; +import org.nd4j.linalg.activations.impl.*; +import org.nd4j.linalg.lossfunctions.ILossFunction; +import org.nd4j.linalg.lossfunctions.impl.*; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class LegacyJsonFormat { + + private LegacyJsonFormat(){ } + + /** + * Get a mapper (minus general config) suitable for loading old format JSON - 1.0.0-alpha and before + * @return Object mapper + */ + public static ObjectMapper getMapper100alpha(){ + //After 1.0.0-alpha, we switched from wrapper object to @class for subtype information + ObjectMapper om = new ObjectMapper(); + + om.addMixIn(InputPreProcessor.class, InputPreProcessorMixin.class); + om.addMixIn(GraphVertex.class, GraphVertexMixin.class); + om.addMixIn(Layer.class, LayerMixin.class); + om.addMixIn(ReconstructionDistribution.class, ReconstructionDistributionMixin.class); + om.addMixIn(IActivation.class, IActivationMixin.class); + om.addMixIn(ILossFunction.class, ILossFunctionMixin.class); + + return om; + } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = CnnToFeedForwardPreProcessor.class, name = "cnnToFeedForward"), + @JsonSubTypes.Type(value = CnnToRnnPreProcessor.class, name = "cnnToRnn"), + @JsonSubTypes.Type(value = ComposableInputPreProcessor.class, name = "composableInput"), + @JsonSubTypes.Type(value = FeedForwardToCnnPreProcessor.class, name = "feedForwardToCnn"), + @JsonSubTypes.Type(value = FeedForwardToRnnPreProcessor.class, name = "feedForwardToRnn"), + @JsonSubTypes.Type(value = RnnToFeedForwardPreProcessor.class, name = "rnnToFeedForward"), + @JsonSubTypes.Type(value = RnnToCnnPreProcessor.class, name = "rnnToCnn")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class InputPreProcessorMixin { } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = ElementWiseVertex.class, name = "ElementWiseVertex"), + @JsonSubTypes.Type(value = MergeVertex.class, name = "MergeVertex"), + @JsonSubTypes.Type(value = SubsetVertex.class, name = "SubsetVertex"), + @JsonSubTypes.Type(value = LayerVertex.class, name = "LayerVertex"), + @JsonSubTypes.Type(value = LastTimeStepVertex.class, name = "LastTimeStepVertex"), + @JsonSubTypes.Type(value = ReverseTimeSeriesVertex.class, name = "ReverseTimeSeriesVertex"), + @JsonSubTypes.Type(value = DuplicateToTimeSeriesVertex.class, name = "DuplicateToTimeSeriesVertex"), + @JsonSubTypes.Type(value = PreprocessorVertex.class, name = "PreprocessorVertex"), + @JsonSubTypes.Type(value = StackVertex.class, name = "StackVertex"), + @JsonSubTypes.Type(value = UnstackVertex.class, name = "UnstackVertex"), + @JsonSubTypes.Type(value = L2Vertex.class, name = "L2Vertex"), + @JsonSubTypes.Type(value = ScaleVertex.class, name = "ScaleVertex"), + @JsonSubTypes.Type(value = L2NormalizeVertex.class, name = "L2NormalizeVertex")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class GraphVertexMixin{ } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = AutoEncoder.class, name = "autoEncoder"), + @JsonSubTypes.Type(value = ConvolutionLayer.class, name = "convolution"), + @JsonSubTypes.Type(value = Convolution1DLayer.class, name = "convolution1d"), + @JsonSubTypes.Type(value = GravesLSTM.class, name = "gravesLSTM"), + @JsonSubTypes.Type(value = LSTM.class, name = "LSTM"), + @JsonSubTypes.Type(value = GravesBidirectionalLSTM.class, name = "gravesBidirectionalLSTM"), + @JsonSubTypes.Type(value = OutputLayer.class, name = "output"), + @JsonSubTypes.Type(value = CenterLossOutputLayer.class, name = "CenterLossOutputLayer"), + @JsonSubTypes.Type(value = RnnOutputLayer.class, name = "rnnoutput"), + @JsonSubTypes.Type(value = LossLayer.class, name = "loss"), + @JsonSubTypes.Type(value = DenseLayer.class, name = "dense"), + @JsonSubTypes.Type(value = SubsamplingLayer.class, name = "subsampling"), + @JsonSubTypes.Type(value = Subsampling1DLayer.class, name = "subsampling1d"), + @JsonSubTypes.Type(value = BatchNormalization.class, name = "batchNormalization"), + @JsonSubTypes.Type(value = LocalResponseNormalization.class, name = "localResponseNormalization"), + @JsonSubTypes.Type(value = EmbeddingLayer.class, name = "embedding"), + @JsonSubTypes.Type(value = ActivationLayer.class, name = "activation"), + @JsonSubTypes.Type(value = VariationalAutoencoder.class, name = "VariationalAutoencoder"), + @JsonSubTypes.Type(value = DropoutLayer.class, name = "dropout"), + @JsonSubTypes.Type(value = GlobalPoolingLayer.class, name = "GlobalPooling"), + @JsonSubTypes.Type(value = ZeroPaddingLayer.class, name = "zeroPadding"), + @JsonSubTypes.Type(value = ZeroPadding1DLayer.class, name = "zeroPadding1d"), + @JsonSubTypes.Type(value = FrozenLayer.class, name = "FrozenLayer"), + @JsonSubTypes.Type(value = Upsampling2D.class, name = "Upsampling2D"), + @JsonSubTypes.Type(value = Yolo2OutputLayer.class, name = "Yolo2OutputLayer"), + @JsonSubTypes.Type(value = RnnLossLayer.class, name = "RnnLossLayer"), + @JsonSubTypes.Type(value = CnnLossLayer.class, name = "CnnLossLayer"), + @JsonSubTypes.Type(value = Bidirectional.class, name = "Bidirectional"), + @JsonSubTypes.Type(value = SimpleRnn.class, name = "SimpleRnn"), + @JsonSubTypes.Type(value = ElementWiseMultiplicationLayer.class, name = "ElementWiseMult"), + @JsonSubTypes.Type(value = MaskLayer.class, name = "MaskLayer"), + @JsonSubTypes.Type(value = MaskZeroLayer.class, name = "MaskZeroLayer"), + @JsonSubTypes.Type(value = Cropping1D.class, name = "Cropping1D"), + @JsonSubTypes.Type(value = Cropping2D.class, name = "Cropping2D")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class LayerMixin {} + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = GaussianReconstructionDistribution.class, name = "Gaussian"), + @JsonSubTypes.Type(value = BernoulliReconstructionDistribution.class, name = "Bernoulli"), + @JsonSubTypes.Type(value = ExponentialReconstructionDistribution.class, name = "Exponential"), + @JsonSubTypes.Type(value = CompositeReconstructionDistribution.class, name = "Composite"), + @JsonSubTypes.Type(value = LossFunctionWrapper.class, name = "LossWrapper")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class ReconstructionDistributionMixin {} + + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = ActivationCube.class, name = "Cube"), + @JsonSubTypes.Type(value = ActivationELU.class, name = "ELU"), + @JsonSubTypes.Type(value = ActivationHardSigmoid.class, name = "HardSigmoid"), + @JsonSubTypes.Type(value = ActivationHardTanH.class, name = "HardTanh"), + @JsonSubTypes.Type(value = ActivationIdentity.class, name = "Identity"), + @JsonSubTypes.Type(value = ActivationLReLU.class, name = "LReLU"), + @JsonSubTypes.Type(value = ActivationRationalTanh.class, name = "RationalTanh"), + @JsonSubTypes.Type(value = ActivationRectifiedTanh.class, name = "RectifiedTanh"), + @JsonSubTypes.Type(value = ActivationSELU.class, name = "SELU"), + @JsonSubTypes.Type(value = ActivationSwish.class, name = "SWISH"), + @JsonSubTypes.Type(value = ActivationReLU.class, name = "ReLU"), + @JsonSubTypes.Type(value = ActivationRReLU.class, name = "RReLU"), + @JsonSubTypes.Type(value = ActivationSigmoid.class, name = "Sigmoid"), + @JsonSubTypes.Type(value = ActivationSoftmax.class, name = "Softmax"), + @JsonSubTypes.Type(value = ActivationSoftPlus.class, name = "SoftPlus"), + @JsonSubTypes.Type(value = ActivationSoftSign.class, name = "SoftSign"), + @JsonSubTypes.Type(value = ActivationTanH.class, name = "TanH")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class IActivationMixin {} + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) + @JsonSubTypes(value = {@JsonSubTypes.Type(value = LossBinaryXENT.class, name = "BinaryXENT"), + @JsonSubTypes.Type(value = LossCosineProximity.class, name = "CosineProximity"), + @JsonSubTypes.Type(value = LossHinge.class, name = "Hinge"), + @JsonSubTypes.Type(value = LossKLD.class, name = "KLD"), + @JsonSubTypes.Type(value = LossMAE.class, name = "MAE"), + @JsonSubTypes.Type(value = LossL1.class, name = "L1"), + @JsonSubTypes.Type(value = LossMAPE.class, name = "MAPE"), + @JsonSubTypes.Type(value = LossMCXENT.class, name = "MCXENT"), + @JsonSubTypes.Type(value = LossMSE.class, name = "MSE"), + @JsonSubTypes.Type(value = LossL2.class, name = "L2"), + @JsonSubTypes.Type(value = LossMSLE.class, name = "MSLE"), + @JsonSubTypes.Type(value = LossNegativeLogLikelihood.class, name = "NegativeLogLikelihood"), + @JsonSubTypes.Type(value = LossPoisson.class, name = "Poisson"), + @JsonSubTypes.Type(value = LossSquaredHinge.class, name = "SquaredHinge"), + @JsonSubTypes.Type(value = LossFMeasure.class, name = "FMeasure")}) + @NoArgsConstructor(access = AccessLevel.PRIVATE) + public static class ILossFunctionMixin {} +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/DefaultStepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/DefaultStepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/DefaultStepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/DefaultStepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/GradientStepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/GradientStepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/GradientStepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/GradientStepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/NegativeDefaultStepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/NegativeDefaultStepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/NegativeDefaultStepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/NegativeDefaultStepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/NegativeGradientStepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/NegativeGradientStepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/NegativeGradientStepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/NegativeGradientStepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/StepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/StepFunction.java similarity index 89% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/StepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/StepFunction.java index 2a7e94a79..3fbbefc3a 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/StepFunction.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/stepfunctions/StepFunction.java @@ -20,10 +20,10 @@ package org.deeplearning4j.nn.conf.stepfunctions; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo.As; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo.Id; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/DropConnect.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/DropConnect.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/DropConnect.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/DropConnect.java index 77a554fd8..cabb01843 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/DropConnect.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/DropConnect.java @@ -29,7 +29,7 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.schedule.ISchedule; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class DropConnect implements IWeightNoise { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/IWeightNoise.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/IWeightNoise.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/IWeightNoise.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/IWeightNoise.java index db0d5cc2f..4c45b762f 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/IWeightNoise.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/IWeightNoise.java @@ -23,7 +23,7 @@ package org.deeplearning4j.nn.conf.weightnoise; import org.deeplearning4j.nn.api.Layer; import org.nd4j.linalg.api.ndarray.INDArray; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java index 1d40bd2b0..0e789749b 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/conf/weightnoise/WeightNoise.java @@ -31,7 +31,7 @@ import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.MulOp; import org.nd4j.linalg.factory.Nd4j; import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @Data public class WeightNoise implements IWeightNoise { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/gradient/DefaultGradient.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/gradient/DefaultGradient.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/gradient/DefaultGradient.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/gradient/DefaultGradient.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/gradient/Gradient.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/gradient/Gradient.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/gradient/Gradient.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/gradient/Gradient.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/ComputationGraph.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/util/ComputationGraphUtil.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/util/ComputationGraphUtil.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/util/ComputationGraphUtil.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/util/ComputationGraphUtil.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/util/GraphIndices.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/util/GraphIndices.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/util/GraphIndices.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/util/GraphIndices.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/BaseGraphVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/BaseGraphVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/BaseGraphVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/BaseGraphVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/BaseWrapperVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/BaseWrapperVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/BaseWrapperVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/BaseWrapperVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/GraphVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/GraphVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/GraphVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/GraphVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/VertexIndices.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/VertexIndices.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/VertexIndices.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/VertexIndices.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ElementWiseVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ElementWiseVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ElementWiseVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ElementWiseVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/FrozenVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/FrozenVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/FrozenVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/FrozenVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/InputVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/InputVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/InputVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/InputVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/L2NormalizeVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/L2NormalizeVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/L2NormalizeVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/L2NormalizeVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/L2Vertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/L2Vertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/L2Vertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/L2Vertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/LayerVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/LayerVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/LayerVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/LayerVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/MergeVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/MergeVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/MergeVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/MergeVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/PoolHelperVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/PoolHelperVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/PoolHelperVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/PoolHelperVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/PreprocessorVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/PreprocessorVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/PreprocessorVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/PreprocessorVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ReshapeVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ReshapeVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ReshapeVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ReshapeVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ScaleVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ScaleVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ScaleVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ScaleVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ShiftVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ShiftVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ShiftVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/ShiftVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/StackVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/StackVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/StackVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/StackVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/SubsetVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/SubsetVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/SubsetVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/SubsetVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/UnstackVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/UnstackVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/UnstackVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/UnstackVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/DuplicateToTimeSeriesVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/DuplicateToTimeSeriesVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/DuplicateToTimeSeriesVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/DuplicateToTimeSeriesVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/LastTimeStepVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/LastTimeStepVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/LastTimeStepVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/LastTimeStepVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/ReverseTimeSeriesVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/ReverseTimeSeriesVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/ReverseTimeSeriesVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/graph/vertex/impl/rnn/ReverseTimeSeriesVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/AbstractLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/ActivationLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/ActivationLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/ActivationLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/ActivationLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/BaseLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/BaseLayer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/BaseLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/BaseLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/BaseOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/BaseOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/BaseOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/BaseOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/BasePretrainNetwork.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/BasePretrainNetwork.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/BasePretrainNetwork.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/BasePretrainNetwork.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/DropoutLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/DropoutLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/DropoutLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/DropoutLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/FrozenLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/FrozenLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/FrozenLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/FrozenLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackprop.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackprop.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackprop.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackprop.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/LayerHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/LayerHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/LayerHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/LayerHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/LossLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/LossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/LossLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/LossLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/OutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/OutputLayer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/OutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/OutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/RepeatVector.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/RepeatVector.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/RepeatVector.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/RepeatVector.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cnn3DLossLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cnn3DLossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cnn3DLossLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cnn3DLossLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/CnnLossLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/CnnLossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/CnnLossLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/CnnLossLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Convolution1DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Convolution1DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Convolution1DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Convolution1DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Convolution3DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Convolution3DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Convolution3DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Convolution3DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ConvolutionHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ConvolutionHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ConvolutionHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ConvolutionHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping1DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping1DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping1DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping1DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping2DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping2DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping2DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping2DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping3DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping3DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping3DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Cropping3DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Deconvolution2DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Deconvolution2DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Deconvolution2DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Deconvolution2DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Deconvolution3DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Deconvolution3DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Deconvolution3DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/Deconvolution3DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/DepthwiseConvolution2DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/DepthwiseConvolution2DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/DepthwiseConvolution2DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/DepthwiseConvolution2DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SeparableConvolution2DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SeparableConvolution2DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SeparableConvolution2DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SeparableConvolution2DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SpaceToBatch.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SpaceToBatch.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SpaceToBatch.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SpaceToBatch.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepth.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepth.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepth.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/SpaceToDepth.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPadding1DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPadding1DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPadding1DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPadding1DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPadding3DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPadding3DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPadding3DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPadding3DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPaddingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPaddingLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPaddingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/ZeroPaddingLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/Subsampling1DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/Subsampling1DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/Subsampling1DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/Subsampling1DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/Subsampling3DLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/Subsampling3DLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/Subsampling3DLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/Subsampling3DLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/subsampling/SubsamplingLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling1D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling1D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling1D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling1D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling2D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling2D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling2D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling2D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling3D.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling3D.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling3D.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling3D.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/PReLU.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/PReLU.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/PReLU.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/PReLU.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/autoencoder/AutoEncoder.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/autoencoder/AutoEncoder.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/autoencoder/AutoEncoder.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/autoencoder/AutoEncoder.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/autoencoder/recursive/Tree.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/autoencoder/recursive/Tree.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/autoencoder/recursive/Tree.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/autoencoder/recursive/Tree.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/dense/DenseLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/elementwise/ElementWiseMultiplicationLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/elementwise/ElementWiseMultiplicationLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/elementwise/ElementWiseMultiplicationLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/elementwise/ElementWiseMultiplicationLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingSequenceLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingSequenceLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingSequenceLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingSequenceLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/BaseMKLDNNHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/BaseMKLDNNHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/BaseMKLDNNHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/BaseMKLDNNHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNBatchNormHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNBatchNormHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNBatchNormHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNBatchNormHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNConvHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNConvHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNConvHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNConvHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNLSTMHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNLSTMHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNLSTMHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNLSTMHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNLocalResponseNormalizationHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNLocalResponseNormalizationHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNLocalResponseNormalizationHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNLocalResponseNormalizationHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNSubsamplingHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNSubsamplingHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNSubsamplingHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/mkldnn/MKLDNNSubsamplingHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/BatchNormalization.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/BatchNormalization.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/BatchNormalization.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/BatchNormalization.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/LocalResponseNormalization.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/LocalResponseNormalization.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/LocalResponseNormalization.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/LocalResponseNormalization.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/LocalResponseNormalizationHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/LocalResponseNormalizationHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/LocalResponseNormalizationHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/normalization/LocalResponseNormalizationHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/DetectedObject.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/DetectedObject.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/DetectedObject.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/DetectedObject.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/Yolo2OutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/Yolo2OutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/Yolo2OutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/Yolo2OutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/YoloUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/YoloUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/YoloUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/objdetect/YoloUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/ocnn/OCNNParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/ocnn/OCNNParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/ocnn/OCNNParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/ocnn/OCNNParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/pooling/GlobalPoolingLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/BaseRecurrentLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/BaseRecurrentLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/BaseRecurrentLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/BaseRecurrentLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/FwdPassReturn.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/FwdPassReturn.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/FwdPassReturn.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/FwdPassReturn.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTM.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTM.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTM.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTM.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTM.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTM.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTM.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTM.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTM.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelpers.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelpers.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelpers.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LSTMHelpers.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LastTimeStepLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LastTimeStepLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LastTimeStepLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/LastTimeStepLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/RnnLossLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/RnnLossLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/RnnLossLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/RnnLossLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/RnnOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/RnnOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/RnnOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/RnnOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/SimpleRnn.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/SimpleRnn.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/SimpleRnn.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/SimpleRnn.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/TimeDistributedLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/TimeDistributedLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/TimeDistributedLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/recurrent/TimeDistributedLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/DL4JSameDiffMemoryMgr.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/DL4JSameDiffMemoryMgr.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/DL4JSameDiffMemoryMgr.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/DL4JSameDiffMemoryMgr.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffGraphVertex.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffGraphVertex.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffGraphVertex.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffGraphVertex.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffOutputLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/samediff/SameDiffOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/training/CenterLossOutputLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/training/CenterLossOutputLayer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/training/CenterLossOutputLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/training/CenterLossOutputLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/util/IdentityLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/util/IdentityLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/util/IdentityLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/util/IdentityLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/util/MaskLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/util/MaskLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/util/MaskLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/util/MaskLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/variational/VariationalAutoencoder.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/variational/VariationalAutoencoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/variational/VariationalAutoencoder.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/variational/VariationalAutoencoder.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/wrapper/BaseWrapperLayer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/wrapper/BaseWrapperLayer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/wrapper/BaseWrapperLayer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/layers/wrapper/BaseWrapperLayer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/multilayer/MultiLayerNetwork.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/BatchNormalizationParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/BatchNormalizationParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/BatchNormalizationParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/BatchNormalizationParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/BidirectionalParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/BidirectionalParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/BidirectionalParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/BidirectionalParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/CenterLossParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/CenterLossParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/CenterLossParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/CenterLossParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/Convolution3DParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/Convolution3DParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/Convolution3DParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/Convolution3DParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/ConvolutionParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/ConvolutionParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/ConvolutionParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/ConvolutionParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/Deconvolution3DParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/Deconvolution3DParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/Deconvolution3DParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/Deconvolution3DParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/DeconvolutionParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DeconvolutionParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/DeconvolutionParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DeconvolutionParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/DefaultParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DefaultParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/DefaultParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DefaultParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/DepthwiseConvolutionParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DepthwiseConvolutionParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/DepthwiseConvolutionParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/DepthwiseConvolutionParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/ElementWiseParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/ElementWiseParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/ElementWiseParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/ElementWiseParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/EmbeddingLayerParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/EmbeddingLayerParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/EmbeddingLayerParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/EmbeddingLayerParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/EmptyParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/EmptyParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/EmptyParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/EmptyParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/FrozenLayerParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/FrozenLayerParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/FrozenLayerParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/FrozenLayerParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/FrozenLayerWithBackpropParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/FrozenLayerWithBackpropParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/FrozenLayerWithBackpropParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/FrozenLayerWithBackpropParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/GravesBidirectionalLSTMParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/GravesBidirectionalLSTMParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/GravesBidirectionalLSTMParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/GravesBidirectionalLSTMParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/GravesLSTMParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/GravesLSTMParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/GravesLSTMParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/GravesLSTMParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/LSTMParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/LSTMParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/LSTMParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/LSTMParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/PReLUParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/PReLUParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/PReLUParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/PReLUParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/PretrainParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/PretrainParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/PretrainParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/PretrainParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/SameDiffParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/SameDiffParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/SameDiffParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/SameDiffParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/SeparableConvolutionParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/SeparableConvolutionParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/SeparableConvolutionParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/SeparableConvolutionParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/SimpleRnnParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/SimpleRnnParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/SimpleRnnParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/SimpleRnnParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/VariationalAutoencoderParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/VariationalAutoencoderParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/VariationalAutoencoderParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/VariationalAutoencoderParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/WrapperLayerParamInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/WrapperLayerParamInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/params/WrapperLayerParamInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/params/WrapperLayerParamInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java index 71678f380..3f2ddd88b 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/FineTuneConfiguration.java @@ -45,9 +45,9 @@ import org.nd4j.linalg.learning.regularization.L2Regularization; import org.nd4j.linalg.learning.regularization.Regularization; import org.nd4j.linalg.learning.regularization.WeightDecay; import org.nd4j.common.primitives.Optional; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; import java.util.ArrayList; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/TransferLearning.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/TransferLearning.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/TransferLearning.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/TransferLearning.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelper.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelper.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelper.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/BaseMultiLayerUpdater.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/BaseMultiLayerUpdater.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/BaseMultiLayerUpdater.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/BaseMultiLayerUpdater.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/LayerUpdater.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/LayerUpdater.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/LayerUpdater.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/LayerUpdater.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/MultiLayerUpdater.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/MultiLayerUpdater.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/MultiLayerUpdater.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/MultiLayerUpdater.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterBlock.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterBlock.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterBlock.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterBlock.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterCreator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterCreator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterCreator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterCreator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/UpdaterUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/graph/ComputationGraphUpdater.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/graph/ComputationGraphUpdater.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/updater/graph/ComputationGraphUpdater.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/updater/graph/ComputationGraphUpdater.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/IWeightInit.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/IWeightInit.java similarity index 95% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/IWeightInit.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/IWeightInit.java index 0b67fa165..d0c524c22 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/IWeightInit.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/IWeightInit.java @@ -21,8 +21,8 @@ package org.deeplearning4j.nn.weights; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInit.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInit.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInit.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInit.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitConstant.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitConstant.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitConstant.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitConstant.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitDistribution.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitDistribution.java similarity index 97% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitDistribution.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitDistribution.java index fbd292265..296305e17 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitDistribution.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitDistribution.java @@ -25,7 +25,7 @@ import org.deeplearning4j.nn.conf.distribution.Distribution; import org.deeplearning4j.nn.conf.distribution.Distributions; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.rng.distribution.impl.OrthogonalDistribution; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; @EqualsAndHashCode public class WeightInitDistribution implements IWeightInit { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitIdentity.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitIdentity.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitIdentity.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitIdentity.java index 95996c7df..2bb8a9ba1 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitIdentity.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitIdentity.java @@ -27,7 +27,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.indexing.INDArrayIndex; import org.nd4j.linalg.indexing.NDArrayIndex; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitLecunUniform.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitLecunUniform.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitLecunUniform.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitLecunUniform.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitNormal.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitNormal.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitNormal.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitNormal.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitRelu.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitRelu.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitRelu.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitRelu.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitReluUniform.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitReluUniform.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitReluUniform.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitReluUniform.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitSigmoidUniform.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitSigmoidUniform.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitSigmoidUniform.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitSigmoidUniform.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitUniform.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitUniform.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitUniform.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitUniform.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitUtil.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitUtil.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitUtil.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitUtil.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanAvg.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanAvg.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanAvg.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanAvg.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanIn.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanIn.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanIn.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanIn.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanOut.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanOut.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanOut.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingNormalFanOut.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanAvg.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanAvg.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanAvg.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanAvg.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanIn.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanIn.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanIn.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanIn.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanOut.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanOut.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanOut.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitVarScalingUniformFanOut.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavier.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavier.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavier.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavier.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavierLegacy.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavierLegacy.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavierLegacy.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavierLegacy.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavierUniform.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavierUniform.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavierUniform.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/WeightInitXavierUniform.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/ArrayEmbeddingInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/ArrayEmbeddingInitializer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/ArrayEmbeddingInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/ArrayEmbeddingInitializer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/EmbeddingInitializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/EmbeddingInitializer.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/EmbeddingInitializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/EmbeddingInitializer.java index ccad6edbf..afae83c80 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/EmbeddingInitializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/EmbeddingInitializer.java @@ -21,7 +21,7 @@ package org.deeplearning4j.nn.weights.embeddings; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.io.Serializable; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/WeightInitEmbedding.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/WeightInitEmbedding.java similarity index 96% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/WeightInitEmbedding.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/WeightInitEmbedding.java index 889443510..6e92b2187 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/WeightInitEmbedding.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/weights/embeddings/WeightInitEmbedding.java @@ -25,8 +25,8 @@ import lombok.NonNull; import org.deeplearning4j.nn.weights.IWeightInit; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties("nonSerializableInit") @EqualsAndHashCode diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/workspace/ArrayType.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/workspace/ArrayType.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/workspace/ArrayType.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/workspace/ArrayType.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/workspace/LayerWorkspaceMgr.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/workspace/LayerWorkspaceMgr.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/workspace/LayerWorkspaceMgr.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/workspace/LayerWorkspaceMgr.java index 23a801ea7..bf8126ed5 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/workspace/LayerWorkspaceMgr.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/nn/workspace/LayerWorkspaceMgr.java @@ -20,7 +20,7 @@ package org.deeplearning4j.nn.workspace; -import org.nd4j.shade.guava.base.Preconditions; +import com.google.common.base.Preconditions; import lombok.Getter; import lombok.NonNull; import lombok.Setter; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/Solver.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/Solver.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/Solver.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/Solver.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/BaseTrainingListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/BaseTrainingListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/BaseTrainingListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/BaseTrainingListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/ConvexOptimizer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/ConvexOptimizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/ConvexOptimizer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/ConvexOptimizer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/InvocationType.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/InvocationType.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/InvocationType.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/InvocationType.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/IterationListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/IterationListener.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/IterationListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/IterationListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/LineOptimizer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/LineOptimizer.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/LineOptimizer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/LineOptimizer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/StepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/StepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/StepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/StepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/TrainingListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/TrainingListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/api/TrainingListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/api/TrainingListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/Checkpoint.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/Checkpoint.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/Checkpoint.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/Checkpoint.java diff --git a/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/CheckpointListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/CheckpointListener.java new file mode 100644 index 000000000..5871b99a0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/CheckpointListener.java @@ -0,0 +1,652 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.optimize.listeners; + +import com.google.common.io.Files; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.optimize.api.BaseTrainingListener; +import org.deeplearning4j.util.ModelSerializer; +import org.nd4j.common.base.Preconditions; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.*; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class CheckpointListener extends BaseTrainingListener implements Serializable { + + private enum KeepMode {ALL, LAST, LAST_AND_EVERY}; + private static final String[] MODEL_TYPES = new String[]{"MultiLayerNetwork", "ComputationGraph", "Model"}; + + private File rootDir; + private KeepMode keepMode; + private int keepLast; + private int keepEvery; + private boolean logSaving; + private boolean deleteExisting; + + private Integer saveEveryNEpochs; + private Integer saveEveryNIterations; + private boolean saveEveryNIterSinceLast; + private Long saveEveryAmount; + private TimeUnit saveEveryUnit; + private Long saveEveryMs; + private boolean saveEverySinceLast; + + private int lastCheckpointNum = -1; + private File checkpointRecordFile; + + private Checkpoint lastCheckpoint; + private long startTime = -1; + private int startIter = -1; + private Long lastSaveEveryMsNoSinceLast; + + private CheckpointListener(Builder builder){ + this.rootDir = builder.rootDir; + this.keepMode = builder.keepMode; + this.keepLast = builder.keepLast; + this.keepEvery = builder.keepEvery; + this.logSaving = builder.logSaving; + this.deleteExisting = builder.deleteExisting; + + this.saveEveryNEpochs = builder.saveEveryNEpochs; + this.saveEveryNIterations = builder.saveEveryNIterations; + this.saveEveryNIterSinceLast = builder.saveEveryNIterSinceLast; + this.saveEveryAmount = builder.saveEveryAmount; + this.saveEveryUnit = builder.saveEveryUnit; + this.saveEverySinceLast = builder.saveEverySinceLast; + + if(saveEveryAmount != null){ + saveEveryMs = TimeUnit.MILLISECONDS.convert(saveEveryAmount, saveEveryUnit); + } + + this.checkpointRecordFile = new File(rootDir, "checkpointInfo.txt"); + if(this.checkpointRecordFile.exists() && this.checkpointRecordFile.length() > 0){ + + if(deleteExisting){ + //Delete any files matching: + //"checkpoint_" + checkpointNum + "_" + modelType + ".zip"; + this.checkpointRecordFile.delete(); + File[] files = rootDir.listFiles(); + if(files != null && files.length > 0){ + for(File f : files){ + String name = f.getName(); + if(name.startsWith("checkpoint_") && (name.endsWith("MultiLayerNetwork.zip") || name.endsWith("ComputationGraph.zip"))){ + f.delete(); + } + } + } + } else { + throw new IllegalStateException("Detected existing checkpoint files at directory " + rootDir.getAbsolutePath() + + ". Use deleteExisting(true) to delete existing checkpoint files when present."); + } + } + } + + @Override + public void onEpochEnd(Model model) { + int epochsDone = getEpoch(model) + 1; + if(saveEveryNEpochs != null && epochsDone > 0 && epochsDone % saveEveryNEpochs == 0){ + //Save: + saveCheckpoint(model); + } + //General saving conditions: don't need to check here - will check in iterationDone + } + + @Override + public void iterationDone(Model model, int iteration, int epoch) { + if (startTime < 0) { + startTime = System.currentTimeMillis(); + startIter = iteration; + return; + } + + //Check iterations saving condition: + if(saveEveryNIterations != null){ + if(saveEveryNIterSinceLast){ + //Consider last saved model when deciding whether to save + long lastSaveIter = (lastCheckpoint != null ? lastCheckpoint.getIteration() : startIter); + if(iteration - lastSaveIter >= saveEveryNIterations){ + saveCheckpoint(model); + return; + } + } else { + //Same every N iterations, regardless of saving time + if(iteration > 0 && iteration % saveEveryNIterations == 0){ + saveCheckpoint(model); + return; + } + } + } + + //Check time saving condition: + long time = System.currentTimeMillis(); + if(saveEveryUnit != null){ + if(saveEverySinceLast){ + //Consider last saved when deciding whether to save + long lastSaveTime = (lastCheckpoint != null ? lastCheckpoint.getTimestamp() : startTime); + if((time - lastSaveTime) >= saveEveryMs){ + saveCheckpoint(model); + return; + } + } else { + //Save periodically, regardless of when last model was saved + long lastSave = (lastSaveEveryMsNoSinceLast != null ? lastSaveEveryMsNoSinceLast : startTime); + if((time - lastSave) > saveEveryMs){ + saveCheckpoint(model); + lastSaveEveryMsNoSinceLast = time; + return; + } + } + } + } + + private void saveCheckpoint(Model model) { + try{ + saveCheckpointHelper(model); + } catch (Exception e){ + throw new RuntimeException("Error saving checkpoint", e); + } + } + + private void saveCheckpointHelper(Model model) throws Exception { + if(!checkpointRecordFile.exists()){ + checkpointRecordFile.createNewFile(); + write(Checkpoint.getFileHeader() + "\n", checkpointRecordFile); + } + + Checkpoint c = new Checkpoint(++lastCheckpointNum, System.currentTimeMillis(), getIter(model), getEpoch(model), + getModelType(model), null); + setFileName(c); + + ModelSerializer.writeModel(model, new File(rootDir, c.getFilename()), true); + + String s = c.toFileString(); + write(s + "\n", checkpointRecordFile); + + if(logSaving){ + log.info("Model checkpoint saved: epoch {}, iteration {}, path: {}", c.getEpoch(), c.getIteration(), + new File(rootDir, c.getFilename()).getPath() ); + } + this.lastCheckpoint = c; + + + //Finally: determine if we should delete some old models... + if(keepMode == null || keepMode == KeepMode.ALL){ + return; + } else if(keepMode == KeepMode.LAST){ + List checkpoints = availableCheckpoints(); + Iterator iter = checkpoints.iterator(); + while(checkpoints.size() > keepLast){ + Checkpoint toRemove = iter.next(); + File f = getFileForCheckpoint(toRemove); + f.delete(); + iter.remove(); + } + } else { + //Keep mode: last N and every M + for(Checkpoint cp : availableCheckpoints()){ + if(cp.getCheckpointNum() > 0 && (cp.getCheckpointNum()+1) % keepEvery == 0){ + //One of the "every M to keep" models + continue; + } else if(cp.getCheckpointNum() > lastCheckpointNum - keepLast ){ //Example: latest is 5, keep last 2 -> keep checkpoints 4 and 5 + //One of last N to keep + continue; + } + //Otherwise: delete file + File f = getFileForCheckpoint(cp); + f.delete(); + } + } + } + + private static void setFileName(Checkpoint c){ + String filename = getFileName(c.getCheckpointNum(), c.getModelType()); + c.setFilename(filename); + } + + private static String getFileName(int checkpointNum, String modelType){ + return "checkpoint_" + checkpointNum + "_" + modelType + ".zip"; + } + + private static String write(String str, File f){ + try { + if(!f.exists()){ + f.createNewFile(); + } + Files.append(str, f, Charset.defaultCharset()); + } catch (IOException e){ + throw new RuntimeException(e); + } + return str; + } + + protected static int getIter(Model model) { + if (model instanceof MultiLayerNetwork) { + return ((MultiLayerNetwork) model).getLayerWiseConfigurations().getIterationCount(); + } else if (model instanceof ComputationGraph) { + return ((ComputationGraph) model).getConfiguration().getIterationCount(); + } else { + return model.conf().getIterationCount(); + } + } + + protected static int getEpoch(Model model) { + if (model instanceof MultiLayerNetwork) { + return ((MultiLayerNetwork) model).getLayerWiseConfigurations().getEpochCount(); + } else if (model instanceof ComputationGraph) { + return ((ComputationGraph) model).getConfiguration().getEpochCount(); + } else { + return model.conf().getEpochCount(); + } + } + + protected static String getModelType(Model model){ + if(model.getClass() == MultiLayerNetwork.class){ + return "MultiLayerNetwork"; + } else if(model.getClass() == ComputationGraph.class){ + return "ComputationGraph"; + } else { + return "Model"; + } + } + + /** + * List all available checkpoints. A checkpoint is 'available' if the file can be loaded. Any checkpoint files that + * have been automatically deleted (given the configuration) will not be returned here. + * + * @return List of checkpoint files that can be loaded + */ + public List availableCheckpoints(){ + if(!checkpointRecordFile.exists()){ + return Collections.emptyList(); + } + + return availableCheckpoints(rootDir); + } + + /** + * List all available checkpoints. A checkpoint is 'available' if the file can be loaded. Any checkpoint files that + * have been automatically deleted (given the configuration) will not be returned here. + * Note that the checkpointInfo.txt file must exist, as this stores checkpoint information + * + * @return List of checkpoint files that can be loaded from the specified directory + */ + public static List availableCheckpoints(File directory){ + File checkpointRecordFile = new File(directory, "checkpointInfo.txt"); + Preconditions.checkState(checkpointRecordFile.exists(), "Could not find checkpoint record file at expected path %s", checkpointRecordFile.getAbsolutePath()); + + List lines; + try(InputStream is = new BufferedInputStream(new FileInputStream(checkpointRecordFile))){ + lines = IOUtils.readLines(is); + } catch (IOException e){ + throw new RuntimeException("Error loading checkpoint data from file: " + checkpointRecordFile.getAbsolutePath(), e); + } + + List out = new ArrayList<>(lines.size()-1); //Assume first line is header + for( int i=1; i all = availableCheckpoints(rootDir); + if(all.isEmpty()){ + return null; + } + return all.get(all.size()-1); + } + + /** + * Get the model file for the given checkpoint. Checkpoint model file must exist + * + * @param checkpoint Checkpoint to get the model file for + * @return Model file for the checkpoint + */ + public File getFileForCheckpoint(Checkpoint checkpoint){ + return getFileForCheckpoint(checkpoint.getCheckpointNum()); + } + + /** + * Get the model file for the given checkpoint number. Checkpoint model file must exist + * + * @param checkpointNum Checkpoint number to get the model file for + * @return Model file for the checkpoint + */ + public File getFileForCheckpoint(int checkpointNum) { + return getFileForCheckpoint(rootDir, checkpointNum); + } + + public static File getFileForCheckpoint(File rootDir, int checkpointNum){ + if(checkpointNum < 0){ + throw new IllegalArgumentException("Invalid checkpoint number: " + checkpointNum); + } + File f = null; + for(String s : MODEL_TYPES){ + f = new File(rootDir, getFileName(checkpointNum, s)); + if(f.exists()){ + return f; + } + } + throw new IllegalStateException("Model file for checkpoint " + checkpointNum + " does not exist"); + } + + /** + * Load a MultiLayerNetwork for the given checkpoint + * + * @param checkpoint Checkpoint model to load + * @return The loaded model + */ + public MultiLayerNetwork loadCheckpointMLN(Checkpoint checkpoint){ + return loadCheckpointMLN(checkpoint.getCheckpointNum()); + } + + /** + * Load a MultiLayerNetwork for the given checkpoint number + * + * @param checkpointNum Checkpoint model to load + * @return The loaded model + */ + public MultiLayerNetwork loadCheckpointMLN(int checkpointNum) { + return loadCheckpointMLN(rootDir, checkpointNum); + } + + /** + * Load a MultiLayerNetwork for the given checkpoint that resides in the specified root directory + * + * @param rootDir Root directory for the checkpoint + * @param checkpoint Checkpoint model to load + * @return The loaded model + */ + public static MultiLayerNetwork loadCheckpointMLN(File rootDir, Checkpoint checkpoint) { + return loadCheckpointMLN(rootDir, checkpoint.getCheckpointNum()); + } + + /** + * Load a MultiLayerNetwork for the given checkpoint number + * + * @param rootDir The directory that the checkpoint resides in + * @param checkpointNum Checkpoint model to load + * @return The loaded model + */ + public static MultiLayerNetwork loadCheckpointMLN(File rootDir, int checkpointNum){ + File f = getFileForCheckpoint(rootDir, checkpointNum); + try { + return ModelSerializer.restoreMultiLayerNetwork(f, true); + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + /** + * Load the last (most recent) checkpoint from the specified root directory + * @param rootDir Root directory to load checpoint from + * @return MultiLayerNetwork for last checkpoint + */ + public static MultiLayerNetwork loadLastCheckpointMLN(File rootDir){ + Checkpoint last = lastCheckpoint(rootDir); + return loadCheckpointMLN(rootDir, last); + } + + /** + * Load a ComputationGraph for the given checkpoint + * + * @param checkpoint Checkpoint model to load + * @return The loaded model + */ + public ComputationGraph loadCheckpointCG(Checkpoint checkpoint){ + return loadCheckpointCG(checkpoint.getCheckpointNum()); + } + + /** + * Load a ComputationGraph for the given checkpoint from the specified root direcotry + * + * @param checkpoint Checkpoint model to load + * @return The loaded model + */ + public static ComputationGraph loadCheckpointCG(File rootDir, Checkpoint checkpoint){ + return loadCheckpointCG(rootDir, checkpoint.getCheckpointNum()); + } + + /** + * Load a ComputationGraph for the given checkpoint + * + * @param checkpointNum Checkpoint model number to load + * @return The loaded model + */ + public ComputationGraph loadCheckpointCG(int checkpointNum) { + return loadCheckpointCG(rootDir, checkpointNum); + } + + /** + * Load a ComputationGraph for the given checkpoint that resides in the specified root directory + * + * @param rootDir Directory that the checkpoint resides in + * @param checkpointNum Checkpoint model number to load + * @return The loaded model + */ + public static ComputationGraph loadCheckpointCG(File rootDir, int checkpointNum){ + File f = getFileForCheckpoint(rootDir, checkpointNum); + try { + return ModelSerializer.restoreComputationGraph(f, true); + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + /** + * Load the last (most recent) checkpoint from the specified root directory + * @param rootDir Root directory to load checpoint from + * @return ComputationGraph for last checkpoint + */ + public static ComputationGraph loadLastCheckpointCG(File rootDir){ + Checkpoint last = lastCheckpoint(rootDir); + return loadCheckpointCG(rootDir, last); + } + + public static class Builder { + + private File rootDir; + private KeepMode keepMode; + private int keepLast; + private int keepEvery; + private boolean logSaving = true; + private boolean deleteExisting = false; + + private Integer saveEveryNEpochs; + private Integer saveEveryNIterations; + private boolean saveEveryNIterSinceLast; + private Long saveEveryAmount; + private TimeUnit saveEveryUnit; + private boolean saveEverySinceLast; + + /** + * @param rootDir Root directory to save models to + */ + public Builder(@NonNull String rootDir){ + this(new File(rootDir)); + } + + /** + * @param rootDir Root directory to save models to + */ + public Builder(@NonNull File rootDir){ + this.rootDir = rootDir; + } + + /** + * Save a model at the end of every epoch + */ + public Builder saveEveryEpoch(){ + return saveEveryNEpochs(1); + } + + /** + * Save a model at the end of every N epochs + */ + public Builder saveEveryNEpochs(int n){ + this.saveEveryNEpochs = n; + return this; + } + + /** + * Save a model every N iterations + */ + public Builder saveEveryNIterations(int n){ + return saveEveryNIterations(n, false); + } + + /** + * Save a model every N iterations (if sinceLast == false), or if N iterations have passed since + * the last model vas saved (if sinceLast == true) + */ + public Builder saveEveryNIterations(int n, boolean sinceLast){ + this.saveEveryNIterations = n; + this.saveEveryNIterSinceLast = sinceLast; + return this; + } + + /** + * Save a model periodically + * + * @param amount Quantity of the specified time unit + * @param timeUnit Time unit + */ + public Builder saveEvery(long amount, TimeUnit timeUnit){ + return saveEvery(amount, timeUnit, false); + } + + /** + * Save a model periodically (if sinceLast == false), or if the specified amount of time has elapsed since + * the last model was saved (if sinceLast == true) + * + * @param amount Quantity of the specified time unit + * @param timeUnit Time unit + */ + public Builder saveEvery(long amount, TimeUnit timeUnit, boolean sinceLast){ + this.saveEveryAmount = amount; + this.saveEveryUnit = timeUnit; + this.saveEverySinceLast = sinceLast; + return this; + } + + /** + * Keep all model checkpoints - i.e., don't delete any. Note that this is the default. + */ + public Builder keepAll(){ + this.keepMode = KeepMode.ALL; + return this; + } + + /** + * Keep only the last N most recent model checkpoint files. Older checkpoints will automatically be deleted. + * @param n Number of most recent checkpoints to keep + */ + public Builder keepLast(int n){ + if(n <= 0){ + throw new IllegalArgumentException("Number of model files to keep should be > 0 (got: " + n + ")"); + } + this.keepMode = KeepMode.LAST; + this.keepLast = n; + return this; + } + + /** + * Keep the last N most recent model checkpoint files, and every M checkpoint files.
+ * For example: suppose you save every 100 iterations, for 2050 iteration, and use keepLastAndEvery(3,5). + * This means after 2050 iterations you would have saved 20 checkpoints - some of which will be deleted. + * Those remaining in this example: iterations 500, 1000, 1500, 1800, 1900, 2000. + * @param nLast Most recent checkpoints to keep + * @param everyN Every N checkpoints to keep (regardless of age) + */ + public Builder keepLastAndEvery(int nLast, int everyN){ + if(nLast <= 0){ + throw new IllegalArgumentException("Most recent number of model files to keep should be > 0 (got: " + + nLast + ")"); + } + if(everyN <= 0){ + throw new IllegalArgumentException("Every n model files to keep should be > 0 (got: " + + everyN + ")"); + } + + this.keepMode = KeepMode.LAST_AND_EVERY; + this.keepLast = nLast; + this.keepEvery = everyN; + return this; + } + + /** + * If true (the default) log a message every time a model is saved + * + * @param logSaving Whether checkpoint saves should be logged or not + */ + public Builder logSaving(boolean logSaving){ + this.logSaving = logSaving; + return this; + } + + /** + * If the checkpoint listener is set to save to a non-empty directory, should the CheckpointListener-related + * content be deleted?
+ * This is disabled by default (and instead, an exception will be thrown if existing data is found)
+ * WARNING: Be careful when enabling this, as it deletes all saved checkpoint models in the specified directory! + */ + public Builder deleteExisting(boolean deleteExisting){ + this.deleteExisting = deleteExisting; + return this; + } + + public CheckpointListener build(){ + if(saveEveryNEpochs == null && saveEveryAmount == null && saveEveryNIterations == null){ + throw new IllegalStateException("Cannot construct listener: no models will be saved (must use at least" + + " one of: save every N epochs, every N iterations, or every T time periods)"); + } + + return new CheckpointListener(this); + } + } +} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/CollectScoresIterationListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/CollectScoresIterationListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/CollectScoresIterationListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/CollectScoresIterationListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/CollectScoresListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/CollectScoresListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/CollectScoresListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/CollectScoresListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/ComposableIterationListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/ComposableIterationListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/ComposableIterationListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/ComposableIterationListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/EvaluativeListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/EvaluativeListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/EvaluativeListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/EvaluativeListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/FailureTestingListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/FailureTestingListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/FailureTestingListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/FailureTestingListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/PerformanceListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/PerformanceListener.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/PerformanceListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/PerformanceListener.java index ccf625658..23d0e81fe 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/PerformanceListener.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/PerformanceListener.java @@ -20,7 +20,7 @@ package org.deeplearning4j.optimize.listeners; -import org.nd4j.shade.guava.base.Preconditions; +import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.nn.api.Model; import org.deeplearning4j.nn.graph.ComputationGraph; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/ScoreIterationListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/ScoreIterationListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/ScoreIterationListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/ScoreIterationListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/SharedGradient.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/SharedGradient.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/SharedGradient.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/SharedGradient.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/SleepyTrainingListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/SleepyTrainingListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/SleepyTrainingListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/SleepyTrainingListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/TimeIterationListener.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/TimeIterationListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/TimeIterationListener.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/TimeIterationListener.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/callbacks/EvaluationCallback.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/callbacks/EvaluationCallback.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/callbacks/EvaluationCallback.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/callbacks/EvaluationCallback.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/callbacks/ModelSavingCallback.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/callbacks/ModelSavingCallback.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/callbacks/ModelSavingCallback.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/listeners/callbacks/ModelSavingCallback.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/BackTrackLineSearch.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/BackTrackLineSearch.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/BackTrackLineSearch.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/BackTrackLineSearch.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/BaseOptimizer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/BaseOptimizer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/BaseOptimizer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/BaseOptimizer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/ConjugateGradient.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/ConjugateGradient.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/ConjugateGradient.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/ConjugateGradient.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/LBFGS.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/LBFGS.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/LBFGS.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/LBFGS.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/LineGradientDescent.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/LineGradientDescent.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/LineGradientDescent.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/LineGradientDescent.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/StochasticGradientDescent.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/StochasticGradientDescent.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/StochasticGradientDescent.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/StochasticGradientDescent.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/BasicGradientsAccumulator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/BasicGradientsAccumulator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/BasicGradientsAccumulator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/BasicGradientsAccumulator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodedGradientsAccumulator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodedGradientsAccumulator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodedGradientsAccumulator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodedGradientsAccumulator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodingHandler.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodingHandler.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodingHandler.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodingHandler.java index 66c37b8fc..fc3e9a9e0 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodingHandler.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/EncodingHandler.java @@ -21,7 +21,7 @@ package org.deeplearning4j.optimize.solvers.accumulation; import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.shade.guava.util.concurrent.AtomicDouble; +import com.google.common.util.concurrent.AtomicDouble; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.optimize.solvers.accumulation.encoding.ResidualPostProcessor; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/FancyBlockingQueue.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/FancyBlockingQueue.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/FancyBlockingQueue.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/FancyBlockingQueue.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/GradientsAccumulator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/GradientsAccumulator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/GradientsAccumulator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/GradientsAccumulator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/IndexedTail.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/IndexedTail.java similarity index 98% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/IndexedTail.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/IndexedTail.java index 23e89fea7..cc16c78d7 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/IndexedTail.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/IndexedTail.java @@ -24,7 +24,6 @@ import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; -import lombok.var; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -183,7 +182,7 @@ public class IndexedTail { * @return */ public boolean hasAnything(long threadId) { - var threadPosition = getLocalPosition(threadId); + long threadPosition = getLocalPosition(threadId); val r = threadPosition < updatesCounter.get(); log.trace("hasAnything({}): {}; position: {}; updates: {}", threadId, r, threadPosition, updatesCounter.get()); @@ -218,7 +217,7 @@ public class IndexedTail { } protected long getLocalPosition(long threadId) { - var threadPosition = positions.get(threadId); + AtomicLong threadPosition = positions.get(threadId); // will be instantiated on first call from any given thread if (threadPosition == null) { @@ -230,7 +229,7 @@ public class IndexedTail { } public boolean drainTo(long threadId, @NonNull INDArray array) { - var threadPosition = positions.get(threadId); + AtomicLong threadPosition = positions.get(threadId); // will be instantiated on first call from any given thread if (threadPosition == null) { diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/LocalHandler.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/LocalHandler.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/LocalHandler.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/LocalHandler.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/MessageHandler.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/MessageHandler.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/MessageHandler.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/MessageHandler.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/Registerable.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/Registerable.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/Registerable.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/Registerable.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/SmartFancyBlockingQueue.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/SmartFancyBlockingQueue.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/SmartFancyBlockingQueue.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/SmartFancyBlockingQueue.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ResidualPostProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ResidualPostProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ResidualPostProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ResidualPostProcessor.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ThresholdAlgorithm.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ThresholdAlgorithm.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ThresholdAlgorithm.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ThresholdAlgorithm.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ThresholdAlgorithmReducer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ThresholdAlgorithmReducer.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ThresholdAlgorithmReducer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/ThresholdAlgorithmReducer.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/residual/NoOpResidualPostProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/residual/NoOpResidualPostProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/residual/NoOpResidualPostProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/residual/NoOpResidualPostProcessor.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/residual/ResidualClippingPostProcessor.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/residual/ResidualClippingPostProcessor.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/residual/ResidualClippingPostProcessor.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/residual/ResidualClippingPostProcessor.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/AdaptiveThresholdAlgorithm.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/AdaptiveThresholdAlgorithm.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/AdaptiveThresholdAlgorithm.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/AdaptiveThresholdAlgorithm.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/FixedThresholdAlgorithm.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/FixedThresholdAlgorithm.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/FixedThresholdAlgorithm.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/FixedThresholdAlgorithm.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/TargetSparsityThresholdAlgorithm.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/TargetSparsityThresholdAlgorithm.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/TargetSparsityThresholdAlgorithm.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/solvers/accumulation/encoding/threshold/TargetSparsityThresholdAlgorithm.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/DefaultStepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/DefaultStepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/DefaultStepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/DefaultStepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/GradientStepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/GradientStepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/GradientStepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/GradientStepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/NegativeDefaultStepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/NegativeDefaultStepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/NegativeDefaultStepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/NegativeDefaultStepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/NegativeGradientStepFunction.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/NegativeGradientStepFunction.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/NegativeGradientStepFunction.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/NegativeGradientStepFunction.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/StepFunctions.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/StepFunctions.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/StepFunctions.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/optimize/stepfunctions/StepFunctions.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/CapsuleUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/CapsuleUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/CapsuleUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/CapsuleUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/Convolution1DUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/Convolution1DUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/Convolution1DUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/Convolution1DUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/Convolution3DUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/Convolution3DUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/Convolution3DUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/Convolution3DUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/ConvolutionUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/ConvolutionUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/ConvolutionUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/ConvolutionUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/CrashReportingUtil.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/CrashReportingUtil.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/CrashReportingUtil.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/CrashReportingUtil.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/DL4JModelValidator.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/DL4JModelValidator.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/DL4JModelValidator.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/DL4JModelValidator.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/MaskedReductionUtil.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/MaskedReductionUtil.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/MaskedReductionUtil.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/MaskedReductionUtil.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/ModelSerializer.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/ModelSerializer.java similarity index 99% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/ModelSerializer.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/ModelSerializer.java index 573b3fe89..ae7e2e2df 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/ModelSerializer.java +++ b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/ModelSerializer.java @@ -22,7 +22,7 @@ package org.deeplearning4j.util; import org.apache.commons.io.input.CloseShieldInputStream; import org.deeplearning4j.common.util.DL4JFileUtils; -import org.nd4j.shade.guava.io.Files; +import com.google.common.io.Files; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/NetworkUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/NetworkUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/NetworkUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/NetworkUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/OutputLayerUtil.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/OutputLayerUtil.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/OutputLayerUtil.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/OutputLayerUtil.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/TimeSeriesUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/TimeSeriesUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/TimeSeriesUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/TimeSeriesUtils.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/ValidationUtils.java b/cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/ValidationUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/ValidationUtils.java rename to cavis-dnn/cavis-dnn-nn/src/main/java/org/deeplearning4j/util/ValidationUtils.java diff --git a/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/build.gradle b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/build.gradle new file mode 100644 index 000000000..db7afeb4d --- /dev/null +++ b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/build.gradle @@ -0,0 +1,43 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation "io.reactivex.rxjava2:rxjava:2.2.21" + implementation projects.cavisDnn.cavisDnnParallelwrapper + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerClient + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode + implementation projects.cavisNd4j.cavisNd4jAeron + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators + + implementation 'org.scala-lang:scala-library' + implementation "io.aeron:aeron-all:1.32.0" + implementation "org.slf4j:slf4j-api" + + testImplementation 'ch.qos.logback:logback-classic' + testImplementation projects.cavisDnn.cavisDnnCommonTests + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/main/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerTrainer.java b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/main/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerTrainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/main/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerTrainer.java rename to cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/main/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerTrainer.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/main/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerTrainerContext.java b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/main/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerTrainerContext.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/main/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerTrainerContext.java rename to cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/main/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerTrainerContext.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java similarity index 94% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java rename to cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java index bee9bb710..d92cdf753 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/java/org/deeplearning4j/parallelism/parameterserver/ParameterServerParallelWrapperTest.java @@ -33,21 +33,13 @@ import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.parallelism.ParallelWrapper; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.learning.config.Nesterovs; import org.nd4j.linalg.lossfunctions.LossFunctions; @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) public class ParameterServerParallelWrapperTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/resources/aeron.properties b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/resources/aeron.properties similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/resources/aeron.properties rename to cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/resources/aeron.properties diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/resources/log4j.properties b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/resources/log4j.properties similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/resources/log4j.properties rename to cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/resources/log4j.properties diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/resources/logback.xml b/cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/resources/logback.xml similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/src/test/resources/logback.xml rename to cavis-dnn/cavis-dnn-parallelwrapper-parameterserver/src/test/resources/logback.xml diff --git a/cavis-dnn/cavis-dnn-parallelwrapper/build.gradle b/cavis-dnn/cavis-dnn-parallelwrapper/build.gradle new file mode 100644 index 000000000..c039ab783 --- /dev/null +++ b/cavis-dnn/cavis-dnn-parallelwrapper/build.gradle @@ -0,0 +1,43 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation "com.beust:jcommander:1.27" + implementation 'org.slf4j:slf4j-api' + implementation "com.google.guava:guava" + + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerClient + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators + + testImplementation projects.cavisUi.cavisUiStandalone + + + testImplementation projects.cavisDnn.cavisDnnCommonTests + testImplementation projects.cavisUi.cavisUiModel + testImplementation projects.cavisUi.cavisUiVertx +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/EarlyStoppingParallelTrainer.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/EarlyStoppingParallelTrainer.java similarity index 99% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/EarlyStoppingParallelTrainer.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/EarlyStoppingParallelTrainer.java index 2cfa01f36..683db198a 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/EarlyStoppingParallelTrainer.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/EarlyStoppingParallelTrainer.java @@ -20,7 +20,7 @@ package org.deeplearning4j.parallelism; -import org.nd4j.shade.guava.util.concurrent.AtomicDouble; +import com.google.common.util.concurrent.AtomicDouble; import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; import org.deeplearning4j.earlystopping.EarlyStoppingResult; diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/InplaceParallelInference.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/InplaceParallelInference.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/InplaceParallelInference.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/InplaceParallelInference.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelInference.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelInference.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelInference.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelInference.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelWrapper.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelWrapper.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelWrapper.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/ParallelWrapper.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContext.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContext.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContext.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContext.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContext.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContext.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContext.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContext.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/TrainerContext.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/TrainerContext.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/TrainerContext.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/factory/TrainerContext.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/InferenceMode.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/InferenceMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/InferenceMode.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/InferenceMode.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/InferenceObservable.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/InferenceObservable.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/InferenceObservable.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/InferenceObservable.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/LoadBalanceMode.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/LoadBalanceMode.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/LoadBalanceMode.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/LoadBalanceMode.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObservable.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObservable.java similarity index 98% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObservable.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObservable.java index 9200a1353..1211e1719 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObservable.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObservable.java @@ -20,7 +20,7 @@ package org.deeplearning4j.parallelism.inference.observers; -import org.nd4j.shade.guava.base.Preconditions; +import com.google.common.base.Preconditions; import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObserver.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObserver.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObserver.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BasicInferenceObserver.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservable.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservable.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservable.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservable.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/DataSetIteratorProviderFactory.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/DataSetIteratorProviderFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/DataSetIteratorProviderFactory.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/DataSetIteratorProviderFactory.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/MultiDataSetProviderFactory.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/MultiDataSetProviderFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/MultiDataSetProviderFactory.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/MultiDataSetProviderFactory.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/ParallelWrapperMain.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/ParallelWrapperMain.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/ParallelWrapperMain.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/main/ParallelWrapperMain.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/CommunicativeTrainer.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/CommunicativeTrainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/CommunicativeTrainer.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/CommunicativeTrainer.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/DefaultTrainer.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/DefaultTrainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/DefaultTrainer.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/DefaultTrainer.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/SymmetricTrainer.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/SymmetricTrainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/SymmetricTrainer.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/SymmetricTrainer.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/Trainer.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/Trainer.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/Trainer.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/main/java/org/deeplearning4j/parallelism/trainer/Trainer.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java similarity index 96% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java index 7e35c4cd2..a8db019b4 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/InplaceParallelInferenceTest.java @@ -27,19 +27,13 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.graph.ComputationGraph; import org.deeplearning4j.parallelism.inference.InferenceMode; import org.deeplearning4j.parallelism.inference.LoadBalanceMode; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) + public class InplaceParallelInferenceTest extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java similarity index 97% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java index f5a3eaaeb..6f694286d 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelInferenceTest.java @@ -32,9 +32,11 @@ import org.deeplearning4j.nn.conf.MultiLayerConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.graph.ComputationGraph; -import org.junit.jupiter.api.*; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.linalg.activations.Activation; import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; import org.deeplearning4j.eval.Evaluation; @@ -62,16 +64,12 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) +@Timeout(30) public class ParallelInferenceTest extends BaseDL4JTest { private static MultiLayerNetwork model; private static DataSetIterator iterator; - @BeforeEach public void setUp() throws Exception { if (model == null) { @@ -87,8 +85,7 @@ public class ParallelInferenceTest extends BaseDL4JTest { iterator.reset(); } - @Test() - @Timeout(30000) + @Test public void testInferenceSequential1() throws Exception { long count0 = 0; @@ -133,8 +130,7 @@ public class ParallelInferenceTest extends BaseDL4JTest { assertTrue(count1 > 0L); } - @Test() - @Timeout(30000) + @Test public void testInferenceSequential2() throws Exception { long count0 = 0; @@ -179,8 +175,7 @@ public class ParallelInferenceTest extends BaseDL4JTest { } - @Test() - @Timeout(30000) + @Test public void testInferenceBatched1() throws Exception { long count0 = 0; long count1 = 0; @@ -412,8 +407,7 @@ public class ParallelInferenceTest extends BaseDL4JTest { } - @Test() - @Timeout(120000) + @Test public void testParallelInferenceVariableLengthTS() throws Exception { Nd4j.getRandom().setSeed(12345); @@ -459,8 +453,7 @@ public class ParallelInferenceTest extends BaseDL4JTest { } } - @Test() - @Timeout(120000) + @Test public void testParallelInferenceVariableLengthTS2() throws Exception { Nd4j.getRandom().setSeed(12345); @@ -515,8 +508,8 @@ public class ParallelInferenceTest extends BaseDL4JTest { } - @Test() - @Timeout(30000) + + @Test public void testParallelInferenceVariableSizeCNN() throws Exception { //Variable size input for CNN model - for example, YOLO models //In these cases, we can't batch and have to execute the different size inputs separately @@ -571,8 +564,8 @@ public class ParallelInferenceTest extends BaseDL4JTest { } } - @Test() - @Timeout(30000) + + @Test public void testParallelInferenceVariableSizeCNN2() throws Exception { //Variable size input for CNN model - for example, YOLO models //In these cases, we can't batch and have to execute the different size inputs separately @@ -626,8 +619,7 @@ public class ParallelInferenceTest extends BaseDL4JTest { } } - @Test() - @Timeout(20000) + @Test public void testParallelInferenceErrorPropagation(){ int nIn = 10; @@ -761,8 +753,7 @@ public class ParallelInferenceTest extends BaseDL4JTest { } } - @Test() - @Timeout(20000) + @Test public void testModelUpdate_1() throws Exception { int nIn = 5; @@ -800,7 +791,7 @@ public class ParallelInferenceTest extends BaseDL4JTest { // model can be null for some of the workers yet, due to race condition if (m != null) { Thread.sleep(500); - assertEquals(net.params(), m.params(), "Failed at model [" + cnt0 + "]"); + assertEquals( net.params(), m.params(), "Failed at model [" + cnt0 + "]"); passed = true; } cnt0++; @@ -827,15 +818,14 @@ public class ParallelInferenceTest extends BaseDL4JTest { cnt0 = 0; for (val m:modelsAfter) { - assertNotNull(m,"Failed at model [" + cnt0 + "]"); - assertEquals(net2.params(), m.params(), "Failed at model [" + cnt0++ + "]"); + assertNotNull( m, "Failed at model [" + cnt0 + "]"); + assertEquals( net2.params(), m.params(), "Failed at model [" + cnt0++ + "]"); } inf.shutdown(); } - @Test() - @Timeout(120000) + @Test public void testMultiOutputNet() throws Exception { int nIn = 5; diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelWrapperTest.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelWrapperTest.java similarity index 96% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelWrapperTest.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelWrapperTest.java index 512eba0df..458b9dab1 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelWrapperTest.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/ParallelWrapperTest.java @@ -35,10 +35,7 @@ import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -50,10 +47,7 @@ import org.slf4j.LoggerFactory; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) + public class ParallelWrapperTest extends BaseDL4JTest { private static final Logger log = LoggerFactory.getLogger(ParallelWrapperTest.class); @@ -143,7 +137,7 @@ public class ParallelWrapperTest extends BaseDL4JTest { mnistTest.reset(); double acc = eval.accuracy(); - assertTrue(acc > 0.5, String.valueOf(acc)); + assertTrue( acc > 0.5, String.valueOf(acc)); wrapper.shutdown(); } diff --git a/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestListeners.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestListeners.java new file mode 100644 index 000000000..eb3ccfef8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestListeners.java @@ -0,0 +1,239 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.parallelism; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.core.storage.StatsStorage; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.listener.RoutingIterationListener; +import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.optimize.api.BaseTrainingListener; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.ui.model.stats.StatsListener; +import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestListeners extends BaseDL4JTest { + + @Test + public void testListeners() { + TestListener.clearCounts(); + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(0, + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10) + .activation(Activation.TANH).build()); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + testListenersForModel(model, Collections.singletonList(new TestListener())); + } + + @Test + public void testListenersGraph() { + TestListener.clearCounts(); + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder() + .addInputs("in").addLayer("0", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10) + .activation(Activation.TANH).build(), + "in") + .setOutputs("0").build(); + + ComputationGraph model = new ComputationGraph(conf); + model.init(); + + testListenersForModel(model, Collections.singletonList(new TestListener())); + } + + @Test + public void testListenersViaModel() { + TestListener.clearCounts(); + + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(0, + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10) + .activation(Activation.TANH).build()); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + StatsStorage ss = new InMemoryStatsStorage(); + model.setListeners(new TestListener(), new StatsListener(ss)); + + testListenersForModel(model, null); + + assertEquals(1, ss.listSessionIDs().size()); + assertEquals(2, ss.listWorkerIDsForSession(ss.listSessionIDs().get(0)).size()); + } + + @Test + public void testListenersViaModelGraph() { + TestListener.clearCounts(); + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder() + .addInputs("in").addLayer("0", + new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10) + .activation(Activation.TANH).build(), + "in") + .setOutputs("0").build(); + + ComputationGraph model = new ComputationGraph(conf); + model.init(); + + StatsStorage ss = new InMemoryStatsStorage(); + model.setListeners(new TestListener(), new StatsListener(ss)); + + testListenersForModel(model, null); + + assertEquals(1, ss.listSessionIDs().size()); + assertEquals(2, ss.listWorkerIDsForSession(ss.listSessionIDs().get(0)).size()); + } + + private static void testListenersForModel(Model model, List listeners) { + + int nWorkers = 2; + ParallelWrapper wrapper = new ParallelWrapper.Builder(model).workers(nWorkers).averagingFrequency(1) + .reportScoreAfterAveraging(true).build(); + + if (listeners != null) { + wrapper.setListeners(listeners); + } + + List data = new ArrayList<>(); + for (int i = 0; i < nWorkers; i++) { + data.add(new DataSet(Nd4j.rand(1, 10), Nd4j.rand(1, 10))); + } + + DataSetIterator iter = new ExistingDataSetIterator(data); + + TestListener.clearCounts(); + wrapper.fit(iter); + + assertEquals(2, TestListener.workerIDs.size()); + assertEquals(1, TestListener.sessionIDs.size()); + assertEquals(2, TestListener.forwardPassCount.get()); + assertEquals(2, TestListener.backwardPassCount.get()); + } + + + private static class TestListener extends BaseTrainingListener implements RoutingIterationListener { + + private static final AtomicInteger forwardPassCount = new AtomicInteger(); + private static final AtomicInteger backwardPassCount = new AtomicInteger(); + private static final AtomicInteger instanceCount = new AtomicInteger(); + private static final Set workerIDs = Collections.newSetFromMap(new ConcurrentHashMap()); + private static final Set sessionIDs = Collections.newSetFromMap(new ConcurrentHashMap()); + + public static void clearCounts() { + forwardPassCount.set(0); + backwardPassCount.set(0); + instanceCount.set(0); + workerIDs.clear(); + sessionIDs.clear(); + } + + public TestListener() { + instanceCount.incrementAndGet(); + } + + @Override + public void onEpochStart(Model model) {} + + @Override + public void onEpochEnd(Model model) {} + + @Override + public void onForwardPass(Model model, List activations) { + forwardPassCount.incrementAndGet(); + } + + @Override + public void onForwardPass(Model model, Map activations) { + forwardPassCount.incrementAndGet(); + } + + @Override + public void onGradientCalculation(Model model) {} + + @Override + public void onBackwardPass(Model model) { + backwardPassCount.getAndIncrement(); + } + + @Override + public void setStorageRouter(StatsStorageRouter router) {} + + @Override + public StatsStorageRouter getStorageRouter() { + return null; + } + + @Override + public void setWorkerID(String workerID) { + workerIDs.add(workerID); + } + + @Override + public String getWorkerID() { + return null; + } + + @Override + public void setSessionID(String sessionID) { + sessionIDs.add(sessionID); + } + + @Override + public String getSessionID() { + return "session_id"; + } + + @Override + public RoutingIterationListener clone() { + return new TestListener(); + } + + @Override + public void iterationDone(Model model, int iteration, int epoch) {} + } + +} diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStopping.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStopping.java similarity index 97% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStopping.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStopping.java index 2972f8b60..2eaf2e850 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStopping.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStopping.java @@ -38,10 +38,7 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; @@ -51,10 +48,7 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) + public class TestParallelEarlyStopping extends BaseDL4JTest { // parallel training results vary wildly with expected result diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStoppingUI.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStoppingUI.java similarity index 94% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStoppingUI.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStoppingUI.java index b84537968..7bea67ef6 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStoppingUI.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestParallelEarlyStoppingUI.java @@ -40,25 +40,19 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.ui.api.UIServer; import org.deeplearning4j.ui.model.stats.StatsListener; import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.learning.config.Sgd; import org.nd4j.linalg.lossfunctions.LossFunctions; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) + public class TestParallelEarlyStoppingUI extends BaseDL4JTest { @Test - @Disabled //To be run manually + //@Ignore //To be run manually public void testParallelStatsListenerCompatibility() throws Exception { UIServer uiServer = UIServer.getInstance(); diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContextTest.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContextTest.java similarity index 96% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContextTest.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContextTest.java index 826068285..3a85b4b34 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContextTest.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/DefaultTrainerContextTest.java @@ -34,17 +34,13 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.parallelism.ParallelWrapper; import org.deeplearning4j.parallelism.trainer.SymmetricTrainer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.learning.config.Nesterovs; import org.nd4j.linalg.lossfunctions.LossFunctions; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag + public class DefaultTrainerContextTest extends BaseDL4JTest { int nChannels = 1; int outputNum = 10; diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContextTest.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContextTest.java similarity index 96% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContextTest.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContextTest.java index 159494805..ec82896df 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContextTest.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/factory/SymmetricTrainerContextTest.java @@ -34,17 +34,13 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.parallelism.ParallelWrapper; import org.deeplearning4j.parallelism.trainer.SymmetricTrainer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.learning.config.Nesterovs; import org.nd4j.linalg.lossfunctions.LossFunctions; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@NativeTag + public class SymmetricTrainerContextTest extends BaseDL4JTest { int nChannels = 1; int outputNum = 10; diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservableTest.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservableTest.java similarity index 97% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservableTest.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservableTest.java index 74f9ddcab..1a49fa3b1 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservableTest.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/inference/observers/BatchedInferenceObservableTest.java @@ -24,10 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.BaseDL4JTest; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -41,8 +38,6 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag public class BatchedInferenceObservableTest extends BaseDL4JTest { @BeforeEach public void setUp() throws Exception {} diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/MnistDataSetIteratorProviderFactory.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/MnistDataSetIteratorProviderFactory.java similarity index 100% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/MnistDataSetIteratorProviderFactory.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/MnistDataSetIteratorProviderFactory.java diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/ParallelWrapperMainTest.java b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/ParallelWrapperMainTest.java similarity index 89% rename from deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/ParallelWrapperMainTest.java rename to cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/ParallelWrapperMainTest.java index ccdc8a1a0..bf525ac67 100644 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/ParallelWrapperMainTest.java +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/main/ParallelWrapperMainTest.java @@ -34,33 +34,24 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.util.ModelSerializer; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.learning.config.Nesterovs; import org.nd4j.linalg.lossfunctions.LossFunctions; import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; @Slf4j -@Disabled("Permissions issues on CI") -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) public class ParallelWrapperMainTest extends BaseDL4JTest { + @TempDir + public File testDir; @Test - public void runParallelWrapperMain(@TempDir Path testDir) throws Exception { + public void runParallelWrapperMain() throws Exception { int nChannels = 1; int outputNum = 10; @@ -97,10 +88,10 @@ public class ParallelWrapperMainTest extends BaseDL4JTest { MultiLayerConfiguration conf = builder.build(); MultiLayerNetwork model = new MultiLayerNetwork(conf); model.init(); - File tempModel = Files.createTempFile(testDir,"tmpmodel","zip").toFile(); + File tempModel = new File(testDir, "tmpmodel.zip"); tempModel.deleteOnExit(); ModelSerializer.writeModel(model, tempModel, false); - File tmp = Files.createTempFile(testDir,"tmpmodel","bin").toFile(); + File tmp = new File(testDir, "tmpmodel.bin"); tmp.deleteOnExit(); ParallelWrapperMain parallelWrapperMain = new ParallelWrapperMain(); try { diff --git a/cavis-dnn/cavis-dnn-parallelwrapper/src/test/resources/junit-platform.properties b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/resources/junit-platform.properties new file mode 100644 index 000000000..863ded8ac --- /dev/null +++ b/cavis-dnn/cavis-dnn-parallelwrapper/src/test/resources/junit-platform.properties @@ -0,0 +1,37 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + +# configuration parameter to configure when timeouts are applied. There are three modes. +#enabled, disabled, disabled_on_debug +junit.jupiter.execution.timeout.mode=enabled + +#Default timeout for all testable and lifecycle methods. +junit.jupiter.execution.timeout.default=60 s + +#junit.jupiter.execution.timeout.testable.method.default – Default timeout for all testable methods. +#junit.jupiter.execution.timeout.test.method.default – Default timeout for @Test methods. +#junit.jupiter.execution.timeout.testtemplate.method.default – Default timeout for @TestTemplate methods. +#junit.jupiter.execution.timeout.testfactory.method.default – Default timeout for @TestFactory methods. +#junit.jupiter.execution.timeout.lifecycle.method.default – Default timeout for all lifecycle methods. +#junit.jupiter.execution.timeout.beforeall.method.default – Default timeout for @BeforeAll methods. +#junit.jupiter.execution.timeout.beforeeach.method.default – Default timeout for @BeforeEach methods. +#junit.jupiter.execution.timeout.afterall.method.default – Default timeout for @AfterAll methods. +#junit.jupiter.execution.timeout.aftereach.method.default – Default timeout for @AfterEach methods. \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/build.gradle b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/build.gradle new file mode 100644 index 000000000..fc7f4e7a8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/build.gradle @@ -0,0 +1,32 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +ext { + buildTarget = rootProject.ext.buildTarget +} + +dependencies { + implementation "org.slf4j:slf4j-api" + implementation "commons-io:commons-io" + implementation 'org.json:json:20190722' + implementation group:"org.bytedeco", name:"cpython" + implementation group:"org.bytedeco", name:"cpython", classifier: buildTarget + testImplementation "com.google.code.findbugs:jsr305:3.0.2" +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/Python.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/Python.java new file mode 100644 index 000000000..5f9cffd8b --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/Python.java @@ -0,0 +1,615 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + + +package org.nd4j.python4j; + +import org.bytedeco.cpython.PyObject; + +import java.util.Collections; +import java.util.List; + +import static org.bytedeco.cpython.global.python.*; + + +public class Python { + + static { + new PythonExecutioner(); + } + + /** + * Imports a python module, similar to python import statement. + * + * @param moduleName name of the module to be imported + * @return reference to the module object + */ + public static PythonObject importModule(String moduleName) { + PythonGIL.assertThreadSafe(); + PythonObject module = new PythonObject(PyImport_ImportModule(moduleName)); + if (module.isNone()) { + throw new PythonException("Error importing module: " + moduleName); + } + return module; + } + + /** + * Gets a builtins attribute + * + * @param attrName Attribute name + * @return + */ + public static PythonObject attr(String attrName) { + PythonGIL.assertThreadSafe(); + PyObject builtins = PyImport_ImportModule("builtins"); + try { + return new PythonObject(PyObject_GetAttrString(builtins, attrName)); + } finally { + Py_DecRef(builtins); + } + } + + + /** + * Gets the size of a PythonObject. similar to len() in python. + * + * @param pythonObject + * @return + */ + public static PythonObject len(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + long n = PyObject_Size(pythonObject.getNativePythonObject()); + if (n < 0) { + throw new PythonException("Object has no length: " + pythonObject); + } + return PythonTypes.INT.toPython(n); + } + + /** + * Gets the string representation of an object. + * + * @param pythonObject + * @return + */ + public static PythonObject str(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + try { + return PythonTypes.STR.toPython(pythonObject.toString()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + + } + + /** + * Returns an empty string + * + * @return + */ + public static PythonObject str() { + PythonGIL.assertThreadSafe(); + try { + return PythonTypes.STR.toPython(""); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Returns the str type object + * @return + */ + public static PythonObject strType() { + return attr("str"); + } + + /** + * Returns a floating point number from a number or a string. + * @param pythonObject + * @return + */ + public static PythonObject float_(PythonObject pythonObject) { + return PythonTypes.FLOAT.toPython(PythonTypes.FLOAT.toJava(pythonObject)); + } + + /** + * Reutrns 0. + * @return + */ + public static PythonObject float_() { + try { + return PythonTypes.FLOAT.toPython(0d); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + /** + * Returns the float type object + * @return + */ + public static PythonObject floatType() { + return attr("float"); + } + + + /** + * Converts a value to a Boolean value i.e., True or False, using the standard truth testing procedure. + * @param pythonObject + * @return + */ + public static PythonObject bool(PythonObject pythonObject) { + return PythonTypes.BOOL.toPython(PythonTypes.BOOL.toJava(pythonObject)); + + } + + /** + * Returns False. + * @return + */ + public static PythonObject bool() { + return PythonTypes.BOOL.toPython(false); + + } + + /** + * Returns the bool type object + * @return + */ + public static PythonObject boolType() { + return attr("bool"); + } + + /** + * Returns an integer from a number or a string. + * @param pythonObject + * @return + */ + public static PythonObject int_(PythonObject pythonObject) { + return PythonTypes.INT.toPython(PythonTypes.INT.toJava(pythonObject)); + } + + /** + * Returns 0 + * @return + */ + public static PythonObject int_() { + return PythonTypes.INT.toPython(0L); + + } + + /** + * Returns the int type object + * @return + */ + public static PythonObject intType() { + return attr("int"); + } + + /** + * Takes sequence types and converts them to lists. + * @param pythonObject + * @return + */ + public static PythonObject list(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + try (PythonGC pgc = PythonGC.watch()) { + PythonObject listF = attr("list"); + PythonObject ret = listF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Object is not iterable: " + pythonObject.toString()); + } + return ret; + } + } + + /** + * Returns empty list. + * @return + */ + public static PythonObject list() { + return PythonTypes.LIST.toPython(Collections.emptyList()); + } + + /** + * Returns list type object. + * @return + */ + public static PythonObject listType() { + return attr("list"); + } + + /** + * Creates a dictionary. + * @param pythonObject + * @return + */ + public static PythonObject dict(PythonObject pythonObject) { + PythonObject dictF = attr("dict"); + PythonObject ret = dictF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build dict from object: " + pythonObject.toString()); + } + dictF.del(); + return ret; + } + + /** + * Returns empty dict + * @return + */ + public static PythonObject dict() { + return PythonTypes.DICT.toPython(Collections.emptyMap()); + } + + /** + * Returns dict type object. + * @return + */ + public static PythonObject dictType() { + return attr("dict"); + } + + /** + * Creates a set. + * @param pythonObject + * @return + */ + public static PythonObject set(PythonObject pythonObject) { + PythonObject setF = attr("set"); + PythonObject ret = setF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build set from object: " + pythonObject.toString()); + } + setF.del(); + return ret; + } + + /** + * Returns empty set. + * @return + */ + public static PythonObject set() { + PythonObject setF = attr("set"); + PythonObject ret; + ret = setF.call(); + setF.del(); + return ret; + } + + /** + * Returns empty set. + * @return + */ + public static PythonObject setType() { + return attr("set"); + } + + /** + * Creates a bytearray. + * @param pythonObject + * @return + */ + public static PythonObject bytearray(PythonObject pythonObject) { + PythonObject baF = attr("bytearray"); + PythonObject ret = baF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build bytearray from object: " + pythonObject.toString()); + } + baF.del(); + return ret; + } + + /** + * Returns empty bytearray. + * @return + */ + public static PythonObject bytearray() { + PythonObject baF = attr("bytearray"); + PythonObject ret; + ret = baF.call(); + baF.del(); + return ret; + } + + /** + * Returns bytearray type object + * @return + */ + public static PythonObject bytearrayType() { + return attr("bytearray"); + } + + /** + * Creates a memoryview. + * @param pythonObject + * @return + */ + public static PythonObject memoryview(PythonObject pythonObject) { + PythonObject mvF = attr("memoryview"); + PythonObject ret = mvF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build memoryview from object: " + pythonObject.toString()); + } + mvF.del(); + return ret; + } + + /** + * Returns memoryview type object. + * @return + */ + public static PythonObject memoryviewType() { + return attr("memoryview"); + } + + /** + * Creates a byte string. + * @param pythonObject + * @return + */ + public static PythonObject bytes(PythonObject pythonObject) { + PythonObject bytesF = attr("bytes"); + PythonObject ret = bytesF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build bytes from object: " + pythonObject.toString()); + } + bytesF.del(); + return ret; + } + + /** + * Returns empty byte string. + * @return + */ + public static PythonObject bytes() { + PythonObject bytesF = attr("bytes"); + PythonObject ret; + ret = bytesF.call(); + bytesF.del(); + return ret; + } + + /** + * Returns bytes type object + * @return + */ + public static PythonObject bytesType() { + return attr("bytes"); + } + + /** + * Creates a tuple. + * @param pythonObject + * @return + */ + public static PythonObject tuple(PythonObject pythonObject) { + PythonObject tupleF = attr("tupleF"); + PythonObject ret = tupleF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build tuple from object: " + pythonObject.toString()); + } + tupleF.del(); + return ret; + } + + /** + * Returns empty tuple. + * @return + */ + public static PythonObject tuple() { + PythonObject tupleF = attr("tuple"); + PythonObject ret; + ret = tupleF.call(); + tupleF.del(); + return ret; + } + + /** + * Returns tuple type object + * @return + */ + public static PythonObject tupleType() { + return attr("tuple"); + } + + /** + * Creates an Exception + * @param pythonObject + * @return + */ + public static PythonObject Exception(PythonObject pythonObject) { + PythonObject excF = attr("Exception"); + PythonObject ret = excF.call(pythonObject); + excF.del(); + return ret; + } + + /** + * Creates an Exception + * @return + */ + public static PythonObject Exception() { + PythonObject excF = attr("Exception"); + PythonObject ret; + ret = excF.call(); + excF.del(); + return ret; + } + + /** + * Returns Exception type object + * @return + */ + public static PythonObject ExceptionType() { + return attr("Exception"); + } + + + /** + * Returns the globals dictionary. + * @return + */ + public static PythonObject globals() { + PythonGIL.assertThreadSafe(); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + Py_DecRef(main); + return new PythonObject(globals, false); + } + + /** + * Returns the type of an object. + * @param pythonObject + * @return + */ + public static PythonObject type(PythonObject pythonObject) { + PythonObject typeF = attr("type"); + PythonObject ret = typeF.call(pythonObject); + typeF.del(); + return ret; + } + + /** + * Returns True if the specified object is of the specified type, otherwise False. + * @param obj + * @param type + * @return + */ + public static boolean isinstance(PythonObject obj, PythonObject... type) { + PythonGIL.assertThreadSafe(); + PyObject argsTuple = PyTuple_New(type.length); + try { + for (int i = 0; i < type.length; i++) { + PythonObject x = type[i]; + Py_IncRef(x.getNativePythonObject()); + PyTuple_SetItem(argsTuple, i, x.getNativePythonObject()); + } + return PyObject_IsInstance(obj.getNativePythonObject(), argsTuple) != 0; + } finally { + Py_DecRef(argsTuple); + } + + } + + /** + * Evaluates the specified expression. + * @param expression + * @return + */ + public static PythonObject eval(String expression) { + + PythonGIL.assertThreadSafe(); + PyObject compiledCode = Py_CompileString(expression, "", Py_eval_input); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + PyObject locals = PyDict_New(); + try { + return new PythonObject(PyEval_EvalCode(compiledCode, globals, locals)); + } finally { + Py_DecRef(main); + Py_DecRef(locals); + Py_DecRef(compiledCode); + } + + } + + /** + * Returns the builtins module + * @return + */ + public static PythonObject builtins() { + return importModule("builtins"); + + } + + /** + * Returns None. + * @return + */ + public static PythonObject None() { + return eval("None"); + } + + /** + * Returns True. + * @return + */ + public static PythonObject True() { + return eval("True"); + } + + /** + * Returns False. + * @return + */ + public static PythonObject False() { + return eval("False"); + } + + /** + * Returns True if the object passed is callable callable, otherwise False. + * @param pythonObject + * @return + */ + public static boolean callable(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + return PyCallable_Check(pythonObject.getNativePythonObject()) == 1; + } + + + public static void setContext(String context){ + PythonContextManager.setContext(context); + } + + public static String getCurrentContext() { + return PythonContextManager.getCurrentContext(); + } + + public static void deleteContext(String context){ + PythonContextManager.deleteContext(context); + } + public static void resetContext() { + PythonContextManager.reset(); + } + + /** + * Executes a string of code. + * @param code + * @throws PythonException + */ + public static void exec(String code) throws PythonException { + PythonExecutioner.exec(code); + } + + /** + * Executes a string of code. + * @param code + * @param inputs + * @param outputs + */ + public static void exec(String code, List inputs, List outputs){ + PythonExecutioner.exec(code, inputs, outputs); + } + + +} diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonContextManager.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonContextManager.java new file mode 100644 index 000000000..4a4ac3aaa --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonContextManager.java @@ -0,0 +1,271 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.python4j; + + +import java.io.Closeable; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; + +public class PythonContextManager { + + private static Set contexts = new HashSet<>(); + private static AtomicBoolean init = new AtomicBoolean(false); + private static String currentContext; + private static final String MAIN_CONTEXT = "main"; + private static final String COLLAPSED_KEY = "__collapsed__"; + + static { + init(); + } + + + public static class Context implements Closeable{ + private final String name; + private final String previous; + private final boolean temp; + public Context(){ + name = "temp_" + UUID.randomUUID().toString().replace("-", "_"); + temp = true; + previous = getCurrentContext(); + setContext(name); + } + public Context(String name){ + this.name = name; + temp = false; + previous = getCurrentContext(); + setContext(name); + } + + @Override + public void close(){ + setContext(previous); + if (temp) deleteContext(name); + } + } + + private static void init() { + if (init.get()) return; + new PythonExecutioner(); + init.set(true); + currentContext = MAIN_CONTEXT; + contexts.add(currentContext); + } + + + /** + * Adds a new context. + * @param contextName + */ + public static void addContext(String contextName) { + if (!validateContextName(contextName)) { + throw new PythonException("Invalid context name: " + contextName); + } + contexts.add(contextName); + } + + /** + * Returns true if context exists, else false. + * @param contextName + * @return + */ + public static boolean hasContext(String contextName) { + return contexts.contains(contextName); + } + + private static boolean validateContextName(String s) { + for (int i=0; i= '0' && c <= '9'){ + return false; + } + } + if (!(c=='_' || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'))){ + return false; + } + } + return true; + } + + private static String getContextPrefix(String contextName) { + return COLLAPSED_KEY + contextName + "__"; + } + + private static String getCollapsedVarNameForContext(String varName, String contextName) { + return getContextPrefix(contextName) + varName; + } + + private static String expandCollapsedVarName(String varName, String contextName) { + String prefix = COLLAPSED_KEY + contextName + "__"; + return varName.substring(prefix.length()); + + } + + private static void collapseContext(String contextName) { + try (PythonGC pgc = PythonGC.watch()) { + PythonObject globals = Python.globals(); + PythonObject pop = globals.attr("pop"); + PythonObject keysF = globals.attr("keys"); + PythonObject keys = keysF.call(); + PythonObject keysList = Python.list(keys); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (!((keyStr.startsWith("__") && keyStr.endsWith("__")) || keyStr.startsWith("__collapsed_"))) { + String collapsedKey = getCollapsedVarNameForContext(keyStr, contextName); + PythonObject val = pop.call(key); + + PythonObject pyNewKey = new PythonObject(collapsedKey); + globals.set(pyNewKey, val); + } + } + } catch (Exception pe) { + throw new RuntimeException(pe); + } + } + + private static void expandContext(String contextName) { + try (PythonGC pgc = PythonGC.watch()) { + String prefix = getContextPrefix(contextName); + PythonObject globals = Python.globals(); + PythonObject pop = globals.attr("pop"); + PythonObject keysF = globals.attr("keys"); + + PythonObject keys = keysF.call(); + + PythonObject keysList = Python.list(keys); + try (PythonGC pgc2 = PythonGC.pause()) { + int numKeys = Python.len(keysList).toInt(); + + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (keyStr.startsWith(prefix)) { + String expandedKey = expandCollapsedVarName(keyStr, contextName); + PythonObject val = pop.call(key); + PythonObject newKey = new PythonObject(expandedKey); + globals.set(newKey, val); + } + } + } + } + } + + + /** + * Activates the specified context + * @param contextName + */ + public static void setContext(String contextName) { + if (contextName.equals(currentContext)) { + return; + } + if (!hasContext(contextName)) { + addContext(contextName); + } + + + collapseContext(currentContext); + + expandContext(contextName); + currentContext = contextName; + + } + + /** + * Activates the main context + */ + public static void setMainContext() { + setContext(MAIN_CONTEXT); + + } + + /** + * Returns the current context's name. + * @return + */ + public static String getCurrentContext() { + return currentContext; + } + + /** + * Resets the current context. + */ + public static void reset() { + String tempContext = "___temp__context___"; + String currContext = currentContext; + setContext(tempContext); + deleteContext(currContext); + setContext(currContext); + deleteContext(tempContext); + } + + /** + * Deletes the specified context. + * @param contextName + */ + public static void deleteContext(String contextName) { + if (contextName.equals(currentContext)) { + throw new PythonException("Cannot delete current context!"); + } + if (!contexts.contains(contextName)) { + return; + } + String prefix = getContextPrefix(contextName); + PythonObject globals = Python.globals(); + PythonObject keysList = Python.list(globals.attr("keys").call()); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (keyStr.startsWith(prefix)) { + globals.attr("__delitem__").call(key); + } + } + contexts.remove(contextName); + } + + /** + * Deletes all contexts except the main context. + */ + public static void deleteNonMainContexts() { + setContext(MAIN_CONTEXT); // will never fail + for (String c : contexts.toArray(new String[0])) { + if (!c.equals(MAIN_CONTEXT)) { + deleteContext(c); // will never fail + } + } + + } + + /** + * Returns the names of all contexts. + * @return + */ + public String[] getContexts() { + return contexts.toArray(new String[0]); + } + +} diff --git a/python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonException.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonException.java similarity index 100% rename from python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonException.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonException.java diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonExecutioner.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonExecutioner.java new file mode 100644 index 000000000..40131a237 --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonExecutioner.java @@ -0,0 +1,354 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + + +package org.nd4j.python4j; + +import org.bytedeco.cpython.PyObject; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.io.IOUtils; +import org.bytedeco.cpython.global.python; + +import static org.bytedeco.cpython.global.python.*; +import static org.bytedeco.cpython.helper.python.Py_SetPath; + + +public class PythonExecutioner { + private final static String PYTHON_EXCEPTION_KEY = "__python_exception__"; + private static AtomicBoolean init = new AtomicBoolean(false); + public final static String DEFAULT_PYTHON_PATH_PROPERTY = "org.eclipse.python4j.path"; + public final static String JAVACPP_PYTHON_APPEND_TYPE = "org.eclipse.python4j.path.append"; + public final static String DEFAULT_APPEND_TYPE = "before"; + + static { + init(); + } + + private static synchronized void init() { + if (init.get()) { + return; + } + + init.set(true); + initPythonPath(); + PyEval_InitThreads(); + Py_InitializeEx(0); + for (PythonType type: PythonTypes.get()) { + type.init(); + } + + //set the main thread state for the gil + PythonGIL.setMainThreadState(); + PyEval_SaveThread(); + + } + + /** + * Sets a variable. + * + * @param name + * @param value + */ + public static void setVariable(String name, PythonObject value) { + PythonGIL.assertThreadSafe(); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + PyDict_SetItemString(globals, name, value.getNativePythonObject()); + Py_DecRef(main); + + } + + /** + * Sets given list of PythonVariables in the interpreter. + * + * @param pyVars + */ + public static void setVariables(List pyVars) { + for (PythonVariable pyVar : pyVars) + setVariable(pyVar.getName(), pyVar.getPythonObject()); + } + + /** + * Sets given list of PythonVariables in the interpreter. + * + * @param pyVars + */ + public static void setVariables(PythonVariable... pyVars) { + setVariables(Arrays.asList(pyVars)); + } + + /** + * Gets the given list of PythonVariables from the interpreter. + * + * @param pyVars + */ + public static void getVariables(List pyVars) { + for (PythonVariable pyVar : pyVars) + pyVar.setValue(getVariable(pyVar.getName(), pyVar.getType()).getValue()); + } + + /** + * Gets the given list of PythonVariables from the interpreter. + * + * @param pyVars + */ + public static void getVariables(PythonVariable... pyVars) { + getVariables(Arrays.asList(pyVars)); + } + + + + /** + * Gets the variable with the given name from the interpreter. + * + * @param name + * @return + */ + public static PythonObject getVariable(String name) { + PythonGIL.assertThreadSafe(); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + PyObject pyName = PyUnicode_FromString(name); + try { + if (PyDict_Contains(globals, pyName) == 1) { + return new PythonObject(PyObject_GetItem(globals, pyName), false); + } + } finally { + Py_DecRef(main); + //Py_DecRef(globals); + Py_DecRef(pyName); + } + return new PythonObject(null); + } + + /** + * Gets the variable with the given name from the interpreter. + * + * @param name + * @return + */ + public static PythonVariable getVariable(String name, PythonType type) { + PythonObject val = getVariable(name); + return new PythonVariable<>(name, type, type.toJava(val)); + } + + /** + * Executes a string of code + * + * @param code + */ + public static synchronized void simpleExec(String code) { + PythonGIL.assertThreadSafe(); + + int result = PyRun_SimpleStringFlags(code, null); + if (result != 0) { + throw new PythonException("Execution failed, unable to retrieve python exception."); + } + } + + private static void throwIfExecutionFailed() { + PythonObject ex = getVariable(PYTHON_EXCEPTION_KEY); + if (ex != null && !ex.isNone() && !ex.toString().isEmpty()) { + setVariable(PYTHON_EXCEPTION_KEY, PythonTypes.STR.toPython("")); + throw new PythonException(ex); + } + } + + + private static String getWrappedCode(String code) { + + try (InputStream is = PythonExecutioner.class + .getResourceAsStream("pythonexec/pythonexec.py")) { + String base = IOUtils.toString(is, StandardCharsets.UTF_8); + String indentedCode = " " + code.replace("\n", "\n "); + String out = base.replace(" pass", indentedCode); + return out; + } catch (IOException e) { + throw new IllegalStateException("Unable to read python code!", e); + } + + } + + /** + * Executes a string of code. Throws PythonException if execution fails. + * + * @param code + */ + public static void exec(String code) { + simpleExec(getWrappedCode(code)); + throwIfExecutionFailed(); + } + + public static void exec(String code, List inputs, List outputs) { + if (inputs != null) { + setVariables(inputs.toArray(new PythonVariable[0])); + } + exec(code); + if (outputs != null) { + getVariables(outputs.toArray(new PythonVariable[0])); + } + } + + /** + * Return list of all supported variables in the interpreter. + * + * @return + */ + public static PythonVariables getAllVariables() { + PythonGIL.assertThreadSafe(); + PythonVariables ret = new PythonVariables(); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + PyObject keys = PyDict_Keys(globals); + PyObject keysIter = PyObject_GetIter(keys); + try { + + long n = PyObject_Size(globals); + for (int i = 0; i < n; i++) { + PyObject pyKey = PyIter_Next(keysIter); + try { + if (!new PythonObject(pyKey, false).toString().startsWith("_")) { + + PyObject pyVal = PyObject_GetItem(globals, pyKey); // TODO check ref count + PythonType pt; + try { + pt = PythonTypes.getPythonTypeForPythonObject(new PythonObject(pyVal, false)); + + } catch (PythonException pe) { + pt = null; + } + if (pt != null) { + ret.add( + new PythonVariable<>( + new PythonObject(pyKey, false).toString(), + pt, + pt.toJava(new PythonObject(pyVal, false)) + ) + ); + } + } + } finally { + Py_DecRef(pyKey); + } + } + } finally { + Py_DecRef(keysIter); + Py_DecRef(keys); + Py_DecRef(main); + return ret; + } + + } + + + /** + * Executes a string of code and returns a list of all supported variables. + * + * @param code + * @param inputs + * @return + */ + public static PythonVariables execAndReturnAllVariables(String code, List inputs) { + setVariables(inputs); + simpleExec(getWrappedCode(code)); + return getAllVariables(); + } + + /** + * Executes a string of code and returns a list of all supported variables. + * + * @param code + * @return + */ + public static PythonVariables execAndReturnAllVariables(String code) { + simpleExec(getWrappedCode(code)); + return getAllVariables(); + } + + private static synchronized void initPythonPath() { + try { + String path = System.getProperty(DEFAULT_PYTHON_PATH_PROPERTY); + + List packagesList = new ArrayList<>(); + packagesList.addAll(Arrays.asList(cachePackages())); + for (PythonType type: PythonTypes.get()){ + packagesList.addAll(Arrays.asList(type.packages())); + } + //// TODO: fix in javacpp + packagesList.add(new File(python.cachePackage(), "site-packages")); + + File[] packages = packagesList.toArray(new File[0]); + + if (path == null) { + Py_SetPath(packages); + } else { + StringBuffer sb = new StringBuffer(); + + JavaCppPathType pathAppendValue = JavaCppPathType.valueOf(System.getProperty(JAVACPP_PYTHON_APPEND_TYPE, DEFAULT_APPEND_TYPE).toUpperCase()); + switch (pathAppendValue) { + case BEFORE: + for (File cacheDir : packages) { + sb.append(cacheDir); + sb.append(java.io.File.pathSeparator); + } + + sb.append(path); + break; + case AFTER: + sb.append(path); + + for (File cacheDir : packages) { + sb.append(cacheDir); + sb.append(java.io.File.pathSeparator); + } + break; + case NONE: + sb.append(path); + break; + } + + Py_SetPath(sb.toString()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private enum JavaCppPathType { + BEFORE, AFTER, NONE + } + + private static File[] cachePackages() throws IOException { + File[] path = org.bytedeco.cpython.global.python.cachePackages(); + path = Arrays.copyOf(path, path.length + 1); + path[path.length - 1] = cachePackage(); + return path; + } + +} \ No newline at end of file diff --git a/python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonGC.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonGC.java similarity index 100% rename from python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonGC.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonGC.java diff --git a/python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonGIL.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonGIL.java similarity index 100% rename from python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonGIL.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonGIL.java diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonObject.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonObject.java new file mode 100644 index 000000000..bd0893a72 --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonObject.java @@ -0,0 +1,296 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.python4j; + + +import org.bytedeco.cpython.PyObject; +import org.bytedeco.javacpp.Pointer; + +import java.util.*; + +import static org.bytedeco.cpython.global.python.*; + +public class PythonObject { + + static { + new PythonExecutioner(); + } + + private boolean owned = true; + private PyObject nativePythonObject; + + + public PythonObject(PyObject nativePythonObject, boolean owned) { + PythonGIL.assertThreadSafe(); + this.nativePythonObject = nativePythonObject; + this.owned = owned; + if (owned && nativePythonObject != null) { + PythonGC.register(this); + } + } + + public PythonObject(PyObject nativePythonObject) { + PythonGIL.assertThreadSafe(); + this.nativePythonObject = nativePythonObject; + if (nativePythonObject != null) { + PythonGC.register(this); + } + + } + + public PyObject getNativePythonObject() { + return nativePythonObject; + } + + public String toString() { + return PythonTypes.STR.toJava(this); + + } + + public boolean isNone() { + if (nativePythonObject == null || Pointer.isNull(nativePythonObject)) { + return true; + } + try (PythonGC pgc = PythonGC.pause()) { + PythonObject type = Python.type(this); + boolean ret = Python.type(this).toString().equals("") && toString().equals("None"); + Py_DecRef(type.nativePythonObject); + return ret; + } + } + + public void del() { + PythonGIL.assertThreadSafe(); + if (owned && nativePythonObject != null && !PythonGC.isWatching()) { + Py_DecRef(nativePythonObject); + nativePythonObject = null; + } + } + + public PythonObject callWithArgs(PythonObject args) { + return callWithArgsAndKwargs(args, null); + } + + public PythonObject callWithKwargs(PythonObject kwargs) { + if (!Python.callable(this)) { + throw new PythonException("Object is not callable: " + toString()); + } + PyObject tuple = PyTuple_New(0); + PyObject dict = kwargs.nativePythonObject; + if (PyObject_IsInstance(dict, new PyObject(PyDict_Type())) != 1) { + throw new PythonException("Expected kwargs to be dict. Received: " + kwargs.toString()); + } + PythonObject ret = new PythonObject(PyObject_Call(nativePythonObject, tuple, dict)); + Py_DecRef(tuple); + return ret; + } + + public PythonObject callWithArgsAndKwargs(PythonObject args, PythonObject kwargs) { + PythonGIL.assertThreadSafe(); + PyObject tuple = null; + boolean ownsTuple = false; + try { + if (!Python.callable(this)) { + throw new PythonException("Object is not callable: " + toString()); + } + + if (PyObject_IsInstance(args.nativePythonObject, new PyObject(PyTuple_Type())) == 1) { + tuple = args.nativePythonObject; + } else if (PyObject_IsInstance(args.nativePythonObject, new PyObject(PyList_Type())) == 1) { + tuple = PyList_AsTuple(args.nativePythonObject); + ownsTuple = true; + } else { + throw new PythonException("Expected args to be tuple or list. Received: " + args.toString()); + } + if (kwargs != null && PyObject_IsInstance(kwargs.nativePythonObject, new PyObject(PyDict_Type())) != 1) { + throw new PythonException("Expected kwargs to be dict. Received: " + kwargs.toString()); + } + return new PythonObject(PyObject_Call(nativePythonObject, tuple, kwargs == null ? null : kwargs.nativePythonObject)); + } finally { + if (ownsTuple) Py_DecRef(tuple); + } + + } + + + public PythonObject call(Object... args) { + return callWithArgsAndKwargs(Arrays.asList(args), null); + } + + public PythonObject callWithArgs(List args) { + return call(args, null); + } + + public PythonObject callWithKwargs(Map kwargs) { + return call(null, kwargs); + } + + public PythonObject callWithArgsAndKwargs(List args, Map kwargs) { + PythonGIL.assertThreadSafe(); + try (PythonGC pgc = PythonGC.watch()) { + if (!Python.callable(this)) { + throw new PythonException("Object is not callable: " + toString()); + } + PythonObject pyArgs; + PythonObject pyKwargs; + + if (args == null || args.isEmpty()) { + pyArgs = new PythonObject(PyTuple_New(0)); + } else { + PythonObject argsList = PythonTypes.convert(args); + pyArgs = new PythonObject(PyList_AsTuple(argsList.getNativePythonObject())); + } + if (kwargs == null) { + pyKwargs = null; + } else { + pyKwargs = PythonTypes.convert(kwargs); + } + + PythonObject ret = new PythonObject( + PyObject_Call( + nativePythonObject, + pyArgs.nativePythonObject, + pyKwargs == null ? null : pyKwargs.nativePythonObject + ) + ); + + PythonGC.keep(ret); + + return ret; + } + + } + + + public PythonObject attr(String attrName) { + PythonGIL.assertThreadSafe(); + return new PythonObject(PyObject_GetAttrString(nativePythonObject, attrName)); + } + + + public PythonObject(Object javaObject) { + PythonGIL.assertThreadSafe(); + if (javaObject instanceof PythonObject) { + owned = false; + nativePythonObject = ((PythonObject) javaObject).nativePythonObject; + } else { + try (PythonGC pgc = PythonGC.pause()) { + nativePythonObject = PythonTypes.convert(javaObject).getNativePythonObject(); + } + PythonGC.register(this); + } + + } + + public int toInt() { + return PythonTypes.INT.toJava(this).intValue(); + } + + public long toLong() { + return PythonTypes.INT.toJava(this); + } + + public float toFloat() { + return PythonTypes.FLOAT.toJava(this).floatValue(); + } + + public double toDouble() { + return PythonTypes.FLOAT.toJava(this); + } + + public boolean toBoolean() { + return PythonTypes.BOOL.toJava(this); + + } + + public List toList() { + return PythonTypes.LIST.toJava(this); + } + + public Map toMap() { + return PythonTypes.DICT.toJava(this); + } + + public PythonObject get(int key) { + PythonGIL.assertThreadSafe(); + return new PythonObject(PyObject_GetItem(nativePythonObject, PyLong_FromLong(key))); + } + + public PythonObject get(String key) { + PythonGIL.assertThreadSafe(); + return new PythonObject(PyObject_GetItem(nativePythonObject, PyUnicode_FromString(key))); + } + + public PythonObject get(PythonObject key) { + PythonGIL.assertThreadSafe(); + return new PythonObject(PyObject_GetItem(nativePythonObject, key.nativePythonObject)); + } + + public void set(PythonObject key, PythonObject value) { + PythonGIL.assertThreadSafe(); + PyObject_SetItem(nativePythonObject, key.nativePythonObject, value.nativePythonObject); + } + + + public PythonObject abs(){ + return new PythonObject(PyNumber_Absolute(nativePythonObject)); + } + public PythonObject add(PythonObject pythonObject){ + return new PythonObject(PyNumber_Add(nativePythonObject, pythonObject.nativePythonObject)); + } + public PythonObject sub(PythonObject pythonObject){ + return new PythonObject(PyNumber_Subtract(nativePythonObject, pythonObject.nativePythonObject)); + } + public PythonObject mod(PythonObject pythonObject){ + return new PythonObject(PyNumber_Divmod(nativePythonObject, pythonObject.nativePythonObject)); + } + public PythonObject mul(PythonObject pythonObject){ + return new PythonObject(PyNumber_Multiply(nativePythonObject, pythonObject.nativePythonObject)); + } + public PythonObject trueDiv(PythonObject pythonObject){ + return new PythonObject(PyNumber_TrueDivide(nativePythonObject, pythonObject.nativePythonObject)); + } + public PythonObject floorDiv(PythonObject pythonObject){ + return new PythonObject(PyNumber_FloorDivide(nativePythonObject, pythonObject.nativePythonObject)); + } + public PythonObject matMul(PythonObject pythonObject){ + return new PythonObject(PyNumber_MatrixMultiply(nativePythonObject, pythonObject.nativePythonObject)); + } + + public void addi(PythonObject pythonObject){ + PyNumber_InPlaceAdd(nativePythonObject, pythonObject.nativePythonObject); + } + public void subi(PythonObject pythonObject){ + PyNumber_InPlaceSubtract(nativePythonObject, pythonObject.nativePythonObject); + } + public void muli(PythonObject pythonObject){ + PyNumber_InPlaceMultiply(nativePythonObject, pythonObject.nativePythonObject); + } + public void trueDivi(PythonObject pythonObject){ + PyNumber_InPlaceTrueDivide(nativePythonObject, pythonObject.nativePythonObject); + } + public void floorDivi(PythonObject pythonObject){ + PyNumber_InPlaceFloorDivide(nativePythonObject, pythonObject.nativePythonObject); + } + public void matMuli(PythonObject pythonObject){ + PyNumber_InPlaceMatrixMultiply(nativePythonObject, pythonObject.nativePythonObject); + } +} diff --git a/python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonProcess.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonProcess.java similarity index 100% rename from python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonProcess.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonProcess.java diff --git a/python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonType.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonType.java similarity index 100% rename from python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonType.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonType.java diff --git a/python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonTypes.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonTypes.java similarity index 100% rename from python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonTypes.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonTypes.java diff --git a/python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonVariable.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonVariable.java similarity index 100% rename from python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonVariable.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonVariable.java diff --git a/python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonVariables.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonVariables.java similarity index 100% rename from python4j/python4j-core/src/main/java/org/nd4j/python4j/PythonVariables.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/java/org/nd4j/python4j/PythonVariables.java diff --git a/libnd4j/include/graph/generated/nd4j/__init__.py b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/resources/org/nd4j/python4j/pythonexec/__init__.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/__init__.py rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/resources/org/nd4j/python4j/pythonexec/__init__.py diff --git a/python4j/python4j-core/src/main/resources/org/nd4j/python4j/pythonexec/pythonexec.py b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/resources/org/nd4j/python4j/pythonexec/pythonexec.py similarity index 100% rename from python4j/python4j-core/src/main/resources/org/nd4j/python4j/pythonexec/pythonexec.py rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/main/resources/org/nd4j/python4j/pythonexec/pythonexec.py diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonBasicExecutionTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonBasicExecutionTest.java new file mode 100644 index 000000000..3aeadb61a --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonBasicExecutionTest.java @@ -0,0 +1,135 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.python4j.*; + +import javax.annotation.concurrent.NotThreadSafe; +import java.util.*; + +@NotThreadSafe +public class PythonBasicExecutionTest { + + @Test + public void testSimpleExecIllegal() { + String code = "print('Hello World')"; + Assertions.assertThrows(IllegalStateException.class, () -> { + PythonExecutioner.exec(code); + }); + } + + @Test + public void testSimpleExec() { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + String code = "print('Hello World')"; + PythonExecutioner.exec(code); + } + + } + + @Test + public void testBadCode() throws Exception { + try { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + String code = "printx('Hello world')"; + PythonExecutioner.exec(code); + } + + } catch (Exception e) { + Assertions.assertEquals("NameError: name 'printx' is not defined", e.getMessage()); + return; + } + throw new Exception("Bad code did not throw!"); + } + + @Test + public void testExecWithInputs() { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("x", PythonTypes.STR, "Hello ")); + inputs.add(new PythonVariable<>("y", PythonTypes.STR, "World")); + String code = "print(x + y)"; + PythonExecutioner.exec(code, inputs, null); + } + + } + + @Test + public void testExecWithInputsAndOutputs() { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("x", PythonTypes.STR, "Hello ")); + inputs.add(new PythonVariable<>("y", PythonTypes.STR, "World")); + PythonVariable out = new PythonVariable<>("z", PythonTypes.STR); + String code = "z = x + y"; + PythonExecutioner.exec(code, inputs, Collections.singletonList(out)); + Assertions.assertEquals("Hello World", out.getValue()); + } + } + + @Test + public void testExecAndReturnAllVariables() { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + PythonContextManager.reset(); + String code = "a = 5\nb = '10'\nc = 20.0"; + List vars = PythonExecutioner.execAndReturnAllVariables(code); + + Assertions.assertEquals("a", vars.get(0).getName()); + Assertions.assertEquals(PythonTypes.INT, vars.get(0).getType()); + Assertions.assertEquals(5L, (long) vars.get(0).getValue()); + + Assertions.assertEquals("b", vars.get(1).getName()); + Assertions.assertEquals(PythonTypes.STR, vars.get(1).getType()); + Assertions.assertEquals("10", vars.get(1).getValue().toString()); + + Assertions.assertEquals("c", vars.get(2).getName()); + Assertions.assertEquals(PythonTypes.FLOAT, vars.get(2).getType()); + Assertions.assertEquals(20.0, (double) vars.get(2).getValue(), 1e-5); + + } + } + + @Test + public void testExecWithInputsAndReturnAllVariables() { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + PythonContextManager.reset(); + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 5)); + String code = "b = '10'\nc = 20.0 + a"; + List vars = PythonExecutioner.execAndReturnAllVariables(code, inputs); + + Assertions.assertEquals("a", vars.get(0).getName()); + Assertions.assertEquals(PythonTypes.INT, vars.get(0).getType()); + Assertions.assertEquals(5L, (long) vars.get(0).getValue()); + + Assertions.assertEquals("b", vars.get(1).getName()); + Assertions.assertEquals(PythonTypes.STR, vars.get(1).getType()); + Assertions.assertEquals("10", vars.get(1).getValue().toString()); + + Assertions.assertEquals("c", vars.get(2).getName()); + Assertions.assertEquals(PythonTypes.FLOAT, vars.get(2).getType()); + Assertions.assertEquals(25.0, (double) vars.get(2).getValue(), 1e-5); + + } + } + +} diff --git a/python4j/python4j-core/src/test/java/PythonCollectionsTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonCollectionsTest.java similarity index 86% rename from python4j/python4j-core/src/test/java/PythonCollectionsTest.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonCollectionsTest.java index 1a85fad4f..70c8d5b0e 100644 --- a/python4j/python4j-core/src/test/java/PythonCollectionsTest.java +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonCollectionsTest.java @@ -19,22 +19,14 @@ */ -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.python4j.*; - +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.*; -import static org.junit.jupiter.api.Assertions.assertEquals; - @javax.annotation.concurrent.NotThreadSafe -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.PYTHON) public class PythonCollectionsTest { @@ -52,7 +44,7 @@ public class PythonCollectionsTest { map.put("list2", Arrays.asList(4, "5", innerMap, false, true)); PythonObject dict = PythonTypes.convert(map); Map map2 = PythonTypes.DICT.toJava(dict); - assertEquals(map.toString(), map2.toString()); + Assertions.assertEquals(map.toString(), map2.toString()); } } @@ -71,7 +63,7 @@ public class PythonCollectionsTest { list.add(map); PythonObject dict = PythonTypes.convert(list); List list2 = PythonTypes.LIST.toJava(dict); - assertEquals(list.toString(), list2.toString()); + Assertions.assertEquals(list.toString(), list2.toString()); } } diff --git a/python4j/python4j-core/src/test/java/PythonContextManagerTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonContextManagerTest.java similarity index 78% rename from python4j/python4j-core/src/test/java/PythonContextManagerTest.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonContextManagerTest.java index 1498ecc76..b1a7fc7f4 100644 --- a/python4j/python4j-core/src/test/java/PythonContextManagerTest.java +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonContextManagerTest.java @@ -20,25 +20,16 @@ */ -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.python4j.Python; import org.nd4j.python4j.PythonContextManager; import org.nd4j.python4j.PythonExecutioner; - +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.nd4j.python4j.PythonGIL; import javax.annotation.concurrent.NotThreadSafe; -import static org.junit.jupiter.api.Assertions.assertEquals; - @NotThreadSafe -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.PYTHON) public class PythonContextManagerTest { @Test @@ -53,13 +44,13 @@ public class PythonContextManagerTest { Python.setContext("context1"); - assertEquals(1, PythonExecutioner.getVariable("a").toInt()); + Assertions.assertEquals(1, PythonExecutioner.getVariable("a").toInt()); Python.setContext("context2"); - assertEquals(2, PythonExecutioner.getVariable("a").toInt()); + Assertions.assertEquals(2, PythonExecutioner.getVariable("a").toInt()); Python.setContext("context3"); - assertEquals(3, PythonExecutioner.getVariable("a").toInt()); + Assertions.assertEquals(3, PythonExecutioner.getVariable("a").toInt()); PythonContextManager.deleteNonMainContexts(); diff --git a/python4j/python4j-core/src/test/java/PythonGCTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonGCTest.java similarity index 87% rename from python4j/python4j-core/src/test/java/PythonGCTest.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonGCTest.java index 048929664..9e853e4b6 100644 --- a/python4j/python4j-core/src/test/java/PythonGCTest.java +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonGCTest.java @@ -18,24 +18,17 @@ * ***************************************************************************** */ -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.python4j.Python; import org.nd4j.python4j.PythonGC; import org.nd4j.python4j.PythonGIL; import org.nd4j.python4j.PythonObject; - +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import javax.annotation.concurrent.NotThreadSafe; -import static org.junit.jupiter.api.Assertions.assertTrue; - @NotThreadSafe -@Tag(TagNames.FILE_IO) -@NativeTag public class PythonGCTest { @Test @@ -52,7 +45,7 @@ public class PythonGCTest { PythonObject pyObjCount2 = Python.len(getObjects.call()); long objCount2 = pyObjCount2.toLong(); long diff = objCount2 - objCount1; - assertTrue(diff > 2); + Assertions.assertTrue(diff > 2); try(PythonGC gc = PythonGC.watch()){ PythonObject pyList2 = Python.list(); pyList2.attr("append").call("a"); @@ -62,7 +55,7 @@ public class PythonGCTest { PythonObject pyObjCount3 = Python.len(getObjects.call()); long objCount3 = pyObjCount3.toLong(); diff = objCount3 - objCount2; - assertTrue(diff <= 2);// 2 objects created during function call + Assertions.assertTrue(diff <= 2);// 2 objects created during function call } } diff --git a/python4j/python4j-core/src/test/java/PythonMultiThreadTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonMultiThreadTest.java similarity index 80% rename from python4j/python4j-core/src/test/java/PythonMultiThreadTest.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonMultiThreadTest.java index a4c067740..b7bbfc201 100644 --- a/python4j/python4j-core/src/test/java/PythonMultiThreadTest.java +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonMultiThreadTest.java @@ -18,14 +18,14 @@ * ***************************************************************************** */ -import org.junit.jupiter.api.Tag; +import org.bytedeco.cpython.PyThreadState; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.python4j.*; import javax.annotation.concurrent.NotThreadSafe; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; @@ -33,35 +33,34 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.bytedeco.cpython.global.python.PyGILState_Check; +import static org.bytedeco.cpython.global.python.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @NotThreadSafe -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.PYTHON) -@Tag(TagNames.MULTI_THREADED) public class PythonMultiThreadTest { @Test public void testMultiThreading1()throws Throwable{ final List exceptions = Collections.synchronizedList(new ArrayList()); - Runnable runnable = () -> { - try(PythonGIL gil = PythonGIL.lock()){ - try(PythonGC gc = PythonGC.watch()){ - List inputs = new ArrayList<>(); - inputs.add(new PythonVariable<>("x", PythonTypes.STR, "Hello ")); - inputs.add(new PythonVariable<>("y", PythonTypes.STR, "World")); - PythonVariable out = new PythonVariable<>("z", PythonTypes.STR); - String code = "z = x + y"; - PythonExecutioner.exec(code, inputs, Collections.singletonList(out)); - assertEquals("Hello World", out.getValue()); - System.out.println(out.getValue() + " From thread " + Thread.currentThread().getId()); + Runnable runnable = new Runnable() { + @Override + public void run() { + try(PythonGIL gil = PythonGIL.lock()){ + try(PythonGC gc = PythonGC.watch()){ + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("x", PythonTypes.STR, "Hello ")); + inputs.add(new PythonVariable<>("y", PythonTypes.STR, "World")); + PythonVariable out = new PythonVariable<>("z", PythonTypes.STR); + String code = "z = x + y"; + PythonExecutioner.exec(code, inputs, Collections.singletonList(out)); + assertEquals("Hello World", out.getValue()); + System.out.println(out.getValue() + " From thread " + Thread.currentThread().getId()); + } + }catch (Throwable e){ + exceptions.add(e); } - }catch (Throwable e){ - exceptions.add(e); } }; @@ -147,7 +146,7 @@ public class PythonMultiThreadTest { public void run() { try(PythonGIL pythonGIL = PythonGIL.lock()) { System.out.println("Using thread " + Thread.currentThread().getId() + " to invoke python"); - assertTrue(PyGILState_Check() > 0,"Thread " + Thread.currentThread().getId() + " does not hold the gil."); + Assertions.assertTrue(PyGILState_Check() > 0, "Thread " + Thread.currentThread().getId() + " does not hold the gil."); PythonExecutioner.exec("import time; time.sleep(10)"); System.out.println("Finished execution on thread " + Thread.currentThread().getId()); finishedExecutionCount.incrementAndGet(); diff --git a/python4j/python4j-core/src/test/java/PythonPrimitiveTypesTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonPrimitiveTypesTest.java similarity index 82% rename from python4j/python4j-core/src/test/java/PythonPrimitiveTypesTest.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonPrimitiveTypesTest.java index 7de05de88..c1a2956e9 100644 --- a/python4j/python4j-core/src/test/java/PythonPrimitiveTypesTest.java +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-core/src/test/java/PythonPrimitiveTypesTest.java @@ -19,21 +19,13 @@ */ -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.python4j.*; - +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.PYTHON) public class PythonPrimitiveTypesTest { @Test @@ -43,12 +35,12 @@ public class PythonPrimitiveTypesTest { PythonObject p = PythonTypes.INT.toPython(j); long j2 = PythonTypes.INT.toJava(p); - assertEquals(j, j2); + Assertions.assertEquals(j, j2); PythonObject p2 = PythonTypes.convert(j); long j3 = PythonTypes.INT.toJava(p2); - assertEquals(j, j3); + Assertions.assertEquals(j, j3); } } @@ -60,12 +52,12 @@ public class PythonPrimitiveTypesTest { PythonObject p = PythonTypes.STR.toPython(s); String s2 = PythonTypes.STR.toJava(p); - assertEquals(s, s2); + Assertions.assertEquals(s, s2); PythonObject p2 = PythonTypes.convert(s); String s3 = PythonTypes.STR.toJava(p2); - assertEquals(s, s3); + Assertions.assertEquals(s, s3); } } @@ -77,12 +69,12 @@ public class PythonPrimitiveTypesTest { PythonObject p = PythonTypes.FLOAT.toPython(f); double f2 = PythonTypes.FLOAT.toJava(p); - assertEquals(f, f2, 1e-5); + Assertions.assertEquals(f, f2, 1e-5); PythonObject p2 = PythonTypes.convert(f); double f3 = PythonTypes.FLOAT.toJava(p2); - assertEquals(f, f3, 1e-5); + Assertions.assertEquals(f, f3, 1e-5); } } @@ -94,12 +86,12 @@ public class PythonPrimitiveTypesTest { PythonObject p = PythonTypes.BOOL.toPython(b); boolean b2 = PythonTypes.BOOL.toJava(p); - assertEquals(b, b2); + Assertions.assertEquals(b, b2); PythonObject p2 = PythonTypes.convert(b); boolean b3 = PythonTypes.BOOL.toJava(p2); - assertEquals(b, b3); + Assertions.assertEquals(b, b3); } } @@ -116,7 +108,7 @@ public class PythonPrimitiveTypesTest { outputs.add(new PythonVariable<>("b2", PythonTypes.BYTES)); String code = "b2=b1"; PythonExecutioner.exec(code, inputs, outputs); - assertArrayEquals(bytes, (byte[]) outputs.get(0).getValue()); + Assertions.assertArrayEquals(bytes, (byte[]) outputs.get(0).getValue()); } } @@ -132,8 +124,8 @@ public class PythonPrimitiveTypesTest { outputs.add(new PythonVariable<>("b2", PythonTypes.BYTES)); String code = "s1 = ''.join(chr(c) for c in b1)\nb2=b'def'"; PythonExecutioner.exec(code, inputs, outputs); - assertEquals("abc", outputs.get(0).getValue()); - assertArrayEquals(new byte[]{100, 101, 102}, (byte[]) outputs.get(1).getValue()); + Assertions.assertEquals("abc", outputs.get(0).getValue()); + Assertions.assertArrayEquals(new byte[]{100, 101, 102}, (byte[]) outputs.get(1).getValue()); } } diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/build.gradle b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/build.gradle new file mode 100644 index 000000000..c45d8fce9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/build.gradle @@ -0,0 +1,39 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +ext{ + buildTarget = rootProject.ext.buildTarget +} + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + + +dependencies { + implementation "org.slf4j:slf4j-api" + implementation "commons-io:commons-io" + testImplementation "com.google.code.findbugs:jsr305:3.0.2" + implementation group: "org.bytedeco", name:"numpy" + implementation group: "org.bytedeco", name:"numpy", classifier: buildTarget + implementation projects.cavisNative.cavisNativeBlas + implementation projects.cavisDnn.cavisDnnApi + testImplementation projects.cavisDnn.cavisDnnCommonTests + implementation projects.cavisDnn.cavisDnnPython4j.cavisPython4jCore +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/pom.xml b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/pom.xml new file mode 100644 index 000000000..62e1ff1ae --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/pom.xml @@ -0,0 +1,183 @@ + + + + + + 4.0.0 + + + net.brutex.ai + python4j-parent + 1.0.0-SNAPSHOT + + + python4j-numpy + + + + org.bytedeco + numpy-platform + ${numpy.javacpp.version} + + + net.brutex.ai + nd4j-native-api + ${project.version} + + + net.brutex.ai + nd4j-common-tests + ${project.version} + test + + + net.brutex.ai + python4j-core + 1.0.0-SNAPSHOT + + + + + + test-nd4j-native + + + net.brutex.ai + nd4j-native + ${project.version} + test + + + net.brutex.ai + dl4j-test-resources + 1.0.1 + test + + + + + + org.apache.maven.plugins + maven-surefire-plugin + true + + + net.brutex.ai + nd4j-native + ${project.version} + + + + + + + src/test/java + + *.java + **/*.java + **/Test*.java + **/*Test.java + **/*TestCase.java + + junit:junit + + + org.org.nd4j.linalg.cpu.nativecpu.CpuBackend + + + org.org.nd4j.linalg.cpu.nativecpu.CpuBackend + + + + " + + + + + + + + test-nd4j-cuda-11.2 + + + net.brutex.ai + nd4j-cuda-${cuda.version} + ${project.version} + test + + + net.brutex.ai + dl4j-test-resources + ${dl4j-test-resources.version} + test + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.surefire + surefire-junit47 + 2.19.1 + + + + + + src/test/java + + *.java + **/*.java + **/Test*.java + **/*Test.java + **/*TestCase.java + + junit:junit + + + org.org.nd4j.linalg.jcublas.JCublasBackend + + + org.org.nd4j.linalg.jcublas.JCublasBackend + + + + -Djava.library.path="${nd4j.basedir}/nd4j-backends/nd4j-backend-impls/nd4j-cuda/target/classes" + + + + + + + diff --git a/python4j/python4j-numpy/src/main/java/org/nd4j/python4j/NumpyArray.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/main/java/org/nd4j/python4j/NumpyArray.java similarity index 100% rename from python4j/python4j-numpy/src/main/java/org/nd4j/python4j/NumpyArray.java rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/main/java/org/nd4j/python4j/NumpyArray.java diff --git a/python4j/python4j-numpy/src/main/resources/META-INF/services/org.nd4j.python4j.PythonType b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/main/resources/META-INF/services/org.nd4j.python4j.PythonType similarity index 100% rename from python4j/python4j-numpy/src/main/resources/META-INF/services/org.nd4j.python4j.PythonType rename to cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/main/resources/META-INF/services/org.nd4j.python4j.PythonType diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyBasicTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyBasicTest.java new file mode 100644 index 000000000..2d9851977 --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyBasicTest.java @@ -0,0 +1,182 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + + +import org.nd4j.python4j.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.nativeblas.OpaqueDataBuffer; + +import javax.annotation.concurrent.NotThreadSafe; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +@NotThreadSafe +////@RunWith(Parameterized.class) +public class PythonNumpyBasicTest { + private DataType dataType; + private long[] shape; + + public PythonNumpyBasicTest(DataType dataType, long[] shape, String dummyArg) { + this.dataType = dataType; + this.shape = shape; + } + + ////@Parameterized.Parameters(name = "{index}: Testing with DataType={0}, shape={2}") + public static Collection params() { + DataType[] types = new DataType[] { + DataType.BOOL, + DataType.FLOAT16, + DataType.BFLOAT16, + DataType.FLOAT, + DataType.DOUBLE, + DataType.INT8, + DataType.INT16, + DataType.INT32, + DataType.INT64, + DataType.UINT8, + DataType.UINT16, + DataType.UINT32, + DataType.UINT64 + }; + + long[][] shapes = new long[][]{ + new long[]{2, 3}, + new long[]{3}, + new long[]{1}, + new long[]{} // scalar + }; + + + List ret = new ArrayList<>(); + for (DataType type: types){ + for (long[] shape: shapes){ + ret.add(new Object[]{type, shape, Arrays.toString(shape)}); + } + } + return ret; + } + + @Test + public void testConversion(){ + try(PythonGIL pythonGIL = PythonGIL.lock()) { + INDArray arr = Nd4j.zeros(dataType, shape); + PythonObject npArr = PythonTypes.convert(arr); + INDArray arr2 = PythonTypes.getPythonTypeForPythonObject(npArr).toJava(npArr); + if (dataType == DataType.BFLOAT16){ + arr = arr.castTo(DataType.FLOAT); + } + Assertions.assertEquals(arr,arr2); + } + + } + + + @Test + public void testExecution() { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + List inputs = new ArrayList<>(); + INDArray x = Nd4j.ones(dataType, shape); + INDArray y = Nd4j.zeros(dataType, shape); + INDArray z = (dataType == DataType.BOOL)?x:x.mul(y.add(2)); + z = (dataType == DataType.BFLOAT16)? z.castTo(DataType.FLOAT): z; + PythonType arrType = PythonTypes.get("numpy.ndarray"); + inputs.add(new PythonVariable<>("x", arrType, x)); + inputs.add(new PythonVariable<>("y", arrType, y)); + List outputs = new ArrayList<>(); + PythonVariable output = new PythonVariable<>("z", arrType); + outputs.add(output); + String code = (dataType == DataType.BOOL)?"z = x":"z = x * (y + 2)"; + if (shape.length == 0){ // scalar special case + code += "\nimport numpy as np\nz = np.asarray(float(z), dtype=x.dtype)"; + } + PythonExecutioner.exec(code, inputs, outputs); + INDArray z2 = output.getValue(); + + Assertions.assertEquals(z.dataType(), z2.dataType()); + Assertions.assertEquals(z, z2); + } + + + } + + + @Test + public void testInplaceExecution() { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + if (dataType == DataType.BOOL || dataType == DataType.BFLOAT16)return; + if (shape.length == 0) return; + List inputs = new ArrayList<>(); + INDArray x = Nd4j.ones(dataType, shape); + INDArray y = Nd4j.zeros(dataType, shape); + INDArray z = x.mul(y.add(2)); + // Nd4j.getAffinityManager().ensureLocation(z, AffinityManager.Location.HOST); + PythonType arrType = PythonTypes.get("numpy.ndarray"); + inputs.add(new PythonVariable<>("x", arrType, x)); + inputs.add(new PythonVariable<>("y", arrType, y)); + List outputs = new ArrayList<>(); + PythonVariable output = new PythonVariable<>("x", arrType); + outputs.add(output); + String code = "x *= y + 2"; + PythonExecutioner.exec(code, inputs, outputs); + INDArray z2 = output.getValue(); + Assertions.assertEquals(x.dataType(), z2.dataType()); + Assertions.assertEquals(z.dataType(), z2.dataType()); + Assertions.assertEquals(x, z2); + Assertions.assertEquals(z, z2); + Assertions.assertEquals(x.data().pointer().address(), z2.data().pointer().address()); + if("CUDA".equalsIgnoreCase(Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"))){ + Assertions.assertEquals(getDeviceAddress(x), getDeviceAddress(z2)); + } + + } + + + } + + + private static long getDeviceAddress(INDArray array) { + if(!"CUDA".equalsIgnoreCase(Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"))){ + throw new IllegalStateException("Cannot ge device pointer for non-CUDA device"); + } + + //Use reflection here as OpaqueDataBuffer is only available on BaseCudaDataBuffer and BaseCpuDataBuffer - not DataBuffer/BaseDataBuffer + // due to it being defined in nd4j-native-api, not nd4j-api + try { + Class c = Class.forName("org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer"); + Method m = c.getMethod("getOpaqueDataBuffer"); + OpaqueDataBuffer db = (OpaqueDataBuffer) m.invoke(array.data()); + long address = db.specialBuffer().address(); + return address; + } catch (Throwable t){ + throw new RuntimeException("Error getting OpaqueDataBuffer", t); + } + } + + + + +} diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyCollectionsTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyCollectionsTest.java new file mode 100644 index 000000000..58c466d13 --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyCollectionsTest.java @@ -0,0 +1,105 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + + +import org.nd4j.python4j.PythonException; +import org.nd4j.python4j.PythonGIL; +import org.nd4j.python4j.PythonObject; +import org.nd4j.python4j.PythonTypes; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.factory.Nd4j; + +import javax.annotation.concurrent.NotThreadSafe; +import java.util.*; + + +@NotThreadSafe +////@RunWith(Parameterized.class) +public class PythonNumpyCollectionsTest { + private DataType dataType; + + public PythonNumpyCollectionsTest(DataType dataType){ + this.dataType = dataType; + } + + ////@Parameterized.Parameters(name = "{index}: Testing with DataType={0}") + public static DataType[] params() { + return new DataType[]{ + DataType.BOOL, + DataType.FLOAT16, + //DataType.BFLOAT16, + DataType.FLOAT, + DataType.DOUBLE, + DataType.INT8, + DataType.INT16, + DataType.INT32, + DataType.INT64, + DataType.UINT8, + DataType.UINT16, + DataType.UINT32, + DataType.UINT64 + }; + } + @Test + public void testPythonDictFromMap() throws PythonException { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + Map map = new HashMap(); + map.put("a", 1); + map.put(1, "a"); + map.put("arr", Nd4j.ones(dataType, 2, 3)); + map.put("list1", Arrays.asList(1, 2.0, 3, 4f, Nd4j.zeros(dataType,3,2))); + Map innerMap = new HashMap(); + innerMap.put("b", 2); + innerMap.put(2, "b"); + innerMap.put(5, Nd4j.ones(dataType, 5)); + map.put("innermap", innerMap); + map.put("list2", Arrays.asList(4, "5", innerMap, false, true)); + PythonObject dict = PythonTypes.convert(map); + Map map2 = PythonTypes.DICT.toJava(dict); + Assertions.assertEquals(map.toString(), map2.toString()); + } + + } + + @Test + public void testPythonListFromList() throws PythonException { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + List list = new ArrayList<>(); + list.add(1); + list.add("2"); + list.add(Nd4j.ones(dataType, 2, 3)); + list.add(Arrays.asList("a", + Nd4j.ones(dataType, 1, 2),1.0, 2f, 10, true, false, + Nd4j.zeros(dataType, 3, 2))); + Map map = new HashMap(); + map.put("a", 1); + map.put(1, "a"); + map.put(5, Nd4j.ones(dataType,4, 5)); + map.put("list1", Arrays.asList(1, 2.0, 3, 4f, Nd4j.zeros(dataType, 3, 1))); + list.add(map); + PythonObject dict = PythonTypes.convert(list); + List list2 = PythonTypes.LIST.toJava(dict); + Assertions.assertEquals(list.toString(), list2.toString()); + } + + } +} diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyGCTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyGCTest.java new file mode 100644 index 000000000..3ff9c60bb --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyGCTest.java @@ -0,0 +1,65 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +import org.junit.jupiter.api.Disabled; +import org.nd4j.python4j.Python; +import org.nd4j.python4j.PythonGC; +import org.nd4j.python4j.PythonGIL; +import org.nd4j.python4j.PythonObject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.factory.Nd4j; + +import javax.annotation.concurrent.NotThreadSafe; + + +@NotThreadSafe +public class PythonNumpyGCTest { + + @Test + @Disabled //crashes, to be investigated + public void testGC() { + try(PythonGIL pythonGIL = PythonGIL.lock()) { + PythonObject gcModule = Python.importModule("gc"); + PythonObject getObjects = gcModule.attr("get_objects"); + PythonObject pyObjCount1 = Python.len(getObjects.call()); + long objCount1 = pyObjCount1.toLong(); + PythonObject pyList = Python.list(); + pyList.attr("append").call(new PythonObject(Nd4j.linspace(1, 10, 10))); + pyList.attr("append").call(1.0); + pyList.attr("append").call(true); + PythonObject pyObjCount2 = Python.len(getObjects.call()); + long objCount2 = pyObjCount2.toLong(); + long diff = objCount2 - objCount1; + Assertions.assertTrue(diff > 2); + try(PythonGC gc = PythonGC.watch()){ + PythonObject pyList2 = Python.list(); + pyList2.attr("append").call(new PythonObject(Nd4j.linspace(1, 10, 10))); + pyList2.attr("append").call(1.0); + pyList2.attr("append").call(true); + } + PythonObject pyObjCount3 = Python.len(getObjects.call()); + long objCount3 = pyObjCount3.toLong(); + diff = objCount3 - objCount2; + Assertions.assertTrue(diff <= 2);// 2 objects created during function call + } + + } +} diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyImportTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyImportTest.java new file mode 100644 index 000000000..b536cd6ae --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyImportTest.java @@ -0,0 +1,44 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +import org.junit.jupiter.api.Disabled; +import org.nd4j.python4j.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +public class PythonNumpyImportTest { + + @Test + @Disabled + public void testNumpyImport(){ + try(PythonGIL pythonGIL = PythonGIL.lock()) { + try(PythonGC gc = PythonGC.watch()){ + PythonObject np = Python.importModule("numpy"); + PythonObject zeros = np.attr("zeros").call(5); + INDArray arr = NumpyArray.INSTANCE.toJava(zeros); + Assertions.assertEquals(arr, Nd4j.zeros(DataType.DOUBLE, 5)); + } + } + + } +} diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyMultiThreadTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyMultiThreadTest.java new file mode 100644 index 000000000..c18d0a925 --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyMultiThreadTest.java @@ -0,0 +1,146 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +import org.nd4j.python4j.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import javax.annotation.concurrent.NotThreadSafe; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + + +@NotThreadSafe + +public class PythonNumpyMultiThreadTest { + private DataType dataType; + + public PythonNumpyMultiThreadTest(DataType dataType) { + this.dataType = dataType; + } + + ////@Parameterized.Parameters(name = "{index}: Testing with DataType={0}") + public static DataType[] params() { + return new DataType[]{ +// DataType.BOOL, +// DataType.FLOAT16, +// DataType.BFLOAT16, + DataType.FLOAT, + DataType.DOUBLE, +// DataType.INT8, +// DataType.INT16, + DataType.INT32, + DataType.INT64, +// DataType.UINT8, +// DataType.UINT16, +// DataType.UINT32, +// DataType.UINT64 + }; + } + + + @Test + public void testMultiThreading1() throws Throwable { + final List exceptions = Collections.synchronizedList(new ArrayList()); + Runnable runnable = new Runnable() { + @Override + public void run() { + try (PythonGIL gil = PythonGIL.lock()) { + try (PythonGC gc = PythonGC.watch()) { + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("x", NumpyArray.INSTANCE, Nd4j.ones(dataType, 2, 3).mul(3))); + inputs.add(new PythonVariable<>("y", NumpyArray.INSTANCE, Nd4j.ones(dataType, 2, 3).mul(4))); + PythonVariable out = new PythonVariable<>("z", NumpyArray.INSTANCE); + String code = "z = x + y"; + PythonExecutioner.exec(code, inputs, Collections.singletonList(out)); + Assertions.assertEquals(Nd4j.ones(dataType, 2, 3).mul(7), out.getValue()); + } + } catch (Throwable e) { + exceptions.add(e); + } + } + }; + + int numThreads = 10; + Thread[] threads = new Thread[numThreads]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(runnable); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + Thread.sleep(100); + for (int i = 0; i < threads.length; i++) { + threads[i].join(); + } + if (!exceptions.isEmpty()) { + throw (exceptions.get(0)); + } + + } + + @Test + public void testMultiThreading2() throws Throwable { + final List exceptions = Collections.synchronizedList(new ArrayList()); + Runnable runnable = new Runnable() { + @Override + public void run() { + try (PythonGIL gil = PythonGIL.lock()) { + try (PythonGC gc = PythonGC.watch()) { + PythonContextManager.reset(); + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("x", NumpyArray.INSTANCE, Nd4j.ones(dataType, 2, 3).mul(3))); + inputs.add(new PythonVariable<>("y", NumpyArray.INSTANCE, Nd4j.ones(dataType, 2, 3).mul(4))); + String code = "z = x + y"; + List outputs = PythonExecutioner.execAndReturnAllVariables(code, inputs); + Assertions.assertEquals(Nd4j.ones(dataType, 2, 3).mul(3), outputs.get(0).getValue()); + Assertions.assertEquals(Nd4j.ones(dataType, 2, 3).mul(4), outputs.get(1).getValue()); + Assertions.assertEquals(Nd4j.ones(dataType, 2, 3).mul(7), outputs.get(2).getValue()); + } + } catch (Throwable e) { + exceptions.add(e); + } + } + }; + + int numThreads = 10; + Thread[] threads = new Thread[numThreads]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(runnable); + } + for (int i = 0; i < threads.length; i++) { + threads[i].start(); + } + Thread.sleep(100); + for (int i = 0; i < threads.length; i++) { + threads[i].join(); + } + if (!exceptions.isEmpty()) { + throw (exceptions.get(0)); + } + } + + +} diff --git a/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyServiceLoaderTest.java b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyServiceLoaderTest.java new file mode 100644 index 000000000..717307c1e --- /dev/null +++ b/cavis-dnn/cavis-dnn-python4j/cavis-python4j-numpy/src/test/java/PythonNumpyServiceLoaderTest.java @@ -0,0 +1,41 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.python4j.NumpyArray; +import org.nd4j.python4j.PythonTypes; + +import javax.annotation.concurrent.NotThreadSafe; + +@NotThreadSafe +@Disabled +public class PythonNumpyServiceLoaderTest { + + @Test + public void testServiceLoader(){ + Assertions.assertEquals(NumpyArray.INSTANCE, PythonTypes.get("numpy.ndarray")); + Assertions.assertEquals(NumpyArray.INSTANCE, PythonTypes.getPythonTypeForJavaObject(Nd4j.zeros(1))); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/build.gradle b/cavis-dnn/cavis-dnn-spark/build.gradle new file mode 100644 index 000000000..a15409758 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/build.gradle @@ -0,0 +1,20 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/build.gradle b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/build.gradle new file mode 100644 index 000000000..3b4b2ae41 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/build.gradle @@ -0,0 +1,55 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +ext { + scalaVersion = rootProject.ext.scalaVersion +} + +dependencies { + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators + implementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators + implementation "org.apache.hadoop:hadoop-common:3.2.0" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" + implementation projects.cavisDatavec.cavisDatavecApi + implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore + implementation "commons-io:commons-io" + + implementation projects.cavisUi.cavisUiComponents + + implementation projects.cavisUi.cavisUiModel + + testImplementation 'ch.qos.logback:logback-classic' + + compileOnly "org.apache.spark:spark-core_${scalaVersion}" + compileOnly "org.apache.spark:spark-mllib_${scalaVersion}" + testCompileOnly "org.apache.spark:spark-core_${scalaVersion}" + testCompileOnly "org.apache.spark:spark-mllib_${scalaVersion}" + testRuntimeOnly projects.cavisDnn.cavisDnnCommonTests + testImplementation projects.cavisDnn.cavisDnnCommonTests + testImplementation "net.java.dev.jna:jna:5.9.0" + testImplementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage + testCompileOnly "org.scala-lang:scala-library" +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/apache/spark/TaskContextHelper.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/apache/spark/TaskContextHelper.java new file mode 100644 index 000000000..02d166e18 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/apache/spark/TaskContextHelper.java @@ -0,0 +1,29 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.apache.spark; + +public abstract class TaskContextHelper extends TaskContext { + + public static void setTaskContext(TaskContext tc) { + TaskContextHelper.setTaskContext(tc); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/RDDTrainingApproach.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/RDDTrainingApproach.java new file mode 100644 index 000000000..cfe35dd87 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/RDDTrainingApproach.java @@ -0,0 +1,25 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +public enum RDDTrainingApproach { + Export, Direct +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/Repartition.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/Repartition.java new file mode 100644 index 000000000..62b877435 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/Repartition.java @@ -0,0 +1,25 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +public enum Repartition { + Never, Always, NumPartitionsWorkersDiffers +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/RepartitionStrategy.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/RepartitionStrategy.java new file mode 100644 index 000000000..ff9c4a70e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/RepartitionStrategy.java @@ -0,0 +1,26 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +public enum RepartitionStrategy { + SparkDefault, Balanced, ApproximateBalanced + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/Repartitioner.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/Repartitioner.java new file mode 100644 index 000000000..b53ed74d6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/Repartitioner.java @@ -0,0 +1,31 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +import org.apache.spark.api.java.JavaRDD; + +import java.io.Serializable; + +public interface Repartitioner extends Serializable { + + JavaRDD repartition(JavaRDD input, int minObjectsPerPartition, int numExecutors); + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingHook.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingHook.java new file mode 100644 index 000000000..8ea9738db --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingHook.java @@ -0,0 +1,62 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +import org.deeplearning4j.nn.api.Model; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.io.Serializable; + +public interface TrainingHook extends Serializable { + /** + * A hook method for pre update. + * @param minibatch the inibatch + * that was used for the update + * @param model themodel that was update + */ + void preUpdate(DataSet minibatch, Model model); + + /** + * A hook method for post update + * @param minibatch the minibatch + * that was usd for the update + * @param model the model that was updated + */ + void postUpdate(DataSet minibatch, Model model); + + /** + * A hook method for pre update. + * @param minibatch the inibatch + * that was used for the update + * @param model the model that was update + */ + void preUpdate(MultiDataSet minibatch, Model model); + + /** + * A hook method for post update + * @param minibatch the minibatch + * that was usd for the update + * @param model the model that was updated + */ + void postUpdate(MultiDataSet minibatch, Model model); + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingMaster.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingMaster.java new file mode 100644 index 000000000..12441db94 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingMaster.java @@ -0,0 +1,172 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.util.Collection; + +public interface TrainingMaster> { + + + /** + * Remove a training hook from the worker + * @param trainingHook the training hook to remove + */ + void removeHook(TrainingHook trainingHook); + + /** + * Add a hook for the master for pre and post training + * @param trainingHook the training hook to add + */ + void addHook(TrainingHook trainingHook); + + /** + * Get the TrainingMaster configuration as JSON + */ + String toJson(); + + /** + * Get the TrainingMaster configuration as YAML + */ + String toYaml(); + + /** + * Get the worker instance for this training master + * + * @param network Current SparkDl4jMultiLayer + * @return Worker instance + */ + W getWorkerInstance(SparkDl4jMultiLayer network); + + /** + * Get the worker instance for this training master + * + * @param graph Current SparkComputationGraph + * @return Worker instance + */ + W getWorkerInstance(SparkComputationGraph graph); + + /** + * Train the SparkDl4jMultiLayer with the specified data set + * + * @param network Current network state + * @param trainingData Data to train on + */ + void executeTraining(SparkDl4jMultiLayer network, JavaRDD trainingData); + + + /** + * Fit the network using a list of paths for serialized DataSet objects. + * + * @param network Current network state + * @param trainingDataPaths Data to train on + */ + void executeTrainingPaths(SparkDl4jMultiLayer network, SparkComputationGraph graph, JavaRDD trainingDataPaths, DataSetLoader dsLoader, MultiDataSetLoader mdsLoader); + + /** + * Train the SparkComputationGraph with the specified data set + * + * @param graph Current network state + * @param trainingData Data to train on + */ + void executeTraining(SparkComputationGraph graph, JavaRDD trainingData); + + /** + * Train the SparkComputationGraph with the specified data set + * + * @param graph Current network state + * @param trainingData Data to train on + */ + void executeTrainingMDS(SparkComputationGraph graph, JavaRDD trainingData); + + /** + * Set whether the training statistics should be collected. Training statistics may include things like per-epoch run times, + * time spent waiting for data, etc. + *

+ * These statistics are primarily used for debugging and optimization, in order to gain some insight into what aspects + * of network training are taking the most time. + * + * @param collectTrainingStats If true: collecting training statistics will be + */ + void setCollectTrainingStats(boolean collectTrainingStats); + + /** + * Get the current setting for collectTrainingStats + */ + boolean getIsCollectTrainingStats(); + + /** + * Return the training statistics. Note that this may return null, unless setCollectTrainingStats has been set first + * + * @return Training statistics + */ + SparkTrainingStats getTrainingStats(); + + /** + * Set the iteration listeners. These should be called after every averaging (or similar) operation in the TrainingMaster, + * though the exact behaviour may be dependent on each TrainingListener + * + * @param listeners Listeners to set + */ + void setListeners(Collection listeners); + + + /** + * Set the iteration listeners and the StatsStorageRouter. This is typically used for UI functionality: for example, + * setListeners(new FileStatsStorage(myFile), Collections.singletonList(new StatsListener(null))). This will pass a + * StatsListener to each worker, and then shuffle the results back to the specified FileStatsStorage instance (which + * can then be attached to the UI or loaded later) + * + * @param router StatsStorageRouter in which to place the results + * @param listeners Listeners + */ + void setListeners(StatsStorageRouter router, Collection listeners); + + /** + * Attempt to delete any temporary files generated by this TrainingMaster. + * Depending on the configuration, no temporary files may be generated. + * + * @param sc JavaSparkContext (used to access HDFS etc file systems, when required) + * @return True if deletion was successful (or, no files to delete); false otherwise. + */ + boolean deleteTempFiles(JavaSparkContext sc); + + /** + * Attempt to delete any temporary files generated by this TrainingMaster. + * Depending on the configuration, no temporary files may be generated. + * + * @param sc SparkContext (used to access HDFS etc file systems, when required) + * @return True if deletion was successful (or, no files to delete); false otherwise. + */ + boolean deleteTempFiles(SparkContext sc); +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingResult.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingResult.java new file mode 100644 index 000000000..42bb29261 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingResult.java @@ -0,0 +1,32 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; + +public interface TrainingResult { + + /** + * + * @param sparkTrainingStats + */ + void setStats(SparkTrainingStats sparkTrainingStats); +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingWorker.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingWorker.java new file mode 100644 index 000000000..70b2d4826 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/TrainingWorker.java @@ -0,0 +1,152 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.common.primitives.Pair; + +import java.io.Serializable; + +public interface TrainingWorker extends Serializable { + + /** + * Remove a training hook from the worker + * @param trainingHook the training hook to remove + */ + void removeHook(TrainingHook trainingHook); + + /** + * Add a training hook to be used + * during training of the worker + * @param trainingHook the training hook to add + */ + void addHook(TrainingHook trainingHook); + + /** + * Get the initial model when training a MultiLayerNetwork/SparkDl4jMultiLayer + * + * @return Initial model for this worker + */ + MultiLayerNetwork getInitialModel(); + + /** + * Get the initial model when training a ComputationGraph/SparkComputationGraph + * + * @return Initial model for this worker + */ + ComputationGraph getInitialModelGraph(); + + /** + * Process (fit) a minibatch for a MultiLayerNetwork + * + * @param dataSet Data set to train on + * @param network Network to train + * @param isLast If true: last data set currently available. If false: more data sets will be processed for this executor + * @return Null, or a training result if training should be terminated immediately. + */ + R processMinibatch(DataSet dataSet, MultiLayerNetwork network, boolean isLast); + + /** + * Process (fit) a minibatch for a ComputationGraph + * + * @param dataSet Data set to train on + * @param graph Network to train + * @param isLast If true: last data set currently available. If false: more data sets will be processed for this executor + * @return Null, or a training result if training should be terminated immediately. + */ + R processMinibatch(DataSet dataSet, ComputationGraph graph, boolean isLast); + + /** + * Process (fit) a minibatch for a ComputationGraph using a MultiDataSet + * + * @param dataSet Data set to train on + * @param graph Network to train + * @param isLast If true: last data set currently available. If false: more data sets will be processed for this executor + * @return Null, or a training result if training should be terminated immediately. + */ + R processMinibatch(MultiDataSet dataSet, ComputationGraph graph, boolean isLast); + + /** + * As per {@link #processMinibatch(DataSet, MultiLayerNetwork, boolean)} but used when {@link SparkTrainingStats} are being collecte + */ + Pair processMinibatchWithStats(DataSet dataSet, MultiLayerNetwork network, boolean isLast); + + /** + * As per {@link #processMinibatch(DataSet, ComputationGraph, boolean)} but used when {@link SparkTrainingStats} are being collected + */ + Pair processMinibatchWithStats(DataSet dataSet, ComputationGraph graph, boolean isLast); + + /** + * As per {@link #processMinibatch(MultiDataSet, ComputationGraph, boolean)} but used when {@link SparkTrainingStats} are being collected + */ + Pair processMinibatchWithStats(MultiDataSet dataSet, ComputationGraph graph, boolean isLast); + + /** + * Get the final result to be returned to the driver + * + * @param network Current state of the network + * @return Result to return to the driver + */ + R getFinalResult(MultiLayerNetwork network); + + /** + * Get the final result to be returned to the driver + * + * @param graph Current state of the network + * @return Result to return to the driver + */ + R getFinalResult(ComputationGraph graph); + + /** + * Get the final result to be returned to the driver, if no data was available for this executor + * + * @return Result to return to the driver + */ + R getFinalResultNoData(); + + /** + * As per {@link #getFinalResultNoData()} but used when {@link SparkTrainingStats} are being collected + */ + Pair getFinalResultNoDataWithStats(); + + /** + * As per {@link #getFinalResult(MultiLayerNetwork)} but used when {@link SparkTrainingStats} are being collected + */ + Pair getFinalResultWithStats(MultiLayerNetwork network); + + /** + * As per {@link #getFinalResult(ComputationGraph)} but used when {@link SparkTrainingStats} are being collected + */ + Pair getFinalResultWithStats(ComputationGraph graph); + + /** + * Get the {@link WorkerConfiguration} that contains information such as minibatch sizes, etc + * + * @return Worker configuration + */ + WorkerConfiguration getDataConfiguration(); + + long getInstanceId(); +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/WorkerConfiguration.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/WorkerConfiguration.java new file mode 100644 index 000000000..8836ec26c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/WorkerConfiguration.java @@ -0,0 +1,39 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.io.Serializable; + +@AllArgsConstructor +@Data +public class WorkerConfiguration implements Serializable { + + protected final boolean isGraphNetwork; + protected final int dataSetObjectSizeExamples; //Number of examples in each DataSet object + protected final int batchSizePerWorker; + protected final int maxBatchesPerWorker; + protected final int prefetchNumBatches; + protected final boolean collectTrainingStats; + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/CommonSparkTrainingStats.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/CommonSparkTrainingStats.java new file mode 100644 index 000000000..e1355b40f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/CommonSparkTrainingStats.java @@ -0,0 +1,253 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.stats; + +import lombok.Data; +import org.apache.commons.io.FilenameUtils; +import org.apache.spark.SparkContext; +import org.deeplearning4j.spark.stats.EventStats; +import org.deeplearning4j.spark.stats.StatsUtils; + +import java.io.IOException; +import java.util.*; + +@Data +public class CommonSparkTrainingStats implements SparkTrainingStats { + + public static final String DEFAULT_DELIMITER = ","; + public static final String FILENAME_TOTAL_TIME_STATS = "workerFlatMapTotalTimeMs.txt"; + public static final String FILENAME_GET_INITIAL_MODEL_STATS = "workerFlatMapGetInitialModelTimeMs.txt"; + public static final String FILENAME_DATASET_GET_TIME_STATS = "workerFlatMapDataSetGetTimesMs.txt"; + public static final String FILENAME_PROCESS_MINIBATCH_TIME_STATS = "workerFlatMapProcessMiniBatchTimesMs.txt"; + + public static final String WORKER_FLAT_MAP_TOTAL_TIME_MS = "WorkerFlatMapTotalTimeMs"; + public static final String WORKER_FLAT_MAP_GET_INITIAL_MODEL_TIME_MS = "WorkerFlatMapGetInitialModelTimeMs"; + public static final String WORKER_FLAT_MAP_DATA_SET_GET_TIMES_MS = "WorkerFlatMapDataSetGetTimesMs"; + public static final String WORKER_FLAT_MAP_PROCESS_MINI_BATCH_TIMES_MS = "WorkerFlatMapProcessMiniBatchTimesMs"; + private static Set columnNames = + Collections.unmodifiableSet(new LinkedHashSet<>(Arrays.asList(WORKER_FLAT_MAP_TOTAL_TIME_MS, + WORKER_FLAT_MAP_GET_INITIAL_MODEL_TIME_MS, WORKER_FLAT_MAP_DATA_SET_GET_TIMES_MS, + WORKER_FLAT_MAP_PROCESS_MINI_BATCH_TIMES_MS))); + + private SparkTrainingStats trainingWorkerSpecificStats; + private List workerFlatMapTotalTimeMs; + private List workerFlatMapGetInitialModelTimeMs; + private List workerFlatMapDataSetGetTimesMs; + private List workerFlatMapProcessMiniBatchTimesMs; + + + + public CommonSparkTrainingStats() { + + } + + private CommonSparkTrainingStats(Builder builder) { + this.trainingWorkerSpecificStats = builder.trainingMasterSpecificStats; + this.workerFlatMapTotalTimeMs = builder.workerFlatMapTotalTimeMs; + this.workerFlatMapGetInitialModelTimeMs = builder.workerFlatMapGetInitialModelTimeMs; + this.workerFlatMapDataSetGetTimesMs = builder.workerFlatMapDataSetGetTimesMs; + this.workerFlatMapProcessMiniBatchTimesMs = builder.workerFlatMapProcessMiniBatchTimesMs; + } + + + @Override + public Set getKeySet() { + Set set = new LinkedHashSet<>(columnNames); + if (trainingWorkerSpecificStats != null) + set.addAll(trainingWorkerSpecificStats.getKeySet()); + + return set; + } + + @Override + public List getValue(String key) { + switch (key) { + case WORKER_FLAT_MAP_TOTAL_TIME_MS: + return workerFlatMapTotalTimeMs; + case WORKER_FLAT_MAP_GET_INITIAL_MODEL_TIME_MS: + return workerFlatMapGetInitialModelTimeMs; + case WORKER_FLAT_MAP_DATA_SET_GET_TIMES_MS: + return workerFlatMapDataSetGetTimesMs; + case WORKER_FLAT_MAP_PROCESS_MINI_BATCH_TIMES_MS: + return workerFlatMapProcessMiniBatchTimesMs; + default: + if (trainingWorkerSpecificStats != null) + return trainingWorkerSpecificStats.getValue(key); + throw new IllegalArgumentException("Unknown key: \"" + key + "\""); + } + } + + @Override + public String getShortNameForKey(String key) { + switch (key) { + case WORKER_FLAT_MAP_TOTAL_TIME_MS: + return "Total"; + case WORKER_FLAT_MAP_GET_INITIAL_MODEL_TIME_MS: + return "GetInitModel"; + case WORKER_FLAT_MAP_DATA_SET_GET_TIMES_MS: + return "GetDataSet"; + case WORKER_FLAT_MAP_PROCESS_MINI_BATCH_TIMES_MS: + return "ProcessBatch"; + default: + if (trainingWorkerSpecificStats != null) + return trainingWorkerSpecificStats.getShortNameForKey(key); + throw new IllegalArgumentException("Unknown key: \"" + key + "\""); + } + } + + @Override + public boolean defaultIncludeInPlots(String key) { + switch (key) { + case WORKER_FLAT_MAP_TOTAL_TIME_MS: + case WORKER_FLAT_MAP_GET_INITIAL_MODEL_TIME_MS: + case WORKER_FLAT_MAP_PROCESS_MINI_BATCH_TIMES_MS: + return false; //Covered by worker stats generally + case WORKER_FLAT_MAP_DATA_SET_GET_TIMES_MS: + return true; + default: + if (trainingWorkerSpecificStats != null) + return trainingWorkerSpecificStats.defaultIncludeInPlots(key); + return false; + } + } + + @Override + public void addOtherTrainingStats(SparkTrainingStats other) { + if (!(other instanceof CommonSparkTrainingStats)) + throw new IllegalArgumentException( + "Cannot add other training stats: not an instance of CommonSparkTrainingStats"); + + CommonSparkTrainingStats o = (CommonSparkTrainingStats) other; + + workerFlatMapTotalTimeMs.addAll(o.workerFlatMapTotalTimeMs); + workerFlatMapGetInitialModelTimeMs.addAll(o.workerFlatMapGetInitialModelTimeMs); + workerFlatMapDataSetGetTimesMs.addAll(o.workerFlatMapDataSetGetTimesMs); + workerFlatMapProcessMiniBatchTimesMs.addAll(o.workerFlatMapProcessMiniBatchTimesMs); + + if (trainingWorkerSpecificStats != null) + trainingWorkerSpecificStats.addOtherTrainingStats(o.trainingWorkerSpecificStats); + else if (o.trainingWorkerSpecificStats != null) + throw new IllegalStateException( + "Cannot merge: training master specific stats is null in one, but not the other"); + } + + @Override + public SparkTrainingStats getNestedTrainingStats() { + return trainingWorkerSpecificStats; + } + + @Override + public String statsAsString() { + StringBuilder sb = new StringBuilder(); + String f = SparkTrainingStats.DEFAULT_PRINT_FORMAT; + + sb.append(String.format(f, WORKER_FLAT_MAP_TOTAL_TIME_MS)); + if (workerFlatMapTotalTimeMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(workerFlatMapTotalTimeMs, ",")).append("\n"); + + sb.append(String.format(f, WORKER_FLAT_MAP_GET_INITIAL_MODEL_TIME_MS)); + if (workerFlatMapGetInitialModelTimeMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(workerFlatMapGetInitialModelTimeMs, ",")).append("\n"); + + sb.append(String.format(f, WORKER_FLAT_MAP_DATA_SET_GET_TIMES_MS)); + if (workerFlatMapDataSetGetTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(workerFlatMapDataSetGetTimesMs, ",")).append("\n"); + + sb.append(String.format(f, WORKER_FLAT_MAP_PROCESS_MINI_BATCH_TIMES_MS)); + if (workerFlatMapProcessMiniBatchTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(workerFlatMapProcessMiniBatchTimesMs, ",")).append("\n"); + + if (trainingWorkerSpecificStats != null) + sb.append(trainingWorkerSpecificStats.statsAsString()).append("\n"); + + return sb.toString(); + } + + @Override + public void exportStatFiles(String outputPath, SparkContext sc) throws IOException { + String d = DEFAULT_DELIMITER; + + + //Total time stats (includes total example counts) + String totalTimeStatsPath = FilenameUtils.concat(outputPath, FILENAME_TOTAL_TIME_STATS); + StatsUtils.exportStats(workerFlatMapTotalTimeMs, totalTimeStatsPath, d, sc); + + //"Get initial model" stats: + String getInitialModelStatsPath = FilenameUtils.concat(outputPath, FILENAME_GET_INITIAL_MODEL_STATS); + StatsUtils.exportStats(workerFlatMapGetInitialModelTimeMs, getInitialModelStatsPath, d, sc); + + //"DataSet get time" stats: + String getDataSetStatsPath = FilenameUtils.concat(outputPath, FILENAME_DATASET_GET_TIME_STATS); + StatsUtils.exportStats(workerFlatMapDataSetGetTimesMs, getDataSetStatsPath, d, sc); + + //Process minibatch time stats: + String processMiniBatchStatsPath = FilenameUtils.concat(outputPath, FILENAME_PROCESS_MINIBATCH_TIME_STATS); + StatsUtils.exportStats(workerFlatMapProcessMiniBatchTimesMs, processMiniBatchStatsPath, d, sc); + + if (trainingWorkerSpecificStats != null) + trainingWorkerSpecificStats.exportStatFiles(outputPath, sc); + } + + public static class Builder { + private SparkTrainingStats trainingMasterSpecificStats; + private List workerFlatMapTotalTimeMs; + private List workerFlatMapGetInitialModelTimeMs; + private List workerFlatMapDataSetGetTimesMs; + private List workerFlatMapProcessMiniBatchTimesMs; + + public Builder trainingMasterSpecificStats(SparkTrainingStats trainingMasterSpecificStats) { + this.trainingMasterSpecificStats = trainingMasterSpecificStats; + return this; + } + + public Builder workerFlatMapTotalTimeMs(List workerFlatMapTotalTimeMs) { + this.workerFlatMapTotalTimeMs = workerFlatMapTotalTimeMs; + return this; + } + + public Builder workerFlatMapGetInitialModelTimeMs(List workerFlatMapGetInitialModelTimeMs) { + this.workerFlatMapGetInitialModelTimeMs = workerFlatMapGetInitialModelTimeMs; + return this; + } + + public Builder workerFlatMapDataSetGetTimesMs(List workerFlatMapDataSetGetTimesMs) { + this.workerFlatMapDataSetGetTimesMs = workerFlatMapDataSetGetTimesMs; + return this; + } + + public Builder workerFlatMapProcessMiniBatchTimesMs(List workerFlatMapProcessMiniBatchTimesMs) { + this.workerFlatMapProcessMiniBatchTimesMs = workerFlatMapProcessMiniBatchTimesMs; + return this; + } + + public CommonSparkTrainingStats build() { + return new CommonSparkTrainingStats(this); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/SparkTrainingStats.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/SparkTrainingStats.java new file mode 100644 index 000000000..e45496cd9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/SparkTrainingStats.java @@ -0,0 +1,108 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.stats; + +import org.apache.spark.SparkContext; +import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingMasterStats; +import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingWorkerStats; +import org.deeplearning4j.spark.stats.EventStats; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; +import java.util.Set; + +public interface SparkTrainingStats extends Serializable { + + /** + * Default indentation for {@link #statsAsString()} + */ + int PRINT_INDENT = 55; + + /** + * Default formatter used for {@link #statsAsString()} + */ + String DEFAULT_PRINT_FORMAT = "%-" + PRINT_INDENT + "s"; + + /** + * @return Set of keys that can be used with {@link #getValue(String)} + */ + Set getKeySet(); + + /** + * Get the statistic value for this key + * + * @param key Key to get the value for + * @return Statistic for this key, or an exception if key is invalid + */ + List getValue(String key); + + /** + * Return a short (display) name for the given key. + * + * @param key Key + * @return Short/display name for key + */ + String getShortNameForKey(String key); + + /** + * When plotting statistics, we don't necessarily want to plot everything. + * For example, some statistics/measurements are made up multiple smaller components; it does not always make sense + * to plot both the larger stat, and the components that make it up + * + * @param key Key to check for default plotting behaviour + * @return Whether the specified key should be included in plots by default or not + */ + boolean defaultIncludeInPlots(String key); + + /** + * Combine the two training stats instances. Usually, the two objects must be of the same type + * + * @param other Other training stats to return + */ + void addOtherTrainingStats(SparkTrainingStats other); + + /** + * Return the nested training stats - if any. + * + * @return The nested stats, if present/applicable, or null otherwise + */ + SparkTrainingStats getNestedTrainingStats(); + + /** + * Get a String representation of the stats. This functionality is implemented as a separate method (as opposed to toString()) + * as the resulting String can be very large.
+ * + * NOTE: The String representation typically includes only duration information. To get full statistics (including + * machine IDs, etc) use {@link #getValue(String)} or export full data via {@link #exportStatFiles(String, SparkContext)} + * + * @return A String representation of the training statistics + */ + String statsAsString(); + + + /** + * Export the stats as a collection of files. Stats are comma-delimited (CSV) with 1 header line + * + * @param outputPath Base directory to write files to + */ + void exportStatFiles(String outputPath, SparkContext sc) throws IOException; +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/StatsCalculationHelper.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/StatsCalculationHelper.java new file mode 100644 index 000000000..7e769cbb5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/stats/StatsCalculationHelper.java @@ -0,0 +1,96 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.stats; + +import org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap; +import org.deeplearning4j.spark.api.worker.ExecuteWorkerMultiDataSetFlatMap; +import org.deeplearning4j.spark.stats.BaseEventStats; +import org.deeplearning4j.spark.stats.EventStats; +import org.deeplearning4j.spark.stats.ExampleCountEventStats; +import org.deeplearning4j.spark.time.TimeSource; +import org.deeplearning4j.spark.time.TimeSourceProvider; + +import java.util.ArrayList; +import java.util.List; + +public class StatsCalculationHelper { + private long methodStartTime; + private long returnTime; + private long initalModelBefore; + private long initialModelAfter; + private long lastDataSetBefore; + private long lastProcessBefore; + private long totalExampleCount; + private List dataSetGetTimes = new ArrayList<>(); + private List processMiniBatchTimes = new ArrayList<>(); + + private TimeSource timeSource = TimeSourceProvider.getInstance(); + + public void logMethodStartTime() { + methodStartTime = timeSource.currentTimeMillis(); + } + + public void logReturnTime() { + returnTime = timeSource.currentTimeMillis(); + } + + public void logInitialModelBefore() { + initalModelBefore = timeSource.currentTimeMillis(); + } + + public void logInitialModelAfter() { + initialModelAfter = timeSource.currentTimeMillis(); + } + + public void logNextDataSetBefore() { + lastDataSetBefore = timeSource.currentTimeMillis(); + } + + public void logNextDataSetAfter(long numExamples) { + long now = timeSource.currentTimeMillis(); + long duration = now - lastDataSetBefore; + dataSetGetTimes.add(new BaseEventStats(lastDataSetBefore, duration)); + totalExampleCount += numExamples; + } + + public void logProcessMinibatchBefore() { + lastProcessBefore = timeSource.currentTimeMillis(); + } + + public void logProcessMinibatchAfter() { + long now = timeSource.currentTimeMillis(); + long duration = now - lastProcessBefore; + processMiniBatchTimes.add(new BaseEventStats(lastProcessBefore, duration)); + } + + public CommonSparkTrainingStats build(SparkTrainingStats masterSpecificStats) { + + List totalTime = new ArrayList<>(); + totalTime.add(new ExampleCountEventStats(methodStartTime, returnTime - methodStartTime, totalExampleCount)); + List initTime = new ArrayList<>(); + initTime.add(new BaseEventStats(initalModelBefore, initialModelAfter - initalModelBefore)); + + return new CommonSparkTrainingStats.Builder().trainingMasterSpecificStats(masterSpecificStats) + .workerFlatMapTotalTimeMs(totalTime).workerFlatMapGetInitialModelTimeMs(initTime) + .workerFlatMapDataSetGetTimesMs(dataSetGetTimes) + .workerFlatMapProcessMiniBatchTimesMs(processMiniBatchTimes).build(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerFlatMap.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerFlatMap.java new file mode 100644 index 000000000..6d52b8103 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerFlatMap.java @@ -0,0 +1,157 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.worker; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.nd4j.linalg.dataset.AsyncDataSetIterator; +import org.deeplearning4j.datasets.iterator.IteratorDataSetIterator; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.api.WorkerConfiguration; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.api.stats.StatsCalculationHelper; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.util.Collections; +import java.util.Iterator; + +public class ExecuteWorkerFlatMap implements FlatMapFunction, R> { + + private final TrainingWorker worker; + + public ExecuteWorkerFlatMap(TrainingWorker worker) { + this.worker = worker; + } + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + WorkerConfiguration dataConfig = worker.getDataConfiguration(); + final boolean isGraph = dataConfig.isGraphNetwork(); + + boolean stats = dataConfig.isCollectTrainingStats(); + StatsCalculationHelper s = (stats ? new StatsCalculationHelper() : null); + if (stats) + s.logMethodStartTime(); + + if (!dataSetIterator.hasNext()) { + if (stats) { + s.logReturnTime(); + + Pair pair = worker.getFinalResultNoDataWithStats(); + pair.getFirst().setStats(s.build(pair.getSecond())); + return Collections.singletonList(pair.getFirst()).iterator(); + } else { + return Collections.singletonList(worker.getFinalResultNoData()).iterator(); + } + } + + int batchSize = dataConfig.getBatchSizePerWorker(); + final int prefetchCount = dataConfig.getPrefetchNumBatches(); + + DataSetIterator batchedIterator = new IteratorDataSetIterator(dataSetIterator, batchSize); + if (prefetchCount > 0) { + batchedIterator = new AsyncDataSetIterator(batchedIterator, prefetchCount); + } + + try { + MultiLayerNetwork net = null; + ComputationGraph graph = null; + if (stats) + s.logInitialModelBefore(); + if (isGraph) + graph = worker.getInitialModelGraph(); + else + net = worker.getInitialModel(); + if (stats) + s.logInitialModelAfter(); + + int miniBatchCount = 0; + int maxMinibatches = (dataConfig.getMaxBatchesPerWorker() > 0 ? dataConfig.getMaxBatchesPerWorker() + : Integer.MAX_VALUE); + + while (batchedIterator.hasNext() && miniBatchCount++ < maxMinibatches) { + if (stats) + s.logNextDataSetBefore(); + DataSet next = batchedIterator.next(); + if (stats) + s.logNextDataSetAfter(next.numExamples()); + + if (stats) { + s.logProcessMinibatchBefore(); + Pair result; + if (isGraph) + result = worker.processMinibatchWithStats(next, graph, !batchedIterator.hasNext()); + else + result = worker.processMinibatchWithStats(next, net, !batchedIterator.hasNext()); + s.logProcessMinibatchAfter(); + if (result != null) { + //Terminate training immediately + s.logReturnTime(); + SparkTrainingStats workerStats = result.getSecond(); + SparkTrainingStats returnStats = s.build(workerStats); + result.getFirst().setStats(returnStats); + + return Collections.singletonList(result.getFirst()).iterator(); + } + } else { + R result; + if (isGraph) + result = worker.processMinibatch(next, graph, !batchedIterator.hasNext()); + else + result = worker.processMinibatch(next, net, !batchedIterator.hasNext()); + if (result != null) { + //Terminate training immediately + return Collections.singletonList(result).iterator(); + } + } + } + + //For some reason, we didn't return already. Normally this shouldn't happen + if (stats) { + s.logReturnTime(); + Pair pair; + if (isGraph) + pair = worker.getFinalResultWithStats(graph); + else + pair = worker.getFinalResultWithStats(net); + pair.getFirst().setStats(s.build(pair.getSecond())); + return Collections.singletonList(pair.getFirst()).iterator(); + } else { + if (isGraph) + return Collections.singletonList(worker.getFinalResult(graph)).iterator(); + else + return Collections.singletonList(worker.getFinalResult(net)).iterator(); + } + } finally { + //Make sure we shut down the async thread properly... + Nd4j.getExecutioner().commit(); + + if (batchedIterator instanceof AsyncDataSetIterator) { + ((AsyncDataSetIterator) batchedIterator).shutdown(); + } + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerMultiDataSetFlatMap.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerMultiDataSetFlatMap.java new file mode 100644 index 000000000..a570dd436 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerMultiDataSetFlatMap.java @@ -0,0 +1,130 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.worker; + +import lombok.AllArgsConstructor; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.nd4j.linalg.dataset.AsyncMultiDataSetIterator; +import org.deeplearning4j.datasets.iterator.IteratorMultiDataSetIterator; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.api.WorkerConfiguration; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.api.stats.StatsCalculationHelper; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.util.Collections; +import java.util.Iterator; + +@AllArgsConstructor +public class ExecuteWorkerMultiDataSetFlatMap implements FlatMapFunction, R> { + + private final TrainingWorker worker; + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + WorkerConfiguration dataConfig = worker.getDataConfiguration(); + + boolean stats = dataConfig.isCollectTrainingStats(); + StatsCalculationHelper s = (stats ? new StatsCalculationHelper() : null); + if (stats) + s.logMethodStartTime(); + + if (!dataSetIterator.hasNext()) { + if (stats) + s.logReturnTime(); + //TODO return the results... + return Collections.emptyIterator(); //Sometimes: no data + } + + int batchSize = dataConfig.getBatchSizePerWorker(); + final int prefetchCount = dataConfig.getPrefetchNumBatches(); + + MultiDataSetIterator batchedIterator = new IteratorMultiDataSetIterator(dataSetIterator, batchSize); + if (prefetchCount > 0) { + batchedIterator = new AsyncMultiDataSetIterator(batchedIterator, prefetchCount); + } + + try { + if (stats) + s.logInitialModelBefore(); + ComputationGraph net = worker.getInitialModelGraph(); + if (stats) + s.logInitialModelAfter(); + + int miniBatchCount = 0; + int maxMinibatches = (dataConfig.getMaxBatchesPerWorker() > 0 ? dataConfig.getMaxBatchesPerWorker() + : Integer.MAX_VALUE); + + while (batchedIterator.hasNext() && miniBatchCount++ < maxMinibatches) { + if (stats) + s.logNextDataSetBefore(); + MultiDataSet next = batchedIterator.next(); + + if (stats) + s.logNextDataSetAfter(next.getFeatures(0).size(0)); + + if (stats) { + s.logProcessMinibatchBefore(); + Pair result = + worker.processMinibatchWithStats(next, net, !batchedIterator.hasNext()); + s.logProcessMinibatchAfter(); + if (result != null) { + //Terminate training immediately + s.logReturnTime(); + SparkTrainingStats workerStats = result.getSecond(); + SparkTrainingStats returnStats = s.build(workerStats); + result.getFirst().setStats(returnStats); + + return Collections.singletonList(result.getFirst()).iterator(); + } + } else { + R result = worker.processMinibatch(next, net, !batchedIterator.hasNext()); + if (result != null) { + //Terminate training immediately + return Collections.singletonList(result).iterator(); + } + } + } + + //For some reason, we didn't return already. Normally this shouldn't happen + if (stats) { + s.logReturnTime(); + Pair pair = worker.getFinalResultWithStats(net); + pair.getFirst().setStats(s.build(pair.getSecond())); + return Collections.singletonList(pair.getFirst()).iterator(); + } else { + return Collections.singletonList(worker.getFinalResult(net)).iterator(); + } + } finally { + Nd4j.getExecutioner().commit(); + + //Make sure we shut down the async thread properly... + if (batchedIterator instanceof AsyncMultiDataSetIterator) { + ((AsyncMultiDataSetIterator) batchedIterator).shutdown(); + } + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPDSFlatMap.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPDSFlatMap.java new file mode 100644 index 000000000..451ea203e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPDSFlatMap.java @@ -0,0 +1,44 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.worker; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.input.PortableDataStream; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.iterator.PortableDataStreamDataSetIterator; +import org.nd4j.linalg.dataset.DataSet; + +import java.util.Iterator; + +@Deprecated +public class ExecuteWorkerPDSFlatMap implements FlatMapFunction, R> { + private final FlatMapFunction, R> workerFlatMap; + + public ExecuteWorkerPDSFlatMap(TrainingWorker worker) { + this.workerFlatMap = new ExecuteWorkerFlatMap<>(worker); + } + + @Override + public Iterator call(Iterator iter) throws Exception { + return workerFlatMap.call(new PortableDataStreamDataSetIterator(iter)); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPDSMDSFlatMap.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPDSMDSFlatMap.java new file mode 100644 index 000000000..49ab59cfe --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPDSMDSFlatMap.java @@ -0,0 +1,44 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.worker; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.input.PortableDataStream; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.iterator.PortableDataStreamMultiDataSetIterator; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.util.Iterator; + +@Deprecated +public class ExecuteWorkerPDSMDSFlatMap implements FlatMapFunction, R> { + private final FlatMapFunction, R> workerFlatMap; + + public ExecuteWorkerPDSMDSFlatMap(TrainingWorker worker) { + this.workerFlatMap = new ExecuteWorkerMultiDataSetFlatMap<>(worker); + } + + @Override + public Iterator call(Iterator iter) throws Exception { + return workerFlatMap.call(new PortableDataStreamMultiDataSetIterator(iter)); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPathFlatMap.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPathFlatMap.java new file mode 100644 index 000000000..5dcf6813f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPathFlatMap.java @@ -0,0 +1,77 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.worker; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.api.WorkerConfiguration; +import org.deeplearning4j.spark.iterator.PathSparkDataSetIterator; +import org.nd4j.linalg.dataset.DataSet; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class ExecuteWorkerPathFlatMap implements FlatMapFunction, R> { + + private final FlatMapFunction, R> workerFlatMap; + private final DataSetLoader dataSetLoader; + private final int maxDataSetObjects; + private final Broadcast hadoopConfig; + + public ExecuteWorkerPathFlatMap(TrainingWorker worker, DataSetLoader dataSetLoader, Broadcast hadoopConfig) { + this.workerFlatMap = new ExecuteWorkerFlatMap<>(worker); + this.dataSetLoader = dataSetLoader; + this.hadoopConfig = hadoopConfig; + + //How many dataset objects of size 'dataSetObjectNumExamples' should we load? + //Only pass on the required number, not all of them (to avoid async preloading data that won't be used) + //Most of the time we'll get exactly the number we want, but this isn't guaranteed all the time for all + // splitting strategies + WorkerConfiguration conf = worker.getDataConfiguration(); + int dataSetObjectNumExamples = conf.getDataSetObjectSizeExamples(); + int workerMinibatchSize = conf.getBatchSizePerWorker(); + int maxMinibatches = (conf.getMaxBatchesPerWorker() > 0 ? conf.getMaxBatchesPerWorker() : Integer.MAX_VALUE); + + if (maxMinibatches == Integer.MAX_VALUE) { + maxDataSetObjects = Integer.MAX_VALUE; + } else { + //Required: total number of examples / examples per dataset object + maxDataSetObjects = + (int) Math.ceil(maxMinibatches * workerMinibatchSize / ((double) dataSetObjectNumExamples)); + } + } + + @Override + public Iterator call(Iterator iter) throws Exception { + List list = new ArrayList<>(); + int count = 0; + while (iter.hasNext() && count++ < maxDataSetObjects) { + list.add(iter.next()); + } + + return workerFlatMap.call(new PathSparkDataSetIterator(list.iterator(), dataSetLoader, hadoopConfig)); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPathMDSFlatMap.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPathMDSFlatMap.java new file mode 100644 index 000000000..b012f3a0d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/ExecuteWorkerPathMDSFlatMap.java @@ -0,0 +1,76 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.worker; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.api.WorkerConfiguration; +import org.deeplearning4j.spark.iterator.PathSparkMultiDataSetIterator; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class ExecuteWorkerPathMDSFlatMap implements FlatMapFunction, R> { + private final FlatMapFunction, R> workerFlatMap; + private MultiDataSetLoader loader; + private final int maxDataSetObjects; + private final Broadcast hadoopConfig; + + public ExecuteWorkerPathMDSFlatMap(TrainingWorker worker, MultiDataSetLoader loader, Broadcast hadoopConfig) { + this.workerFlatMap = new ExecuteWorkerMultiDataSetFlatMap<>(worker); + this.loader = loader; + this.hadoopConfig = hadoopConfig; + + //How many dataset objects of size 'dataSetObjectNumExamples' should we load? + //Only pass on the required number, not all of them (to avoid async preloading data that won't be used) + //Most of the time we'll get exactly the number we want, but this isn't guaranteed all the time for all + // splitting strategies + WorkerConfiguration conf = worker.getDataConfiguration(); + int dataSetObjectNumExamples = conf.getDataSetObjectSizeExamples(); + int workerMinibatchSize = conf.getBatchSizePerWorker(); + int maxMinibatches = (conf.getMaxBatchesPerWorker() > 0 ? conf.getMaxBatchesPerWorker() : Integer.MAX_VALUE); + + if (maxMinibatches == Integer.MAX_VALUE) { + maxDataSetObjects = Integer.MAX_VALUE; + } else { + //Required: total number of examples / examples per dataset object + maxDataSetObjects = + (int) Math.ceil(maxMinibatches * workerMinibatchSize / ((double) dataSetObjectNumExamples)); + } + } + + @Override + public Iterator call(Iterator iter) throws Exception { + List list = new ArrayList<>(); + int count = 0; + while (iter.hasNext() && count++ < maxDataSetObjects) { + list.add(iter.next()); + } + + return workerFlatMap.call(new PathSparkMultiDataSetIterator(list.iterator(), loader, hadoopConfig)); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/NetBroadcastTuple.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/NetBroadcastTuple.java new file mode 100644 index 000000000..9fa317026 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/api/worker/NetBroadcastTuple.java @@ -0,0 +1,63 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.api.worker; + +import lombok.Data; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicInteger; + +@Data +public class NetBroadcastTuple implements Serializable { + + private final MultiLayerConfiguration configuration; + private final ComputationGraphConfiguration graphConfiguration; + private final INDArray parameters; + private final INDArray updaterState; + private final AtomicInteger counter; + + public NetBroadcastTuple(MultiLayerConfiguration configuration, INDArray parameters, INDArray updaterState) { + this(configuration, null, parameters, updaterState); + } + + public NetBroadcastTuple(ComputationGraphConfiguration graphConfiguration, INDArray parameters, + INDArray updaterState) { + this(null, graphConfiguration, parameters, updaterState); + + } + + public NetBroadcastTuple(MultiLayerConfiguration configuration, ComputationGraphConfiguration graphConfiguration, + INDArray parameters, INDArray updaterState) { + this(configuration, graphConfiguration, parameters, updaterState, new AtomicInteger(0)); + } + + public NetBroadcastTuple(MultiLayerConfiguration configuration, ComputationGraphConfiguration graphConfiguration, + INDArray parameters, INDArray updaterState, AtomicInteger counter) { + this.configuration = configuration; + this.graphConfiguration = graphConfiguration; + this.parameters = parameters; + this.updaterState = updaterState; + this.counter = counter; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchAndExportDataSetsFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchAndExportDataSetsFunction.java new file mode 100644 index 000000000..ac9a0a256 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchAndExportDataSetsFunction.java @@ -0,0 +1,153 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.function.Function2; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.common.primitives.Pair; + +import java.net.URI; +import java.util.*; + +public class BatchAndExportDataSetsFunction implements Function2, Iterator> { + private final int minibatchSize; + private final String exportBaseDirectory; + private final String jvmuid; + private final Broadcast conf; + + /** + * @param minibatchSize Minibatch size to combine examples to (if necessary) + * @param exportBaseDirectory Base directory for exporting + */ + public BatchAndExportDataSetsFunction(int minibatchSize, String exportBaseDirectory) { + this(minibatchSize, exportBaseDirectory, null); + } + + /** + * @param minibatchSize Minibatch size to combine examples to (if necessary) + * @param exportBaseDirectory Base directory for exporting + * @param configuration Hadoop Configuration + */ + public BatchAndExportDataSetsFunction(int minibatchSize, String exportBaseDirectory, Broadcast configuration) { + this.minibatchSize = minibatchSize; + this.exportBaseDirectory = exportBaseDirectory; + String fullUID = UIDProvider.getJVMUID(); + this.jvmuid = (fullUID.length() <= 8 ? fullUID : fullUID.substring(0, 8)); + this.conf = configuration; + } + + @Override + public Iterator call(Integer partitionIdx, Iterator iterator) throws Exception { + + List outputPaths = new ArrayList<>(); + LinkedList tempList = new LinkedList<>(); + + int count = 0; + while (iterator.hasNext()) { + DataSet next = iterator.next(); + if (next.numExamples() == minibatchSize) { + outputPaths.add(export(next, partitionIdx, count++)); + continue; + } + //DataSet must be either smaller or larger than minibatch size... + tempList.add(next); + Pair> countAndPaths = processList(tempList, partitionIdx, count, false); + if (countAndPaths.getSecond() != null && !countAndPaths.getSecond().isEmpty()) { + outputPaths.addAll(countAndPaths.getSecond()); + } + count = countAndPaths.getFirst(); + } + + //We might have some left-over examples... + Pair> countAndPaths = processList(tempList, partitionIdx, count, true); + if (countAndPaths.getSecond() != null && !countAndPaths.getSecond().isEmpty()) { + outputPaths.addAll(countAndPaths.getSecond()); + } + + return outputPaths.iterator(); + } + + private Pair> processList(LinkedList tempList, int partitionIdx, int countBefore, + boolean finalExport) throws Exception { + //Go through the list. If we have enough examples: remove the DataSet objects, merge and export them. Otherwise: do nothing + int numExamples = 0; + for (DataSet ds : tempList) { + numExamples += ds.numExamples(); + } + + if (tempList.isEmpty() || (numExamples < minibatchSize && !finalExport)) { + //No op + return new Pair<>(countBefore, Collections.emptyList()); + } + + List exportPaths = new ArrayList<>(); + + int countAfter = countBefore; + + //Batch the required number together + int countSoFar = 0; + List tempToMerge = new ArrayList<>(); + while (!tempList.isEmpty() && countSoFar != minibatchSize) { + DataSet next = tempList.removeFirst(); + if (countSoFar + next.numExamples() <= minibatchSize) { + //Add the entire DataSet object + tempToMerge.add(next); + countSoFar += next.numExamples(); + } else { + //Split the DataSet + List examples = next.asList(); + for (DataSet ds : examples) { + tempList.addFirst(ds); + } + } + } + //At this point: we should have the required number of examples in tempToMerge (unless it's a final export) + DataSet toExport = DataSet.merge(tempToMerge); + exportPaths.add(export(toExport, partitionIdx, countAfter++)); + + return new Pair<>(countAfter, exportPaths); + } + + private String export(DataSet dataSet, int partitionIdx, int outputCount) throws Exception { + String filename = "dataset_" + partitionIdx + jvmuid + "_" + outputCount + ".bin"; + + URI uri = new URI(exportBaseDirectory + + (exportBaseDirectory.endsWith("/") || exportBaseDirectory.endsWith("\\") ? "" : "/") + + filename); + + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + + FileSystem file = FileSystem.get(uri, c); + try (FSDataOutputStream out = file.create(new Path(uri))) { + dataSet.save(out); + } + + return uri.toString(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchAndExportMultiDataSetsFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchAndExportMultiDataSetsFunction.java new file mode 100644 index 000000000..b7e30b351 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchAndExportMultiDataSetsFunction.java @@ -0,0 +1,155 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.function.Function2; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.common.primitives.Pair; + +import java.net.URI; +import java.util.*; + +public class BatchAndExportMultiDataSetsFunction + implements Function2, Iterator> { + + private final int minibatchSize; + private final String exportBaseDirectory; + private final String jvmuid; + private final Broadcast conf; + + /** + * @param minibatchSize Minibatch size to combine examples to (if necessary) + * @param exportBaseDirectory Base directory for exporting + */ + public BatchAndExportMultiDataSetsFunction(int minibatchSize, String exportBaseDirectory) { + this(minibatchSize, exportBaseDirectory, null); + } + + /** + * @param minibatchSize Minibatch size to combine examples to (if necessary) + * @param exportBaseDirectory Base directory for exporting + * @param configuration Hadoop Configuration + */ + public BatchAndExportMultiDataSetsFunction(int minibatchSize, String exportBaseDirectory, Broadcast configuration) { + this.minibatchSize = minibatchSize; + this.exportBaseDirectory = exportBaseDirectory; + String fullUID = UIDProvider.getJVMUID(); + this.jvmuid = (fullUID.length() <= 8 ? fullUID : fullUID.substring(0, 8)); + this.conf = configuration; + } + + @Override + public Iterator call(Integer partitionIdx, Iterator iterator) throws Exception { + + List outputPaths = new ArrayList<>(); + LinkedList tempList = new LinkedList<>(); + + int count = 0; + while (iterator.hasNext()) { + MultiDataSet next = iterator.next(); + if (next.getFeatures(0).size(0) == minibatchSize) { + outputPaths.add(export(next, partitionIdx, count++)); + continue; + } + //DataSet must be either smaller or larger than minibatch size... + tempList.add(next); + Pair> countAndPaths = processList(tempList, partitionIdx, count, false); + if (countAndPaths.getSecond() != null && !countAndPaths.getSecond().isEmpty()) { + outputPaths.addAll(countAndPaths.getSecond()); + } + count = countAndPaths.getFirst(); + } + + //We might have some left-over examples... + Pair> countAndPaths = processList(tempList, partitionIdx, count, true); + if (countAndPaths.getSecond() != null && !countAndPaths.getSecond().isEmpty()) { + outputPaths.addAll(countAndPaths.getSecond()); + } + + return outputPaths.iterator(); + } + + private Pair> processList(LinkedList tempList, int partitionIdx, + int countBefore, boolean finalExport) throws Exception { + //Go through the list. If we have enough examples: remove the DataSet objects, merge and export them. Otherwise: do nothing + int numExamples = 0; + for (MultiDataSet ds : tempList) { + numExamples += ds.getFeatures(0).size(0); + } + + if (tempList.isEmpty() || (numExamples < minibatchSize && !finalExport)) { + //No op + return new Pair<>(countBefore, Collections.emptyList()); + } + + List exportPaths = new ArrayList<>(); + + int countAfter = countBefore; + + //Batch the required number together + int countSoFar = 0; + List tempToMerge = new ArrayList<>(); + while (!tempList.isEmpty() && countSoFar != minibatchSize) { + MultiDataSet next = tempList.removeFirst(); + if (countSoFar + next.getFeatures(0).size(0) <= minibatchSize) { + //Add the entire DataSet object + tempToMerge.add(next); + countSoFar += next.getFeatures(0).size(0); + } else { + //Split the DataSet + List examples = next.asList(); + for (MultiDataSet ds : examples) { + tempList.addFirst(ds); + } + } + } + //At this point: we should have the required number of examples in tempToMerge (unless it's a final export) + MultiDataSet toExport = org.nd4j.linalg.dataset.MultiDataSet.merge(tempToMerge); + exportPaths.add(export(toExport, partitionIdx, countAfter++)); + + return new Pair<>(countAfter, exportPaths); + } + + private String export(MultiDataSet dataSet, int partitionIdx, int outputCount) throws Exception { + String filename = "mds_" + partitionIdx + jvmuid + "_" + outputCount + ".bin"; + + URI uri = new URI(exportBaseDirectory + + (exportBaseDirectory.endsWith("/") || exportBaseDirectory.endsWith("\\") ? "" : "/") + + filename); + + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + + FileSystem file = FileSystem.get(uri, c); + try (FSDataOutputStream out = file.create(new Path(uri))) { + dataSet.save(out); + } + + return uri.toString(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchDataSetsFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchDataSetsFunction.java new file mode 100644 index 000000000..4b05e63eb --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/BatchDataSetsFunction.java @@ -0,0 +1,58 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import lombok.AllArgsConstructor; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.nd4j.linalg.dataset.DataSet; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +@AllArgsConstructor +public class BatchDataSetsFunction implements FlatMapFunction, DataSet> { + private final int minibatchSize; + + @Override + public Iterator call(Iterator iter) throws Exception { + List out = new ArrayList<>(); + while (iter.hasNext()) { + List list = new ArrayList<>(); + + int count = 0; + while (count < minibatchSize && iter.hasNext()) { + DataSet ds = iter.next(); + count += ds.getFeatures().size(0); + list.add(ds); + } + + DataSet next; + if (list.isEmpty()) + next = list.get(0); + else + next = DataSet.merge(list); + + out.add(next); + } + return out.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/DataSetExportFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/DataSetExportFunction.java new file mode 100644 index 000000000..8009075e1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/DataSetExportFunction.java @@ -0,0 +1,74 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.function.VoidFunction; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.linalg.dataset.DataSet; + +import java.net.URI; +import java.util.Iterator; + +public class DataSetExportFunction implements VoidFunction> { + + private final URI outputDir; + private final Broadcast conf; + private String uid = null; + + private int outputCount; + + public DataSetExportFunction(URI outputDir) { + this(outputDir, null); + } + + public DataSetExportFunction(URI outputDir, Broadcast configuration) { + this.outputDir = outputDir; + this.conf = configuration; + } + + @Override + public void call(Iterator iter) throws Exception { + String jvmuid = UIDProvider.getJVMUID(); + uid = Thread.currentThread().getId() + jvmuid.substring(0, Math.min(8, jvmuid.length())); + + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + + while (iter.hasNext()) { + DataSet next = iter.next(); + + String filename = "dataset_" + uid + "_" + (outputCount++) + ".bin"; + + String path = outputDir.getPath(); + URI uri = new URI(path + (path.endsWith("/") || path.endsWith("\\") ? "" : "/") + filename); + FileSystem file = FileSystem.get(uri, c); + try (FSDataOutputStream out = file.create(new Path(uri))) { + next.save(out); + } + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/DataSetProvider.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/DataSetProvider.java new file mode 100644 index 000000000..3aba24292 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/DataSetProvider.java @@ -0,0 +1,48 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.spark.SparkContext; +import org.apache.spark.rdd.RDD; +import org.datavec.api.transform.TransformProcess; +import org.nd4j.linalg.dataset.DataSet; + +/** + * A provider for an {@link DataSet} + * rdd. + * @author Adam Gibson + */ +public interface DataSetProvider { + + /** + * Return an rdd of type dataset + * @return + */ + RDD data(SparkContext sparkContext); + + /** + * (Optional) The transform process + * for the dataset provider. + * @return + */ + TransformProcess transformProcess(); + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/MultiDataSetExportFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/MultiDataSetExportFunction.java new file mode 100644 index 000000000..31c29562f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/MultiDataSetExportFunction.java @@ -0,0 +1,73 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.function.VoidFunction; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.net.URI; +import java.util.Iterator; + +public class MultiDataSetExportFunction implements VoidFunction> { + private final URI outputDir; + private final Broadcast conf; + private String uid = null; + + private int outputCount; + + public MultiDataSetExportFunction(URI outputDir) { + this(outputDir, null); + } + + public MultiDataSetExportFunction(URI outputDir, Broadcast configuration) { + this.outputDir = outputDir; + this.conf = configuration; + } + + @Override + public void call(Iterator iter) throws Exception { + String jvmuid = UIDProvider.getJVMUID(); + uid = Thread.currentThread().getId() + jvmuid.substring(0, Math.min(8, jvmuid.length())); + + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + + while (iter.hasNext()) { + MultiDataSet next = iter.next(); + + String filename = "mds_" + uid + "_" + (outputCount++) + ".bin"; + + String path = outputDir.getPath(); + URI uri = new URI(path + (path.endsWith("/") || path.endsWith("\\") ? "" : "/") + filename); + FileSystem file = FileSystem.get(uri, c); + try (FSDataOutputStream out = file.create(new Path(uri))) { + next.save(out); + } + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/MultiDataSetProvider.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/MultiDataSetProvider.java new file mode 100644 index 000000000..628d6d698 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/MultiDataSetProvider.java @@ -0,0 +1,50 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.spark.SparkContext; +import org.apache.spark.rdd.RDD; +import org.datavec.api.transform.TransformProcess; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +/** + * A provider for an {@link MultiDataSet} + * rdd. + * @author Adam Gibson + */ +public interface MultiDataSetProvider { + + + /** + * Return an rdd of type dataset + * @return + */ + RDD data(SparkContext sparkContext); + + + /** + * (Optional) The transform process + * for the dataset provider. + * @return + */ + TransformProcess transformProcess(); + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/PathToDataSetFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/PathToDataSetFunction.java new file mode 100644 index 000000000..8b2cd43d7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/PathToDataSetFunction.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.IOException; +import java.net.URI; + +public class PathToDataSetFunction implements Function { + public static final int BUFFER_SIZE = 4194304; //4 MB + + private transient FileSystem fileSystem; + private final Broadcast conf; + + public PathToDataSetFunction(){ + this(null); + } + + public PathToDataSetFunction(Broadcast configuration){ + this.conf = configuration; + } + + @Override + public DataSet call(String path) throws Exception { + if (fileSystem == null) { + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + try { + fileSystem = FileSystem.get(new URI(path), c); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + DataSet ds = new DataSet(); + try (FSDataInputStream inputStream = fileSystem.open(new Path(path), BUFFER_SIZE)) { + ds.load(inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return ds; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/PathToMultiDataSetFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/PathToMultiDataSetFunction.java new file mode 100644 index 000000000..e437a7da0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/PathToMultiDataSetFunction.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.io.IOException; +import java.net.URI; + +public class PathToMultiDataSetFunction implements Function { + public static final int BUFFER_SIZE = 4194304; //4 MB + + private transient FileSystem fileSystem; + private final Broadcast conf; + + public PathToMultiDataSetFunction(){ + this(null); + } + + public PathToMultiDataSetFunction(Broadcast configuration){ + this.conf = configuration; + } + + @Override + public MultiDataSet call(String path) throws Exception { + if (fileSystem == null) { + try { + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + fileSystem = FileSystem.get(new URI(path), c); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + MultiDataSet ds = new org.nd4j.linalg.dataset.MultiDataSet(); + try (FSDataInputStream inputStream = fileSystem.open(new Path(path), BUFFER_SIZE)) { + ds.load(inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return ds; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/SplitDataSetsFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/SplitDataSetsFunction.java new file mode 100644 index 000000000..fdd65b661 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/SplitDataSetsFunction.java @@ -0,0 +1,39 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.nd4j.linalg.dataset.DataSet; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class SplitDataSetsFunction implements FlatMapFunction, DataSet> { + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + List out = new ArrayList<>(); + while (dataSetIterator.hasNext()) { + out.addAll(dataSetIterator.next().asList()); + } + return out.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/loader/RemoteFileSource.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/loader/RemoteFileSource.java new file mode 100644 index 000000000..d0573686e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/loader/RemoteFileSource.java @@ -0,0 +1,48 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data.loader; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.nd4j.common.loader.Source; + +import java.io.IOException; +import java.io.InputStream; + +@AllArgsConstructor +public class RemoteFileSource implements Source { + public static final int DEFAULT_BUFFER_SIZE = 4*1024*2014; + @Getter + private String path; + private final FileSystem fileSystem; + private final int bufferSize; + + public RemoteFileSource(String path, FileSystem fileSystem){ + this(path, fileSystem, DEFAULT_BUFFER_SIZE); + } + + @Override + public InputStream getInputStream() throws IOException { + return fileSystem.open(new Path(path), bufferSize); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/loader/RemoteFileSourceFactory.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/loader/RemoteFileSourceFactory.java new file mode 100644 index 000000000..fe3e94cac --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/loader/RemoteFileSourceFactory.java @@ -0,0 +1,57 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data.loader; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.nd4j.common.loader.Source; +import org.nd4j.common.loader.SourceFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + + +public class RemoteFileSourceFactory implements SourceFactory { + private transient FileSystem fileSystem; + private final Broadcast conf; + + public RemoteFileSourceFactory(Broadcast configuration){ + this.conf = configuration; + } + + @Override + public Source getSource(String path) { + if(fileSystem == null){ + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + try { + fileSystem = FileSystem.get(new URI(path), c); + } catch (IOException | URISyntaxException u){ + throw new RuntimeException(u); + } + } + + return new RemoteFileSource(path, fileSystem); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/shuffle/SplitDataSetExamplesPairFlatMapFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/shuffle/SplitDataSetExamplesPairFlatMapFunction.java new file mode 100644 index 000000000..f6b12a1eb --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/data/shuffle/SplitDataSetExamplesPairFlatMapFunction.java @@ -0,0 +1,57 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data.shuffle; + +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.api.java.function.PairFlatMapFunction; +import org.nd4j.linalg.dataset.DataSet; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +public class SplitDataSetExamplesPairFlatMapFunction implements PairFlatMapFunction { + + private transient Random r; + private int maxKeyIndex; + + public SplitDataSetExamplesPairFlatMapFunction(int maxKeyIndex) { + this.maxKeyIndex = maxKeyIndex; + } + + @Override + public Iterator> call(DataSet dataSet) throws Exception { + if (r == null) { + r = new Random(); + } + + List singleExamples = dataSet.asList(); + List> out = new ArrayList<>(singleExamples.size()); + for (DataSet ds : singleExamples) { + out.add(new Tuple2<>(r.nextInt(maxKeyIndex), ds)); + } + + return out.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecByteDataSetFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecByteDataSetFunction.java new file mode 100644 index 000000000..f8413037b --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecByteDataSetFunction.java @@ -0,0 +1,132 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.Text; +import org.apache.spark.api.java.function.PairFunction; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.DataSetPreProcessor; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.util.FeatureUtil; +import scala.Tuple2; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + */ +@Slf4j +public class DataVecByteDataSetFunction implements PairFunction, Double, DataSet> { + + private int labelIndex = 0; + private int numPossibleLabels; + private int byteFileLen; + private int batchSize; + private int numExamples; + private boolean regression = false; + private DataSetPreProcessor preProcessor; + + public DataVecByteDataSetFunction(int labelIndex, int numPossibleLabels, int batchSize, int byteFileLen) { + this(labelIndex, numPossibleLabels, batchSize, byteFileLen, false, null); + } + + public DataVecByteDataSetFunction(int labelIndex, int numPossibleLabels, int batchSize, int byteFileLen, + boolean regression) { + this(labelIndex, numPossibleLabels, batchSize, byteFileLen, regression, null); + } + + /** + * @param labelIndex Index of the label column + * @param numPossibleLabels Number of classes for classification (not used if regression = true) + * @param batchSize size of examples in DataSet. Pass in total examples if including all + * @param byteFileLen number of bytes per individual file + * @param regression False for classification, true for regression + * @param preProcessor DataSetPreprocessor (may be null) + */ + public DataVecByteDataSetFunction(int labelIndex, int numPossibleLabels, int batchSize, int byteFileLen, + boolean regression, DataSetPreProcessor preProcessor) { + this.labelIndex = labelIndex; + this.numPossibleLabels = numPossibleLabels; + this.batchSize = batchSize; + this.byteFileLen = byteFileLen; + this.regression = regression; + this.preProcessor = preProcessor; + + } + + @Override + public Tuple2 call(Tuple2 inputTuple) throws Exception { + int lenFeatureVector = 0; + + if (numPossibleLabels >= 1) { + lenFeatureVector = byteFileLen - 1; + if (labelIndex < 0) + labelIndex = byteFileLen - 1; + } + + InputStream inputStream = new DataInputStream(new ByteArrayInputStream(inputTuple._2().getBytes())); + + int batchNumCount = 0; + byte[] byteFeature = new byte[byteFileLen]; + List dataSets = new ArrayList<>(); + INDArray label; + int featureCount; + + try { + INDArray featureVector = Nd4j.create(lenFeatureVector); + while ((inputStream.read(byteFeature)) != -1 && batchNumCount != batchSize) { + featureCount = 0; + label = FeatureUtil.toOutcomeVector(byteFeature[labelIndex], numPossibleLabels); + for (int j = 1; j <= featureVector.length(); j++) + featureVector.putScalar(featureCount++, byteFeature[j]); + dataSets.add(new DataSet(featureVector, label)); + batchNumCount++; + byteFeature = new byte[byteFileLen]; + featureVector = Nd4j.create(lenFeatureVector); + } + } catch (IOException e) { + log.error("",e); + } + + List inputs = new ArrayList<>(); + List labels = new ArrayList<>(); + + for (DataSet data : dataSets) { + inputs.add(data.getFeatures()); + labels.add(data.getLabels()); + } + + DataSet ds = new DataSet(Nd4j.vstack(inputs.toArray(new INDArray[0])), + Nd4j.vstack(labels.toArray(new INDArray[0]))); + if (preProcessor != null) + preProcessor.preProcess(ds); + return new Tuple2<>((double) batchNumCount, ds); + + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecDataSetFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecDataSetFunction.java new file mode 100644 index 000000000..bc72ced72 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecDataSetFunction.java @@ -0,0 +1,185 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.function.Function; +import org.datavec.api.io.WritableConverter; +import org.datavec.api.io.converters.WritableConverterException; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.DataSetPreProcessor; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.util.FeatureUtil; + +import java.io.Serializable; +import java.util.List; + +@Slf4j +public class DataVecDataSetFunction implements Function, DataSet>, Serializable { + + private final int labelIndex; + private final int labelIndexTo; + private final int numPossibleLabels; + private final boolean regression; + private final DataSetPreProcessor preProcessor; + private final WritableConverter converter; + protected int batchSize = -1; + + public DataVecDataSetFunction(int labelIndex, int numPossibleLabels, boolean regression) { + this(labelIndex, numPossibleLabels, regression, null, null); + } + + /** + * @param labelIndex Index of the label column + * @param numPossibleLabels Number of classes for classification (not used if regression = true) + * @param regression False for classification, true for regression + * @param preProcessor DataSetPreprocessor (may be null) + * @param converter WritableConverter (may be null) + */ + public DataVecDataSetFunction(int labelIndex, int numPossibleLabels, boolean regression, + DataSetPreProcessor preProcessor, WritableConverter converter) { + this(labelIndex, labelIndex, numPossibleLabels, regression, preProcessor, converter); + } + + /** + * Main constructor, including for multi-label regression + * + * @param labelIndexFrom Index of the first target + * @param labelIndexTo Index of the last target, inclusive (for classification or single-output regression: same as labelIndexFrom) + * @param numPossibleLabels Unused for regression, or number of classes for classification + * @param regression If true: regression. false: classification + */ + public DataVecDataSetFunction(int labelIndexFrom, int labelIndexTo, int numPossibleLabels, boolean regression, + DataSetPreProcessor preProcessor, WritableConverter converter) { + this.labelIndex = labelIndexFrom; + this.labelIndexTo = labelIndexTo; + this.numPossibleLabels = numPossibleLabels; + this.regression = regression; + this.preProcessor = preProcessor; + this.converter = converter; + } + + @Override + public DataSet call(List currList) throws Exception { + + //allow people to specify label index as -1 and infer the last possible label + int labelIndex = this.labelIndex; + if (numPossibleLabels >= 1 && labelIndex < 0) { + labelIndex = currList.size() - 1; + } + + INDArray label = null; + INDArray featureVector = null; + int featureCount = 0; + int labelCount = 0; + + //no labels + if (currList.size() == 2 && currList.get(1) instanceof NDArrayWritable + && currList.get(0) instanceof NDArrayWritable && currList.get(0) == currList.get(1)) { + NDArrayWritable writable = (NDArrayWritable) currList.get(0); + DataSet ds = new DataSet(writable.get(), writable.get()); + if (preProcessor != null) + preProcessor.preProcess(ds); + return ds; + } + if (currList.size() == 2 && currList.get(0) instanceof NDArrayWritable) { + if (!regression) + label = FeatureUtil.toOutcomeVector((int) Double.parseDouble(currList.get(1).toString()), + numPossibleLabels); + else + label = Nd4j.scalar(Double.parseDouble(currList.get(1).toString())).reshape(1,1); + NDArrayWritable ndArrayWritable = (NDArrayWritable) currList.get(0); + featureVector = ndArrayWritable.get(); + DataSet ds = new DataSet(featureVector, label); + if (preProcessor != null) + preProcessor.preProcess(ds); + return ds; + } + + for (int j = 0; j < currList.size(); j++) { + Writable current = currList.get(j); + //ndarray writable is an insane slow down here + if (!(current instanceof NDArrayWritable) && current.toString().isEmpty()) + continue; + + if (labelIndex >= 0 && j >= labelIndex && j <= labelIndexTo) { + //single label case (classification, single label regression etc) + if (converter != null) { + try { + current = converter.convert(current); + } catch (WritableConverterException e) { + + log.error("",e); + } + } + if (regression) { + //single and multi-label regression + if (label == null) { + label = Nd4j.zeros(1, labelIndexTo - labelIndex + 1); + } + label.putScalar(0, labelCount++, current.toDouble()); + } else { + if (numPossibleLabels < 1) + throw new IllegalStateException( + "Number of possible labels invalid, must be >= 1 for classification"); + int curr = current.toInt(); + if (curr >= numPossibleLabels) + throw new IllegalStateException( + "Invalid index: got index " + curr + " but numPossibleLabels is " + + numPossibleLabels + " (must be 0 <= idx < numPossibleLabels"); + label = FeatureUtil.toOutcomeVector(curr, numPossibleLabels); + } + } else { + try { + double value = current.toDouble(); + if (featureVector == null) { + if (regression && labelIndex >= 0) { + //Handle the possibly multi-label regression case here: + int nLabels = labelIndexTo - labelIndex + 1; + featureVector = Nd4j.create(1, currList.size() - nLabels); + } else { + //Classification case, and also no-labels case + featureVector = Nd4j.create(1, labelIndex >= 0 ? currList.size() - 1 : currList.size()); + } + } + featureVector.putScalar(featureCount++, value); + } catch (UnsupportedOperationException e) { + // This isn't a scalar, so check if we got an array already + if (current instanceof NDArrayWritable) { + Preconditions.checkState(featureVector == null, "Already got an array"); + featureVector = ((NDArrayWritable) current).get(); + } else { + throw e; + } + } + } + } + + DataSet ds = new DataSet(featureVector, (labelIndex >= 0 ? label : featureVector)); + if (preProcessor != null) + preProcessor.preProcess(ds); + return ds; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecSequenceDataSetFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecSequenceDataSetFunction.java new file mode 100644 index 000000000..025a1f4c0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecSequenceDataSetFunction.java @@ -0,0 +1,131 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import org.apache.spark.api.java.function.Function; +import org.datavec.api.io.WritableConverter; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; +import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.DataSetPreProcessor; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.util.FeatureUtil; + +import java.io.Serializable; +import java.util.Iterator; +import java.util.List; + +public class DataVecSequenceDataSetFunction implements Function>, DataSet>, Serializable { + + private final boolean regression; + private final int labelIndex; + private final int numPossibleLabels; + private final DataSetPreProcessor preProcessor; + private final WritableConverter converter; + + /** + * @param labelIndex Index of the label column + * @param numPossibleLabels Number of classes for classification (not used if regression = true) + * @param regression False for classification, true for regression + */ + public DataVecSequenceDataSetFunction(int labelIndex, int numPossibleLabels, boolean regression) { + this(labelIndex, numPossibleLabels, regression, null, null); + } + + /** + * @param labelIndex Index of the label column + * @param numPossibleLabels Number of classes for classification (not used if regression = true) + * @param regression False for classification, true for regression + * @param preProcessor DataSetPreprocessor (may be null) + * @param converter WritableConverter (may be null) + */ + public DataVecSequenceDataSetFunction(int labelIndex, int numPossibleLabels, boolean regression, + DataSetPreProcessor preProcessor, WritableConverter converter) { + this.labelIndex = labelIndex; + this.numPossibleLabels = numPossibleLabels; + this.regression = regression; + this.preProcessor = preProcessor; + this.converter = converter; + } + + + @Override + public DataSet call(List> input) throws Exception { + Iterator> iter = input.iterator(); + + INDArray features = null; + INDArray labels = Nd4j.zeros(1, (regression ? 1 : numPossibleLabels), input.size()); + + int[] fIdx = new int[3]; + int[] lIdx = new int[3]; + + int i = 0; + while (iter.hasNext()) { + List step = iter.next(); + if (i == 0) { + features = Nd4j.zeros(1, step.size() - 1, input.size()); + } + + Iterator timeStepIter = step.iterator(); + int countIn = 0; + int countFeatures = 0; + while (timeStepIter.hasNext()) { + Writable current = timeStepIter.next(); + if (converter != null) + current = converter.convert(current); + if (countIn++ == labelIndex) { + //label + if (regression) { + lIdx[2] = i; + labels.putScalar(lIdx, current.toDouble()); + } else { + INDArray line = FeatureUtil.toOutcomeVector(current.toInt(), numPossibleLabels); + labels.tensorAlongDimension(i, 1).assign(line); //1d from [1,nOut,timeSeriesLength] -> tensor i along dimension 1 is at time i + } + } else { + //feature + fIdx[1] = countFeatures++; + fIdx[2] = i; + try { + features.putScalar(fIdx, current.toDouble()); + } catch (UnsupportedOperationException e) { + // This isn't a scalar, so check if we got an array already + if (current instanceof NDArrayWritable) { + features.get(NDArrayIndex.point(fIdx[0]), NDArrayIndex.all(), NDArrayIndex.point(fIdx[2])) + .putRow(0, ((NDArrayWritable) current).get()); + } else { + throw e; + } + } + } + } + i++; + } + + DataSet ds = new DataSet(features, labels); + if (preProcessor != null) + preProcessor.preProcess(ds); + return ds; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecSequencePairDataSetFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecSequencePairDataSetFunction.java new file mode 100644 index 000000000..1ad5a7cfd --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/DataVecSequencePairDataSetFunction.java @@ -0,0 +1,230 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import org.apache.spark.api.java.function.Function; +import org.datavec.api.io.WritableConverter; +import org.datavec.api.writable.NDArrayWritable; +import org.datavec.api.writable.Writable; +import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.DataSetPreProcessor; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.util.FeatureUtil; +import scala.Tuple2; + +import java.io.Serializable; +import java.util.Iterator; +import java.util.List; + +public class DataVecSequencePairDataSetFunction + implements Function>, List>>, DataSet>, Serializable { + /**Alignment mode for dealing with input/labels of differing lengths (for example, one-to-many and many-to-one type situations). + * For example, might have 10 time steps total but only one label at end for sequence classification.
+ * EQUAL_LENGTH: Default. Assume that label and input time series are of equal length
+ * ALIGN_START: Align the label/input time series at the first time step, and zero pad either the labels or + * the input at the end (pad whichever is shorter)
+ * ALIGN_END: Align the label/input at the last time step, zero padding either the input or the labels as required
+ */ + public enum AlignmentMode { + EQUAL_LENGTH, ALIGN_START, ALIGN_END + } + + private final boolean regression; + private final int numPossibleLabels; + private final AlignmentMode alignmentMode; + private final DataSetPreProcessor preProcessor; + private final WritableConverter converter; + + /** Constructor for equal length and no conversion of labels (i.e., regression or already in one-hot representation). + * No data set proprocessor or writable converter + */ + public DataVecSequencePairDataSetFunction() { + this(-1, true); + } + + /**Constructor for equal length, no data set preprocessor or writable converter + * @see #DataVecSequencePairDataSetFunction(int, boolean, AlignmentMode, DataSetPreProcessor, WritableConverter) + */ + public DataVecSequencePairDataSetFunction(int numPossibleLabels, boolean regression) { + this(numPossibleLabels, regression, AlignmentMode.EQUAL_LENGTH); + } + + /**Constructor for data with a specified alignment mode, no data set preprocessor or writable converter + * @see #DataVecSequencePairDataSetFunction(int, boolean, AlignmentMode, DataSetPreProcessor, WritableConverter) + */ + public DataVecSequencePairDataSetFunction(int numPossibleLabels, boolean regression, AlignmentMode alignmentMode) { + this(numPossibleLabels, regression, alignmentMode, null, null); + } + + /** + * @param numPossibleLabels Number of classes for classification (not used if regression = true) + * @param regression False for classification, true for regression + * @param alignmentMode Alignment mode for data. See {@link DataVecSequencePairDataSetFunction.AlignmentMode} + * @param preProcessor DataSetPreprocessor (may be null) + * @param converter WritableConverter (may be null) + */ + public DataVecSequencePairDataSetFunction(int numPossibleLabels, boolean regression, AlignmentMode alignmentMode, + DataSetPreProcessor preProcessor, WritableConverter converter) { + this.numPossibleLabels = numPossibleLabels; + this.regression = regression; + this.alignmentMode = alignmentMode; + this.preProcessor = preProcessor; + this.converter = converter; + } + + + @Override + public DataSet call(Tuple2>, List>> input) throws Exception { + List> featuresSeq = input._1(); + List> labelsSeq = input._2(); + + int featuresLength = featuresSeq.size(); + int labelsLength = labelsSeq.size(); + + + Iterator> fIter = featuresSeq.iterator(); + Iterator> lIter = labelsSeq.iterator(); + + INDArray inputArr = null; + INDArray outputArr = null; + + int[] idx = new int[3]; + int i = 0; + while (fIter.hasNext()) { + List step = fIter.next(); + if (i == 0) { + int[] inShape = new int[] {1, step.size(), featuresLength}; + inputArr = Nd4j.create(inShape); + } + Iterator timeStepIter = step.iterator(); + int f = 0; + idx[1] = 0; + while (timeStepIter.hasNext()) { + Writable current = timeStepIter.next(); + if (converter != null) + current = converter.convert(current); + try { + inputArr.putScalar(idx, current.toDouble()); + } catch (UnsupportedOperationException e) { + // This isn't a scalar, so check if we got an array already + if (current instanceof NDArrayWritable) { + inputArr.get(NDArrayIndex.point(idx[0]), NDArrayIndex.all(), NDArrayIndex.point(idx[2])) + .putRow(0, ((NDArrayWritable) current).get()); + } else { + throw e; + } + } + idx[1] = ++f; + } + idx[2] = ++i; + } + + idx = new int[3]; + i = 0; + while (lIter.hasNext()) { + List step = lIter.next(); + if (i == 0) { + int[] outShape = new int[] {1, (regression ? step.size() : numPossibleLabels), labelsLength}; + outputArr = Nd4j.create(outShape); + } + Iterator timeStepIter = step.iterator(); + int f = 0; + idx[1] = 0; + if (regression) { + //Load all values without modification + while (timeStepIter.hasNext()) { + Writable current = timeStepIter.next(); + if (converter != null) + current = converter.convert(current); + outputArr.putScalar(idx, current.toDouble()); + idx[1] = ++f; + } + } else { + //Expect a single value (index) -> convert to one-hot vector + Writable value = timeStepIter.next(); + int labelClassIdx = value.toInt(); + INDArray line = FeatureUtil.toOutcomeVector(labelClassIdx, numPossibleLabels); + outputArr.tensorAlongDimension(i, 1).assign(line); //1d from [1,nOut,timeSeriesLength] -> tensor i along dimension 1 is at time i + } + + idx[2] = ++i; + } + + DataSet ds; + if (alignmentMode == AlignmentMode.EQUAL_LENGTH || featuresLength == labelsLength) { + ds = new DataSet(inputArr, outputArr); + } else if (alignmentMode == AlignmentMode.ALIGN_END) { + if (featuresLength > labelsLength) { + //Input longer, pad output + INDArray newOutput = Nd4j.create(1, outputArr.size(1), featuresLength); + newOutput.get(NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.interval(featuresLength - labelsLength, featuresLength)).assign(outputArr); + //Need an output mask array, but not an input mask array + INDArray outputMask = Nd4j.create(1, featuresLength); + for (int j = featuresLength - labelsLength; j < featuresLength; j++) + outputMask.putScalar(j, 1.0); + ds = new DataSet(inputArr, newOutput, Nd4j.ones(outputMask.shape()), outputMask); + } else { + //Output longer, pad input + INDArray newInput = Nd4j.create(1, inputArr.size(1), labelsLength); + newInput.get(NDArrayIndex.point(0), NDArrayIndex.all(), + NDArrayIndex.interval(labelsLength - featuresLength, labelsLength)).assign(inputArr); + //Need an input mask array, but not an output mask array + INDArray inputMask = Nd4j.create(1, labelsLength); + for (int j = labelsLength - featuresLength; j < labelsLength; j++) + inputMask.putScalar(j, 1.0); + ds = new DataSet(newInput, outputArr, inputMask, Nd4j.ones(inputMask.shape())); + } + } else if (alignmentMode == AlignmentMode.ALIGN_START) { + if (featuresLength > labelsLength) { + //Input longer, pad output + INDArray newOutput = Nd4j.create(1, outputArr.size(1), featuresLength); + newOutput.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.interval(0, labelsLength)) + .assign(outputArr); + //Need an output mask array, but not an input mask array + INDArray outputMask = Nd4j.create(1, featuresLength); + for (int j = 0; j < labelsLength; j++) + outputMask.putScalar(j, 1.0); + ds = new DataSet(inputArr, newOutput, Nd4j.ones(outputMask.shape()), outputMask); + } else { + //Output longer, pad input + INDArray newInput = Nd4j.create(1, inputArr.size(1), labelsLength); + newInput.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.interval(0, featuresLength)) + .assign(inputArr); + //Need an input mask array, but not an output mask array + INDArray inputMask = Nd4j.create(1, labelsLength); + for (int j = 0; j < featuresLength; j++) + inputMask.putScalar(j, 1.0); + ds = new DataSet(newInput, outputArr, inputMask, Nd4j.ones(inputMask.shape())); + } + } else { + throw new UnsupportedOperationException("Invalid alignment mode: " + alignmentMode); + } + + + if (preProcessor != null) + preProcessor.preProcess(ds); + return ds; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/RDDMiniBatches.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/RDDMiniBatches.java new file mode 100644 index 000000000..4c0da6832 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/RDDMiniBatches.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import lombok.AllArgsConstructor; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class RDDMiniBatches implements Serializable { + private int miniBatches; + private JavaRDD toSplitJava; + + public RDDMiniBatches(int miniBatches, JavaRDD toSplit) { + this.miniBatches = miniBatches; + this.toSplitJava = toSplit; + } + + public JavaRDD miniBatchesJava() { + //need a new mapping function, doesn't handle mini batches properly + return toSplitJava.mapPartitions(new MiniBatchFunction(miniBatches)); + } + + @AllArgsConstructor + public static class MiniBatchFunction implements FlatMapFunction, DataSet> { + private int batchSize; + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + List ret = new ArrayList<>(); + List temp = new ArrayList<>(); + while (dataSetIterator.hasNext()) { + temp.add(dataSetIterator.next().copy()); + if (temp.size() == batchSize) { + ret.add(DataSet.merge(temp)); + temp.clear(); + } + } + + //Add remaining ('left over') data + if (temp.size() > 0) + ret.add(DataSet.merge(temp)); + + return ret.iterator(); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/RecordReaderFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/RecordReaderFunction.java new file mode 100644 index 000000000..8d24bba6a --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/RecordReaderFunction.java @@ -0,0 +1,94 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import org.apache.spark.api.java.function.Function; +import org.datavec.api.io.WritableConverter; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.StringSplit; +import org.datavec.api.writable.Writable; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.util.FeatureUtil; + +import java.util.ArrayList; +import java.util.List; + +public class RecordReaderFunction implements Function { + private RecordReader recordReader; + private int labelIndex = -1; + private int numPossibleLabels = -1; + private WritableConverter converter; + + public RecordReaderFunction(RecordReader recordReader, int labelIndex, int numPossibleLabels, + WritableConverter converter) { + this.recordReader = recordReader; + this.labelIndex = labelIndex; + this.numPossibleLabels = numPossibleLabels; + this.converter = converter; + + } + + public RecordReaderFunction(RecordReader recordReader, int labelIndex, int numPossibleLabels) { + this(recordReader, labelIndex, numPossibleLabels, null); + } + + @Override + public DataSet call(String v1) throws Exception { + recordReader.initialize(new StringSplit(v1)); + List dataSets = new ArrayList<>(); + List currList = recordReader.next(); + + INDArray label = null; + INDArray featureVector = Nd4j.create(1, labelIndex >= 0 ? currList.size() - 1 : currList.size()); + int count = 0; + for (int j = 0; j < currList.size(); j++) { + if (labelIndex >= 0 && j == labelIndex) { + if (numPossibleLabels < 1) + throw new IllegalStateException("Number of possible labels invalid, must be >= 1"); + Writable current = currList.get(j); + if (converter != null) + current = converter.convert(current); + label = FeatureUtil.toOutcomeVector(current.toInt(), numPossibleLabels); + } else { + Writable current = currList.get(j); + featureVector.putScalar(count++, current.toDouble()); + } + } + + dataSets.add(new DataSet(featureVector, labelIndex >= 0 ? label : featureVector)); + + + + List inputs = new ArrayList<>(); + List labels = new ArrayList<>(); + for (DataSet data : dataSets) { + inputs.add(data.getFeatures()); + labels.add(data.getLabels()); + } + + + DataSet ret = new DataSet(Nd4j.vstack(inputs.toArray(new INDArray[inputs.size()])), + Nd4j.vstack(labels.toArray(new INDArray[inputs.size()]))); + return ret; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/export/StringToDataSetExportFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/export/StringToDataSetExportFunction.java new file mode 100644 index 000000000..9242ab798 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/export/StringToDataSetExportFunction.java @@ -0,0 +1,112 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec.export; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.function.VoidFunction; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.collection.CollectionRecordReader; +import org.datavec.api.split.StringSplit; +import org.datavec.api.writable.Writable; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.linalg.dataset.DataSet; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class StringToDataSetExportFunction implements VoidFunction> { + + private final Broadcast conf; + + private final URI outputDir; + private final RecordReader recordReader; + private final int batchSize; + private final boolean regression; + private final int labelIndex; + private final int numPossibleLabels; + private String uid = null; + + private int outputCount; + + public StringToDataSetExportFunction(URI outputDir, RecordReader recordReader, int batchSize, boolean regression, + int labelIndex, int numPossibleLabels) { + this(outputDir, recordReader, batchSize, regression, labelIndex, numPossibleLabels, null); + } + + public StringToDataSetExportFunction(URI outputDir, RecordReader recordReader, int batchSize, boolean regression, + int labelIndex, int numPossibleLabels, Broadcast configuration) { + this.outputDir = outputDir; + this.recordReader = recordReader; + this.batchSize = batchSize; + this.regression = regression; + this.labelIndex = labelIndex; + this.numPossibleLabels = numPossibleLabels; + this.conf = configuration; + } + + @Override + public void call(Iterator stringIterator) throws Exception { + String jvmuid = UIDProvider.getJVMUID(); + uid = Thread.currentThread().getId() + jvmuid.substring(0, Math.min(8, jvmuid.length())); + + List> list = new ArrayList<>(batchSize); + + while (stringIterator.hasNext()) { + String next = stringIterator.next(); + recordReader.initialize(new StringSplit(next)); + list.add(recordReader.next()); + + processBatchIfRequired(list, !stringIterator.hasNext()); + } + } + + private void processBatchIfRequired(List> list, boolean finalRecord) throws Exception { + if (list.isEmpty()) + return; + if (list.size() < batchSize && !finalRecord) + return; + + RecordReader rr = new CollectionRecordReader(list); + RecordReaderDataSetIterator iter = new RecordReaderDataSetIterator(rr, null, batchSize, labelIndex, labelIndex, numPossibleLabels, -1, regression); + + DataSet ds = iter.next(); + + String filename = "dataset_" + uid + "_" + (outputCount++) + ".bin"; + + URI uri = new URI(outputDir.getPath() + "/" + filename); + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + FileSystem file = FileSystem.get(uri, c); + try (FSDataOutputStream out = file.create(new Path(uri))) { + ds.save(out); + } + + list.clear(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/DataVecRecord.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/DataVecRecord.java new file mode 100644 index 000000000..7fb70736b --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/DataVecRecord.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec.iterator; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.datavec.api.writable.Writable; + +import java.io.Serializable; +import java.util.List; + +@AllArgsConstructor +@Data +public class DataVecRecord implements Serializable { + private int readerIdx; + private List record; + private List> seqRecord; +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/DataVecRecords.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/DataVecRecords.java new file mode 100644 index 000000000..a950527e1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/DataVecRecords.java @@ -0,0 +1,35 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec.iterator; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.datavec.api.writable.Writable; + +import java.io.Serializable; +import java.util.List; + +@AllArgsConstructor +@Data +public class DataVecRecords implements Serializable { + private List> records; + private List>> seqRecords; +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/IteratorUtils.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/IteratorUtils.java new file mode 100644 index 000000000..b2a7592e8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/IteratorUtils.java @@ -0,0 +1,309 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec.iterator; + +import lombok.AllArgsConstructor; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.api.java.function.PairFunction; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.writable.Writable; +import org.deeplearning4j.datasets.datavec.RecordReaderMultiDataSetIterator; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import scala.Tuple2; + +import java.util.*; + +public class IteratorUtils { + + /** + * Apply a single reader {@link RecordReaderMultiDataSetIterator} to a {@code JavaRDD>}. + * NOTE: The RecordReaderMultiDataSetIterator must use {@link SparkSourceDummyReader} in place of + * "real" RecordReader instances + * + * @param rdd RDD with writables + * @param iterator RecordReaderMultiDataSetIterator with {@link SparkSourceDummyReader} readers + */ + public static JavaRDD mapRRMDSI(JavaRDD> rdd, RecordReaderMultiDataSetIterator iterator){ + checkIterator(iterator, 1, 0); + return mapRRMDSIRecords(rdd.map(new Function,DataVecRecords>(){ + @Override + public DataVecRecords call(List v1) throws Exception { + return new DataVecRecords(Collections.singletonList(v1), null); + } + }), iterator); + } + + /** + * Apply a single sequence reader {@link RecordReaderMultiDataSetIterator} to sequence data, in the form of + * {@code JavaRDD>>}. + * NOTE: The RecordReaderMultiDataSetIterator must use {@link SparkSourceDummySeqReader} in place of + * "real" SequenceRecordReader instances + * + * @param rdd RDD with writables + * @param iterator RecordReaderMultiDataSetIterator with {@link SparkSourceDummySeqReader} sequence readers + */ + public static JavaRDD mapRRMDSISeq(JavaRDD>> rdd, RecordReaderMultiDataSetIterator iterator){ + checkIterator(iterator, 0, 1); + return mapRRMDSIRecords(rdd.map(new Function>,DataVecRecords>(){ + @Override + public DataVecRecords call(List> v1) throws Exception { + return new DataVecRecords(null, Collections.singletonList(v1)); + } + }), iterator); + } + + /** + * Apply to an arbitrary mix of non-sequence and sequence data, in the form of {@code JavaRDD>} + * and {@code JavaRDD>>}.
+ * Note: this method performs a join by key. To perform this, we require that each record (and every step of + * sequence records) contain the same key value (could be any Writable).
+ * NOTE: The RecordReaderMultiDataSetIterator must use {@link SparkSourceDummyReader} and + * {@link SparkSourceDummySeqReader} instances in place of "real" RecordReader and SequenceRecordReader instances + * + * @param rdds RDD with non-sequence data. May be null. + * @param seqRdds RDDs with sequence data. May be null. + * @param rddsKeyColumns Column indices for the keys in the (non-sequence) RDDs data + * @param seqRddsKeyColumns Column indices for the keys in the sequence RDDs data + * @param filterMissing If true: filter out any records that don't have matching keys in all RDDs + * @param iterator RecordReaderMultiDataSetIterator with {@link SparkSourceDummyReader} and {@link SparkSourceDummySeqReader}readers + */ + public static JavaRDD mapRRMDSI(List>> rdds, List>>> seqRdds, + int[] rddsKeyColumns, int[] seqRddsKeyColumns, boolean filterMissing, + RecordReaderMultiDataSetIterator iterator){ + checkIterator(iterator, (rdds == null ? 0 : rdds.size()), (seqRdds == null ? 0 : seqRdds.size())); + assertNullOrSameLength(rdds, rddsKeyColumns, false); + assertNullOrSameLength(seqRdds, seqRddsKeyColumns, true); + if((rdds == null || rdds.isEmpty()) && (seqRdds == null || seqRdds.isEmpty()) ){ + throw new IllegalArgumentException(); + } + + JavaPairRDD allPairs = null; + if(rdds != null){ + for( int i=0; i> rdd = rdds.get(i); + JavaPairRDD currPairs = rdd.mapToPair(new MapToPairFn(i, rddsKeyColumns[i])); + if(allPairs == null){ + allPairs = currPairs; + } else { + allPairs = allPairs.union(currPairs); + } + } + } + + + if(seqRdds != null){ + for( int i=0; i>> rdd = seqRdds.get(i); + JavaPairRDD currPairs = rdd.mapToPair(new MapToPairSeqFn(i, seqRddsKeyColumns[i])); + if(allPairs == null){ + allPairs = currPairs; + } else { + allPairs = allPairs.union(currPairs); + } + } + } + + int expNumRec = (rddsKeyColumns == null ? 0 : rddsKeyColumns.length); + int expNumSeqRec = (seqRddsKeyColumns == null ? 0 : seqRddsKeyColumns.length); + + //Finally: group by key, filter (if necessary), convert + JavaPairRDD> grouped = allPairs.groupByKey(); + if(filterMissing){ + //TODO + grouped = grouped.filter(new FilterMissingFn(expNumRec, expNumSeqRec)); + } + + JavaRDD combined = grouped.map(new CombineFunction(expNumRec, expNumSeqRec)); + return mapRRMDSIRecords(combined, iterator); + } + + @AllArgsConstructor + private static class MapToPairFn implements PairFunction, Writable, DataVecRecord> { + private int readerIdx; + private int keyIndex; + @Override + public Tuple2 call(List writables) throws Exception { + return new Tuple2<>(writables.get(keyIndex), new DataVecRecord(readerIdx, writables, null)); + } + } + + @AllArgsConstructor + private static class MapToPairSeqFn implements PairFunction>, Writable, DataVecRecord> { + private int readerIdx; + private int keyIndex; + @Override + public Tuple2 call(List> seq) throws Exception { + if(seq.isEmpty()){ + throw new IllegalStateException("Sequence of length 0 encountered"); + } + return new Tuple2<>(seq.get(0).get(keyIndex), new DataVecRecord(readerIdx, null, seq)); + } + } + + @AllArgsConstructor + private static class CombineFunction implements Function>, DataVecRecords>{ + private int expNumRecords; + private int expNumSeqRecords; + @Override + public DataVecRecords call(Tuple2> all) throws Exception { + + List[] allRecordsArr = null; + if(expNumRecords > 0){ + allRecordsArr = (List[])new List[expNumRecords]; //Array.newInstance(List.class, expNumRecords); + } + List>[] allRecordsSeqArr = null; + if(expNumSeqRecords > 0){ + allRecordsSeqArr = (List>[])new List[expNumSeqRecords]; + } + + for(DataVecRecord rec : all._2()){ + if(rec.getRecord() != null){ + allRecordsArr[rec.getReaderIdx()] = rec.getRecord(); + } else { + allRecordsSeqArr[rec.getReaderIdx()] = rec.getSeqRecord(); + } + } + + if(allRecordsArr != null){ + for(int i=0; i> r = (allRecordsArr == null ? null : Arrays.asList(allRecordsArr)); + List>> sr = (allRecordsSeqArr == null ? null : Arrays.asList(allRecordsSeqArr)); + return new DataVecRecords(r, sr); + } + } + + + @AllArgsConstructor + private static class FilterMissingFn implements Function>, Boolean>{ + private final int expNumRec; + private final int expNumSeqRec; + private transient ThreadLocal> recIdxs; + private transient ThreadLocal> seqRecIdxs; + + private FilterMissingFn(int expNumRec, int expNumSeqRec){ + this.expNumRec = expNumRec; + this.expNumSeqRec = expNumSeqRec; + } + + @Override + public Boolean call(Tuple2> iter) throws Exception { + if(recIdxs == null) recIdxs = new ThreadLocal<>(); + if(seqRecIdxs == null) seqRecIdxs = new ThreadLocal<>(); + + Set ri = recIdxs.get(); + if(ri == null){ + ri = new HashSet<>(); + recIdxs.set(ri); + } + Set sri = seqRecIdxs.get(); + if(sri == null){ + sri = new HashSet<>(); + seqRecIdxs.set(sri); + } + + for(DataVecRecord r : iter._2()){ + if(r.getRecord() != null){ + ri.add(r.getReaderIdx()); + } else if(r.getSeqRecord() != null){ + sri.add(r.getReaderIdx()); + } + } + + int count = ri.size(); + int count2 = sri.size(); + + ri.clear(); + sri.clear(); + + return (count == expNumRec) && (count2 == expNumSeqRec); + } + } + + + private static void assertNullOrSameLength(List list, int[] arr, boolean isSeq){ + if(list != null && arr == null){ + throw new IllegalStateException(); + } + if(list == null && (arr != null && arr.length > 0)){ + throw new IllegalStateException(); + } + if(list != null && list.size() != arr.length){ + throw new IllegalStateException(); + } + } + + + public static JavaRDD mapRRMDSIRecords(JavaRDD rdd, RecordReaderMultiDataSetIterator iterator){ + return rdd.map(new RRMDSIFunction(iterator)); + } + + private static void checkIterator( RecordReaderMultiDataSetIterator iterator, int maxReaders, int maxSeqReaders ){ + + + Map rrs = iterator.getRecordReaders(); + Map seqRRs = iterator.getSequenceRecordReaders(); + + + if(rrs != null && rrs.size() > maxReaders){ + throw new IllegalStateException("Invalid state: iterator has " + rrs.size() + " readers but " + maxReaders + + " RDDs of List were provided"); + } + if(seqRRs != null && seqRRs.size() > maxSeqReaders){ + throw new IllegalStateException("Invalid state: iterator has " + seqRRs.size() + " sequence readers but " + + maxSeqReaders + " RDDs of sequences - List> were provided"); + } + + if(rrs != null && rrs.size() > 0){ + for(Map.Entry e : rrs.entrySet()){ + if(!(e.getValue() instanceof SparkSourceDummyReader)){ + throw new IllegalStateException("Invalid state: expected SparkSourceDummyReader for reader with name \"" + + e.getKey() + "\", but got reader type: " + e.getKey().getClass()); + } + } + } + + if(seqRRs != null && seqRRs.size() > 0){ + for(Map.Entry e : seqRRs.entrySet()){ + if(!(e.getValue() instanceof SparkSourceDummySeqReader)){ + throw new IllegalStateException("Invalid state: expected SparkSourceDummySeqReader for sequence reader with name \"" + + e.getKey() + "\", but got reader type: " + e.getKey().getClass()); + } + } + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/RRMDSIFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/RRMDSIFunction.java new file mode 100644 index 000000000..1c3a47039 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/RRMDSIFunction.java @@ -0,0 +1,78 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec.iterator; + +import lombok.AllArgsConstructor; +import org.apache.spark.api.java.function.Function; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.writable.Writable; +import org.deeplearning4j.datasets.datavec.RecordReaderMultiDataSetIterator; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@AllArgsConstructor +public class RRMDSIFunction implements Function { + + private RecordReaderMultiDataSetIterator iterator; + + @Override + public MultiDataSet call(DataVecRecords records) throws Exception { + + + + Map>> nextRRVals = Collections.emptyMap(); + Map>>> nextSeqRRVals = Collections.emptyMap(); + + if(records.getRecords() != null && !records.getRecords().isEmpty()){ + nextRRVals = new HashMap<>(); + + Map m = iterator.getRecordReaders(); + for(Map.Entry e : m.entrySet()){ + SparkSourceDummyReader dr = (SparkSourceDummyReader)e.getValue(); + int idx = dr.getReaderIdx(); + nextRRVals.put(e.getKey(), Collections.singletonList(records.getRecords().get(idx))); + } + + } + if(records.getSeqRecords() != null && !records.getSeqRecords().isEmpty()){ + nextSeqRRVals = new HashMap<>(); + + Map m = iterator.getSequenceRecordReaders(); + for(Map.Entry e : m.entrySet()){ + SparkSourceDummySeqReader dr = (SparkSourceDummySeqReader)e.getValue(); + int idx = dr.getReaderIdx(); + nextSeqRRVals.put(e.getKey(), Collections.singletonList(records.getSeqRecords().get(idx))); + } + } + + + MultiDataSet mds = iterator.nextMultiDataSet(nextRRVals, null, nextSeqRRVals, null); + Nd4j.getExecutioner().commit(); + + return mds; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/SparkSourceDummyReader.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/SparkSourceDummyReader.java new file mode 100644 index 000000000..0f5519f35 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/SparkSourceDummyReader.java @@ -0,0 +1,135 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec.iterator; + +import lombok.Data; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.Record; +import org.datavec.api.records.listener.RecordListener; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.InputSplit; +import org.datavec.api.writable.Writable; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.Serializable; +import java.net.URI; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +@Data +public class SparkSourceDummyReader implements RecordReader, Serializable { + private int readerIdx; + + /** + * @param readerIdx Index of the reader, in terms of the RDD that matches it. For a single RDD as input, this + * is always 0; for 2 RDDs used as input, this would be 0 or 1, depending on whether it should pull + * values from the first or second RDD. Note that the indexing for RDDs doesn't depend on the + * presence of sequence RDDs - they are indexed separately. + */ + public SparkSourceDummyReader(int readerIdx) { + this.readerIdx = readerIdx; + } + + + @Override + public void initialize(InputSplit inputSplit) throws IOException, InterruptedException { + /* No op */ + } + + @Override + public void initialize(Configuration configuration, InputSplit inputSplit) throws IOException, InterruptedException { + /* No op */ + } + + @Override + public boolean batchesSupported() { + return false; + } + + @Override + public List> next(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public List next() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasNext() { + return false; + } + + @Override + public List getLabels() { + return null; + } + + @Override + public void reset() { /* No op */ } + + @Override + public boolean resetSupported() { + return true; + } + + @Override + public List record(URI uri, DataInputStream dataInputStream) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Record nextRecord() { + throw new UnsupportedOperationException(); + } + + @Override + public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public List loadFromMetaData(List list) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public List getListeners() { return Collections.emptyList(); } + + @Override + public void setListeners(RecordListener... recordListeners) { /* No op */ } + + @Override + public void setListeners(Collection collection) { } + + @Override + public void close() throws IOException { /* No op */} + + @Override + public void setConf(Configuration configuration) { /* No op */ } + + @Override + public Configuration getConf() { throw new UnsupportedOperationException(); } +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/SparkSourceDummySeqReader.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/SparkSourceDummySeqReader.java new file mode 100644 index 000000000..54671ec37 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/datavec/iterator/SparkSourceDummySeqReader.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec.iterator; + +import lombok.Data; +import org.datavec.api.records.SequenceRecord; +import org.datavec.api.records.metadata.RecordMetaData; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.writable.Writable; + +import java.io.DataInputStream; +import java.io.IOException; +import java.net.URI; +import java.util.List; + +@Data +public class SparkSourceDummySeqReader extends SparkSourceDummyReader implements SequenceRecordReader { + + /** + * @param readerIdx Index of the reader, in terms of the sequence RDD that it should use. For a single sequence RDD + * as input, this is always 0; for 2 sequence RDDs used as input, this would be 0 or 1, depending + * on whether it should pull values from the first or second sequence RDD. Note that the indexing + * for sequence RDDs doesn't depend on the presence of non-sequence RDDs - they are indexed separately. + */ + public SparkSourceDummySeqReader(int readerIdx) { + super(readerIdx); + } + + @Override + public List> sequenceRecord() { + throw new UnsupportedOperationException(); + } + + @Override + public List> sequenceRecord(URI uri, DataInputStream dataInputStream) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public SequenceRecord nextSequence() { + throw new UnsupportedOperationException(); + } + + @Override + public SequenceRecord loadSequenceFromMetaData(RecordMetaData recordMetaData) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public List loadSequenceFromMetaData(List list) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/BaseSparkEarlyStoppingTrainer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/BaseSparkEarlyStoppingTrainer.java new file mode 100644 index 000000000..5f1029131 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/BaseSparkEarlyStoppingTrainer.java @@ -0,0 +1,234 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.earlystopping; + +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.EarlyStoppingResult; +import org.deeplearning4j.earlystopping.listener.EarlyStoppingListener; +import org.deeplearning4j.earlystopping.scorecalc.ScoreCalculator; +import org.deeplearning4j.earlystopping.termination.EpochTerminationCondition; +import org.deeplearning4j.earlystopping.termination.IterationTerminationCondition; +import org.deeplearning4j.earlystopping.trainer.IEarlyStoppingTrainer; +import org.deeplearning4j.nn.api.Model; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +public abstract class BaseSparkEarlyStoppingTrainer implements IEarlyStoppingTrainer { + + private static Logger log = LoggerFactory.getLogger(BaseSparkEarlyStoppingTrainer.class); + + private JavaSparkContext sc; + private final EarlyStoppingConfiguration esConfig; + private T net; + private final JavaRDD train; + private final JavaRDD trainMulti; + private EarlyStoppingListener listener; + + private double bestModelScore = Double.MAX_VALUE; + private int bestModelEpoch = -1; + + protected BaseSparkEarlyStoppingTrainer(JavaSparkContext sc, EarlyStoppingConfiguration esConfig, T net, + JavaRDD train, JavaRDD trainMulti, EarlyStoppingListener listener) { + if ((esConfig.getEpochTerminationConditions() == null || esConfig.getEpochTerminationConditions().isEmpty()) + && (esConfig.getIterationTerminationConditions() == null + || esConfig.getIterationTerminationConditions().isEmpty())) { + throw new IllegalArgumentException( + "Cannot conduct early stopping without a termination condition (both Iteration " + + "and Epoch termination conditions are null/empty)"); + } + + this.sc = sc; + this.esConfig = esConfig; + this.net = net; + this.train = train; + this.trainMulti = trainMulti; + this.listener = listener; + } + + protected abstract void fit(JavaRDD data); + + protected abstract void fitMulti(JavaRDD data); + + protected abstract double getScore(); + + @Override + public EarlyStoppingResult fit() { + log.info("Starting early stopping training"); + if (esConfig.getScoreCalculator() == null) + log.warn("No score calculator provided for early stopping. Score will be reported as 0.0 to epoch termination conditions"); + + //Initialize termination conditions: + if (esConfig.getIterationTerminationConditions() != null) { + for (IterationTerminationCondition c : esConfig.getIterationTerminationConditions()) { + c.initialize(); + } + } + if (esConfig.getEpochTerminationConditions() != null) { + for (EpochTerminationCondition c : esConfig.getEpochTerminationConditions()) { + c.initialize(); + } + } + + if (listener != null) + listener.onStart(esConfig, net); + + Map scoreVsEpoch = new LinkedHashMap<>(); + + int epochCount = 0; + while (true) { //Iterate (do epochs) until termination condition hit + double lastScore; + boolean terminate = false; + IterationTerminationCondition terminationReason = null; + + if (train != null) + fit(train); + else + fitMulti(trainMulti); + + //TODO revisit per iteration termination conditions, ensuring they are evaluated *per averaging* not per epoch + //Check per-iteration termination conditions + lastScore = getScore(); + for (IterationTerminationCondition c : esConfig.getIterationTerminationConditions()) { + if (c.terminate(lastScore)) { + terminate = true; + terminationReason = c; + break; + } + } + + if (terminate) { + //Handle termination condition: + log.info("Hit per iteration epoch termination condition at epoch {}, iteration {}. Reason: {}", + epochCount, epochCount, terminationReason); + + if (esConfig.isSaveLastModel()) { + //Save last model: + try { + esConfig.getModelSaver().saveLatestModel(net, 0.0); + } catch (IOException e) { + throw new RuntimeException("Error saving most recent model", e); + } + } + + T bestModel; + try { + bestModel = esConfig.getModelSaver().getBestModel(); + } catch (IOException e2) { + throw new RuntimeException(e2); + } + EarlyStoppingResult result = new EarlyStoppingResult<>( + EarlyStoppingResult.TerminationReason.IterationTerminationCondition, + terminationReason.toString(), scoreVsEpoch, bestModelEpoch, bestModelScore, epochCount, + bestModel); + if (listener != null) + listener.onCompletion(result); + return result; + } + + + + log.info("Completed training epoch {}", epochCount); + + + if ((epochCount == 0 && esConfig.getEvaluateEveryNEpochs() == 1) + || epochCount % esConfig.getEvaluateEveryNEpochs() == 0) { + //Calculate score at this epoch: + ScoreCalculator sc = esConfig.getScoreCalculator(); + double score = (sc == null ? 0.0 : esConfig.getScoreCalculator().calculateScore(net)); + scoreVsEpoch.put(epochCount - 1, score); + + if (sc != null && score < bestModelScore) { + //Save best model: + if (bestModelEpoch == -1) { + //First calculated/reported score + log.info("Score at epoch {}: {}", epochCount, score); + } else { + log.info("New best model: score = {}, epoch = {} (previous: score = {}, epoch = {})", score, + epochCount, bestModelScore, bestModelEpoch); + } + bestModelScore = score; + bestModelEpoch = epochCount; + + try { + esConfig.getModelSaver().saveBestModel(net, score); + } catch (IOException e) { + throw new RuntimeException("Error saving best model", e); + } + } + + if (esConfig.isSaveLastModel()) { + //Save last model: + try { + esConfig.getModelSaver().saveLatestModel(net, score); + } catch (IOException e) { + throw new RuntimeException("Error saving most recent model", e); + } + } + + if (listener != null) + listener.onEpoch(epochCount, score, esConfig, net); + + //Check per-epoch termination conditions: + boolean epochTerminate = false; + EpochTerminationCondition termReason = null; + for (EpochTerminationCondition c : esConfig.getEpochTerminationConditions()) { + if (c.terminate(epochCount, score, esConfig.getScoreCalculator().minimizeScore())) { + epochTerminate = true; + termReason = c; + break; + } + } + if (epochTerminate) { + log.info("Hit epoch termination condition at epoch {}. Details: {}", epochCount, + termReason.toString()); + T bestModel; + try { + bestModel = esConfig.getModelSaver().getBestModel(); + } catch (IOException e2) { + throw new RuntimeException(e2); + } + EarlyStoppingResult result = new EarlyStoppingResult<>( + EarlyStoppingResult.TerminationReason.EpochTerminationCondition, + termReason.toString(), scoreVsEpoch, bestModelEpoch, bestModelScore, epochCount + 1, + bestModel); + if (listener != null) + listener.onCompletion(result); + return result; + } + + epochCount++; + } + } + } + + @Override + public void setListener(EarlyStoppingListener listener) { + this.listener = listener; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkDataSetLossCalculator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkDataSetLossCalculator.java new file mode 100644 index 000000000..be71c408c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkDataSetLossCalculator.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.earlystopping; + +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.earlystopping.scorecalc.ScoreCalculator; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.nd4j.linalg.dataset.DataSet; + +public class SparkDataSetLossCalculator implements ScoreCalculator { + + + private JavaRDD data; + private boolean average; + private SparkContext sc; + + /**Calculate the score (loss function value) on a given data set (usually a test set) + * + * @param data Data set to calculate the score for + * @param average Whether to return the average (sum of loss / N) or just (sum of loss) + */ + public SparkDataSetLossCalculator(JavaRDD data, boolean average, SparkContext sc) { + this.data = data; + this.average = average; + this.sc = sc; + } + + @Override + public double calculateScore(MultiLayerNetwork network) { + SparkDl4jMultiLayer net = new SparkDl4jMultiLayer(sc, network, null); + return net.calculateScore(data, average); + } + + @Override + public boolean minimizeScore() { + return true; //Minimize loss + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkEarlyStoppingGraphTrainer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkEarlyStoppingGraphTrainer.java new file mode 100644 index 000000000..efdab70aa --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkEarlyStoppingGraphTrainer.java @@ -0,0 +1,91 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.earlystopping; + +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.EarlyStoppingResult; +import org.deeplearning4j.earlystopping.listener.EarlyStoppingListener; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.graph.dataset.DataSetToMultiDataSetFn; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +public class SparkEarlyStoppingGraphTrainer extends BaseSparkEarlyStoppingTrainer { + + private SparkComputationGraph sparkNet; + + public SparkEarlyStoppingGraphTrainer(SparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, ComputationGraph net, + JavaRDD train, int examplesPerFit, int totalExamples) { + this(new JavaSparkContext(sc), trainingMaster, esConfig, net, train, null); + } + + public SparkEarlyStoppingGraphTrainer(JavaSparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, ComputationGraph net, + JavaRDD train, int examplesPerFit, int totalExamples) { + this(sc, trainingMaster, esConfig, net, train, null); + } + + public SparkEarlyStoppingGraphTrainer(SparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, ComputationGraph net, + JavaRDD train) { + this(new JavaSparkContext(sc), trainingMaster, esConfig, net, train, null); + } + + public SparkEarlyStoppingGraphTrainer(JavaSparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, ComputationGraph net, + JavaRDD train) { + this(sc, trainingMaster, esConfig, net, train, null); + } + + public SparkEarlyStoppingGraphTrainer(JavaSparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, ComputationGraph net, + JavaRDD train, EarlyStoppingListener listener) { + super(sc, esConfig, net, null, train, listener); + this.sparkNet = new SparkComputationGraph(sc, net, trainingMaster); + } + + + @Override + protected void fit(JavaRDD data) { + fitMulti(data.map(new DataSetToMultiDataSetFn())); + } + + @Override + protected void fitMulti(JavaRDD data) { + sparkNet.fitMultiDataSet(data); + } + + @Override + protected double getScore() { + return sparkNet.getScore(); + } + + @Override + public EarlyStoppingResult pretrain() { + throw new UnsupportedOperationException("Not supported"); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkEarlyStoppingTrainer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkEarlyStoppingTrainer.java new file mode 100644 index 000000000..3e61bd7cd --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkEarlyStoppingTrainer.java @@ -0,0 +1,84 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.earlystopping; + +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.EarlyStoppingResult; +import org.deeplearning4j.earlystopping.listener.EarlyStoppingListener; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +public class SparkEarlyStoppingTrainer extends BaseSparkEarlyStoppingTrainer { + + private SparkDl4jMultiLayer sparkNet; + + public SparkEarlyStoppingTrainer(SparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, MultiLayerNetwork net, + JavaRDD train) { + this(new JavaSparkContext(sc), trainingMaster, esConfig, net, train, null); + } + + public SparkEarlyStoppingTrainer(JavaSparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, MultiLayerNetwork net, + JavaRDD train) { + this(sc, trainingMaster, esConfig, net, train, null); + } + + public SparkEarlyStoppingTrainer(SparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, MultiLayerNetwork net, + JavaRDD train, EarlyStoppingListener listener) { + this(new JavaSparkContext(sc), trainingMaster, esConfig, net, train, listener); + } + + public SparkEarlyStoppingTrainer(JavaSparkContext sc, TrainingMaster trainingMaster, + EarlyStoppingConfiguration esConfig, MultiLayerNetwork net, + JavaRDD train, EarlyStoppingListener listener) { + super(sc, esConfig, net, train, null, listener); + sparkNet = new SparkDl4jMultiLayer(sc, net, trainingMaster); + } + + + @Override + protected void fit(JavaRDD data) { + sparkNet.fit(data); + } + + @Override + protected void fitMulti(JavaRDD data) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + protected double getScore() { + return sparkNet.getScore(); + } + + @Override + public EarlyStoppingResult pretrain() { + throw new UnsupportedOperationException("Not supported"); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkLossCalculatorComputationGraph.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkLossCalculatorComputationGraph.java new file mode 100644 index 000000000..be03c85af --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/earlystopping/SparkLossCalculatorComputationGraph.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.earlystopping; + +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.earlystopping.scorecalc.ScoreCalculator; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +public class SparkLossCalculatorComputationGraph implements ScoreCalculator { + + private JavaRDD data; + private boolean average; + private SparkContext sc; + + /** + * Calculate the score (loss function value) on a given data set (usually a test set) + * + * @param data Data set to calculate the score for + * @param average Whether to return the average (sum of loss / N) or just (sum of loss) + */ + public SparkLossCalculatorComputationGraph(JavaRDD data, boolean average, SparkContext sc) { + this.data = data; + this.average = average; + this.sc = sc; + } + + + @Override + public double calculateScore(ComputationGraph network) { + SparkComputationGraph net = new SparkComputationGraph(sc, network, null); + return net.calculateScoreMultiDataSet(data, average); + } + + @Override + public boolean minimizeScore() { + return true; //Minimize loss + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/SparkListenable.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/SparkListenable.java new file mode 100644 index 000000000..36011825d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/SparkListenable.java @@ -0,0 +1,119 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.StatsStorageRouterProvider; +import org.deeplearning4j.core.storage.listener.RoutingIterationListener; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.listeners.VanillaStatsStorageRouterProvider; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +@Slf4j +public class SparkListenable { + + protected TrainingMaster trainingMaster; + private List listeners = new ArrayList<>(); + + + /** + * This method allows you to specify trainingListeners for this model. + * + * @param listeners Iteration listeners + */ + public void setListeners(@NonNull Collection listeners) { + this.listeners.clear(); + this.listeners.addAll(listeners); + if (trainingMaster != null) + trainingMaster.setListeners(this.listeners); + } + + /** + * This method allows you to specify trainingListeners for this model. Note that for listeners + * like StatsListener (that have state that will be sent somewhere), consider instead using {@link + * #setListeners(StatsStorageRouter, Collection)} + * + * @param listeners Listeners to set + */ + public void setListeners(@NonNull TrainingListener... listeners) { + setListeners(Arrays.asList(listeners)); + } + + /** + * Set the listeners, along with a StatsStorageRouter that the results will be shuffled to (in the + * case of any listeners that implement the {@link RoutingIterationListener} interface) + * + * @param statsStorage Stats storage router to place the results into + * @param listeners Listeners to set + */ + public void setListeners(StatsStorageRouter statsStorage, TrainingListener... listeners) { + setListeners(statsStorage, Arrays.asList(listeners)); + } + + /** + * Set the listeners, along with a StatsStorageRouter that the results will be shuffled to (in the + * case of any listeners that implement the {@link RoutingIterationListener} interface) + * + * @param statsStorage Stats storage router to place the results into + * @param listeners Listeners to set + */ + public void setListeners(StatsStorageRouter statsStorage, Collection listeners) { + //Check if we have any RoutingIterationListener instances that need a StatsStorage implementation... + StatsStorageRouterProvider routerProvider = null; + if (listeners != null) { + for (TrainingListener l : listeners) { + if (l instanceof RoutingIterationListener) { + RoutingIterationListener rl = (RoutingIterationListener) l; + if (statsStorage == null && rl.getStorageRouter() == null) { + log.warn("RoutingIterationListener provided without providing any StatsStorage instance. Iterator may not function without one. Listener: {}", + l); + } else if (rl.getStorageRouter() != null && !(rl.getStorageRouter() instanceof Serializable)) { + //Spark would throw a (probably cryptic) serialization exception later anyway... + throw new IllegalStateException( + "RoutingIterationListener provided with non-serializable storage router " + + "\nRoutingIterationListener class: " + rl.getClass().getName() + + "\nStatsStorageRouter class: " + + rl.getStorageRouter().getClass().getName()); + } + + //Need to give workers a router provider... + if (routerProvider == null) { + routerProvider = new VanillaStatsStorageRouterProvider(); + } + } + } + } + this.listeners.clear(); + if (listeners != null) { + this.listeners.addAll(listeners); + if (trainingMaster != null) + trainingMaster.setListeners(statsStorage, this.listeners); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/Add.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/Add.java new file mode 100644 index 000000000..8ce9c31b8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/Add.java @@ -0,0 +1,40 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common; + +import org.apache.spark.api.java.function.Function2; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +/** + * Adds 2 ndarrays + * @author Adam Gibson + */ +public class Add implements Function2 { + @Override + public INDArray call(INDArray v1, INDArray v2) throws Exception { + INDArray res = v1.addi(v2); + + Nd4j.getExecutioner().commit(); + + return res; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/CountPartitionsFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/CountPartitionsFunction.java new file mode 100644 index 000000000..2f14c04e7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/CountPartitionsFunction.java @@ -0,0 +1,43 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common; + +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.Function2; +import org.deeplearning4j.spark.api.Repartition; +import scala.Tuple2; + +import java.util.Collections; +import java.util.Iterator; + +public class CountPartitionsFunction implements Function2, Iterator>> { + @Override + public Iterator> call(Integer v1, Iterator v2) throws Exception { + + int count = 0; + while (v2.hasNext()) { + v2.next(); + count++; + } + + return Collections.singletonList(new Tuple2<>(v1, count)).iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/LoadDataSetFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/LoadDataSetFunction.java new file mode 100644 index 000000000..6e6795838 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/LoadDataSetFunction.java @@ -0,0 +1,43 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common; + +import lombok.AllArgsConstructor; +import org.apache.spark.api.java.function.Function; +import org.nd4j.common.loader.Loader; +import org.nd4j.common.loader.Source; +import org.nd4j.common.loader.SourceFactory; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.InputStream; + +@AllArgsConstructor +public class LoadDataSetFunction implements Function { + + private final Loader loader; + private final SourceFactory factory; + + @Override + public DataSet call(String path) throws Exception { + Source s = factory.getSource(path); + return loader.load(s); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/SplitPartitionsFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/SplitPartitionsFunction.java new file mode 100644 index 000000000..65539fe60 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/SplitPartitionsFunction.java @@ -0,0 +1,58 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common; + +import lombok.AllArgsConstructor; +import org.apache.spark.api.java.function.Function2; + +import java.util.*; + +@AllArgsConstructor +public class SplitPartitionsFunction implements Function2, Iterator> { + private final int splitIndex; + private final int numSplits; + private final long baseRngSeed; + + @Override + public Iterator call(Integer v1, Iterator iter) throws Exception { + long thisRngSeed = baseRngSeed + v1; + + Random r = new Random(thisRngSeed); + List list = new ArrayList<>(); + for (int i = 0; i < numSplits; i++) { + list.add(i); + } + + List outputList = new ArrayList<>(); + int i = 0; + while (iter.hasNext()) { + if (i % numSplits == 0) + Collections.shuffle(list, r); + + T next = iter.next(); + if (list.get(i % numSplits) == splitIndex) + outputList.add(next); + i++; + } + + return outputList.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/SplitPartitionsFunction2.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/SplitPartitionsFunction2.java new file mode 100644 index 000000000..c565cdcd6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/SplitPartitionsFunction2.java @@ -0,0 +1,60 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common; + +import lombok.AllArgsConstructor; +import org.apache.spark.api.java.function.Function2; +import scala.Tuple2; + +import java.util.*; + +@AllArgsConstructor +public class SplitPartitionsFunction2 + implements Function2>, Iterator>> { + private final int splitIndex; + private final int numSplits; + private final long baseRngSeed; + + @Override + public Iterator> call(Integer v1, Iterator> iter) throws Exception { + long thisRngSeed = baseRngSeed + v1; + + Random r = new Random(thisRngSeed); + List list = new ArrayList<>(); + for (int i = 0; i < numSplits; i++) { + list.add(i); + } + + List> outputList = new ArrayList<>(); + int i = 0; + while (iter.hasNext()) { + if (i % numSplits == 0) + Collections.shuffle(list, r); + + Tuple2 next = iter.next(); + if (list.get(i % numSplits) == splitIndex) + outputList.add(next); + i++; + } + + return outputList.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/reduce/IntDoubleReduceFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/reduce/IntDoubleReduceFunction.java new file mode 100644 index 000000000..558128f7c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/reduce/IntDoubleReduceFunction.java @@ -0,0 +1,32 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.reduce; + +import org.apache.spark.api.java.function.Function2; +import scala.Tuple2; + +public class IntDoubleReduceFunction + implements Function2, Tuple2, Tuple2> { + @Override + public Tuple2 call(Tuple2 f, Tuple2 s) throws Exception { + return new Tuple2<>(f._1() + s._1(), f._2() + s._2()); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/reduce/LongDoubleReduceFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/reduce/LongDoubleReduceFunction.java new file mode 100644 index 000000000..4cbcc75f0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/reduce/LongDoubleReduceFunction.java @@ -0,0 +1,32 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.reduce; + +import org.apache.spark.api.java.function.Function2; +import scala.Tuple2; + +public class LongDoubleReduceFunction + implements Function2, Tuple2, Tuple2> { + @Override + public Tuple2 call(Tuple2 f, Tuple2 s) throws Exception { + return new Tuple2<>(f._1() + s._1(), f._2() + s._2()); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitioner.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitioner.java new file mode 100644 index 000000000..0f26e52e4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitioner.java @@ -0,0 +1,86 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.repartition; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.Partitioner; + +import java.util.Random; + +@Slf4j +public class BalancedPartitioner extends Partitioner { + private final int numPartitions; //Total number of partitions + private final int elementsPerPartition; + private final int remainder; + private Random r; + + public BalancedPartitioner(int numPartitions, int elementsPerPartition, int remainder) { + this.numPartitions = numPartitions; + this.elementsPerPartition = elementsPerPartition; + this.remainder = remainder; + } + + @Override + public int numPartitions() { + return numPartitions; + } + + @Override + public int getPartition(Object key) { + int elementIdx = key.hashCode(); + + //First 'remainder' executors get elementsPerPartition+1 each; the remainder get + // elementsPerPartition each. This is because the total number of examples might not be an exact multiple + // of the number of cores in the cluster + + //Work out: which partition it belongs to... + if (elementIdx <= (elementsPerPartition + 1) * remainder) { + //This goes into one of the larger partitions (of size elementsPerPartition+1) + int outputPartition = elementIdx / (elementsPerPartition + 1); + if (outputPartition >= numPartitions) { + //Should never happen, unless there's some up-stream problem with calculating elementsPerPartition + outputPartition = getRandom().nextInt(numPartitions); + log.trace("Random partition assigned (1): elementIdx={}, numPartitions={}, elementsPerPartition={}, remainder={}", + elementIdx, numPartitions, elementsPerPartition, remainder); + } + return outputPartition; + } else { + //This goes into one of the standard size partitions (of size elementsPerPartition) + int numValsInLargerPartitions = remainder * (elementsPerPartition + 1); + int idxInSmallerPartitions = elementIdx - numValsInLargerPartitions; + int smallPartitionIdx = idxInSmallerPartitions / elementsPerPartition; + int outputPartition = remainder + smallPartitionIdx; + if (outputPartition >= numPartitions) { + //Should never happen, unless there's some up-stream problem with calculating elementsPerPartition + outputPartition = getRandom().nextInt(numPartitions); + log.trace("Random partition assigned (2): elementIdx={}, numPartitions={}, elementsPerPartition={}, remainder={}", + elementIdx, numPartitions, elementsPerPartition, remainder); + } + return outputPartition; + } + } + + private synchronized Random getRandom() { + if (r == null) + r = new Random(); + return r; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/EqualPartitioner.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/EqualPartitioner.java new file mode 100644 index 000000000..0e7de28a0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/EqualPartitioner.java @@ -0,0 +1,60 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.repartition; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.Partitioner; +import org.apache.spark.api.java.JavaRDD; + +import java.util.Random; + +@Slf4j +@AllArgsConstructor +public class EqualPartitioner extends Partitioner { + private final int numPartitions; //Total number of partitions + private final int partitionSizeExRemainder; + private final int[] remainderPositions; + + @Override + public int numPartitions() { + return numPartitions; + } + + @Override + public int getPartition(Object key) { + int elementIdx = key.hashCode(); + + //Assign an equal number of elements to each partition, sequentially + // For any remainder, use the specified remainder indexes + + //Work out: which partition it belongs to... + if(elementIdx < numPartitions * partitionSizeExRemainder){ + //Standard element + return elementIdx / partitionSizeExRemainder; + } else { + //Is a 'remainder' element + int remainderNum = elementIdx % numPartitions; + return remainderPositions[remainderNum %remainderPositions.length]; //Final mod here shouldn't be necessary, but is here for safety... + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitioner.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitioner.java new file mode 100644 index 000000000..e2f5814bd --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitioner.java @@ -0,0 +1,152 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.repartition; + +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; +import org.apache.spark.Partitioner; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +public class HashingBalancedPartitioner extends Partitioner { + private final int numClasses; // Total number of element classes + private final int numPartitions; // Total number of partitions + // partitionWeightsByClass : numClasses lists of numPartitions elements + // where each element is the partition's relative share of its # of elements w.r.t the per-partition mean + // e.g. we have 3 partitions, with red and blue elements, red is indexed by 0, blue by 1: + // [ r, r, r, r, b, b, b ], [r, b, b], [b, b, b, b, b, r, r] + // avg # red elems per partition : 2.33 + // avg # blue elems per partition : 3.33 + // partitionWeightsByClass = [[1.714, .429, .857], [0.9, 0.6, 1.5]] + private List> partitionWeightsByClass; + + // The cumulative distribution of jump probabilities of extra elements by partition, by class + // 0 for partitions that already have enough elements + private List> jumpTable; + private Random r; + + public HashingBalancedPartitioner(List> partitionWeightsByClass) { + List> pw = checkNotNull(partitionWeightsByClass); + checkArgument(!pw.isEmpty(), "Partition weights are required"); + checkArgument(pw.size() >= 1, "There should be at least one element class"); + checkArgument(!checkNotNull(pw.get(0)).isEmpty(), "At least one partition is required"); + this.numClasses = pw.size(); + this.numPartitions = pw.get(0).size(); + for (int i = 1; i < pw.size(); i++) { + checkArgument(checkNotNull(pw.get(i)).size() == this.numPartitions, + "Non-consistent partition weight specification"); + // you also should have sum(pw.get(i)) = this.numPartitions + } + this.partitionWeightsByClass = partitionWeightsByClass; // p_(j, i) + + List> jumpsByClass = new ArrayList<>();; + for (int j = 0; j < numClasses; j++) { + Double totalImbalance = 0D; // i_j = sum(max(1 - p_(j, i), 0) , i = 1..numPartitions) + for (int i = 0; i < numPartitions; i++) { + totalImbalance += partitionWeightsByClass.get(j).get(i) >= 0 + ? Math.max(1 - partitionWeightsByClass.get(j).get(i), 0) : 0; + } + Double sumProb = 0D; + List cumulProbsThisClass = new ArrayList<>(); + for (int i = 0; i < numPartitions; i++) { + if (partitionWeightsByClass.get(j).get(i) >= 0 && (totalImbalance > 0 || sumProb >= 1)) { + Double thisPartitionRelProb = + Math.max(1 - partitionWeightsByClass.get(j).get(i), 0) / totalImbalance; + if (thisPartitionRelProb > 0) { + sumProb += thisPartitionRelProb; + cumulProbsThisClass.add(sumProb); + } else { + cumulProbsThisClass.add(0D); + } + } else { + // There's no more imbalance, every jumpProb is > 1 + cumulProbsThisClass.add(0D); + } + } + jumpsByClass.add(cumulProbsThisClass); + } + + this.jumpTable = jumpsByClass; + } + + @Override + public int numPartitions() { + List list = partitionWeightsByClass.get(0); + int count = 0; + for(Double d : list){ + if(d >= 0) + count++; + } + return count; + } + + @Override + public int getPartition(Object key) { + checkArgument(key instanceof Tuple2, "The key should be in the form: Tuple2(SparkUID, class) ..."); + Tuple2 uidNclass = (Tuple2) key; + Long uid = uidNclass._1(); + Integer partitionId = (int) (uid % numPartitions); + Integer elementClass = uidNclass._2(); + + Double jumpProbability = Math.max(1D - 1D / partitionWeightsByClass.get(elementClass).get(partitionId), 0D); + LinearCongruentialGenerator rand = new LinearCongruentialGenerator(uid); + + Double thisJumps = rand.nextDouble(); + Integer thisPartition = partitionId; + if (thisJumps < jumpProbability) { + // Where do we jump ? + List jumpsTo = jumpTable.get(elementClass); + Double destination = rand.nextDouble(); + Integer probe = 0; + + while (jumpsTo.get(probe) < destination) { + probe++; + } + thisPartition = probe; + } + + return thisPartition; + } + + // Multiplier chosen for nice distribution properties when successive random values are used to form tuples, + // which is the case with Spark's uid. + // See P. L'Ecuyer. Tables of Linear Congruential Generators of Different Sizes and Good Lattice + // Structure. In Mathematics of Computation 68 (225): pages 249–260 + // + static final class LinearCongruentialGenerator { + private long state; + + public LinearCongruentialGenerator(long seed) { + this.state = seed; + } + + public double nextDouble() { + state = 2862933555777941757L * state + 1; + return ((double) ((int) (state >>> 33) + 1)) / (0x1.0p31); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/MapTupleToPairFlatMap.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/MapTupleToPairFlatMap.java new file mode 100644 index 000000000..e78d8b321 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/repartition/MapTupleToPairFlatMap.java @@ -0,0 +1,40 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.repartition; + +import org.apache.spark.api.java.function.PairFlatMapFunction; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class MapTupleToPairFlatMap implements PairFlatMapFunction>, T, U> { + + @Override + public Iterator> call(Iterator> tuple2Iterator) throws Exception { + List> list = new ArrayList<>(); + while (tuple2Iterator.hasNext()) { + list.add(tuple2Iterator.next()); + } + return list.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/score/BaseVaeReconstructionProbWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/score/BaseVaeReconstructionProbWithKeyFunction.java new file mode 100644 index 000000000..ed302a351 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/score/BaseVaeReconstructionProbWithKeyFunction.java @@ -0,0 +1,54 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.score; + +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.layers.variational.VariationalAutoencoder; +import org.nd4j.linalg.api.ndarray.INDArray; + +public abstract class BaseVaeReconstructionProbWithKeyFunction extends BaseVaeScoreWithKeyFunction { + + private final boolean useLogProbability; + private final int numSamples; + + /** + * @param params MultiLayerNetwork parameters + * @param jsonConfig MultiLayerConfiguration, as json + * @param useLogProbability If true: use log probability. False: use raw probability. + * @param batchSize Batch size to use when scoring + * @param numSamples Number of samples to use when calling {@link VariationalAutoencoder#reconstructionLogProbability(INDArray, int)} + */ + public BaseVaeReconstructionProbWithKeyFunction(Broadcast params, Broadcast jsonConfig, + boolean useLogProbability, int batchSize, int numSamples) { + super(params, jsonConfig, batchSize); + this.useLogProbability = useLogProbability; + this.numSamples = numSamples; + } + + @Override + public INDArray computeScore(VariationalAutoencoder vae, INDArray toScore) { + if (useLogProbability) { + return vae.reconstructionLogProbability(toScore, numSamples); + } else { + return vae.reconstructionProbability(toScore, numSamples); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/score/BaseVaeScoreWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/score/BaseVaeScoreWithKeyFunction.java new file mode 100644 index 000000000..4140b8a53 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/common/score/BaseVaeScoreWithKeyFunction.java @@ -0,0 +1,110 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.score; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.api.java.function.PairFlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.layers.variational.VariationalAutoencoder; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +@Slf4j +public abstract class BaseVaeScoreWithKeyFunction implements PairFlatMapFunction>, K, Double> { + + protected final Broadcast params; + protected final Broadcast jsonConfig; + private final int batchSize; + + + /** + * @param params MultiLayerNetwork parameters + * @param jsonConfig MultiLayerConfiguration, as json + * @param batchSize Batch size to use when scoring + */ + public BaseVaeScoreWithKeyFunction(Broadcast params, Broadcast jsonConfig, int batchSize) { + this.params = params; + this.jsonConfig = jsonConfig; + this.batchSize = batchSize; + } + + public abstract VariationalAutoencoder getVaeLayer(); + + public abstract INDArray computeScore(VariationalAutoencoder vae, INDArray toScore); + + + @Override + public Iterator> call(Iterator> iterator) throws Exception { + if (!iterator.hasNext()) { + return Collections.emptyIterator(); + } + + VariationalAutoencoder vae = getVaeLayer(); + + List> ret = new ArrayList<>(); + + List collect = new ArrayList<>(batchSize); + List collectKey = new ArrayList<>(batchSize); + int totalCount = 0; + while (iterator.hasNext()) { + collect.clear(); + collectKey.clear(); + int nExamples = 0; + while (iterator.hasNext() && nExamples < batchSize) { + Tuple2 t2 = iterator.next(); + INDArray features = t2._2(); + val n = features.size(0); + if (n != 1) + throw new IllegalStateException("Cannot score examples with one key per data set if " + + "data set contains more than 1 example (numExamples: " + n + ")"); + collect.add(features); + collectKey.add(t2._1()); + nExamples += n; + } + totalCount += nExamples; + + INDArray toScore = Nd4j.vstack(collect); + INDArray scores = computeScore(vae, toScore); + + double[] doubleScores = scores.data().asDouble(); + + for (int i = 0; i < doubleScores.length; i++) { + ret.add(new Tuple2<>(collectKey.get(i), doubleScores[i])); + } + } + + Nd4j.getExecutioner().commit(); + + if (log.isDebugEnabled()) { + log.debug("Scored {} examples ", totalCount); + } + + return ret.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/evaluation/EvaluationRunner.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/evaluation/EvaluationRunner.java new file mode 100644 index 000000000..8550c6e3c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/evaluation/EvaluationRunner.java @@ -0,0 +1,246 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.evaluation; + +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.datasets.iterator.IteratorDataSetIterator; +import org.deeplearning4j.datasets.iterator.IteratorMultiDataSetIterator; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.common.base.Preconditions; +import org.nd4j.evaluation.IEvaluation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.util.DeviceLocalNDArray; + +import java.io.ByteArrayInputStream; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class EvaluationRunner { + + private static final EvaluationRunner INSTANCE = new EvaluationRunner(); + + public static EvaluationRunner getInstance(){ + return INSTANCE; + } + + private final AtomicInteger workerCount = new AtomicInteger(0); + private Queue queue = new ConcurrentLinkedQueue<>(); + //parameters map for device local parameters for a given broadcast + //Note: byte[] doesn't override Object.equals hence this is effectively an *identity* weak hash map, which is what we want here + //i.e., DeviceLocal can be GC'd once the Broadcast is no longer referenced anywhere + //This approach relies on the fact that a single Broadcast object's *content* will be shared by all of Spark's threads, + // even though the Broadcast object itself mayb not be + //Also by storing params as a byte[] (i.e., in serialized form), we sidestep a lot of the thread locality issues + private Map paramsMap = new WeakHashMap<>(); + + + private EvaluationRunner(){ } + + /** + * Evaluate the data using the specified evaluations + * @param evals Evaluations to perform + * @param evalWorkers Number of concurrent workers + * @param evalBatchSize Evaluation batch size to use + * @param ds DataSet iterator + * @param mds MultiDataSet iterator + * @param isCG True if ComputationGraph, false otherwise + * @param json JSON for the network + * @param params Parameters for the network + * @return Future for the results + */ + public Future execute(IEvaluation[] evals, int evalWorkers, int evalBatchSize, Iterator ds, Iterator mds, + boolean isCG, Broadcast json, Broadcast params){ + Preconditions.checkArgument(evalWorkers > 0, "Invalid number of evaluation workers: must be > 0. Got: %s", evalWorkers); + Preconditions.checkState(ds != null || mds != null, "No data provided - both DataSet and MultiDataSet iterators were null"); + + //For multi-GPU we'll use a round robbin approach for worker thread/GPU affinity + int numDevices = Nd4j.getAffinityManager().getNumberOfDevices(); + if(numDevices <= 0) + numDevices = 1; + + //Create the device local params if required + DeviceLocalNDArray deviceLocalParams; + synchronized (this){ + if(!paramsMap.containsKey(params.getValue())){ + //Due to singleton pattern, this block should execute only once (first thread) + //Initially put on device 0. For CPU, this means we only have a single copy of the params INDArray shared by + // all threads, which is both safe and uses the least amount of memory + //For CUDA, we can't share threads otherwise arrays will be continually relocated, causing a crash + //Nd4j.getMemoryManager().releaseCurrentContext(); + //NativeOpsHolder.getInstance().getDeviceNativeOps().setDevice(0); + //Nd4j.getAffinityManager().attachThreadToDevice(Thread.currentThread(), 0); + byte[] pBytes = params.getValue(); + INDArray p; + try{ + p = Nd4j.read(new ByteArrayInputStream(pBytes)); + } catch (RuntimeException e){ + throw new RuntimeException(e); //Should never happen + } + DeviceLocalNDArray dlp = new DeviceLocalNDArray(p); + paramsMap.put(params.getValue(), dlp); + //log.info("paramsMap: size {}", paramsMap.size()); + } + deviceLocalParams = paramsMap.get(params.getValue()); + } + + int currentWorkerCount; + while((currentWorkerCount = workerCount.get()) < evalWorkers){ + //For load balancing: we're relying on the fact that threads are mapped to devices in a round-robbin approach + // the first time they touch an INDArray. If we assume this method is called by new threads, + // then the first N workers will be distributed evenly across available devices. + + if (workerCount.compareAndSet(currentWorkerCount, currentWorkerCount + 1)) { + log.debug("Starting evaluation in thread {}", Thread.currentThread().getId()); + //This thread is now a worker + EvaluationFuture f = new EvaluationFuture(); + f.setResult(evals); + try { + Model m; + if (isCG) { + ComputationGraphConfiguration conf = ComputationGraphConfiguration.fromJson(json.getValue()); + ComputationGraph cg = new ComputationGraph(conf); + cg.init(deviceLocalParams.get(), false); + m = cg; + } else { + MultiLayerConfiguration conf = MultiLayerConfiguration.fromJson(json.getValue()); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(deviceLocalParams.get(), false); + m = net; + } + + //Perform eval on this thread's data + try { + doEval(m, evals, ds, mds, evalBatchSize); + } catch (Throwable t) { + f.setException(t); + } finally { + f.getSemaphore().release(1); + } + + //Perform eval on other thread's data + while (!queue.isEmpty()) { + Eval e = queue.poll(); //Use poll not remove to avoid race condition on last element + if (e == null) + continue; + try { + doEval(m, evals, e.getDs(), e.getMds(), evalBatchSize); + } catch (Throwable t) { + e.getFuture().setException(t); + } finally { + e.getFuture().getSemaphore().release(1); + } + } + } finally { + workerCount.decrementAndGet(); + log.debug("Finished evaluation in thread {}", Thread.currentThread().getId()); + } + + Nd4j.getExecutioner().commit(); + return f; + } + } + + //At this point: not a worker thread (otherwise, would have returned already) + log.debug("Submitting evaluation from thread {} for processing in evaluation thread", Thread.currentThread().getId()); + EvaluationFuture f = new EvaluationFuture(); + queue.add(new Eval(ds, mds, evals, f)); + return f; + } + + private static void doEval(Model m, IEvaluation[] e, Iterator ds, Iterator mds, int evalBatchSize){ + if(m instanceof MultiLayerNetwork){ + MultiLayerNetwork mln = (MultiLayerNetwork)m; + if(ds != null){ + mln.doEvaluation(new IteratorDataSetIterator(ds, evalBatchSize), e); + } else { + mln.doEvaluation(new IteratorMultiDataSetIterator(mds, evalBatchSize), e); + } + } else { + ComputationGraph cg = (ComputationGraph)m; + if(ds != null){ + cg.doEvaluation(new IteratorDataSetIterator(ds, evalBatchSize), e); + } else { + cg.doEvaluation(new IteratorMultiDataSetIterator(mds, evalBatchSize), e); + } + } + } + + + + @AllArgsConstructor + @Data + private static class Eval { + private Iterator ds; + private Iterator mds; + private IEvaluation[] evaluations; + private EvaluationFuture future; + } + + @Setter + @Getter + private static class EvaluationFuture implements Future { + + private Semaphore semaphore = new Semaphore(0); + private IEvaluation[] result; + private Throwable exception; + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return semaphore.availablePermits() > 0; + } + + @Override + public IEvaluation[] get() throws InterruptedException, ExecutionException { + if(result == null && exception == null) + semaphore.acquire(); //Block until completion (or failure) is reported + if(exception != null){ + throw new ExecutionException(exception); + } + return result; + } + + @Override + public IEvaluation[] get(long timeout, @NonNull TimeUnit unit) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/SparkComputationGraph.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/SparkComputationGraph.java new file mode 100644 index 000000000..14d08dc99 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/SparkComputationGraph.java @@ -0,0 +1,915 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaDoubleRDD; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.rdd.RDD; +import org.datavec.spark.util.BroadcastHadoopConfigHolder; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.core.loader.impl.SerializedDataSetLoader; +import org.deeplearning4j.core.loader.impl.SerializedMultiDataSetLoader; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.impl.SparkListenable; +import org.deeplearning4j.spark.impl.common.reduce.LongDoubleReduceFunction; +import org.deeplearning4j.spark.impl.graph.dataset.DataSetToMultiDataSetFn; +import org.deeplearning4j.spark.impl.graph.dataset.PairDataSetToMultiDataSetFn; +import org.deeplearning4j.spark.impl.graph.evaluation.IEvaluateMDSFlatMapFunction; +import org.deeplearning4j.spark.impl.graph.evaluation.IEvaluateMDSPathsFlatMapFunction; +import org.deeplearning4j.spark.impl.graph.scoring.*; +import org.deeplearning4j.spark.impl.multilayer.evaluation.IEvaluateAggregateFunction; +import org.deeplearning4j.spark.impl.multilayer.evaluation.IEvaluateFlatMapFunction; +import org.deeplearning4j.spark.util.SparkUtils; +import org.deeplearning4j.util.ModelSerializer; +import org.nd4j.common.base.Preconditions; +import org.nd4j.evaluation.IEvaluation; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.evaluation.classification.ROC; +import org.nd4j.evaluation.classification.ROCMultiClass; +import org.nd4j.evaluation.regression.RegressionEvaluation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.executioner.GridExecutioner; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.heartbeat.Heartbeat; +import org.nd4j.linalg.heartbeat.reports.Environment; +import org.nd4j.linalg.heartbeat.reports.Event; +import org.nd4j.linalg.heartbeat.reports.Task; +import org.nd4j.linalg.heartbeat.utils.EnvironmentUtils; +import scala.Tuple2; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class SparkComputationGraph extends SparkListenable { + public static final int DEFAULT_ROC_THRESHOLD_STEPS = 32; + public static final int DEFAULT_EVAL_SCORE_BATCH_SIZE = 64; + public static final int DEFAULT_EVAL_WORKERS = 4; + private transient JavaSparkContext sc; + private ComputationGraphConfiguration conf; + private ComputationGraph network; + private double lastScore; + private int defaultEvaluationWorkers = DEFAULT_EVAL_WORKERS; + + private transient AtomicInteger iterationsCount = new AtomicInteger(0); + + /** + * Instantiate a ComputationGraph instance with the given context, network and training master. + * + * @param sparkContext the spark context to use + * @param network the network to use + * @param trainingMaster Required for training. May be null if the SparkComputationGraph is only to be used + * for evaluation or inference + */ + public SparkComputationGraph(SparkContext sparkContext, ComputationGraph network, TrainingMaster trainingMaster) { + this(new JavaSparkContext(sparkContext), network, trainingMaster); + } + + public SparkComputationGraph(JavaSparkContext javaSparkContext, ComputationGraph network, + TrainingMaster trainingMaster) { + sc = javaSparkContext; + this.trainingMaster = trainingMaster; + this.conf = network.getConfiguration().clone(); + this.network = network; + this.network.init(); + + //Check if kryo configuration is correct: + SparkUtils.checkKryoConfiguration(javaSparkContext, log); + } + + + public SparkComputationGraph(SparkContext sparkContext, ComputationGraphConfiguration conf, + TrainingMaster trainingMaster) { + this(new JavaSparkContext(sparkContext), conf, trainingMaster); + } + + public SparkComputationGraph(JavaSparkContext sparkContext, ComputationGraphConfiguration conf, + TrainingMaster trainingMaster) { + sc = sparkContext; + this.trainingMaster = trainingMaster; + this.conf = conf.clone(); + this.network = new ComputationGraph(conf); + this.network.init(); + + //Check if kryo configuration is correct: + SparkUtils.checkKryoConfiguration(sparkContext, log); + } + + public JavaSparkContext getSparkContext() { + return sc; + } + + public void setCollectTrainingStats(boolean collectTrainingStats) { + trainingMaster.setCollectTrainingStats(collectTrainingStats); + } + + public SparkTrainingStats getSparkTrainingStats() { + return trainingMaster.getTrainingStats(); + } + + /** + * @return The trained ComputationGraph + */ + public ComputationGraph getNetwork() { + return network; + } + + /** + * @return The TrainingMaster for this network + */ + public TrainingMaster getTrainingMaster() { + return trainingMaster; + } + + /** + * @param network The network to be used for any subsequent training, inference and evaluation steps + */ + public void setNetwork(ComputationGraph network) { + this.network = network; + } + + /** + * Returns the currently set default number of evaluation workers/threads. + * Note that when the number of workers is provided explicitly in an evaluation method, the default value + * is not used.
+ * In many cases, we may want this to be smaller than the number of Spark threads, to reduce memory requirements. + * For example, with 32 Spark threads and a large network, we don't want to spin up 32 instances of the network + * to perform evaluation. Better (for memory requirements, and reduced cache thrashing) to use say 4 workers.
+ * If it is not set explicitly, {@link #DEFAULT_EVAL_WORKERS} will be used + * + * @return Default number of evaluation workers (threads). + */ + public int getDefaultEvaluationWorkers(){ + return defaultEvaluationWorkers; + } + + /** + * Set the default number of evaluation workers/threads. + * Note that when the number of workers is provided explicitly in an evaluation method, the default value + * is not used.
+ * In many cases, we may want this to be smaller than the number of Spark threads, to reduce memory requirements. + * For example, with 32 Spark threads and a large network, we don't want to spin up 32 instances of the network + * to perform evaluation. Better (for memory requirements, and reduced cache thrashing) to use say 4 workers.
+ * If it is not set explicitly, {@link #DEFAULT_EVAL_WORKERS} will be used + * + * @return Default number of evaluation workers (threads). + */ + public void setDefaultEvaluationWorkers(int workers){ + Preconditions.checkArgument(workers > 0, "Number of workers must be > 0: got %s", workers); + this.defaultEvaluationWorkers = workers; + } + + /** + * Fit the ComputationGraph with the given data set + * + * @param rdd Data to train on + * @return Trained network + */ + public ComputationGraph fit(RDD rdd) { + return fit(rdd.toJavaRDD()); + } + + /** + * Fit the ComputationGraph with the given data set + * + * @param rdd Data to train on + * @return Trained network + */ + public ComputationGraph fit(JavaRDD rdd) { + if (Nd4j.getExecutioner() instanceof GridExecutioner) + ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); + + trainingMaster.executeTraining(this, rdd); + network.incrementEpochCount(); + return network; + } + + /** + * Fit the SparkComputationGraph network using a directory of serialized DataSet objects + * The assumption here is that the directory contains a number of {@link DataSet} objects, each serialized using + * {@link DataSet#save(OutputStream)} + * + * @param path Path to the directory containing the serialized DataSet objcets + * @return The MultiLayerNetwork after training + */ + public ComputationGraph fit(String path) { + if (Nd4j.getExecutioner() instanceof GridExecutioner) + ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); + + JavaRDD paths; + try { + paths = SparkUtils.listPaths(sc, path); + } catch (IOException e) { + throw new RuntimeException("Error listing paths in directory", e); + } + + return fitPaths(paths); + } + + /** + * @deprecated Use {@link #fit(String)} + */ + @Deprecated + public ComputationGraph fit(String path, int minPartitions) { + return fit(path); + } + + /** + * Fit the network using a list of paths for serialized DataSet objects. + * + * @param paths List of paths + * @return trained network + */ + public ComputationGraph fitPaths(JavaRDD paths) { + return fitPaths(paths, new SerializedDataSetLoader()); + } + + public ComputationGraph fitPaths(JavaRDD paths, DataSetLoader loader) { + trainingMaster.executeTrainingPaths(null,this, paths, loader, null); + network.incrementEpochCount(); + return network; + } + + /** + * Fit the ComputationGraph with the given data set + * + * @param rdd Data to train on + * @return Trained network + */ + public ComputationGraph fitMultiDataSet(RDD rdd) { + return fitMultiDataSet(rdd.toJavaRDD()); + } + + /** + * Fit the ComputationGraph with the given data set + * + * @param rdd Data to train on + * @return Trained network + */ + public ComputationGraph fitMultiDataSet(JavaRDD rdd) { + if (Nd4j.getExecutioner() instanceof GridExecutioner) + ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); + + trainingMaster.executeTrainingMDS(this, rdd); + network.incrementEpochCount(); + return network; + } + + /** + * Fit the SparkComputationGraph network using a directory of serialized MultiDataSet objects + * The assumption here is that the directory contains a number of serialized {@link MultiDataSet} objects + * + * @param path Path to the directory containing the serialized MultiDataSet objcets + * @return The MultiLayerNetwork after training + */ + public ComputationGraph fitMultiDataSet(String path) { + if (Nd4j.getExecutioner() instanceof GridExecutioner) + ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); + + JavaRDD paths; + try { + paths = SparkUtils.listPaths(sc, path); + } catch (IOException e) { + throw new RuntimeException("Error listing paths in directory", e); + } + + return fitPathsMultiDataSet(paths); + } + + /** + * Fit the network using a list of paths for serialized MultiDataSet objects. + * + * @param paths List of paths + * @return trained network + */ + public ComputationGraph fitPathsMultiDataSet(JavaRDD paths) { + return fitPaths(paths, new SerializedMultiDataSetLoader()); + } + + public ComputationGraph fitPaths(JavaRDD paths, MultiDataSetLoader loader) { + trainingMaster.executeTrainingPaths(null, this, paths, null, loader); + network.incrementEpochCount(); + return network; + } + + /** + * @deprecated use {@link #fitMultiDataSet(String)} + */ + @Deprecated + public ComputationGraph fitMultiDataSet(String path, int minPartitions) { + return fitMultiDataSet(path); + } + + /** + * Gets the last (average) minibatch score from calling fit. This is the average score across all executors for the + * last minibatch executed in each worker + */ + public double getScore() { + return lastScore; + } + + public void setScore(double lastScore) { + this.lastScore = lastScore; + } + + /** + * Calculate the score for all examples in the provided {@code JavaRDD}, either by summing + * or averaging over the entire data set. To calculate a score for each example individually, use {@link #scoreExamples(JavaPairRDD, boolean)} + * or one of the similar methods. Uses default minibatch size in each worker, {@link SparkComputationGraph#DEFAULT_EVAL_SCORE_BATCH_SIZE} + * + * @param data Data to score + * @param average Whether to sum the scores, or average them + */ + public double calculateScore(JavaRDD data, boolean average) { + return calculateScore(data, average, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Calculate the score for all examples in the provided {@code JavaRDD}, either by summing + * or averaging over the entire data set. To calculate a score for each example individually, use {@link #scoreExamples(JavaPairRDD, boolean)} + * or one of the similar methods + * + * @param data Data to score + * @param average Whether to sum the scores, or average them + * @param minibatchSize The number of examples to use in each minibatch when scoring. If more examples are in a partition than + * this, multiple scoring operations will be done (to avoid using too much memory by doing the whole partition + * in one go) + */ + public double calculateScore(JavaRDD data, boolean average, int minibatchSize) { + JavaRDD> rdd = data.mapPartitions(new ScoreFlatMapFunctionCGDataSet(conf.toJson(), + sc.broadcast(network.params()), minibatchSize)); + + //Reduce to a single tuple, with example count + sum of scores + Tuple2 countAndSumScores = rdd.reduce(new LongDoubleReduceFunction()); + if (average) { + return countAndSumScores._2() / countAndSumScores._1(); + } else { + return countAndSumScores._2(); + } + } + + /** + * Calculate the score for all examples in the provided {@code JavaRDD}, either by summing + * or averaging over the entire data set. + * Uses default minibatch size in each worker, {@link SparkComputationGraph#DEFAULT_EVAL_SCORE_BATCH_SIZE} + * + * @param data Data to score + * @param average Whether to sum the scores, or average them + */ + public double calculateScoreMultiDataSet(JavaRDD data, boolean average) { + return calculateScoreMultiDataSet(data, average, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Calculate the score for all examples in the provided {@code JavaRDD}, either by summing + * or averaging over the entire data set. + * * + * @param data Data to score + * @param average Whether to sum the scores, or average them + * @param minibatchSize The number of examples to use in each minibatch when scoring. If more examples are in a partition than + * this, multiple scoring operations will be done (to avoid using too much memory by doing the whole partition + * in one go) + */ + public double calculateScoreMultiDataSet(JavaRDD data, boolean average, int minibatchSize) { + JavaRDD> rdd = data.mapPartitions(new ScoreFlatMapFunctionCGMultiDataSet(conf.toJson(), + sc.broadcast(network.params()), minibatchSize)); + //Reduce to a single tuple, with example count + sum of scores + Tuple2 countAndSumScores = rdd.reduce(new LongDoubleReduceFunction()); + if (average) { + return countAndSumScores._2() / countAndSumScores._1(); + } else { + return countAndSumScores._2(); + } + } + + /** + * DataSet version of {@link #scoreExamples(JavaRDD, boolean)} + */ + public JavaDoubleRDD scoreExamples(JavaRDD data, boolean includeRegularizationTerms) { + return scoreExamplesMultiDataSet(data.map(new DataSetToMultiDataSetFn()), includeRegularizationTerms); + } + + /** + * DataSet version of {@link #scoreExamples(JavaPairRDD, boolean, int)} + */ + public JavaDoubleRDD scoreExamples(JavaRDD data, boolean includeRegularizationTerms, int batchSize) { + return scoreExamplesMultiDataSet(data.map(new DataSetToMultiDataSetFn()), includeRegularizationTerms, + batchSize); + } + + /** + * DataSet version of {@link #scoreExamples(JavaPairRDD, boolean)} + */ + public JavaPairRDD scoreExamples(JavaPairRDD data, boolean includeRegularizationTerms) { + return scoreExamplesMultiDataSet(data.mapToPair(new PairDataSetToMultiDataSetFn()), + includeRegularizationTerms, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * DataSet version of {@link #scoreExamples(JavaPairRDD, boolean, int)} + */ + public JavaPairRDD scoreExamples(JavaPairRDD data, boolean includeRegularizationTerms, + int batchSize) { + return scoreExamplesMultiDataSet(data.mapToPair(new PairDataSetToMultiDataSetFn()), + includeRegularizationTerms, batchSize); + } + + /** + * Score the examples individually, using the default batch size {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE}. Unlike {@link #calculateScore(JavaRDD, boolean)}, + * this method returns a score for each example separately. If scoring is needed for specific examples use either + * {@link #scoreExamples(JavaPairRDD, boolean)} or {@link #scoreExamples(JavaPairRDD, boolean, int)} which can have + * a key for each example. + * + * @param data Data to score + * @param includeRegularizationTerms If true: include the l1/l2 regularization terms with the score (if any) + * @return A JavaDoubleRDD containing the scores of each example + * @see ComputationGraph#scoreExamples(MultiDataSet, boolean) + */ + public JavaDoubleRDD scoreExamplesMultiDataSet(JavaRDD data, boolean includeRegularizationTerms) { + return scoreExamplesMultiDataSet(data, includeRegularizationTerms, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Score the examples individually, using a specified batch size. Unlike {@link #calculateScore(JavaRDD, boolean)}, + * this method returns a score for each example separately. If scoring is needed for specific examples use either + * {@link #scoreExamples(JavaPairRDD, boolean)} or {@link #scoreExamples(JavaPairRDD, boolean, int)} which can have + * a key for each example. + * + * @param data Data to score + * @param includeRegularizationTerms If true: include the l1/l2 regularization terms with the score (if any) + * @param batchSize Batch size to use when doing scoring + * @return A JavaDoubleRDD containing the scores of each example + * @see ComputationGraph#scoreExamples(MultiDataSet, boolean) + */ + public JavaDoubleRDD scoreExamplesMultiDataSet(JavaRDD data, boolean includeRegularizationTerms, + int batchSize) { + return data.mapPartitionsToDouble(new ScoreExamplesFunction(sc.broadcast(network.params()), + sc.broadcast(conf.toJson()), includeRegularizationTerms, batchSize)); + } + + /** + * Score the examples individually, using the default batch size {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE}. Unlike {@link #calculateScore(JavaRDD, boolean)}, + * this method returns a score for each example separately
+ * Note: The provided JavaPairRDD has a key that is associated with each example and returned score.
+ * Note: The DataSet objects passed in must have exactly one example in them (otherwise: can't have a 1:1 association + * between keys and data sets to score) + * + * @param data Data to score + * @param includeRegularizationTerms If true: include the l1/l2 regularization terms with the score (if any) + * @param Key type + * @return A {@code JavaPairRDD} containing the scores of each example + * @see MultiLayerNetwork#scoreExamples(DataSet, boolean) + */ + public JavaPairRDD scoreExamplesMultiDataSet(JavaPairRDD data, + boolean includeRegularizationTerms) { + return scoreExamplesMultiDataSet(data, includeRegularizationTerms, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Feed-forward the specified data, with the given keys. i.e., get the network output/predictions for the specified data + * + * @param featuresData Features data to feed through the network + * @param batchSize Batch size to use when doing feed forward operations + * @param Type of data for key - may be anything + * @return Network output given the input, by key + */ + public JavaPairRDD feedForwardWithKeySingle(JavaPairRDD featuresData, int batchSize) { + if (network.getNumInputArrays() != 1 || network.getNumOutputArrays() != 1) { + throw new IllegalStateException( + "Cannot use this method with computation graphs with more than 1 input or output " + + "( has: " + network.getNumInputArrays() + " inputs, " + + network.getNumOutputArrays() + " outputs"); + } + PairToArrayPair p = new PairToArrayPair<>(); + JavaPairRDD rdd = featuresData.mapToPair(p); + return feedForwardWithKey(rdd, batchSize).mapToPair(new ArrayPairToPair()); + } + + /** + * Feed-forward the specified data, with the given keys. i.e., get the network output/predictions for the specified data + * + * @param featuresData Features data to feed through the network + * @param batchSize Batch size to use when doing feed forward operations + * @param Type of data for key - may be anything + * @return Network output given the input, by key + */ + public JavaPairRDD feedForwardWithKey(JavaPairRDD featuresData, int batchSize) { + return featuresData.mapPartitionsToPair(new GraphFeedForwardWithKeyFunction(sc.broadcast(network.params()), + sc.broadcast(conf.toJson()), batchSize)); + } + + private void update(int mr, long mg) { + Environment env = EnvironmentUtils.buildEnvironment(); + env.setNumCores(mr); + env.setAvailableMemory(mg); + Task task = ModelSerializer.taskByModel(network); + Heartbeat.getInstance().reportEvent(Event.SPARK, env, task); + } + + /** + * Score the examples individually, using a specified batch size. Unlike {@link #calculateScore(JavaRDD, boolean)}, + * this method returns a score for each example separately
+ * Note: The provided JavaPairRDD has a key that is associated with each example and returned score.
+ * Note: The DataSet objects passed in must have exactly one example in them (otherwise: can't have a 1:1 association + * between keys and data sets to score) + * + * @param data Data to score + * @param includeRegularizationTerms If true: include the l1/l2 regularization terms with the score (if any) + * @param Key type + * @return A {@code JavaPairRDD} containing the scores of each example + * @see MultiLayerNetwork#scoreExamples(DataSet, boolean) + */ + public JavaPairRDD scoreExamplesMultiDataSet(JavaPairRDD data, + boolean includeRegularizationTerms, int batchSize) { + return data.mapPartitionsToPair(new ScoreExamplesWithKeyFunction(sc.broadcast(network.params()), + sc.broadcast(conf.toJson()), includeRegularizationTerms, batchSize)); + } + + /** + * Evaluate the single-output network on a directory containing a set of DataSet objects to be loaded with a {@link DataSetLoader}. + * Uses default batch size of {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE} + * @param path Path/URI to the directory containing the datasets to load + * @return Evaluation + */ + public Evaluation evaluate(String path, DataSetLoader loader){ + JavaRDD data; + try { + data = SparkUtils.listPaths(sc, path); + } catch (IOException e){ + throw new RuntimeException("Error listing files for evaluation of files at path: " + path, e); + } + return (Evaluation) doEvaluation(data, DEFAULT_EVAL_WORKERS, DEFAULT_EVAL_SCORE_BATCH_SIZE, loader, (MultiDataSetLoader)null, new Evaluation())[0]; + } + + /** + * Evaluate the single-output network on a directory containing a set of MultiDataSet objects to be loaded with a {@link MultiDataSetLoader}. + * Uses default batch size of {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE} + * @param path Path/URI to the directory containing the datasets to load + * @return Evaluation + */ + public Evaluation evaluate(String path, MultiDataSetLoader loader){ + JavaRDD data; + try { + data = SparkUtils.listPaths(sc, path); + } catch (IOException e){ + throw new RuntimeException("Error listing files for evaluation of files at path: " + path, e); + } + return (Evaluation) doEvaluation(data, DEFAULT_EVAL_WORKERS, DEFAULT_EVAL_SCORE_BATCH_SIZE, null, loader, new Evaluation())[0]; + } + + /** + * {@code RDD} overload of {@link #evaluate(JavaRDD)} + */ + public T evaluate(RDD data) { + return evaluate(data.toJavaRDD()); + } + + /** + * Evaluate the network (classification performance) in a distributed manner on the provided data + * + * @param data Data to evaluate on + * @return Evaluation object; results of evaluation on all examples in the data set + */ + public T evaluate(JavaRDD data) { + return evaluate(data, null); + } + + /** + * {@code RDD} overload of {@link #evaluate(JavaRDD, List)} + */ + public T evaluate(RDD data, List labelsList) { + return evaluate(data.toJavaRDD(), labelsList); + } + + /** + * Evaluate the network (regression performance) in a distributed manner on the provided data + * + * @param data Data to evaluate + * @return {@link RegressionEvaluation} instance with regression performance + */ + public T evaluateRegression(JavaRDD data) { + return evaluateRegression(data, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Evaluate the network (regression performance) in a distributed manner on the provided data + * + * @param data Data to evaluate + * @param minibatchSize Minibatch size to use when doing performing evaluation + * @return {@link RegressionEvaluation} instance with regression performance + */ + public T evaluateRegression(JavaRDD data, int minibatchSize) { + val nOut = ((FeedForwardLayer) network.getOutputLayer(0).conf().getLayer()).getNOut(); + return (T)doEvaluation(data, new org.deeplearning4j.eval.RegressionEvaluation(nOut), minibatchSize); + } + + /** + * Evaluate the network (classification performance) in a distributed manner, using default batch size and a provided + * list of labels + * + * @param data Data to evaluate on + * @param labelsList List of labels used for evaluation + * @return Evaluation object; results of evaluation on all examples in the data set + */ + public T evaluate(JavaRDD data, List labelsList) { + return evaluate(data, labelsList, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Perform ROC analysis/evaluation on the given DataSet in a distributed manner, using the default number of + * threshold steps ({@link #DEFAULT_ROC_THRESHOLD_STEPS}) and the default minibatch size ({@link #DEFAULT_EVAL_SCORE_BATCH_SIZE}) + * + * @param data Test set data (to evaluate on) + * @return ROC for the entire data set + */ + public T evaluateROC(JavaRDD data) { + return evaluateROC(data, DEFAULT_ROC_THRESHOLD_STEPS, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Perform ROC analysis/evaluation on the given DataSet in a distributed manner + * + * @param data Test set data (to evaluate on) + * @param thresholdSteps Number of threshold steps for ROC - see {@link ROC} + * @param evaluationMinibatchSize Minibatch size to use when performing ROC evaluation + * @return ROC for the entire data set + */ + public T evaluateROC(JavaRDD data, int thresholdSteps, int evaluationMinibatchSize) { + return (T)doEvaluation(data, new org.deeplearning4j.eval.ROC(thresholdSteps), evaluationMinibatchSize); + } + + /** + * Perform ROC analysis/evaluation (for the multi-class case, using {@link ROCMultiClass} on the given DataSet in a distributed manner + * + * @param data Test set data (to evaluate on) + * @return ROC for the entire data set + */ + public T evaluateROCMultiClass(JavaRDD data) { + return evaluateROCMultiClass(data, DEFAULT_ROC_THRESHOLD_STEPS, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Perform ROC analysis/evaluation (for the multi-class case, using {@link ROCMultiClass} on the given DataSet in a distributed manner + * + * @param data Test set data (to evaluate on) + * @param thresholdSteps Number of threshold steps for ROC - see {@link ROC} + * @param evaluationMinibatchSize Minibatch size to use when performing ROC evaluation + * @return ROCMultiClass for the entire data set + */ + public T evaluateROCMultiClass(JavaRDD data, int thresholdSteps, int evaluationMinibatchSize) { + return (T)doEvaluation(data, new org.deeplearning4j.eval.ROCMultiClass(thresholdSteps), evaluationMinibatchSize); + } + + + + /** + * Evaluate the network (classification performance) in a distributed manner, using specified batch size and a provided + * list of labels + * + * @param data Data to evaluate on + * @param labelsList List of labels used for evaluation + * @param evalBatchSize Batch size to use when conducting evaluations + * @return Evaluation object; results of evaluation on all examples in the data set + */ + public T evaluate(JavaRDD data, List labelsList, int evalBatchSize) { + Evaluation e = new org.deeplearning4j.eval.Evaluation(); + e = doEvaluation(data, e, evalBatchSize); + if (labelsList != null) { + e.setLabelsList(labelsList); + } + return (T)e; + } + + + + /** + * Evaluate the network (classification performance) in a distributed manner on the provided data + */ + public T evaluateMDS(JavaRDD data) { + return evaluateMDS(data, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Evaluate the network (classification performance) in a distributed manner on the provided data + */ + public T evaluateMDS(JavaRDD data, int minibatchSize) { + return (T)doEvaluationMDS(data, minibatchSize, new org.deeplearning4j.eval.Evaluation())[0]; + } + + /** + * Evaluate the network (regression performance) in a distributed manner on the provided data + * + * @param data Data to evaluate + * @return {@link RegressionEvaluation} instance with regression performance + */ + public T evaluateRegressionMDS(JavaRDD data) { + return evaluateRegressionMDS(data, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Evaluate the network (regression performance) in a distributed manner on the provided data + * + * @param data Data to evaluate + * @param minibatchSize Minibatch size to use when doing performing evaluation + * @return {@link RegressionEvaluation} instance with regression performance + */ + public T evaluateRegressionMDS(JavaRDD data, int minibatchSize) { + return (T)doEvaluationMDS(data, minibatchSize, new org.deeplearning4j.eval.RegressionEvaluation())[0]; + } + + /** + * Perform ROC analysis/evaluation on the given DataSet in a distributed manner, using the default number of + * threshold steps ({@link #DEFAULT_ROC_THRESHOLD_STEPS}) and the default minibatch size ({@link #DEFAULT_EVAL_SCORE_BATCH_SIZE}) + * + * @param data Test set data (to evaluate on) + * @return ROC for the entire data set + */ + public ROC evaluateROCMDS(JavaRDD data) { + return evaluateROCMDS(data, DEFAULT_ROC_THRESHOLD_STEPS, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Perform ROC analysis/evaluation on the given DataSet in a distributed manner, using the specified number of + * steps and minibatch size + * + * @param data Test set data (to evaluate on) + * @param rocThresholdNumSteps See {@link ROC} for details + * @param minibatchSize Minibatch size for evaluation + * @return ROC for the entire data set + */ + public T evaluateROCMDS(JavaRDD data, int rocThresholdNumSteps, int minibatchSize) { + return (T)doEvaluationMDS(data, minibatchSize, new org.deeplearning4j.eval.ROC(rocThresholdNumSteps))[0]; + } + + + /** + * Perform distributed evaluation of any type of {@link IEvaluation}. For example, {@link Evaluation}, {@link RegressionEvaluation}, + * {@link ROC}, {@link ROCMultiClass} etc. + * + * @param data Data to evaluate on + * @param emptyEvaluation Empty evaluation instance. This is the starting point (serialized/duplicated, then merged) + * @param evalBatchSize Evaluation batch size + * @param Type of evaluation instance to return + * @return IEvaluation instance + */ + @SuppressWarnings("unchecked") + public T doEvaluation(JavaRDD data, T emptyEvaluation, int evalBatchSize) { + IEvaluation[] arr = new IEvaluation[] {emptyEvaluation}; + return (T) doEvaluation(data, evalBatchSize, arr)[0]; + } + + /** + * Perform distributed evaluation on a single output ComputationGraph form DataSet objects using Spark. + * Can be used to perform multiple evaluations on this single output (for example, {@link Evaluation} and + * {@link ROC}) at the same time.
+ * Note that the default number of worker threads {@link #getDefaultEvaluationWorkers()} will be used + * + * @param data Data to evaluatie + * @param evalBatchSize Minibatch size for evaluation + * @param emptyEvaluations Evaluations to perform + * @return Evaluations + */ + public T[] doEvaluation(JavaRDD data, int evalBatchSize, T... emptyEvaluations) { + return doEvaluation(data, getDefaultEvaluationWorkers(), evalBatchSize, emptyEvaluations); + } + + /** + * Perform distributed evaluation on a single output ComputationGraph form DataSet objects using Spark. + * Can be used to perform multiple evaluations on this single output (for example, {@link Evaluation} and + * {@link ROC}) at the same time.
+ * + * @param data Data to evaluatie + * @param evalNumWorkers Number of worker threads (per machine) to use for evaluation. May want tis to be less than + * the number of Spark threads per machine/JVM to reduce memory requirements + * @param evalBatchSize Minibatch size for evaluation + * @param emptyEvaluations Evaluations to perform + * @return Evaluations + */ + public T[] doEvaluation(JavaRDD data, int evalNumWorkers, int evalBatchSize, T... emptyEvaluations) { + IEvaluateFlatMapFunction evalFn = new IEvaluateFlatMapFunction<>(true, sc.broadcast(conf.toJson()), + SparkUtils.asByteArrayBroadcast(sc, network.params()), evalNumWorkers, evalBatchSize, emptyEvaluations); + JavaRDD evaluations = data.mapPartitions(evalFn); + return evaluations.treeAggregate(null, new IEvaluateAggregateFunction(), + new IEvaluateAggregateFunction()); + } + + /** + * Perform distributed evaluation on a single output ComputationGraph form MultiDataSet objects using Spark. + * Can be used to perform multiple evaluations on this single output (for example, {@link Evaluation} and + * {@link ROC}) at the same time. + * + * @param data Data to evaluatie + * @param evalBatchSize Minibatch size for evaluation + * @param emptyEvaluations Evaluations to perform + * @return Evaluations + */ + @SuppressWarnings("unchecked") + public T[] doEvaluationMDS(JavaRDD data, int evalBatchSize, T... emptyEvaluations) { + return doEvaluationMDS(data, getDefaultEvaluationWorkers(), evalBatchSize, emptyEvaluations); + } + + public T[] doEvaluationMDS(JavaRDD data, int evalNumWorkers, int evalBatchSize, T... emptyEvaluations) { + Preconditions.checkArgument(evalNumWorkers > 0, "Invalid number of evaulation workers: require at least 1 - got %s", evalNumWorkers); + IEvaluateMDSFlatMapFunction evalFn = new IEvaluateMDSFlatMapFunction<>(sc.broadcast(conf.toJson()), + SparkUtils.asByteArrayBroadcast(sc, network.params()), evalNumWorkers, evalBatchSize, emptyEvaluations); + JavaRDD evaluations = data.mapPartitions(evalFn); + return evaluations.treeAggregate(null, new IEvaluateAggregateFunction(), + new IEvaluateAggregateFunction()); + } + + /** + * Perform evaluation on serialized DataSet objects on disk, (potentially in any format), that are loaded using an {@link DataSetLoader}.
+ * Uses the default number of workers (model replicas per JVM) of {@link #DEFAULT_EVAL_WORKERS} with the default + * minibatch size of {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE} + * @param data List of paths to the data (that can be loaded as / converted to DataSets) + * @param loader Used to load DataSets from their paths + * @param emptyEvaluations Evaluations to perform + * @return Evaluation + */ + public IEvaluation[] doEvaluation(JavaRDD data, DataSetLoader loader, IEvaluation... emptyEvaluations) { + return doEvaluation(data, DEFAULT_EVAL_WORKERS, DEFAULT_EVAL_SCORE_BATCH_SIZE, loader, emptyEvaluations); + } + + /** + * Perform evaluation on serialized DataSet objects on disk, (potentially in any format), that are loaded using an {@link DataSetLoader}. + * @param data List of paths to the data (that can be loaded as / converted to DataSets) + * @param evalNumWorkers Number of workers to perform evaluation with. To reduce memory requirements and cache thrashing, + * it is common to set this to a lower value than the number of spark threads per JVM/executor + * @param evalBatchSize Batch size to use when performing evaluation + * @param loader Used to load DataSets from their paths + * @param emptyEvaluations Evaluations to perform + * @return Evaluation + */ + public IEvaluation[] doEvaluation(JavaRDD data, int evalNumWorkers, int evalBatchSize, DataSetLoader loader, IEvaluation... emptyEvaluations) { + return doEvaluation(data, evalNumWorkers, evalBatchSize, loader, null, emptyEvaluations); + } + + /** + * Perform evaluation on serialized MultiDataSet objects on disk, (potentially in any format), that are loaded using an {@link MultiDataSetLoader}.
+ * Uses the default number of workers (model replicas per JVM) of {@link #DEFAULT_EVAL_WORKERS} with the default + * minibatch size of {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE} + * @param data List of paths to the data (that can be loaded as / converted to DataSets) + * @param loader Used to load MultiDataSets from their paths + * @param emptyEvaluations Evaluations to perform + * @return Evaluation + */ + public IEvaluation[] doEvaluation(JavaRDD data, MultiDataSetLoader loader, IEvaluation... emptyEvaluations) { + return doEvaluation(data, DEFAULT_EVAL_WORKERS, DEFAULT_EVAL_SCORE_BATCH_SIZE, null, loader, emptyEvaluations); + } + + /** + * Perform evaluation on serialized MultiDataSet objects on disk, (potentially in any format), that are loaded using an {@link MultiDataSetLoader} + * @param data List of paths to the data (that can be loaded as / converted to DataSets) + * @param evalNumWorkers Number of workers to perform evaluation with. To reduce memory requirements and cache thrashing, + * it is common to set this to a lower value than the number of spark threads per JVM/executor + * @param evalBatchSize Batch size to use when performing evaluation + * @param loader Used to load MultiDataSets from their paths + * @param emptyEvaluations Evaluations to perform + * @return Evaluation + */ + public IEvaluation[] doEvaluation(JavaRDD data, int evalNumWorkers, int evalBatchSize, MultiDataSetLoader loader, IEvaluation... emptyEvaluations) { + return doEvaluation(data, evalNumWorkers, evalBatchSize, null, loader, emptyEvaluations); + } + + protected IEvaluation[] doEvaluation(JavaRDD data, int evalNumWorkers, int evalBatchSize, DataSetLoader loader, MultiDataSetLoader mdsLoader, IEvaluation... emptyEvaluations){ + IEvaluateMDSPathsFlatMapFunction evalFn = new IEvaluateMDSPathsFlatMapFunction(sc.broadcast(conf.toJson()), + SparkUtils.asByteArrayBroadcast(sc, network.params()), evalNumWorkers, evalBatchSize, loader, mdsLoader, + BroadcastHadoopConfigHolder.get(sc), emptyEvaluations); + Preconditions.checkArgument(evalNumWorkers > 0, "Invalid number of evaulation workers: require at least 1 - got %s", evalNumWorkers); + JavaRDD evaluations = data.mapPartitions(evalFn); + return evaluations.treeAggregate(null, new IEvaluateAggregateFunction<>(), new IEvaluateAggregateFunction<>()); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/dataset/DataSetToMultiDataSetFn.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/dataset/DataSetToMultiDataSetFn.java new file mode 100644 index 000000000..81e384836 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/dataset/DataSetToMultiDataSetFn.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.dataset; + +import org.apache.spark.api.java.function.Function; +import org.deeplearning4j.nn.graph.util.ComputationGraphUtil; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +public class DataSetToMultiDataSetFn implements Function { + @Override + public MultiDataSet call(DataSet d) throws Exception { + return ComputationGraphUtil.toMultiDataSet(d); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/dataset/PairDataSetToMultiDataSetFn.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/dataset/PairDataSetToMultiDataSetFn.java new file mode 100644 index 000000000..94a0b1fb4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/dataset/PairDataSetToMultiDataSetFn.java @@ -0,0 +1,35 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.dataset; + +import org.apache.spark.api.java.function.PairFunction; +import org.deeplearning4j.nn.graph.util.ComputationGraphUtil; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import scala.Tuple2; + +public class PairDataSetToMultiDataSetFn implements PairFunction, K, MultiDataSet> { + + @Override + public Tuple2 call(Tuple2 in) throws Exception { + return new Tuple2<>(in._1(), ComputationGraphUtil.toMultiDataSet(in._2())); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/evaluation/IEvaluateMDSFlatMapFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/evaluation/IEvaluateMDSFlatMapFunction.java new file mode 100644 index 000000000..31bc1333d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/evaluation/IEvaluateMDSFlatMapFunction.java @@ -0,0 +1,78 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.evaluation; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.spark.impl.evaluation.EvaluationRunner; +import org.nd4j.evaluation.IEvaluation; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.util.Collections; +import java.util.Iterator; +import java.util.concurrent.Future; + +public class IEvaluateMDSFlatMapFunction implements FlatMapFunction, T[]> { + + protected Broadcast json; + protected Broadcast params; + protected int evalNumWorkers; + protected int evalBatchSize; + protected T[] evaluations; + + /** + * @param json Network configuration (json format) + * @param params Network parameters + * @param evalBatchSize Max examples per evaluation. Do multiple separate forward passes if data exceeds + * this. Used to avoid doing too many at once (and hence memory issues) + * @param evaluations Initial evaulation instance (i.e., empty Evaluation or RegressionEvaluation instance) + */ + public IEvaluateMDSFlatMapFunction(Broadcast json, Broadcast params, int evalNumWorkers, + int evalBatchSize, T[] evaluations) { + this.json = json; + this.params = params; + this.evalNumWorkers = evalNumWorkers; + this.evalBatchSize = evalBatchSize; + this.evaluations = evaluations; + } + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + if (!dataSetIterator.hasNext()) { + return Collections.emptyIterator(); + } + + if (!dataSetIterator.hasNext()) { + return Collections.emptyIterator(); + } + + Future f = EvaluationRunner.getInstance().execute( + evaluations, evalNumWorkers, evalBatchSize, null, dataSetIterator, true, json, params); + + IEvaluation[] result = f.get(); + if(result == null){ + return Collections.emptyIterator(); + } else { + return Collections.singletonList((T[])result).iterator(); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/evaluation/IEvaluateMDSPathsFlatMapFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/evaluation/IEvaluateMDSPathsFlatMapFunction.java new file mode 100644 index 000000000..70e1532d2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/evaluation/IEvaluateMDSPathsFlatMapFunction.java @@ -0,0 +1,93 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.evaluation; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.datasets.iterator.loader.DataSetLoaderIterator; +import org.deeplearning4j.datasets.iterator.loader.MultiDataSetLoaderIterator; +import org.deeplearning4j.spark.data.loader.RemoteFileSourceFactory; +import org.deeplearning4j.spark.impl.evaluation.EvaluationRunner; +import org.nd4j.evaluation.IEvaluation; +import org.nd4j.linalg.dataset.adapter.MultiDataSetIteratorAdapter; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +import java.util.Collections; +import java.util.Iterator; +import java.util.concurrent.Future; + +public class IEvaluateMDSPathsFlatMapFunction implements FlatMapFunction, IEvaluation[]> { + + protected Broadcast json; + protected Broadcast params; + protected int evalNumWorkers; + protected int evalBatchSize; + protected DataSetLoader dsLoader; + protected MultiDataSetLoader mdsLoader; + protected Broadcast conf; + protected IEvaluation[] evaluations; + + /** + * @param json Network configuration (json format) + * @param params Network parameters + * @param evalBatchSize Max examples per evaluation. Do multiple separate forward passes if data exceeds + * this. Used to avoid doing too many at once (and hence memory issues) + * @param evaluations Initial evaulation instance (i.e., empty Evaluation or RegressionEvaluation instance) + */ + public IEvaluateMDSPathsFlatMapFunction(Broadcast json, Broadcast params, int evalNumWorkers, int evalBatchSize, + DataSetLoader dsLoader, MultiDataSetLoader mdsLoader, Broadcast configuration, IEvaluation[] evaluations) { + this.json = json; + this.params = params; + this.evalNumWorkers = evalNumWorkers; + this.evalBatchSize = evalBatchSize; + this.dsLoader = dsLoader; + this.mdsLoader = mdsLoader; + this.conf = configuration; + this.evaluations = evaluations; + } + + @Override + public Iterator call(Iterator paths) throws Exception { + if (!paths.hasNext()) { + return Collections.emptyIterator(); + } + + MultiDataSetIterator iter; + if(dsLoader != null){ + DataSetIterator dsIter = new DataSetLoaderIterator(paths, dsLoader, new RemoteFileSourceFactory(conf)); + iter = new MultiDataSetIteratorAdapter(dsIter); + } else { + iter = new MultiDataSetLoaderIterator(paths, mdsLoader, new RemoteFileSourceFactory(conf)); + } + + Future f = EvaluationRunner.getInstance().execute(evaluations, evalNumWorkers, evalBatchSize, null, iter, true, json, params); + IEvaluation[] result = f.get(); + if(result == null){ + return Collections.emptyIterator(); + } else { + return Collections.singletonList(result).iterator(); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ArrayPairToPair.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ArrayPairToPair.java new file mode 100644 index 000000000..7ef99a0e5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ArrayPairToPair.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import org.apache.spark.api.java.function.PairFunction; +import org.nd4j.linalg.api.ndarray.INDArray; +import scala.Tuple2; + +public class ArrayPairToPair implements PairFunction, K, INDArray> { + @Override + public Tuple2 call(Tuple2 v1) throws Exception { + INDArray arr = (v1._2() == null ? null : v1._2()[0]); + return new Tuple2<>(v1._1(), arr); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/CGVaeReconstructionErrorWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/CGVaeReconstructionErrorWithKeyFunction.java new file mode 100644 index 000000000..d8aadc3f1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/CGVaeReconstructionErrorWithKeyFunction.java @@ -0,0 +1,68 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.spark.impl.common.score.BaseVaeScoreWithKeyFunction; +import org.nd4j.linalg.api.ndarray.INDArray; + +public class CGVaeReconstructionErrorWithKeyFunction extends BaseVaeScoreWithKeyFunction { + + + /** + * @param params MultiLayerNetwork parameters + * @param jsonConfig MultiLayerConfiguration, as json + * @param batchSize Batch size to use when scoring + */ + public CGVaeReconstructionErrorWithKeyFunction(Broadcast params, Broadcast jsonConfig, + int batchSize) { + super(params, jsonConfig, batchSize); + } + + @Override + public VariationalAutoencoder getVaeLayer() { + ComputationGraph network = + new ComputationGraph(ComputationGraphConfiguration.fromJson((String) jsonConfig.getValue())); + network.init(); + INDArray val = ((INDArray) params.value()).unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcasted set parameters"); + network.setParams(val); + + Layer l = network.getLayer(0); + if (!(l instanceof VariationalAutoencoder)) { + throw new RuntimeException( + "Cannot use CGVaeReconstructionErrorWithKeyFunction on network that doesn't have a VAE " + + "layer as layer 0. Layer type: " + l.getClass()); + } + return (VariationalAutoencoder) l; + } + + @Override + public INDArray computeScore(VariationalAutoencoder vae, INDArray toScore) { + return vae.reconstructionError(toScore); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/CGVaeReconstructionProbWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/CGVaeReconstructionProbWithKeyFunction.java new file mode 100644 index 000000000..57c568239 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/CGVaeReconstructionProbWithKeyFunction.java @@ -0,0 +1,65 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.spark.impl.common.score.BaseVaeReconstructionProbWithKeyFunction; +import org.nd4j.linalg.api.ndarray.INDArray; + +public class CGVaeReconstructionProbWithKeyFunction extends BaseVaeReconstructionProbWithKeyFunction { + + + /** + * @param params MultiLayerNetwork parameters + * @param jsonConfig MultiLayerConfiguration, as json + * @param useLogProbability If true: use log probability. False: use raw probability. + * @param batchSize Batch size to use when scoring + * @param numSamples Number of samples to use when calling {@link VariationalAutoencoder#reconstructionLogProbability(INDArray, int)} + */ + public CGVaeReconstructionProbWithKeyFunction(Broadcast params, Broadcast jsonConfig, + boolean useLogProbability, int batchSize, int numSamples) { + super(params, jsonConfig, useLogProbability, batchSize, numSamples); + } + + @Override + public VariationalAutoencoder getVaeLayer() { + ComputationGraph network = + new ComputationGraph(ComputationGraphConfiguration.fromJson((String) jsonConfig.getValue())); + network.init(); + INDArray val = ((INDArray) params.value()).unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcasted set parameters"); + network.setParams(val); + + Layer l = network.getLayer(0); + if (!(l instanceof VariationalAutoencoder)) { + throw new RuntimeException( + "Cannot use CGVaeReconstructionProbWithKeyFunction on network that doesn't have a VAE " + + "layer as layer 0. Layer type: " + l.getClass()); + } + return (VariationalAutoencoder) l; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/GraphFeedForwardWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/GraphFeedForwardWithKeyFunction.java new file mode 100644 index 000000000..5b99afac5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/GraphFeedForwardWithKeyFunction.java @@ -0,0 +1,187 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.api.java.function.PairFlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +@Slf4j +@AllArgsConstructor +public class GraphFeedForwardWithKeyFunction implements PairFlatMapFunction>, K, INDArray[]> { + + private final Broadcast params; + private final Broadcast jsonConfig; + private final int batchSize; + + + @Override + public Iterator> call(Iterator> iterator) throws Exception { + if (!iterator.hasNext()) { + return Collections.emptyIterator(); + } + + ComputationGraph network = new ComputationGraph(ComputationGraphConfiguration.fromJson(jsonConfig.getValue())); + network.init(); + INDArray val = params.value().unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParams(val); + + //Issue: for 2d data (MLPs etc) we can just stack the examples. + //But: for 3d and 4d: in principle the data sizes could be different + //We could handle that with mask arrays - but it gets messy. The approach used here is simpler but less efficient + + List featuresList = new ArrayList<>(batchSize); + List keyList = new ArrayList<>(batchSize); + List origSizeList = new ArrayList<>(); + + long[][] firstShapes = null; + boolean sizesDiffer = false; + int tupleCount = 0; + while (iterator.hasNext()) { + Tuple2 t2 = iterator.next(); + if (firstShapes == null) { + firstShapes = new long[t2._2().length][0]; + for (int i = 0; i < firstShapes.length; i++) { + firstShapes[i] = t2._2()[i].shape(); + } + } else if (!sizesDiffer) { + for (int i = 0; i < firstShapes.length; i++) { + for (int j = 1; j < firstShapes[i].length; j++) { + if (firstShapes[i][j] != featuresList.get(tupleCount - 1)[i].size(j)) { + sizesDiffer = true; + break; + } + } + } + } + featuresList.add(t2._2()); + keyList.add(t2._1()); + + origSizeList.add(t2._2()[0].size(0)); + tupleCount++; + } + + if (tupleCount == 0) { + return Collections.emptyIterator(); + } + + List> output = new ArrayList<>(tupleCount); + int currentArrayIndex = 0; + + while (currentArrayIndex < featuresList.size()) { + int firstIdx = currentArrayIndex; + int nextIdx = currentArrayIndex; + int examplesInBatch = 0; + List toMerge = new ArrayList<>(); + firstShapes = null; + while (nextIdx < featuresList.size() && examplesInBatch < batchSize) { + INDArray[] f = featuresList.get(nextIdx); + if (firstShapes == null) { + firstShapes = new long[f.length][0]; + for (int i = 0; i < firstShapes.length; i++) { + firstShapes[i] = f[i].shape(); + } + } else if (sizesDiffer) { + boolean breakWhile = false; + for (int i = 0; i < firstShapes.length; i++) { + for (int j = 1; j < firstShapes[i].length; j++) { + if (firstShapes[i][j] != featuresList.get(nextIdx)[i].size(j)) { + //Next example has a different size. So: don't add it to the current batch, just process what we have + breakWhile = true; + break; + } + } + } + if (breakWhile) { + break; + } + } + + toMerge.add(f); + examplesInBatch += f[0].size(0); + nextIdx++; + } + + INDArray[] batchFeatures = new INDArray[toMerge.get(0).length]; + for (int i = 0; i < batchFeatures.length; i++) { + INDArray[] tempArr = new INDArray[toMerge.size()]; + for (int j = 0; j < tempArr.length; j++) { + tempArr[j] = toMerge.get(j)[i]; + } + batchFeatures[i] = Nd4j.concat(0, tempArr); + } + + + INDArray[] out = network.output(false, batchFeatures); + + examplesInBatch = 0; + for (int i = firstIdx; i < nextIdx; i++) { + long numExamples = origSizeList.get(i); + INDArray[] outSubset = new INDArray[out.length]; + for (int j = 0; j < out.length; j++) { + outSubset[j] = getSubset(examplesInBatch, examplesInBatch + numExamples, out[j]); + } + examplesInBatch += numExamples; + + output.add(new Tuple2<>(keyList.get(i), outSubset)); + } + + currentArrayIndex += (nextIdx - firstIdx); + } + + Nd4j.getExecutioner().commit(); + + return output.iterator(); + } + + private INDArray getSubset(long exampleStart, long exampleEnd, INDArray from) { + switch (from.rank()) { + case 2: + return from.get(NDArrayIndex.interval(exampleStart, exampleEnd), NDArrayIndex.all()); + case 3: + return from.get(NDArrayIndex.interval(exampleStart, exampleEnd), NDArrayIndex.all(), + NDArrayIndex.all()); + case 4: + return from.get(NDArrayIndex.interval(exampleStart, exampleEnd), NDArrayIndex.all(), NDArrayIndex.all(), + NDArrayIndex.all()); + default: + throw new RuntimeException("Invalid rank: " + from.rank()); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/PairToArrayPair.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/PairToArrayPair.java new file mode 100644 index 000000000..bc38d0522 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/PairToArrayPair.java @@ -0,0 +1,32 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import org.apache.spark.api.java.function.PairFunction; +import org.nd4j.linalg.api.ndarray.INDArray; +import scala.Tuple2; + +public class PairToArrayPair implements PairFunction, K, INDArray[]> { + @Override + public Tuple2 call(Tuple2 v1) throws Exception { + return new Tuple2<>(v1._1(), new INDArray[] {v1._2()}); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreExamplesFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreExamplesFunction.java new file mode 100644 index 000000000..68f645ffc --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreExamplesFunction.java @@ -0,0 +1,108 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.function.DoubleFlatMapFunction; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import lombok.val; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + + +@Slf4j +public class ScoreExamplesFunction implements DoubleFlatMapFunction> { + + private final Broadcast params; + private final Broadcast jsonConfig; + private final boolean addRegularization; + private final int batchSize; + + public ScoreExamplesFunction(Broadcast params, Broadcast jsonConfig, + boolean addRegularizationTerms, int batchSize) { + this.params = params; + this.jsonConfig = jsonConfig; + this.addRegularization = addRegularizationTerms; + this.batchSize = batchSize; + } + + + @Override + public Iterator call(Iterator iterator) throws Exception { + if (!iterator.hasNext()) { + return Collections.emptyIterator(); + } + + ComputationGraph network = new ComputationGraph(ComputationGraphConfiguration.fromJson(jsonConfig.getValue())); + network.init(); + INDArray val = params.value().unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParams(val); + + List ret = new ArrayList<>(); + + List collect = new ArrayList<>(batchSize); + int totalCount = 0; + while (iterator.hasNext()) { + collect.clear(); + int nExamples = 0; + while (iterator.hasNext() && nExamples < batchSize) { + MultiDataSet ds = iterator.next(); + val n = ds.getFeatures(0).size(0); + collect.add(ds); + nExamples += n; + } + totalCount += nExamples; + + + MultiDataSet data = org.nd4j.linalg.dataset.MultiDataSet.merge(collect); + + + INDArray scores = network.scoreExamples(data, addRegularization); + double[] doubleScores = scores.data().asDouble(); + + for (double doubleScore : doubleScores) { + ret.add(doubleScore); + } + } + + Nd4j.getExecutioner().commit(); + + if (log.isDebugEnabled()) { + log.debug("Scored {} examples ", totalCount); + } + + return ret.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreExamplesWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreExamplesWithKeyFunction.java new file mode 100644 index 000000000..e0a3eba0a --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreExamplesWithKeyFunction.java @@ -0,0 +1,117 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.spark.api.java.function.PairFlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +@Slf4j +public class ScoreExamplesWithKeyFunction implements PairFlatMapFunction>, K, Double> { + + private final Broadcast params; + private final Broadcast jsonConfig; + private final boolean addRegularization; + private final int batchSize; + + /** + * @param params ComputationGraph parameters + * @param jsonConfig ComputationGraphConfiguration, as json + * @param addRegularizationTerms if true: add regularization terms (l1/l2) if applicable; false: don't add regularization terms + * @param batchSize Batch size to use when scoring examples + */ + public ScoreExamplesWithKeyFunction(Broadcast params, Broadcast jsonConfig, + boolean addRegularizationTerms, int batchSize) { + this.params = params; + this.jsonConfig = jsonConfig; + this.addRegularization = addRegularizationTerms; + this.batchSize = batchSize; + } + + + @Override + public Iterator> call(Iterator> iterator) throws Exception { + if (!iterator.hasNext()) { + return Collections.emptyIterator(); + } + + ComputationGraph network = new ComputationGraph(ComputationGraphConfiguration.fromJson(jsonConfig.getValue())); + network.init(); + INDArray val = params.value().unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParams(val); + + List> ret = new ArrayList<>(); + + List collect = new ArrayList<>(batchSize); + List collectKey = new ArrayList<>(batchSize); + int totalCount = 0; + while (iterator.hasNext()) { + collect.clear(); + collectKey.clear(); + int nExamples = 0; + while (iterator.hasNext() && nExamples < batchSize) { + Tuple2 t2 = iterator.next(); + MultiDataSet ds = t2._2(); + val n = ds.getFeatures(0).size(0); + if (n != 1) + throw new IllegalStateException("Cannot score examples with one key per data set if " + + "data set contains more than 1 example (numExamples: " + n + ")"); + collect.add(ds); + collectKey.add(t2._1()); + nExamples += n; + } + totalCount += nExamples; + + MultiDataSet data = org.nd4j.linalg.dataset.MultiDataSet.merge(collect); + + + INDArray scores = network.scoreExamples(data, addRegularization); + double[] doubleScores = scores.data().asDouble(); + + for (int i = 0; i < doubleScores.length; i++) { + ret.add(new Tuple2<>(collectKey.get(i), doubleScores[i])); + } + } + + Nd4j.getExecutioner().commit(); + + if (log.isDebugEnabled()) { + log.debug("Scored {} examples ", totalCount); + } + + return ret.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreFlatMapFunctionCGDataSet.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreFlatMapFunctionCGDataSet.java new file mode 100644 index 000000000..7acae9d8f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreFlatMapFunctionCGDataSet.java @@ -0,0 +1,84 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.datasets.iterator.IteratorDataSetIterator; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; +import lombok.val; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +public class ScoreFlatMapFunctionCGDataSet implements FlatMapFunction, Tuple2> { + private static final Logger log = LoggerFactory.getLogger(ScoreFlatMapFunctionCGDataSet.class); + private String json; + private Broadcast params; + private int minibatchSize; + + + public ScoreFlatMapFunctionCGDataSet(String json, Broadcast params, int minibatchSize) { + this.json = json; + this.params = params; + this.minibatchSize = minibatchSize; + } + + @Override + public Iterator> call(Iterator dataSetIterator) throws Exception { + if (!dataSetIterator.hasNext()) { + return Collections.singletonList(new Tuple2<>(0L, 0.0)).iterator(); + } + + DataSetIterator iter = new IteratorDataSetIterator(dataSetIterator, minibatchSize); //Does batching where appropriate + + ComputationGraph network = new ComputationGraph(ComputationGraphConfiguration.fromJson(json)); + network.init(); + INDArray val = params.value().unsafeDuplication(); //.value() is shared by all executors on single machine -> OK, as params are not changed in score function + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParams(val); + + List> out = new ArrayList<>(); + while (iter.hasNext()) { + DataSet ds = iter.next(); + double score = network.score(ds, false); + + long numExamples = ds.getFeatures().size(0); + out.add(new Tuple2<>(numExamples, score * numExamples)); + } + + Nd4j.getExecutioner().commit(); + + return out.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreFlatMapFunctionCGMultiDataSet.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreFlatMapFunctionCGMultiDataSet.java new file mode 100644 index 000000000..60ba08857 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/graph/scoring/ScoreFlatMapFunctionCGMultiDataSet.java @@ -0,0 +1,85 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph.scoring; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.datasets.iterator.IteratorMultiDataSetIterator; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +public class ScoreFlatMapFunctionCGMultiDataSet implements FlatMapFunction, Tuple2> { + + private static final Logger log = LoggerFactory.getLogger(ScoreFlatMapFunctionCGMultiDataSet.class); + private String json; + private Broadcast params; + private int minibatchSize; + + + public ScoreFlatMapFunctionCGMultiDataSet(String json, Broadcast params, int minibatchSize) { + this.json = json; + this.params = params; + this.minibatchSize = minibatchSize; + } + + @Override + public Iterator> call(Iterator dataSetIterator) throws Exception { + if (!dataSetIterator.hasNext()) { + return Collections.singletonList(new Tuple2<>(0L, 0.0)).iterator(); + } + + MultiDataSetIterator iter = new IteratorMultiDataSetIterator(dataSetIterator, minibatchSize); //Does batching where appropriate + + + ComputationGraph network = new ComputationGraph(ComputationGraphConfiguration.fromJson(json)); + network.init(); + INDArray val = params.value().unsafeDuplication(); //.value() is shared by all executors on single machine -> OK, as params are not changed in score function + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParams(val); + + List> out = new ArrayList<>(); + while (iter.hasNext()) { + MultiDataSet ds = iter.next(); + double score = network.score(ds, false); + + long numExamples = ds.getFeatures(0).size(0); + out.add(new Tuple2<>(numExamples, score * numExamples)); + } + + Nd4j.getExecutioner().commit(); + + return out.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/listeners/VanillaStatsStorageRouter.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/listeners/VanillaStatsStorageRouter.java new file mode 100644 index 000000000..a1f7c7f17 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/listeners/VanillaStatsStorageRouter.java @@ -0,0 +1,89 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.listeners; + +import lombok.Data; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.StorageMetaData; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +@Data +public class VanillaStatsStorageRouter implements StatsStorageRouter { + + private final List storageMetaData = + Collections.synchronizedList(new ArrayList()); + private final List staticInfo = Collections.synchronizedList(new ArrayList()); + private final List updates = Collections.synchronizedList(new ArrayList()); + + @Override + public void putStorageMetaData(StorageMetaData storageMetaData) { + this.storageMetaData.add(storageMetaData); + } + + @Override + public void putStorageMetaData(Collection storageMetaData) { + this.storageMetaData.addAll(storageMetaData); + } + + @Override + public void putStaticInfo(Persistable staticInfo) { + this.staticInfo.add(staticInfo); + } + + @Override + public void putStaticInfo(Collection staticInfo) { + this.staticInfo.addAll(staticInfo); + } + + @Override + public void putUpdate(Persistable update) { + this.updates.add(update); + } + + @Override + public void putUpdate(Collection updates) { + this.updates.addAll(updates); + } + + + public List getStorageMetaData() { + //We can't return synchronized lists list this for Kryo: with default config, it will fail to deserialize the + // synchronized lists, throwing an obscure null pointer exception + return new ArrayList<>(storageMetaData); + } + + public List getStaticInfo() { + //We can't return synchronized lists list this for Kryo: with default config, it will fail to deserialize the + // synchronized lists, throwing an obscure null pointer exception + return new ArrayList<>(staticInfo); + } + + public List getUpdates() { + //We can't return synchronized lists list this for Kryo: with default config, it will fail to deserialize the + // synchronized lists, throwing an obscure null pointer exception + return new ArrayList<>(updates); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/listeners/VanillaStatsStorageRouterProvider.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/listeners/VanillaStatsStorageRouterProvider.java new file mode 100644 index 000000000..1f0ab8184 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/listeners/VanillaStatsStorageRouterProvider.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.listeners; + +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.StatsStorageRouterProvider; + +public class VanillaStatsStorageRouterProvider implements StatsStorageRouterProvider { + + private StatsStorageRouter router = null; + + @Override + public synchronized StatsStorageRouter getRouter() { + if (router == null) + router = new VanillaStatsStorageRouter(); + return router; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/SparkDl4jMultiLayer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/SparkDl4jMultiLayer.java new file mode 100644 index 000000000..be7780f2f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/SparkDl4jMultiLayer.java @@ -0,0 +1,780 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer; + +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.conf.Configuration; +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaDoubleRDD; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.mllib.linalg.Matrix; +import org.apache.spark.mllib.linalg.Vector; +import org.apache.spark.mllib.regression.LabeledPoint; +import org.apache.spark.rdd.RDD; +import org.datavec.spark.util.BroadcastHadoopConfigHolder; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.core.loader.impl.SerializedDataSetLoader; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.data.loader.RemoteFileSourceFactory; +import org.deeplearning4j.spark.impl.SparkListenable; +import org.deeplearning4j.spark.impl.common.LoadDataSetFunction; +import org.deeplearning4j.spark.impl.common.reduce.IntDoubleReduceFunction; +import org.deeplearning4j.spark.impl.graph.evaluation.IEvaluateMDSPathsFlatMapFunction; +import org.deeplearning4j.spark.impl.multilayer.evaluation.IEvaluateAggregateFunction; +import org.deeplearning4j.spark.impl.multilayer.evaluation.IEvaluateFlatMapFunction; +import org.deeplearning4j.spark.impl.multilayer.evaluation.IEvaluationReduceFunction; +import org.deeplearning4j.spark.impl.multilayer.scoring.*; +import org.deeplearning4j.spark.util.MLLibUtil; +import org.deeplearning4j.spark.util.SparkUtils; +import org.deeplearning4j.util.ModelSerializer; +import org.nd4j.common.base.Preconditions; +import org.nd4j.evaluation.IEvaluation; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.evaluation.classification.ROC; +import org.nd4j.evaluation.classification.ROCMultiClass; +import org.nd4j.evaluation.regression.RegressionEvaluation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.executioner.GridExecutioner; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.heartbeat.Heartbeat; +import org.nd4j.linalg.heartbeat.reports.Environment; +import org.nd4j.linalg.heartbeat.reports.Event; +import org.nd4j.linalg.heartbeat.reports.Task; +import org.nd4j.linalg.heartbeat.utils.EnvironmentUtils; +import scala.Tuple2; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +@Slf4j +public class SparkDl4jMultiLayer extends SparkListenable { + public static final int DEFAULT_EVAL_SCORE_BATCH_SIZE = 64; + public static final int DEFAULT_ROC_THRESHOLD_STEPS = 32; + public static final int DEFAULT_EVAL_WORKERS = 4; + private transient JavaSparkContext sc; + private MultiLayerConfiguration conf; + private MultiLayerNetwork network; + private double lastScore; + private int defaultEvaluationWorkers = DEFAULT_EVAL_WORKERS; + + /** + * Instantiate a multi layer spark instance + * with the given context and network. + * This is the prediction constructor + * + * @param sparkContext the spark context to use + * @param network the network to use + */ + public SparkDl4jMultiLayer(SparkContext sparkContext, MultiLayerNetwork network, + TrainingMaster trainingMaster) { + this(new JavaSparkContext(sparkContext), network, trainingMaster); + } + + /** + * Training constructor. Instantiate with a configuration + * + * @param sparkContext the spark context to use + * @param conf the configuration of the network + */ + public SparkDl4jMultiLayer(SparkContext sparkContext, MultiLayerConfiguration conf, + TrainingMaster trainingMaster) { + this(new JavaSparkContext(sparkContext), initNetwork(conf), trainingMaster); + } + + /** + * Training constructor. Instantiate with a configuration + * + * @param sc the spark context to use + * @param conf the configuration of the network + */ + public SparkDl4jMultiLayer(JavaSparkContext sc, MultiLayerConfiguration conf, TrainingMaster trainingMaster) { + this(sc.sc(), conf, trainingMaster); + } + + public SparkDl4jMultiLayer(JavaSparkContext javaSparkContext, MultiLayerNetwork network, + TrainingMaster trainingMaster) { + sc = javaSparkContext; + this.conf = network.getLayerWiseConfigurations().clone(); + this.network = network; + if (!network.isInitCalled()) + network.init(); + this.trainingMaster = trainingMaster; + + //Check if kryo configuration is correct: + SparkUtils.checkKryoConfiguration(javaSparkContext, log); + } + + private static MultiLayerNetwork initNetwork(MultiLayerConfiguration conf) { + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + return net; + } + + public JavaSparkContext getSparkContext() { + return sc; + } + + /** + * @return The MultiLayerNetwork underlying the SparkDl4jMultiLayer + */ + public MultiLayerNetwork getNetwork() { + return network; + } + + /** + * @return The TrainingMaster for this network + */ + public TrainingMaster getTrainingMaster() { + return trainingMaster; + } + + /** + * Set the network that underlies this SparkDl4jMultiLayer instacne + * + * @param network network to set + */ + public void setNetwork(MultiLayerNetwork network) { + this.network = network; + } + + + /** + * Returns the currently set default number of evaluation workers/threads. + * Note that when the number of workers is provided explicitly in an evaluation method, the default value + * is not used.
+ * In many cases, we may want this to be smaller than the number of Spark threads, to reduce memory requirements. + * For example, with 32 Spark threads and a large network, we don't want to spin up 32 instances of the network + * to perform evaluation. Better (for memory requirements, and reduced cache thrashing) to use say 4 workers.
+ * If it is not set explicitly, {@link #DEFAULT_EVAL_WORKERS} will be used + * + * @return Default number of evaluation workers (threads). + */ + public int getDefaultEvaluationWorkers(){ + return defaultEvaluationWorkers; + } + + /** + * Set the default number of evaluation workers/threads. + * Note that when the number of workers is provided explicitly in an evaluation method, the default value + * is not used.
+ * In many cases, we may want this to be smaller than the number of Spark threads, to reduce memory requirements. + * For example, with 32 Spark threads and a large network, we don't want to spin up 32 instances of the network + * to perform evaluation. Better (for memory requirements, and reduced cache thrashing) to use say 4 workers.
+ * If it is not set explicitly, {@link #DEFAULT_EVAL_WORKERS} will be used + * + * @return Default number of evaluation workers (threads). + */ + public void setDefaultEvaluationWorkers(int workers){ + Preconditions.checkArgument(workers > 0, "Number of workers must be > 0: got %s", workers); + this.defaultEvaluationWorkers = workers; + } + + /** + * Set whether training statistics should be collected for debugging purposes. Statistics collection is disabled by default + * + * @param collectTrainingStats If true: collect training statistics. If false: don't collect. + */ + public void setCollectTrainingStats(boolean collectTrainingStats) { + trainingMaster.setCollectTrainingStats(collectTrainingStats); + } + + /** + * Get the training statistics, after collection of stats has been enabled using {@link #setCollectTrainingStats(boolean)} + * + * @return Training statistics + */ + public SparkTrainingStats getSparkTrainingStats() { + return trainingMaster.getTrainingStats(); + } + + /** + * Predict the given feature matrix + * + * @param features the given feature matrix + * @return the predictions + */ + public Matrix predict(Matrix features) { + return MLLibUtil.toMatrix(network.output(MLLibUtil.toMatrix(features))); + } + + + /** + * Predict the given vector + * + * @param point the vector to predict + * @return the predicted vector + */ + public Vector predict(Vector point) { + return MLLibUtil.toVector(network.output(MLLibUtil.toVector(point))); + } + + /** + * Fit the DataSet RDD. Equivalent to fit(trainingData.toJavaRDD()) + * + * @param trainingData the training data RDD to fitDataSet + * @return the MultiLayerNetwork after training + */ + public MultiLayerNetwork fit(RDD trainingData) { + return fit(trainingData.toJavaRDD()); + } + + /** + * Fit the DataSet RDD + * + * @param trainingData the training data RDD to fitDataSet + * @return the MultiLayerNetwork after training + */ + public MultiLayerNetwork fit(JavaRDD trainingData) { + if (Nd4j.getExecutioner() instanceof GridExecutioner) + ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); + + trainingMaster.executeTraining(this, trainingData); + network.incrementEpochCount(); + return network; + } + + /** + * Fit the SparkDl4jMultiLayer network using a directory of serialized DataSet objects + * The assumption here is that the directory contains a number of {@link DataSet} objects, each serialized using + * {@link DataSet#save(OutputStream)} + * + * @param path Path to the directory containing the serialized DataSet objcets + * @return The MultiLayerNetwork after training + */ + public MultiLayerNetwork fit(String path) { + if (Nd4j.getExecutioner() instanceof GridExecutioner) + ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); + + JavaRDD paths; + try { + paths = SparkUtils.listPaths(sc, path); + } catch (IOException e) { + throw new RuntimeException("Error listing paths in directory", e); + } + + return fitPaths(paths); + } + + /** + * @deprecated Use {@link #fit(String)} + */ + @Deprecated + public MultiLayerNetwork fit(String path, int minPartitions) { + return fit(path); + } + + /** + * Fit the network using a list of paths for serialized DataSet objects. + * + * @param paths List of paths + * @return trained network + */ + public MultiLayerNetwork fitPaths(JavaRDD paths) { + return fitPaths(paths, new SerializedDataSetLoader()); + } + + public MultiLayerNetwork fitPaths(JavaRDD paths, DataSetLoader loader) { + trainingMaster.executeTrainingPaths(this, null, paths, loader, null); + network.incrementEpochCount(); + return network; + } + + /** + * Fit a MultiLayerNetwork using Spark MLLib LabeledPoint instances. + * This will convert the labeled points to the internal DL4J data format and train the model on that + * + * @param rdd the rdd to fitDataSet + * @return the multi layer network that was fitDataSet + */ + public MultiLayerNetwork fitLabeledPoint(JavaRDD rdd) { + int nLayers = network.getLayerWiseConfigurations().getConfs().size(); + FeedForwardLayer ffl = (FeedForwardLayer) network.getLayerWiseConfigurations().getConf(nLayers - 1).getLayer(); + JavaRDD ds = MLLibUtil.fromLabeledPoint(sc, rdd, ffl.getNOut()); + return fit(ds); + } + + /** + * Fits a MultiLayerNetwork using Spark MLLib LabeledPoint instances + * This will convert labeled points that have continuous labels used for regression to the internal + * DL4J data format and train the model on that + * @param rdd the javaRDD containing the labeled points + * @return a MultiLayerNetwork + */ + public MultiLayerNetwork fitContinuousLabeledPoint(JavaRDD rdd) { + return fit(MLLibUtil.fromContinuousLabeledPoint(sc, rdd)); + } + + /** + * Gets the last (average) minibatch score from calling fit. This is the average score across all executors for the + * last minibatch executed in each worker + */ + public double getScore() { + return lastScore; + } + + public void setScore(double lastScore) { + this.lastScore = lastScore; + } + + /** + * Overload of {@link #calculateScore(JavaRDD, boolean)} for {@code RDD} instead of {@code JavaRDD} + */ + public double calculateScore(RDD data, boolean average) { + return calculateScore(data.toJavaRDD(), average); + } + + /** + * Calculate the score for all examples in the provided {@code JavaRDD}, either by summing + * or averaging over the entire data set. To calculate a score for each example individually, use {@link #scoreExamples(JavaPairRDD, boolean)} + * or one of the similar methods. Uses default minibatch size in each worker, {@link SparkDl4jMultiLayer#DEFAULT_EVAL_SCORE_BATCH_SIZE} + * + * @param data Data to score + * @param average Whether to sum the scores, or average them + */ + public double calculateScore(JavaRDD data, boolean average) { + return calculateScore(data, average, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Calculate the score for all examples in the provided {@code JavaRDD}, either by summing + * or averaging over the entire data set. To calculate a score for each example individually, use {@link #scoreExamples(JavaPairRDD, boolean)} + * or one of the similar methods + * + * @param data Data to score + * @param average Whether to sum the scores, or average them + * @param minibatchSize The number of examples to use in each minibatch when scoring. If more examples are in a partition than + * this, multiple scoring operations will be done (to avoid using too much memory by doing the whole partition + * in one go) + */ + public double calculateScore(JavaRDD data, boolean average, int minibatchSize) { + JavaRDD> rdd = data.mapPartitions( + new ScoreFlatMapFunction(conf.toJson(), sc.broadcast(network.params(false)), minibatchSize)); + + //Reduce to a single tuple, with example count + sum of scores + Tuple2 countAndSumScores = rdd.reduce(new IntDoubleReduceFunction()); + if (average) { + return countAndSumScores._2() / countAndSumScores._1(); + } else { + return countAndSumScores._2(); + } + } + + /** + * {@code RDD} overload of {@link #scoreExamples(JavaPairRDD, boolean)} + */ + public JavaDoubleRDD scoreExamples(RDD data, boolean includeRegularizationTerms) { + return scoreExamples(data.toJavaRDD(), includeRegularizationTerms); + } + + /** + * Score the examples individually, using the default batch size {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE}. Unlike {@link #calculateScore(JavaRDD, boolean)}, + * this method returns a score for each example separately. If scoring is needed for specific examples use either + * {@link #scoreExamples(JavaPairRDD, boolean)} or {@link #scoreExamples(JavaPairRDD, boolean, int)} which can have + * a key for each example. + * + * @param data Data to score + * @param includeRegularizationTerms If true: include the l1/l2 regularization terms with the score (if any) + * @return A JavaDoubleRDD containing the scores of each example + * @see MultiLayerNetwork#scoreExamples(DataSet, boolean) + */ + public JavaDoubleRDD scoreExamples(JavaRDD data, boolean includeRegularizationTerms) { + return scoreExamples(data, includeRegularizationTerms, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * {@code RDD} + * overload of {@link #scoreExamples(JavaRDD, boolean, int)} + */ + public JavaDoubleRDD scoreExamples(RDD data, boolean includeRegularizationTerms, int batchSize) { + return scoreExamples(data.toJavaRDD(), includeRegularizationTerms, batchSize); + } + + /** + * Score the examples individually, using a specified batch size. Unlike {@link #calculateScore(JavaRDD, boolean)}, + * this method returns a score for each example separately. If scoring is needed for specific examples use either + * {@link #scoreExamples(JavaPairRDD, boolean)} or {@link #scoreExamples(JavaPairRDD, boolean, int)} which can have + * a key for each example. + * + * @param data Data to score + * @param includeRegularizationTerms If true: include the l1/l2 regularization terms with the score (if any) + * @param batchSize Batch size to use when doing scoring + * @return A JavaDoubleRDD containing the scores of each example + * @see MultiLayerNetwork#scoreExamples(DataSet, boolean) + */ + public JavaDoubleRDD scoreExamples(JavaRDD data, boolean includeRegularizationTerms, int batchSize) { + return data.mapPartitionsToDouble(new ScoreExamplesFunction(sc.broadcast(network.params()), + sc.broadcast(conf.toJson()), includeRegularizationTerms, batchSize)); + } + + /** + * Score the examples individually, using the default batch size {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE}. Unlike {@link #calculateScore(JavaRDD, boolean)}, + * this method returns a score for each example separately
+ * Note: The provided JavaPairRDD has a key that is associated with each example and returned score.
+ * Note: The DataSet objects passed in must have exactly one example in them (otherwise: can't have a 1:1 association + * between keys and data sets to score) + * + * @param data Data to score + * @param includeRegularizationTerms If true: include the l1/l2 regularization terms with the score (if any) + * @param Key type + * @return A {@code JavaPairRDD} containing the scores of each example + * @see MultiLayerNetwork#scoreExamples(DataSet, boolean) + */ + public JavaPairRDD scoreExamples(JavaPairRDD data, boolean includeRegularizationTerms) { + return scoreExamples(data, includeRegularizationTerms, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Score the examples individually, using a specified batch size. Unlike {@link #calculateScore(JavaRDD, boolean)}, + * this method returns a score for each example separately
+ * Note: The provided JavaPairRDD has a key that is associated with each example and returned score.
+ * Note: The DataSet objects passed in must have exactly one example in them (otherwise: can't have a 1:1 association + * between keys and data sets to score) + * + * @param data Data to score + * @param includeRegularizationTerms If true: include the l1/l2 regularization terms with the score (if any) + * @param Key type + * @return A {@code JavaPairRDD} containing the scores of each example + * @see MultiLayerNetwork#scoreExamples(DataSet, boolean) + */ + public JavaPairRDD scoreExamples(JavaPairRDD data, boolean includeRegularizationTerms, + int batchSize) { + return data.mapPartitionsToPair(new ScoreExamplesWithKeyFunction(sc.broadcast(network.params()), + sc.broadcast(conf.toJson()), includeRegularizationTerms, batchSize)); + } + + /** + * Feed-forward the specified data, with the given keys. i.e., get the network output/predictions for the specified data + * + * @param featuresData Features data to feed through the network + * @param batchSize Batch size to use when doing feed forward operations + * @param Type of data for key - may be anything + * @return Network output given the input, by key + */ + public JavaPairRDD feedForwardWithKey(JavaPairRDD featuresData, int batchSize) { + return feedForwardWithMaskAndKey(featuresData.mapToPair(new SingleToPairFunction()), batchSize); + } + + /** + * Feed-forward the specified data (and optionally mask array), with the given keys. i.e., get the network + * output/predictions for the specified data + * + * @param featuresDataAndMask Features data to feed through the network. The Tuple2 is of the network input (features), + * and optionally the feature mask arrays + * @param batchSize Batch size to use when doing feed forward operations + * @param Type of data for key - may be anything + * @return Network output given the input (and optionally mask), by key + */ + public JavaPairRDD feedForwardWithMaskAndKey(JavaPairRDD> featuresDataAndMask, int batchSize) { + return featuresDataAndMask + .mapPartitionsToPair(new FeedForwardWithKeyFunction(sc.broadcast(network.params()), + sc.broadcast(conf.toJson()), batchSize)); + } + + /** + * {@code RDD} overload of {@link #evaluate(JavaRDD)} + */ + public T evaluate(RDD data) { + return evaluate(data.toJavaRDD()); + } + + /** + * Evaluate on a directory containing a set of DataSet objects serialized with {@link DataSet#save(OutputStream)} + * @param path Path/URI to the directory containing the dataset objects + * @return Evaluation + */ + public T evaluate(String path){ + return evaluate(path, new SerializedDataSetLoader()); + } + + /** + * Evaluate on a directory containing a set of DataSet objects to be loaded with a {@link DataSetLoader}. + * Uses default batch size of {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE} + * @param path Path/URI to the directory containing the datasets to load + * @return Evaluation + */ + public T evaluate(String path, DataSetLoader loader) { + return evaluate(path, DEFAULT_EVAL_SCORE_BATCH_SIZE, loader); + } + + /** + * Evaluate on a directory containing a set of DataSet objects to be loaded with a {@link DataSetLoader}. + * Uses default batch size of {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE} + * @param path Path/URI to the directory containing the datasets to load + * @return Evaluation + */ + public T evaluate(String path, int batchSize, DataSetLoader loader){ + JavaRDD paths; + try { + paths = SparkUtils.listPaths(sc, path); + } catch (IOException e) { + throw new RuntimeException("Error listing paths in directory", e); + } + + JavaRDD rdd = paths.map(new LoadDataSetFunction(loader, new RemoteFileSourceFactory(BroadcastHadoopConfigHolder.get(sc)))); + return (T)doEvaluation(rdd, batchSize, new org.deeplearning4j.eval.Evaluation())[0]; + } + + /** + * Evaluate the network (classification performance) in a distributed manner on the provided data + * + * @param data Data to evaluate on + * @return Evaluation object; results of evaluation on all examples in the data set + */ + public T evaluate(JavaRDD data) { + return evaluate(data, null); + } + + /** + * {@code RDD} overload of {@link #evaluate(JavaRDD, List)} + */ + public T evaluate(RDD data, List labelsList) { + return evaluate(data.toJavaRDD(), labelsList); + } + + /** + * Evaluate the network (regression performance) in a distributed manner on the provided data + * + * @param data Data to evaluate + * @return {@link RegressionEvaluation} instance with regression performance + */ + public T evaluateRegression(JavaRDD data) { + return evaluateRegression(data, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Evaluate the network (regression performance) in a distributed manner on the provided data + * + * @param data Data to evaluate + * @param minibatchSize Minibatch size to use when doing performing evaluation + * @return {@link RegressionEvaluation} instance with regression performance + */ + public T evaluateRegression(JavaRDD data, int minibatchSize) { + long nOut = ((FeedForwardLayer) network.getOutputLayer().conf().getLayer()).getNOut(); + return (T)doEvaluation(data, new org.deeplearning4j.eval.RegressionEvaluation(nOut), minibatchSize); + } + + /** + * Evaluate the network (classification performance) in a distributed manner, using default batch size and a provided + * list of labels + * + * @param data Data to evaluate on + * @param labelsList List of labels used for evaluation + * @return Evaluation object; results of evaluation on all examples in the data set + */ + public T evaluate(JavaRDD data, List labelsList) { + return evaluate(data, labelsList, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Perform ROC analysis/evaluation on the given DataSet in a distributed manner, using the default number of + * threshold steps ({@link #DEFAULT_ROC_THRESHOLD_STEPS}) and the default minibatch size ({@link #DEFAULT_EVAL_SCORE_BATCH_SIZE}) + * + * @param data Test set data (to evaluate on) + * @return ROC for the entire data set + */ + public T evaluateROC(JavaRDD data) { + return evaluateROC(data, DEFAULT_ROC_THRESHOLD_STEPS, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Perform ROC analysis/evaluation on the given DataSet in a distributed manner + * + * @param data Test set data (to evaluate on) + * @param thresholdSteps Number of threshold steps for ROC - see {@link ROC} + * @param evaluationMinibatchSize Minibatch size to use when performing ROC evaluation + * @return ROC for the entire data set + */ + public T evaluateROC(JavaRDD data, int thresholdSteps, int evaluationMinibatchSize) { + return (T)doEvaluation(data, new org.deeplearning4j.eval.ROC(thresholdSteps), evaluationMinibatchSize); + } + + /** + * Perform ROC analysis/evaluation (for the multi-class case, using {@link ROCMultiClass} on the given DataSet in a distributed manner + * + * @param data Test set data (to evaluate on) + * @return ROC for the entire data set + */ + public T evaluateROCMultiClass(JavaRDD data) { + return evaluateROCMultiClass(data, DEFAULT_ROC_THRESHOLD_STEPS, DEFAULT_EVAL_SCORE_BATCH_SIZE); + } + + /** + * Perform ROC analysis/evaluation (for the multi-class case, using {@link ROCMultiClass} on the given DataSet in a distributed manner + * + * @param data Test set data (to evaluate on) + * @param thresholdSteps Number of threshold steps for ROC - see {@link ROC} + * @param evaluationMinibatchSize Minibatch size to use when performing ROC evaluation + * @return ROCMultiClass for the entire data set + */ + public T evaluateROCMultiClass(JavaRDD data, int thresholdSteps, int evaluationMinibatchSize) { + return (T)doEvaluation(data, new org.deeplearning4j.eval.ROCMultiClass(thresholdSteps), evaluationMinibatchSize); + } + + private void update(int mr, long mg) { + Environment env = EnvironmentUtils.buildEnvironment(); + env.setNumCores(mr); + env.setAvailableMemory(mg); + Task task = ModelSerializer.taskByModel(network); + Heartbeat.getInstance().reportEvent(Event.SPARK, env, task); + } + + /** + * Evaluate the network (classification performance) in a distributed manner, using specified batch size and a provided + * list of labels + * + * @param data Data to evaluate on + * @param labelsList List of labels used for evaluation + * @param evalBatchSize Batch size to use when conducting evaluations + * @return Evaluation object; results of evaluation on all examples in the data set + */ + public T evaluate(JavaRDD data, List labelsList, int evalBatchSize) { + Evaluation e = new org.deeplearning4j.eval.Evaluation(); + e = doEvaluation(data, e, evalBatchSize); + if (labelsList != null) { + e.setLabelsList(labelsList); + } + return (T)e; + } + + /** + * Perform distributed evaluation of any type of {@link IEvaluation}. For example, {@link Evaluation}, {@link RegressionEvaluation}, + * {@link ROC}, {@link ROCMultiClass} etc. + * + * @param data Data to evaluate on + * @param emptyEvaluation Empty evaluation instance. This is the starting point (serialized/duplicated, then merged) + * @param evalBatchSize Evaluation batch size + * @param Type of evaluation instance to return + * @return IEvaluation instance + */ + @SuppressWarnings("unchecked") + public T doEvaluation(JavaRDD data, T emptyEvaluation, int evalBatchSize) { + return doEvaluation(data, evalBatchSize, emptyEvaluation)[0]; + } + + /** + * Perform distributed evaluation of any type of {@link IEvaluation} - or multiple IEvaluation instances. + * Distributed equivalent of {@link MultiLayerNetwork#doEvaluation(DataSetIterator, IEvaluation[])} + * + * @param data Data to evaluate on + * @param emptyEvaluations Empty evaluation instances. Starting point (serialized/duplicated, then merged) + * @param evalBatchSize Evaluation batch size + * @param Type of evaluation instance to return + * @return IEvaluation instances + */ + @SuppressWarnings("unchecked") + public T[] doEvaluation(JavaRDD data, int evalBatchSize, T... emptyEvaluations) { + return doEvaluation(data, getDefaultEvaluationWorkers(), evalBatchSize, emptyEvaluations ); + } + /** + * Perform distributed evaluation of any type of {@link IEvaluation} - or multiple IEvaluation instances. + * Distributed equivalent of {@link MultiLayerNetwork#doEvaluation(DataSetIterator, IEvaluation[])} + * + * @param data Data to evaluate on + * @param emptyEvaluations Empty evaluation instances. Starting point (serialized/duplicated, then merged) + * @param evalNumWorkers Number of workers (copies of the MultiLayerNetwork) model to use. Generally this should + * be smaller than the number of threads - 2 to 4 is often good enough. If using CUDA GPUs, + * this should ideally be set to the number of GPUs on each node (i.e., 1 for a single GPU node) + * @param evalBatchSize Evaluation batch size + * @param Type of evaluation instance to return + * @return IEvaluation instances + */ + public T[] doEvaluation(JavaRDD data, int evalNumWorkers, int evalBatchSize, T... emptyEvaluations) { + IEvaluateFlatMapFunction evalFn = new IEvaluateFlatMapFunction<>(false, sc.broadcast(conf.toJson()), + SparkUtils.asByteArrayBroadcast(sc, network.params()), evalNumWorkers, evalBatchSize, emptyEvaluations); + JavaRDD evaluations = data.mapPartitions(evalFn); + return evaluations.treeAggregate(null, new IEvaluateAggregateFunction(), new IEvaluationReduceFunction()); + } + + + /** + * Perform evaluation on serialized DataSet objects on disk, (potentially in any format), that are loaded using an {@link DataSetLoader}.
+ * Uses the default number of workers (model replicas per JVM) of {@link #DEFAULT_EVAL_WORKERS} with the default + * minibatch size of {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE} + * @param data List of paths to the data (that can be loaded as / converted to DataSets) + * @param loader Used to load DataSets from their paths + * @param emptyEvaluations Evaluations to perform + * @return Evaluation + */ + public IEvaluation[] doEvaluation(JavaRDD data, DataSetLoader loader, IEvaluation... emptyEvaluations) { + return doEvaluation(data, DEFAULT_EVAL_WORKERS, DEFAULT_EVAL_SCORE_BATCH_SIZE, loader, emptyEvaluations); + } + + /** + * Perform evaluation on serialized DataSet objects on disk, (potentially in any format), that are loaded using an {@link DataSetLoader}. + * @param data List of paths to the data (that can be loaded as / converted to DataSets) + * @param evalNumWorkers Number of workers to perform evaluation with. To reduce memory requirements and cache thrashing, + * it is common to set this to a lower value than the number of spark threads per JVM/executor + * @param evalBatchSize Batch size to use when performing evaluation + * @param loader Used to load DataSets from their paths + * @param emptyEvaluations Evaluations to perform + * @return Evaluation + */ + public IEvaluation[] doEvaluation(JavaRDD data, int evalNumWorkers, int evalBatchSize, DataSetLoader loader, IEvaluation... emptyEvaluations) { + return doEvaluation(data, evalNumWorkers, evalBatchSize, loader, null, emptyEvaluations); + } + + /** + * Perform evaluation on serialized MultiDataSet objects on disk, (potentially in any format), that are loaded using an {@link MultiDataSetLoader}.
+ * Uses the default number of workers (model replicas per JVM) of {@link #DEFAULT_EVAL_WORKERS} with the default + * minibatch size of {@link #DEFAULT_EVAL_SCORE_BATCH_SIZE} + * @param data List of paths to the data (that can be loaded as / converted to DataSets) + * @param loader Used to load MultiDataSets from their paths + * @param emptyEvaluations Evaluations to perform + * @return Evaluation + */ + public IEvaluation[] doEvaluation(JavaRDD data, MultiDataSetLoader loader, IEvaluation... emptyEvaluations) { + return doEvaluation(data, DEFAULT_EVAL_WORKERS, DEFAULT_EVAL_SCORE_BATCH_SIZE, null, loader, emptyEvaluations); + } + + /** + * Perform evaluation on serialized MultiDataSet objects on disk, (potentially in any format), that are loaded using an {@link MultiDataSetLoader} + * @param data List of paths to the data (that can be loaded as / converted to DataSets) + * @param evalNumWorkers Number of workers to perform evaluation with. To reduce memory requirements and cache thrashing, + * it is common to set this to a lower value than the number of spark threads per JVM/executor + * @param evalBatchSize Batch size to use when performing evaluation + * @param loader Used to load MultiDataSets from their paths + * @param emptyEvaluations Evaluations to perform + * @return Evaluation + */ + public IEvaluation[] doEvaluation(JavaRDD data, int evalNumWorkers, int evalBatchSize, MultiDataSetLoader loader, IEvaluation... emptyEvaluations) { + return doEvaluation(data, evalNumWorkers, evalBatchSize, null, loader, emptyEvaluations); + } + + protected IEvaluation[] doEvaluation(JavaRDD data, int evalNumWorkers, int evalBatchSize, DataSetLoader loader, MultiDataSetLoader mdsLoader, IEvaluation... emptyEvaluations){ + Configuration config = sc.hadoopConfiguration(); + IEvaluateMDSPathsFlatMapFunction evalFn = new IEvaluateMDSPathsFlatMapFunction(sc.broadcast(conf.toJson()), + SparkUtils.asByteArrayBroadcast(sc, network.params()), evalNumWorkers, evalBatchSize, loader, mdsLoader, + BroadcastHadoopConfigHolder.get(sc), emptyEvaluations); + Preconditions.checkArgument(evalNumWorkers > 0, "Invalid number of evaulation workers: require at least 1 - got %s", evalNumWorkers); + JavaRDD evaluations = data.mapPartitions(evalFn); + return evaluations.treeAggregate(null, new IEvaluateAggregateFunction<>(), new IEvaluateAggregateFunction<>()); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluateAggregateFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluateAggregateFunction.java new file mode 100644 index 000000000..4e5f7d127 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluateAggregateFunction.java @@ -0,0 +1,38 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.evaluation; + +import org.apache.spark.api.java.function.Function2; +import org.nd4j.evaluation.IEvaluation; + +public class IEvaluateAggregateFunction implements Function2 { + @Override + public T[] call(T[] v1, T[] v2) throws Exception { + if (v1 == null) + return v2; + if (v2 == null) + return v1; + for (int i = 0; i < v1.length; i++) { + v1[i].merge(v2[i]); + } + return v1; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluateFlatMapFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluateFlatMapFunction.java new file mode 100644 index 000000000..3ea14aaa5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluateFlatMapFunction.java @@ -0,0 +1,77 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.evaluation; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.spark.impl.evaluation.EvaluationRunner; +import org.nd4j.evaluation.IEvaluation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; + +import java.util.Collections; +import java.util.Iterator; +import java.util.concurrent.Future; + +public class IEvaluateFlatMapFunction implements FlatMapFunction, T[]> { + + protected boolean isCompGraph; + protected Broadcast json; + protected Broadcast params; + protected int evalNumWorkers; + protected int evalBatchSize; + protected T[] evaluations; + + /** + * @param json Network configuration (json format) + * @param params Network parameters + * @param evalBatchSize Max examples per evaluation. Do multiple separate forward passes if data exceeds + * this. Used to avoid doing too many at once (and hence memory issues) + * @param evaluations Initial evaulation instance (i.e., empty Evaluation or RegressionEvaluation instance) + */ + public IEvaluateFlatMapFunction(boolean isCompGraph, Broadcast json, Broadcast params, + int evalNumWorkers, int evalBatchSize, T[] evaluations) { + this.isCompGraph = isCompGraph; + this.json = json; + this.params = params; + this.evalNumWorkers = evalNumWorkers; + this.evalBatchSize = evalBatchSize; + this.evaluations = evaluations; + } + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + if (!dataSetIterator.hasNext()) { + return Collections.emptyIterator(); + } + + Future f = EvaluationRunner.getInstance().execute( + evaluations, evalNumWorkers, evalBatchSize, dataSetIterator, null, isCompGraph, json, params); + + IEvaluation[] result = f.get(); + if(result == null){ + return Collections.emptyIterator(); + } else { + return Collections.singletonList((T[])result).iterator(); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluationReduceFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluationReduceFunction.java new file mode 100644 index 000000000..44901b20e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/evaluation/IEvaluationReduceFunction.java @@ -0,0 +1,50 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.evaluation; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.function.Function2; +import org.nd4j.evaluation.IEvaluation; + +@Slf4j +public class IEvaluationReduceFunction implements Function2 { + public IEvaluationReduceFunction() {} + + @Override + public T[] call(T[] eval1, T[] eval2) throws Exception { + //Shouldn't *usually* happen... + if(eval1 == null){ + return eval2; + } else if(eval2 == null){ + return eval1; + } + + + for (int i = 0; i < eval1.length; i++) { + if(eval1[i] == null){ + eval1[i] = eval2[i]; + } else if(eval2[i] != null){ + eval1[i].merge(eval2[i]); + } + } + return eval1; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/FeedForwardWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/FeedForwardWithKeyFunction.java new file mode 100644 index 000000000..510f2e4d4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/FeedForwardWithKeyFunction.java @@ -0,0 +1,183 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.scoring; + +import org.apache.spark.api.java.function.PairFlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.DataSetUtil; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.common.primitives.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +public class FeedForwardWithKeyFunction + implements PairFlatMapFunction>>, K, INDArray> { + + protected static Logger log = LoggerFactory.getLogger(FeedForwardWithKeyFunction.class); + + private final Broadcast params; + private final Broadcast jsonConfig; + private final int batchSize; + + /** + * @param params MultiLayerNetwork parameters + * @param jsonConfig MultiLayerConfiguration, as json + * @param batchSize Batch size to use for forward pass (use > 1 for efficiency) + */ + public FeedForwardWithKeyFunction(Broadcast params, Broadcast jsonConfig, int batchSize) { + this.params = params; + this.jsonConfig = jsonConfig; + this.batchSize = batchSize; + } + + + @Override + public Iterator> call(Iterator>> iterator) throws Exception { + if (!iterator.hasNext()) { + return Collections.emptyIterator(); + } + + MultiLayerNetwork network = new MultiLayerNetwork(MultiLayerConfiguration.fromJson(jsonConfig.getValue())); + network.init(); + INDArray val = params.value().unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcasted set parameters"); + network.setParameters(val); + + //Issue: for 2d data (MLPs etc) we can just stack the examples. + //But: for 3d and 4d: in principle the data sizes could be different + //We could handle that with mask arrays - but it gets messy. The approach used here is simpler but less efficient + + List featuresList = new ArrayList<>(batchSize); + List fMaskList = new ArrayList<>(batchSize); + List keyList = new ArrayList<>(batchSize); + List origSizeList = new ArrayList<>(); + + long[] firstShape = null; + boolean sizesDiffer = false; + int tupleCount = 0; + while (iterator.hasNext()) { + Tuple2> t2 = iterator.next(); + + if (firstShape == null) { + firstShape = t2._2()._1().shape(); + } else if (!sizesDiffer) { + for (int i = 1; i < firstShape.length; i++) { + if (firstShape[i] != featuresList.get(tupleCount - 1).size(i)) { + sizesDiffer = true; + break; + } + } + } + featuresList.add(t2._2()._1()); + fMaskList.add(t2._2()._2()); + keyList.add(t2._1()); + + origSizeList.add((int) t2._2()._1().size(0)); + tupleCount++; + } + + if (tupleCount == 0) { + return Collections.emptyIterator(); + } + + List> output = new ArrayList<>(tupleCount); + int currentArrayIndex = 0; + + while (currentArrayIndex < featuresList.size()) { + int firstIdx = currentArrayIndex; + int nextIdx = currentArrayIndex; + int examplesInBatch = 0; + List toMerge = new ArrayList<>(); + List toMergeMask = new ArrayList<>(); + firstShape = null; + while (nextIdx < featuresList.size() && examplesInBatch < batchSize) { + if (firstShape == null) { + firstShape = featuresList.get(nextIdx).shape(); + } else if (sizesDiffer) { + boolean breakWhile = false; + for (int i = 1; i < firstShape.length; i++) { + if (firstShape[i] != featuresList.get(nextIdx).size(i)) { + //Next example has a different size. So: don't add it to the current batch, just process what we have + breakWhile = true; + break; + } + } + if (breakWhile) { + break; + } + } + + INDArray f = featuresList.get(nextIdx); + INDArray fm = fMaskList.get(nextIdx); + nextIdx++; + toMerge.add(f); + toMergeMask.add(fm); + examplesInBatch += f.size(0); + } + + Pair p = DataSetUtil.mergeFeatures(toMerge.toArray(new INDArray[toMerge.size()]), toMergeMask.toArray(new INDArray[toMergeMask.size()])); +// INDArray batchFeatures = Nd4j.concat(0, toMerge.toArray(new INDArray[toMerge.size()])); + INDArray out = network.output(p.getFirst(), false, p.getSecond(), null); + + examplesInBatch = 0; + for (int i = firstIdx; i < nextIdx; i++) { + int numExamples = origSizeList.get(i); + INDArray outputSubset = getSubset(examplesInBatch, examplesInBatch + numExamples, out); + examplesInBatch += numExamples; + + output.add(new Tuple2<>(keyList.get(i), outputSubset)); + } + + currentArrayIndex += (nextIdx - firstIdx); + } + + Nd4j.getExecutioner().commit(); + + return output.iterator(); + } + + private INDArray getSubset(int exampleStart, int exampleEnd, INDArray from) { + switch (from.rank()) { + case 2: + return from.get(NDArrayIndex.interval(exampleStart, exampleEnd), NDArrayIndex.all()); + case 3: + return from.get(NDArrayIndex.interval(exampleStart, exampleEnd), NDArrayIndex.all(), + NDArrayIndex.all()); + case 4: + return from.get(NDArrayIndex.interval(exampleStart, exampleEnd), NDArrayIndex.all(), NDArrayIndex.all(), + NDArrayIndex.all()); + default: + throw new RuntimeException("Invalid rank: " + from.rank()); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreExamplesFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreExamplesFunction.java new file mode 100644 index 000000000..6c3878da5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreExamplesFunction.java @@ -0,0 +1,105 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.scoring; + +import org.apache.spark.api.java.function.DoubleFlatMapFunction; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +public class ScoreExamplesFunction implements DoubleFlatMapFunction> { + + protected static Logger log = LoggerFactory.getLogger(ScoreExamplesFunction.class); + + private final Broadcast params; + private final Broadcast jsonConfig; + private final boolean addRegularization; + private final int batchSize; + + public ScoreExamplesFunction(Broadcast params, Broadcast jsonConfig, + boolean addRegularizationTerms, int batchSize) { + this.params = params; + this.jsonConfig = jsonConfig; + this.addRegularization = addRegularizationTerms; + this.batchSize = batchSize; + } + + + @Override + public Iterator call(Iterator iterator) throws Exception { + if (!iterator.hasNext()) { + return Collections.emptyIterator(); + } + + MultiLayerNetwork network = new MultiLayerNetwork(MultiLayerConfiguration.fromJson(jsonConfig.getValue())); + network.init(); + INDArray val = params.value().unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParameters(val); + + List ret = new ArrayList<>(); + + List collect = new ArrayList<>(batchSize); + int totalCount = 0; + while (iterator.hasNext()) { + collect.clear(); + int nExamples = 0; + while (iterator.hasNext() && nExamples < batchSize) { + DataSet ds = iterator.next(); + int n = ds.numExamples(); + collect.add(ds); + nExamples += n; + } + totalCount += nExamples; + + DataSet data = DataSet.merge(collect); + + + INDArray scores = network.scoreExamples(data, addRegularization); + double[] doubleScores = scores.data().asDouble(); + + for (double doubleScore : doubleScores) { + ret.add(doubleScore); + } + } + + Nd4j.getExecutioner().commit(); + + if (log.isDebugEnabled()) { + log.debug("Scored {} examples ", totalCount); + } + + return ret.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreExamplesWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreExamplesWithKeyFunction.java new file mode 100644 index 000000000..4c54cedf4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreExamplesWithKeyFunction.java @@ -0,0 +1,115 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.scoring; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.function.PairFlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +@Slf4j +public class ScoreExamplesWithKeyFunction implements PairFlatMapFunction>, K, Double> { + + private final Broadcast params; + private final Broadcast jsonConfig; + private final boolean addRegularization; + private final int batchSize; + + /** + * @param params MultiLayerNetwork parameters + * @param jsonConfig MultiLayerConfiguration, as json + * @param addRegularizationTerms if true: add regularization terms (L1, L2) to the score + * @param batchSize Batch size to use when scoring + */ + public ScoreExamplesWithKeyFunction(Broadcast params, Broadcast jsonConfig, boolean addRegularizationTerms, int batchSize) { + this.params = params; + this.jsonConfig = jsonConfig; + this.addRegularization = addRegularizationTerms; + this.batchSize = batchSize; + } + + + @Override + public Iterator> call(Iterator> iterator) throws Exception { + if (!iterator.hasNext()) { + return Collections.emptyIterator(); + } + + MultiLayerNetwork network = new MultiLayerNetwork(MultiLayerConfiguration.fromJson(jsonConfig.getValue())); + network.init(); + INDArray val = params.value().unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParameters(val); + + List> ret = new ArrayList<>(); + + List collect = new ArrayList<>(batchSize); + List collectKey = new ArrayList<>(batchSize); + int totalCount = 0; + while (iterator.hasNext()) { + collect.clear(); + collectKey.clear(); + int nExamples = 0; + while (iterator.hasNext() && nExamples < batchSize) { + Tuple2 t2 = iterator.next(); + DataSet ds = t2._2(); + int n = ds.numExamples(); + if (n != 1) + throw new IllegalStateException("Cannot score examples with one key per data set if " + + "data set contains more than 1 example (numExamples: " + n + ")"); + collect.add(ds); + collectKey.add(t2._1()); + nExamples += n; + } + totalCount += nExamples; + + DataSet data = DataSet.merge(collect); + + + INDArray scores = network.scoreExamples(data, addRegularization); + double[] doubleScores = scores.data().asDouble(); + + for (int i = 0; i < doubleScores.length; i++) { + ret.add(new Tuple2<>(collectKey.get(i), doubleScores[i])); + } + } + + Nd4j.getExecutioner().commit(); + + if (log.isDebugEnabled()) { + log.debug("Scored {} examples ", totalCount); + } + + return ret.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreFlatMapFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreFlatMapFunction.java new file mode 100644 index 000000000..3676390da --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/ScoreFlatMapFunction.java @@ -0,0 +1,79 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.scoring; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.datasets.iterator.IteratorDataSetIterator; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +@Slf4j +@AllArgsConstructor +public class ScoreFlatMapFunction implements FlatMapFunction, Tuple2> { + + private String json; + private Broadcast params; + private int minibatchSize; + + @Override + public Iterator> call(Iterator dataSetIterator) throws Exception { + if (!dataSetIterator.hasNext()) { + return Collections.singletonList(new Tuple2<>(0, 0.0)).iterator(); + } + + DataSetIterator iter = new IteratorDataSetIterator(dataSetIterator, minibatchSize); //Does batching where appropriate + + MultiLayerNetwork network = new MultiLayerNetwork(MultiLayerConfiguration.fromJson(json)); + network.init(); + INDArray val = params.value().unsafeDuplication(); //.value() object will be shared by all executors on each machine -> OK, as params are not modified by score function + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParameters(val); + + List> out = new ArrayList<>(); + while (iter.hasNext()) { + DataSet ds = iter.next(); + double score = network.score(ds, false); + + val numExamples = (int) ds.getFeatures().size(0); + out.add(new Tuple2<>(numExamples, score * numExamples)); + } + + Nd4j.getExecutioner().commit(); + + return out.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/SingleToPairFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/SingleToPairFunction.java new file mode 100644 index 000000000..c3ffc7173 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/SingleToPairFunction.java @@ -0,0 +1,34 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.scoring; + +import org.apache.spark.api.java.function.PairFunction; +import org.nd4j.linalg.api.ndarray.INDArray; +import scala.Tuple2; + +public class SingleToPairFunction implements PairFunction, T,Tuple2> { + + + @Override + public Tuple2> call(Tuple2 t2) throws Exception { + return new Tuple2<>(t2._1(), new Tuple2(t2._2(), null)); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/VaeReconstructionErrorWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/VaeReconstructionErrorWithKeyFunction.java new file mode 100644 index 000000000..3f7c5ba6c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/VaeReconstructionErrorWithKeyFunction.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.scoring; + +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.spark.impl.common.score.BaseVaeScoreWithKeyFunction; +import org.nd4j.linalg.api.ndarray.INDArray; +import scala.Tuple2; + +import java.util.Iterator; + + +public class VaeReconstructionErrorWithKeyFunction extends BaseVaeScoreWithKeyFunction { + + /** + * @param params MultiLayerNetwork parameters + * @param jsonConfig MultiLayerConfiguration, as json + * @param batchSize Batch size to use when scoring + */ + public VaeReconstructionErrorWithKeyFunction(Broadcast params, Broadcast jsonConfig, + int batchSize) { + super(params, jsonConfig, batchSize); + } + + @Override + public VariationalAutoencoder getVaeLayer() { + MultiLayerNetwork network = + new MultiLayerNetwork(MultiLayerConfiguration.fromJson((String) jsonConfig.getValue())); + network.init(); + INDArray val = ((INDArray) params.value()).unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParameters(val); + + Layer l = network.getLayer(0); + if (!(l instanceof VariationalAutoencoder)) { + throw new RuntimeException( + "Cannot use VaeReconstructionErrorWithKeyFunction on network that doesn't have a VAE " + + "layer as layer 0. Layer type: " + l.getClass()); + } + return (VariationalAutoencoder) l; + } + + @Override + public INDArray computeScore(VariationalAutoencoder vae, INDArray toScore) { + return vae.reconstructionError(toScore); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/VaeReconstructionProbWithKeyFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/VaeReconstructionProbWithKeyFunction.java new file mode 100644 index 000000000..d9dd8a155 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/multilayer/scoring/VaeReconstructionProbWithKeyFunction.java @@ -0,0 +1,66 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer.scoring; + +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.spark.impl.common.score.BaseVaeReconstructionProbWithKeyFunction; +import org.nd4j.linalg.api.ndarray.INDArray; + + +public class VaeReconstructionProbWithKeyFunction extends BaseVaeReconstructionProbWithKeyFunction { + + + /** + * @param params MultiLayerNetwork parameters + * @param jsonConfig MultiLayerConfiguration, as json + * @param useLogProbability If true: use log probability. False: use raw probability. + * @param batchSize Batch size to use when scoring + * @param numSamples Number of samples to use when calling {@link VariationalAutoencoder#reconstructionLogProbability(INDArray, int)} + */ + public VaeReconstructionProbWithKeyFunction(Broadcast params, Broadcast jsonConfig, + boolean useLogProbability, int batchSize, int numSamples) { + super(params, jsonConfig, useLogProbability, batchSize, numSamples); + } + + @Override + public VariationalAutoencoder getVaeLayer() { + MultiLayerNetwork network = + new MultiLayerNetwork(MultiLayerConfiguration.fromJson((String) jsonConfig.getValue())); + network.init(); + INDArray val = ((INDArray) params.value()).unsafeDuplication(); + if (val.length() != network.numParams(false)) + throw new IllegalStateException( + "Network did not have same number of parameters as the broadcast set parameters"); + network.setParameters(val); + + Layer l = network.getLayer(0); + if (!(l instanceof VariationalAutoencoder)) { + throw new RuntimeException( + "Cannot use VaeReconstructionProbWithKeyFunction on network that doesn't have a VAE " + + "layer as layer 0. Layer type: " + l.getClass()); + } + return (VariationalAutoencoder) l; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingMaster.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingMaster.java new file mode 100644 index 000000000..dbe2cbb27 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingMaster.java @@ -0,0 +1,280 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.storage.StorageLevel; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.spark.api.*; +import org.deeplearning4j.spark.data.BatchAndExportDataSetsFunction; +import org.deeplearning4j.spark.data.BatchAndExportMultiDataSetsFunction; +import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingMasterStats; +import org.deeplearning4j.spark.impl.paramavg.util.ExportSupport; +import org.deeplearning4j.spark.util.serde.StorageLevelDeserializer; +import org.deeplearning4j.spark.util.serde.StorageLevelSerializer; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Random; + +@Slf4j +public abstract class BaseTrainingMaster> + implements TrainingMaster { + protected static ObjectMapper jsonMapper; + protected static ObjectMapper yamlMapper; + + protected boolean collectTrainingStats; + protected ParameterAveragingTrainingMasterStats.ParameterAveragingTrainingMasterStatsHelper stats; + + protected int lastExportedRDDId = Integer.MIN_VALUE; + protected String lastRDDExportPath; + protected int batchSizePerWorker; + protected String exportDirectory = null; + protected Random rng; + + protected String trainingMasterUID; + + @Setter @Getter + protected Boolean workerTogglePeriodicGC; + @Setter @Getter + protected Integer workerPeriodicGCFrequency; + protected StatsStorageRouter statsStorage; + + //Listeners etc + protected List listeners; + + + protected Repartition repartition; + protected RepartitionStrategy repartitionStrategy; + @JsonSerialize(using = StorageLevelSerializer.class) + @JsonDeserialize(using = StorageLevelDeserializer.class) + protected StorageLevel storageLevel; + @JsonSerialize(using = StorageLevelSerializer.class) + @JsonDeserialize(using = StorageLevelDeserializer.class) + protected StorageLevel storageLevelStreams = StorageLevel.MEMORY_ONLY(); + protected RDDTrainingApproach rddTrainingApproach = RDDTrainingApproach.Export; + + protected Broadcast broadcastHadoopConfig; + + protected BaseTrainingMaster() { + + } + + + protected static synchronized ObjectMapper getJsonMapper() { + if (jsonMapper == null) { + jsonMapper = getNewMapper(new JsonFactory()); + } + return jsonMapper; + } + + protected static synchronized ObjectMapper getYamlMapper() { + if (yamlMapper == null) { + yamlMapper = getNewMapper(new YAMLFactory()); + } + return yamlMapper; + } + + protected static ObjectMapper getNewMapper(JsonFactory jsonFactory) { + ObjectMapper om = new ObjectMapper(jsonFactory); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + om.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); + om.enable(SerializationFeature.INDENT_OUTPUT); + om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); + om.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + return om; + } + + + + protected JavaRDD exportIfRequired(JavaSparkContext sc, JavaRDD trainingData) { + ExportSupport.assertExportSupported(sc); + if (collectTrainingStats) + stats.logExportStart(); + + //Two possibilities here: + // 1. We've seen this RDD before (i.e., multiple epochs training case) + // 2. We have not seen this RDD before + // (a) And we haven't got any stored data -> simply export + // (b) And we previously exported some data from a different RDD -> delete the last data + int currentRDDUid = trainingData.id(); //Id is a "A unique ID for this RDD (within its SparkContext)." + + String baseDir; + if (lastExportedRDDId == Integer.MIN_VALUE) { + //Haven't seen a RDD yet in this training master -> export data + baseDir = export(trainingData); + } else { + if (lastExportedRDDId == currentRDDUid) { + //Use the already-exported data again for another epoch + baseDir = getBaseDirForRDD(trainingData); + } else { + //The new RDD is different to the last one + // Clean up the data for the last one, and export + deleteTempDir(sc, lastRDDExportPath); + baseDir = export(trainingData); + } + } + + if (collectTrainingStats) + stats.logExportEnd(); + + return sc.textFile(baseDir + "paths/"); + } + + protected JavaRDD exportIfRequiredMDS(JavaSparkContext sc, JavaRDD trainingData) { + ExportSupport.assertExportSupported(sc); + if (collectTrainingStats) + stats.logExportStart(); + + //Two possibilities here: + // 1. We've seen this RDD before (i.e., multiple epochs training case) + // 2. We have not seen this RDD before + // (a) And we haven't got any stored data -> simply export + // (b) And we previously exported some data from a different RDD -> delete the last data + int currentRDDUid = trainingData.id(); //Id is a "A unique ID for this RDD (within its SparkContext)." + + String baseDir; + if (lastExportedRDDId == Integer.MIN_VALUE) { + //Haven't seen a RDD yet in this training master -> export data + baseDir = exportMDS(trainingData); + } else { + if (lastExportedRDDId == currentRDDUid) { + //Use the already-exported data again for another epoch + baseDir = getBaseDirForRDD(trainingData); + } else { + //The new RDD is different to the last one + // Clean up the data for the last one, and export + deleteTempDir(sc, lastRDDExportPath); + baseDir = exportMDS(trainingData); + } + } + + if (collectTrainingStats) + stats.logExportEnd(); + + return sc.textFile(baseDir + "paths/"); + } + + protected String export(JavaRDD trainingData) { + String baseDir = getBaseDirForRDD(trainingData); + String dataDir = baseDir + "data/"; + String pathsDir = baseDir + "paths/"; + + log.info("Initiating RDD export at {}", baseDir); + JavaRDD paths = trainingData + .mapPartitionsWithIndex(new BatchAndExportDataSetsFunction(batchSizePerWorker, dataDir), true); + paths.saveAsTextFile(pathsDir); + log.info("RDD export complete at {}", baseDir); + + lastExportedRDDId = trainingData.id(); + lastRDDExportPath = baseDir; + return baseDir; + } + + protected String exportMDS(JavaRDD trainingData) { + String baseDir = getBaseDirForRDD(trainingData); + String dataDir = baseDir + "data/"; + String pathsDir = baseDir + "paths/"; + + log.info("Initiating RDD export at {}", baseDir); + JavaRDD paths = trainingData.mapPartitionsWithIndex( + new BatchAndExportMultiDataSetsFunction(batchSizePerWorker, dataDir), true); + paths.saveAsTextFile(pathsDir); + log.info("RDD export complete at {}", baseDir); + + lastExportedRDDId = trainingData.id(); + lastRDDExportPath = baseDir; + return baseDir; + } + + protected String getBaseDirForRDD(JavaRDD rdd) { + if (exportDirectory == null) { + exportDirectory = getDefaultExportDirectory(rdd.context()); + } + + return exportDirectory + (exportDirectory.endsWith("/") ? "" : "/") + trainingMasterUID + "/" + rdd.id() + "/"; + } + + protected boolean deleteTempDir(JavaSparkContext sc, String tempDirPath) { + log.info("Attempting to delete temporary directory: {}", tempDirPath); + + Configuration hadoopConfiguration = sc.hadoopConfiguration(); + FileSystem fileSystem; + try { + fileSystem = FileSystem.get(new URI(tempDirPath), hadoopConfiguration); + } catch (URISyntaxException | IOException e) { + throw new RuntimeException(e); + } + + try { + fileSystem.delete(new Path(tempDirPath), true); + log.info("Deleted temporary directory: {}", tempDirPath); + return true; + } catch (IOException e) { + log.warn("Could not delete temporary directory: {}", tempDirPath, e); + return false; + } + } + + protected String getDefaultExportDirectory(SparkContext sc) { + String hadoopTmpDir = sc.hadoopConfiguration().get("hadoop.tmp.dir"); + if (!hadoopTmpDir.endsWith("/") && !hadoopTmpDir.endsWith("\\")) + hadoopTmpDir = hadoopTmpDir + "/"; + return hadoopTmpDir + "dl4j/"; + } + + + @Override + public boolean deleteTempFiles(JavaSparkContext sc) { + return lastRDDExportPath == null || deleteTempDir(sc, lastRDDExportPath); + } + + @Override + public boolean deleteTempFiles(SparkContext sc) { + return deleteTempFiles(new JavaSparkContext(sc)); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingResult.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingResult.java new file mode 100644 index 000000000..70546b9df --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingResult.java @@ -0,0 +1,26 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + +import org.deeplearning4j.spark.api.TrainingResult; + +public abstract class BaseTrainingResult implements TrainingResult { +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingWorker.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingWorker.java new file mode 100644 index 000000000..b9d7d3c19 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingWorker.java @@ -0,0 +1,27 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; + +public abstract class BaseTrainingWorker implements TrainingWorker { +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java new file mode 100644 index 000000000..8d8532e0b --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java @@ -0,0 +1,1008 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaRDDLike; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.input.PortableDataStream; +import org.apache.spark.storage.StorageLevel; +import org.datavec.spark.util.BroadcastHadoopConfigHolder; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.core.loader.impl.SerializedDataSetLoader; +import org.deeplearning4j.core.loader.impl.SerializedMultiDataSetLoader; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.StatsStorageRouterProvider; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.spark.api.*; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.api.worker.*; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.graph.dataset.DataSetToMultiDataSetFn; +import org.deeplearning4j.spark.impl.listeners.VanillaStatsStorageRouterProvider; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.aggregator.ParameterAveragingAggregationTuple; +import org.deeplearning4j.spark.impl.paramavg.aggregator.ParameterAveragingElementAddFunction; +import org.deeplearning4j.spark.impl.paramavg.aggregator.ParameterAveragingElementCombineFunction; +import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingMasterStats; +import org.deeplearning4j.spark.util.SparkUtils; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; + +import static com.google.common.base.Preconditions.checkArgument; + +@Data +@JsonIgnoreProperties({"stats", "listeners", "iterationCount", "rng", "lastExportedRDDId", "lastRDDExportPath", + "trainingMasterUID"}) +@EqualsAndHashCode(exclude = {"stats", "listeners", "iterationCount", "rng", "lastExportedRDDId", "lastRDDExportPath", + "trainingMasterUID"}) +@Slf4j +public class ParameterAveragingTrainingMaster + extends BaseTrainingMaster + implements TrainingMaster { + + protected static final int COALESCE_THRESHOLD = 3; + + + protected boolean saveUpdater; + protected Integer numWorkers; + protected int rddDataSetNumExamples; + + protected int averagingFrequency; + protected int aggregationDepth; + protected int prefetchNumBatches; + protected int iterationCount = 0; + + protected Collection trainingHookList; + + protected ParameterAveragingTrainingMaster() { + // no-arg constructor for Jackson + + String jvmuid = UIDProvider.getJVMUID(); + this.trainingMasterUID = + System.currentTimeMillis() + "_" + (jvmuid.length() <= 8 ? jvmuid : jvmuid.substring(0, 8)); + this.rng = new Random(); + } + + protected ParameterAveragingTrainingMaster(Builder builder) { + this.saveUpdater = builder.saveUpdater; + this.numWorkers = builder.numWorkers; + this.rddDataSetNumExamples = builder.rddDataSetNumExamples; + this.batchSizePerWorker = builder.batchSizePerWorker; + this.averagingFrequency = builder.averagingFrequency; + this.aggregationDepth = builder.aggregationDepth; + this.prefetchNumBatches = builder.prefetchNumBatches; + this.repartition = builder.repartition; + this.repartitionStrategy = builder.repartitionStrategy; + this.storageLevel = builder.storageLevel; + this.storageLevelStreams = builder.storageLevelStreams; + this.rddTrainingApproach = builder.rddTrainingApproach; + this.exportDirectory = builder.exportDirectory; + this.trainingHookList = builder.trainingHooks; + this.collectTrainingStats = builder.collectTrainingStats; + if (collectTrainingStats) + stats = new ParameterAveragingTrainingMasterStats.ParameterAveragingTrainingMasterStatsHelper(); + + + if (builder.rngSeed == null) { + this.rng = new Random(); + } else { + this.rng = new Random(builder.rngSeed); + } + + String jvmuid = UIDProvider.getJVMUID(); + this.trainingMasterUID = + System.currentTimeMillis() + "_" + (jvmuid.length() <= 8 ? jvmuid : jvmuid.substring(0, 8)); + } + + public ParameterAveragingTrainingMaster(boolean saveUpdater, Integer numWorkers, int rddDataSetNumExamples, + int batchSizePerWorker, int averagingFrequency, int prefetchNumBatches) { + this(saveUpdater, numWorkers, rddDataSetNumExamples, batchSizePerWorker, averagingFrequency, 2, + prefetchNumBatches, Repartition.Always, RepartitionStrategy.Balanced, false); + } + + /** + * @param saveUpdater If true: save (and average) the updater state when doing parameter averaging + * @param numWorkers Number of workers (executors * threads per executor) for the cluster + * @param rddDataSetNumExamples Number of examples in each DataSet object in the {@code RDD} + * @param batchSizePerWorker Number of examples to use per worker per fit + * @param averagingFrequency Frequency (in number of minibatches) with which to average parameters + * @param aggregationDepth Number of aggregation levels used in parameter aggregation + * @param prefetchNumBatches Number of batches to asynchronously prefetch (0: disable) + * @param repartition Set if/when repartitioning should be conducted for the training data + * @param repartitionStrategy Repartitioning strategy to use. See {@link RepartitionStrategy} + * @param collectTrainingStats If true: collect training statistics for debugging/optimization purposes + */ + public ParameterAveragingTrainingMaster(boolean saveUpdater, Integer numWorkers, int rddDataSetNumExamples, + int batchSizePerWorker, int averagingFrequency, int aggregationDepth, int prefetchNumBatches, + Repartition repartition, RepartitionStrategy repartitionStrategy, boolean collectTrainingStats) { + this(saveUpdater, numWorkers, rddDataSetNumExamples, batchSizePerWorker, averagingFrequency, aggregationDepth, + prefetchNumBatches, repartition, repartitionStrategy, StorageLevel.MEMORY_ONLY_SER(), + collectTrainingStats); + } + + public ParameterAveragingTrainingMaster(boolean saveUpdater, Integer numWorkers, int rddDataSetNumExamples, + int batchSizePerWorker, int averagingFrequency, int aggregationDepth, int prefetchNumBatches, + Repartition repartition, RepartitionStrategy repartitionStrategy, StorageLevel storageLevel, + boolean collectTrainingStats) { + checkArgument(numWorkers > 0, "Invalid number of workers: " + numWorkers + " (must be >= 1)"); + checkArgument(rddDataSetNumExamples > 0, + "Invalid rdd data set size: " + rddDataSetNumExamples + " (must be >= 1)"); + checkArgument(averagingFrequency > 0, "Invalid input: averaging frequency must be >= 1"); + checkArgument(aggregationDepth > 0, "Invalid input: tree aggregation channels must be >= 1"); + + this.saveUpdater = saveUpdater; + this.numWorkers = numWorkers; + this.rddDataSetNumExamples = rddDataSetNumExamples; + this.batchSizePerWorker = batchSizePerWorker; + this.averagingFrequency = averagingFrequency; + this.aggregationDepth = aggregationDepth; + this.prefetchNumBatches = prefetchNumBatches; + this.collectTrainingStats = collectTrainingStats; + this.repartition = repartition; + this.repartitionStrategy = repartitionStrategy; + this.storageLevel = storageLevel; + if (collectTrainingStats) + stats = new ParameterAveragingTrainingMasterStats.ParameterAveragingTrainingMasterStatsHelper(); + + String jvmuid = UIDProvider.getJVMUID(); + this.trainingMasterUID = + System.currentTimeMillis() + "_" + (jvmuid.length() <= 8 ? jvmuid : jvmuid.substring(0, 8)); + this.rng = new Random(); + } + + + + /** + * Remove a training hook from the worker + * + * @param trainingHook the training hook to remove + */ + @Override + public void removeHook(TrainingHook trainingHook) { + if (trainingHookList == null) + return; + trainingHookList.remove(trainingHook); + } + + /** + * Add a hook for the master for pre and post training + * + * @param trainingHook the training hook to add + */ + @Override + public void addHook(TrainingHook trainingHook) { + if (trainingHookList == null) { + trainingHookList = new ArrayList<>(); + } + trainingHookList.add(trainingHook); + } + + @Override + public String toJson() { + ObjectMapper om = getJsonMapper(); + + try { + return om.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException("Error producing JSON representation for ParameterAveragingTrainingMaster", e); + } + } + + @Override + public String toYaml() { + ObjectMapper om = getYamlMapper(); + + try { + return om.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException("Error producing YAML representation for ParameterAveragingTrainingMaster", e); + } + } + + /** + * Create a ParameterAveragingTrainingMaster instance by deserializing a JSON string that has been serialized with + * {@link #toJson()} + * + * @param jsonStr ParameterAveragingTrainingMaster configuration serialized as JSON + */ + public static ParameterAveragingTrainingMaster fromJson(String jsonStr) { + ObjectMapper om = getJsonMapper(); + try { + return om.readValue(jsonStr, ParameterAveragingTrainingMaster.class); + } catch (IOException e) { + throw new RuntimeException("Could not parse JSON", e); + } + } + + /** + * Create a ParameterAveragingTrainingMaster instance by deserializing a YAML string that has been serialized with + * {@link #toYaml()} + * + * @param yamlStr ParameterAveragingTrainingMaster configuration serialized as YAML + */ + public static ParameterAveragingTrainingMaster fromYaml(String yamlStr) { + ObjectMapper om = getYamlMapper(); + try { + return om.readValue(yamlStr, ParameterAveragingTrainingMaster.class); + } catch (IOException e) { + throw new RuntimeException("Could not parse YAML", e); + } + } + + + @Override + public ParameterAveragingTrainingWorker getWorkerInstance(SparkDl4jMultiLayer network) { + NetBroadcastTuple tuple = new NetBroadcastTuple(network.getNetwork().getLayerWiseConfigurations(), + network.getNetwork().params(), network.getNetwork().getUpdater().getStateViewArray()); + + if (collectTrainingStats) + stats.logBroadcastStart(); + Broadcast broadcast = network.getSparkContext().broadcast(tuple); + if (collectTrainingStats) + stats.logBroadcastEnd(); + + WorkerConfiguration configuration = new WorkerConfiguration(false, rddDataSetNumExamples, batchSizePerWorker, + averagingFrequency, prefetchNumBatches, collectTrainingStats); + return new ParameterAveragingTrainingWorker(broadcast, saveUpdater, configuration, trainingHookList, listeners, + getRouterProvider()); + } + + @Override + public ParameterAveragingTrainingWorker getWorkerInstance(SparkComputationGraph graph) { + NetBroadcastTuple tuple = new NetBroadcastTuple(graph.getNetwork().getConfiguration(), + graph.getNetwork().params(), graph.getNetwork().getUpdater().getStateViewArray()); + + if (collectTrainingStats) + stats.logBroadcastStart(); + Broadcast broadcast = graph.getSparkContext().broadcast(tuple); + if (collectTrainingStats) + stats.logBroadcastEnd(); + + WorkerConfiguration configuration = new WorkerConfiguration(true, rddDataSetNumExamples, batchSizePerWorker, + averagingFrequency, prefetchNumBatches, collectTrainingStats); + return new ParameterAveragingTrainingWorker(broadcast, saveUpdater, configuration, trainingHookList, listeners, + getRouterProvider()); + } + + protected int numObjectsEachWorker(int numExamplesEachRddObject) { + return batchSizePerWorker * averagingFrequency / numExamplesEachRddObject; + } + + protected int getNumDataSetObjectsPerSplit(int numExamplesEachRddObject) { + int dataSetObjectsPerSplit; + if (numExamplesEachRddObject == 1) { + dataSetObjectsPerSplit = numWorkers * batchSizePerWorker * averagingFrequency; + } else { + int numDataSetObjsReqEachWorker = numObjectsEachWorker(numExamplesEachRddObject); + if (numDataSetObjsReqEachWorker < 1) { + //In this case: more examples in a DataSet object than we actually require + //For example, 100 examples in DataSet, with batchSizePerWorker=50 and averagingFrequency=1 + numDataSetObjsReqEachWorker = 1; + } + + dataSetObjectsPerSplit = numDataSetObjsReqEachWorker * numWorkers; + } + return dataSetObjectsPerSplit; + } + + @Override + public void executeTraining(SparkDl4jMultiLayer network, JavaRDD trainingData) { + if (numWorkers == null) + numWorkers = network.getSparkContext().defaultParallelism(); + + if (rddTrainingApproach == RDDTrainingApproach.Direct) { + executeTrainingDirect(network, trainingData); + } else { + //Export data if required (or, use cached export) + JavaRDD paths = exportIfRequired(network.getSparkContext(), trainingData); + executeTrainingPathsHelper(network, null, paths, new SerializedDataSetLoader(), null, batchSizePerWorker); //Originally (pre-export): had rddDataSetNumExamples per DataSet. Now we have batchSizePerWorker per exported DataSet + } + } + + protected > long getTotalDataSetObjectCount( + JavaRDDLike trainingData) { + if (collectTrainingStats) + stats.logCountStart(); + long totalDataSetObjectCount = trainingData.count(); + if (collectTrainingStats) + stats.logCountEnd(); + return totalDataSetObjectCount; + } + + protected JavaPairRDD[] getSplitRDDs(JavaPairRDD trainingData, + int totalDataSetObjectCount) { + int dataSetObjectsPerSplit = getNumDataSetObjectsPerSplit(rddDataSetNumExamples); + + if (collectTrainingStats) + stats.logSplitStart(); + JavaPairRDD[] splits = SparkUtils.balancedRandomSplit(totalDataSetObjectCount, dataSetObjectsPerSplit, + trainingData, rng.nextLong()); + if (collectTrainingStats) + stats.logSplitEnd(); + return splits; + } + + protected JavaRDD[] getSplitRDDs(JavaRDD trainingData, int totalDataSetObjectCount, + int examplesPerDataSetObject) { + int dataSetObjectsPerSplit = getNumDataSetObjectsPerSplit(examplesPerDataSetObject); + + if (collectTrainingStats) + stats.logSplitStart(); + JavaRDD[] splits = SparkUtils.balancedRandomSplit(totalDataSetObjectCount, dataSetObjectsPerSplit, + trainingData, rng.nextLong()); + if (collectTrainingStats) + stats.logSplitEnd(); + return splits; + } + + protected void executeTrainingDirect(SparkDl4jMultiLayer network, JavaRDD trainingData) { + if (collectTrainingStats) + stats.logFitStart(); + //For "vanilla" parameter averaging training, we need to split the full data set into batches of size N, such that we can process the specified + // number of minibatches between averagings + //But to do that, wee need to know: (a) the number of examples, and (b) the number of workers + if (storageLevel != null) + trainingData.persist(storageLevel); + + long totalDataSetObjectCount = getTotalDataSetObjectCount(trainingData); + JavaRDD[] splits = getSplitRDDs(trainingData, (int) totalDataSetObjectCount, rddDataSetNumExamples); + + int splitNum = 1; + for (JavaRDD split : splits) { + doIteration(network, split, splitNum++, splits.length); + } + + if (collectTrainingStats) + stats.logFitEnd((int) totalDataSetObjectCount); + } + + @Override + public void executeTrainingPaths(SparkDl4jMultiLayer network, SparkComputationGraph graph, JavaRDD trainingDataPaths, DataSetLoader dsLoader, MultiDataSetLoader mdsLoader){ + executeTrainingPathsHelper(network, graph, trainingDataPaths, dsLoader, mdsLoader, rddDataSetNumExamples); + } + + protected void executeTrainingPathsHelper(SparkDl4jMultiLayer network, SparkComputationGraph graph, JavaRDD trainingDataPaths, + DataSetLoader dsLoader, MultiDataSetLoader mdsLoader, int dataSetObjectsNumExamples) { + if (numWorkers == null) + numWorkers = network.getSparkContext().defaultParallelism(); + + if (collectTrainingStats) + stats.logFitStart(); + if (storageLevelStreams != null) + trainingDataPaths.persist(storageLevelStreams); + + long totalDataSetObjectCount = getTotalDataSetObjectCount(trainingDataPaths); + JavaRDD[] splits = + getSplitRDDs(trainingDataPaths, (int) totalDataSetObjectCount, dataSetObjectsNumExamples); + + int splitNum = 1; + for (JavaRDD split : splits) { + doIterationPaths(network, graph, split, splitNum++, splits.length, dataSetObjectsNumExamples, dsLoader, mdsLoader); + } + + if (collectTrainingStats) + stats.logFitEnd((int) totalDataSetObjectCount); + } + + @Override + public void executeTraining(SparkComputationGraph graph, JavaRDD trainingData) { + if (numWorkers == null) + numWorkers = graph.getSparkContext().defaultParallelism(); + + JavaRDD mdsTrainingData = trainingData.map(new DataSetToMultiDataSetFn()); + + executeTrainingMDS(graph, mdsTrainingData); + } + + @Override + public void executeTrainingMDS(SparkComputationGraph graph, JavaRDD trainingData) { + if (numWorkers == null) + numWorkers = graph.getSparkContext().defaultParallelism(); + + if (rddTrainingApproach == RDDTrainingApproach.Direct) { + executeTrainingDirect(graph, trainingData); + } else { + //Export data if required (or, use cached export) + JavaRDD paths = exportIfRequiredMDS(graph.getSparkContext(), trainingData); + executeTrainingPathsHelper(null, graph, paths, null, new SerializedMultiDataSetLoader(), batchSizePerWorker); + } + } + + protected void executeTrainingDirect(SparkComputationGraph graph, JavaRDD trainingData) { + if (collectTrainingStats) + stats.logFitStart(); + //For "vanilla" parameter averaging training, we need to split the full data set into batches of size N, such that we can process the specified + // number of minibatches between averaging + //But to do that, we need to know: (a) the number of examples, and (b) the number of workers + if (storageLevel != null) + trainingData.persist(storageLevel); + + long totalDataSetObjectCount = getTotalDataSetObjectCount(trainingData); + + JavaRDD[] splits = + getSplitRDDs(trainingData, (int) totalDataSetObjectCount, rddDataSetNumExamples); + + int splitNum = 1; + for (JavaRDD split : splits) { + doIteration(graph, split, splitNum++, splits.length); + } + + if (collectTrainingStats) + stats.logFitEnd((int) totalDataSetObjectCount); + } + + @Override + public void setCollectTrainingStats(boolean collectTrainingStats) { + this.collectTrainingStats = collectTrainingStats; + if (collectTrainingStats) { + if (this.stats == null) + this.stats = new ParameterAveragingTrainingMasterStats.ParameterAveragingTrainingMasterStatsHelper(); + } else { + this.stats = null; + } + } + + @Override + public boolean getIsCollectTrainingStats() { + return collectTrainingStats; + } + + @Override + public SparkTrainingStats getTrainingStats() { + if (stats != null) + return stats.build(); + return null; + } + + @Override + public void setListeners(Collection listeners) { + setListeners(null, listeners); + } + + @Override + public void setListeners(StatsStorageRouter statsStorage, Collection listeners) { + this.statsStorage = statsStorage; + this.listeners = listeners == null ? null : new ArrayList<>(listeners); + } + + + + protected void doIteration(SparkDl4jMultiLayer network, JavaRDD split, int splitNum, int numSplits) { + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, averagingFreq={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, averagingFrequency, numWorkers); + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + JavaRDD splitData = split; + if (collectTrainingStats) + stats.logRepartitionStart(); + splitData = SparkUtils.repartition(splitData, repartition, repartitionStrategy, + numObjectsEachWorker(rddDataSetNumExamples), numWorkers); + int nPartitions = splitData.partitions().size(); + if (collectTrainingStats && repartition != Repartition.Never) + stats.logRepartitionEnd(); + + + FlatMapFunction, ParameterAveragingTrainingResult> function = + new ExecuteWorkerFlatMap<>(getWorkerInstance(network)); + JavaRDD result = splitData.mapPartitions(function); + processResults(network, null, result, splitNum, numSplits); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + @Deprecated + protected void doIterationPDS(SparkDl4jMultiLayer network, SparkComputationGraph graph, + JavaRDD split, int splitNum, int numSplits) { + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, averagingFreq={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, averagingFrequency, numWorkers); + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + JavaRDD splitData = split; + if (collectTrainingStats) + stats.logRepartitionStart(); + splitData = SparkUtils.repartition(splitData, repartition, repartitionStrategy, + numObjectsEachWorker(rddDataSetNumExamples), numWorkers); + int nPartitions = splitData.partitions().size(); + if (collectTrainingStats && repartition != Repartition.Never) + stats.logRepartitionEnd(); + + FlatMapFunction, ParameterAveragingTrainingResult> function; + if (network != null) + function = new ExecuteWorkerPDSFlatMap<>(getWorkerInstance(network)); + else + function = new ExecuteWorkerPDSFlatMap<>(getWorkerInstance(graph)); + + JavaRDD result = splitData.mapPartitions(function); + processResults(network, graph, result, splitNum, numSplits); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + protected void doIterationPaths(SparkDl4jMultiLayer network, SparkComputationGraph graph, JavaRDD split, + int splitNum, int numSplits, int dataSetObjectNumExamples, DataSetLoader dsLoader, MultiDataSetLoader mdsLoader) { + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, averagingFreq={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, averagingFrequency, numWorkers); + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + JavaRDD splitData = split; + if (collectTrainingStats) + stats.logRepartitionStart(); + splitData = SparkUtils.repartition(splitData, repartition, repartitionStrategy, + numObjectsEachWorker(dataSetObjectNumExamples), numWorkers); + int nPartitions = splitData.partitions().size(); + if (collectTrainingStats && repartition != Repartition.Never) + stats.logRepartitionEnd(); + + JavaSparkContext sc = (network != null ? network.getSparkContext() : graph.getSparkContext()); + FlatMapFunction, ParameterAveragingTrainingResult> function; + if (network != null) { + if(dsLoader != null){ + function = new ExecuteWorkerPathFlatMap<>(getWorkerInstance(network), dsLoader, BroadcastHadoopConfigHolder.get(sc)); + } else { + function = new ExecuteWorkerPathMDSFlatMap<>(getWorkerInstance(network), mdsLoader, BroadcastHadoopConfigHolder.get(sc)); + } + } else { + if(dsLoader != null){ + function = new ExecuteWorkerPathFlatMap<>(getWorkerInstance(graph), dsLoader, BroadcastHadoopConfigHolder.get(sc)); + } else { + function = new ExecuteWorkerPathMDSFlatMap<>(getWorkerInstance(graph), mdsLoader, BroadcastHadoopConfigHolder.get(sc)); + } + } + + JavaRDD result = splitData.mapPartitions(function); + processResults(network, graph, result, splitNum, numSplits); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + protected void doIteration(SparkComputationGraph graph, JavaRDD split, int splitNum, int numSplits) { + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, averagingFreq={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, averagingFrequency, numWorkers); + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + JavaRDD splitData = split; + + splitData = SparkUtils.repartition(splitData, repartition, repartitionStrategy, + numObjectsEachWorker(rddDataSetNumExamples), numWorkers); + int nPartitions = split.partitions().size(); + + FlatMapFunction, ParameterAveragingTrainingResult> function = + new ExecuteWorkerMultiDataSetFlatMap<>(getWorkerInstance(graph)); + JavaRDD result = splitData.mapPartitions(function); + processResults(null, graph, result, splitNum, numSplits); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + protected void doIterationPDS_MDS(SparkComputationGraph graph, JavaRDD split, int splitNum, + int numSplits) { + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, averagingFreq={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, averagingFrequency, numWorkers); + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + JavaRDD splitData = split; + if (collectTrainingStats) + stats.logRepartitionStart(); + splitData = SparkUtils.repartition(splitData, repartition, repartitionStrategy, + numObjectsEachWorker(rddDataSetNumExamples), numWorkers); + int nPartitions = splitData.partitions().size(); + if (collectTrainingStats && repartition != Repartition.Never) + stats.logRepartitionEnd(); + + FlatMapFunction, ParameterAveragingTrainingResult> function = + new ExecuteWorkerPDSMDSFlatMap<>(getWorkerInstance(graph)); + + JavaRDD result = splitData.mapPartitions(function); + processResults(null, graph, result, splitNum, numSplits); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + + protected void processResults(SparkDl4jMultiLayer network, SparkComputationGraph graph, + JavaRDD results, int splitNum, int totalSplits) { + //Need to do parameter averaging, and where necessary also do averaging of the updaters + //Let's do all of this in ONE step, such that we don't have extra synchronization costs + + if (collectTrainingStats) + stats.logAggregateStartTime(); + ParameterAveragingAggregationTuple tuple = + results.treeAggregate(null, new ParameterAveragingElementAddFunction(), + new ParameterAveragingElementCombineFunction(), this.aggregationDepth); + INDArray params = tuple.getParametersSum(); + int aggCount = tuple.getAggregationsCount(); + SparkTrainingStats aggregatedStats = tuple.getSparkTrainingStats(); + if (collectTrainingStats) + stats.logAggregationEndTime(); + + + if (collectTrainingStats) + stats.logProcessParamsUpdaterStart(); + if (params != null) { + params.divi(aggCount); + INDArray updaterState = tuple.getUpdaterStateSum(); + if (updaterState != null) + updaterState.divi(aggCount); //May be null if all SGD updaters, for example + + if (network != null) { + MultiLayerNetwork net = network.getNetwork(); + net.setParameters(params); + if (updaterState != null) + net.getUpdater().setStateViewArray(null, updaterState, false); + + network.setScore(tuple.getScoreSum() / tuple.getAggregationsCount()); + } else { + ComputationGraph g = graph.getNetwork(); + g.setParams(params); + if (updaterState != null) + g.getUpdater().setStateViewArray(updaterState); + + graph.setScore(tuple.getScoreSum() / tuple.getAggregationsCount()); + } + } else { + log.info("Skipping imbalanced split with no data for all executors"); + } + + + + if (collectTrainingStats) { + stats.logProcessParamsUpdaterEnd(); + stats.addWorkerStats(aggregatedStats); + } + + if (statsStorage != null) { + Collection meta = tuple.getListenerMetaData(); + if (meta != null && !meta.isEmpty()) { + statsStorage.putStorageMetaData(meta); + } + + Collection staticInfo = tuple.getListenerStaticInfo(); + if (staticInfo != null && !staticInfo.isEmpty()) { + statsStorage.putStaticInfo(staticInfo); + } + + Collection updates = tuple.getListenerUpdates(); + if (updates != null && !updates.isEmpty()) { + statsStorage.putUpdate(updates); + } + } + + Nd4j.getExecutioner().commit(); + + log.info("Completed training of split {} of {}", splitNum, totalSplits); + + if (params != null) { + //Params may be null for edge case (empty RDD) + if (network != null) { + MultiLayerConfiguration conf = network.getNetwork().getLayerWiseConfigurations(); + int numUpdates = averagingFrequency; + conf.setIterationCount(conf.getIterationCount() + numUpdates); + } else { + ComputationGraphConfiguration conf = graph.getNetwork().getConfiguration(); + int numUpdates = averagingFrequency; + conf.setIterationCount(conf.getIterationCount() + numUpdates); + } + } + } + + + + protected StatsStorageRouterProvider getRouterProvider() { + if (statsStorage == null) + return null; //Not needed + return new VanillaStatsStorageRouterProvider(); + } + + + public static class Builder { + protected boolean saveUpdater; + protected Integer numWorkers; + protected int rddDataSetNumExamples; + protected int batchSizePerWorker = 16; + protected int averagingFrequency = 5; + protected int aggregationDepth = 2; + protected int prefetchNumBatches = 0; + protected Repartition repartition = Repartition.Always; + protected RepartitionStrategy repartitionStrategy = RepartitionStrategy.Balanced; + protected StorageLevel storageLevel = StorageLevel.MEMORY_ONLY_SER(); + protected StorageLevel storageLevelStreams = StorageLevel.MEMORY_ONLY(); + protected RDDTrainingApproach rddTrainingApproach = RDDTrainingApproach.Export; + protected String exportDirectory = null; + protected Long rngSeed; + protected Collection trainingHooks; + protected boolean collectTrainingStats = false; + + + /** + * Adds training hooks to the master. + * The training master will setup the workers + * with the desired hooks for training. + * This can allow for tings like parameter servers + * and async updates as well as collecting statistics. + * + * @param trainingHooks the training hooks to ad + * @return + */ + public Builder trainingHooks(Collection trainingHooks) { + this.trainingHooks = trainingHooks; + return this; + } + + /** + * Adds training hooks to the master. + * The training master will setup the workers + * with the desired hooks for training. + * This can allow for tings like parameter servers + * and async updates as well as collecting statistics. + * @param hooks the training hooks to ad + * @return + */ + public Builder trainingHooks(TrainingHook... hooks) { + this.trainingHooks = Arrays.asList(hooks); + return this; + } + + /** + * Same as {@link #Builder(Integer, int)} but automatically set number of workers based on JavaSparkContext.defaultParallelism() + * + * @param rddDataSetNumExamples Number of examples in each DataSet object in the {@code RDD} + */ + public Builder(int rddDataSetNumExamples) { + this(null, rddDataSetNumExamples); + } + + /** + * Create a builder, where the following number of workers (Spark executors * number of threads per executor) are + * being used.
+ * Note: this should match the configuration of the cluster.
+ *

+ * It is also necessary to specify how many examples are in each DataSet that appears in the {@code RDD} + * or {@code JavaRDD} used for training.
+ * Two most common cases here:
+ * (a) Preprocessed data pipelines will often load binary DataSet objects with N > 1 examples in each; in this case, + * rddDataSetNumExamples should be set to N
+ * (b) "In line" data pipelines (for example, CSV String -> record reader -> DataSet just before training) will + * typically have exactly 1 example in each DataSet object. In this case, rddDataSetNumExamples should be set to 1 + * + * @param numWorkers Number of Spark execution threads in the cluster. May be null. If null: number of workers will + * be obtained from JavaSparkContext.defaultParallelism(), which should provide the number of cores + * in the cluster. + * @param rddDataSetNumExamples Number of examples in each DataSet object in the {@code RDD} + */ + public Builder(Integer numWorkers, int rddDataSetNumExamples) { + checkArgument(numWorkers == null || numWorkers > 0, + "Invalid number of workers: " + numWorkers + " (must be >= 1)"); + checkArgument(rddDataSetNumExamples > 0, + "Invalid rdd data set size: " + rddDataSetNumExamples + " (must be >= 1)"); + this.numWorkers = numWorkers; + this.rddDataSetNumExamples = rddDataSetNumExamples; + } + + /** + * Batch size (in number of examples) per worker, for each fit(DataSet) call. + * + * @param batchSizePerWorker Size of each minibatch to use for each worker + * @return + */ + public Builder batchSizePerWorker(int batchSizePerWorker) { + this.batchSizePerWorker = batchSizePerWorker; + return this; + } + + /** + * Frequency with which to average worker parameters.
+ * Note: Too high or too low can be bad for different reasons.
+ * - Too low (such as 1) can result in a lot of network traffic
+ * - Too high (>> 20 or so) can result in accuracy issues or problems with network convergence + * + * @param averagingFrequency Frequency (in number of minibatches of size 'batchSizePerWorker') to average parameters + */ + public Builder averagingFrequency(int averagingFrequency) { + checkArgument(averagingFrequency > 0, "Invalid input: averaging frequency must be >= 1"); + this.averagingFrequency = averagingFrequency; + return this; + } + + /** + * The number of levels in the aggregation tree for parameter synchronization. (default: 2) + * Note: For large models trained with many partitions, increasing this number + * will reduce the load on the driver and help prevent it from becoming a bottleneck.
+ * + * @param aggregationDepth RDD tree aggregation channels when averaging parameter updates. + */ + public Builder aggregationDepth(int aggregationDepth) { + checkArgument(aggregationDepth > 0, "Invalid input: tree aggregation channels must be >= 1"); + this.aggregationDepth = aggregationDepth; + return this; + } + + /** + * Set the number of minibatches to asynchronously prefetch in the worker. + *

+ * Default: 0 (no prefetching) + * + * @param prefetchNumBatches Number of minibatches (DataSets of size batchSizePerWorker) to fetch + */ + public Builder workerPrefetchNumBatches(int prefetchNumBatches) { + this.prefetchNumBatches = prefetchNumBatches; + return this; + } + + /** + * Set whether the updater (i.e., historical state for momentum, adagrad, etc should be saved). + * NOTE: This can double (or more) the amount of network traffic in each direction, but might + * improve network training performance (and can be more stable for certain updaters such as adagrad).
+ *

+ * This is enabled by default. + * + * @param saveUpdater If true: retain the updater state (default). If false, don't retain (updaters will be + * reinitalized in each worker after averaging). + */ + public Builder saveUpdater(boolean saveUpdater) { + this.saveUpdater = saveUpdater; + return this; + } + + /** + * Set if/when repartitioning should be conducted for the training data.
+ * Default value: always repartition (if required to guarantee correct number of partitions and correct number + * of examples in each partition). + * + * @param repartition Setting for repartitioning + */ + public Builder repartionData(Repartition repartition) { + this.repartition = repartition; + return this; + } + + /** + * Used in conjunction with {@link #repartionData(Repartition)} (which defines when repartitioning should be + * conducted), repartitionStrategy defines how the repartitioning should be done. See {@link RepartitionStrategy} + * for details + * + * @param repartitionStrategy Repartitioning strategy to use + */ + public Builder repartitionStrategy(RepartitionStrategy repartitionStrategy) { + this.repartitionStrategy = repartitionStrategy; + return this; + } + + /** + * Set the storage level for {@code RDD}s.
+ * Default: StorageLevel.MEMORY_ONLY_SER() - i.e., store in memory, in serialized form
+ * To use no RDD persistence, use {@code null}
+ *

+ * Note: Spark's StorageLevel.MEMORY_ONLY() and StorageLevel.MEMORY_AND_DISK() can be problematic when + * it comes to off-heap data (which DL4J/ND4J uses extensively). Spark does not account for off-heap memory + * when deciding if/when to drop blocks to ensure enough free memory; consequently, for DataSet RDDs that are + * larger than the total amount of (off-heap) memory, this can lead to OOM issues. Put another way: Spark counts + * the on-heap size of DataSet and INDArray objects only (which is negligible) resulting in a significant + * underestimate of the true DataSet object sizes. More DataSets are thus kept in memory than we can really afford. + * + * @param storageLevel Storage level to use for DataSet RDDs + */ + public Builder storageLevel(StorageLevel storageLevel) { + this.storageLevel = storageLevel; + return this; + } + + /** + * Set the storage level RDDs used when fitting data from Streams: either PortableDataStreams (sc.binaryFiles via + * {@link SparkDl4jMultiLayer#fit(String)} and {@link SparkComputationGraph#fit(String)}) or String paths + * (via {@link SparkDl4jMultiLayer#fitPaths(JavaRDD)}, {@link SparkComputationGraph#fitPaths(JavaRDD)} and + * {@link SparkComputationGraph#fitPathsMultiDataSet(JavaRDD)}).
+ *

+ * Default storage level is StorageLevel.MEMORY_ONLY() which should be appropriate in most cases. + * + * @param storageLevelStreams Storage level to use + */ + public Builder storageLevelStreams(StorageLevel storageLevelStreams) { + this.storageLevelStreams = storageLevelStreams; + return this; + } + + /** + * The approach to use when training on a {@code RDD} or {@code RDD}. + * Default: {@link RDDTrainingApproach#Export}, which exports data to a temporary directory first + * + * @param rddTrainingApproach Training approach to use when training from a {@code RDD} or {@code RDD} + */ + public Builder rddTrainingApproach(RDDTrainingApproach rddTrainingApproach) { + this.rddTrainingApproach = rddTrainingApproach; + return this; + } + + /** + * When {@link #rddTrainingApproach(RDDTrainingApproach)} is set to {@link RDDTrainingApproach#Export} (as it is by default) + * the data is exported to a temporary directory first. + *

+ * Default: null. -> use {hadoop.tmp.dir}/dl4j/. In this case, data is exported to {hadoop.tmp.dir}/dl4j/SOME_UNIQUE_ID/
+ * If you specify a directory, the directory {exportDirectory}/SOME_UNIQUE_ID/ will be used instead. + * + * @param exportDirectory Base directory to export data + */ + public Builder exportDirectory(String exportDirectory) { + this.exportDirectory = exportDirectory; + return this; + } + + /** + * Random number generator seed, used mainly for enforcing repeatable splitting on RDDs + * Default: no seed set (i.e., random seed) + * + * @param rngSeed RNG seed + * @return + */ + public Builder rngSeed(long rngSeed) { + this.rngSeed = rngSeed; + return this; + } + + /** + * Whether training stats collection should be enabled (disabled by default). + * @see ParameterAveragingTrainingMaster#setCollectTrainingStats(boolean) + * @see org.deeplearning4j.spark.stats.StatsUtils#exportStatsAsHTML(SparkTrainingStats, OutputStream) + * @param collectTrainingStats + */ + public Builder collectTrainingStats(boolean collectTrainingStats){ + this.collectTrainingStats = collectTrainingStats; + return this; + } + + public ParameterAveragingTrainingMaster build() { + return new ParameterAveragingTrainingMaster(this); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingResult.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingResult.java new file mode 100644 index 000000000..777f41d81 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingResult.java @@ -0,0 +1,68 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + +import lombok.Data; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.util.Collection; + +@Data +public class ParameterAveragingTrainingResult implements TrainingResult { + + private final INDArray parameters; + private final INDArray updaterState; + private final double score; + private SparkTrainingStats sparkTrainingStats; + + private final Collection listenerMetaData; + private final Collection listenerStaticInfo; + private final Collection listenerUpdates; + + + public ParameterAveragingTrainingResult(INDArray parameters, INDArray updaterState, double score, + Collection listenerMetaData, Collection listenerStaticInfo, + Collection listenerUpdates) { + this(parameters, updaterState, score, null, listenerMetaData, listenerStaticInfo, listenerUpdates); + } + + public ParameterAveragingTrainingResult(INDArray parameters, INDArray updaterState, double score, + SparkTrainingStats sparkTrainingStats, Collection listenerMetaData, + Collection listenerStaticInfo, Collection listenerUpdates) { + this.parameters = parameters; + this.updaterState = updaterState; + this.score = score; + this.sparkTrainingStats = sparkTrainingStats; + + this.listenerMetaData = listenerMetaData; + this.listenerStaticInfo = listenerStaticInfo; + this.listenerUpdates = listenerUpdates; + } + + @Override + public void setStats(SparkTrainingStats sparkTrainingStats) { + this.sparkTrainingStats = sparkTrainingStats; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingWorker.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingWorker.java new file mode 100644 index 000000000..5030a21b6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingWorker.java @@ -0,0 +1,394 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + +import lombok.val; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.StatsStorageRouterProvider; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.core.storage.listener.RoutingIterationListener; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.api.Updater; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.graph.util.ComputationGraphUtil; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.updater.MultiLayerUpdater; +import org.deeplearning4j.nn.updater.graph.ComputationGraphUpdater; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.spark.api.TrainingHook; +import org.deeplearning4j.spark.api.WorkerConfiguration; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.api.worker.NetBroadcastTuple; +import org.deeplearning4j.spark.impl.listeners.VanillaStatsStorageRouter; +import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingWorkerStats; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class ParameterAveragingTrainingWorker extends BaseTrainingWorker { + + private final Broadcast broadcast; + private final boolean saveUpdater; + private Collection trainingHooks; + private final WorkerConfiguration configuration; + private ParameterAveragingTrainingWorkerStats.ParameterAveragingTrainingWorkerStatsHelper stats = null; + private Collection trainingListeners; + private StatsStorageRouterProvider listenerRouterProvider; + + public ParameterAveragingTrainingWorker(Broadcast broadcast, boolean saveUpdater, + WorkerConfiguration configuration, Collection trainingHooks, + Collection listeners, StatsStorageRouterProvider routerProvider) { + + this.broadcast = broadcast; + this.saveUpdater = saveUpdater; + this.configuration = configuration; + this.trainingHooks = trainingHooks; + this.trainingListeners = listeners; + this.listenerRouterProvider = routerProvider; + } + + /** + * Remove a training hook from the worker + * + * @param trainingHook the training hook to remove + */ + @Override + public void removeHook(TrainingHook trainingHook) { + if (trainingHooks == null) + return; + trainingHooks.remove(trainingHook); + } + + /** + * Add a training hook to be used + * during training of the worker + * + * @param trainingHook the training hook to add + */ + @Override + public void addHook(TrainingHook trainingHook) { + if (trainingHooks == null) + trainingHooks = new ArrayList<>(); + trainingHooks.add(trainingHook); + } + + @Override + public MultiLayerNetwork getInitialModel() { + if (configuration.isCollectTrainingStats()) + stats = new ParameterAveragingTrainingWorkerStats.ParameterAveragingTrainingWorkerStatsHelper(); + + if (configuration.isCollectTrainingStats()) + stats.logBroadcastGetValueStart(); + NetBroadcastTuple tuple = broadcast.getValue(); + if (configuration.isCollectTrainingStats()) + stats.logBroadcastGetValueEnd(); + + //Don't want to have shared configuration object: each may update its iteration count (for LR schedule etc) individually + MultiLayerNetwork net = new MultiLayerNetwork(tuple.getConfiguration().clone()); + //Can't have shared parameter array across executors for parameter averaging, hence the 'true' for clone parameters array arg + net.init(tuple.getParameters().unsafeDuplication(), false); + + if (tuple.getUpdaterState() != null) { + net.setUpdater(new MultiLayerUpdater(net, tuple.getUpdaterState().unsafeDuplication())); //Can't have shared updater state + } + + Nd4j.getExecutioner().commit(); + + configureListeners(net, tuple.getCounter().getAndIncrement()); + + if (configuration.isCollectTrainingStats()) + stats.logInitEnd(); + + return net; + } + + @Override + public ComputationGraph getInitialModelGraph() { + if (configuration.isCollectTrainingStats()) + stats = new ParameterAveragingTrainingWorkerStats.ParameterAveragingTrainingWorkerStatsHelper(); + + if (configuration.isCollectTrainingStats()) + stats.logBroadcastGetValueStart(); + NetBroadcastTuple tuple = broadcast.getValue(); + if (configuration.isCollectTrainingStats()) + stats.logBroadcastGetValueEnd(); + + //Don't want to have shared configuration object: each may update its iteration count (for LR schedule etc) individually + ComputationGraph net = new ComputationGraph(tuple.getGraphConfiguration().clone()); + //Can't have shared parameter array across executors for parameter averaging, hence the 'true' for clone parameters array arg + net.init(tuple.getParameters().unsafeDuplication(), false); + + if (tuple.getUpdaterState() != null) { + net.setUpdater(new ComputationGraphUpdater(net, tuple.getUpdaterState().unsafeDuplication())); //Again: can't have shared updater state + } + + Nd4j.getExecutioner().commit(); + + configureListeners(net, tuple.getCounter().getAndIncrement()); + + if (configuration.isCollectTrainingStats()) + stats.logInitEnd(); + + return net; + } + + private void configureListeners(Model m, int counter) { + if (trainingListeners != null) { + List list = new ArrayList<>(trainingListeners.size()); + for (TrainingListener l : trainingListeners) { + if (listenerRouterProvider != null && l instanceof RoutingIterationListener) { + RoutingIterationListener rl = (RoutingIterationListener) l; + rl.setStorageRouter(listenerRouterProvider.getRouter()); + String workerID = UIDProvider.getJVMUID() + "_" + counter; + rl.setWorkerID(workerID); + } + list.add(l); //Don't need to clone listeners: not from broadcast, so deserialization handles + } + if (m instanceof MultiLayerNetwork) + ((MultiLayerNetwork) m).setListeners(list); + else + ((ComputationGraph) m).setListeners(list); + } + } + + @Override + public ParameterAveragingTrainingResult processMinibatch(DataSet dataSet, MultiLayerNetwork network, + boolean isLast) { + if (configuration.isCollectTrainingStats()) + stats.logFitStart(); + + if (trainingHooks != null) { + for (TrainingHook trainingHook : trainingHooks) { + trainingHook.preUpdate(dataSet, network); + } + } + + network.fit(dataSet); + + if (trainingHooks != null) { + for (TrainingHook trainingHook : trainingHooks) { + trainingHook.postUpdate(dataSet, network); + } + } + + + if (configuration.isCollectTrainingStats()) + stats.logFitEnd(dataSet.numExamples()); + + Nd4j.getExecutioner().commit(); + + if (isLast) { + val result = getFinalResult(network); + + // releasing Context here +// Nd4j.getMemoryManager().releaseCurrentContext(); + + return result; + } + + // releasing Context here +// Nd4j.getMemoryManager().releaseCurrentContext(); + + return null; + } + + @Override + public ParameterAveragingTrainingResult processMinibatch(DataSet dataSet, ComputationGraph graph, boolean isLast) { + return processMinibatch(ComputationGraphUtil.toMultiDataSet(dataSet), graph, isLast); + } + + @Override + public ParameterAveragingTrainingResult processMinibatch(MultiDataSet dataSet, ComputationGraph graph, + boolean isLast) { + if (configuration.isCollectTrainingStats()) + stats.logFitStart(); + //pre training hooks + if (trainingHooks != null) { + for (TrainingHook trainingHook : trainingHooks) { + trainingHook.preUpdate(dataSet, graph); + } + } + + graph.fit(dataSet); + + //post training hooks + if (trainingHooks != null) { + for (TrainingHook trainingHook : trainingHooks) { + trainingHook.postUpdate(dataSet, graph); + } + } + if (configuration.isCollectTrainingStats()) + stats.logFitEnd(dataSet.getFeatures(0).size(0)); + + Nd4j.getExecutioner().commit(); + + if (isLast) { + val result = getFinalResult(graph); + + // releasing Context here +// Nd4j.getMemoryManager().releaseCurrentContext(); + + return result; + } + + // releasing Context here +// Nd4j.getMemoryManager().releaseCurrentContext(); + + return null; + } + + + @Override + public Pair processMinibatchWithStats(DataSet dataSet, + MultiLayerNetwork network, boolean isLast) { + ParameterAveragingTrainingResult result = processMinibatch(dataSet, network, isLast); + if (result == null) + return null; + + SparkTrainingStats statsToReturn = (stats != null ? stats.build() : null); + return new Pair<>(result, statsToReturn); + } + + @Override + public Pair processMinibatchWithStats(DataSet dataSet, + ComputationGraph graph, boolean isLast) { + return processMinibatchWithStats(ComputationGraphUtil.toMultiDataSet(dataSet), graph, isLast); + } + + @Override + public Pair processMinibatchWithStats(MultiDataSet dataSet, + ComputationGraph graph, boolean isLast) { + ParameterAveragingTrainingResult result = processMinibatch(dataSet, graph, isLast); + if (result == null) + return null; + + SparkTrainingStats statsToReturn = (stats != null ? stats.build() : null); + return new Pair<>(result, statsToReturn); + } + + @Override + public ParameterAveragingTrainingResult getFinalResult(MultiLayerNetwork network) { + INDArray updaterState = null; + if (saveUpdater) { + Updater u = network.getUpdater(); + if (u != null) + updaterState = u.getStateViewArray(); + } + + Nd4j.getExecutioner().commit(); + + Collection storageMetaData = null; + Collection listenerStaticInfo = null; + Collection listenerUpdates = null; + if (listenerRouterProvider != null) { + StatsStorageRouter r = listenerRouterProvider.getRouter(); + if (r instanceof VanillaStatsStorageRouter) { //TODO this is ugly... need to find a better solution + VanillaStatsStorageRouter ssr = (VanillaStatsStorageRouter) r; + storageMetaData = ssr.getStorageMetaData(); + listenerStaticInfo = ssr.getStaticInfo(); + listenerUpdates = ssr.getUpdates(); + } + } + return new ParameterAveragingTrainingResult(network.params(), updaterState, network.score(), storageMetaData, + listenerStaticInfo, listenerUpdates); + } + + @Override + public ParameterAveragingTrainingResult getFinalResult(ComputationGraph network) { + INDArray updaterState = null; + if (saveUpdater) { + ComputationGraphUpdater u = network.getUpdater(); + if (u != null) + updaterState = u.getStateViewArray(); + } + + Nd4j.getExecutioner().commit(); + + Collection storageMetaData = null; + Collection listenerStaticInfo = null; + Collection listenerUpdates = null; + if (listenerRouterProvider != null) { + StatsStorageRouter r = listenerRouterProvider.getRouter(); + if (r instanceof VanillaStatsStorageRouter) { //TODO this is ugly... need to find a better solution + VanillaStatsStorageRouter ssr = (VanillaStatsStorageRouter) r; + storageMetaData = ssr.getStorageMetaData(); + listenerStaticInfo = ssr.getStaticInfo(); + listenerUpdates = ssr.getUpdates(); + } + } + + return new ParameterAveragingTrainingResult(network.params(), updaterState, network.score(), storageMetaData, + listenerStaticInfo, listenerUpdates); + } + + @Override + public ParameterAveragingTrainingResult getFinalResultNoData() { + return new ParameterAveragingTrainingResult(null, null, 0.0, null, null, null); + } + + @Override + public Pair getFinalResultNoDataWithStats() { + return new Pair<>(getFinalResultNoData(), null); + } + + @Override + public Pair getFinalResultWithStats( + MultiLayerNetwork network) { + ParameterAveragingTrainingResult result = getFinalResult(network); + if (result == null) + return null; + + SparkTrainingStats statsToReturn = (stats != null ? stats.build() : null); + return new Pair<>(result, statsToReturn); + } + + @Override + public Pair getFinalResultWithStats(ComputationGraph graph) { + ParameterAveragingTrainingResult result = getFinalResult(graph); + if (result == null) + return null; + + SparkTrainingStats statsToReturn = (stats != null ? stats.build() : null); + return new Pair<>(result, statsToReturn); + } + + @Override + public WorkerConfiguration getDataConfiguration() { + return configuration; + } + + @Override + public long getInstanceId() { + //Not used for parameter averaging + return 0; + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingAggregationTuple.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingAggregationTuple.java new file mode 100644 index 000000000..c493f967f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingAggregationTuple.java @@ -0,0 +1,46 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg.aggregator; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.Serializable; +import java.util.Collection; + +@AllArgsConstructor +@Data +@Builder +public class ParameterAveragingAggregationTuple implements Serializable { + private final INDArray parametersSum; + private final INDArray updaterStateSum; + private final double scoreSum; + private final int aggregationsCount; + private final SparkTrainingStats sparkTrainingStats; + private final Collection listenerMetaData; + private final Collection listenerStaticInfo; + private final Collection listenerUpdates; +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingElementAddFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingElementAddFunction.java new file mode 100644 index 000000000..cde384b89 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingElementAddFunction.java @@ -0,0 +1,101 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg.aggregator; + +import org.apache.spark.api.java.function.Function2; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingResult; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collection; + +public class ParameterAveragingElementAddFunction implements + Function2 { + + @Override + public ParameterAveragingAggregationTuple call(ParameterAveragingAggregationTuple tuple, + ParameterAveragingTrainingResult result) throws Exception { + if (tuple == null) { + return ParameterAveragingAggregationTuple.builder().parametersSum(result.getParameters()) + .updaterStateSum(result.getUpdaterState()).scoreSum(result.getScore()).aggregationsCount(1) + .sparkTrainingStats(result.getSparkTrainingStats()) + .listenerMetaData(result.getListenerMetaData()) + .listenerStaticInfo(result.getListenerStaticInfo()) + .listenerUpdates(result.getListenerUpdates()).build(); + } + + INDArray params = tuple.getParametersSum().addi(result.getParameters()); + INDArray updaterStateSum; + if (tuple.getUpdaterStateSum() == null) { + updaterStateSum = result.getUpdaterState(); + } else { + updaterStateSum = tuple.getUpdaterStateSum(); + if (result.getUpdaterState() != null) + updaterStateSum.addi(result.getUpdaterState()); + } + + double scoreSum = tuple.getScoreSum() + result.getScore(); + SparkTrainingStats stats = tuple.getSparkTrainingStats(); + if (result.getSparkTrainingStats() != null) { + if (stats == null) + stats = result.getSparkTrainingStats(); + else + stats.addOtherTrainingStats(result.getSparkTrainingStats()); + } + + Nd4j.getExecutioner().commit(); + + Collection listenerMetaData = tuple.getListenerMetaData(); + if (listenerMetaData == null) + listenerMetaData = result.getListenerMetaData(); + else { + Collection newMeta = result.getListenerMetaData(); + if (newMeta != null) + listenerMetaData.addAll(newMeta); + } + + Collection listenerStaticInfo = tuple.getListenerStaticInfo(); + if (listenerStaticInfo == null) + listenerStaticInfo = result.getListenerStaticInfo(); + else { + Collection newStatic = tuple.getListenerStaticInfo(); + if (newStatic != null) + listenerStaticInfo.addAll(newStatic); + } + + Collection listenerUpdates = tuple.getListenerUpdates(); + if (listenerUpdates == null) + listenerUpdates = result.getListenerUpdates(); + else { + Collection newUpdates = result.getListenerUpdates(); + if (newUpdates != null) + listenerUpdates.addAll(newUpdates); + } + + + + return new ParameterAveragingAggregationTuple(params, updaterStateSum, scoreSum, + tuple.getAggregationsCount() + 1, stats, listenerMetaData, listenerStaticInfo, listenerUpdates); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingElementCombineFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingElementCombineFunction.java new file mode 100644 index 000000000..3da530a31 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/aggregator/ParameterAveragingElementCombineFunction.java @@ -0,0 +1,102 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg.aggregator; + +import org.apache.spark.api.java.function.Function2; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collection; + +public class ParameterAveragingElementCombineFunction implements + Function2 { + @Override + public ParameterAveragingAggregationTuple call(ParameterAveragingAggregationTuple v1, + ParameterAveragingAggregationTuple v2) throws Exception { + if (v1 == null) + return v2; + else if (v2 == null) + return v1; + + //Handle edge case of less data than executors: in this case, one (or both) of v1 and v2 might not have any contents... + if (v1.getParametersSum() == null) + return v2; + else if (v2.getParametersSum() == null) + return v1; + + INDArray newParams = v1.getParametersSum().addi(v2.getParametersSum()); + INDArray updaterStateSum; + if (v1.getUpdaterStateSum() == null) { + updaterStateSum = v2.getUpdaterStateSum(); + } else { + updaterStateSum = v1.getUpdaterStateSum(); + if (v2.getUpdaterStateSum() != null) + updaterStateSum.addi(v2.getUpdaterStateSum()); + } + + + double scoreSum = v1.getScoreSum() + v2.getScoreSum(); + int aggregationCount = v1.getAggregationsCount() + v2.getAggregationsCount(); + + SparkTrainingStats stats = v1.getSparkTrainingStats(); + if (v2.getSparkTrainingStats() != null) { + if (stats == null) + stats = v2.getSparkTrainingStats(); + else + stats.addOtherTrainingStats(v2.getSparkTrainingStats()); + } + + Nd4j.getExecutioner().commit(); + + Collection listenerMetaData = v1.getListenerMetaData(); + if (listenerMetaData == null) + listenerMetaData = v2.getListenerMetaData(); + else { + Collection newMeta = v2.getListenerMetaData(); + if (newMeta != null) + listenerMetaData.addAll(newMeta); + } + + Collection listenerStaticInfo = v1.getListenerStaticInfo(); + if (listenerStaticInfo == null) + listenerStaticInfo = v2.getListenerStaticInfo(); + else { + Collection newStatic = v2.getListenerStaticInfo(); + if (newStatic != null) + listenerStaticInfo.addAll(newStatic); + } + + Collection listenerUpdates = v1.getListenerUpdates(); + if (listenerUpdates == null) + listenerUpdates = v2.getListenerUpdates(); + else { + Collection listenerUpdates2 = v2.getListenerUpdates(); + if (listenerUpdates2 != null) + listenerUpdates.addAll(listenerUpdates2); + } + + return new ParameterAveragingAggregationTuple(newParams, updaterStateSum, scoreSum, aggregationCount, stats, + listenerMetaData, listenerStaticInfo, listenerUpdates); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/stats/ParameterAveragingTrainingMasterStats.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/stats/ParameterAveragingTrainingMasterStats.java new file mode 100644 index 000000000..3488d8a66 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/stats/ParameterAveragingTrainingMasterStats.java @@ -0,0 +1,471 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg.stats; + +import lombok.Data; +import org.apache.commons.io.FilenameUtils; +import org.apache.spark.SparkContext; +import org.deeplearning4j.spark.api.stats.CommonSparkTrainingStats; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.stats.*; +import org.deeplearning4j.spark.time.TimeSource; +import org.deeplearning4j.spark.time.TimeSourceProvider; + +import java.io.IOException; +import java.util.*; + +@Data +public class ParameterAveragingTrainingMasterStats implements SparkTrainingStats { + + public static final String DEFAULT_DELIMITER = CommonSparkTrainingStats.DEFAULT_DELIMITER; + public static final String FILENAME_EXPORT_RDD_TIME = "parameterAveragingMasterExportTimesMs.txt"; + public static final String FILENAME_COUNT_RDD_SIZE = "parameterAveragingMasterCountRddSizeTimesMs.txt"; + public static final String FILENAME_BROADCAST_CREATE = "parameterAveragingMasterBroadcastCreateTimesMs.txt"; + public static final String FILENAME_FIT_TIME = "parameterAveragingMasterFitTimesMs.txt"; + public static final String FILENAME_SPLIT_TIME = "parameterAveragingMasterSplitTimesMs.txt"; + public static final String FILENAME_MAP_PARTITIONS_TIME = "parameterAveragingMasterMapPartitionsTimesMs.txt"; + public static final String FILENAME_AGGREGATE_TIME = "parameterAveragingMasterAggregateTimesMs.txt"; + public static final String FILENAME_PROCESS_PARAMS_TIME = "parameterAveragingMasterProcessParamsUpdaterTimesMs.txt"; + public static final String FILENAME_REPARTITION_STATS = "parameterAveragingMasterRepartitionTimesMs.txt"; + + public static final String PARAMETER_AVERAGING_MASTER_EXPORT_RDD_TIMES_MS = "parameterAveragingMasterExportTimesMs"; + public static final String PARAMETER_AVERAGING_MASTER_COUNT_RDD_TIMES_MS = + "ParameterAveragingMasterCountRddSizeTimesMs"; + public static final String PARAMETER_AVERAGING_MASTER_BROADCAST_CREATE_TIMES_MS = + "ParameterAveragingMasterBroadcastCreateTimesMs"; + public static final String PARAMETER_AVERAGING_MASTER_FIT_TIMES_MS = "ParameterAveragingMasterFitTimesMs"; + public static final String PARAMETER_AVERAGING_MASTER_SPLIT_TIMES_MS = "ParameterAveragingMasterSplitTimesMs"; + public static final String PARAMETER_AVERAGING_MASTER_MAP_PARTITIONS_TIMES_MS = + "ParameterAveragingMasterMapPartitionsTimesMs"; + public static final String PARAMETER_AVERAGING_MASTER_AGGREGATE_TIMES_MS = + "ParameterAveragingMasterAggregateTimesMs"; + public static final String PARAMETER_AVERAGING_MASTER_PROCESS_PARAMS_UPDATER_TIMES_MS = + "ParameterAveragingMasterProcessParamsUpdaterTimesMs"; + public static final String PARAMETER_AVERAGING_MASTER_REPARTITION_TIMES_MS = + "ParameterAveragingMasterRepartitionTimesMs"; + + private static Set columnNames = Collections.unmodifiableSet(new LinkedHashSet<>(Arrays.asList( + PARAMETER_AVERAGING_MASTER_EXPORT_RDD_TIMES_MS, PARAMETER_AVERAGING_MASTER_COUNT_RDD_TIMES_MS, + PARAMETER_AVERAGING_MASTER_BROADCAST_CREATE_TIMES_MS, PARAMETER_AVERAGING_MASTER_FIT_TIMES_MS, + PARAMETER_AVERAGING_MASTER_SPLIT_TIMES_MS, PARAMETER_AVERAGING_MASTER_MAP_PARTITIONS_TIMES_MS, + PARAMETER_AVERAGING_MASTER_AGGREGATE_TIMES_MS, + PARAMETER_AVERAGING_MASTER_PROCESS_PARAMS_UPDATER_TIMES_MS, + PARAMETER_AVERAGING_MASTER_REPARTITION_TIMES_MS))); + + private SparkTrainingStats workerStats; + private List parameterAveragingMasterExportTimesMs; + private List parameterAveragingMasterCountRddSizeTimesMs; + private List parameterAveragingMasterBroadcastCreateTimesMs; + private List parameterAveragingMasterFitTimesMs; + private List parameterAveragingMasterSplitTimesMs; + private List parameterAveragingMasterMapPartitionsTimesMs; + private List paramaterAveragingMasterAggregateTimesMs; + private List parameterAveragingMasterProcessParamsUpdaterTimesMs; + private List parameterAveragingMasterRepartitionTimesMs; + + + public ParameterAveragingTrainingMasterStats(SparkTrainingStats workerStats, + List parameterAveragingMasterExportTimesMs, + List parameterAveragingMasterCountRddSizeTimesMs, + List parameterAveragingMasterBroadcastCreateTimeMs, + List parameterAveragingMasterFitTimeMs, + List parameterAveragingMasterSplitTimeMs, + List parameterAveragingMasterMapPartitionsTimesMs, + List parameterAveragingMasterAggregateTimesMs, + List parameterAveragingMasterProcessParamsUpdaterTimesMs, + List parameterAveragingMasterRepartitionTimesMs) { + this.workerStats = workerStats; + this.parameterAveragingMasterExportTimesMs = parameterAveragingMasterExportTimesMs; + this.parameterAveragingMasterCountRddSizeTimesMs = parameterAveragingMasterCountRddSizeTimesMs; + this.parameterAveragingMasterBroadcastCreateTimesMs = parameterAveragingMasterBroadcastCreateTimeMs; + this.parameterAveragingMasterFitTimesMs = parameterAveragingMasterFitTimeMs; + this.parameterAveragingMasterSplitTimesMs = parameterAveragingMasterSplitTimeMs; + this.parameterAveragingMasterMapPartitionsTimesMs = parameterAveragingMasterMapPartitionsTimesMs; + this.paramaterAveragingMasterAggregateTimesMs = parameterAveragingMasterAggregateTimesMs; + this.parameterAveragingMasterProcessParamsUpdaterTimesMs = parameterAveragingMasterProcessParamsUpdaterTimesMs; + this.parameterAveragingMasterRepartitionTimesMs = parameterAveragingMasterRepartitionTimesMs; + } + + + @Override + public Set getKeySet() { + Set out = new LinkedHashSet<>(columnNames); + if (workerStats != null) + out.addAll(workerStats.getKeySet()); + return out; + } + + @Override + public List getValue(String key) { + switch (key) { + case PARAMETER_AVERAGING_MASTER_EXPORT_RDD_TIMES_MS: + return parameterAveragingMasterExportTimesMs; + case PARAMETER_AVERAGING_MASTER_COUNT_RDD_TIMES_MS: + return parameterAveragingMasterCountRddSizeTimesMs; + case PARAMETER_AVERAGING_MASTER_BROADCAST_CREATE_TIMES_MS: + return parameterAveragingMasterBroadcastCreateTimesMs; + case PARAMETER_AVERAGING_MASTER_FIT_TIMES_MS: + return parameterAveragingMasterFitTimesMs; + case PARAMETER_AVERAGING_MASTER_SPLIT_TIMES_MS: + return parameterAveragingMasterSplitTimesMs; + case PARAMETER_AVERAGING_MASTER_MAP_PARTITIONS_TIMES_MS: + return parameterAveragingMasterMapPartitionsTimesMs; + case PARAMETER_AVERAGING_MASTER_AGGREGATE_TIMES_MS: + return paramaterAveragingMasterAggregateTimesMs; + case PARAMETER_AVERAGING_MASTER_PROCESS_PARAMS_UPDATER_TIMES_MS: + return parameterAveragingMasterProcessParamsUpdaterTimesMs; + case PARAMETER_AVERAGING_MASTER_REPARTITION_TIMES_MS: + return parameterAveragingMasterRepartitionTimesMs; + default: + if (workerStats != null) + return workerStats.getValue(key); + throw new IllegalArgumentException("Unknown key: \"" + key + "\""); + } + } + + @Override + public String getShortNameForKey(String key) { + switch (key) { + case PARAMETER_AVERAGING_MASTER_EXPORT_RDD_TIMES_MS: + return "Export"; + case PARAMETER_AVERAGING_MASTER_COUNT_RDD_TIMES_MS: + return "CountRDD"; + case PARAMETER_AVERAGING_MASTER_BROADCAST_CREATE_TIMES_MS: + return "CreateBroadcast"; + case PARAMETER_AVERAGING_MASTER_FIT_TIMES_MS: + return "Fit"; + case PARAMETER_AVERAGING_MASTER_SPLIT_TIMES_MS: + return "Split"; + case PARAMETER_AVERAGING_MASTER_MAP_PARTITIONS_TIMES_MS: + return "MapPart"; + case PARAMETER_AVERAGING_MASTER_AGGREGATE_TIMES_MS: + return "Aggregate"; + case PARAMETER_AVERAGING_MASTER_PROCESS_PARAMS_UPDATER_TIMES_MS: + return "ProcessParams"; + case PARAMETER_AVERAGING_MASTER_REPARTITION_TIMES_MS: + return "Repartition"; + default: + if (workerStats != null) + return workerStats.getShortNameForKey(key); + throw new IllegalArgumentException("Unknown key: \"" + key + "\""); + } + } + + @Override + public boolean defaultIncludeInPlots(String key) { + switch (key) { + case PARAMETER_AVERAGING_MASTER_FIT_TIMES_MS: + case PARAMETER_AVERAGING_MASTER_MAP_PARTITIONS_TIMES_MS: + return false; + case PARAMETER_AVERAGING_MASTER_EXPORT_RDD_TIMES_MS: + case PARAMETER_AVERAGING_MASTER_COUNT_RDD_TIMES_MS: + case PARAMETER_AVERAGING_MASTER_SPLIT_TIMES_MS: + case PARAMETER_AVERAGING_MASTER_BROADCAST_CREATE_TIMES_MS: + case PARAMETER_AVERAGING_MASTER_AGGREGATE_TIMES_MS: + case PARAMETER_AVERAGING_MASTER_PROCESS_PARAMS_UPDATER_TIMES_MS: + case PARAMETER_AVERAGING_MASTER_REPARTITION_TIMES_MS: + return true; + default: + if (workerStats != null) + return workerStats.defaultIncludeInPlots(key); + return false; + } + } + + @Override + public void addOtherTrainingStats(SparkTrainingStats other) { + if (!(other instanceof ParameterAveragingTrainingMasterStats)) + throw new IllegalArgumentException("Expected ParameterAveragingTrainingMasterStats, got " + + (other != null ? other.getClass() : null)); + + ParameterAveragingTrainingMasterStats o = (ParameterAveragingTrainingMasterStats) other; + + if (workerStats != null) { + if (o.workerStats != null) + workerStats.addOtherTrainingStats(o.workerStats); + } else { + if (o.workerStats != null) + workerStats = o.workerStats; + } + + this.parameterAveragingMasterExportTimesMs.addAll(o.parameterAveragingMasterExportTimesMs); + this.parameterAveragingMasterCountRddSizeTimesMs.addAll(o.parameterAveragingMasterCountRddSizeTimesMs); + this.parameterAveragingMasterBroadcastCreateTimesMs.addAll(o.parameterAveragingMasterBroadcastCreateTimesMs); + this.parameterAveragingMasterRepartitionTimesMs.addAll(o.parameterAveragingMasterRepartitionTimesMs); + this.parameterAveragingMasterFitTimesMs.addAll(o.parameterAveragingMasterFitTimesMs); + if (parameterAveragingMasterRepartitionTimesMs == null) { + if (o.parameterAveragingMasterRepartitionTimesMs != null) + parameterAveragingMasterRepartitionTimesMs = o.parameterAveragingMasterRepartitionTimesMs; + } else { + if (o.parameterAveragingMasterRepartitionTimesMs != null) + parameterAveragingMasterRepartitionTimesMs.addAll(o.parameterAveragingMasterRepartitionTimesMs); + } + } + + @Override + public SparkTrainingStats getNestedTrainingStats() { + return workerStats; + } + + @Override + public String statsAsString() { + StringBuilder sb = new StringBuilder(); + String f = SparkTrainingStats.DEFAULT_PRINT_FORMAT; + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_EXPORT_RDD_TIMES_MS)); + if (parameterAveragingMasterExportTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingMasterExportTimesMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_COUNT_RDD_TIMES_MS)); + if (parameterAveragingMasterCountRddSizeTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingMasterCountRddSizeTimesMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_BROADCAST_CREATE_TIMES_MS)); + if (parameterAveragingMasterBroadcastCreateTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingMasterBroadcastCreateTimesMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_REPARTITION_TIMES_MS)); + if (parameterAveragingMasterRepartitionTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingMasterRepartitionTimesMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_FIT_TIMES_MS)); + if (parameterAveragingMasterFitTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingMasterFitTimesMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_SPLIT_TIMES_MS)); + if (parameterAveragingMasterSplitTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingMasterSplitTimesMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_MAP_PARTITIONS_TIMES_MS)); + if (parameterAveragingMasterMapPartitionsTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingMasterMapPartitionsTimesMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_AGGREGATE_TIMES_MS)); + if (paramaterAveragingMasterAggregateTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(paramaterAveragingMasterAggregateTimesMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_MASTER_PROCESS_PARAMS_UPDATER_TIMES_MS)); + if (parameterAveragingMasterProcessParamsUpdaterTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingMasterProcessParamsUpdaterTimesMs, ",")) + .append("\n"); + + if (workerStats != null) + sb.append(workerStats.statsAsString()); + + return sb.toString(); + } + + @Override + public void exportStatFiles(String outputPath, SparkContext sc) throws IOException { + String d = DEFAULT_DELIMITER; + + //Export times + String exportRddPath = FilenameUtils.concat(outputPath, FILENAME_EXPORT_RDD_TIME); + StatsUtils.exportStats(parameterAveragingMasterExportTimesMs, exportRddPath, d, sc); + + //Count RDD times: + String countRddPath = FilenameUtils.concat(outputPath, FILENAME_COUNT_RDD_SIZE); + StatsUtils.exportStats(parameterAveragingMasterCountRddSizeTimesMs, countRddPath, d, sc); + + //broadcast create time: + String broadcastTimePath = FilenameUtils.concat(outputPath, FILENAME_BROADCAST_CREATE); + StatsUtils.exportStats(parameterAveragingMasterBroadcastCreateTimesMs, broadcastTimePath, d, sc); + + //repartition + String repartitionTime = FilenameUtils.concat(outputPath, FILENAME_REPARTITION_STATS); + StatsUtils.exportStats(parameterAveragingMasterRepartitionTimesMs, repartitionTime, d, sc); + + //Fit time: + String fitTimePath = FilenameUtils.concat(outputPath, FILENAME_FIT_TIME); + StatsUtils.exportStats(parameterAveragingMasterFitTimesMs, fitTimePath, d, sc); + + //Split time: + String splitTimePath = FilenameUtils.concat(outputPath, FILENAME_SPLIT_TIME); + StatsUtils.exportStats(parameterAveragingMasterSplitTimesMs, splitTimePath, d, sc); + + //Map partitions: + String mapPartitionsPath = FilenameUtils.concat(outputPath, FILENAME_MAP_PARTITIONS_TIME); + StatsUtils.exportStats(parameterAveragingMasterMapPartitionsTimesMs, mapPartitionsPath, d, sc); + + //Aggregate time: + String aggregatePath = FilenameUtils.concat(outputPath, FILENAME_AGGREGATE_TIME); + StatsUtils.exportStats(paramaterAveragingMasterAggregateTimesMs, aggregatePath, d, sc); + + //broadcast create time: + String processParamsPath = FilenameUtils.concat(outputPath, FILENAME_PROCESS_PARAMS_TIME); + StatsUtils.exportStats(parameterAveragingMasterProcessParamsUpdaterTimesMs, processParamsPath, d, sc); + + //Repartition + if (parameterAveragingMasterRepartitionTimesMs != null) { + String repartitionPath = FilenameUtils.concat(outputPath, FILENAME_REPARTITION_STATS); + StatsUtils.exportStats(parameterAveragingMasterRepartitionTimesMs, repartitionPath, d, sc); + } + + if (workerStats != null) + workerStats.exportStatFiles(outputPath, sc); + } + + public static class ParameterAveragingTrainingMasterStatsHelper { + + private long lastExportStartTime; + private long lastCountStartTime; + private long lastBroadcastStartTime; + private long lastRepartitionStartTime; + private long lastFitStartTime; + private long lastSplitStartTime; + private long lastMapPartitionsStartTime; + private long lastAggregateStartTime; + private long lastProcessParamsUpdaterStartTime; + + private SparkTrainingStats workerStats; + + private List exportTimes = new ArrayList<>(); //Starts for exporting data + private List countTimes = new ArrayList<>(); + private List broadcastTimes = new ArrayList<>(); + private List repartitionTimes = new ArrayList<>(); + private List fitTimes = new ArrayList<>(); + private List splitTimes = new ArrayList<>(); + private List mapPartitions = new ArrayList<>(); + private List aggregateTimes = new ArrayList<>(); + private List processParamsUpdaterTimes = new ArrayList<>(); + + private final TimeSource timeSource = TimeSourceProvider.getInstance(); + + public void logExportStart() { + this.lastExportStartTime = timeSource.currentTimeMillis(); + } + + public void logExportEnd() { + long now = timeSource.currentTimeMillis(); + + exportTimes.add(new BaseEventStats(lastExportStartTime, now - lastExportStartTime)); + } + + public void logCountStart() { + this.lastCountStartTime = timeSource.currentTimeMillis(); + } + + public void logCountEnd() { + long now = timeSource.currentTimeMillis(); + + countTimes.add(new BaseEventStats(lastCountStartTime, now - lastCountStartTime)); + } + + public void logBroadcastStart() { + this.lastBroadcastStartTime = timeSource.currentTimeMillis(); + } + + public void logBroadcastEnd() { + long now = timeSource.currentTimeMillis(); + + broadcastTimes.add(new BaseEventStats(lastBroadcastStartTime, now - lastBroadcastStartTime)); + } + + public void logRepartitionStart() { + lastRepartitionStartTime = timeSource.currentTimeMillis(); + } + + public void logRepartitionEnd() { + long now = timeSource.currentTimeMillis(); + repartitionTimes.add(new BaseEventStats(lastRepartitionStartTime, now - lastRepartitionStartTime)); + } + + public void logFitStart() { + lastFitStartTime = timeSource.currentTimeMillis(); + } + + public void logFitEnd(int examplesCount) { + long now = timeSource.currentTimeMillis(); + fitTimes.add(new ExampleCountEventStats(lastFitStartTime, now - lastFitStartTime, examplesCount)); + } + + public void logSplitStart() { + lastSplitStartTime = timeSource.currentTimeMillis(); + } + + public void logSplitEnd() { + long now = timeSource.currentTimeMillis(); + splitTimes.add(new BaseEventStats(lastSplitStartTime, now - lastSplitStartTime)); + } + + public void logMapPartitionsStart() { + lastMapPartitionsStartTime = timeSource.currentTimeMillis(); + } + + public void logMapPartitionsEnd(int nPartitions) { + long now = timeSource.currentTimeMillis(); + mapPartitions.add(new PartitionCountEventStats(lastMapPartitionsStartTime, + (now - lastMapPartitionsStartTime), nPartitions)); + } + + public void logAggregateStartTime() { + lastAggregateStartTime = timeSource.currentTimeMillis(); + } + + public void logAggregationEndTime() { + long now = timeSource.currentTimeMillis(); + aggregateTimes.add(new BaseEventStats(lastAggregateStartTime, now - lastAggregateStartTime)); + } + + public void logProcessParamsUpdaterStart() { + lastProcessParamsUpdaterStartTime = timeSource.currentTimeMillis(); + } + + public void logProcessParamsUpdaterEnd() { + long now = timeSource.currentTimeMillis(); + processParamsUpdaterTimes.add(new BaseEventStats(lastProcessParamsUpdaterStartTime, + now - lastProcessParamsUpdaterStartTime)); + } + + public void addWorkerStats(SparkTrainingStats workerStats) { + if (this.workerStats == null) + this.workerStats = workerStats; + else if (workerStats != null) + this.workerStats.addOtherTrainingStats(workerStats); + } + + public ParameterAveragingTrainingMasterStats build() { + return new ParameterAveragingTrainingMasterStats(workerStats, exportTimes, countTimes, broadcastTimes, + fitTimes, splitTimes, mapPartitions, aggregateTimes, processParamsUpdaterTimes, + repartitionTimes); + } + + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/stats/ParameterAveragingTrainingWorkerStats.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/stats/ParameterAveragingTrainingWorkerStats.java new file mode 100644 index 000000000..fce3ec751 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/stats/ParameterAveragingTrainingWorkerStats.java @@ -0,0 +1,212 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg.stats; + +import lombok.Data; +import org.apache.spark.SparkContext; +import org.deeplearning4j.spark.api.stats.CommonSparkTrainingStats; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.stats.BaseEventStats; +import org.deeplearning4j.spark.stats.EventStats; +import org.deeplearning4j.spark.stats.ExampleCountEventStats; +import org.deeplearning4j.spark.stats.StatsUtils; +import org.deeplearning4j.spark.time.TimeSource; +import org.deeplearning4j.spark.time.TimeSourceProvider; + +import java.io.IOException; +import java.util.*; + +@Data +public class ParameterAveragingTrainingWorkerStats implements SparkTrainingStats { + + public static final String DEFAULT_DELIMITER = CommonSparkTrainingStats.DEFAULT_DELIMITER; + public static final String FILENAME_BROADCAST_GET_STATS = "parameterAveragingWorkerBroadcastGetValueTimeMs.txt"; + public static final String FILENAME_INIT_STATS = "parameterAveragingWorkerInitTimeMs.txt"; + public static final String FILENAME_FIT_STATS = "parameterAveragingWorkerFitTimesMs.txt"; + + private List parameterAveragingWorkerBroadcastGetValueTimeMs; + private List parameterAveragingWorkerInitTimeMs; + private List parameterAveragingWorkerFitTimesMs; + + public static final String PARAMETER_AVERAGING_WORKER_BROADCAST_GET_VALUE_TIME_MS = + "ParameterAveragingWorkerBroadcastGetValueTimeMs"; + public static final String PARAMETER_AVERAGING_WORKER_INIT_TIME_MS = "ParameterAveragingWorkerInitTimeMs"; + public static final String PARAMETER_AVERAGING_WORKER_FIT_TIMES_MS = "ParameterAveragingWorkerFitTimesMs"; + private static Set columnNames = Collections.unmodifiableSet( + new LinkedHashSet<>(Arrays.asList(PARAMETER_AVERAGING_WORKER_BROADCAST_GET_VALUE_TIME_MS, + PARAMETER_AVERAGING_WORKER_INIT_TIME_MS, PARAMETER_AVERAGING_WORKER_FIT_TIMES_MS))); + + public ParameterAveragingTrainingWorkerStats(List parameterAveragingWorkerBroadcastGetValueTimeMs, + List parameterAveragingWorkerInitTimeMs, + List parameterAveragingWorkerFitTimesMs) { + this.parameterAveragingWorkerBroadcastGetValueTimeMs = parameterAveragingWorkerBroadcastGetValueTimeMs; + this.parameterAveragingWorkerInitTimeMs = parameterAveragingWorkerInitTimeMs; + this.parameterAveragingWorkerFitTimesMs = parameterAveragingWorkerFitTimesMs; + } + + @Override + public Set getKeySet() { + return columnNames; + } + + @Override + public List getValue(String key) { + switch (key) { + case PARAMETER_AVERAGING_WORKER_BROADCAST_GET_VALUE_TIME_MS: + return parameterAveragingWorkerBroadcastGetValueTimeMs; + case PARAMETER_AVERAGING_WORKER_INIT_TIME_MS: + return parameterAveragingWorkerInitTimeMs; + case PARAMETER_AVERAGING_WORKER_FIT_TIMES_MS: + return parameterAveragingWorkerFitTimesMs; + default: + throw new IllegalArgumentException("Unknown key: \"" + key + "\""); + } + } + + @Override + public String getShortNameForKey(String key) { + switch (key) { + case PARAMETER_AVERAGING_WORKER_BROADCAST_GET_VALUE_TIME_MS: + return "BroadcastGet"; + case PARAMETER_AVERAGING_WORKER_INIT_TIME_MS: + return "ModelInit"; + case PARAMETER_AVERAGING_WORKER_FIT_TIMES_MS: + return "Fit"; + default: + throw new IllegalArgumentException("Unknown key: \"" + key + "\""); + } + } + + @Override + public boolean defaultIncludeInPlots(String key) { + switch (key) { + case PARAMETER_AVERAGING_WORKER_BROADCAST_GET_VALUE_TIME_MS: + case PARAMETER_AVERAGING_WORKER_INIT_TIME_MS: + case PARAMETER_AVERAGING_WORKER_FIT_TIMES_MS: + return true; + default: + throw new IllegalArgumentException("Unknown key: \"" + key + "\""); + } + } + + @Override + public void addOtherTrainingStats(SparkTrainingStats other) { + if (!(other instanceof ParameterAveragingTrainingWorkerStats)) + throw new IllegalArgumentException("Cannot merge ParameterAveragingTrainingWorkerStats with " + + (other != null ? other.getClass() : null)); + + ParameterAveragingTrainingWorkerStats o = (ParameterAveragingTrainingWorkerStats) other; + + this.parameterAveragingWorkerBroadcastGetValueTimeMs.addAll(o.parameterAveragingWorkerBroadcastGetValueTimeMs); + this.parameterAveragingWorkerInitTimeMs.addAll(o.parameterAveragingWorkerInitTimeMs); + this.parameterAveragingWorkerFitTimesMs.addAll(o.parameterAveragingWorkerFitTimesMs); + } + + @Override + public SparkTrainingStats getNestedTrainingStats() { + return null; + } + + @Override + public String statsAsString() { + StringBuilder sb = new StringBuilder(); + String f = SparkTrainingStats.DEFAULT_PRINT_FORMAT; + + sb.append(String.format(f, PARAMETER_AVERAGING_WORKER_BROADCAST_GET_VALUE_TIME_MS)); + if (parameterAveragingWorkerBroadcastGetValueTimeMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingWorkerBroadcastGetValueTimeMs, ",")) + .append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_WORKER_INIT_TIME_MS)); + if (parameterAveragingWorkerInitTimeMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingWorkerInitTimeMs, ",")).append("\n"); + + sb.append(String.format(f, PARAMETER_AVERAGING_WORKER_FIT_TIMES_MS)); + if (parameterAveragingWorkerFitTimesMs == null) + sb.append("-\n"); + else + sb.append(StatsUtils.getDurationAsString(parameterAveragingWorkerFitTimesMs, ",")).append("\n"); + + return sb.toString(); + } + + @Override + public void exportStatFiles(String outputPath, SparkContext sc) throws IOException { + String d = DEFAULT_DELIMITER; + + //Broadcast get time: + StatsUtils.exportStats(parameterAveragingWorkerBroadcastGetValueTimeMs, outputPath, + FILENAME_BROADCAST_GET_STATS, d, sc); + + //Network init time: + StatsUtils.exportStats(parameterAveragingWorkerInitTimeMs, outputPath, FILENAME_INIT_STATS, d, sc); + + //Network fit time: + StatsUtils.exportStats(parameterAveragingWorkerFitTimesMs, outputPath, FILENAME_FIT_STATS, d, sc); + } + + public static class ParameterAveragingTrainingWorkerStatsHelper { + private long broadcastStartTime; + private long broadcastEndTime; + private long initEndTime; + private long lastFitStartTime; + //TODO replace with fast int collection (no boxing) + private List fitTimes = new ArrayList<>(); + + private final TimeSource timeSource = TimeSourceProvider.getInstance(); + + + public void logBroadcastGetValueStart() { + broadcastStartTime = timeSource.currentTimeMillis(); + } + + public void logBroadcastGetValueEnd() { + broadcastEndTime = timeSource.currentTimeMillis(); + } + + public void logInitEnd() { + initEndTime = timeSource.currentTimeMillis(); + } + + public void logFitStart() { + lastFitStartTime = timeSource.currentTimeMillis(); + } + + public void logFitEnd(long numExamples) { + long now = timeSource.currentTimeMillis(); + fitTimes.add(new ExampleCountEventStats(lastFitStartTime, now - lastFitStartTime, numExamples)); + } + + public ParameterAveragingTrainingWorkerStats build() { + //Using ArrayList not Collections.singletonList() etc so we can add to them later (during merging) + List bList = new ArrayList<>(); + bList.add(new BaseEventStats(broadcastStartTime, broadcastEndTime - broadcastStartTime)); + List initList = new ArrayList<>(); + initList.add(new BaseEventStats(broadcastEndTime, initEndTime - broadcastEndTime)); //Init starts at same time that broadcast ends + + return new ParameterAveragingTrainingWorkerStats(bList, initList, fitTimes); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupport.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupport.java new file mode 100644 index 000000000..1255cc7ed --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupport.java @@ -0,0 +1,82 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg.util; + +import lombok.NonNull; +import org.apache.hadoop.fs.FileSystem; +import org.apache.spark.api.java.JavaSparkContext; + +import java.io.IOException; + +public class ExportSupport { + /** + * Verify that exporting data is supported, and throw an informative exception if not. + * + * @param sc the Spark context + */ + public static void assertExportSupported(@NonNull JavaSparkContext sc) { + if (!exportSupported(sc)) { + throw new RuntimeException("Export training approach is not supported in the current environment. " + + "This means that the default Hadoop file system is the local file system and Spark is running " + + "in a non-local mode. You can fix this by either adding hadoop configuration to your environment " + + "or using the Direct training approach. Configuring Hadoop can be done by adding config files (" + + "https://spark.apache.org/docs/1.6.3/configuration.html#inheriting-hadoop-cluster-configuration" + + ") or adding a setting to your SparkConf object with " + + "`sparkConf.set(\"spark.hadoop.fs.defaultFS\", \"hdfs://my-hdfs-host:9000\");`. Alternatively, " + + "you can use some other non-local storage like S3."); + } + } + + /** + * Check if exporting data is supported in the current environment. Exporting is possible in two cases: + * - The master is set to local. In this case any file system, including local FS, will work for exporting. + * - The file system is not local. Local file systems do not work in cluster modes. + * + * @param sc the Spark context + * @return if export is supported + */ + public static boolean exportSupported(@NonNull JavaSparkContext sc) { + try { + return exportSupported(sc.master(), FileSystem.get(sc.hadoopConfiguration())); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + /** + * Check if exporting data is supported in the current environment. Exporting is possible in two cases: + * - The master is set to local. In this case any file system, including local FS, will work for exporting. + * - The file system is not local. Local file systems do not work in cluster modes. + * + * @param sparkMaster the Spark master + * @param fs the Hadoop file system + * @return if export is supported + */ + public static boolean exportSupported(@NonNull String sparkMaster, @NonNull FileSystem fs) { + // Anything is supported with a local master. Regex matches 'local', 'local[DIGITS]' or 'local[*]' + if (sparkMaster.matches("^local(\\[(\\d+|\\*)])?$")) { + return true; + } + // Clustered mode is supported as long as the file system is not a local one + // ToDo: Brian it could also be a shared "local" file system accessible to all worker nodes and driver. + return !fs.getUri().getScheme().equals("file"); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/DefaultRepartitioner.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/DefaultRepartitioner.java new file mode 100644 index 000000000..042e76abe --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/DefaultRepartitioner.java @@ -0,0 +1,80 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.repartitioner; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.spark.api.Repartitioner; +import org.deeplearning4j.spark.impl.common.CountPartitionsFunction; +import scala.Tuple2; + +import java.util.List; + +@Slf4j +public class DefaultRepartitioner implements Repartitioner { + public static final int DEFAULT_MAX_PARTITIONS = 5000; + + private final int maxPartitions; + + /** + * Create a DefaultRepartitioner with the default maximum number of partitions, {@link #DEFAULT_MAX_PARTITIONS} + */ + public DefaultRepartitioner(){ + this(DEFAULT_MAX_PARTITIONS); + } + + /** + * + * @param maxPartitions Maximum number of partitions + */ + public DefaultRepartitioner(int maxPartitions){ + this.maxPartitions = maxPartitions; + } + + + @Override + public JavaRDD repartition(JavaRDD rdd, int minObjectsPerPartition, int numExecutors) { + //Num executors intentionally not used + + //Count each partition... + List> partitionCounts = + rdd.mapPartitionsWithIndex(new CountPartitionsFunction(), true).collect(); + int totalObjects = 0; + for(Tuple2 t2 : partitionCounts){ + totalObjects += t2._2(); + } + + //Now, we want 'minObjectsPerPartition' in each partition... up to a maximum number of partitions + int numPartitions; + if(totalObjects / minObjectsPerPartition > maxPartitions){ + //Need more than the minimum, to avoid exceeding the maximum + numPartitions = maxPartitions; + } else { + numPartitions = (int)Math.ceil(totalObjects / (double)minObjectsPerPartition); + } + return EqualRepartitioner.repartition(rdd, numPartitions, partitionCounts); + } + + @Override + public String toString(){ + return "DefaultRepartitioner(maxPartitions=" + maxPartitions + ")"; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/EqualRepartitioner.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/EqualRepartitioner.java new file mode 100644 index 000000000..254273dfa --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/EqualRepartitioner.java @@ -0,0 +1,103 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.repartitioner; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.spark.api.Repartitioner; +import org.deeplearning4j.spark.impl.common.CountPartitionsFunction; +import org.deeplearning4j.spark.impl.common.repartition.EqualPartitioner; +import org.deeplearning4j.spark.util.SparkUtils; +import org.nd4j.common.util.MathUtils; +import scala.Tuple2; + +import java.util.List; +import java.util.Random; + +@Slf4j +public class EqualRepartitioner implements Repartitioner { + @Override + public JavaRDD repartition(JavaRDD rdd, int minObjectsPerPartition, int numExecutors) { + //minObjectsPerPartition: intentionally not used here + + //Repartition: either always, or origNumPartitions != numWorkers + + //First: count number of elements in each partition. Need to know this so we can work out how to properly index each example, + // so we can in turn create properly balanced partitions after repartitioning + //Because the objects (DataSets etc) should be small, this should be OK + + //Count each partition... + List> partitionCounts = + rdd.mapPartitionsWithIndex(new CountPartitionsFunction(), true).collect(); + return repartition(rdd, numExecutors, partitionCounts); + } + + + public static JavaRDD repartition(JavaRDD rdd, int numPartitions, List> partitionCounts){ + int totalObjects = 0; + int initialPartitions = partitionCounts.size(); + + for (Tuple2 t2 : partitionCounts) { + totalObjects += t2._2(); + } + + //Check if already correct + int minAllowable = (int)Math.floor(totalObjects / (double) numPartitions); + int maxAllowable = (int)Math.ceil(totalObjects / (double) numPartitions); + + boolean repartitionRequired = false; + for (Tuple2 t2 : partitionCounts) { + if(t2._2() < minAllowable || t2._2() > maxAllowable ){ + repartitionRequired = true; + break; + } + } + + if (initialPartitions == numPartitions && !repartitionRequired) { + //Don't need to do any repartitioning here - already in the format we want + return rdd; + } + + //Index each element for repartitioning (can only do manual repartitioning on a JavaPairRDD) + JavaPairRDD pairIndexed = SparkUtils.indexedRDD(rdd); + + //Handle remainder. + //We'll randomly allocate one of these to a single partition, with no partition getting more than 1 (otherwise, imbalanced) + //Given that we don't know exactly how Spark will allocate partitions to workers, we are probably better off doing + // this randomly rather than "first N get +1" or "every M get +1" as this could introduce poor load balancing + int remainder = totalObjects % numPartitions; + int[] remainderPartitions = null; + if (remainder > 0) { + remainderPartitions = new int[remainder]; + int[] temp = new int[numPartitions]; + for( int i=0; i< temp.length; i++ ){ + temp[i] = i; + } + MathUtils.shuffleArray(temp, new Random()); + System.arraycopy(temp, 0, remainderPartitions, 0, remainder); + } + + int partitionSizeExRemainder = totalObjects / numPartitions; + pairIndexed = pairIndexed.partitionBy(new EqualPartitioner(numPartitions, partitionSizeExRemainder, remainderPartitions)); + return pairIndexed.values(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/NoOpRepartitioner.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/NoOpRepartitioner.java new file mode 100644 index 000000000..903382811 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/impl/repartitioner/NoOpRepartitioner.java @@ -0,0 +1,31 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.repartitioner; + +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.spark.api.Repartitioner; + +public class NoOpRepartitioner implements Repartitioner { + @Override + public JavaRDD repartition(JavaRDD input, int minObjectsPerPartition, int numExecutors) { + return input; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/BaseDataSetIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/BaseDataSetIterator.java new file mode 100644 index 000000000..936009272 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/BaseDataSetIterator.java @@ -0,0 +1,126 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.iterator; + +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.DataSetPreProcessor; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.exception.ND4JArraySizeException; + +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +public abstract class BaseDataSetIterator implements DataSetIterator { + protected Collection dataSetStreams; + protected DataSetPreProcessor preprocessor; + protected Iterator iter; + protected long totalOutcomes = -1; + protected long inputColumns = -1; + protected int batch = -1; + protected DataSet preloadedDataSet; + protected int cursor = 0; + + @Override + public DataSet next(int num) { + return next(); + } + + @Override + public int inputColumns() { + if (inputColumns == -1) + preloadDataSet(); + return (int)inputColumns; + } + + @Override + public int totalOutcomes() { + if (totalOutcomes == -1) + preloadDataSet(); + if(preloadedDataSet == null || preloadedDataSet.getLabels() == null){ + return 0; + } + return (int)preloadedDataSet.getLabels().size(1); + } + + @Override + public boolean resetSupported() { + return dataSetStreams != null; + } + + @Override + public boolean asyncSupported() { + return true; + } + + @Override + public void reset() { + if (dataSetStreams == null) + throw new IllegalStateException("Cannot reset iterator constructed with an iterator"); + iter = dataSetStreams.iterator(); + cursor = 0; + } + + @Override + public int batch() { + if (batch == -1) + preloadDataSet(); + return batch; + } + + @Override + public void setPreProcessor(DataSetPreProcessor preProcessor) { + this.preprocessor = preProcessor; + } + + @Override + public DataSetPreProcessor getPreProcessor() { + return this.preprocessor; + } + + @Override + public List getLabels() { + return null; + } + + @Override + public boolean hasNext() { + return preloadedDataSet != null || iter.hasNext(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + private void preloadDataSet() { + preloadedDataSet = load(iter.next()); + + if (preloadedDataSet.getLabels().size(1) > Integer.MAX_VALUE || + preloadedDataSet.getFeatures().size(1) > Integer.MAX_VALUE) + throw new ND4JArraySizeException(); + totalOutcomes = (int) preloadedDataSet.getLabels().size(1); + inputColumns = (int) preloadedDataSet.getFeatures().size(1); + } + + + protected abstract DataSet load(T ds); +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PathSparkDataSetIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PathSparkDataSetIterator.java new file mode 100644 index 000000000..2e7c6bad5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PathSparkDataSetIterator.java @@ -0,0 +1,94 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.iterator; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.spark.data.loader.RemoteFileSource; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.OutputStream; +import java.net.URI; +import java.util.Collection; +import java.util.Iterator; + +public class PathSparkDataSetIterator extends BaseDataSetIterator { + + public static final int BUFFER_SIZE = 4194304; //4 MB + private FileSystem fileSystem; + private DataSetLoader dataSetLoader; + private Broadcast hadoopConfig; + + public PathSparkDataSetIterator(Iterator iter, DataSetLoader dataSetLoader, Broadcast hadoopConfig) { + this.dataSetStreams = null; + this.iter = iter; + this.dataSetLoader = dataSetLoader; + this.hadoopConfig = hadoopConfig; + } + + public PathSparkDataSetIterator(Collection dataSetStreams, DataSetLoader dataSetLoader, Broadcast hadoopConfig) { + this.dataSetStreams = dataSetStreams; + iter = dataSetStreams.iterator(); + this.dataSetLoader = dataSetLoader; + this.hadoopConfig = hadoopConfig; + } + + @Override + public DataSet next() { + DataSet ds; + if (preloadedDataSet != null) { + ds = preloadedDataSet; + preloadedDataSet = null; + } else { + ds = load(iter.next()); + } + + totalOutcomes = ds.getLabels() == null ? 0 : (int) ds.getLabels().size(1); //May be null for layerwise pretraining + inputColumns = (int) ds.getFeatures().size(1); + batch = ds.numExamples(); + + if (preprocessor != null) + preprocessor.preProcess(ds); + return ds; + } + + protected synchronized DataSet load(String path) { + if (fileSystem == null) { + try { + Configuration c = hadoopConfig == null ? DefaultHadoopConfig.get() : hadoopConfig.getValue().getConfiguration(); + fileSystem = FileSystem.get(new URI(path), c); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + cursor++; + try{ + return dataSetLoader.load(new RemoteFileSource(path, fileSystem, BUFFER_SIZE)); + } catch (Exception e){ + throw new RuntimeException("Error loading DataSet at path " + path + " - DataSet may be corrupt or invalid." + + " Spark DataSets can be validated using org.deeplearning4j.spark.util.data.SparkDataValidation", e); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PathSparkMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PathSparkMultiDataSetIterator.java new file mode 100644 index 000000000..94f023f6e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PathSparkMultiDataSetIterator.java @@ -0,0 +1,134 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.iterator; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.spark.data.loader.RemoteFileSource; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.Collection; +import java.util.Iterator; + +public class PathSparkMultiDataSetIterator implements MultiDataSetIterator { + + public static final int BUFFER_SIZE = 4194304; //4 MB + + private final Collection dataSetStreams; + private MultiDataSetPreProcessor preprocessor; + private Iterator iter; + private FileSystem fileSystem; + private final MultiDataSetLoader loader; + private final Broadcast hadoopConfig; + + public PathSparkMultiDataSetIterator(Iterator iter, MultiDataSetLoader loader, Broadcast hadoopConfig) { + this.dataSetStreams = null; + this.iter = iter; + this.loader = loader; + this.hadoopConfig = hadoopConfig; + } + + public PathSparkMultiDataSetIterator(Collection dataSetStreams, MultiDataSetLoader loader, Broadcast hadoopConfig) { + this.dataSetStreams = dataSetStreams; + iter = dataSetStreams.iterator(); + this.loader = loader; + this.hadoopConfig = hadoopConfig; + } + + @Override + public MultiDataSet next(int num) { + return next(); + } + + @Override + public boolean resetSupported() { + return dataSetStreams != null; + } + + @Override + public boolean asyncSupported() { + return true; + } + + @Override + public void reset() { + if (dataSetStreams == null) + throw new IllegalStateException("Cannot reset iterator constructed with an iterator"); + iter = dataSetStreams.iterator(); + } + + @Override + public void setPreProcessor(MultiDataSetPreProcessor preProcessor) { + this.preprocessor = preProcessor; + } + + @Override + public MultiDataSetPreProcessor getPreProcessor() { + return preprocessor; + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public MultiDataSet next() { + MultiDataSet ds = load(iter.next()); + + if (preprocessor != null) + preprocessor.preProcess(ds); + return ds; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + + private synchronized MultiDataSet load(String path) { + if (fileSystem == null) { + try { + Configuration c = hadoopConfig == null ? DefaultHadoopConfig.get() : hadoopConfig.getValue().getConfiguration(); + fileSystem = FileSystem.get(new URI(path), c); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + try{ + return loader.load(new RemoteFileSource(path, fileSystem, BUFFER_SIZE)); + } catch (IOException e) { + throw new RuntimeException("Error loading MultiDataSet at path " + path + " - DataSet may be corrupt or invalid." + + " Spark MultiDataSets can be validated using org.deeplearning4j.spark.util.data.SparkDataValidation", e); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PortableDataStreamDataSetIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PortableDataStreamDataSetIterator.java new file mode 100644 index 000000000..78ea43fc5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PortableDataStreamDataSetIterator.java @@ -0,0 +1,79 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.iterator; + +import org.apache.spark.input.PortableDataStream; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.exception.ND4JArraySizeException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collection; +import java.util.Iterator; + +public class PortableDataStreamDataSetIterator extends BaseDataSetIterator { + + public PortableDataStreamDataSetIterator(Iterator iter) { + this.dataSetStreams = null; + this.iter = iter; + } + + public PortableDataStreamDataSetIterator(Collection dataSetStreams) { + this.dataSetStreams = dataSetStreams; + iter = dataSetStreams.iterator(); + } + + @Override + public DataSet next() { + DataSet ds; + if (preloadedDataSet != null) { + ds = preloadedDataSet; + preloadedDataSet = null; + } else { + ds = load(iter.next()); + } + + if (ds.getLabels().size(1) > Integer.MAX_VALUE || + ds.getFeatures().size(1) > Integer.MAX_VALUE) + throw new ND4JArraySizeException(); + totalOutcomes = (int) ds.getLabels().size(1); + inputColumns = (int) ds.getFeatures().size(1); + batch = ds.numExamples(); + + if (preprocessor != null) + preprocessor.preProcess(ds); + return ds; + } + + protected DataSet load(PortableDataStream pds) { + DataSet ds = new DataSet(); + try (InputStream is = pds.open()) { + ds.load(is); + } catch (IOException e) { + throw new RuntimeException("Error loading DataSet at path " + pds.getPath() + " - DataSet may be corrupt or invalid." + + " Spark DataSets can be validated using org.deeplearning4j.spark.util.data.SparkDataValidation", e); + } + cursor++; + return ds; + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PortableDataStreamMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PortableDataStreamMultiDataSetIterator.java new file mode 100644 index 000000000..d1f983b6e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/PortableDataStreamMultiDataSetIterator.java @@ -0,0 +1,107 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.iterator; + +import org.apache.spark.input.PortableDataStream; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collection; +import java.util.Iterator; + +public class PortableDataStreamMultiDataSetIterator implements MultiDataSetIterator { + + private final Collection dataSetStreams; + private MultiDataSetPreProcessor preprocessor; + private Iterator iter; + + public PortableDataStreamMultiDataSetIterator(Iterator iter) { + this.dataSetStreams = null; + this.iter = iter; + } + + public PortableDataStreamMultiDataSetIterator(Collection dataSetStreams) { + this.dataSetStreams = dataSetStreams; + iter = dataSetStreams.iterator(); + } + + @Override + public MultiDataSet next(int num) { + return next(); + } + + @Override + public boolean resetSupported() { + return dataSetStreams != null; + } + + @Override + public boolean asyncSupported() { + return true; + } + + @Override + public void reset() { + if (dataSetStreams == null) + throw new IllegalStateException("Cannot reset iterator constructed with an iterator"); + iter = dataSetStreams.iterator(); + } + + @Override + public void setPreProcessor(MultiDataSetPreProcessor preProcessor) { + this.preprocessor = preProcessor; + } + + @Override + public MultiDataSetPreProcessor getPreProcessor() { + return preprocessor; + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public MultiDataSet next() { + MultiDataSet ds = new org.nd4j.linalg.dataset.MultiDataSet(); + PortableDataStream pds = iter.next(); + try (InputStream is = pds.open()) { + ds.load(is); + } catch (IOException e) { + throw new RuntimeException("Error loading MultiDataSet at path " + pds.getPath() + " - MultiDataSet may be corrupt or invalid." + + " Spark MultiDataSets can be validated using org.deeplearning4j.spark.util.data.SparkDataValidation", e); + } + + if (preprocessor != null) + preprocessor.preProcess(ds); + return ds; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/SparkADSI.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/SparkADSI.java new file mode 100644 index 000000000..09ed9973c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/SparkADSI.java @@ -0,0 +1,124 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.deeplearning4j.spark.iterator; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.TaskContext; +import org.apache.spark.TaskContextHelper; +import org.nd4j.linalg.dataset.AsyncDataSetIterator; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.callbacks.DataSetCallback; +import org.nd4j.linalg.dataset.callbacks.DefaultCallback; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +@Slf4j +public class SparkADSI extends AsyncDataSetIterator { + protected TaskContext context; + + protected SparkADSI() { + super(); + } + + public SparkADSI(DataSetIterator baseIterator) { + this(baseIterator, 8); + } + + public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue queue) { + this(iterator, queueSize, queue, true); + } + + public SparkADSI(DataSetIterator baseIterator, int queueSize) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize)); + } + + public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace); + } + + public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, Integer deviceId) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace, new DefaultCallback(), + deviceId); + } + + public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, DataSetCallback callback) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace, callback); + } + + public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue queue, boolean useWorkspace) { + this(iterator, queueSize, queue, useWorkspace, new DefaultCallback()); + } + + public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue queue, boolean useWorkspace, + DataSetCallback callback) { + this(iterator, queueSize, queue, useWorkspace, callback, Nd4j.getAffinityManager().getDeviceForCurrentThread()); + } + + public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue queue, boolean useWorkspace, + DataSetCallback callback, Integer deviceId) { + this(); + + if (queueSize < 2) + queueSize = 2; + + this.deviceId = deviceId; + this.callback = callback; + this.useWorkspace = useWorkspace; + this.buffer = queue; + this.prefetchSize = queueSize; + this.backedIterator = iterator; + this.workspaceId = "SADSI_ITER-" + java.util.UUID.randomUUID().toString(); + + if (iterator.resetSupported()) + this.backedIterator.reset(); + + context = TaskContext.get(); + + this.thread = new SparkPrefetchThread(buffer, iterator, terminator, null, Nd4j.getAffinityManager().getDeviceForCurrentThread()); + + /** + * We want to ensure, that background thread will have the same thread->device affinity, as master thread + */ + + thread.setDaemon(true); + thread.start(); + } + + @Override + protected void externalCall() { + TaskContextHelper.setTaskContext(context); + + } + + public class SparkPrefetchThread extends AsyncPrefetchThread { + + protected SparkPrefetchThread(BlockingQueue queue, DataSetIterator iterator, DataSet terminator, MemoryWorkspace workspace, int deviceId) { + super(queue, iterator, terminator, workspace, deviceId); + } + + + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/SparkAMDSI.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/SparkAMDSI.java new file mode 100644 index 000000000..128db97a7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/iterator/SparkAMDSI.java @@ -0,0 +1,119 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.deeplearning4j.spark.iterator; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.TaskContext; +import org.apache.spark.TaskContextHelper; +import org.nd4j.linalg.dataset.AsyncMultiDataSetIterator; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.dataset.callbacks.DataSetCallback; +import org.nd4j.linalg.dataset.callbacks.DefaultCallback; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +@Slf4j +public class SparkAMDSI extends AsyncMultiDataSetIterator { + protected TaskContext context; + + protected SparkAMDSI() { + super(); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator) { + this(baseIterator, 8); + } + + public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue queue) { + this(iterator, queueSize, queue, true); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize)); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace, Integer deviceId) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace, + new DefaultCallback(), deviceId); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace, + DataSetCallback callback) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace, callback); + } + + public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue queue, + boolean useWorkspace) { + this(iterator, queueSize, queue, useWorkspace, null); + } + + public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue queue, + boolean useWorkspace, DataSetCallback callback) { + this(iterator, queueSize, queue, useWorkspace, callback, Nd4j.getAffinityManager().getDeviceForCurrentThread()); + } + + public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue queue, + boolean useWorkspace, DataSetCallback callback, Integer deviceId) { + this(); + + if (queueSize < 2) + queueSize = 2; + + this.callback = callback; + this.buffer = queue; + this.backedIterator = iterator; + this.useWorkspaces = useWorkspace; + this.prefetchSize = queueSize; + this.workspaceId = "SAMDSI_ITER-" + java.util.UUID.randomUUID().toString(); + this.deviceId = deviceId; + + if (iterator.resetSupported()) + this.backedIterator.reset(); + + this.thread = new SparkPrefetchThread(buffer, iterator, terminator, Nd4j.getAffinityManager().getDeviceForCurrentThread()); + + context = TaskContext.get(); + + thread.setDaemon(true); + thread.start(); + } + + @Override + protected void externalCall() { + TaskContextHelper.setTaskContext(context); + } + + protected class SparkPrefetchThread extends AsyncPrefetchThread { + + protected SparkPrefetchThread(@NonNull BlockingQueue queue, @NonNull MultiDataSetIterator iterator, @NonNull MultiDataSet terminator, int deviceId) { + super(queue, iterator, terminator, deviceId); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/ordering/DataSetOrdering.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/ordering/DataSetOrdering.java new file mode 100644 index 000000000..00084f931 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/ordering/DataSetOrdering.java @@ -0,0 +1,85 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.ordering; + +import org.nd4j.linalg.dataset.DataSet; +import scala.Function1; +import scala.Some; +import scala.math.Ordering; + +public class DataSetOrdering implements Ordering { + @Override + public Some tryCompare(DataSet dataSet, DataSet t1) { + return null; + } + + @Override + public int compare(DataSet dataSet, DataSet t1) { + return 0; + } + + @Override + public boolean lteq(DataSet dataSet, DataSet t1) { + return dataSet.numExamples() >= t1.numExamples(); + } + + @Override + public boolean gteq(DataSet dataSet, DataSet t1) { + return !lteq(dataSet, t1); + } + + @Override + public boolean lt(DataSet dataSet, DataSet t1) { + return dataSet.numExamples() >= t1.numExamples(); + } + + @Override + public boolean gt(DataSet dataSet, DataSet t1) { + return !lt(dataSet, t1); + } + + @Override + public boolean equiv(DataSet dataSet, DataSet t1) { + return dataSet.numExamples() == t1.numExamples(); + } + + @Override + public DataSet max(DataSet dataSet, DataSet t1) { + return gt(dataSet, t1) ? dataSet : t1; + } + + @Override + public DataSet min(DataSet dataSet, DataSet t1) { + return max(dataSet, t1) == dataSet ? t1 : dataSet; + } + + @Override + public Ordering reverse() { + return null; + } + + @Override + public Ordering on(Function1 function1) { + return null; + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/BaseEventStats.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/BaseEventStats.java new file mode 100644 index 000000000..880fa41a9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/BaseEventStats.java @@ -0,0 +1,81 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.stats; + +import org.deeplearning4j.core.util.UIDProvider; + +public class BaseEventStats implements EventStats { + + protected final String machineId; + protected final String jvmId; + protected final long threadId; + protected final long startTime; + protected final long durationMs; + + public BaseEventStats(long startTime, long durationMs) { + this(UIDProvider.getHardwareUID(), UIDProvider.getJVMUID(), Thread.currentThread().getId(), startTime, + durationMs); + } + + public BaseEventStats(String machineId, String jvmId, long threadId, long startTime, long durationMs) { + this.machineId = machineId; + this.jvmId = jvmId; + this.threadId = threadId; + this.startTime = startTime; + this.durationMs = durationMs; + } + + @Override + public String getMachineID() { + return machineId; + } + + @Override + public String getJvmID() { + return jvmId; + } + + @Override + public long getThreadID() { + return threadId; + } + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public long getDurationMs() { + return durationMs; + } + + @Override + public String asString(String delimiter) { + return machineId + delimiter + jvmId + delimiter + threadId + delimiter + startTime + delimiter + durationMs; + } + + @Override + public String getStringHeader(String delimiter) { + return "machineId" + delimiter + "jvmId" + delimiter + "threadId" + delimiter + "startTime" + delimiter + + "durationMs"; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/EventStats.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/EventStats.java new file mode 100644 index 000000000..06597aa95 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/EventStats.java @@ -0,0 +1,53 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.stats; + +import java.io.Serializable; + +public interface EventStats extends Serializable { + + String getMachineID(); + + String getJvmID(); + + long getThreadID(); + + long getStartTime(); + + long getDurationMs(); + + /** + * Get a String representation of the EventStats. This should be a single line delimited representation, suitable + * for exporting (such as CSV). Should not contain a new-line character at the end of each line + * + * @param delimiter Delimiter to use for the data + * @return String representation of the EventStats object + */ + String asString(String delimiter); + + /** + * Get a header line for exporting the EventStats object, for use with {@link #asString(String)} + * + * @param delimiter Delimiter to use for the header + * @return Header line + */ + String getStringHeader(String delimiter); +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/ExampleCountEventStats.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/ExampleCountEventStats.java new file mode 100644 index 000000000..97b7254f4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/ExampleCountEventStats.java @@ -0,0 +1,50 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.stats; + +import lombok.Getter; + +public class ExampleCountEventStats extends BaseEventStats { + + @Getter + private final long totalExampleCount; + + public ExampleCountEventStats(long startTime, long durationMs, long totalExampleCount) { + super(startTime, durationMs); + this.totalExampleCount = totalExampleCount; + } + + public ExampleCountEventStats(String machineId, String jvmId, long threadId, long startTime, long durationMs, + int totalExampleCount) { + super(machineId, jvmId, threadId, startTime, durationMs); + this.totalExampleCount = totalExampleCount; + } + + @Override + public String asString(String delimiter) { + return super.asString(delimiter) + delimiter + totalExampleCount; + } + + @Override + public String getStringHeader(String delimiter) { + return super.getStringHeader(delimiter) + delimiter + "totalExampleCount"; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/PartitionCountEventStats.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/PartitionCountEventStats.java new file mode 100644 index 000000000..2018d1912 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/PartitionCountEventStats.java @@ -0,0 +1,50 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.stats; + +import lombok.Getter; + +public class PartitionCountEventStats extends BaseEventStats { + + @Getter + private final int numPartitions; + + public PartitionCountEventStats(long startTime, long durationMs, int numPartitions) { + super(startTime, durationMs); + this.numPartitions = numPartitions; + } + + public PartitionCountEventStats(String machineId, String jvmId, long threadId, long startTime, long durationMs, + int numPartitions) { + super(machineId, jvmId, threadId, startTime, durationMs); + this.numPartitions = numPartitions; + } + + @Override + public String asString(String delimiter) { + return super.asString(delimiter) + delimiter + numPartitions; + } + + @Override + public String getStringHeader(String delimiter) { + return super.getStringHeader(delimiter) + delimiter + "numPartitions"; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/StatsUtils.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/StatsUtils.java new file mode 100644 index 000000000..867d89795 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/stats/StatsUtils.java @@ -0,0 +1,460 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.stats; + +import org.apache.commons.io.FilenameUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.util.SparkUtils; +import org.deeplearning4j.ui.api.Component; +import org.deeplearning4j.ui.api.LengthUnit; +import org.deeplearning4j.ui.components.chart.ChartHistogram; +import org.deeplearning4j.ui.components.chart.ChartLine; +import org.deeplearning4j.ui.components.chart.ChartTimeline; +import org.deeplearning4j.ui.components.chart.style.StyleChart; +import org.deeplearning4j.ui.components.component.ComponentDiv; +import org.deeplearning4j.ui.components.component.style.StyleDiv; +import org.deeplearning4j.ui.components.text.ComponentText; +import org.deeplearning4j.ui.components.text.style.StyleText; +import org.deeplearning4j.ui.standalone.StaticPageUtil; +import scala.Tuple3; + +import java.awt.*; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; +import java.util.List; + +public class StatsUtils { + + public static final long DEFAULT_MAX_TIMELINE_SIZE_MS = 20 * 60 * 1000; //20 minutes + + private StatsUtils() {} + + public static void exportStats(List list, String outputDirectory, String filename, String delimiter, + SparkContext sc) throws IOException { + String path = FilenameUtils.concat(outputDirectory, filename); + exportStats(list, path, delimiter, sc); + } + + public static void exportStats(List list, String outputPath, String delimiter, SparkContext sc) + throws IOException { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (EventStats e : list) { + if (first) + sb.append(e.getStringHeader(delimiter)).append("\n"); + sb.append(e.asString(delimiter)).append("\n"); + first = false; + } + SparkUtils.writeStringToFile(outputPath, sb.toString(), sc); + } + + public static String getDurationAsString(List list, String delim) { + StringBuilder sb = new StringBuilder(); + int num = list.size(); + int count = 0; + for (EventStats e : list) { + sb.append(e.getDurationMs()); + if (count++ < num - 1) + sb.append(delim); + } + return sb.toString(); + } + + public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, String path, JavaSparkContext sc) + throws Exception { + exportStatsAsHtml(sparkTrainingStats, path, sc.sc()); + } + + /** + * Generate and export a HTML representation (including charts, etc) of the Spark training statistics
+ * Note: exporting is done via Spark, so the path here can be a local file, HDFS, etc. + * + * @param sparkTrainingStats Stats to generate HTML page for + * @param path Path to export. May be local or HDFS + * @param sc Spark context + * @throws Exception IO errors or error generating HTML file + */ + public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, String path, SparkContext sc) + throws Exception { + exportStatsAsHtml(sparkTrainingStats, DEFAULT_MAX_TIMELINE_SIZE_MS, path, sc); + } + + /** + * Generate and export a HTML representation (including charts, etc) of the Spark training statistics
+ * Note: exporting is done via Spark, so the path here can be a local file, HDFS, etc. + * + * @param sparkTrainingStats Stats to generate HTML page for + * @param path Path to export. May be local or HDFS + * @param maxTimelineSizeMs maximum amount of activity to show in a single timeline plot (multiple plots will be used if training exceeds this amount of time) + * @param sc Spark context + * @throws Exception IO errors or error generating HTML file + */ + public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, long maxTimelineSizeMs, String path, + SparkContext sc) throws Exception { + FileSystem fileSystem = FileSystem.get(sc.hadoopConfiguration()); + try (BufferedOutputStream bos = new BufferedOutputStream(fileSystem.create(new Path(path)))) { + exportStatsAsHTML(sparkTrainingStats, maxTimelineSizeMs, bos); + } + } + + /** + * Generate and export a HTML representation (including charts, etc) of the Spark training statistics
+ * This overload is for writing to an output stream + * + * @param sparkTrainingStats Stats to generate HTML page for + * @throws Exception IO errors or error generating HTML file + */ + public static void exportStatsAsHTML(SparkTrainingStats sparkTrainingStats, OutputStream outputStream) + throws Exception { + exportStatsAsHTML(sparkTrainingStats, DEFAULT_MAX_TIMELINE_SIZE_MS, outputStream); + } + + /** + * Generate and export a HTML representation (including charts, etc) of the Spark training statistics
+ * This overload is for writing to an output stream + * + * @param sparkTrainingStats Stats to generate HTML page for + * @param maxTimelineSizeMs maximum amount of activity to show in a single timeline plot (multiple plots will be used if training exceeds this amount of time) + * @throws Exception IO errors or error generating HTML file + */ + public static void exportStatsAsHTML(SparkTrainingStats sparkTrainingStats, long maxTimelineSizeMs, + OutputStream outputStream) throws Exception { + Set keySet = sparkTrainingStats.getKeySet(); + + List components = new ArrayList<>(); + + StyleChart styleChart = new StyleChart.Builder().backgroundColor(Color.WHITE).width(700, LengthUnit.Px) + .height(400, LengthUnit.Px).build(); + + StyleText styleText = new StyleText.Builder().color(Color.BLACK).fontSize(20).build(); + Component headerText = new ComponentText("Deeplearning4j - Spark Training Analysis", styleText); + Component header = new ComponentDiv( + new StyleDiv.Builder().height(40, LengthUnit.Px).width(100, LengthUnit.Percent).build(), + headerText); + components.add(header); + + Set keySetInclude = new HashSet<>(); + for (String s : keySet) + if (sparkTrainingStats.defaultIncludeInPlots(s)) + keySetInclude.add(s); + + Collections.addAll(components, + getTrainingStatsTimelineChart(sparkTrainingStats, keySetInclude, maxTimelineSizeMs)); + + for (String s : keySet) { + List list = new ArrayList<>(sparkTrainingStats.getValue(s)); + Collections.sort(list, new StartTimeComparator()); + + double[] x = new double[list.size()]; + double[] duration = new double[list.size()]; + double minDur = Double.MAX_VALUE; + double maxDur = -Double.MAX_VALUE; + for (int i = 0; i < duration.length; i++) { + x[i] = i; + duration[i] = list.get(i).getDurationMs(); + minDur = Math.min(minDur, duration[i]); + maxDur = Math.max(maxDur, duration[i]); + } + + Component line = new ChartLine.Builder(s, styleChart).addSeries("Duration", x, duration) + .setYMin(minDur == maxDur ? minDur - 1 : null).setYMax(minDur == maxDur ? minDur + 1 : null) + .build(); + + //Also build a histogram... + Component hist = null; + if (minDur != maxDur && !list.isEmpty()) + hist = getHistogram(duration, 20, s, styleChart); + + Component[] temp; + if (hist != null) { + temp = new Component[] {line, hist}; + } else { + temp = new Component[] {line}; + } + + components.add(new ComponentDiv(new StyleDiv.Builder().width(100, LengthUnit.Percent).build(), temp)); + + + //TODO this is really ugly + if (!list.isEmpty() && (list.get(0) instanceof ExampleCountEventStats + || list.get(0) instanceof PartitionCountEventStats)) { + boolean exCount = list.get(0) instanceof ExampleCountEventStats; + + double[] y = new double[list.size()]; + double miny = Double.MAX_VALUE; + double maxy = -Double.MAX_VALUE; + for (int i = 0; i < y.length; i++) { + y[i] = (exCount ? ((ExampleCountEventStats) list.get(i)).getTotalExampleCount() + : ((PartitionCountEventStats) list.get(i)).getNumPartitions()); + miny = Math.min(miny, y[i]); + maxy = Math.max(maxy, y[i]); + } + + String title = s + " / " + (exCount ? "Number of Examples" : "Number of Partitions"); + Component line2 = new ChartLine.Builder(title, styleChart) + .addSeries((exCount ? "Examples" : "Partitions"), x, y) + .setYMin(miny == maxy ? miny - 1 : null).setYMax(miny == maxy ? miny + 1 : null) + .build(); + + + //Also build a histogram... + Component hist2 = null; + if (miny != maxy) + hist2 = getHistogram(y, 20, title, styleChart); + + Component[] temp2; + if (hist2 != null) { + temp2 = new Component[] {line2, hist2}; + } else { + temp2 = new Component[] {line2}; + } + + components.add(new ComponentDiv(new StyleDiv.Builder().width(100, LengthUnit.Percent).build(), temp2)); + } + } + + String html = StaticPageUtil.renderHTML(components); + outputStream.write(html.getBytes("UTF-8")); + } + + + public static class StartTimeComparator implements Comparator { + @Override + public int compare(EventStats o1, EventStats o2) { + return Long.compare(o1.getStartTime(), o2.getStartTime()); + } + } + + + private static Component[] getTrainingStatsTimelineChart(SparkTrainingStats stats, Set includeSet, + long maxDurationMs) { + Set> uniqueTuples = new HashSet<>(); + Set machineIDs = new HashSet<>(); + Set jvmIDs = new HashSet<>(); + + Map machineShortNames = new HashMap<>(); + Map jvmShortNames = new HashMap<>(); + + long earliestStart = Long.MAX_VALUE; + long latestEnd = Long.MIN_VALUE; + for (String s : includeSet) { + List list = stats.getValue(s); + for (EventStats e : list) { + machineIDs.add(e.getMachineID()); + jvmIDs.add(e.getJvmID()); + uniqueTuples.add(new Tuple3(e.getMachineID(), e.getJvmID(), e.getThreadID())); + earliestStart = Math.min(earliestStart, e.getStartTime()); + latestEnd = Math.max(latestEnd, e.getStartTime() + e.getDurationMs()); + } + } + int count = 0; + for (String s : machineIDs) { + machineShortNames.put(s, "PC " + count++); + } + count = 0; + for (String s : jvmIDs) { + jvmShortNames.put(s, "JVM " + count++); + } + + int nLanes = uniqueTuples.size(); + List> outputOrder = new ArrayList<>(uniqueTuples); + Collections.sort(outputOrder, new TupleComparator()); + + Color[] colors = getColors(includeSet.size()); + Map colorMap = new HashMap<>(); + count = 0; + for (String s : includeSet) { + colorMap.put(s, colors[count++]); + } + + //Create key for charts: + List tempList = new ArrayList<>(); + for (String s : includeSet) { + String key = stats.getShortNameForKey(s) + " - " + s; + + tempList.add(new ComponentDiv( + new StyleDiv.Builder().backgroundColor(colorMap.get(s)).width(33.3, LengthUnit.Percent) + .height(25, LengthUnit.Px).floatValue(StyleDiv.FloatValue.left).build(), + new ComponentText(key, new StyleText.Builder().fontSize(11).build()))); + } + Component key = new ComponentDiv(new StyleDiv.Builder().width(100, LengthUnit.Percent).build(), tempList); + + //How many charts? + int nCharts = (int) ((latestEnd - earliestStart) / maxDurationMs); + if (nCharts < 1) + nCharts = 1; + long[] chartStartTimes = new long[nCharts]; + long[] chartEndTimes = new long[nCharts]; + for (int i = 0; i < nCharts; i++) { + chartStartTimes[i] = earliestStart + i * maxDurationMs; + chartEndTimes[i] = earliestStart + (i + 1) * maxDurationMs; + } + + + List>> entriesByLane = new ArrayList<>(); + for (int c = 0; c < nCharts; c++) { + entriesByLane.add(new ArrayList>()); + for (int i = 0; i < nLanes; i++) { + entriesByLane.get(c).add(new ArrayList()); + } + } + + for (String s : includeSet) { + + List list = stats.getValue(s); + for (EventStats e : list) { + if (e.getDurationMs() == 0) + continue; + + long start = e.getStartTime(); + long end = start + e.getDurationMs(); + + int chartIdx = -1; + for (int j = 0; j < nCharts; j++) { + if (start >= chartStartTimes[j] && start < chartEndTimes[j]) { + chartIdx = j; + } + } + if (chartIdx == -1) + chartIdx = nCharts - 1; + + + Tuple3 tuple = new Tuple3<>(e.getMachineID(), e.getJvmID(), e.getThreadID()); + + int idx = outputOrder.indexOf(tuple); + Color c = colorMap.get(s); + // ChartTimeline.TimelineEntry entry = new ChartTimeline.TimelineEntry(null, start, end, c); + ChartTimeline.TimelineEntry entry = + new ChartTimeline.TimelineEntry(stats.getShortNameForKey(s), start, end, c); + entriesByLane.get(chartIdx).get(idx).add(entry); + } + } + + //Sort each lane by start time: + for (int i = 0; i < nCharts; i++) { + for (List l : entriesByLane.get(i)) { + Collections.sort(l, new Comparator() { + @Override + public int compare(ChartTimeline.TimelineEntry o1, ChartTimeline.TimelineEntry o2) { + return Long.compare(o1.getStartTimeMs(), o2.getStartTimeMs()); + } + }); + } + } + + StyleChart sc = new StyleChart.Builder().width(1280, LengthUnit.Px) + .height(35 * nLanes + (60 + 20 + 25), LengthUnit.Px).margin(LengthUnit.Px, 60, 20, 200, 10) //top, bottom, left, right + .build(); + + List list = new ArrayList<>(nCharts); + for (int j = 0; j < nCharts; j++) { + ChartTimeline.Builder b = new ChartTimeline.Builder("Timeline: Training Activities", sc); + int i = 0; + for (List l : entriesByLane.get(j)) { + Tuple3 t3 = outputOrder.get(i); + String name = machineShortNames.get(t3._1()) + ", " + jvmShortNames.get(t3._2()) + ", Thread " + + t3._3(); + b.addLane(name, l); + i++; + } + list.add(b.build()); + } + + list.add(key); + + return list.toArray(new Component[list.size()]); + } + + private static class TupleComparator implements Comparator> { + @Override + public int compare(Tuple3 o1, Tuple3 o2) { + if (o1._1().equals(o2._1())) { + //Equal machine IDs, so sort on JVM ids + if (o1._2().equals(o2._2())) { + //Equal machine AND JVM IDs, so sort on thread ID + return Long.compare(o1._3(), o2._3()); + } else { + return o1._2().compareTo(o2._2()); + } + } else { + return o1._1().compareTo(o2._1()); + } + } + } + + private static Color[] getColors(int nColors) { + Color[] c = new Color[nColors]; + double step; + if (nColors <= 1) + step = 1.0; + else + step = 1.0 / (nColors + 1); + for (int i = 0; i < nColors; i++) { + // c[i] = Color.getHSBColor((float) step * i, 0.4f, 0.75f); //step hue; fixed saturation + variance to (hopefully) ensure readability of labels + if (i % 2 == 0) + c[i] = Color.getHSBColor((float) step * i, 0.4f, 0.75f); //step hue; fixed saturation + variance to (hopefully) ensure readability of labels + else + c[i] = Color.getHSBColor((float) step * i, 1.0f, 1.0f); //step hue; fixed saturation + variance to (hopefully) ensure readability of labels + } + return c; + } + + private static Component getHistogram(double[] data, int nBins, String title, StyleChart styleChart) { + double min = Double.MAX_VALUE; + double max = -Double.MAX_VALUE; + for (double d : data) { + min = Math.min(min, d); + max = Math.max(max, d); + } + + if (min == max) + return null; + double[] bins = new double[nBins + 1]; + int[] counts = new int[nBins]; + double step = (max - min) / nBins; + for (int i = 0; i < bins.length; i++) + bins[i] = min + i * step; + + for (double d : data) { + for (int i = 0; i < bins.length - 1; i++) { + if (d >= bins[i] && d < bins[i + 1]) { + counts[i]++; + break; + } + } + if (d == bins[bins.length - 1]) + counts[counts.length - 1]++; + } + + ChartHistogram.Builder b = new ChartHistogram.Builder(title, styleChart); + for (int i = 0; i < bins.length - 1; i++) { + b.addBin(bins[i], bins[i + 1], counts[i]); + } + + return b.build(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/NTPTimeSource.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/NTPTimeSource.java new file mode 100644 index 000000000..8b6332ba4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/NTPTimeSource.java @@ -0,0 +1,175 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.time; + +import org.apache.commons.net.ntp.NTPUDPClient; +import org.apache.commons.net.ntp.TimeInfo; +import org.deeplearning4j.common.config.DL4JSystemProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.util.Timer; +import java.util.TimerTask; + +public class NTPTimeSource implements TimeSource { + + /** + * @deprecated Use {@link DL4JSystemProperties#NTP_SOURCE_UPDATE_FREQUENCY_MS_PROPERTY} + */ + @Deprecated + public static final String NTP_SOURCE_UPDATE_FREQUENCY_MS_PROPERTY = DL4JSystemProperties.NTP_SOURCE_UPDATE_FREQUENCY_MS_PROPERTY; + /** + * @deprecated Use {@link DL4JSystemProperties#NTP_SOURCE_SERVER_PROPERTY} + */ + @Deprecated + public static final String NTP_SOURCE_SERVER_PROPERTY = DL4JSystemProperties.NTP_SOURCE_SERVER_PROPERTY; + public static final int MAX_QUERY_RETRIES = 10; + public static final int DEFAULT_NTP_TIMEOUT_MS = 10000; + public static final long DEFAULT_UPDATE_FREQUENCY = 30 * 60 * 1000L; //30 Minutes + public static final long MIN_UPDATE_FREQUENCY = 30000L; //30 sec + + public static final String DEFAULT_NTP_SERVER = "0.pool.ntp.org"; + + private static Logger log = LoggerFactory.getLogger(NTPTimeSource.class); + private static NTPTimeSource instance; + + public static synchronized TimeSource getInstance() { + if (instance == null) + instance = new NTPTimeSource(); + return instance; + } + + private volatile long lastOffsetGetTimeSystemMS = -1; + private volatile long lastOffsetMilliseconds; + + private final long synchronizationFreqMS; + private final String ntpServer; + + private NTPTimeSource() { + this(getUpdateFrequencyConfiguration(), getServerConfiguration()); + } + + private NTPTimeSource(long synchronizationFreqMS, String ntpServer) { + this.synchronizationFreqMS = synchronizationFreqMS; + this.ntpServer = ntpServer; + + log.debug("Initializing NTPTimeSource with query frequency {} ms using server {}", synchronizationFreqMS, + ntpServer); + + queryServerNow(); + + //Start a Timer to periodically query the server + Timer timer = new Timer(true); + timer.scheduleAtFixedRate(new QueryServerTask(), synchronizationFreqMS, synchronizationFreqMS); + + log.debug("Initialized NTPTimeSource with query frequency {} ms using server {}", synchronizationFreqMS, + ntpServer); + } + + //Query and parse the system property + private static long getUpdateFrequencyConfiguration() { + String property = System.getProperty(DL4JSystemProperties.NTP_SOURCE_UPDATE_FREQUENCY_MS_PROPERTY); + Long parseAttempt = null; + long updateFreq; + if (property != null) { + try { + parseAttempt = Long.parseLong(property); + } catch (Exception e) { + log.info("Error parsing system property \"{}\" with value \"{}\"", + DL4JSystemProperties.NTP_SOURCE_UPDATE_FREQUENCY_MS_PROPERTY, property); + } + if (parseAttempt != null) { + if (parseAttempt < MIN_UPDATE_FREQUENCY) { + log.info("Invalid update frequency (milliseconds): {} is less than minimum {}. Using default update frequency: {} ms", + parseAttempt, MIN_UPDATE_FREQUENCY, DEFAULT_UPDATE_FREQUENCY); + updateFreq = DEFAULT_UPDATE_FREQUENCY; + } else { + updateFreq = parseAttempt; + } + } else { + updateFreq = DEFAULT_UPDATE_FREQUENCY; + } + } else { + updateFreq = DEFAULT_UPDATE_FREQUENCY; + } + return updateFreq; + } + + private static String getServerConfiguration() { + return System.getProperty(DL4JSystemProperties.NTP_SOURCE_SERVER_PROPERTY, DEFAULT_NTP_SERVER); + } + + + private void queryServerNow() { + Long offsetResult = null; + for (int i = 0; i < MAX_QUERY_RETRIES; i++) { + try { + NTPUDPClient client = new NTPUDPClient(); + client.setDefaultTimeout(DEFAULT_NTP_TIMEOUT_MS);// Timeout if a response takes longer than 10 seconds + + client.open(); + InetAddress address = InetAddress.getByName(ntpServer); + TimeInfo info = client.getTime(address); + info.computeDetails(); + Long offset = info.getOffset(); + if (offset == null) { + throw new Exception("Could not calculate time offset (offset is null)"); + } else { + offsetResult = offset; + break; + } + } catch (Exception e) { + log.error("Error querying NTP server, attempt {} of {}", (i + 1), MAX_QUERY_RETRIES, e); + } + } + + if (offsetResult == null) { + log.error("Could not successfully query NTP server after " + MAX_QUERY_RETRIES + " tries"); + throw new RuntimeException("Could not successfully query NTP server after " + MAX_QUERY_RETRIES + " tries"); + } + + lastOffsetGetTimeSystemMS = System.currentTimeMillis(); + lastOffsetMilliseconds = offsetResult; + log.debug("Updated local time offset based on NTP server result. Offset = {}", lastOffsetMilliseconds); + } + + //Timer task to be run periodically + private class QueryServerTask extends TimerTask { + public void run() { + queryServerNow(); + } + } + + + + //Get system offset. Note: positive offset means system clock is behind time server; negative offset means system + // clock is ahead of time server + private synchronized long getSystemOffset() { + return lastOffsetMilliseconds; + } + + public long currentTimeMillis() { + long offset = getSystemOffset(); + long systemTime = System.currentTimeMillis(); + return systemTime + offset; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/SystemClockTimeSource.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/SystemClockTimeSource.java new file mode 100644 index 000000000..02c7c0fb8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/SystemClockTimeSource.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.time; + + +public class SystemClockTimeSource implements TimeSource { + + public static TimeSource getInstance() { + return new SystemClockTimeSource(); + } + + public long currentTimeMillis() { + return System.currentTimeMillis(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/TimeSource.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/TimeSource.java new file mode 100644 index 000000000..c236aea68 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/TimeSource.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.time; + +public interface TimeSource { + + /** + * Get the current time in milliseconds, according to this TimeSource + * @return Current time, since epoch + */ + long currentTimeMillis(); + + //TODO add methods related to accuracy etc + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/TimeSourceProvider.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/TimeSourceProvider.java new file mode 100644 index 000000000..688ddef19 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/time/TimeSourceProvider.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.time; + +import org.deeplearning4j.common.config.DL4JClassLoading; +import org.deeplearning4j.common.config.DL4JSystemProperties; + +import java.lang.reflect.Method; + +public class TimeSourceProvider { + + /** + * Default class to use when getting a TimeSource instance + */ + public static final String DEFAULT_TIMESOURCE_CLASS_NAME = NTPTimeSource.class.getName(); + + /** + * @deprecated Use {@link DL4JSystemProperties#TIMESOURCE_CLASSNAME_PROPERTY} + */ + @Deprecated + public static final String TIMESOURCE_CLASSNAME_PROPERTY = DL4JSystemProperties.TIMESOURCE_CLASSNAME_PROPERTY; + + private TimeSourceProvider() {} + + /** + * Get a TimeSource + * the default TimeSource instance (default: {@link NTPTimeSource} + * + * @return TimeSource + */ + public static TimeSource getInstance() { + String className = System.getProperty(DL4JSystemProperties.TIMESOURCE_CLASSNAME_PROPERTY, DEFAULT_TIMESOURCE_CLASS_NAME); + + return getInstance(className); + } + + /** + * Get a specific TimeSource by class name + * + * @param className Class name of the TimeSource to return the instance for + * @return TimeSource instance + */ + public static TimeSource getInstance(String className) { + try { + Class clazz = DL4JClassLoading.loadClassByName(className); + Method getInstance = clazz.getMethod("getInstance"); + return (TimeSource) getInstance.invoke(null); + } catch (Exception e) { + throw new RuntimeException("Error getting TimeSource instance for class \"" + className + "\"", e); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/MLLibUtil.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/MLLibUtil.java new file mode 100644 index 000000000..dbde9f862 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/MLLibUtil.java @@ -0,0 +1,436 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util; + +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.input.PortableDataStream; +import org.apache.spark.mllib.linalg.Matrices; +import org.apache.spark.mllib.linalg.Matrix; +import org.apache.spark.mllib.linalg.Vector; +import org.apache.spark.mllib.linalg.Vectors; +import org.apache.spark.mllib.regression.LabeledPoint; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.split.InputStreamInputSplit; +import org.datavec.api.writable.Writable; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.exception.ND4JArraySizeException; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.util.FeatureUtil; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + + +/** + * Dl4j <----> MLLib + * + * @author Adam Gibson + */ +public class MLLibUtil { + + + private MLLibUtil() {} + + /** + * This is for the edge case where + * you have a single output layer + * and need to convert the output layer to + * an index + * @param vector the vector to get the classifier prediction for + * @return the prediction for the given vector + */ + public static double toClassifierPrediction(Vector vector) { + double max = Double.NEGATIVE_INFINITY; + int maxIndex = 0; + for (int i = 0; i < vector.size(); i++) { + double curr = vector.apply(i); + if (curr > max) { + maxIndex = i; + max = curr; + } + } + + return maxIndex; + } + + /** + * Convert an ndarray to a matrix. + * Note that the matrix will be con + * @param arr the array + * @return an mllib vector + */ + public static INDArray toMatrix(Matrix arr) { + + // we assume that Matrix always has F order + return Nd4j.create(arr.toArray(), new int[] {arr.numRows(), arr.numCols()}, 'f'); + } + + /** + * Convert an ndarray to a vector + * @param arr the array + * @return an mllib vector + */ + public static INDArray toVector(Vector arr) { + return Nd4j.create(Nd4j.createBuffer(arr.toArray())); + } + + + /** + * Convert an ndarray to a matrix. + * Note that the matrix will be con + * @param arr the array + * @return an mllib vector + */ + public static Matrix toMatrix(INDArray arr) { + if (!arr.isMatrix()) { + throw new IllegalArgumentException("passed in array must be a matrix"); + } + + // if arr is a view - we have to dup anyway + if (arr.isView()) { + return Matrices.dense(arr.rows(), arr.columns(), arr.dup('f').data().asDouble()); + } else // if not a view - we must ensure data is F ordered + return Matrices.dense(arr.rows(), arr.columns(), + arr.ordering() == 'f' ? arr.data().asDouble() : arr.dup('f').data().asDouble()); + } + + /** + * Convert an ndarray to a vector + * @param arr the array + * @return an mllib vector + */ + public static Vector toVector(INDArray arr) { + if (!arr.isVector()) { + throw new IllegalArgumentException("passed in array must be a vector"); + } + if (arr.length() > Integer.MAX_VALUE) + throw new ND4JArraySizeException(); + double[] ret = new double[(int) arr.length()]; + for (int i = 0; i < arr.length(); i++) { + ret[i] = arr.getDouble(i); + } + + return Vectors.dense(ret); + } + + + /** + * Convert a traditional sc.binaryFiles + * in to something usable for machine learning + * @param binaryFiles the binary files to convert + * @param reader the reader to use + * @return the labeled points based on the given rdd + */ + public static JavaRDD fromBinary(JavaPairRDD binaryFiles, + final RecordReader reader) { + JavaRDD> records = + binaryFiles.map(new Function, Collection>() { + @Override + public Collection call( + Tuple2 stringPortableDataStreamTuple2) + throws Exception { + reader.initialize(new InputStreamInputSplit(stringPortableDataStreamTuple2._2().open(), + stringPortableDataStreamTuple2._1())); + return reader.next(); + } + }); + + JavaRDD ret = records.map(new Function, LabeledPoint>() { + @Override + public LabeledPoint call(Collection writables) throws Exception { + return pointOf(writables); + } + }); + return ret; + } + + /** + * Convert a traditional sc.binaryFiles + * in to something usable for machine learning + * @param binaryFiles the binary files to convert + * @param reader the reader to use + * @return the labeled points based on the given rdd + */ + public static JavaRDD fromBinary(JavaRDD> binaryFiles, + final RecordReader reader) { + return fromBinary(JavaPairRDD.fromJavaRDD(binaryFiles), reader); + } + + + /** + * Returns a labeled point of the writables + * where the final item is the point and the rest of the items are + * features + * @param writables the writables + * @return the labeled point + */ + public static LabeledPoint pointOf(Collection writables) { + double[] ret = new double[writables.size() - 1]; + int count = 0; + double target = 0; + for (Writable w : writables) { + if (count < writables.size() - 1) + ret[count++] = Float.parseFloat(w.toString()); + else + target = Float.parseFloat(w.toString()); + } + + if (target < 0) + throw new IllegalStateException("Target must be >= 0"); + return new LabeledPoint(target, Vectors.dense(ret)); + } + + /** + * Convert an rdd + * of labeled point + * based on the specified batch size + * in to data set + * @param data the data to convert + * @param numPossibleLabels the number of possible labels + * @param batchSize the batch size + * @return the new rdd + */ + public static JavaRDD fromLabeledPoint(JavaRDD data, final long numPossibleLabels, + long batchSize) { + + JavaRDD mappedData = data.map(new Function() { + @Override + public DataSet call(LabeledPoint lp) { + return fromLabeledPoint(lp, numPossibleLabels); + } + }); + + return mappedData.repartition((int) (mappedData.count() / batchSize)); + } + + /** + * From labeled point + * @param sc the org.deeplearning4j.spark context used for creating the rdd + * @param data the data to convert + * @param numPossibleLabels the number of possible labels + * @return + * @deprecated Use {@link #fromLabeledPoint(JavaRDD, int)} + */ + @Deprecated + public static JavaRDD fromLabeledPoint(JavaSparkContext sc, JavaRDD data, + final long numPossibleLabels) { + return data.map(new Function() { + @Override + public DataSet call(LabeledPoint lp) { + return fromLabeledPoint(lp, numPossibleLabels); + } + }); + } + + /** + * Convert rdd labeled points to a rdd dataset with continuous features + * @param data the java rdd labeled points ready to convert + * @return a JavaRDD with a continuous label + * @deprecated Use {@link #fromContinuousLabeledPoint(JavaRDD)} + */ + @Deprecated + public static JavaRDD fromContinuousLabeledPoint(JavaSparkContext sc, JavaRDD data) { + + return data.map(new Function() { + @Override + public DataSet call(LabeledPoint lp) { + return convertToDataset(lp); + } + }); + } + + private static DataSet convertToDataset(LabeledPoint lp) { + Vector features = lp.features(); + double label = lp.label(); + return new DataSet(Nd4j.create(features.toArray()), Nd4j.create(new double[] {label})); + } + + /** + * Convert an rdd of data set in to labeled point + * @param sc the spark context to use + * @param data the dataset to convert + * @return an rdd of labeled point + * @deprecated Use {@link #fromDataSet(JavaRDD)} + * + */ + @Deprecated + public static JavaRDD fromDataSet(JavaSparkContext sc, JavaRDD data) { + + return data.map(new Function() { + @Override + public LabeledPoint call(DataSet pt) { + return toLabeledPoint(pt); + } + }); + } + + /** + * Convert a list of dataset in to a list of labeled points + * @param labeledPoints the labeled points to convert + * @return the labeled point list + */ + private static List toLabeledPoint(List labeledPoints) { + List ret = new ArrayList<>(); + for (DataSet point : labeledPoints) { + ret.add(toLabeledPoint(point)); + } + return ret; + } + + /** + * Convert a dataset (feature vector) to a labeled point + * @param point the point to convert + * @return the labeled point derived from this dataset + */ + private static LabeledPoint toLabeledPoint(DataSet point) { + if (!point.getFeatures().isVector()) { + throw new IllegalArgumentException("Feature matrix must be a vector"); + } + + Vector features = toVector(point.getFeatures().dup()); + + double label = Nd4j.getBlasWrapper().iamax(point.getLabels()); + return new LabeledPoint(label, features); + } + + /** + * Converts a continuous JavaRDD LabeledPoint to a JavaRDD DataSet. + * @param data JavaRDD LabeledPoint + * @return JavaRdd DataSet + */ + public static JavaRDD fromContinuousLabeledPoint(JavaRDD data) { + return fromContinuousLabeledPoint(data, false); + } + + /** + * Converts a continuous JavaRDD LabeledPoint to a JavaRDD DataSet. + * @param data JavaRdd LabeledPoint + * @param preCache boolean pre-cache rdd before operation + * @return + */ + public static JavaRDD fromContinuousLabeledPoint(JavaRDD data, boolean preCache) { + if (preCache && !data.getStorageLevel().useMemory()) { + data.cache(); + } + return data.map(new Function() { + @Override + public DataSet call(LabeledPoint lp) { + return convertToDataset(lp); + } + }); + } + + /** + * Converts JavaRDD labeled points to JavaRDD datasets. + * @param data JavaRDD LabeledPoints + * @param numPossibleLabels number of possible labels + * @return + */ + public static JavaRDD fromLabeledPoint(JavaRDD data, final long numPossibleLabels) { + return fromLabeledPoint(data, numPossibleLabels, false); + } + + /** + * Converts JavaRDD labeled points to JavaRDD DataSets. + * @param data JavaRDD LabeledPoints + * @param numPossibleLabels number of possible labels + * @param preCache boolean pre-cache rdd before operation + * @return + */ + public static JavaRDD fromLabeledPoint(JavaRDD data, final long numPossibleLabels, + boolean preCache) { + if (preCache && !data.getStorageLevel().useMemory()) { + data.cache(); + } + return data.map(new Function() { + @Override + public DataSet call(LabeledPoint lp) { + return fromLabeledPoint(lp, numPossibleLabels); + } + }); + } + + /** + * Convert an rdd of data set in to labeled point. + * @param data the dataset to convert + * @return an rdd of labeled point + */ + public static JavaRDD fromDataSet(JavaRDD data) { + return fromDataSet(data, false); + } + + /** + * Convert an rdd of data set in to labeled point. + * @param data the dataset to convert + * @param preCache boolean pre-cache rdd before operation + * @return an rdd of labeled point + */ + public static JavaRDD fromDataSet(JavaRDD data, boolean preCache) { + if (preCache && !data.getStorageLevel().useMemory()) { + data.cache(); + } + return data.map(new Function() { + @Override + public LabeledPoint call(DataSet dataSet) { + return toLabeledPoint(dataSet); + } + }); + } + + + /** + * + * @param labeledPoints + * @param numPossibleLabels + * @return List of {@link DataSet} + */ + private static List fromLabeledPoint(List labeledPoints, long numPossibleLabels) { + List ret = new ArrayList<>(); + for (LabeledPoint point : labeledPoints) { + ret.add(fromLabeledPoint(point, numPossibleLabels)); + } + return ret; + } + + /** + * + * @param point + * @param numPossibleLabels + * @return {@link DataSet} + */ + private static DataSet fromLabeledPoint(LabeledPoint point, long numPossibleLabels) { + Vector features = point.features(); + double label = point.label(); + + // FIXMEL int cast + double[] fArr = features.toArray(); + return new DataSet(Nd4j.create(fArr, new long[]{1,fArr.length}), + FeatureUtil.toOutcomeVector((int) label, (int) numPossibleLabels)); + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/SparkDataUtils.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/SparkDataUtils.java new file mode 100644 index 000000000..eb919864b --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/SparkDataUtils.java @@ -0,0 +1,238 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util; + +import lombok.NonNull; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.VoidFunction; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.loader.impl.RecordReaderFileBatchLoader; +import org.nd4j.common.loader.FileBatch; + +import java.io.*; +import java.util.*; + +public class SparkDataUtils { + + private SparkDataUtils() { + } + + /** + * See {@link #createFileBatchesLocal(File, String[], boolean, File, int)}.
+ * The directory filtering (extensions arg) is null when calling this method. + */ + public static void createFileBatchesLocal(File inputDirectory, boolean recursive, File outputDirectory, int batchSize) throws IOException { + createFileBatchesLocal(inputDirectory, null, recursive, outputDirectory, batchSize); + } + + /** + * Create a number of {@link FileBatch} files from local files (in random order).
+ * Use cases: distributed training on compressed file formats such as images, that need to be loaded to a remote + * file storage system such as HDFS. Local files can be created using this method and then copied to HDFS for training.
+ * FileBatch is also compressed (zip file format) so space may be saved in some cases (such as CSV sequences) + * For example, if we were training with a minibatch size of 64 images, reading the raw images would result in 64 + * different disk reads (one for each file) - which could clearly be a bottleneck during training.
+ * Alternatively, we could create and save DataSet/INDArray objects containing a batch of images - however, storing + * images in FP32 (or ever UINT8) format - effectively a bitmap - is still much less efficient than the raw image files.
+ * Instead, can create minibatches of {@link FileBatch} objects: these objects contain the raw file content for + * multiple files (as byte[]s) along with their original paths, which can then be used for distributed training using + * {@link RecordReaderFileBatchLoader}.
+ * This approach gives us the benefits of the original file format (i.e., small size, compression) along with + * the benefits of a batched DataSet/INDArray format - i.e., disk reads are reduced by a factor of the minibatch size.
+ *
+ * See {@link #createFileBatchesSpark(JavaRDD, String, int, JavaSparkContext)} for the distributed (Spark) version of this method.
+ *
+ * Usage - image classification example - assume each FileBatch object contains a number of jpg/png etc image files + * 
+     * {@code
+     * JavaSparkContext sc = ...
+     * SparkDl4jMultiLayer net = ...
+     * String baseFileBatchDir = ...
+     * JavaRDD paths = org.deeplearning4j.spark.util.SparkUtils.listPaths(sc, baseFileBatchDir);
+     *
+     * //Image record reader:
+     * PathLabelGenerator labelMaker = new ParentPathLabelGenerator();
+     * ImageRecordReader rr = new ImageRecordReader(32, 32, 1, labelMaker);
+     * rr.setLabels();
+     *
+     * //Create DataSetLoader:
+     * int batchSize = 32;
+     * int numClasses = 1000;
+     * DataSetLoader loader = RecordReaderFileBatchLoader(rr, batchSize, 1, numClasses);
+     *
+     * //Fit the network
+     * net.fitPaths(paths, loader);
+     * }
+     *
+ * + * @param inputDirectory Directory containing the files to convert + * @param extensions Optional (may be null). If non-null, only those files with the specified extension will be included + * @param recursive If true: convert the files recursively + * @param outputDirectory Output directory to save the created FileBatch objects + * @param batchSize Batch size - i.e., minibatch size to be used for training, and the number of files to + * include in each FileBatch object + * @throws IOException If an error occurs while reading the files + * @see #createFileBatchesSpark(JavaRDD, String, int, JavaSparkContext) + * @see org.datavec.api.records.reader.impl.filebatch.FileBatchRecordReader FileBatchRecordReader for local training on these files, if required + * @see org.datavec.api.records.reader.impl.filebatch.FileBatchSequenceRecordReader for local training on these files, if required + */ + public static void createFileBatchesLocal(File inputDirectory, String[] extensions, boolean recursive, File outputDirectory, int batchSize) throws IOException { + if(!outputDirectory.exists()) + outputDirectory.mkdirs(); + //Local version + List c = new ArrayList<>(FileUtils.listFiles(inputDirectory, extensions, recursive)); + Collections.shuffle(c); + + //Construct file batch + List list = new ArrayList<>(); + List bytes = new ArrayList<>(); + for (int i = 0; i < c.size(); i++) { + list.add(c.get(i).toURI().toString()); + bytes.add(FileUtils.readFileToByteArray(c.get(i))); + + if (list.size() == batchSize) { + process(list, bytes, outputDirectory); + } + } + if (list.size() > 0) { + process(list, bytes, outputDirectory); + } + } + + private static void process(List paths, List bytes, File outputDirectory) throws IOException { + FileBatch fb = new FileBatch(bytes, paths); + String name = UUID.randomUUID().toString().replaceAll("-", "") + ".zip"; + File f = new File(outputDirectory, name); + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f))) { + fb.writeAsZip(bos); + } + + paths.clear(); + bytes.clear(); + } + + /** + * Create a number of {@link FileBatch} files from files on network storage such as HDFS (in random order).
+ * Use cases: distributed training on compressed file formats such as images, that need to be loaded to a remote + * file storage system such as HDFS.
+ * For example, if we were training with a minibatch size of 64 images, reading the raw images would result in 64 + * different disk reads (one for each file) - which could clearly be a bottleneck during training.
+ * Alternatively, we could create and save DataSet/INDArray objects containing a batch of images - however, storing + * images in FP32 (or ever UINT8) format - effectively a bitmap - is still much less efficient than the raw image files.
+ * Instead, can create minibatches of {@link FileBatch} objects: these objects contain the raw file content for + * multiple files (as byte[]s) along with their original paths, which can then be used for distributed training using + * {@link RecordReaderFileBatchLoader}.
+ * This approach gives us the benefits of the original file format (i.e., small size, compression) along with + * the benefits of a batched DataSet/INDArray format - i.e., disk reads are reduced by a factor of the minibatch size.
+ *
+ * See {@link #createFileBatchesLocal(File, String[], boolean, File, int)} for the local (non-Spark) version of this method. + *
+ * Usage - image classification example - assume each FileBatch object contains a number of jpg/png etc image files + * 
+     * {@code
+     * JavaSparkContext sc = ...
+     * SparkDl4jMultiLayer net = ...
+     * String baseFileBatchDir = ...
+     * JavaRDD paths = org.deeplearning4j.spark.util.SparkUtils.listPaths(sc, baseFileBatchDir);
+     *
+     * //Image record reader:
+     * PathLabelGenerator labelMaker = new ParentPathLabelGenerator();
+     * ImageRecordReader rr = new ImageRecordReader(32, 32, 1, labelMaker);
+     * rr.setLabels();
+     *
+     * //Create DataSetLoader:
+     * int batchSize = 32;
+     * int numClasses = 1000;
+     * DataSetLoader loader = RecordReaderFileBatchLoader(rr, batchSize, 1, numClasses);
+     *
+     * //Fit the network
+     * net.fitPaths(paths, loader);
+     * }
+     *
+ * + * @param batchSize Batch size - i.e., minibatch size to be used for training, and the number of files to + * include in each FileBatch object + * @throws IOException If an error occurs while reading the files + * @see #createFileBatchesLocal(File, String[], boolean, File, int) + * @see org.datavec.api.records.reader.impl.filebatch.FileBatchRecordReader FileBatchRecordReader for local training on these files, if required + * @see org.datavec.api.records.reader.impl.filebatch.FileBatchSequenceRecordReader for local training on these files, if required + */ + public static void createFileBatchesSpark(JavaRDD filePaths, final String rootOutputDir, final int batchSize, JavaSparkContext sc) { + createFileBatchesSpark(filePaths, rootOutputDir, batchSize, sc.hadoopConfiguration()); + } + + /** + * See {@link #createFileBatchesSpark(JavaRDD, String, int, JavaSparkContext)} + */ + public static void createFileBatchesSpark(JavaRDD filePaths, final String rootOutputDir, final int batchSize, @NonNull final org.apache.hadoop.conf.Configuration hadoopConfig) { + final SerializableHadoopConfig conf = new SerializableHadoopConfig(hadoopConfig); + //Here: assume input is images. We can't store them as Float32 arrays - that's too inefficient + // instead: let's store the raw file content in a batch. + long count = filePaths.count(); + long maxPartitions = count / batchSize; + JavaRDD repartitioned = filePaths.repartition(Math.max(filePaths.getNumPartitions(), (int) maxPartitions)); + repartitioned.foreachPartition(new VoidFunction>() { + @Override + public void call(Iterator stringIterator) throws Exception { + //Construct file batch + List list = new ArrayList<>(); + List bytes = new ArrayList<>(); + FileSystem fs = FileSystem.get(conf.getConfiguration()); + while (stringIterator.hasNext()) { + String inFile = stringIterator.next(); + byte[] fileBytes; + try (BufferedInputStream bis = new BufferedInputStream(fs.open(new Path(inFile)))) { + fileBytes = IOUtils.toByteArray(bis); + } + list.add(inFile); + bytes.add(fileBytes); + + if (list.size() == batchSize) { + process(list, bytes); + } + } + if (list.size() > 0) { + process(list, bytes); + } + } + + private void process(List paths, List bytes) throws IOException { + FileBatch fb = new FileBatch(bytes, paths); + String name = UUID.randomUUID().toString().replaceAll("-", "") + ".zip"; + String outPath = FilenameUtils.concat(rootOutputDir, name); + FileSystem fileSystem = FileSystem.get(conf.getConfiguration()); + try (BufferedOutputStream bos = new BufferedOutputStream(fileSystem.create(new Path(outPath)))) { + fb.writeAsZip(bos); + } + + paths.clear(); + bytes.clear(); + } + }); + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/SparkUtils.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/SparkUtils.java new file mode 100644 index 000000000..6e88fbfa8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/SparkUtils.java @@ -0,0 +1,669 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.spark.HashPartitioner; +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.api.java.function.Function2; +import org.apache.spark.api.java.function.PairFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.serializer.SerializerInstance; +import org.deeplearning4j.spark.api.Repartition; +import org.deeplearning4j.spark.api.RepartitionStrategy; +import org.deeplearning4j.spark.data.BatchDataSetsFunction; +import org.deeplearning4j.spark.data.shuffle.SplitDataSetExamplesPairFlatMapFunction; +import org.deeplearning4j.spark.impl.common.CountPartitionsFunction; +import org.deeplearning4j.spark.impl.common.SplitPartitionsFunction; +import org.deeplearning4j.spark.impl.common.SplitPartitionsFunction2; +import org.deeplearning4j.spark.impl.common.repartition.BalancedPartitioner; +import org.deeplearning4j.spark.impl.common.repartition.HashingBalancedPartitioner; +import org.deeplearning4j.spark.impl.common.repartition.MapTupleToPairFlatMap; +import org.deeplearning4j.spark.impl.repartitioner.EqualRepartitioner; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.slf4j.Logger; +import scala.Tuple2; + +import java.io.*; +import java.lang.reflect.Array; +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.*; + +@Slf4j +public class SparkUtils { + + private static final String KRYO_EXCEPTION_MSG = "Kryo serialization detected without an appropriate registrator " + + "for ND4J INDArrays.\nWhen using Kryo, An appropriate Kryo registrator must be used to avoid" + + " serialization issues (NullPointerException) with off-heap data in INDArrays.\n" + + "Use nd4j-kryo_2.10 or _2.11 artifact, with sparkConf.set(\"spark.kryo.registrator\", \"org.nd4j.kryo.Nd4jRegistrator\");\n" + + "See https://deeplearning4j.konduit.ai/distributed-deep-learning/howto#how-to-use-kryo-serialization-with-dl-4-j-and-nd-4-j for more details"; + + private static String sparkExecutorId; + + private SparkUtils() {} + + /** + * Check the spark configuration for incorrect Kryo configuration, logging a warning message if necessary + * + * @param javaSparkContext Spark context + * @param log Logger to log messages to + * @return True if ok (no kryo, or correct kryo setup) + */ + public static boolean checkKryoConfiguration(JavaSparkContext javaSparkContext, Logger log) { + //Check if kryo configuration is correct: + String serializer = javaSparkContext.getConf().get("spark.serializer", null); + if (serializer != null && serializer.equals("org.apache.spark.serializer.KryoSerializer")) { + String kryoRegistrator = javaSparkContext.getConf().get("spark.kryo.registrator", null); + if (kryoRegistrator == null || !kryoRegistrator.equals("org.nd4j.kryo.Nd4jRegistrator")) { + + //It's probably going to fail later due to Kryo failing on the INDArray deserialization (off-heap data) + //But: the user might be using a custom Kryo registrator that can handle ND4J INDArrays, even if they + // aren't using the official ND4J-provided one + //Either way: Let's test serialization now of INDArrays now, and fail early if necessary + SerializerInstance si; + ByteBuffer bb; + try { + si = javaSparkContext.env().serializer().newInstance(); + bb = si.serialize(Nd4j.linspace(1, 5, 5), null); + } catch (Exception e) { + //Failed for some unknown reason during serialization - should never happen + throw new RuntimeException(KRYO_EXCEPTION_MSG, e); + } + + if (bb == null) { + //Should probably never happen + throw new RuntimeException( + KRYO_EXCEPTION_MSG + "\n(Got: null ByteBuffer from Spark SerializerInstance)"); + } else { + //Could serialize successfully, but still may not be able to deserialize if kryo config is wrong + boolean equals; + INDArray deserialized; + try { + deserialized = (INDArray) si.deserialize(bb, null); + //Equals method may fail on malformed INDArrays, hence should be within the try-catch + equals = Nd4j.linspace(1, 5, 5).equals(deserialized); + } catch (Exception e) { + throw new RuntimeException(KRYO_EXCEPTION_MSG, e); + } + if (!equals) { + throw new RuntimeException(KRYO_EXCEPTION_MSG + "\n(Error during deserialization: test array" + + " was not deserialized successfully)"); + } + + //Otherwise: serialization/deserialization was successful using Kryo + return true; + } + } + } + return true; + } + + /** + * Write a String to a file (on HDFS or local) in UTF-8 format + * + * @param path Path to write to + * @param toWrite String to write + * @param sc Spark context + */ + public static void writeStringToFile(String path, String toWrite, JavaSparkContext sc) throws IOException { + writeStringToFile(path, toWrite, sc.sc()); + } + + /** + * Write a String to a file (on HDFS or local) in UTF-8 format + * + * @param path Path to write to + * @param toWrite String to write + * @param sc Spark context + */ + public static void writeStringToFile(String path, String toWrite, SparkContext sc) throws IOException { + FileSystem fileSystem = FileSystem.get(sc.hadoopConfiguration()); + try (BufferedOutputStream bos = new BufferedOutputStream(fileSystem.create(new Path(path)))) { + bos.write(toWrite.getBytes("UTF-8")); + } + } + + /** + * Read a UTF-8 format String from HDFS (or local) + * + * @param path Path to write the string + * @param sc Spark context + */ + public static String readStringFromFile(String path, JavaSparkContext sc) throws IOException { + return readStringFromFile(path, sc.sc()); + } + + /** + * Read a UTF-8 format String from HDFS (or local) + * + * @param path Path to write the string + * @param sc Spark context + */ + public static String readStringFromFile(String path, SparkContext sc) throws IOException { + FileSystem fileSystem = FileSystem.get(sc.hadoopConfiguration()); + try (BufferedInputStream bis = new BufferedInputStream(fileSystem.open(new Path(path)))) { + byte[] asBytes = IOUtils.toByteArray(bis); + return new String(asBytes, "UTF-8"); + } + } + + /** + * Write an object to HDFS (or local) using default Java object serialization + * + * @param path Path to write the object to + * @param toWrite Object to write + * @param sc Spark context + */ + public static void writeObjectToFile(String path, Object toWrite, JavaSparkContext sc) throws IOException { + writeObjectToFile(path, toWrite, sc.sc()); + } + + /** + * Write an object to HDFS (or local) using default Java object serialization + * + * @param path Path to write the object to + * @param toWrite Object to write + * @param sc Spark context + */ + public static void writeObjectToFile(String path, Object toWrite, SparkContext sc) throws IOException { + FileSystem fileSystem = FileSystem.get(sc.hadoopConfiguration()); + try (BufferedOutputStream bos = new BufferedOutputStream(fileSystem.create(new Path(path)))) { + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(toWrite); + } + } + + /** + * Read an object from HDFS (or local) using default Java object serialization + * + * @param path File to read + * @param type Class of the object to read + * @param sc Spark context + * @param Type of the object to read + */ + public static T readObjectFromFile(String path, Class type, JavaSparkContext sc) throws IOException { + return readObjectFromFile(path, type, sc.sc()); + } + + /** + * Read an object from HDFS (or local) using default Java object serialization + * + * @param path File to read + * @param type Class of the object to read + * @param sc Spark context + * @param Type of the object to read + */ + public static T readObjectFromFile(String path, Class type, SparkContext sc) throws IOException { + FileSystem fileSystem = FileSystem.get(sc.hadoopConfiguration()); + try (ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(fileSystem.open(new Path(path))))) { + Object o; + try { + o = ois.readObject(); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + + return (T) o; + } + } + + /** + * Repartition the specified RDD (or not) using the given {@link Repartition} and {@link RepartitionStrategy} settings + * + * @param rdd RDD to repartition + * @param repartition Setting for when repartiting is to be conducted + * @param repartitionStrategy Setting for how repartitioning is to be conducted + * @param objectsPerPartition Desired number of objects per partition + * @param numPartitions Total number of partitions + * @param Type of the RDD + * @return Repartitioned RDD, or original RDD if no repartitioning was conducted + */ + public static JavaRDD repartition(JavaRDD rdd, Repartition repartition, + RepartitionStrategy repartitionStrategy, int objectsPerPartition, int numPartitions) { + if (repartition == Repartition.Never) + return rdd; + + switch (repartitionStrategy) { + case SparkDefault: + if (repartition == Repartition.NumPartitionsWorkersDiffers && rdd.partitions().size() == numPartitions) + return rdd; + + //Either repartition always, or workers/num partitions differs + return rdd.repartition(numPartitions); + case Balanced: + return repartitionBalanceIfRequired(rdd, repartition, objectsPerPartition, numPartitions); + case ApproximateBalanced: + return repartitionApproximateBalance(rdd, repartition, numPartitions); + default: + throw new RuntimeException("Unknown repartition strategy: " + repartitionStrategy); + } + } + + public static JavaRDD repartitionApproximateBalance(JavaRDD rdd, Repartition repartition, + int numPartitions) { + int origNumPartitions = rdd.partitions().size(); + switch (repartition) { + case Never: + return rdd; + case NumPartitionsWorkersDiffers: + if (origNumPartitions == numPartitions) + return rdd; + case Always: + // Count each partition... + List partitionCounts = + rdd.mapPartitionsWithIndex(new Function2, Iterator>() { + @Override + public Iterator call(Integer integer, Iterator tIterator) + throws Exception { + int count = 0; + while (tIterator.hasNext()) { + tIterator.next(); + count++; + } + return Collections.singletonList(count).iterator(); + } + }, true).collect(); + + Integer totalCount = 0; + for (Integer i : partitionCounts) + totalCount += i; + List partitionWeights = new ArrayList<>(Math.max(numPartitions, origNumPartitions)); + Double ideal = (double) totalCount / numPartitions; + // partitions in the initial set and not in the final one get -1 => elements always jump + // partitions in the final set not in the initial one get 0 => aim to receive the average amount + for (int i = 0; i < Math.min(origNumPartitions, numPartitions); i++) { + partitionWeights.add((double) partitionCounts.get(i) / ideal); + } + for (int i = Math.min(origNumPartitions, numPartitions); i < Math.max(origNumPartitions, + numPartitions); i++) { + // we shrink the # of partitions + if (i >= numPartitions) + partitionWeights.add(-1D); + // we enlarge the # of partitions + else + partitionWeights.add(0D); + } + + // this method won't trigger a spark job, which is different from {@link org.apache.spark.rdd.RDD#zipWithIndex} + + JavaPairRDD, T> indexedRDD = rdd.zipWithUniqueId() + .mapToPair(new PairFunction, Tuple2, T>() { + @Override + public Tuple2, T> call(Tuple2 tLongTuple2) { + return new Tuple2<>( + new Tuple2(tLongTuple2._2(), 0), + tLongTuple2._1()); + } + }); + + HashingBalancedPartitioner hbp = + new HashingBalancedPartitioner(Collections.singletonList(partitionWeights)); + JavaPairRDD, T> partitionedRDD = indexedRDD.partitionBy(hbp); + + return partitionedRDD.map(new Function, T>, T>() { + @Override + public T call(Tuple2, T> indexNPayload) { + return indexNPayload._2(); + } + }); + default: + throw new RuntimeException("Unknown setting for repartition: " + repartition); + } + } + + /** + * Repartition a RDD (given the {@link Repartition} setting) such that we have approximately + * {@code numPartitions} partitions, each of which has {@code objectsPerPartition} objects. + * + * @param rdd RDD to repartition + * @param repartition Repartitioning setting + * @param objectsPerPartition Number of objects we want in each partition + * @param numPartitions Number of partitions to have + * @param Type of RDD + * @return Repartitioned RDD, or the original RDD if no repartitioning was performed + */ + public static JavaRDD repartitionBalanceIfRequired(JavaRDD rdd, Repartition repartition, + int objectsPerPartition, int numPartitions) { + int origNumPartitions = rdd.partitions().size(); + switch (repartition) { + case Never: + return rdd; + case NumPartitionsWorkersDiffers: + if (origNumPartitions == numPartitions) + return rdd; + case Always: + //Repartition: either always, or origNumPartitions != numWorkers + + //First: count number of elements in each partition. Need to know this so we can work out how to properly index each example, + // so we can in turn create properly balanced partitions after repartitioning + //Because the objects (DataSets etc) should be small, this should be OK + + //Count each partition... + List> partitionCounts = + rdd.mapPartitionsWithIndex(new CountPartitionsFunction(), true).collect(); + int totalObjects = 0; + int initialPartitions = partitionCounts.size(); + + boolean allCorrectSize = true; + int x = 0; + for (Tuple2 t2 : partitionCounts) { + int partitionSize = t2._2(); + allCorrectSize &= (partitionSize == objectsPerPartition); + totalObjects += t2._2(); + } + + if (numPartitions * objectsPerPartition < totalObjects) { + allCorrectSize = true; + for (Tuple2 t2 : partitionCounts) { + allCorrectSize &= (t2._2() == objectsPerPartition); + } + } + + if (initialPartitions == numPartitions && allCorrectSize) { + //Don't need to do any repartitioning here - already in the format we want + return rdd; + } + + //Index each element for repartitioning (can only do manual repartitioning on a JavaPairRDD) + JavaPairRDD pairIndexed = indexedRDD(rdd); + + int remainder = (totalObjects - numPartitions * objectsPerPartition) % numPartitions; + log.trace("About to rebalance: numPartitions={}, objectsPerPartition={}, remainder={}", numPartitions, objectsPerPartition, remainder); + pairIndexed = pairIndexed + .partitionBy(new BalancedPartitioner(numPartitions, objectsPerPartition, remainder)); + return pairIndexed.values(); + default: + throw new RuntimeException("Unknown setting for repartition: " + repartition); + } + } + + public static JavaPairRDD indexedRDD(JavaRDD rdd) { + return rdd.zipWithIndex().mapToPair(new PairFunction, Integer, T>() { + @Override + public Tuple2 call(Tuple2 elemIdx) { + return new Tuple2<>(elemIdx._2().intValue(), elemIdx._1()); + } + }); + } + + public static JavaRDD repartitionEqually(JavaRDD rdd, Repartition repartition, int numPartitions){ + int origNumPartitions = rdd.partitions().size(); + switch (repartition) { + case Never: + return rdd; + case NumPartitionsWorkersDiffers: + if (origNumPartitions == numPartitions) + return rdd; + case Always: + return new EqualRepartitioner().repartition(rdd, -1, numPartitions); + default: + throw new RuntimeException("Unknown setting for repartition: " + repartition); + } + } + + /** + * Random split the specified RDD into a number of RDDs, where each has {@code numObjectsPerSplit} in them. + *

+ * This similar to how RDD.randomSplit works (i.e., split via filtering), but this should result in more + * equal splits (instead of independent binomial sampling that is used there, based on weighting) + * This balanced splitting approach is important when the number of DataSet objects we want in each split is small, + * as random sampling variance of {@link JavaRDD#randomSplit(double[])} is quite large relative to the number of examples + * in each split. Note however that this method doesn't guarantee that partitions will be balanced + *

+ * Downside is we need total object count (whereas {@link JavaRDD#randomSplit(double[])} does not). However, randomSplit + * requires a full pass of the data anyway (in order to do filtering upon it) so this should not add much overhead in practice + * + * @param totalObjectCount Total number of objects in the RDD to split + * @param numObjectsPerSplit Number of objects in each split + * @param data Data to split + * @param Generic type for the RDD + * @return The RDD split up (without replacement) into a number of smaller RDDs + */ + public static JavaRDD[] balancedRandomSplit(int totalObjectCount, int numObjectsPerSplit, JavaRDD data) { + return balancedRandomSplit(totalObjectCount, numObjectsPerSplit, data, new Random().nextLong()); + } + + /** + * Equivalent to {@link #balancedRandomSplit(int, int, JavaRDD)} with control over the RNG seed + */ + public static JavaRDD[] balancedRandomSplit(int totalObjectCount, int numObjectsPerSplit, JavaRDD data, + long rngSeed) { + JavaRDD[] splits; + if (totalObjectCount <= numObjectsPerSplit) { + splits = (JavaRDD[]) Array.newInstance(JavaRDD.class, 1); + splits[0] = data; + } else { + int numSplits = totalObjectCount / numObjectsPerSplit; //Intentional round down + splits = (JavaRDD[]) Array.newInstance(JavaRDD.class, numSplits); + for (int i = 0; i < numSplits; i++) { + splits[i] = data.mapPartitionsWithIndex(new SplitPartitionsFunction(i, numSplits, rngSeed), true); + } + + } + return splits; + } + + /** + * Equivalent to {@link #balancedRandomSplit(int, int, JavaRDD)} but for Pair RDDs + */ + public static JavaPairRDD[] balancedRandomSplit(int totalObjectCount, int numObjectsPerSplit, + JavaPairRDD data) { + return balancedRandomSplit(totalObjectCount, numObjectsPerSplit, data, new Random().nextLong()); + } + + /** + * Equivalent to {@link #balancedRandomSplit(int, int, JavaRDD)} but for pair RDDs, and with control over the RNG seed + */ + public static JavaPairRDD[] balancedRandomSplit(int totalObjectCount, int numObjectsPerSplit, + JavaPairRDD data, long rngSeed) { + JavaPairRDD[] splits; + if (totalObjectCount <= numObjectsPerSplit) { + splits = (JavaPairRDD[]) Array.newInstance(JavaPairRDD.class, 1); + splits[0] = data; + } else { + int numSplits = totalObjectCount / numObjectsPerSplit; //Intentional round down + + splits = (JavaPairRDD[]) Array.newInstance(JavaPairRDD.class, numSplits); + for (int i = 0; i < numSplits; i++) { + + //What we really need is a .mapPartitionsToPairWithIndex function + //but, of course Spark doesn't provide this + //So we need to do a two-step process here... + + JavaRDD> split = data.mapPartitionsWithIndex( + new SplitPartitionsFunction2(i, numSplits, rngSeed), true); + splits[i] = split.mapPartitionsToPair(new MapTupleToPairFlatMap(), true); + } + } + return splits; + } + + /** + * List of the files in the given directory (path), as a {@code JavaRDD} + * + * @param sc Spark context + * @param path Path to list files in + * @return Paths in the directory + * @throws IOException If error occurs getting directory contents + */ + public static JavaRDD listPaths(JavaSparkContext sc, String path) throws IOException { + return listPaths(sc, path, false); + } + + /** + * List of the files in the given directory (path), as a {@code JavaRDD} + * + * @param sc Spark context + * @param path Path to list files in + * @param recursive Whether to walk the directory tree recursively (i.e., include subdirectories) + * @return Paths in the directory + * @throws IOException If error occurs getting directory contents + */ + public static JavaRDD listPaths(JavaSparkContext sc, String path, boolean recursive) throws IOException { + //NativeImageLoader.ALLOWED_FORMATS + return listPaths(sc, path, recursive, (Set)null); + } + + /** + * List of the files in the given directory (path), as a {@code JavaRDD} + * + * @param sc Spark context + * @param path Path to list files in + * @param recursive Whether to walk the directory tree recursively (i.e., include subdirectories) + * @param allowedExtensions If null: all files will be accepted. If non-null: only files with the specified extension will be allowed. + * Exclude the extension separator - i.e., use "txt" not ".txt" here. + * @return Paths in the directory + * @throws IOException If error occurs getting directory contents + */ + public static JavaRDD listPaths(JavaSparkContext sc, String path, boolean recursive, String[] allowedExtensions) throws IOException { + return listPaths(sc, path, recursive, (allowedExtensions == null ? null : new HashSet<>(Arrays.asList(allowedExtensions)))); + } + + /** + * List of the files in the given directory (path), as a {@code JavaRDD} + * + * @param sc Spark context + * @param path Path to list files in + * @param recursive Whether to walk the directory tree recursively (i.e., include subdirectories) + * @param allowedExtensions If null: all files will be accepted. If non-null: only files with the specified extension will be allowed. + * Exclude the extension separator - i.e., use "txt" not ".txt" here. + * @return Paths in the directory + * @throws IOException If error occurs getting directory contents + */ + public static JavaRDD listPaths(JavaSparkContext sc, String path, boolean recursive, Set allowedExtensions) throws IOException { + return listPaths(sc, path, recursive, allowedExtensions, sc.hadoopConfiguration()); + } + + /** + * List of the files in the given directory (path), as a {@code JavaRDD} + * + * @param sc Spark context + * @param path Path to list files in + * @param recursive Whether to walk the directory tree recursively (i.e., include subdirectories) + * @param allowedExtensions If null: all files will be accepted. If non-null: only files with the specified extension will be allowed. + * Exclude the extension separator - i.e., use "txt" not ".txt" here. + * @param config Hadoop configuration to use. Must not be null. + * @return Paths in the directory + * @throws IOException If error occurs getting directory contents + */ + public static JavaRDD listPaths(@NonNull JavaSparkContext sc, String path, boolean recursive, + Set allowedExtensions, @NonNull Configuration config) throws IOException { + List paths = new ArrayList<>(); + FileSystem hdfs = FileSystem.get(URI.create(path), config); + RemoteIterator fileIter = hdfs.listFiles(new org.apache.hadoop.fs.Path(path), recursive); + + while (fileIter.hasNext()) { + String filePath = fileIter.next().getPath().toString(); + if(allowedExtensions == null){ + paths.add(filePath); + } else { + String ext = FilenameUtils.getExtension(path); + if(allowedExtensions.contains(ext)){ + paths.add(filePath); + } + } + } + return sc.parallelize(paths); + } + + + /** + * Randomly shuffle the examples in each DataSet object, and recombine them into new DataSet objects + * with the specified BatchSize + * + * @param rdd DataSets to shuffle/recombine + * @param newBatchSize New batch size for the DataSet objects, after shuffling/recombining + * @param numPartitions Number of partitions to use when splitting/recombining + * @return A new {@link JavaRDD}, with the examples shuffled/combined in each + */ + public static JavaRDD shuffleExamples(JavaRDD rdd, int newBatchSize, int numPartitions) { + //Step 1: split into individual examples, mapping to a pair RDD (random key in range 0 to numPartitions) + + JavaPairRDD singleExampleDataSets = + rdd.flatMapToPair(new SplitDataSetExamplesPairFlatMapFunction(numPartitions)); + + //Step 2: repartition according to the random keys + singleExampleDataSets = singleExampleDataSets.partitionBy(new HashPartitioner(numPartitions)); + + //Step 3: Recombine + return singleExampleDataSets.values().mapPartitions(new BatchDataSetsFunction(newBatchSize)); + } + + /** + * Get the Spark executor ID
+ * The ID is parsed from the JVM launch args. If that is not specified (or can't be obtained) then the value + * from {@link UIDProvider#getJVMUID()} is returned + * @return + */ + public static String getSparkExecutorId(){ + if(sparkExecutorId != null) + return sparkExecutorId; + + synchronized (SparkUtils.class){ + //re-check, in case some other thread set it while waiting for lock + if(sparkExecutorId != null) + return sparkExecutorId; + + String s = System.getProperty("sun.java.command"); + if(s == null || s.isEmpty() || !s.contains("executor-id")){ + sparkExecutorId = UIDProvider.getJVMUID(); + return sparkExecutorId; + } + + int idx = s.indexOf("executor-id"); + String sub = s.substring(idx); + String[] split = sub.split(" "); + if(split.length < 2){ + sparkExecutorId = UIDProvider.getJVMUID(); + return sparkExecutorId; + } + sparkExecutorId = split[1]; + return sparkExecutorId; + } + } + + public static Broadcast asByteArrayBroadcast(JavaSparkContext sc, INDArray array){ + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + Nd4j.write(array, new DataOutputStream(baos)); + } catch (IOException e){ + throw new RuntimeException(e); //Should never happen + } + byte[] paramBytes = baos.toByteArray(); //See docs in EvaluationRunner for why we use byte[] instead of INDArray (thread locality etc) + return sc.broadcast(paramBytes); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/SparkDataValidation.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/SparkDataValidation.java new file mode 100644 index 000000000..33ccc5107 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/SparkDataValidation.java @@ -0,0 +1,253 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util.data; + +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.spark.util.SparkUtils; +import org.deeplearning4j.spark.util.data.validation.ValidateMultiDataSetFn; +import org.deeplearning4j.spark.util.data.validation.ValidationResultReduceFn; +import org.deeplearning4j.spark.util.data.validation.ValidateDataSetFn; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +public class SparkDataValidation { + + private SparkDataValidation() { + } + + /** + * Validate DataSet objects saved to the specified directory on HDFS by attempting to load them and checking their + * contents. Assumes DataSets were saved using {@link org.nd4j.linalg.dataset.DataSet#save(OutputStream)}.
+ * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @return Results of the validation + */ + public static ValidationResult validateDataSets(JavaSparkContext sc, String path) { + return validateDataSets(sc, path, true, false, null, null); + } + + /** + * Validate DataSet objects saved to the specified directory on HDFS by attempting to load them and checking their + * contents. Assumes DataSets were saved using {@link org.nd4j.linalg.dataset.DataSet#save(OutputStream)}.
+ * This method (optionally) additionally validates the arrays using the specified shapes for the features and labels. + * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @param featuresShape May be null. If non-null: feature arrays must match the specified shape, for all values with + * shape > 0. For example, if featuresShape = {-1,10} then the features must be rank 2, + * can have any size for the first dimension, but must have size 10 for the second dimension. + * @param labelsShape As per featuresShape, but for the labels instead + * @return Results of the validation + */ + public static ValidationResult validateDataSets(JavaSparkContext sc, String path, int[] featuresShape, int[] labelsShape) { + return validateDataSets(sc, path, true, false, featuresShape, labelsShape); + } + + /** + * Validate DataSet objects - and delete any invalid DataSets - that have been previously saved to the + * specified directory on HDFS by attempting to load them and checking their contents. Assumes DataSets were saved + * using {@link org.nd4j.linalg.dataset.DataSet#save(OutputStream)}.
+ * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @return Results of the validation/deletion + */ + public static ValidationResult deleteInvalidDataSets(JavaSparkContext sc, String path) { + return validateDataSets(sc, path, true, true, null, null); + } + + /** + * Validate DataSet objects - and delete any invalid DataSets - that have been previously saved to the + * specified directory on HDFS by attempting to load them and checking their contents. Assumes DataSets were saved + * using {@link org.nd4j.linalg.dataset.DataSet#save(OutputStream)}.
+ * This method (optionally) additionally validates the arrays using the specified shapes for the features and labels. + * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @param featuresShape May be null. If non-null: feature arrays must match the specified shape, for all values with + * shape > 0. For example, if featuresShape = {-1,10} then the features must be rank 2, + * can have any size for the first dimension, but must have size 10 for the second dimension. + * @param labelsShape As per featuresShape, but for the labels instead + * @return Results of the validation + */ + public static ValidationResult deleteInvalidDataSets(JavaSparkContext sc, String path, int[] featuresShape, int[] labelsShape) { + return validateDataSets(sc, path, true, true, featuresShape, labelsShape); + } + + + protected static ValidationResult validateDataSets(SparkContext sc, String path, boolean recursive, boolean deleteInvalid, + int[] featuresShape, int[] labelsShape) { + return validateDataSets(new JavaSparkContext(sc), path, recursive, deleteInvalid, featuresShape, labelsShape); + } + + protected static ValidationResult validateDataSets(JavaSparkContext sc, String path, boolean recursive, boolean deleteInvalid, + int[] featuresShape, int[] labelsShape) { + JavaRDD paths; + try { + paths = SparkUtils.listPaths(sc, path, recursive); + } catch (IOException e) { + throw new RuntimeException("Error listing paths in directory", e); + } + + JavaRDD results = paths.map(new ValidateDataSetFn(deleteInvalid, featuresShape, labelsShape)); + + return results.reduce(new ValidationResultReduceFn()); + } + + + /** + * Validate MultiDataSet objects saved to the specified directory on HDFS by attempting to load them and checking their + * contents. Assumes MultiDataSets were saved using {@link org.nd4j.linalg.dataset.MultiDataSet#save(OutputStream)}.
+ * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @return Results of the validation + */ + public static ValidationResult validateMultiDataSets(JavaSparkContext sc, String path) { + return validateMultiDataSets(sc, path, true, false, -1, -1, null, null); + } + + /** + * Validate MultiDataSet objects saved to the specified directory on HDFS by attempting to load them and checking their + * contents. Assumes MultiDataSets were saved using {@link org.nd4j.linalg.dataset.MultiDataSet#save(OutputStream)}.
+ * This method additionally validates that the expected number of feature/labels arrays are present in all MultiDataSet + * objects
+ * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @param numFeatureArrays Number of feature arrays that are expected for the MultiDataSet (set -1 to not check) + * @param numLabelArrays Number of labels arrays that are expected for the MultiDataSet (set -1 to not check) + * @return Results of the validation + */ + public static ValidationResult validateMultiDataSets(JavaSparkContext sc, String path, int numFeatureArrays, int numLabelArrays) { + return validateMultiDataSets(sc, path, true, false, numFeatureArrays, numLabelArrays, null, null); + } + + + /** + * Validate MultiDataSet objects saved to the specified directory on HDFS by attempting to load them and checking their + * contents. Assumes MultiDataSets were saved using {@link org.nd4j.linalg.dataset.MultiDataSet#save(OutputStream)}.
+ * This method (optionally) additionally validates the arrays using the specified shapes for the features and labels. + * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @param featuresShape May be null. If non-null: feature arrays must match the specified shapes, for all values with + * shape > 0. For example, if featuresShape = {{-1,10}} then there must be 1 features array, + * features array 0 must be rank 2, can have any size for the first dimension, but must have + * size 10 for the second dimension. + * @param labelsShape As per featuresShape, but for the labels instead + * @return Results of the validation + */ + public static ValidationResult validateMultiDataSets(JavaSparkContext sc, String path, List featuresShape, List labelsShape) { + return validateMultiDataSets(sc, path, true, false, (featuresShape == null ? -1 : featuresShape.size()), + (labelsShape == null ? -1 : labelsShape.size()), featuresShape, labelsShape); + } + + /** + * Validate MultiDataSet objects - and delete any invalid MultiDataSets - that have been previously saved to the + * specified directory on HDFS by attempting to load them and checking their contents. Assumes MultiDataSets were saved + * using {@link org.nd4j.linalg.dataset.MultiDataSet#save(OutputStream)}.
+ * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @return Results of the validation/deletion + */ + public static ValidationResult deleteInvalidMultiDataSets(JavaSparkContext sc, String path) { + return validateMultiDataSets(sc, path, true, true, -1, -1, null, null); + } + + /** + * Validate MultiDataSet objects - and delete any invalid MultiDataSets - that have been previously saved + * to the specified directory on HDFS by attempting to load them and checking their contents. Assumes MultiDataSets + * were saved using {@link org.nd4j.linalg.dataset.MultiDataSet#save(OutputStream)}.
+ * This method (optionally) additionally validates the arrays using the specified shapes for the features and labels, + * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @param featuresShape May be null. If non-null: feature arrays must match the specified shapes, for all values with + * shape > 0. For example, if featuresShape = {{-1,10}} then there must be 1 features array, + * features array 0 must be rank 2, can have any size for the first dimension, but must have + * size 10 for the second dimension. + * @param labelsShape As per featuresShape, but for the labels instead + * @return Results of the validation + */ + public static ValidationResult deleteInvalidMultiDataSets(JavaSparkContext sc, String path, List featuresShape, + List labelsShape) { + return validateMultiDataSets(sc, path, true, true, (featuresShape == null ? -1 : featuresShape.size()), + (labelsShape == null ? -1 : labelsShape.size()), featuresShape, labelsShape); + } + + /** + * Validate MultiDataSet objects - and delete any invalid MultiDataSets - that have been previously saved + * to the specified directory on HDFS by attempting to load them and checking their contents. Assumes MultiDataSets + * were saved using {@link org.nd4j.linalg.dataset.MultiDataSet#save(OutputStream)}.
+ * This method (optionally) additionally validates the arrays using the specified shapes for the features and labels. + * Note: this method will also consider all files in subdirectories (i.e., is recursive). + * + * @param sc Spark context + * @param path HDFS path of the directory containing the saved DataSet objects + * @param numFeatureArrays Number of feature arrays that are expected for the MultiDataSet (set -1 to not check) + * @param numLabelArrays Number of labels arrays that are expected for the MultiDataSet (set -1 to not check) + * @return Results of the validation + */ + public static ValidationResult deleteInvalidMultiDataSets(JavaSparkContext sc, String path, int numFeatureArrays, int numLabelArrays) { + return validateMultiDataSets(sc, path, true, true, numFeatureArrays, numLabelArrays, null, null); + } + + protected static ValidationResult validateMultiDataSets(SparkContext sc, String path, boolean recursive, boolean deleteInvalid, + int numFeatureArrays, int numLabelArrays, + List featuresShape, List labelsShape) { + return validateMultiDataSets(new JavaSparkContext(sc), path, recursive, deleteInvalid, numFeatureArrays, numLabelArrays, + featuresShape, labelsShape); + } + + protected static ValidationResult validateMultiDataSets(JavaSparkContext sc, String path, boolean recursive, boolean deleteInvalid, + int numFeatureArrays, int numLabelArrays, + List featuresShape, List labelsShape) { + JavaRDD paths; + try { + paths = SparkUtils.listPaths(sc, path, recursive); + } catch (IOException e) { + throw new RuntimeException("Error listing paths in directory", e); + } + + JavaRDD results = paths.map(new ValidateMultiDataSetFn(deleteInvalid, numFeatureArrays, numLabelArrays, + featuresShape, labelsShape)); + + return results.reduce(new ValidationResultReduceFn()); + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/ValidationResult.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/ValidationResult.java new file mode 100644 index 000000000..615a9ec86 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/ValidationResult.java @@ -0,0 +1,64 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@AllArgsConstructor +@NoArgsConstructor +@Data +@Builder +public class ValidationResult implements Serializable { + private long countTotal; + private long countMissingFile; + private long countTotalValid; + private long countTotalInvalid; + private long countLoadingFailure; + private long countMissingFeatures; + private long countMissingLabels; + private long countInvalidFeatures; + private long countInvalidLabels; + private long countInvalidDeleted; + + public ValidationResult add(ValidationResult o){ + if(o == null){ + return this; + } + + countTotal += o.countTotal; + countMissingFile += o.countMissingFile; + countTotalValid += o.countTotalValid; + countTotalInvalid += o.countTotalInvalid; + countLoadingFailure += o.countLoadingFailure; + countMissingFeatures += o.countMissingFeatures; + countMissingLabels += o.countMissingLabels; + countInvalidFeatures += o.countInvalidFeatures; + countInvalidLabels += o.countInvalidLabels; + countInvalidDeleted += o.countInvalidDeleted; + + return this; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/validation/ValidateDataSetFn.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/validation/ValidateDataSetFn.java new file mode 100644 index 000000000..d1797ac6b --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/data/validation/ValidateDataSetFn.java @@ -0,0 +1,155 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util.data.validation; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.DefaultHadoopConfig; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.spark.util.data.ValidationResult; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.EOFException; +import java.net.URI; + +public class ValidateDataSetFn implements Function { + public static final int BUFFER_SIZE = 4194304; //4 MB + + private final boolean deleteInvalid; + private final int[] featuresShape; + private final int[] labelsShape; + private final Broadcast conf; + private transient FileSystem fileSystem; + + public ValidateDataSetFn(boolean deleteInvalid, int[] featuresShape, int[] labelsShape) { + this(deleteInvalid, featuresShape, labelsShape, null); + } + + public ValidateDataSetFn(boolean deleteInvalid, int[] featuresShape, int[] labelsShape, Broadcast configuration) { + this.deleteInvalid = deleteInvalid; + this.featuresShape = featuresShape; + this.labelsShape = labelsShape; + this.conf = configuration; + } + + @Override + public ValidationResult call(String path) throws Exception { + if (fileSystem == null) { + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + try { + fileSystem = FileSystem.get(new URI(path), c); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + ValidationResult ret = new ValidationResult(); + ret.setCountTotal(1); + + boolean shouldDelete = false; + boolean loadSuccessful = false; + DataSet ds = new DataSet(); + Path p = new Path(path); + + if(fileSystem.isDirectory(p)){ + ret.setCountTotal(0); + return ret; + } + + if (!fileSystem.exists(p)) { + ret.setCountMissingFile(1); + return ret; + } + + try (FSDataInputStream inputStream = fileSystem.open(p, BUFFER_SIZE)) { + ds.load(inputStream); + loadSuccessful = true; + } catch (RuntimeException t) { + shouldDelete = deleteInvalid; + ret.setCountLoadingFailure(1); + } + + boolean isValid = loadSuccessful; + if (loadSuccessful) { + //Validate + if (ds.getFeatures() == null) { + ret.setCountMissingFeatures(1); + isValid = false; + } else { + if(featuresShape != null && !validateArrayShape(featuresShape, ds.getFeatures())){ + ret.setCountInvalidFeatures(1); + isValid = false; + } + } + + if(ds.getLabels() == null){ + ret.setCountMissingLabels(1); + isValid = false; + } else { + if(labelsShape != null && !validateArrayShape(labelsShape, ds.getLabels())){ + ret.setCountInvalidLabels(1); + isValid = false; + } + } + + if(!isValid && deleteInvalid){ + shouldDelete = true; + } + } + + if (isValid) { + ret.setCountTotalValid(1); + } else { + ret.setCountTotalInvalid(1); + } + + if (shouldDelete) { + fileSystem.delete(p, false); + ret.setCountInvalidDeleted(1); + } + + return ret; + } + + protected static boolean validateArrayShape(int[] featuresShape, INDArray array){ + if(featuresShape == null){ + return true; + } + + if(featuresShape.length != array.rank()){ + return false; + } else { + for( int i=0; i { + public static final int BUFFER_SIZE = 4194304; //4 MB + + private final boolean deleteInvalid; + private final int numFeatures; + private final int numLabels; + private final List featuresShape; + private final List labelsShape; + private final Broadcast conf; + private transient FileSystem fileSystem; + + public ValidateMultiDataSetFn(boolean deleteInvalid, int numFeatures, int numLabels, List featuresShape, List labelsShape) { + this(deleteInvalid, numFeatures, numLabels, featuresShape, labelsShape, null); + } + + public ValidateMultiDataSetFn(boolean deleteInvalid, int numFeatures, int numLabels, List featuresShape, List labelsShape, Broadcast configuration) { + this.deleteInvalid = deleteInvalid; + this.numFeatures = numFeatures; + this.numLabels = numLabels; + this.featuresShape = featuresShape; + this.labelsShape = labelsShape; + this.conf = configuration; + } + + @Override + public ValidationResult call(String path) throws Exception { + if (fileSystem == null) { + Configuration c = conf == null ? DefaultHadoopConfig.get() : conf.getValue().getConfiguration(); + try { + fileSystem = FileSystem.get(new URI(path), c); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + ValidationResult ret = new ValidationResult(); + ret.setCountTotal(1); + + boolean shouldDelete = false; + boolean loadSuccessful = false; + MultiDataSet ds = new MultiDataSet(); + Path p = new Path(path); + + if(fileSystem.isDirectory(p)){ + ret.setCountTotal(0); + return ret; + } + + if (!fileSystem.exists(p)) { + ret.setCountMissingFile(1); + return ret; + } + + try (FSDataInputStream inputStream = fileSystem.open(p, BUFFER_SIZE)) { + ds.load(inputStream); + loadSuccessful = true; + } catch (Throwable t) { + shouldDelete = deleteInvalid; + ret.setCountLoadingFailure(1); + } + + + boolean isValid = loadSuccessful; + if (loadSuccessful) { + //Validate + if (invalidArray(ds.getFeatures())) { + ret.setCountMissingFeatures(1); + isValid = false; + } else { + if(featuresShape != null && !validateArrayShapes(numFeatures, featuresShape, ds.getFeatures())){ + ret.setCountInvalidFeatures(1); + isValid = false; + } + } + + if(ds.getLabels() == null){ + ret.setCountMissingLabels(1); + isValid = false; + } else { + if(labelsShape != null && !validateArrayShapes(numLabels, labelsShape, ds.getLabels())){ + ret.setCountInvalidLabels(1); + isValid = false; + } + } + + if(!isValid && deleteInvalid){ + shouldDelete = true; + } + } + + if (isValid) { + ret.setCountTotalValid(1); + } else { + ret.setCountTotalInvalid(1); + } + + if (shouldDelete) { + fileSystem.delete(p, false); + ret.setCountInvalidDeleted(1); + } + + return ret; + } + + private static boolean invalidArray(INDArray[] array){ + if(array == null || array.length == 0) + return true; + for( int i=0; i shapes, INDArray[] arr){ + if(arr.length != numFeatures){ + return false; + } + + if(shapes == null) + return true; + if(shapes.size() != arr.length) + return false; + + for( int i=0; i { + @Override + public ValidationResult call(ValidationResult v1, ValidationResult v2) throws Exception { + return v1.add(v2); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelDeserializer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelDeserializer.java new file mode 100644 index 000000000..cc9490a9a --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelDeserializer.java @@ -0,0 +1,43 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util.serde; + +import org.apache.spark.storage.StorageLevel; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; + +public class StorageLevelDeserializer extends JsonDeserializer { + @Override + public StorageLevel deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException, JsonProcessingException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + String value = node.textValue(); + if (value == null || "null".equals(value)) { + return null; + } + return StorageLevel.fromString(value); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelSerializer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelSerializer.java new file mode 100644 index 000000000..db02ea278 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelSerializer.java @@ -0,0 +1,64 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util.serde; + +import org.apache.spark.storage.StorageLevel; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class StorageLevelSerializer extends JsonSerializer { + + private static final Map map = initMap(); + + private static Map initMap() { + Map map = new HashMap<>(); + map.put(StorageLevel.NONE(), "NONE"); + map.put(StorageLevel.DISK_ONLY(), "DISK_ONLY"); + map.put(StorageLevel.DISK_ONLY_2(), "DISK_ONLY_2"); + map.put(StorageLevel.MEMORY_ONLY(), "MEMORY_ONLY"); + map.put(StorageLevel.MEMORY_ONLY_2(), "MEMORY_ONLY_2"); + map.put(StorageLevel.MEMORY_ONLY_SER(), "MEMORY_ONLY_SER"); + map.put(StorageLevel.MEMORY_ONLY_SER_2(), "MEMORY_ONLY_SER_2"); + map.put(StorageLevel.MEMORY_AND_DISK(), "MEMORY_AND_DISK"); + map.put(StorageLevel.MEMORY_AND_DISK_2(), "MEMORY_AND_DISK_2"); + map.put(StorageLevel.MEMORY_AND_DISK_SER(), "MEMORY_AND_DISK_SER"); + map.put(StorageLevel.MEMORY_AND_DISK_SER_2(), "MEMORY_AND_DISK_SER_2"); + map.put(StorageLevel.OFF_HEAP(), "OFF_HEAP"); + return map; + } + + @Override + public void serialize(StorageLevel storageLevel, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException, JsonProcessingException { + //This is a little ugly, but Spark doesn't provide many options here... + String s = null; + if (storageLevel != null) { + s = map.get(storageLevel); + } + jsonGenerator.writeString(s); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/BaseSparkKryoTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/BaseSparkKryoTest.java new file mode 100644 index 000000000..9cab73c5e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/BaseSparkKryoTest.java @@ -0,0 +1,81 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.Map; + +public class BaseSparkKryoTest extends BaseSparkTest { + + @Override + public long getTimeoutMilliseconds() { + return 120000L; + } + + @Override + public JavaSparkContext getContext() { + if (sc != null) { + return sc; + } + + //Ensure SPARK_USER environment variable is set for Spark Kryo tests + String u = System.getenv("SPARK_USER"); + if(u == null || u.isEmpty()){ + try { + Class[] classes = Collections.class.getDeclaredClasses(); + Map env = System.getenv(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map map = (Map) obj; + String user = System.getProperty("user.name"); + if(user == null || user.isEmpty()) + user = "user"; + map.put("SPARK_USER", user); + } + } + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + + + SparkConf sparkConf = new SparkConf().setMaster("local[" + numExecutors() + "]") + .setAppName("sparktest") + .set("spark.driver.host", "localhost"); + + sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + sparkConf.set("spark.kryo.registrator", "org.nd4j.kryo.Nd4jRegistrator"); + + sc = new JavaSparkContext(sparkConf); + + return sc; + } + +} + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/BaseSparkTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/BaseSparkTest.java new file mode 100644 index 000000000..e00f8d6d3 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/BaseSparkTest.java @@ -0,0 +1,146 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark; + +import org.apache.hadoop.conf.Configuration; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + + +public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable { + protected transient JavaSparkContext sc; + protected transient INDArray labels; + protected transient INDArray input; + protected transient INDArray rowSums; + protected transient int nRows = 200; + protected transient int nIn = 4; + protected transient int nOut = 3; + protected transient DataSet data; + protected transient JavaRDD sparkData; + + @Override + public long getTimeoutMilliseconds() { + return 120000L; + } + + @BeforeEach + public void before() { + + sc = getContext(); + Random r = new Random(12345); + labels = Nd4j.create(nRows, nOut); + input = Nd4j.rand(nRows, nIn); + rowSums = input.sum(1); + input.diviColumnVector(rowSums); + + for (int i = 0; i < nRows; i++) { + int x1 = r.nextInt(nOut); + labels.putScalar(new int[] {i, x1}, 1.0); + } + + sparkData = getBasicSparkDataSet(nRows, input, labels); + } + + @AfterEach + public void after() { + if(sc != null) { + sc.close(); + } + sc = null; + } + + /** + * + * @return + */ + public JavaSparkContext getContext() { + if (sc != null) + return sc; + // set to test mode + SparkConf sparkConf = new SparkConf().setMaster("local[" + numExecutors() + "]") + .set("spark.driver.host", "localhost").setAppName("sparktest"); + + + sc = new JavaSparkContext(sparkConf); + + return sc; + } + + protected JavaRDD getBasicSparkDataSet(int nRows, INDArray input, INDArray labels) { + List list = new ArrayList<>(); + for (int i = 0; i < nRows; i++) { + INDArray inRow = input.getRow(i, true).dup(); + INDArray outRow = labels.getRow(i, true).dup(); + + DataSet ds = new DataSet(inRow, outRow); + list.add(ds); + } + list.iterator(); + + data = DataSet.merge(list); + return sc.parallelize(list); + } + + + protected SparkDl4jMultiLayer getBasicNetwork() { + return new SparkDl4jMultiLayer(sc, getBasicConf(), + new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0)); + } + + protected int numExecutors() { + return 4; + } + + protected MultiLayerConfiguration getBasicConf() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123) + .updater(new Nesterovs(0.1, 0.9)).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(nIn).nOut(3) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(3).nOut(nOut) + .activation(Activation.SOFTMAX).build()) + .build(); + + return conf; + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSpark.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSpark.java new file mode 100644 index 000000000..ed8de3623 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSpark.java @@ -0,0 +1,326 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark; + +import com.sun.jna.Platform; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.EarlyStoppingModelSaver; +import org.deeplearning4j.earlystopping.EarlyStoppingResult; +import org.deeplearning4j.earlystopping.listener.EarlyStoppingListener; +import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; +import org.deeplearning4j.earlystopping.termination.MaxEpochsTerminationCondition; +import org.deeplearning4j.earlystopping.termination.MaxScoreIterationTerminationCondition; +import org.deeplearning4j.earlystopping.termination.MaxTimeIterationTerminationCondition; +import org.deeplearning4j.earlystopping.termination.ScoreImprovementEpochTerminationCondition; +import org.deeplearning4j.earlystopping.trainer.IEarlyStoppingTrainer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.spark.earlystopping.SparkDataSetLossCalculator; +import org.deeplearning4j.spark.earlystopping.SparkEarlyStoppingTrainer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestEarlyStoppingSpark extends BaseSparkTest { + + @Test + public void testEarlyStoppingIris() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd()).weightInit(WeightInit.XAVIER).list() + .layer(0, new OutputLayer.Builder().nIn(4).nOut(3) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.setListeners(new ScoreIterationListener(5)); + + + JavaRDD irisData = getIris(); + + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = + new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(5)) + .iterationTerminationConditions( + new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES)) + .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc())) + .modelSaver(saver).build(); + + IEarlyStoppingTrainer trainer = + new SparkEarlyStoppingTrainer( + getContext().sc(), new ParameterAveragingTrainingMaster.Builder(irisBatchSize()) + .saveUpdater(true).averagingFrequency(1).build(), + esConf, net, irisData); + + EarlyStoppingResult result = trainer.fit(); + System.out.println(result); + + assertEquals(5, result.getTotalEpochs()); + assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason()); + Map scoreVsIter = result.getScoreVsEpoch(); + assertEquals(5, scoreVsIter.size()); + String expDetails = esConf.getEpochTerminationConditions().get(0).toString(); + assertEquals(expDetails, result.getTerminationDetails()); + + MultiLayerNetwork out = result.getBestModel(); + assertNotNull(out); + + //Check that best score actually matches (returned model vs. manually calculated score) + MultiLayerNetwork bestNetwork = result.getBestModel(); + double score = bestNetwork.score(new IrisDataSetIterator(150, 150).next()); + double bestModelScore = result.getBestModelScore(); + assertEquals(bestModelScore, score, 1e-3); + } + + @Test + public void testBadTuning() { + //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(10.0)) //Intentionally huge LR + .weightInit(WeightInit.XAVIER).list() + .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY) + .lossFunction(LossFunctions.LossFunction.MSE).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.setListeners(new ScoreIterationListener(5)); + + JavaRDD irisData = getIris(); + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = + new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(5000)) + .iterationTerminationConditions( + new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES), + new MaxScoreIterationTerminationCondition(7.5)) //Initial score is ~2.5 + .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc())) + .modelSaver(saver).build(); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingTrainer(getContext().sc(), + new ParameterAveragingTrainingMaster(true, 4, 1, 150 / 4, 1, 0), esConf, net, irisData); + EarlyStoppingResult result = trainer.fit(); + + assertTrue(result.getTotalEpochs() < 5); + assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition, + result.getTerminationReason()); + String expDetails = new MaxScoreIterationTerminationCondition(7.5).toString(); + assertEquals(expDetails, result.getTerminationDetails()); + } + + @Test + public void testTimeTermination() { + //test termination after max time + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(1e-6)).weightInit(WeightInit.XAVIER).list() + .layer(0, new OutputLayer.Builder().nIn(4).nOut(3) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.setListeners(new ScoreIterationListener(5)); + + JavaRDD irisData = getIris(); + + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = + new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(10000)) + .iterationTerminationConditions( + new MaxTimeIterationTerminationCondition(3, TimeUnit.SECONDS), + new MaxScoreIterationTerminationCondition(7.5)) //Initial score is ~2.5 + .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc())) + .modelSaver(saver).build(); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingTrainer(getContext().sc(), + new ParameterAveragingTrainingMaster(true, 4, 1, 150 / 15, 1, 0), esConf, net, irisData); + long startTime = System.currentTimeMillis(); + EarlyStoppingResult result = trainer.fit(); + long endTime = System.currentTimeMillis(); + int durationSeconds = (int) (endTime - startTime) / 1000; + + assertTrue(durationSeconds >= 3, "durationSeconds = " + durationSeconds); + assertTrue(durationSeconds <= 20, "durationSeconds = " + durationSeconds); + + assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition, + result.getTerminationReason()); + String expDetails = new MaxTimeIterationTerminationCondition(3, TimeUnit.SECONDS).toString(); + assertEquals(expDetails, result.getTerminationDetails()); + } + + @Test + public void testNoImprovementNEpochsTermination() { + //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs + //Simulate this by setting LR = 0.0 + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).list() + .layer(0, new OutputLayer.Builder().nIn(4).nOut(3) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.setListeners(new ScoreIterationListener(5)); + + JavaRDD irisData = getIris(); + + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = + new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(100), + new ScoreImprovementEpochTerminationCondition(5)) + .iterationTerminationConditions(new MaxScoreIterationTerminationCondition(7.5)) //Initial score is ~2.5 + .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc())) + .modelSaver(saver).build(); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingTrainer(getContext().sc(), + new ParameterAveragingTrainingMaster(true, 4, 1, 150 / 10, 1, 0), esConf, net, irisData); + EarlyStoppingResult result = trainer.fit(); + + //Expect no score change due to 0 LR -> terminate after 6 total epochs + assertTrue(result.getTotalEpochs() < 12); //Normally expect 6 epochs exactly; get a little more than that here due to rounding + order of operations + assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason()); + String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString(); + assertEquals(expDetails, result.getTerminationDetails()); + } + + @Test + public void testListeners() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd()).weightInit(WeightInit.XAVIER).list() + .layer(0, new OutputLayer.Builder().nIn(4).nOut(3) + .lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.setListeners(new ScoreIterationListener(5)); + + + JavaRDD irisData = getIris(); + + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = + new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(5)) + .iterationTerminationConditions( + new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES)) + .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc())) + .modelSaver(saver).build(); + + LoggingEarlyStoppingListener listener = new LoggingEarlyStoppingListener(); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingTrainer( + getContext().sc(), new ParameterAveragingTrainingMaster(true, + Runtime.getRuntime().availableProcessors(), 1, 10, 1, 0), + esConf, net, irisData); + trainer.setListener(listener); + + trainer.fit(); + + assertEquals(1, listener.onStartCallCount); + assertEquals(5, listener.onEpochCallCount); + assertEquals(1, listener.onCompletionCallCount); + } + + private static class LoggingEarlyStoppingListener implements EarlyStoppingListener { + + private static Logger log = LoggerFactory.getLogger(LoggingEarlyStoppingListener.class); + private int onStartCallCount = 0; + private int onEpochCallCount = 0; + private int onCompletionCallCount = 0; + + @Override + public void onStart(EarlyStoppingConfiguration esConfig, MultiLayerNetwork net) { + log.info("EarlyStopping: onStart called"); + onStartCallCount++; + } + + @Override + public void onEpoch(int epochNum, double score, EarlyStoppingConfiguration esConfig, MultiLayerNetwork net) { + log.info("EarlyStopping: onEpoch called (epochNum={}, score={}}", epochNum, score); + onEpochCallCount++; + } + + @Override + public void onCompletion(EarlyStoppingResult esResult) { + log.info("EarlyStopping: onCompletion called (result: {})", esResult); + onCompletionCallCount++; + } + } + + private int irisBatchSize() { + return 1; + } + + private JavaRDD getIris() { + + JavaSparkContext sc = getContext(); + + IrisDataSetIterator iter = new IrisDataSetIterator(irisBatchSize(), 150); + List list = new ArrayList<>(150); + while (iter.hasNext()) + list.add(iter.next()); + + return sc.parallelize(list); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSparkCompGraph.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSparkCompGraph.java new file mode 100644 index 000000000..3de17a742 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSparkCompGraph.java @@ -0,0 +1,328 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark; + +import com.sun.jna.Platform; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; +import org.deeplearning4j.earlystopping.EarlyStoppingModelSaver; +import org.deeplearning4j.earlystopping.EarlyStoppingResult; +import org.deeplearning4j.earlystopping.listener.EarlyStoppingListener; +import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; +import org.deeplearning4j.earlystopping.termination.MaxEpochsTerminationCondition; +import org.deeplearning4j.earlystopping.termination.MaxScoreIterationTerminationCondition; +import org.deeplearning4j.earlystopping.termination.MaxTimeIterationTerminationCondition; +import org.deeplearning4j.earlystopping.termination.ScoreImprovementEpochTerminationCondition; +import org.deeplearning4j.earlystopping.trainer.IEarlyStoppingTrainer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.earlystopping.SparkEarlyStoppingGraphTrainer; +import org.deeplearning4j.spark.earlystopping.SparkLossCalculatorComputationGraph; +import org.deeplearning4j.spark.impl.graph.dataset.DataSetToMultiDataSetFn; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestEarlyStoppingSparkCompGraph extends BaseSparkTest { + + + @Test + public void testEarlyStoppingIris() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd()).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in") + .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in") + .setOutputs("0").build(); + ComputationGraph net = new ComputationGraph(conf); + net.setListeners(new ScoreIterationListener(5)); + + + JavaRDD irisData = getIris(); + + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(5)) + .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES)) + .scoreCalculator(new SparkLossCalculatorComputationGraph( + irisData.map(new DataSetToMultiDataSetFn()), true, sc.sc())) + .modelSaver(saver).build(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingGraphTrainer(getContext().sc(), tm, + esConf, net, irisData.map(new DataSetToMultiDataSetFn())); + + EarlyStoppingResult result = trainer.fit(); + System.out.println(result); + + assertEquals(5, result.getTotalEpochs()); + assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason()); + Map scoreVsIter = result.getScoreVsEpoch(); + assertEquals(5, scoreVsIter.size()); + String expDetails = esConf.getEpochTerminationConditions().get(0).toString(); + assertEquals(expDetails, result.getTerminationDetails()); + + ComputationGraph out = result.getBestModel(); + assertNotNull(out); + + //Check that best score actually matches (returned model vs. manually calculated score) + ComputationGraph bestNetwork = result.getBestModel(); + double score = bestNetwork.score(new IrisDataSetIterator(150, 150).next()); + double bestModelScore = result.getBestModelScore(); + assertEquals(bestModelScore, score, 1e-3); + } + + @Test + public void testBadTuning() { + //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Nd4j.getRandom().setSeed(12345); + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(2.0)) //Intentionally huge LR + .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in") + .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY) + .lossFunction(LossFunctions.LossFunction.MSE).build(), "in") + .setOutputs("0").build(); + ComputationGraph net = new ComputationGraph(conf); + net.setListeners(new ScoreIterationListener(5)); + + JavaRDD irisData = getIris(); + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(5000)) + .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES), + new MaxScoreIterationTerminationCondition(7.5)) //Initial score is ~2.5 + .scoreCalculator(new SparkLossCalculatorComputationGraph( + irisData.map(new DataSetToMultiDataSetFn()), true, sc.sc())) + .modelSaver(saver).build(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingGraphTrainer(getContext().sc(), tm, + esConf, net, irisData.map(new DataSetToMultiDataSetFn())); + EarlyStoppingResult result = trainer.fit(); + + assertTrue(result.getTotalEpochs() < 5); + assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition, + result.getTerminationReason()); + String expDetails = new MaxScoreIterationTerminationCondition(7.5).toString(); + assertEquals(expDetails, result.getTerminationDetails()); + } + + @Test + public void testTimeTermination() { + //test termination after max time + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Nd4j.getRandom().setSeed(12345); + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(1e-6)).weightInit(WeightInit.XAVIER).graphBuilder() + .addInputs("in") + .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in") + .setOutputs("0").build(); + ComputationGraph net = new ComputationGraph(conf); + net.setListeners(new ScoreIterationListener(5)); + + JavaRDD irisData = getIris(); + + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(10000)) + .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(3, TimeUnit.SECONDS), + new MaxScoreIterationTerminationCondition(7.5)) //Initial score is ~2.5 + .scoreCalculator(new SparkLossCalculatorComputationGraph( + irisData.map(new DataSetToMultiDataSetFn()), true, sc.sc())) + .modelSaver(saver).build(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingGraphTrainer(getContext().sc(), tm, + esConf, net, irisData.map(new DataSetToMultiDataSetFn())); + long startTime = System.currentTimeMillis(); + EarlyStoppingResult result = trainer.fit(); + long endTime = System.currentTimeMillis(); + int durationSeconds = (int) (endTime - startTime) / 1000; + + assertTrue(durationSeconds >= 3); + assertTrue(durationSeconds <= 20); + + assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition, + result.getTerminationReason()); + String expDetails = new MaxTimeIterationTerminationCondition(3, TimeUnit.SECONDS).toString(); + assertEquals(expDetails, result.getTerminationDetails()); + } + + @Test + public void testNoImprovementNEpochsTermination() { + //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs + //Simulate this by setting LR = 0.0 + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Nd4j.getRandom().setSeed(12345); + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).graphBuilder() + .addInputs("in") + .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in") + .setOutputs("0").build(); + ComputationGraph net = new ComputationGraph(conf); + net.setListeners(new ScoreIterationListener(5)); + + JavaRDD irisData = getIris(); + + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(100), + new ScoreImprovementEpochTerminationCondition(5)) + .iterationTerminationConditions(new MaxScoreIterationTerminationCondition(7.5)) //Initial score is ~2.5 + .scoreCalculator(new SparkLossCalculatorComputationGraph( + irisData.map(new DataSetToMultiDataSetFn()), true, sc.sc())) + .modelSaver(saver).build(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingGraphTrainer(getContext().sc(), tm, + esConf, net, irisData.map(new DataSetToMultiDataSetFn())); + EarlyStoppingResult result = trainer.fit(); + + //Expect no score change due to 0 LR -> terminate after 6 total epochs + assertTrue(result.getTotalEpochs() < 12); //Normally expect 6 epochs exactly; get a little more than that here due to rounding + order of operations + assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason()); + String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString(); + assertEquals(expDetails, result.getTerminationDetails()); + } + + @Test + public void testListeners() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .updater(new Sgd()).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in") + .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in") + .setOutputs("0").build(); + ComputationGraph net = new ComputationGraph(conf); + net.setListeners(new ScoreIterationListener(5)); + + + JavaRDD irisData = getIris(); + + EarlyStoppingModelSaver saver = new InMemoryModelSaver<>(); + EarlyStoppingConfiguration esConf = new EarlyStoppingConfiguration.Builder() + .epochTerminationConditions(new MaxEpochsTerminationCondition(5)) + .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES)) + .scoreCalculator(new SparkLossCalculatorComputationGraph( + irisData.map(new DataSetToMultiDataSetFn()), true, sc.sc())) + .modelSaver(saver).build(); + + LoggingEarlyStoppingListener listener = new LoggingEarlyStoppingListener(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0); + + IEarlyStoppingTrainer trainer = new SparkEarlyStoppingGraphTrainer(getContext().sc(), tm, + esConf, net, irisData.map(new DataSetToMultiDataSetFn())); + trainer.setListener(listener); + + trainer.fit(); + + assertEquals(1, listener.onStartCallCount); + assertEquals(5, listener.onEpochCallCount); + assertEquals(1, listener.onCompletionCallCount); + } + + private static class LoggingEarlyStoppingListener implements EarlyStoppingListener { + + private static Logger log = LoggerFactory.getLogger(LoggingEarlyStoppingListener.class); + private int onStartCallCount = 0; + private int onEpochCallCount = 0; + private int onCompletionCallCount = 0; + + @Override + public void onStart(EarlyStoppingConfiguration esConfig, ComputationGraph net) { + log.info("EarlyStopping: onStart called"); + onStartCallCount++; + } + + @Override + public void onEpoch(int epochNum, double score, EarlyStoppingConfiguration esConfig, ComputationGraph net) { + log.info("EarlyStopping: onEpoch called (epochNum={}, score={}}", epochNum, score); + onEpochCallCount++; + } + + @Override + public void onCompletion(EarlyStoppingResult esResult) { + log.info("EorlyStopping: onCompletion called (result: {})", esResult); + onCompletionCallCount++; + } + } + + private JavaRDD getIris() { + + JavaSparkContext sc = getContext(); + + IrisDataSetIterator iter = new IrisDataSetIterator(1, 150); + List list = new ArrayList<>(150); + while (iter.hasNext()) + list.add(iter.next()); + + return sc.parallelize(list); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestKryo.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestKryo.java new file mode 100644 index 000000000..48212f814 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/TestKryo.java @@ -0,0 +1,199 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark; + +import org.apache.spark.serializer.SerializerInstance; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.distribution.UniformDistribution; +import org.deeplearning4j.nn.conf.graph.*; +import org.deeplearning4j.nn.conf.graph.rnn.DuplicateToTimeSeriesVertex; +import org.deeplearning4j.nn.conf.graph.rnn.LastTimeStepVertex; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; +import org.junit.jupiter.api.Test; +import org.nd4j.evaluation.IEvaluation; +import org.nd4j.evaluation.classification.*; +import org.nd4j.evaluation.regression.RegressionEvaluation; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Nadam; +import org.nd4j.linalg.schedule.MapSchedule; +import org.nd4j.linalg.schedule.ScheduleType; +import scala.collection.JavaConversions; //TODO: needs changing scala 2.13 + +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestKryo extends BaseSparkKryoTest { + + private void testSerialization(T in, SerializerInstance si) { + ByteBuffer bb = si.serialize(in, null); + T deserialized = (T)si.deserialize(bb, null); + + boolean equals = in.equals(deserialized); + assertTrue(equals, in.getClass() + "\t" + in.toString()); + } + + @Test + public void testSerializationConfigurations() { + + SerializerInstance si = sc.env().serializer().newInstance(); + + //Check network configurations: + Map m = new HashMap<>(); + m.put(0, 0.5); + m.put(10, 0.1); + MultiLayerConfiguration mlc = new NeuralNetConfiguration.Builder() + .updater(new Nadam(new MapSchedule(ScheduleType.ITERATION,m))).list().layer(0, new OutputLayer.Builder().nIn(10).nOut(10).build()) + .build(); + + testSerialization(mlc, si); + + + ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder() + .dist(new UniformDistribution(-1, 1)) + .updater(new Adam(new MapSchedule(ScheduleType.ITERATION,m))) + .graphBuilder() + .addInputs("in").addLayer("out", new OutputLayer.Builder().nIn(10).nOut(10).build(), "in") + .setOutputs("out").build(); + + testSerialization(cgc, si); + + + //Check main layers: + Layer[] layers = new Layer[] {new OutputLayer.Builder().nIn(10).nOut(10).build(), + new RnnOutputLayer.Builder().nIn(10).nOut(10).build(), new LossLayer.Builder().build(), + new CenterLossOutputLayer.Builder().nIn(10).nOut(10).build(), + new DenseLayer.Builder().nIn(10).nOut(10).build(), + new ConvolutionLayer.Builder().nIn(10).nOut(10).build(), new SubsamplingLayer.Builder().build(), + new Convolution1DLayer.Builder(2, 2).nIn(10).nOut(10).build(), + new ActivationLayer.Builder().activation(Activation.TANH).build(), + new GlobalPoolingLayer.Builder().build(), new GravesLSTM.Builder().nIn(10).nOut(10).build(), + new LSTM.Builder().nIn(10).nOut(10).build(), new DropoutLayer.Builder(0.5).build(), + new BatchNormalization.Builder().build(), new LocalResponseNormalization.Builder().build()}; + + for (Layer l : layers) { + testSerialization(l, si); + } + + //Check graph vertices + GraphVertex[] vertices = new GraphVertex[] {new ElementWiseVertex(ElementWiseVertex.Op.Add), + new L2NormalizeVertex(), new LayerVertex(null, null), new MergeVertex(), new PoolHelperVertex(), + new PreprocessorVertex(new CnnToFeedForwardPreProcessor(28, 28, 1)), + new ReshapeVertex(new int[] {1, 1}), new ScaleVertex(1.0), new ShiftVertex(1.0), + new SubsetVertex(1, 1), new UnstackVertex(0, 2), new DuplicateToTimeSeriesVertex("in1"), + new LastTimeStepVertex("in1")}; + + for (GraphVertex gv : vertices) { + testSerialization(gv, si); + } + } + + @Test + public void testSerializationEvaluation() { + + Evaluation e = new Evaluation(); + e.eval(Nd4j.create(new double[] {1, 0, 0}, new long[]{1, 3}), Nd4j.create(new double[] {0.2, 0.5, 0.3}, new long[]{1, 3})); + + EvaluationBinary eb = new EvaluationBinary(); + eb.eval(Nd4j.create(new double[] {1, 0, 0}, new long[]{1, 3}), Nd4j.create(new double[] {0.2, 0.6, 0.3}, new long[]{1, 3})); + + ROC roc = new ROC(30); + roc.eval(Nd4j.create(new double[] {1}, new long[]{1, 1}), Nd4j.create(new double[] {0.2}, new long[]{1, 1})); + ROC roc2 = new ROC(); + roc2.eval(Nd4j.create(new double[] {1}, new long[]{1, 1}), Nd4j.create(new double[] {0.2}, new long[]{1, 1})); + + ROCMultiClass rocM = new ROCMultiClass(30); + rocM.eval(Nd4j.create(new double[] {1, 0, 0}, new long[]{1, 3}), Nd4j.create(new double[] {0.2, 0.5, 0.3}, new long[]{1, 3})); + ROCMultiClass rocM2 = new ROCMultiClass(); + rocM2.eval(Nd4j.create(new double[] {1, 0, 0}, new long[]{1, 3}), Nd4j.create(new double[] {0.2, 0.5, 0.3}, new long[]{1, 3})); + + ROCBinary rocB = new ROCBinary(30); + rocB.eval(Nd4j.create(new double[] {1, 0, 0}, new long[]{1, 3}), Nd4j.create(new double[] {0.2, 0.6, 0.3}, new long[]{1, 3})); + + ROCBinary rocB2 = new ROCBinary(); + rocB2.eval(Nd4j.create(new double[] {1, 0, 0}, new long[]{1, 3}), Nd4j.create(new double[] {0.2, 0.6, 0.3}, new long[]{1, 3})); + + RegressionEvaluation re = new RegressionEvaluation(); + re.eval(Nd4j.rand(1, 5), Nd4j.rand(1, 5)); + + IEvaluation[] evaluations = new IEvaluation[] {new Evaluation(), e, new EvaluationBinary(), eb, new ROC(), roc, + roc2, new ROCMultiClass(), rocM, rocM2, new ROCBinary(), rocB, rocB2, + new RegressionEvaluation(), re}; + + SerializerInstance si = sc.env().serializer().newInstance(); + + for (IEvaluation ie : evaluations) { + //System.out.println(ie.getClass()); + testSerialization(ie, si); + } + } + + @Test + public void testScalaCollections() { + //Scala collections should already work with Spark + kryo; some very basic tests to check this is still the case + SerializerInstance si = sc.env().serializer().newInstance(); + + scala.collection.immutable.Map emptyImmutableMap = + scala.collection.immutable.Map$.MODULE$.empty(); + testSerialization(emptyImmutableMap, si); + + Map m = new HashMap<>(); + m.put(0, 1.0); + + scala.collection.Map m2 = JavaConversions.mapAsScalaMap(m); + testSerialization(m2, si); + } + + @Test + public void testJavaTypes() { + + Map m = new HashMap<>(); + m.put("key", "value"); + + SerializerInstance si = sc.env().serializer().newInstance(); + + testSerialization(Collections.singletonMap("key", "value"), si); + testSerialization(Collections.synchronizedMap(m), si); + testSerialization(Collections.emptyMap(), si); + testSerialization(new ConcurrentHashMap<>(m), si); + testSerialization(Collections.unmodifiableMap(m), si); + + testSerialization(Arrays.asList("s"), si); + testSerialization(Collections.singleton("s"), si); + testSerialization(Collections.synchronizedList(Arrays.asList("s")), si); + testSerialization(Collections.emptyList(), si); + testSerialization(new CopyOnWriteArrayList<>(Arrays.asList("s")), si); + testSerialization(Collections.unmodifiableList(Arrays.asList("s")), si); + + testSerialization(Collections.singleton("s"), si); + testSerialization(Collections.synchronizedSet(new HashSet<>(Arrays.asList("s"))), si); + testSerialization(Collections.emptySet(), si); + testSerialization(Collections.unmodifiableSet(new HashSet<>(Arrays.asList("s"))), si); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/common/AddTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/common/AddTest.java new file mode 100644 index 000000000..f366de5b4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/common/AddTest.java @@ -0,0 +1,47 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.common; + +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.impl.common.Add; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class AddTest extends BaseSparkTest { + + @Test + public void testAdd() { + List list = new ArrayList<>(); + for (int i = 0; i < 5; i++) + list.add(Nd4j.ones(5)); + JavaRDD rdd = sc.parallelize(list); + INDArray sum = rdd.fold(Nd4j.zeros(5), new Add()); + assertEquals(25, sum.sum(Integer.MAX_VALUE).getDouble(0), 1e-1); + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/data/TestShuffleExamples.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/data/TestShuffleExamples.java new file mode 100644 index 000000000..f879cfd29 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/data/TestShuffleExamples.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.apache.spark.HashPartitioner; +import org.apache.spark.Partitioner; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.util.SparkUtils; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestShuffleExamples extends BaseSparkTest { + + @Test + public void testShuffle() { + List list = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + INDArray f = Nd4j.valueArrayOf(new int[] {10, 1}, i); + INDArray l = f.dup(); + + DataSet ds = new DataSet(f, l); + list.add(ds); + } + + JavaRDD rdd = sc.parallelize(list); + + JavaRDD shuffled = SparkUtils.shuffleExamples(rdd, 10, 10); + + List shuffledList = shuffled.collect(); + + int totalExampleCount = 0; + for (DataSet ds : shuffledList) { + totalExampleCount += ds.getFeatures().length(); +// System.out.println(Arrays.toString(ds.getFeatures().data().asFloat())); + + assertEquals(ds.getFeatures(), ds.getLabels()); + } + + assertEquals(100, totalExampleCount); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/data/TestSparkDataUtils.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/data/TestSparkDataUtils.java new file mode 100644 index 000000000..4e9f12dd9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/data/TestSparkDataUtils.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.deeplearning4j.spark.BaseSparkTest; +import org.junit.jupiter.api.Test; + +public class TestSparkDataUtils extends BaseSparkTest { + + @Test + public void testExport(){ + + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/MiniBatchTests.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/MiniBatchTests.java new file mode 100644 index 000000000..43c50fdeb --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/MiniBatchTests.java @@ -0,0 +1,84 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.Function; +import org.datavec.api.conf.Configuration; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader; +import org.deeplearning4j.spark.BaseSparkTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.common.io.ClassPathResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MiniBatchTests extends BaseSparkTest { + private static final Logger log = LoggerFactory.getLogger(MiniBatchTests.class); + + @Test + public void testMiniBatches() throws Exception { + log.info("Setting up Spark Context..."); + JavaRDD lines = sc.textFile(new ClassPathResource("svmLight/iris_svmLight_0.txt") + .getTempFileFromArchive().toURI().toString()).cache(); + long count = lines.count(); + assertEquals(300, count); + // gotta map this to a Matrix/INDArray + RecordReader rr = new SVMLightRecordReader(); + Configuration c = new Configuration(); + c.set(SVMLightRecordReader.NUM_FEATURES, "5"); + rr.setConf(c); + JavaRDD points = lines.map(new RecordReaderFunction(rr, 4, 3)).cache(); + count = points.count(); + assertEquals(300, count); + + List collect = points.collect(); + + points = points.repartition(1); + JavaRDD miniBatches = new RDDMiniBatches(10, points).miniBatchesJava(); + count = miniBatches.count(); + List list = miniBatches.collect(); + assertEquals(30, count); //Expect exactly 30 from 1 partition... could be more for multiple input partitions + + lines.unpersist(); + points.unpersist(); + miniBatches.map(new DataSetAssertionFunction()); + } + + + public static class DataSetAssertionFunction implements Function { + + @Override + public Object call(DataSet dataSet) throws Exception { + assertTrue(dataSet.getFeatures().columns() == 150); + assertTrue(dataSet.numExamples() == 30); + return null; + } + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestDataVecDataSetFunctions.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestDataVecDataSetFunctions.java new file mode 100644 index 000000000..fad1b4092 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestDataVecDataSetFunctions.java @@ -0,0 +1,550 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import com.sun.jna.Platform; +import lombok.val; +import org.apache.commons.io.FilenameUtils; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.input.PortableDataStream; +import org.datavec.api.io.labels.ParentPathLabelGenerator; +import org.datavec.api.records.reader.SequenceRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.split.InputSplit; +import org.datavec.api.split.NumberedFileInputSplit; +import org.datavec.api.writable.Writable; +import org.datavec.image.recordreader.ImageRecordReader; +import org.datavec.spark.functions.SequenceRecordReaderFunction; +import org.datavec.spark.functions.pairdata.*; +import org.datavec.spark.transform.misc.StringToWritablesFunction; +import org.datavec.spark.util.DataVecSparkUtil; +import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; +import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; +import org.deeplearning4j.spark.BaseSparkTest; + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.io.ClassPathResource; +import scala.Tuple2; + +import java.io.File; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestDataVecDataSetFunctions extends BaseSparkTest { + + @TempDir + public File testDir; + + @Test + public void testDataVecDataSetFunction() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + JavaSparkContext sc = getContext(); + + File f = testDir; + ClassPathResource cpr = new ClassPathResource("dl4j-spark/imagetest/"); + cpr.copyDirectory(f); + + //Test Spark record reader functionality vs. local + List labelsList = Arrays.asList("0", "1"); //Need this for Spark: can't infer without init call + + String path = f.getPath() + "/*"; + + JavaPairRDD origData = sc.binaryFiles(path); + assertEquals(4, origData.count()); //4 images + + ImageRecordReader rr = new ImageRecordReader(28, 28, 1, new ParentPathLabelGenerator()); + rr.setLabels(labelsList); + org.datavec.spark.functions.RecordReaderFunction rrf = new org.datavec.spark.functions.RecordReaderFunction(rr); + JavaRDD> rdd = origData.map(rrf); + JavaRDD data = rdd.map(new DataVecDataSetFunction(1, 2, false)); + List collected = data.collect(); + + //Load normally (i.e., not via Spark), and check that we get the same results (order not withstanding) + InputSplit is = new FileSplit(f, new String[] {"bmp"}, true); + ImageRecordReader irr = new ImageRecordReader(28, 28, 1, new ParentPathLabelGenerator()); + irr.initialize(is); + + RecordReaderDataSetIterator iter = new RecordReaderDataSetIterator(irr, 1, 1, 2); + List listLocal = new ArrayList<>(4); + while (iter.hasNext()) { + listLocal.add(iter.next()); + } + + + //Compare: + assertEquals(4, collected.size()); + assertEquals(4, listLocal.size()); + + //Check that results are the same (order not withstanding) + boolean[] found = new boolean[4]; + for (int i = 0; i < 4; i++) { + int foundIndex = -1; + DataSet ds = collected.get(i); + for (int j = 0; j < 4; j++) { + if (ds.equals(listLocal.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + int count = 0; + for (boolean b : found) + if (b) + count++; + assertEquals(4, count); //Expect all 4 and exactly 4 pairwise matches between spark and local versions + } + + @Test + public void testDataVecDataSetFunctionMultiLabelRegression() throws Exception { + JavaSparkContext sc = getContext(); + + List stringData = new ArrayList<>(); + int n = 6; + for (int i = 0; i < 10; i++) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (int j = 0; j < n; j++) { + if (!first) + sb.append(","); + sb.append(10 * i + j); + first = false; + } + stringData.add(sb.toString()); + } + + JavaRDD stringList = sc.parallelize(stringData); + JavaRDD> writables = stringList.map(new StringToWritablesFunction(new CSVRecordReader())); + JavaRDD dataSets = writables.map(new DataVecDataSetFunction(3, 5, -1, true, null, null)); + + List ds = dataSets.collect(); + assertEquals(10, ds.size()); + + boolean[] seen = new boolean[10]; + for (DataSet d : ds) { + INDArray f = d.getFeatures(); + INDArray l = d.getLabels(); + assertEquals(3, f.length()); + assertEquals(3, l.length()); + + int exampleIdx = ((int) f.getDouble(0)) / 10; + seen[exampleIdx] = true; + + for (int j = 0; j < 3; j++) { + assertEquals(10 * exampleIdx + j, (int) f.getDouble(j)); + assertEquals(10 * exampleIdx + j + 3, (int) l.getDouble(j)); + } + } + + int seenCount = 0; + for (boolean b : seen) + if (b) + seenCount++; + assertEquals(10, seenCount); + } + + @Test + public void testDataVecSequenceDataSetFunction() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + JavaSparkContext sc = getContext(); + //Test Spark record reader functionality vs. local + File dir = testDir; + ClassPathResource cpr = new ClassPathResource("dl4j-spark/csvsequence/"); + cpr.copyDirectory(dir); + + JavaPairRDD origData = sc.binaryFiles(dir.getAbsolutePath()); + assertEquals(3, origData.count()); //3 CSV sequences + + + + SequenceRecordReader seqRR = new CSVSequenceRecordReader(1, ","); + SequenceRecordReaderFunction rrf = new SequenceRecordReaderFunction(seqRR); + JavaRDD>> rdd = origData.map(rrf); + JavaRDD data = rdd.map(new DataVecSequenceDataSetFunction(2, -1, true, null, null)); + List collected = data.collect(); + + //Load normally (i.e., not via Spark), and check that we get the same results (order not withstanding) + InputSplit is = new FileSplit(dir, new String[] {"txt"}, true); + SequenceRecordReader seqRR2 = new CSVSequenceRecordReader(1, ","); + seqRR2.initialize(is); + + SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(seqRR2, 1, -1, 2, true); + List listLocal = new ArrayList<>(3); + while (iter.hasNext()) { + listLocal.add(iter.next()); + } + + + //Compare: + assertEquals(3, collected.size()); + assertEquals(3, listLocal.size()); + + //Check that results are the same (order not withstanding) + boolean[] found = new boolean[3]; + for (int i = 0; i < 3; i++) { + int foundIndex = -1; + DataSet ds = collected.get(i); + for (int j = 0; j < 3; j++) { + if (ds.equals(listLocal.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + int count = 0; + for (boolean b : found) + if (b) + count++; + assertEquals(3, count); //Expect all 3 and exactly 3 pairwise matches between spark and local versions + } + + @Test + public void testDataVecSequencePairDataSetFunction() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + JavaSparkContext sc = getContext(); + + File f = testDir; + ClassPathResource cpr = new ClassPathResource("dl4j-spark/csvsequence/"); + cpr.copyDirectory(f); + String path = f.getAbsolutePath() + "/*"; + + PathToKeyConverter pathConverter = new PathToKeyConverterFilename(); + JavaPairRDD toWrite = + DataVecSparkUtil.combineFilesForSequenceFile(sc, path, path, pathConverter); + + Path p = new File(testDir,"dl4j_testSeqPairFn").toPath(); + p.toFile().deleteOnExit(); + String outPath = p.toString() + "/out"; + new File(outPath).deleteOnExit(); + toWrite.saveAsNewAPIHadoopFile(outPath, Text.class, BytesPairWritable.class, SequenceFileOutputFormat.class); + + //Load from sequence file: + JavaPairRDD fromSeq = sc.sequenceFile(outPath, Text.class, BytesPairWritable.class); + + SequenceRecordReader srr1 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader srr2 = new CSVSequenceRecordReader(1, ","); + PairSequenceRecordReaderBytesFunction psrbf = new PairSequenceRecordReaderBytesFunction(srr1, srr2); + JavaRDD>, List>>> writables = fromSeq.map(psrbf); + + //Map to DataSet: + DataVecSequencePairDataSetFunction pairFn = new DataVecSequencePairDataSetFunction(); + JavaRDD data = writables.map(pairFn); + List sparkData = data.collect(); + + + //Now: do the same thing locally (SequenceRecordReaderDataSetIterator) and compare + String featuresPath = FilenameUtils.concat(f.getAbsolutePath(), "csvsequence_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iter = + new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, -1, true); + + List localData = new ArrayList<>(3); + while (iter.hasNext()) + localData.add(iter.next()); + + assertEquals(3, sparkData.size()); + assertEquals(3, localData.size()); + + for (int i = 0; i < 3; i++) { + //Check shapes etc. data sets order may differ for spark vs. local + DataSet dsSpark = sparkData.get(i); + DataSet dsLocal = localData.get(i); + + assertNull(dsSpark.getFeaturesMaskArray()); + assertNull(dsSpark.getLabelsMaskArray()); + + INDArray fSpark = dsSpark.getFeatures(); + INDArray fLocal = dsLocal.getFeatures(); + INDArray lSpark = dsSpark.getLabels(); + INDArray lLocal = dsLocal.getLabels(); + + val s = new long[] {1, 3, 4}; //1 example, 3 values, 3 time steps + assertArrayEquals(s, fSpark.shape()); + assertArrayEquals(s, fLocal.shape()); + assertArrayEquals(s, lSpark.shape()); + assertArrayEquals(s, lLocal.shape()); + } + + + //Check that results are the same (order not withstanding) + boolean[] found = new boolean[3]; + for (int i = 0; i < 3; i++) { + int foundIndex = -1; + DataSet ds = sparkData.get(i); + for (int j = 0; j < 3; j++) { + if (ds.equals(localData.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + int count = 0; + for (boolean b : found) + if (b) + count++; + assertEquals(3, count); //Expect all 3 and exactly 3 pairwise matches between spark and local versions + } + + @Test + public void testDataVecSequencePairDataSetFunctionVariableLength() throws Exception { + //Same sort of test as testDataVecSequencePairDataSetFunction() but with variable length time series (labels shorter, align end) + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + File dirFeatures = testDir; + ClassPathResource cpr = new ClassPathResource("dl4j-spark/csvsequence/"); + cpr.copyDirectory(dirFeatures); + + File dirLabels = testDir; + ClassPathResource cpr2 = new ClassPathResource("dl4j-spark/csvsequencelabels/"); + cpr2.copyDirectory(dirLabels); + + + PathToKeyConverter pathConverter = new PathToKeyConverterNumber(); //Extract a number from the file name + JavaPairRDD toWrite = + DataVecSparkUtil.combineFilesForSequenceFile(sc, dirFeatures.getAbsolutePath(), dirLabels.getAbsolutePath(), pathConverter); + + Path p = new File(testDir, "dl4j_testSeqPairFnVarLength").toPath(); + p.toFile().deleteOnExit(); + String outPath = p.toFile().getAbsolutePath() + "/out"; + new File(outPath).deleteOnExit(); + toWrite.saveAsNewAPIHadoopFile(outPath, Text.class, BytesPairWritable.class, SequenceFileOutputFormat.class); + + //Load from sequence file: + JavaPairRDD fromSeq = sc.sequenceFile(outPath, Text.class, BytesPairWritable.class); + + SequenceRecordReader srr1 = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader srr2 = new CSVSequenceRecordReader(1, ","); + PairSequenceRecordReaderBytesFunction psrbf = new PairSequenceRecordReaderBytesFunction(srr1, srr2); + JavaRDD>, List>>> writables = fromSeq.map(psrbf); + + //Map to DataSet: + DataVecSequencePairDataSetFunction pairFn = new DataVecSequencePairDataSetFunction(4, false, + DataVecSequencePairDataSetFunction.AlignmentMode.ALIGN_END); + JavaRDD data = writables.map(pairFn); + List sparkData = data.collect(); + + + //Now: do the same thing locally (SequenceRecordReaderDataSetIterator) and compare + String featuresPath = FilenameUtils.concat(dirFeatures.getAbsolutePath(), "csvsequence_%d.txt"); + String labelsPath = FilenameUtils.concat(dirLabels.getAbsolutePath(), "csvsequencelabelsShort_%d.txt"); + + SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); + SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + + SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, + 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); + + List localData = new ArrayList<>(3); + while (iter.hasNext()) + localData.add(iter.next()); + + assertEquals(3, sparkData.size()); + assertEquals(3, localData.size()); + + val fShapeExp = new long[] {1, 3, 4}; //1 example, 3 values, 4 time steps + val lShapeExp = new long[] {1, 4, 4}; //1 example, 4 values/classes, 4 time steps (after padding) + for (int i = 0; i < 3; i++) { + //Check shapes etc. data sets order may differ for spark vs. local + DataSet dsSpark = sparkData.get(i); + DataSet dsLocal = localData.get(i); + + assertNotNull(dsSpark.getLabelsMaskArray()); //Expect mask array for labels + + INDArray fSpark = dsSpark.getFeatures(); + INDArray fLocal = dsLocal.getFeatures(); + INDArray lSpark = dsSpark.getLabels(); + INDArray lLocal = dsLocal.getLabels(); + + + assertArrayEquals(fShapeExp, fSpark.shape()); + assertArrayEquals(fShapeExp, fLocal.shape()); + assertArrayEquals(lShapeExp, lSpark.shape()); + assertArrayEquals(lShapeExp, lLocal.shape()); + } + + + //Check that results are the same (order not withstanding) + boolean[] found = new boolean[3]; + for (int i = 0; i < 3; i++) { + int foundIndex = -1; + DataSet ds = sparkData.get(i); + for (int j = 0; j < 3; j++) { + if (dataSetsEqual(ds, localData.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + int count = 0; + for (boolean b : found) { + if (b) { + count++; + } + } + assertEquals(3, count); //Expect all 3 and exactly 3 pairwise matches between spark and local versions + + + //------------------------------------------------- + //NOW: test same thing, but for align start... + DataVecSequencePairDataSetFunction pairFnAlignStart = new DataVecSequencePairDataSetFunction(4, false, + DataVecSequencePairDataSetFunction.AlignmentMode.ALIGN_START); + JavaRDD rddDataAlignStart = writables.map(pairFnAlignStart); + List sparkDataAlignStart = rddDataAlignStart.collect(); + + featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); //re-initialize to reset + labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); + SequenceRecordReaderDataSetIterator iterAlignStart = new SequenceRecordReaderDataSetIterator(featureReader, + labelReader, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_START); + + List localDataAlignStart = new ArrayList<>(3); + while (iterAlignStart.hasNext()) + localDataAlignStart.add(iterAlignStart.next()); + + assertEquals(3, sparkDataAlignStart.size()); + assertEquals(3, localDataAlignStart.size()); + + for (int i = 0; i < 3; i++) { + //Check shapes etc. data sets order may differ for spark vs. local + DataSet dsSpark = sparkDataAlignStart.get(i); + DataSet dsLocal = localDataAlignStart.get(i); + + assertNotNull(dsSpark.getLabelsMaskArray()); //Expect mask array for labels + + INDArray fSpark = dsSpark.getFeatures(); + INDArray fLocal = dsLocal.getFeatures(); + INDArray lSpark = dsSpark.getLabels(); + INDArray lLocal = dsLocal.getLabels(); + + + assertArrayEquals(fShapeExp, fSpark.shape()); + assertArrayEquals(fShapeExp, fLocal.shape()); + assertArrayEquals(lShapeExp, lSpark.shape()); + assertArrayEquals(lShapeExp, lLocal.shape()); + } + + + //Check that results are the same (order not withstanding) + found = new boolean[3]; + for (int i = 0; i < 3; i++) { + int foundIndex = -1; + DataSet ds = sparkData.get(i); + for (int j = 0; j < 3; j++) { + if (dataSetsEqual(ds, localData.get(j))) { + if (foundIndex != -1) + fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) + foundIndex = j; + if (found[foundIndex]) + fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list + found[foundIndex] = true; //mark this one as seen before + } + } + } + count = 0; + for (boolean b : found) + if (b) + count++; + assertEquals(3, count); //Expect all 3 and exactly 3 pairwise matches between spark and local versions + } + + + private static boolean dataSetsEqual(DataSet d1, DataSet d2) { + + if (!d1.getFeatures().equals(d2.getFeatures())) { + return false; + } + if (d1.getLabels() == null && d2.getLabels() != null || d1.getLabels() != null && d2.getLabels() == null) { + return false; + } + if (d1.getLabels() != null && !d1.getLabels().equals(d2.getLabels())) { + return false; + } + + return masksEqual(d1.getFeatures(), d2.getFeatures()) + && masksEqual(d1.getLabelsMaskArray(), d2.getLabelsMaskArray()); + } + + private static boolean masksEqual(INDArray m1, INDArray m2) { + if (m1 == null && m2 == null) { + return true; + } + if (m1 != null && m2 != null) { + return m1.equals(m2); + } + //One is null, other is not. Null and ones mask arrays are equal though + if (m1 != null && !m1.equals(Nd4j.ones(m1.shape()))) { + return false; + } + if (m2 != null && !m2.equals(Nd4j.ones(m2.shape()))) { + return false; + } + + return true; + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestExport.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestExport.java new file mode 100644 index 000000000..b9eef9113 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestExport.java @@ -0,0 +1,170 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import com.sun.jna.Platform; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.data.BatchAndExportDataSetsFunction; +import org.deeplearning4j.spark.data.BatchAndExportMultiDataSetsFunction; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class TestExport extends BaseSparkTest { + + @Test + public void testBatchAndExportDataSetsFunction() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + String baseDir = System.getProperty("java.io.tmpdir"); + baseDir = FilenameUtils.concat(baseDir, "dl4j_spark_testBatchAndExport/"); + baseDir = baseDir.replaceAll("\\\\", "/"); + File f = new File(baseDir); + if (f.exists()) + FileUtils.deleteDirectory(f); + f.mkdir(); + f.deleteOnExit(); + int minibatchSize = 5; + int nIn = 4; + int nOut = 3; + + List dataSets = new ArrayList<>(); + dataSets.add(new DataSet(Nd4j.create(10, nIn), Nd4j.create(10, nOut))); //Larger than minibatch size -> tests splitting + for (int i = 0; i < 98; i++) { + if (i % 2 == 0) { + dataSets.add(new DataSet(Nd4j.create(5, nIn), Nd4j.create(5, nOut))); + } else { + dataSets.add(new DataSet(Nd4j.create(1, nIn), Nd4j.create(1, nOut))); + dataSets.add(new DataSet(Nd4j.create(1, nIn), Nd4j.create(1, nOut))); + dataSets.add(new DataSet(Nd4j.create(3, nIn), Nd4j.create(3, nOut))); + } + } + + Collections.shuffle(dataSets, new Random(12345)); + + JavaRDD rdd = sc.parallelize(dataSets); + rdd = rdd.repartition(1); //For testing purposes (should get exactly 100 out, but maybe more with more partitions) + + + JavaRDD pathsRdd = rdd.mapPartitionsWithIndex( + new BatchAndExportDataSetsFunction(minibatchSize, "file:///" + baseDir), true); + + List paths = pathsRdd.collect(); + assertEquals(100, paths.size()); + + File[] files = f.listFiles(); + assertNotNull(files); + + int count = 0; + for (File file : files) { + if (!file.getPath().endsWith(".bin")) + continue; +// System.out.println(file); + DataSet ds = new DataSet(); + ds.load(file); + assertEquals(minibatchSize, ds.numExamples()); + + count++; + } + + assertEquals(100, count); + + FileUtils.deleteDirectory(f); + } + + @Test + public void testBatchAndExportMultiDataSetsFunction() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + String baseDir = System.getProperty("java.io.tmpdir"); + baseDir = FilenameUtils.concat(baseDir, "dl4j_spark_testBatchAndExportMDS/"); + baseDir = baseDir.replaceAll("\\\\", "/"); + File f = new File(baseDir); + if (f.exists()) + FileUtils.deleteDirectory(f); + f.mkdir(); + f.deleteOnExit(); + int minibatchSize = 5; + int nIn = 4; + int nOut = 3; + + List dataSets = new ArrayList<>(); + dataSets.add(new org.nd4j.linalg.dataset.MultiDataSet(Nd4j.create(10, nIn), Nd4j.create(10, nOut))); //Larger than minibatch size -> tests splitting + for (int i = 0; i < 98; i++) { + if (i % 2 == 0) { + dataSets.add(new org.nd4j.linalg.dataset.MultiDataSet(Nd4j.create(5, nIn), Nd4j.create(5, nOut))); + } else { + dataSets.add(new org.nd4j.linalg.dataset.MultiDataSet(Nd4j.create(1, nIn), Nd4j.create(1, nOut))); + dataSets.add(new org.nd4j.linalg.dataset.MultiDataSet(Nd4j.create(1, nIn), Nd4j.create(1, nOut))); + dataSets.add(new org.nd4j.linalg.dataset.MultiDataSet(Nd4j.create(3, nIn), Nd4j.create(3, nOut))); + } + } + + Collections.shuffle(dataSets, new Random(12345)); + + JavaRDD rdd = sc.parallelize(dataSets); + rdd = rdd.repartition(1); //For testing purposes (should get exactly 100 out, but maybe more with more partitions) + + + JavaRDD pathsRdd = rdd.mapPartitionsWithIndex( + new BatchAndExportMultiDataSetsFunction(minibatchSize, "file:///" + baseDir), true); + + List paths = pathsRdd.collect(); + assertEquals(100, paths.size()); + + File[] files = f.listFiles(); + assertNotNull(files); + + int count = 0; + for (File file : files) { + if (!file.getPath().endsWith(".bin")) + continue; +// System.out.println(file); + MultiDataSet ds = new org.nd4j.linalg.dataset.MultiDataSet(); + ds.load(file); + assertEquals(minibatchSize, ds.getFeatures(0).size(0)); + assertEquals(minibatchSize, ds.getLabels(0).size(0)); + + count++; + } + + assertEquals(100, count); + + FileUtils.deleteDirectory(f); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestPreProcessedData.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestPreProcessedData.java new file mode 100644 index 000000000..714c3ffb6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/TestPreProcessedData.java @@ -0,0 +1,388 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec; + +import com.sun.jna.Platform; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.input.PortableDataStream; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.Updater; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.api.Repartition; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.datavec.export.StringToDataSetExportFunction; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.deeplearning4j.spark.iterator.PortableDataStreamDataSetIterator; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestPreProcessedData extends BaseSparkTest { + + @Test + public void testPreprocessedData() { + //Test _loading_ of preprocessed data + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + int dataSetObjSize = 5; + int batchSizePerExecutor = 10; + + String path = FilenameUtils.concat(System.getProperty("java.io.tmpdir"), "dl4j_testpreprocdata"); + File f = new File(path); + if (f.exists()) + f.delete(); + f.mkdir(); + + DataSetIterator iter = new IrisDataSetIterator(5, 150); + int i = 0; + while (iter.hasNext()) { + File f2 = new File(FilenameUtils.concat(path, "data" + (i++) + ".bin")); + iter.next().save(f2); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(3) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3).activation(Activation.SOFTMAX) + .build()) + .build(); + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1) + .repartionData(Repartition.Always).build()); + sparkNet.setCollectTrainingStats(true); + + sparkNet.fit("file:///" + path.replaceAll("\\\\", "/")); + + SparkTrainingStats sts = sparkNet.getSparkTrainingStats(); + int expNumFits = 12; //4 'fits' per averaging (4 executors, 1 averaging freq); 10 examples each -> 40 examples per fit. 150/40 = 3 averagings (round down); 3*4 = 12 + + //Unfortunately: perfect partitioning isn't guaranteed by SparkUtils.balancedRandomSplit (esp. if original partitions are all size 1 + // which appears to be occurring at least some of the time), but we should get close to what we expect... + assertTrue(Math.abs(expNumFits - sts.getValue("ParameterAveragingWorkerFitTimesMs").size()) < 3); + + assertEquals(3, sts.getValue("ParameterAveragingMasterMapPartitionsTimesMs").size()); + } + + @Test + public void testPreprocessedDataCompGraphDataSet() { + //Test _loading_ of preprocessed DataSet data + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + int dataSetObjSize = 5; + int batchSizePerExecutor = 10; + + String path = FilenameUtils.concat(System.getProperty("java.io.tmpdir"), "dl4j_testpreprocdata2"); + File f = new File(path); + if (f.exists()) + f.delete(); + f.mkdir(); + + DataSetIterator iter = new IrisDataSetIterator(5, 150); + int i = 0; + while (iter.hasNext()) { + File f2 = new File(FilenameUtils.concat(path, "data" + (i++) + ".bin")); + iter.next().save(f2); + } + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .graphBuilder().addInputs("in") + .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(3) + .activation(Activation.TANH).build(), "in") + .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3).activation(Activation.SOFTMAX) + .build(), + "0") + .setOutputs("1").build(); + + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf, + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1) + .repartionData(Repartition.Always).build()); + sparkNet.setCollectTrainingStats(true); + + sparkNet.fit("file:///" + path.replaceAll("\\\\", "/")); + + SparkTrainingStats sts = sparkNet.getSparkTrainingStats(); + int expNumFits = 12; //4 'fits' per averaging (4 executors, 1 averaging freq); 10 examples each -> 40 examples per fit. 150/40 = 3 averagings (round down); 3*4 = 12 + + //Unfortunately: perfect partitioning isn't guaranteed by SparkUtils.balancedRandomSplit (esp. if original partitions are all size 1 + // which appears to be occurring at least some of the time), but we should get close to what we expect... + assertTrue(Math.abs(expNumFits - sts.getValue("ParameterAveragingWorkerFitTimesMs").size()) < 3); + + assertEquals(3, sts.getValue("ParameterAveragingMasterMapPartitionsTimesMs").size()); + } + + @Test + public void testPreprocessedDataCompGraphMultiDataSet() throws IOException { + //Test _loading_ of preprocessed MultiDataSet data + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + int dataSetObjSize = 5; + int batchSizePerExecutor = 10; + + String path = FilenameUtils.concat(System.getProperty("java.io.tmpdir"), "dl4j_testpreprocdata3"); + File f = new File(path); + if (f.exists()) + f.delete(); + f.mkdir(); + + DataSetIterator iter = new IrisDataSetIterator(5, 150); + int i = 0; + while (iter.hasNext()) { + File f2 = new File(FilenameUtils.concat(path, "data" + (i++) + ".bin")); + DataSet ds = iter.next(); + MultiDataSet mds = new MultiDataSet(ds.getFeatures(), ds.getLabels()); + mds.save(f2); + } + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().updater(Updater.RMSPROP) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .graphBuilder().addInputs("in") + .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(3) + .activation(Activation.TANH).build(), "in") + .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3).activation(Activation.SOFTMAX) + .build(), + "0") + .setOutputs("1").build(); + + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf, + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1) + .repartionData(Repartition.Always).build()); + sparkNet.setCollectTrainingStats(true); + + sparkNet.fitMultiDataSet("file:///" + path.replaceAll("\\\\", "/")); + + SparkTrainingStats sts = sparkNet.getSparkTrainingStats(); + int expNumFits = 12; //4 'fits' per averaging (4 executors, 1 averaging freq); 10 examples each -> 40 examples per fit. 150/40 = 3 averagings (round down); 3*4 = 12 + + //Unfortunately: perfect partitioning isn't guaranteed by SparkUtils.balancedRandomSplit (esp. if original partitions are all size 1 + // which appears to be occurring at least some of the time), but we should get close to what we expect... + assertTrue(Math.abs(expNumFits - sts.getValue("ParameterAveragingWorkerFitTimesMs").size()) < 3); + + assertEquals(3, sts.getValue("ParameterAveragingMasterMapPartitionsTimesMs").size()); + } + + @Test + public void testCsvPreprocessedDataGeneration() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + List list = new ArrayList<>(); + DataSetIterator iter = new IrisDataSetIterator(1, 150); + while (iter.hasNext()) { + DataSet ds = iter.next(); + list.add(toString(ds.getFeatures(), Nd4j.argMax(ds.getLabels(), 1).getInt(0))); + } + + JavaRDD rdd = sc.parallelize(list); + int partitions = rdd.partitions().size(); + + URI tempDir = new File(System.getProperty("java.io.tmpdir")).toURI(); + URI outputDir = new URI(tempDir.getPath() + "/dl4j_testPreprocessedData2"); + File temp = new File(outputDir.getPath()); + if (temp.exists()) + FileUtils.deleteDirectory(temp); + + int numBinFiles = 0; + try { + int batchSize = 5; + int labelIdx = 4; + int numPossibleLabels = 3; + + rdd.foreachPartition(new StringToDataSetExportFunction(outputDir, new CSVRecordReader(0), batchSize, false, + labelIdx, numPossibleLabels)); + + File[] fileList = new File(outputDir.getPath()).listFiles(); + + int totalExamples = 0; + for (File f2 : fileList) { + if (!f2.getPath().endsWith(".bin")) + continue; + // System.out.println(f2.getPath()); + numBinFiles++; + + DataSet ds = new DataSet(); + ds.load(f2); + + assertEquals(4, ds.numInputs()); + assertEquals(3, ds.numOutcomes()); + + totalExamples += ds.numExamples(); + } + + assertEquals(150, totalExamples); + assertTrue(Math.abs(150 / batchSize - numBinFiles) <= partitions); //Expect 30, give or take due to partitioning randomness + + + + //Test the PortableDataStreamDataSetIterator: + JavaPairRDD pds = sc.binaryFiles(outputDir.getPath()); + List pdsList = pds.values().collect(); + + DataSetIterator pdsIter = new PortableDataStreamDataSetIterator(pdsList); + int pdsCount = 0; + int totalExamples2 = 0; + while (pdsIter.hasNext()) { + DataSet ds = pdsIter.next(); + pdsCount++; + totalExamples2 += ds.numExamples(); + + assertEquals(4, ds.numInputs()); + assertEquals(3, ds.numOutcomes()); + } + + assertEquals(150, totalExamples2); + assertEquals(numBinFiles, pdsCount); + } finally { + FileUtils.deleteDirectory(temp); + } + } + + private static String toString(INDArray rowVector, int labelIdx) { + StringBuilder sb = new StringBuilder(); + long length = rowVector.length(); + for (int i = 0; i < length; i++) { + sb.append(rowVector.getDouble(i)); + sb.append(","); + } + sb.append(labelIdx); + return sb.toString(); + } + + + @Test + public void testCsvPreprocessedDataGenerationNoLabel() throws Exception { + //Same as above test, but without any labels (in which case: input and output arrays are the same) + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + List list = new ArrayList<>(); + DataSetIterator iter = new IrisDataSetIterator(1, 150); + while (iter.hasNext()) { + DataSet ds = iter.next(); + list.add(toString(ds.getFeatures(), Nd4j.argMax(ds.getLabels(), 1).getInt(0))); + } + + JavaRDD rdd = sc.parallelize(list); + int partitions = rdd.partitions().size(); + + URI tempDir = new File(System.getProperty("java.io.tmpdir")).toURI(); + URI outputDir = new URI(tempDir.getPath() + "/dl4j_testPreprocessedData3"); + File temp = new File(outputDir.getPath()); + if (temp.exists()) + FileUtils.deleteDirectory(temp); + + int numBinFiles = 0; + try { + int batchSize = 5; + int labelIdx = -1; + int numPossibleLabels = -1; + + rdd.foreachPartition(new StringToDataSetExportFunction(outputDir, new CSVRecordReader(0), batchSize, false, + labelIdx, numPossibleLabels)); + + File[] fileList = new File(outputDir.getPath()).listFiles(); + + int totalExamples = 0; + for (File f2 : fileList) { + if (!f2.getPath().endsWith(".bin")) + continue; + // System.out.println(f2.getPath()); + numBinFiles++; + + DataSet ds = new DataSet(); + ds.load(f2); + + assertEquals(5, ds.numInputs()); + assertEquals(5, ds.numOutcomes()); + + totalExamples += ds.numExamples(); + } + + assertEquals(150, totalExamples); + assertTrue(Math.abs(150 / batchSize - numBinFiles) <= partitions); //Expect 30, give or take due to partitioning randomness + + + + //Test the PortableDataStreamDataSetIterator: + JavaPairRDD pds = sc.binaryFiles(outputDir.getPath()); + List pdsList = pds.values().collect(); + + DataSetIterator pdsIter = new PortableDataStreamDataSetIterator(pdsList); + int pdsCount = 0; + int totalExamples2 = 0; + while (pdsIter.hasNext()) { + DataSet ds = pdsIter.next(); + pdsCount++; + totalExamples2 += ds.numExamples(); + + assertEquals(5, ds.numInputs()); + assertEquals(5, ds.numOutcomes()); + } + + assertEquals(150, totalExamples2); + assertEquals(numBinFiles, pdsCount); + } finally { + FileUtils.deleteDirectory(temp); + } + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/iterator/TestIteratorUtils.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/iterator/TestIteratorUtils.java new file mode 100644 index 000000000..30ce34c6b --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/datavec/iterator/TestIteratorUtils.java @@ -0,0 +1,144 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.datavec.iterator; + +import org.apache.spark.api.java.JavaRDD; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.split.FileSplit; +import org.datavec.api.writable.Writable; +import org.datavec.spark.transform.misc.StringToWritablesFunction; +import org.deeplearning4j.datasets.datavec.RecordReaderMultiDataSetIterator; +import org.deeplearning4j.spark.BaseSparkTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestIteratorUtils extends BaseSparkTest { + + @Override + public DataType getDataType() { + return DataType.FLOAT; + } + + @Override + public DataType getDefaultFPDataType() { + return DataType.FLOAT; + } + + @Test + public void testIrisRRMDSI() throws Exception { + + ClassPathResource cpr = new ClassPathResource("iris.txt"); + File f = cpr.getFile(); + RecordReader rr = new CSVRecordReader(); + rr.initialize(new FileSplit(f)); + + RecordReaderMultiDataSetIterator rrmdsi1 = new RecordReaderMultiDataSetIterator.Builder(1) + .addReader("reader", rr) + .addInput("reader", 0, 3) + .addOutputOneHot("reader", 4, 3) + .build(); + + RecordReaderMultiDataSetIterator rrmdsi2 = new RecordReaderMultiDataSetIterator.Builder(1) + .addReader("reader", new SparkSourceDummyReader(0)) + .addInput("reader", 0, 3) + .addOutputOneHot("reader", 4, 3) + .build(); + + List expected = new ArrayList<>(150); + while(rrmdsi1.hasNext()){ + expected.add(rrmdsi1.next()); + } + + JavaRDD> rdd = sc.textFile(f.getPath()).coalesce(1) + .map(new StringToWritablesFunction(new CSVRecordReader())); + + JavaRDD mdsRdd = IteratorUtils.mapRRMDSI(rdd, rrmdsi2); + + List act = mdsRdd.collect(); + + assertEquals(expected, act); + } + + @Test + public void testRRMDSIJoin() throws Exception { + + ClassPathResource cpr1 = new ClassPathResource("spark/rrmdsi/file1.txt"); + ClassPathResource cpr2 = new ClassPathResource("spark/rrmdsi/file2.txt"); + + RecordReader rr1 = new CSVRecordReader(); + rr1.initialize(new FileSplit(cpr1.getFile())); + RecordReader rr2 = new CSVRecordReader(); + rr2.initialize(new FileSplit(cpr2.getFile())); + + RecordReaderMultiDataSetIterator rrmdsi1 = new RecordReaderMultiDataSetIterator.Builder(1) + .addReader("r1", rr1) + .addReader("r2", rr2) + .addInput("r1", 1, 2) + .addOutput("r2",1,2) + .build(); + + RecordReaderMultiDataSetIterator rrmdsi2 = new RecordReaderMultiDataSetIterator.Builder(1) + .addReader("r1", new SparkSourceDummyReader(0)) + .addReader("r2", new SparkSourceDummyReader(1)) + .addInput("r1", 1, 2) + .addOutput("r2",1,2) + .build(); + + List expected = new ArrayList<>(3); + while(rrmdsi1.hasNext()){ + expected.add(rrmdsi1.next()); + } + + JavaRDD> rdd1 = sc.textFile(cpr1.getFile().getPath()).coalesce(1) + .map(new StringToWritablesFunction(new CSVRecordReader())); + JavaRDD> rdd2 = sc.textFile(cpr2.getFile().getPath()).coalesce(1) + .map(new StringToWritablesFunction(new CSVRecordReader())); + + List>> list = Arrays.asList(rdd1, rdd2); + JavaRDD mdsRdd = IteratorUtils.mapRRMDSI(list, null, new int[]{0,0}, null, false, rrmdsi2); + + List act = mdsRdd.collect(); + + + expected = new ArrayList<>(expected); + act = new ArrayList<>(act); + Comparator comp = new Comparator() { + @Override + public int compare(MultiDataSet d1, MultiDataSet d2) { + return Double.compare(d1.getFeatures(0).getDouble(0), d2.getFeatures(0).getDouble(0)); + } + }; + + Collections.sort(expected, comp); + Collections.sort(act, comp); + + assertEquals(expected, act); + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/TestKryoWarning.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/TestKryoWarning.java new file mode 100644 index 000000000..ec2195081 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/TestKryoWarning.java @@ -0,0 +1,142 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; + +import org.junit.jupiter.api.Test; + +public class TestKryoWarning { + + private static void doTestMLN(SparkConf sparkConf) { + JavaSparkContext sc = new JavaSparkContext(sparkConf); + + try { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() + .layer(0, new OutputLayer.Builder().nIn(10).nOut(10).build()) + .build(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1).build(); + + SparkDl4jMultiLayer sml = new SparkDl4jMultiLayer(sc, conf, tm); + } finally { + sc.stop(); + } + } + + private static void doTestCG(SparkConf sparkConf) { + JavaSparkContext sc = new JavaSparkContext(sparkConf); + + try { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in") + .addLayer("0", new OutputLayer.Builder().nIn(10).nOut(10).build(), "in").setOutputs("0") + .build(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1).build(); + + SparkListenable scg = new SparkComputationGraph(sc, conf, tm); + } finally { + sc.stop(); + } + } + + @Test + //@Ignore + public void testKryoMessageMLNIncorrectConfig() { + //Should print warning message + SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest") + .set("spark.driver.host", "localhost") + .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + + doTestMLN(sparkConf); + } + + @Test + //@Ignore + public void testKryoMessageMLNCorrectConfigKryo() { + //Should NOT print warning message + SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest") + .set("spark.driver.host", "localhost") + .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .set("spark.kryo.registrator", "org.nd4j.kryo.Nd4jRegistrator"); + + doTestMLN(sparkConf); + } + + @Test + //@Ignore + public void testKryoMessageMLNCorrectConfigNoKryo() { + //Should NOT print warning message + SparkConf sparkConf = new SparkConf().setMaster("local[*]") + .set("spark.driver.host", "localhost") + .setAppName("sparktest"); + + doTestMLN(sparkConf); + } + + + + @Test + //@Ignore + public void testKryoMessageCGIncorrectConfig() { + //Should print warning message + SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest") + .set("spark.driver.host", "localhost") + .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + + doTestCG(sparkConf); + } + + @Test + //@Ignore + public void testKryoMessageCGCorrectConfigKryo() { + //Should NOT print warning message + SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest") + .set("spark.driver.host", "localhost") + .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .set("spark.kryo.registrator", "org.nd4j.kryo.Nd4jRegistrator"); + + doTestCG(sparkConf); + } + + @Test + //@Ignore + public void testKryoMessageCGCorrectConfigNoKryo() { + //Should NOT print warning message + SparkConf sparkConf = new SparkConf().setMaster("local[*]") + .set("spark.driver.host", "localhost") + .setAppName("sparktest"); + + doTestCG(sparkConf); + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitionerTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitionerTest.java new file mode 100644 index 000000000..8559d5330 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitionerTest.java @@ -0,0 +1,80 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.repartition; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + + +public class BalancedPartitionerTest { + + + @Test + public void balancedPartitionerFirstElements() { + BalancedPartitioner bp = new BalancedPartitioner(10, 10, 0); + // the 10 first elements should go in the 1st partition + for (int i = 0; i < 10; i++) { + int p = bp.getPartition(i); + assertEquals( 0, p, "Found wrong partition output " + p + ", not 0"); + } + } + + @Test + public void balancedPartitionerFirstElementsWithRemainder() { + BalancedPartitioner bp = new BalancedPartitioner(10, 10, 1); + // the 10 first elements should go in the 1st partition + for (int i = 0; i < 10; i++) { + int p = bp.getPartition(i); + assertEquals( 0, p, "Found wrong partition output " + p + ", not 0"); + } + } + + @Test + public void balancedPartitionerDoesBalance() { + BalancedPartitioner bp = new BalancedPartitioner(10, 10, 0); + int[] countPerPartition = new int[10]; + for (int i = 0; i < 10 * 10; i++) { + int p = bp.getPartition(i); + countPerPartition[p] += 1; + } + for (int i = 0; i < 10; i++) { + assertEquals(10, countPerPartition[i]); + } + } + + @Test + public void balancedPartitionerDoesBalanceWithRemainder() { + BalancedPartitioner bp = new BalancedPartitioner(10, 10, 7); + int[] countPerPartition = new int[10]; + for (int i = 0; i < 10 * 10 + 7; i++) { + int p = bp.getPartition(i); + countPerPartition[p] += 1; + } + for (int i = 0; i < 10; i++) { + if (i < 7) + assertEquals(10 + 1, countPerPartition[i]); + else + assertEquals(10, countPerPartition[i]); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitionerTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitionerTest.java new file mode 100644 index 000000000..74e8f03be --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitionerTest.java @@ -0,0 +1,220 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.common.repartition; + +import org.apache.spark.HashPartitioner; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.Function2; +import org.apache.spark.api.java.function.PairFunction; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.impl.common.repartition.HashingBalancedPartitioner.LinearCongruentialGenerator; +import org.junit.jupiter.api.Test; +import scala.Tuple2; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertTrue; + + +public class HashingBalancedPartitionerTest extends BaseSparkTest { + + // e.g. we have 3 partitions, with red and blue elements, red is indexed by 0, blue by 1: + // [ r, r, r, r, b, b, b ], [r, b, b], [b, b, b, b, b, r, r] + // avg # red elems per partition : 2.33 + // avg # blue elems per partition : 3.33 + // partitionWeightsByClass = [[1.714, .429, .857], [0.9, 0.6, 1.5]] + + @Test + public void hashingBalancedPartitionerDoesBalance() { + // partitionWeightsByClass = [[1.714, .429, .857], [0.9, 0.6, 1.5]] + List reds = Arrays.asList(1.714D, 0.429D, .857D); + List blues = Arrays.asList(0.9D, 0.6D, 1.5D); + List> partitionWeights = Arrays.asList(reds, blues); + + HashingBalancedPartitioner hbp = new HashingBalancedPartitioner(partitionWeights); + List> l = new ArrayList<>(); + + for (int i = 0; i < 4; i++) { + l.add(new Tuple2(0, "red")); + } + for (int i = 0; i < 3; i++) { + l.add(new Tuple2(0, "blue")); + } + for (int i = 0; i < 1; i++) { + l.add(new Tuple2(1, "red")); + } + for (int i = 0; i < 2; i++) { + l.add(new Tuple2(1, "blue")); + } + for (int i = 0; i < 2; i++) { + l.add(new Tuple2(2, "red")); + } + for (int i = 0; i < 5; i++) { + l.add(new Tuple2(2, "blue")); + } + // This should give exactly the sought distribution + JavaPairRDD rdd = + JavaPairRDD.fromJavaRDD(sc.parallelize(l)).partitionBy(new HashPartitioner(3)); + + // Let's reproduce UIDs + JavaPairRDD, String> indexedRDD = rdd.zipWithUniqueId().mapToPair( + new PairFunction, Long>, Tuple2, String>() { + @Override + public Tuple2, String> call( + Tuple2, Long> payLoadNuid) { + Long uid = payLoadNuid._2(); + String value = payLoadNuid._1()._2(); + Integer elemClass = value.equals("red") ? 0 : 1; + return new Tuple2, String>( + new Tuple2(uid, elemClass), value); + } + }); + + List, String>> testList = indexedRDD.collect(); + + int[][] colorCountsByPartition = new int[3][2]; + for (final Tuple2, String> val : testList) { +// System.out.println(val); + Integer partition = hbp.getPartition(val._1()); +// System.out.println(partition); + + if (val._2().equals("red")) + colorCountsByPartition[partition][0] += 1; + else + colorCountsByPartition[partition][1] += 1; + } + +// for (int i = 0; i < 3; i++) { +// System.out.println(Arrays.toString(colorCountsByPartition[i])); +// } + for (int i = 0; i < 3; i++) { + // avg red per partition : 2.33 + assertTrue(colorCountsByPartition[i][0] >= 1 && colorCountsByPartition[i][0] < 4); + // avg blue per partition : 3.33 + assertTrue(colorCountsByPartition[i][1] >= 2 && colorCountsByPartition[i][1] < 5); + } + + } + + @Test + public void hashPartitionerBalancesAtScale() { + LinearCongruentialGenerator r = new LinearCongruentialGenerator(10000); + List elements = new ArrayList(); + for (int i = 0; i < 10000; i++) { + // The red occur towards the end + if (r.nextDouble() < ((double) i / 10000D)) + elements.add("red"); + // The blue occur towards the front + if (r.nextDouble() < (1 - (double) i / 10000D)) + elements.add("blue"); + } + Integer countRed = 0; + Integer countBlue = 0; + for (String elem : elements) { + if (elem.equals("red")) + countRed++; + else + countBlue++; + } + JavaRDD rdd = sc.parallelize(elements); + JavaPairRDD, String> indexedRDD = rdd.zipWithUniqueId() + .mapToPair(new PairFunction, Tuple2, String>() { + @Override + public Tuple2, String> call(Tuple2 stringLongTuple2) + throws Exception { + Integer elemClass = stringLongTuple2._1().equals("red") ? 0 : 1; + return new Tuple2, String>( + new Tuple2(stringLongTuple2._2(), elemClass), + stringLongTuple2._1()); + } + }); + + Integer numPartitions = indexedRDD.getNumPartitions(); + + // rdd and indexedRDD have the same partition distribution + List> partitionTuples = + rdd.mapPartitionsWithIndex(new CountRedBluePartitionsFunction(), true).collect(); + List redWeights = new ArrayList(); + List blueWeights = new ArrayList(); + Float avgRed = (float) countRed / numPartitions; + Float avgBlue = (float) countBlue / numPartitions; + for (int i = 0; i < partitionTuples.size(); i++) { + Tuple2 counts = partitionTuples.get(i); + redWeights.add((double) counts._1() / avgRed); + blueWeights.add((double) counts._2() / avgBlue); + } + List> partitionWeights = Arrays.asList(redWeights, blueWeights); + + + HashingBalancedPartitioner hbp = new HashingBalancedPartitioner(partitionWeights); + + List, String>> testList = indexedRDD.collect(); + + int[][] colorCountsByPartition = new int[numPartitions][2]; + for (final Tuple2, String> val : testList) { + Integer partition = hbp.getPartition(val._1()); + + if (val._2().equals("red")) + colorCountsByPartition[partition][0] += 1; + else + colorCountsByPartition[partition][1] += 1; + } + +// for (int i = 0; i < numPartitions; i++) { +// System.out.println(Arrays.toString(colorCountsByPartition[i])); +// } +// +// System.out.println("Ideal red # per partition: " + avgRed); +// System.out.println("Ideal blue # per partition: " + avgBlue); + + for (int i = 0; i < numPartitions; i++) { + // avg red per partition : 2.33 + assertTrue(colorCountsByPartition[i][0] >= Math.round(avgRed * .99) + && colorCountsByPartition[i][0] < Math.round(avgRed * 1.01) + 1); + // avg blue per partition : 3.33 + assertTrue(colorCountsByPartition[i][1] >= Math.round(avgBlue * .99) + && colorCountsByPartition[i][1] < Math.round(avgBlue * 1.01) + 1); + } + + + } + + class CountRedBluePartitionsFunction + implements Function2, Iterator>> { + @Override + public Iterator> call(Integer v1, Iterator v2) throws Exception { + + int redCount = 0; + int blueCount = 0; + while (v2.hasNext()) { + String elem = v2.next(); + if (elem.equals("red")) + redCount++; + else + blueCount++; + } + + return Collections.singletonList(new Tuple2<>(redCount, blueCount)).iterator(); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/TestCustomLayer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/TestCustomLayer.java new file mode 100644 index 000000000..b3c96333d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/TestCustomLayer.java @@ -0,0 +1,80 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.customlayer; + +import com.sun.jna.Platform; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.impl.customlayer.layer.CustomLayer; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +public class TestCustomLayer extends BaseSparkTest { + + @Test + public void testSparkWithCustomLayer() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + //Basic test - checks whether exceptions etc are thrown with custom layers + spark + //Custom layers are tested more extensively in dl4j core + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new CustomLayer(3.14159)).layer(2, + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .nIn(10).nOut(10).build()) + .build(); + + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1).averagingFrequency(2) + .batchSizePerWorker(5).saveUpdater(true).workerPrefetchNumBatches(0).build(); + + SparkDl4jMultiLayer net = new SparkDl4jMultiLayer(sc, conf, tm); + + List testData = new ArrayList<>(); + Random r = new Random(12345); + for (int i = 0; i < 200; i++) { + INDArray f = Nd4j.rand(1, 10); + INDArray l = Nd4j.zeros(1, 10); + l.putScalar(0, r.nextInt(10), 1.0); + testData.add(new DataSet(f, l)); + } + + JavaRDD rdd = sc.parallelize(testData); + net.fit(rdd); + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayer.java new file mode 100644 index 000000000..189e1f529 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayer.java @@ -0,0 +1,90 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.customlayer.layer; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.deeplearning4j.nn.api.ParamInitializer; +import org.deeplearning4j.nn.conf.InputPreProcessor; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; +import org.deeplearning4j.nn.conf.memory.LayerMemoryReport; +import org.deeplearning4j.nn.params.DefaultParamInitializer; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collection; +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CustomLayer extends FeedForwardLayer { + + private final double someCustomParameter; + + public CustomLayer(@JsonProperty("someCustomParameter") double someCustomParameter) { + this.someCustomParameter = someCustomParameter; + this.nIn = 10; + this.nOut = 10; + } + + @Override + public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf, + Collection trainingListeners, int layerIndex, INDArray layerParamsView, + boolean initializeParams, DataType networkDataType) { + CustomLayerImpl ret = new CustomLayerImpl(conf, networkDataType); + ret.setListeners(trainingListeners); + ret.setIndex(layerIndex); + ret.setParamsViewArray(layerParamsView); + Map paramTable = initializer().init(conf, layerParamsView, initializeParams); + ret.setParamTable(paramTable); + ret.setConf(conf); + return ret; + } + + @Override + public ParamInitializer initializer() { + return DefaultParamInitializer.getInstance(); + } + + @Override + public InputType getOutputType(int layerIndex, InputType inputType) { + return InputType.feedForward(10); + } + + @Override + public void setNIn(InputType inputType, boolean override) { + //No op + } + + @Override + public InputPreProcessor getPreProcessorForInputType(InputType inputType) { + return null; + } + + @Override + public LayerMemoryReport getMemoryReport(InputType inputType) { + throw new UnsupportedOperationException(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayerImpl.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayerImpl.java new file mode 100644 index 000000000..55b32d1dc --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayerImpl.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.customlayer.layer; + +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.layers.BaseLayer; +import org.nd4j.linalg.api.buffer.DataType; + +public class CustomLayerImpl extends BaseLayer { + public CustomLayerImpl(NeuralNetConfiguration conf, DataType dataType) { + super(conf, dataType); + } + + @Override + public boolean isPretrainLayer() { + return false; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/graph/TestSparkComputationGraph.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/graph/TestSparkComputationGraph.java new file mode 100644 index 000000000..cc6e5f9ec --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/graph/TestSparkComputationGraph.java @@ -0,0 +1,443 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.graph; + +import lombok.val; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.datavec.api.records.reader.RecordReader; +import org.datavec.api.records.reader.impl.csv.CSVRecordReader; +import org.datavec.api.split.FileSplit; +import org.deeplearning4j.datasets.datavec.RecordReaderMultiDataSetIterator; +import org.deeplearning4j.datasets.iterator.IteratorMultiDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.Updater; +import org.deeplearning4j.nn.conf.graph.ElementWiseVertex; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.api.RDDTrainingApproach; +import org.deeplearning4j.spark.api.Repartition; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.evaluation.IEvaluation; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.evaluation.classification.ROC; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import scala.Tuple2; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +//@Ignore("AB 2019/05/24 - Rarely getting stuck on CI - see issue #7657") +public class TestSparkComputationGraph extends BaseSparkTest { + + public static ComputationGraph getBasicNetIris2Class() { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER) + .graphBuilder().addInputs("in") + .addLayer("l0", new DenseLayer.Builder().nIn(4).nOut(10).build(), "in") + .addLayer("l1", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) + .activation(Activation.SOFTMAX).nIn(10).nOut(2).build(), "l0") + .setOutputs("l1").build(); + + ComputationGraph cg = new ComputationGraph(conf); + cg.init(); + + return cg; + } + + @Test + public void testBasic() throws Exception { + + JavaSparkContext sc = this.sc; + + RecordReader rr = new CSVRecordReader(0, ','); + rr.initialize(new FileSplit(new ClassPathResource("iris.txt").getTempFileFromArchive())); + MultiDataSetIterator iter = new RecordReaderMultiDataSetIterator.Builder(1).addReader("iris", rr) + .addInput("iris", 0, 3).addOutputOneHot("iris", 4, 3).build(); + + List list = new ArrayList<>(150); + while (iter.hasNext()) + list.add(iter.next()); + + ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder() + .updater(new Sgd(0.1)) + .graphBuilder().addInputs("in") + .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out", + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3) + .build(), + "dense") + .setOutputs("out").build(); + + ComputationGraph cg = new ComputationGraph(config); + cg.init(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0); + + SparkComputationGraph scg = new SparkComputationGraph(sc, cg, tm); + scg.setListeners(Collections.singleton((TrainingListener) new ScoreIterationListener(5))); + + JavaRDD rdd = sc.parallelize(list); + scg.fitMultiDataSet(rdd); + + //Try: fitting using DataSet + DataSetIterator iris = new IrisDataSetIterator(1, 150); + List list2 = new ArrayList<>(); + while (iris.hasNext()) + list2.add(iris.next()); + JavaRDD rddDS = sc.parallelize(list2); + + scg.fit(rddDS); + } + + + @Test + public void testDistributedScoring() { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().l1(0.1).l2(0.1) + .seed(123).updater(new Nesterovs(0.1, 0.9)).graphBuilder() + .addInputs("in") + .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(nIn).nOut(3) + .activation(Activation.TANH).build(), "in") + .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(3).nOut(nOut) + .activation(Activation.SOFTMAX).build(), + "0") + .setOutputs("1").build(); + + TrainingMaster tm = new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0); + + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf, tm); + ComputationGraph netCopy = sparkNet.getNetwork().clone(); + + int nRows = 100; + + INDArray features = Nd4j.rand(nRows, nIn); + INDArray labels = Nd4j.zeros(nRows, nOut); + Random r = new Random(12345); + for (int i = 0; i < nRows; i++) { + labels.putScalar(new int[] {i, r.nextInt(nOut)}, 1.0); + } + + INDArray localScoresWithReg = netCopy.scoreExamples(new DataSet(features, labels), true); + INDArray localScoresNoReg = netCopy.scoreExamples(new DataSet(features, labels), false); + + List> dataWithKeys = new ArrayList<>(); + for (int i = 0; i < nRows; i++) { + DataSet ds = new DataSet(features.getRow(i,true).dup(), labels.getRow(i,true).dup()); + dataWithKeys.add(new Tuple2<>(String.valueOf(i), ds)); + } + JavaPairRDD dataWithKeysRdd = sc.parallelizePairs(dataWithKeys); + + JavaPairRDD sparkScoresWithReg = sparkNet.scoreExamples(dataWithKeysRdd, true, 4); + JavaPairRDD sparkScoresNoReg = sparkNet.scoreExamples(dataWithKeysRdd, false, 4); + + Map sparkScoresWithRegMap = sparkScoresWithReg.collectAsMap(); + Map sparkScoresNoRegMap = sparkScoresNoReg.collectAsMap(); + + for (int i = 0; i < nRows; i++) { + double scoreRegExp = localScoresWithReg.getDouble(i); + double scoreRegAct = sparkScoresWithRegMap.get(String.valueOf(i)); + assertEquals(scoreRegExp, scoreRegAct, 1e-5); + + double scoreNoRegExp = localScoresNoReg.getDouble(i); + double scoreNoRegAct = sparkScoresNoRegMap.get(String.valueOf(i)); + assertEquals(scoreNoRegExp, scoreNoRegAct, 1e-5); + + // System.out.println(scoreRegExp + "\t" + scoreRegAct + "\t" + scoreNoRegExp + "\t" + scoreNoRegAct); + } + + List dataNoKeys = new ArrayList<>(); + for (int i = 0; i < nRows; i++) { + dataNoKeys.add(new DataSet(features.getRow(i,true).dup(), labels.getRow(i,true).dup())); + } + JavaRDD dataNoKeysRdd = sc.parallelize(dataNoKeys); + + List scoresWithReg = new ArrayList<>(sparkNet.scoreExamples(dataNoKeysRdd, true, 4).collect()); + List scoresNoReg = new ArrayList<>(sparkNet.scoreExamples(dataNoKeysRdd, false, 4).collect()); + Collections.sort(scoresWithReg); + Collections.sort(scoresNoReg); + double[] localScoresWithRegDouble = localScoresWithReg.data().asDouble(); + double[] localScoresNoRegDouble = localScoresNoReg.data().asDouble(); + Arrays.sort(localScoresWithRegDouble); + Arrays.sort(localScoresNoRegDouble); + + for (int i = 0; i < localScoresWithRegDouble.length; i++) { + assertEquals(localScoresWithRegDouble[i], scoresWithReg.get(i), 1e-5); + assertEquals(localScoresNoRegDouble[i], scoresNoReg.get(i), 1e-5); + + // System.out.println(localScoresWithRegDouble[i] + "\t" + scoresWithReg.get(i) + "\t" + localScoresNoRegDouble[i] + "\t" + scoresNoReg.get(i)); + } + } + + //@Ignore("AB 2019/05/23 - Failing on CI only - passing locally. Possible precision or threading issue") + public void testSeedRepeatability() throws Exception { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(Updater.RMSPROP) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in") + .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(4) + .activation(Activation.TANH).build(), "in") + .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(4).nOut(3).activation(Activation.SOFTMAX) + .build(), + "0") + .setOutputs("1").build(); + + Nd4j.getRandom().setSeed(12345); + ComputationGraph n1 = new ComputationGraph(conf.clone()); + n1.init(); + + Nd4j.getRandom().setSeed(12345); + ComputationGraph n2 = new ComputationGraph(conf.clone()); + n2.init(); + + Nd4j.getRandom().setSeed(12345); + ComputationGraph n3 = new ComputationGraph(conf.clone()); + n3.init(); + + SparkComputationGraph sparkNet1 = new SparkComputationGraph(sc, n1, + new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5) + .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always) + .rngSeed(12345).build()); + + Thread.sleep(100); //Training master IDs are only unique if they are created at least 1 ms apart... + + SparkComputationGraph sparkNet2 = new SparkComputationGraph(sc, n2, + new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5) + .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always) + .rngSeed(12345).build()); + + Thread.sleep(100); + + SparkComputationGraph sparkNet3 = new SparkComputationGraph(sc, n3, + new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5) + .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always) + .rngSeed(98765).build()); + + List data = new ArrayList<>(); + DataSetIterator iter = new IrisDataSetIterator(1, 150); + while (iter.hasNext()) + data.add(iter.next()); + + JavaRDD rdd = sc.parallelize(data); + + + sparkNet1.fit(rdd); + sparkNet2.fit(rdd); + sparkNet3.fit(rdd); + + + INDArray p1 = sparkNet1.getNetwork().params(); + INDArray p2 = sparkNet2.getNetwork().params(); + INDArray p3 = sparkNet3.getNetwork().params(); + + sparkNet1.getTrainingMaster().deleteTempFiles(sc); + sparkNet2.getTrainingMaster().deleteTempFiles(sc); + sparkNet3.getTrainingMaster().deleteTempFiles(sc); + + boolean eq1 = p1.equalsWithEps(p2, 0.01); + boolean eq2 = p1.equalsWithEps(p3, 0.01); + assertTrue(eq1, "Model 1 and 2 params should be equal"); + assertFalse(eq2, "Model 1 and 3 params shoud be different"); + } + + + @Test @Timeout(60) + public void testEvaluationAndRoc() { + for( int evalWorkers : new int[]{1, 4, 8}) { + DataSetIterator iter = new IrisDataSetIterator(5, 150); + + //Make a 2-class version of iris: + List l = new ArrayList<>(); + iter.reset(); + while (iter.hasNext()) { + DataSet ds = iter.next(); + INDArray newL = Nd4j.create(ds.getLabels().size(0), 2); + newL.putColumn(0, ds.getLabels().getColumn(0)); + newL.putColumn(1, ds.getLabels().getColumn(1)); + newL.getColumn(1).addi(ds.getLabels().getColumn(2)); + ds.setLabels(newL); + l.add(ds); + } + + iter = new ListDataSetIterator<>(l); + + ComputationGraph cg = getBasicNetIris2Class(); + + Evaluation e = cg.evaluate(iter); + ROC roc = cg.evaluateROC(iter, 32); + + + SparkComputationGraph scg = new SparkComputationGraph(sc, cg, null); + scg.setDefaultEvaluationWorkers(evalWorkers); + + + JavaRDD rdd = sc.parallelize(l); + rdd = rdd.repartition(20); + + Evaluation e2 = scg.evaluate(rdd); + ROC roc2 = scg.evaluateROC(rdd); + + + assertEquals(e2.accuracy(), e.accuracy(), 1e-3); + assertEquals(e2.f1(), e.f1(), 1e-3); + assertEquals(e2.getNumRowCounter(), e.getNumRowCounter(), 1e-3); + assertEquals(e2.falseNegatives(), e.falseNegatives()); + assertEquals(e2.falsePositives(), e.falsePositives()); + assertEquals(e2.trueNegatives(), e.trueNegatives()); + assertEquals(e2.truePositives(), e.truePositives()); + assertEquals(e2.precision(), e.precision(), 1e-3); + assertEquals(e2.recall(), e.recall(), 1e-3); + assertEquals(e2.getConfusionMatrix(), e.getConfusionMatrix()); + + assertEquals(roc.calculateAUC(), roc2.calculateAUC(), 1e-5); + assertEquals(roc.calculateAUCPR(), roc2.calculateAUCPR(), 1e-5); + } + } + + @Test + public void testEvaluationAndRocMDS() { + for( int evalWorkers : new int[]{1, 4, 8}) { + + DataSetIterator iter = new IrisDataSetIterator(5, 150); + + //Make a 2-class version of iris: + List l = new ArrayList<>(); + iter.reset(); + while (iter.hasNext()) { + DataSet ds = iter.next(); + INDArray newL = Nd4j.create(ds.getLabels().size(0), 2); + newL.putColumn(0, ds.getLabels().getColumn(0)); + newL.putColumn(1, ds.getLabels().getColumn(1)); + newL.getColumn(1).addi(ds.getLabels().getColumn(2)); + + MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(ds.getFeatures(), newL); + l.add(mds); + } + + MultiDataSetIterator mdsIter = new IteratorMultiDataSetIterator(l.iterator(), 5); + + ComputationGraph cg = getBasicNetIris2Class(); + + IEvaluation[] es = cg.doEvaluation(mdsIter, new Evaluation(), new ROC(32)); + Evaluation e = (Evaluation) es[0]; + ROC roc = (ROC) es[1]; + + + SparkComputationGraph scg = new SparkComputationGraph(sc, cg, null); + scg.setDefaultEvaluationWorkers(evalWorkers); + + JavaRDD rdd = sc.parallelize(l); + rdd = rdd.repartition(20); + + IEvaluation[] es2 = scg.doEvaluationMDS(rdd, 5, new Evaluation(), new ROC(32)); + Evaluation e2 = (Evaluation) es2[0]; + ROC roc2 = (ROC) es2[1]; + + + assertEquals(e2.accuracy(), e.accuracy(), 1e-3); + assertEquals(e2.f1(), e.f1(), 1e-3); + assertEquals(e2.getNumRowCounter(), e.getNumRowCounter(), 1e-3); + assertEquals(e2.falseNegatives(), e.falseNegatives()); + assertEquals(e2.falsePositives(), e.falsePositives()); + assertEquals(e2.trueNegatives(), e.trueNegatives()); + assertEquals(e2.truePositives(), e.truePositives()); + assertEquals(e2.precision(), e.precision(), 1e-3); + assertEquals(e2.recall(), e.recall(), 1e-3); + assertEquals(e2.getConfusionMatrix(), e.getConfusionMatrix()); + + assertEquals(roc.calculateAUC(), roc2.calculateAUC(), 1e-5); + assertEquals(roc.calculateAUCPR(), roc2.calculateAUCPR(), 1e-5); + } + } + + @Test + public void testIssue7068() throws Exception { + + val batchSize = 5; + val featSize = 10; + val labelSize = 2; + val random = new Random(0); + + List l = new ArrayList<>(); + for( int i=0; i<10; i++ ) { + org.nd4j.linalg.dataset.MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet( + new INDArray[]{Nd4j.rand(batchSize, featSize).castTo(DataType.DOUBLE), Nd4j.rand(batchSize, featSize).castTo(DataType.DOUBLE)}, + new INDArray[]{Nd4j.rand(batchSize, labelSize).castTo(DataType.DOUBLE)}); + l.add(mds); + } + JavaRDD rdd = sc.parallelize(l); + + // simple model + val modelConf = new NeuralNetConfiguration.Builder() + .updater(new Adam(0.01)) + .weightInit(WeightInit.XAVIER_UNIFORM) + .biasInit(0) + .graphBuilder() + .addInputs("input1", "input2") + .addVertex("avg",new ElementWiseVertex(ElementWiseVertex.Op.Average),"input1","input2") + .addLayer("dense",new DenseLayer.Builder().dropOut(0.9).nIn(featSize).nOut(featSize / 2).build(),"avg") + .addLayer("output",new OutputLayer.Builder().nIn(featSize / 2).nOut(2).lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).hasBias(false).build(),"dense") + .setOutputs("output") + .build(); + + val model = new ComputationGraph(modelConf); + model.init(); + + val trainingMaster = + new ParameterAveragingTrainingMaster.Builder(batchSize) + .rddTrainingApproach(RDDTrainingApproach.Direct) + .build(); + val sparkModel = + new SparkComputationGraph(sc, model, trainingMaster); + + for( int i=0; i<3; i++ ){ + sparkModel.fitMultiDataSet(rdd); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/misc/TestFrozenLayers.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/misc/TestFrozenLayers.java new file mode 100644 index 000000000..887696af3 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/misc/TestFrozenLayers.java @@ -0,0 +1,207 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.misc; + +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.layers.FrozenLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; +import org.deeplearning4j.nn.transferlearning.TransferLearning; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.api.RDDTrainingApproach; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestFrozenLayers extends BaseSparkTest { + + @Test + public void testSparkFrozenLayers() { + + NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.TANH); + + FineTuneConfiguration finetune = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build(); + + int nIn = 6; + int nOut = 3; + + MultiLayerNetwork origModel = new MultiLayerNetwork(overallConf.clone().list() + .layer(0, new DenseLayer.Builder().nIn(6).nOut(5).build()) + .layer(1, new DenseLayer.Builder().nIn(5).nOut(4).build()) + .layer(2, new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build()) + .build()); + origModel.init(); + + MultiLayerNetwork withFrozen = new TransferLearning.Builder(origModel).fineTuneConfiguration(finetune) + .setFeatureExtractor(1).build(); + + Map m = withFrozen.paramTable(); + Map pCopy = new HashMap<>(); + for (Map.Entry entry : m.entrySet()) { + pCopy.put(entry.getKey(), entry.getValue().dup()); + } + + + int avgFreq = 2; + int batchSize = 8; + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(batchSize) + .averagingFrequency(avgFreq).batchSizePerWorker(batchSize) + .rddTrainingApproach(RDDTrainingApproach.Direct).workerPrefetchNumBatches(0).build(); + + SparkDl4jMultiLayer sNet = new SparkDl4jMultiLayer(sc, withFrozen.clone(), tm); + + assertTrue(withFrozen.getLayer(0) instanceof FrozenLayer); + assertTrue(withFrozen.getLayer(1) instanceof FrozenLayer); + + int numMinibatches = 4 * sc.defaultParallelism(); + + List list = new ArrayList<>(); + for (int i = 0; i < numMinibatches; i++) { + INDArray f = Nd4j.rand(batchSize, nIn); + INDArray l = Nd4j.zeros(batchSize, nOut); + for (int j = 0; j < batchSize; j++) { + l.putScalar(j, j % nOut, 1.0); + } + list.add(new DataSet(f, l)); + } + + JavaRDD rdd = sc.parallelize(list); + + sNet.fit(rdd); + + MultiLayerNetwork fitted = sNet.getNetwork(); + + Map fittedParams = fitted.paramTable(); + + for (Map.Entry entry : fittedParams.entrySet()) { + INDArray orig = pCopy.get(entry.getKey()); + INDArray now = entry.getValue(); + boolean isFrozen = entry.getKey().startsWith("0_") || entry.getKey().startsWith("1_"); + + if (isFrozen) { + //Layer should be frozen -> no change + assertEquals(orig, now, entry.getKey()); + } else { + //Not frozen -> should be different + assertNotEquals(orig, now, entry.getKey()); + } + } + } + + + @Test + public void testSparkFrozenLayersCompGraph() { + + FineTuneConfiguration finetune = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build(); + + int nIn = 6; + int nOut = 3; + + ComputationGraph origModel = new ComputationGraph(new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)) + .activation(Activation.TANH).graphBuilder().addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(6).nOut(5).build(), "in") + .addLayer("1", new DenseLayer.Builder().nIn(5).nOut(4).build(), "0") + .addLayer("2", new DenseLayer.Builder().nIn(4).nOut(3).build(), "1") + .addLayer("3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3) + .build(), + "2") + .setOutputs("3").build()); + origModel.init(); + + ComputationGraph withFrozen = new TransferLearning.GraphBuilder(origModel).fineTuneConfiguration(finetune) + .setFeatureExtractor("1").build(); + + Map m = withFrozen.paramTable(); + Map pCopy = new HashMap<>(); + for (Map.Entry entry : m.entrySet()) { + pCopy.put(entry.getKey(), entry.getValue().dup()); + } + + + int avgFreq = 2; + int batchSize = 8; + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(batchSize) + .averagingFrequency(avgFreq).batchSizePerWorker(batchSize) + .rddTrainingApproach(RDDTrainingApproach.Direct).workerPrefetchNumBatches(0).build(); + + SparkComputationGraph sNet = new SparkComputationGraph(sc, withFrozen.clone(), tm); + + assertTrue(withFrozen.getLayer(0) instanceof FrozenLayer); + assertTrue(withFrozen.getLayer(1) instanceof FrozenLayer); + + int numMinibatches = 4 * sc.defaultParallelism(); + + List list = new ArrayList<>(); + for (int i = 0; i < numMinibatches; i++) { + INDArray f = Nd4j.rand(batchSize, nIn); + INDArray l = Nd4j.zeros(batchSize, nOut); + for (int j = 0; j < batchSize; j++) { + l.putScalar(j, j % nOut, 1.0); + } + list.add(new DataSet(f, l)); + } + + JavaRDD rdd = sc.parallelize(list); + + sNet.fit(rdd); + + ComputationGraph fitted = sNet.getNetwork(); + + Map fittedParams = fitted.paramTable(); + + for (Map.Entry entry : fittedParams.entrySet()) { + INDArray orig = pCopy.get(entry.getKey()); + INDArray now = entry.getValue(); + boolean isFrozen = entry.getKey().startsWith("0_") || entry.getKey().startsWith("1_"); + + if (isFrozen) { + //Layer should be frozen -> no change + assertEquals(orig, now, entry.getKey()); + } else { + //Not frozen -> should be different + assertNotEquals(orig, now, entry.getKey()); + } + } + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestMiscFunctions.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestMiscFunctions.java new file mode 100644 index 000000000..550ccc9b2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestMiscFunctions.java @@ -0,0 +1,302 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer; + +import org.apache.spark.api.java.JavaPairRDD; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution; +import org.deeplearning4j.nn.conf.layers.variational.LossFunctionWrapper; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.scoring.VaeReconstructionErrorWithKeyFunction; +import org.deeplearning4j.spark.impl.multilayer.scoring.VaeReconstructionProbWithKeyFunction; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.lossfunctions.impl.LossMSE; +import scala.Tuple2; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestMiscFunctions extends BaseSparkTest { + + @Test + public void testFeedForwardWithKey() { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3) + .activation(Activation.SOFTMAX).build()) + .build(); + + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + DataSetIterator iter = new IrisDataSetIterator(150, 150); + DataSet ds = iter.next(); + + + List expected = new ArrayList<>(); + List> mapFeatures = new ArrayList<>(); + int count = 0; + int arrayCount = 0; + Random r = new Random(12345); + while (count < 150) { + int exampleCount = r.nextInt(5) + 1; //1 to 5 inclusive examples + if (count + exampleCount > 150) + exampleCount = 150 - count; + + INDArray subset = ds.getFeatures().get(NDArrayIndex.interval(count, count + exampleCount), + NDArrayIndex.all()); + + expected.add(net.output(subset, false)); + mapFeatures.add(new Tuple2<>(arrayCount, subset)); + arrayCount++; + count += exampleCount; + } + +// JavaPairRDD rdd = sc.parallelizePairs(mapFeatures); + JavaPairRDD rdd = sc.parallelizePairs(mapFeatures); + + SparkDl4jMultiLayer multiLayer = new SparkDl4jMultiLayer(sc, net, null); + Map map = multiLayer.feedForwardWithKey(rdd, 16).collectAsMap(); + + for (int i = 0; i < expected.size(); i++) { + INDArray exp = expected.get(i); + INDArray act = map.get(i); + + assertEquals(exp, act); + } + } + + @Test + public void testFeedForwardWithKeyInputMask() { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER) + .list() + .layer( new LSTM.Builder().nIn(4).nOut(3).build()) + .layer(new GlobalPoolingLayer(PoolingType.AVG)) + .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(3).nOut(3) + .activation(Activation.SOFTMAX).build()) + .build(); + + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + List ds = Arrays.asList( + new org.nd4j.linalg.dataset.DataSet(Nd4j.rand(new int[]{1, 4, 5}), Nd4j.create(new double[]{1,1,1,0,0})), + new org.nd4j.linalg.dataset.DataSet(Nd4j.rand(new int[]{1, 4, 5}), Nd4j.create(new double[]{1,1,1,1,0})), + new org.nd4j.linalg.dataset.DataSet(Nd4j.rand(new int[]{1, 4, 5}), Nd4j.create(new double[]{1,1,1,1,1})) + ); + + + Map expected = new HashMap<>(); + List>> mapFeatures = new ArrayList<>(); + int count = 0; + int arrayCount = 0; + Random r = new Random(12345); + + + int i=0; + for(org.nd4j.linalg.dataset.DataSet d : ds){ + + INDArray f = d.getFeatures(); + INDArray fm = d.getFeaturesMaskArray(); + + mapFeatures.add(new Tuple2<>(i, new Tuple2<>(f, fm))); + + INDArray out = net.output(f, false, fm, null); + expected.put(i++, out); + } + + JavaPairRDD> rdd = sc.parallelizePairs(mapFeatures); + + SparkDl4jMultiLayer multiLayer = new SparkDl4jMultiLayer(sc, net, null); + Map map = multiLayer.feedForwardWithMaskAndKey(rdd, 16).collectAsMap(); + + for (i = 0; i < expected.size(); i++) { + INDArray exp = expected.get(i); + INDArray act = map.get(i); + + assertEquals(exp, act); + } + } + + + @Test + public void testFeedForwardWithKeyGraph() { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER) + .graphBuilder().addInputs("in1", "in2") + .addLayer("0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "in1") + .addLayer("1", new DenseLayer.Builder().nIn(4).nOut(3).build(), "in2").addLayer("2", + new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(6).nOut(3) + .activation(Activation.SOFTMAX).build(), + "0", "1") + .setOutputs("2").build(); + + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + + DataSetIterator iter = new IrisDataSetIterator(150, 150); + DataSet ds = iter.next(); + + + List expected = new ArrayList<>(); + List> mapFeatures = new ArrayList<>(); + int count = 0; + int arrayCount = 0; + Random r = new Random(12345); + while (count < 150) { + int exampleCount = r.nextInt(5) + 1; //1 to 5 inclusive examples + if (count + exampleCount > 150) + exampleCount = 150 - count; + + INDArray subset = ds.getFeatures().get(NDArrayIndex.interval(count, count + exampleCount), + NDArrayIndex.all()); + + expected.add(net.outputSingle(false, subset, subset)); + mapFeatures.add(new Tuple2<>(arrayCount, new INDArray[] {subset, subset})); + arrayCount++; + count += exampleCount; + } + + JavaPairRDD rdd = sc.parallelizePairs(mapFeatures); + + SparkComputationGraph graph = new SparkComputationGraph(sc, net, null); + Map map = graph.feedForwardWithKey(rdd, 16).collectAsMap(); + + for (int i = 0; i < expected.size(); i++) { + INDArray exp = expected.get(i); + INDArray act = map.get(i)[0]; + + assertEquals(exp, act); + } + } + + + @Test + public void testVaeReconstructionProbabilityWithKey() { + + //Simple test. We can't do a direct comparison, as the reconstruction probabilities are stochastic + // due to sampling + + int nIn = 10; + + MultiLayerConfiguration mlc = new NeuralNetConfiguration.Builder().list() + .layer(0, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder() + .reconstructionDistribution( + new GaussianReconstructionDistribution(Activation.IDENTITY)) + .nIn(nIn).nOut(5).encoderLayerSizes(12).decoderLayerSizes(13).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(mlc); + net.init(); + + List> toScore = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + INDArray arr = Nd4j.rand(1, nIn); + toScore.add(new Tuple2(i, arr)); + } + + JavaPairRDD rdd = sc.parallelizePairs(toScore); + + JavaPairRDD reconstr = + rdd.mapPartitionsToPair(new VaeReconstructionProbWithKeyFunction( + sc.broadcast(net.params()), sc.broadcast(mlc.toJson()), true, 16, 128)); + + Map l = reconstr.collectAsMap(); + + assertEquals(100, l.size()); + + for (int i = 0; i < 100; i++) { + assertTrue(l.containsKey(i)); + assertTrue(l.get(i) < 0.0); //log probability: should be negative + } + } + + + @Test + public void testVaeReconstructionErrorWithKey() { + //Simple test. We CAN do a direct comparison here vs. local, as reconstruction error is deterministic + + int nIn = 10; + + MultiLayerConfiguration mlc = new NeuralNetConfiguration.Builder() + .list().layer(0, + new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder() + .reconstructionDistribution(new LossFunctionWrapper( + Activation.IDENTITY, new LossMSE())) + .nIn(nIn).nOut(5).encoderLayerSizes(12).decoderLayerSizes(13) + .build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(mlc); + net.init(); + + VariationalAutoencoder vae = (VariationalAutoencoder) net.getLayer(0); + + List> toScore = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + INDArray arr = Nd4j.rand(1, nIn); + toScore.add(new Tuple2(i, arr)); + } + + JavaPairRDD rdd = sc.parallelizePairs(toScore); + + JavaPairRDD reconstrErrors = + rdd.mapPartitionsToPair(new VaeReconstructionErrorWithKeyFunction( + sc.broadcast(net.params()), sc.broadcast(mlc.toJson()), 16)); + + Map l = reconstrErrors.collectAsMap(); + + assertEquals(100, l.size()); + + for (int i = 0; i < 100; i++) { + assertTrue(l.containsKey(i)); + + INDArray localToScore = toScore.get(i)._2(); + double localScore = vae.reconstructionError(localToScore).data().asDouble()[0]; + + assertEquals(localScore, l.get(i), 1e-6); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestSparkDl4jMultiLayer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestSparkDl4jMultiLayer.java new file mode 100644 index 000000000..c64618557 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestSparkDl4jMultiLayer.java @@ -0,0 +1,153 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.multilayer; + +import com.sun.jna.Platform; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.junit.jupiter.api.Test; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class TestSparkDl4jMultiLayer extends BaseSparkTest { + + @Override + public long getTimeoutMilliseconds() { + return 120000L; + } + + @Override + public DataType getDataType() { + return DataType.FLOAT; + } + + @Override + public DataType getDefaultFPDataType() { + return DataType.FLOAT; + } + + @Test + public void testEvaluationSimple() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Nd4j.getRandom().setSeed(12345); + + for( int evalWorkers : new int[]{1, 4, 8}) { + //Simple test to validate DL4J issue 4099 is fixed... + + int numEpochs = 1; + int batchSizePerWorker = 8; + + //Load the data into memory then parallelize + //This isn't a good approach in general - but is simple to use for this example + DataSetIterator iterTrain = new MnistDataSetIterator(batchSizePerWorker, true, 12345); + DataSetIterator iterTest = new MnistDataSetIterator(batchSizePerWorker, false, 12345); + List trainDataList = new ArrayList<>(); + List testDataList = new ArrayList<>(); + int count = 0; + while (iterTrain.hasNext() && count++ < 30) { + trainDataList.add(iterTrain.next()); + } + while (iterTest.hasNext()) { + testDataList.add(iterTest.next()); + } + + JavaRDD trainData = sc.parallelize(trainDataList); + JavaRDD testData = sc.parallelize(testDataList); + + + //---------------------------------- + //Create network configuration and conduct network training + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .dataType(DataType.FLOAT) + .seed(12345) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .activation(Activation.LEAKYRELU) + .weightInit(WeightInit.XAVIER) + .updater(new Adam(1e-3)) + .l2(1e-5) + .list() + .layer(0, new DenseLayer.Builder().nIn(28 * 28).nOut(500).build()) + .layer(1, new DenseLayer.Builder().nIn(500).nOut(100).build()) + .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .activation(Activation.SOFTMAX).nIn(100).nOut(10).build()) + .build(); + + //Configuration for Spark training: see https://deeplearning4j.konduit.ai/distributed-deep-learning/howto for explanation of these configuration options + + TrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(batchSizePerWorker) + .averagingFrequency(2) + .build(); + + //Create the Spark network + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, tm); + sparkNet.setDefaultEvaluationWorkers(evalWorkers); + + //Execute training: + for (int i = 0; i < numEpochs; i++) { + sparkNet.fit(trainData); + } + + //Perform evaluation (distributed) + Evaluation evaluation = sparkNet.evaluate(testData); + log.info("***** Evaluation *****"); + log.info(evaluation.stats()); + + //Delete the temp training files, now that we are done with them + tm.deleteTempFiles(sc); + + assertEquals(10000, evaluation.getNumRowCounter()); //10k test set + assertTrue(!Double.isNaN(evaluation.accuracy())); + assertTrue(evaluation.accuracy() >= 0.10); + assertTrue(evaluation.precision() >= 0.10); + assertTrue(evaluation.recall() >= 0.10); + assertTrue(evaluation.f1() >= 0.10); + } + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestCompareParameterAveragingSparkVsSingleMachine.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestCompareParameterAveragingSparkVsSingleMachine.java new file mode 100644 index 000000000..cbe7247bd --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestCompareParameterAveragingSparkVsSingleMachine.java @@ -0,0 +1,595 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + +import com.sun.jna.Platform; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.spark.api.RDDTrainingApproach; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.learning.config.RmsProp; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestCompareParameterAveragingSparkVsSingleMachine { + @BeforeEach + public void setUp() { + //CudaEnvironment.getInstance().getConfiguration().allowMultiGPU(false); + } + + + private static MultiLayerConfiguration getConf(int seed, IUpdater updater) { + Nd4j.getRandom().setSeed(seed); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).build()) + .build(); + return conf; + } + + private static MultiLayerConfiguration getConfCNN(int seed, IUpdater updater) { + Nd4j.getRandom().setSeed(seed); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).list() + .layer(0, new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1).padding(0, 0) + .activation(Activation.TANH).build()) + .layer(1, new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1).padding(0, 0) + .activation(Activation.TANH).build()) + .layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(10) + .build()) + .setInputType(InputType.convolutional(10, 10, 3)).build(); + return conf; + } + + private static ComputationGraphConfiguration getGraphConf(int seed, IUpdater updater) { + Nd4j.getRandom().setSeed(seed); + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).graphBuilder() + .addInputs("in") + .addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in").addLayer("1", + new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(10) + .nOut(10).build(), + "0") + .setOutputs("1").build(); + return conf; + } + + private static ComputationGraphConfiguration getGraphConfCNN(int seed, IUpdater updater) { + Nd4j.getRandom().setSeed(seed); + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).graphBuilder() + .addInputs("in") + .addLayer("0", new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1) + .padding(0, 0).activation(Activation.TANH).build(), "in") + .addLayer("1", new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1) + .padding(0, 0).activation(Activation.TANH).build(), "0") + .addLayer("2", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(10) + .build(), "1") + .setOutputs("2").setInputTypes(InputType.convolutional(10, 10, 3)) + .build(); + return conf; + } + + private static TrainingMaster getTrainingMaster(int avgFreq, int miniBatchSize) { + return getTrainingMaster(avgFreq, miniBatchSize, true); + } + + private static TrainingMaster getTrainingMaster(int avgFreq, int miniBatchSize, boolean saveUpdater) { + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1) + .averagingFrequency(avgFreq).batchSizePerWorker(miniBatchSize).saveUpdater(saveUpdater) + .aggregationDepth(2).workerPrefetchNumBatches(0).build(); + return tm; + } + + private static JavaSparkContext getContext(int nWorkers) { + SparkConf sparkConf = new SparkConf(); + sparkConf.setMaster("local[" + nWorkers + "]"); + sparkConf.setAppName("Test"); + sparkConf.set("spark.driver.host", "localhost"); + + JavaSparkContext sc = new JavaSparkContext(sparkConf); + return sc; + } + + private List getOneDataSetAsIndividalExamples(int totalExamples, int seed) { + Nd4j.getRandom().setSeed(seed); + List list = new ArrayList<>(); + for (int i = 0; i < totalExamples; i++) { + INDArray f = Nd4j.rand(1, 10); + INDArray l = Nd4j.rand(1, 10); + DataSet ds = new DataSet(f, l); + list.add(ds); + } + return list; + } + + private List getOneDataSetAsIndividalExamplesCNN(int totalExamples, int seed) { + Nd4j.getRandom().setSeed(seed); + List list = new ArrayList<>(); + for (int i = 0; i < totalExamples; i++) { + INDArray f = Nd4j.rand(new int[] {1, 3, 10, 10}); + INDArray l = Nd4j.rand(1, 10); + DataSet ds = new DataSet(f, l); + list.add(ds); + } + return list; + } + + private DataSet getOneDataSet(int totalExamples, int seed) { + return DataSet.merge(getOneDataSetAsIndividalExamples(totalExamples, seed)); + } + + private DataSet getOneDataSetCNN(int totalExamples, int seed) { + return DataSet.merge(getOneDataSetAsIndividalExamplesCNN(totalExamples, seed)); + } + + @Test + public void testOneExecutor() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + //Idea: single worker/executor on Spark should give identical results to a single machine + + int miniBatchSize = 10; + int nWorkers = 1; + + for (boolean saveUpdater : new boolean[] {true, false}) { + JavaSparkContext sc = getContext(nWorkers); + + try { + //Do training locally, for 3 minibatches + int[] seeds = {1, 2, 3}; + + MultiLayerNetwork net = new MultiLayerNetwork(getConf(12345, new RmsProp(0.5))); + net.init(); + INDArray initialParams = net.params().dup(); + + for (int i = 0; i < seeds.length; i++) { + DataSet ds = getOneDataSet(miniBatchSize, seeds[i]); + if (!saveUpdater) + net.setUpdater(null); + net.fit(ds); + } + INDArray finalParams = net.params().dup(); + + //Do training on Spark with one executor, for 3 separate minibatches + TrainingMaster tm = getTrainingMaster(1, miniBatchSize, saveUpdater); + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, getConf(12345, new RmsProp(0.5)), tm); + sparkNet.setCollectTrainingStats(true); + INDArray initialSparkParams = sparkNet.getNetwork().params().dup(); + + for (int i = 0; i < seeds.length; i++) { + List list = getOneDataSetAsIndividalExamples(miniBatchSize, seeds[i]); + JavaRDD rdd = sc.parallelize(list); + + sparkNet.fit(rdd); + } + + INDArray finalSparkParams = sparkNet.getNetwork().params().dup(); + + assertEquals(initialParams, initialSparkParams); + assertNotEquals(initialParams, finalParams); + assertEquals(finalParams, finalSparkParams); + } finally { + sc.stop(); + } + } + } + + @Test + public void testOneExecutorGraph() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + //Idea: single worker/executor on Spark should give identical results to a single machine + + int miniBatchSize = 10; + int nWorkers = 1; + + for (boolean saveUpdater : new boolean[] {true, false}) { + JavaSparkContext sc = getContext(nWorkers); + + try { + //Do training locally, for 3 minibatches + int[] seeds = {1, 2, 3}; + + ComputationGraph net = new ComputationGraph(getGraphConf(12345, new RmsProp(0.5))); + net.init(); + INDArray initialParams = net.params().dup(); + + for (int i = 0; i < seeds.length; i++) { + DataSet ds = getOneDataSet(miniBatchSize, seeds[i]); + if (!saveUpdater) + net.setUpdater(null); + net.fit(ds); + } + INDArray finalParams = net.params().dup(); + + //Do training on Spark with one executor, for 3 separate minibatches + TrainingMaster tm = getTrainingMaster(1, miniBatchSize, saveUpdater); + SparkComputationGraph sparkNet = + new SparkComputationGraph(sc, getGraphConf(12345, new RmsProp(0.5)), tm); + sparkNet.setCollectTrainingStats(true); + INDArray initialSparkParams = sparkNet.getNetwork().params().dup(); + + for (int i = 0; i < seeds.length; i++) { + List list = getOneDataSetAsIndividalExamples(miniBatchSize, seeds[i]); + JavaRDD rdd = sc.parallelize(list); + + sparkNet.fit(rdd); + } + + INDArray finalSparkParams = sparkNet.getNetwork().params().dup(); + + assertEquals(initialParams, initialSparkParams); + assertNotEquals(initialParams, finalParams); + assertEquals(finalParams, finalSparkParams); + } finally { + sc.stop(); + } + } + } + + @Test + public void testAverageEveryStep() { + //Idea: averaging every step with SGD (SGD updater + optimizer) is mathematically identical to doing the learning + // on a single machine for synchronous distributed training + //BUT: This is *ONLY* the case if all workers get an identical number of examples. This won't be the case if + // we use RDD.randomSplit (which is what occurs if we use .fit(JavaRDD on a data set that needs splitting), + // which might give a number of examples that isn't divisible by number of workers (like 39 examples on 4 executors) + //This is also ONLY the case using SGD updater + + int miniBatchSizePerWorker = 10; + int nWorkers = 4; + + + for (boolean saveUpdater : new boolean[] {true, false}) { + JavaSparkContext sc = getContext(nWorkers); + + try { + //Do training locally, for 3 minibatches + int[] seeds = {1, 2, 3}; + + // CudaGridExecutioner executioner = (CudaGridExecutioner) Nd4j.getExecutioner(); + + MultiLayerNetwork net = new MultiLayerNetwork(getConf(12345, new Sgd(0.5))); + net.init(); + INDArray initialParams = net.params().dup(); + // executioner.addToWatchdog(initialParams, "initialParams"); + + + for (int i = 0; i < seeds.length; i++) { + DataSet ds = getOneDataSet(miniBatchSizePerWorker * nWorkers, seeds[i]); + if (!saveUpdater) + net.setUpdater(null); + net.fit(ds); + } + INDArray finalParams = net.params().dup(); + + //Do training on Spark with one executor, for 3 separate minibatches + // TrainingMaster tm = getTrainingMaster(1, miniBatchSizePerWorker, saveUpdater); + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1) + .averagingFrequency(1).batchSizePerWorker(miniBatchSizePerWorker) + .saveUpdater(saveUpdater).workerPrefetchNumBatches(0) + // .rddTrainingApproach(RDDTrainingApproach.Direct) + .rddTrainingApproach(RDDTrainingApproach.Export).build(); + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, getConf(12345, new Sgd(0.5)), tm); + sparkNet.setCollectTrainingStats(true); + INDArray initialSparkParams = sparkNet.getNetwork().params().dup(); + + // executioner.addToWatchdog(initialSparkParams, "initialSparkParams"); + + for (int i = 0; i < seeds.length; i++) { + List list = getOneDataSetAsIndividalExamples(miniBatchSizePerWorker * nWorkers, seeds[i]); + JavaRDD rdd = sc.parallelize(list); + + sparkNet.fit(rdd); + } + +// System.out.println(sparkNet.getSparkTrainingStats().statsAsString()); + sparkNet.getSparkTrainingStats().statsAsString(); + + INDArray finalSparkParams = sparkNet.getNetwork().params().dup(); + +// System.out.println("Initial (Local) params: " + Arrays.toString(initialParams.data().asFloat())); +// System.out.println("Initial (Spark) params: " +// + Arrays.toString(initialSparkParams.data().asFloat())); +// System.out.println("Final (Local) params: " + Arrays.toString(finalParams.data().asFloat())); +// System.out.println("Final (Spark) params: " + Arrays.toString(finalSparkParams.data().asFloat())); + assertEquals(initialParams, initialSparkParams); + assertNotEquals(initialParams, finalParams); + assertEquals(finalParams, finalSparkParams); + + double sparkScore = sparkNet.getScore(); + assertTrue(sparkScore > 0.0); + + assertEquals(net.score(), sparkScore, 1e-3); + } finally { + sc.stop(); + } + } + } + + @Test + public void testAverageEveryStepCNN() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + //Idea: averaging every step with SGD (SGD updater + optimizer) is mathematically identical to doing the learning + // on a single machine for synchronous distributed training + //BUT: This is *ONLY* the case if all workers get an identical number of examples. This won't be the case if + // we use RDD.randomSplit (which is what occurs if we use .fit(JavaRDD on a data set that needs splitting), + // which might give a number of examples that isn't divisible by number of workers (like 39 examples on 4 executors) + //This is also ONLY the case using SGD updater + + int miniBatchSizePerWorker = 10; + int nWorkers = 4; + + + for (boolean saveUpdater : new boolean[] {true, false}) { + JavaSparkContext sc = getContext(nWorkers); + + try { + //Do training locally, for 3 minibatches + int[] seeds = {1, 2, 3}; + + MultiLayerNetwork net = new MultiLayerNetwork(getConfCNN(12345, new Sgd(0.5))); + net.init(); + INDArray initialParams = net.params().dup(); + + for (int i = 0; i < seeds.length; i++) { + DataSet ds = getOneDataSetCNN(miniBatchSizePerWorker * nWorkers, seeds[i]); + if (!saveUpdater) + net.setUpdater(null); + net.fit(ds); + } + INDArray finalParams = net.params().dup(); + + //Do training on Spark with one executor, for 3 separate minibatches + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1) + .averagingFrequency(1).batchSizePerWorker(miniBatchSizePerWorker) + .saveUpdater(saveUpdater).workerPrefetchNumBatches(0) + .rddTrainingApproach(RDDTrainingApproach.Export).build(); + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, getConfCNN(12345, new Sgd(0.5)), tm); + sparkNet.setCollectTrainingStats(true); + INDArray initialSparkParams = sparkNet.getNetwork().params().dup(); + + for (int i = 0; i < seeds.length; i++) { + List list = + getOneDataSetAsIndividalExamplesCNN(miniBatchSizePerWorker * nWorkers, seeds[i]); + JavaRDD rdd = sc.parallelize(list); + + sparkNet.fit(rdd); + } + +// System.out.println(sparkNet.getSparkTrainingStats().statsAsString()); + sparkNet.getSparkTrainingStats().statsAsString(); + + INDArray finalSparkParams = sparkNet.getNetwork().params().dup(); + +// System.out.println("Initial (Local) params: " + Arrays.toString(initialParams.data().asFloat())); +// System.out.println("Initial (Spark) params: " +// + Arrays.toString(initialSparkParams.data().asFloat())); +// System.out.println("Final (Local) params: " + Arrays.toString(finalParams.data().asFloat())); +// System.out.println("Final (Spark) params: " + Arrays.toString(finalSparkParams.data().asFloat())); + assertArrayEquals(initialParams.data().asFloat(), initialSparkParams.data().asFloat(), 1e-8f); + assertArrayEquals(finalParams.data().asFloat(), finalSparkParams.data().asFloat(), 1e-6f); + + double sparkScore = sparkNet.getScore(); + assertTrue(sparkScore > 0.0); + + assertEquals(net.score(), sparkScore, 1e-3); + } finally { + sc.stop(); + } + } + } + + @Test + public void testAverageEveryStepGraph() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + //Idea: averaging every step with SGD (SGD updater + optimizer) is mathematically identical to doing the learning + // on a single machine for synchronous distributed training + //BUT: This is *ONLY* the case if all workers get an identical number of examples. This won't be the case if + // we use RDD.randomSplit (which is what occurs if we use .fit(JavaRDD on a data set that needs splitting), + // which might give a number of examples that isn't divisible by number of workers (like 39 examples on 4 executors) + //This is also ONLY the case using SGD updater + + int miniBatchSizePerWorker = 10; + int nWorkers = 4; + + + for (boolean saveUpdater : new boolean[] {true, false}) { + JavaSparkContext sc = getContext(nWorkers); + + try { + //Do training locally, for 3 minibatches + int[] seeds = {1, 2, 3}; + + // CudaGridExecutioner executioner = (CudaGridExecutioner) Nd4j.getExecutioner(); + + ComputationGraph net = new ComputationGraph(getGraphConf(12345, new Sgd(0.5))); + net.init(); + INDArray initialParams = net.params().dup(); + // executioner.addToWatchdog(initialParams, "initialParams"); + + for (int i = 0; i < seeds.length; i++) { + DataSet ds = getOneDataSet(miniBatchSizePerWorker * nWorkers, seeds[i]); + if (!saveUpdater) + net.setUpdater(null); + net.fit(ds); + } + INDArray finalParams = net.params().dup(); + // executioner.addToWatchdog(finalParams, "finalParams"); + + //Do training on Spark with one executor, for 3 separate minibatches + TrainingMaster tm = getTrainingMaster(1, miniBatchSizePerWorker, saveUpdater); + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, getGraphConf(12345, new Sgd(0.5)), tm); + sparkNet.setCollectTrainingStats(true); + INDArray initialSparkParams = sparkNet.getNetwork().params().dup(); + + // executioner.addToWatchdog(initialSparkParams, "initialSparkParams"); + + for (int i = 0; i < seeds.length; i++) { + List list = getOneDataSetAsIndividalExamples(miniBatchSizePerWorker * nWorkers, seeds[i]); + JavaRDD rdd = sc.parallelize(list); + + sparkNet.fit(rdd); + } + +// System.out.println(sparkNet.getSparkTrainingStats().statsAsString()); + sparkNet.getSparkTrainingStats().statsAsString(); + + INDArray finalSparkParams = sparkNet.getNetwork().params().dup(); + // executioner.addToWatchdog(finalSparkParams, "finalSparkParams"); + + float[] fp = finalParams.data().asFloat(); + float[] fps = finalSparkParams.data().asFloat(); +// System.out.println("Initial (Local) params: " + Arrays.toString(initialParams.data().asFloat())); +// System.out.println("Initial (Spark) params: " +// + Arrays.toString(initialSparkParams.data().asFloat())); +// System.out.println("Final (Local) params: " + Arrays.toString(fp)); +// System.out.println("Final (Spark) params: " + Arrays.toString(fps)); + + assertEquals(initialParams, initialSparkParams); + assertNotEquals(initialParams, finalParams); + assertArrayEquals(fp, fps, 1e-5f); + + double sparkScore = sparkNet.getScore(); + assertTrue(sparkScore > 0.0); + + assertEquals(net.score(), sparkScore, 1e-3); + } finally { + sc.stop(); + } + } + } + + @Test + public void testAverageEveryStepGraphCNN() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + //Idea: averaging every step with SGD (SGD updater + optimizer) is mathematically identical to doing the learning + // on a single machine for synchronous distributed training + //BUT: This is *ONLY* the case if all workers get an identical number of examples. This won't be the case if + // we use RDD.randomSplit (which is what occurs if we use .fit(JavaRDD on a data set that needs splitting), + // which might give a number of examples that isn't divisible by number of workers (like 39 examples on 4 executors) + //This is also ONLY the case using SGD updater + + int miniBatchSizePerWorker = 10; + int nWorkers = 4; + + + for (boolean saveUpdater : new boolean[] {true, false}) { + JavaSparkContext sc = getContext(nWorkers); + + try { + //Do training locally, for 3 minibatches + int[] seeds = {1, 2, 3}; + + ComputationGraph net = new ComputationGraph(getGraphConfCNN(12345, new Sgd(0.5))); + net.init(); + INDArray initialParams = net.params().dup(); + + for (int i = 0; i < seeds.length; i++) { + DataSet ds = getOneDataSetCNN(miniBatchSizePerWorker * nWorkers, seeds[i]); + if (!saveUpdater) + net.setUpdater(null); + net.fit(ds); + } + INDArray finalParams = net.params().dup(); + + //Do training on Spark with one executor, for 3 separate minibatches + TrainingMaster tm = getTrainingMaster(1, miniBatchSizePerWorker, saveUpdater); + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, getGraphConfCNN(12345, new Sgd(0.5)), tm); + sparkNet.setCollectTrainingStats(true); + INDArray initialSparkParams = sparkNet.getNetwork().params().dup(); + + for (int i = 0; i < seeds.length; i++) { + List list = + getOneDataSetAsIndividalExamplesCNN(miniBatchSizePerWorker * nWorkers, seeds[i]); + JavaRDD rdd = sc.parallelize(list); + + sparkNet.fit(rdd); + } + +// System.out.println(sparkNet.getSparkTrainingStats().statsAsString()); + sparkNet.getSparkTrainingStats().statsAsString(); + + INDArray finalSparkParams = sparkNet.getNetwork().params().dup(); + +// System.out.println("Initial (Local) params: " + Arrays.toString(initialParams.data().asFloat())); +// System.out.println("Initial (Spark) params: " + Arrays.toString(initialSparkParams.data().asFloat())); +// System.out.println("Final (Local) params: " + Arrays.toString(finalParams.data().asFloat())); +// System.out.println("Final (Spark) params: " + Arrays.toString(finalSparkParams.data().asFloat())); + assertArrayEquals(initialParams.data().asFloat(), initialSparkParams.data().asFloat(), 1e-8f); + assertArrayEquals(finalParams.data().asFloat(), finalSparkParams.data().asFloat(), 1e-6f); + + double sparkScore = sparkNet.getScore(); + assertTrue(sparkScore > 0.0); + + assertEquals(net.score(), sparkScore, 1e-3); + } finally { + sc.stop(); + } + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestJsonYaml.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestJsonYaml.java new file mode 100644 index 000000000..64c984ad7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestJsonYaml.java @@ -0,0 +1,52 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + +import org.apache.spark.storage.StorageLevel; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestJsonYaml { + + @Test + public void testJsonYaml() { + TrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(2).batchSizePerWorker(32) + .exportDirectory("hdfs://SomeDirectory/").saveUpdater(false).averagingFrequency(3) + .storageLevel(StorageLevel.MEMORY_ONLY_SER_2()).storageLevelStreams(StorageLevel.DISK_ONLY()) + .build(); + + String json = tm.toJson(); + String yaml = tm.toYaml(); + +// System.out.println(json); + + TrainingMaster fromJson = ParameterAveragingTrainingMaster.fromJson(json); + TrainingMaster fromYaml = ParameterAveragingTrainingMaster.fromYaml(yaml); + + + assertEquals(tm, fromJson); + assertEquals(tm, fromYaml); + + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java new file mode 100644 index 000000000..bc1ced484 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java @@ -0,0 +1,1092 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg; + + +import com.sun.jna.Platform; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.mllib.linalg.Vectors; +import org.apache.spark.mllib.regression.LabeledPoint; +import org.apache.spark.mllib.util.MLUtils; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.nn.api.Layer; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.BaseLayer; +import org.deeplearning4j.nn.conf.layers.BatchNormalization; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.variational.GaussianReconstructionDistribution; +import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.api.Repartition; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.stats.EventStats; +import org.deeplearning4j.spark.stats.ExampleCountEventStats; + + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.evaluation.classification.Evaluation; +import org.nd4j.evaluation.classification.ROC; +import org.nd4j.evaluation.classification.ROCMultiClass; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.IUpdater; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.learning.config.RmsProp; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import scala.Tuple2; + +import java.io.File; +import java.nio.file.Path; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + + +public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { + + public static class TestFn implements Function{ + @Override + public LabeledPoint call(LabeledPoint v1) throws Exception { + return new LabeledPoint(v1.label(), Vectors.dense(v1.features().toArray())); + } + } + + @TempDir + public File testDir; + + + @Override + public long getTimeoutMilliseconds() { + return 120000L; + } + + @Override + public DataType getDefaultFPDataType() { + return DataType.FLOAT; + } + + @Override + public DataType getDataType() { + return DataType.FLOAT; + } + + @Test + public void testFromSvmLightBackprop() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + JavaRDD data = MLUtils + .loadLibSVMFile(sc.sc(), + new ClassPathResource("svmLight/iris_svmLight_0.txt").getTempFileFromArchive() + .getAbsolutePath()) + .toJavaRDD().map(new TestFn()); + + DataSet d = new IrisDataSetIterator(150, 150).next(); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(100).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(100).nOut(3) + .activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER) + .build()) + .build(); + + + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + System.out.println("Initializing network"); + + SparkDl4jMultiLayer master = new SparkDl4jMultiLayer(sc, conf, + new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 5, 1, 0)); + + MultiLayerNetwork network2 = master.fitLabeledPoint(data); + } + + + @Test + public void testFromSvmLight() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + JavaRDD data = MLUtils + .loadLibSVMFile(sc.sc(), + new ClassPathResource("svmLight/iris_svmLight_0.txt").getTempFileFromArchive() + .getAbsolutePath()) + .toJavaRDD().map(new TestFn()); + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().seed(123) + .updater(new Adam(1e-6)) + .weightInit(WeightInit.XAVIER) + .list() + .layer(new BatchNormalization.Builder().nIn(4).nOut(4).build()) + .layer(new DenseLayer.Builder().nIn(4).nOut(32).activation(Activation.RELU).build()) + .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(32).nOut(3) + .activation(Activation.SOFTMAX).build()) + .build(); + + + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + System.out.println("Initializing network"); + SparkDl4jMultiLayer master = new SparkDl4jMultiLayer(sc, getBasicConf(), + new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 5, 1, 0)); + + master.fitLabeledPoint(data); + } + + @Test + public void testRunIteration() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + DataSet dataSet = new IrisDataSetIterator(5, 5).next(); + List list = dataSet.asList(); + JavaRDD data = sc.parallelize(list); + + SparkDl4jMultiLayer sparkNetCopy = new SparkDl4jMultiLayer(sc, getBasicConf(), + new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 5, 1, 0)); + MultiLayerNetwork networkCopy = sparkNetCopy.fit(data); + + INDArray expectedParams = networkCopy.params(); + + SparkDl4jMultiLayer sparkNet = getBasicNetwork(); + MultiLayerNetwork network = sparkNet.fit(data); + INDArray actualParams = network.params(); + + assertEquals(expectedParams.size(1), actualParams.size(1)); + } + + @Test + public void testUpdaters() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + SparkDl4jMultiLayer sparkNet = getBasicNetwork(); + MultiLayerNetwork netCopy = sparkNet.getNetwork().clone(); + + netCopy.fit(data); + IUpdater expectedUpdater = ((BaseLayer) netCopy.conf().getLayer()).getIUpdater(); + double expectedLR = ((Nesterovs)((BaseLayer) netCopy.conf().getLayer()).getIUpdater()).getLearningRate(); + double expectedMomentum = ((Nesterovs)((BaseLayer) netCopy.conf().getLayer()).getIUpdater()).getMomentum(); + + IUpdater actualUpdater = ((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getIUpdater(); + sparkNet.fit(sparkData); + double actualLR = ((Nesterovs)((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getIUpdater()).getLearningRate(); + double actualMomentum = ((Nesterovs)((BaseLayer) sparkNet.getNetwork().conf().getLayer()).getIUpdater()).getMomentum(); + + assertEquals(expectedUpdater, actualUpdater); + assertEquals(expectedLR, actualLR, 0.01); + assertEquals(expectedMomentum, actualMomentum, 0.01); + + } + + + @Test + public void testEvaluation() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + SparkDl4jMultiLayer sparkNet = getBasicNetwork(); + MultiLayerNetwork netCopy = sparkNet.getNetwork().clone(); + + Evaluation evalExpected = new Evaluation(); + INDArray outLocal = netCopy.output(input, Layer.TrainingMode.TEST); + evalExpected.eval(labels, outLocal); + + Evaluation evalActual = sparkNet.evaluate(sparkData); + + assertEquals(evalExpected.accuracy(), evalActual.accuracy(), 1e-3); + assertEquals(evalExpected.f1(), evalActual.f1(), 1e-3); + assertEquals(evalExpected.getNumRowCounter(), evalActual.getNumRowCounter(), 1e-3); + assertMapEquals(evalExpected.falseNegatives(), evalActual.falseNegatives()); + assertMapEquals(evalExpected.falsePositives(), evalActual.falsePositives()); + assertMapEquals(evalExpected.trueNegatives(), evalActual.trueNegatives()); + assertMapEquals(evalExpected.truePositives(), evalActual.truePositives()); + assertEquals(evalExpected.precision(), evalActual.precision(), 1e-3); + assertEquals(evalExpected.recall(), evalActual.recall(), 1e-3); + assertEquals(evalExpected.getConfusionMatrix(), evalActual.getConfusionMatrix()); + } + + private static void assertMapEquals(Map first, Map second) { + assertEquals(first.keySet(), second.keySet()); + for (Integer i : first.keySet()) { + assertEquals(first.get(i), second.get(i)); + } + } + + @Test + public void testSmallAmountOfData() { + //Idea: Test spark training where some executors don't get any data + //in this case: by having fewer examples (2 DataSets) than executors (local[*]) + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(nIn).nOut(3) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MSE).nIn(3).nOut(nOut).activation(Activation.SOFTMAX) + .build()) + .build(); + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, + new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0)); + + Nd4j.getRandom().setSeed(12345); + DataSet d1 = new DataSet(Nd4j.rand(1, nIn), Nd4j.rand(1, nOut)); + DataSet d2 = new DataSet(Nd4j.rand(1, nIn), Nd4j.rand(1, nOut)); + + JavaRDD rddData = sc.parallelize(Arrays.asList(d1, d2)); + + sparkNet.fit(rddData); + + } + + @Test + public void testDistributedScoring() { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l1(0.1).l2(0.1) + .seed(123).updater(new Nesterovs(0.1, 0.9)).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(nIn).nOut(3) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(3).nOut(nOut) + .activation(Activation.SOFTMAX).build()) + .build(); + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, + new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0)); + MultiLayerNetwork netCopy = sparkNet.getNetwork().clone(); + + int nRows = 100; + + INDArray features = Nd4j.rand(nRows, nIn); + INDArray labels = Nd4j.zeros(nRows, nOut); + Random r = new Random(12345); + for (int i = 0; i < nRows; i++) { + labels.putScalar(new int[] {i, r.nextInt(nOut)}, 1.0); + } + + INDArray localScoresWithReg = netCopy.scoreExamples(new DataSet(features, labels), true); + INDArray localScoresNoReg = netCopy.scoreExamples(new DataSet(features, labels), false); + + List> dataWithKeys = new ArrayList<>(); + for (int i = 0; i < nRows; i++) { + DataSet ds = new DataSet(features.getRow(i,true).dup(), labels.getRow(i,true).dup()); + dataWithKeys.add(new Tuple2<>(String.valueOf(i), ds)); + } + JavaPairRDD dataWithKeysRdd = sc.parallelizePairs(dataWithKeys); + + JavaPairRDD sparkScoresWithReg = sparkNet.scoreExamples(dataWithKeysRdd, true, 4); + JavaPairRDD sparkScoresNoReg = sparkNet.scoreExamples(dataWithKeysRdd, false, 4); + + Map sparkScoresWithRegMap = sparkScoresWithReg.collectAsMap(); + Map sparkScoresNoRegMap = sparkScoresNoReg.collectAsMap(); + + for (int i = 0; i < nRows; i++) { + double scoreRegExp = localScoresWithReg.getDouble(i); + double scoreRegAct = sparkScoresWithRegMap.get(String.valueOf(i)); + assertEquals(scoreRegExp, scoreRegAct, 1e-5); + + double scoreNoRegExp = localScoresNoReg.getDouble(i); + double scoreNoRegAct = sparkScoresNoRegMap.get(String.valueOf(i)); + assertEquals(scoreNoRegExp, scoreNoRegAct, 1e-5); + + // System.out.println(scoreRegExp + "\t" + scoreRegAct + "\t" + scoreNoRegExp + "\t" + scoreNoRegAct); + } + + List dataNoKeys = new ArrayList<>(); + for (int i = 0; i < nRows; i++) { + dataNoKeys.add(new DataSet(features.getRow(i,true).dup(), labels.getRow(i,true).dup())); + } + JavaRDD dataNoKeysRdd = sc.parallelize(dataNoKeys); + + List scoresWithReg = new ArrayList<>(sparkNet.scoreExamples(dataNoKeysRdd, true, 4).collect()); + List scoresNoReg = new ArrayList<>(sparkNet.scoreExamples(dataNoKeysRdd, false, 4).collect()); + Collections.sort(scoresWithReg); + Collections.sort(scoresNoReg); + double[] localScoresWithRegDouble = localScoresWithReg.data().asDouble(); + double[] localScoresNoRegDouble = localScoresNoReg.data().asDouble(); + Arrays.sort(localScoresWithRegDouble); + Arrays.sort(localScoresNoRegDouble); + + for (int i = 0; i < localScoresWithRegDouble.length; i++) { + assertEquals(localScoresWithRegDouble[i], scoresWithReg.get(i), 1e-5); + assertEquals(localScoresNoRegDouble[i], scoresNoReg.get(i), 1e-5); + + //System.out.println(localScoresWithRegDouble[i] + "\t" + scoresWithReg.get(i) + "\t" + localScoresNoRegDouble[i] + "\t" + scoresNoReg.get(i)); + } + } + + + + @Test + public void testParameterAveragingMultipleExamplesPerDataSet() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + int dataSetObjSize = 5; + int batchSizePerExecutor = 25; + List list = new ArrayList<>(); + DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, 1000, false); + while (iter.hasNext()) { + list.add(iter.next()); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10) + .activation(Activation.SOFTMAX).build()) + .build(); + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1) + .aggregationDepth(1).repartionData(Repartition.Always).build()); + sparkNet.setCollectTrainingStats(true); + + JavaRDD rdd = sc.parallelize(list); + + sparkNet.fit(rdd); + + SparkTrainingStats stats = sparkNet.getSparkTrainingStats(); + + List mapPartitionStats = stats.getValue("ParameterAveragingMasterMapPartitionsTimesMs"); + int numSplits = list.size() * dataSetObjSize / (numExecutors() * batchSizePerExecutor); //For an averaging frequency of 1 + assertEquals(numSplits, mapPartitionStats.size()); + + + List workerFitStats = stats.getValue("ParameterAveragingWorkerFitTimesMs"); + for (EventStats e : workerFitStats) { + ExampleCountEventStats eces = (ExampleCountEventStats) e; +// System.out.println(eces.getTotalExampleCount()); + } + + for (EventStats e : workerFitStats) { + ExampleCountEventStats eces = (ExampleCountEventStats) e; + assertEquals(batchSizePerExecutor, eces.getTotalExampleCount()); + } + } + + + @Test + public void testFitViaStringPaths() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Path tempDir = new File(testDir, "DL4J-testFitViaStringPaths").toPath(); + File tempDirF = tempDir.toFile(); + tempDirF.deleteOnExit(); + + int dataSetObjSize = 5; + int batchSizePerExecutor = 25; + DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, 1000, false); + int i = 0; + while (iter.hasNext()) { + File nextFile = new File(tempDirF, i + ".bin"); + DataSet ds = iter.next(); + ds.save(nextFile); + i++; + } + + System.out.println("Saved to: " + tempDirF.getAbsolutePath()); + + + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10) + .activation(Activation.SOFTMAX).build()) + .build(); + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .workerPrefetchNumBatches(5).batchSizePerWorker(batchSizePerExecutor) + .averagingFrequency(1).repartionData(Repartition.Always).build()); + sparkNet.setCollectTrainingStats(true); + + + //List files: + Configuration config = new Configuration(); + FileSystem hdfs = FileSystem.get(tempDir.toUri(), config); + RemoteIterator fileIter = + hdfs.listFiles(new org.apache.hadoop.fs.Path(tempDir.toString()), false); + + List paths = new ArrayList<>(); + while (fileIter.hasNext()) { + String path = fileIter.next().getPath().toString(); + paths.add(path); + } + + INDArray paramsBefore = sparkNet.getNetwork().params().dup(); + JavaRDD pathRdd = sc.parallelize(paths); + sparkNet.fitPaths(pathRdd); + + INDArray paramsAfter = sparkNet.getNetwork().params().dup(); + assertNotEquals(paramsBefore, paramsAfter); + + SparkTrainingStats stats = sparkNet.getSparkTrainingStats(); +// System.out.println(stats.statsAsString()); + stats.statsAsString(); + + sparkNet.getTrainingMaster().deleteTempFiles(sc); + } + + @Test + public void testFitViaStringPathsSize1() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Path tempDir = new File(testDir, "DL4J-testFitViaStringPathsSize1").toPath(); + File tempDirF = tempDir.toFile(); + tempDirF.deleteOnExit(); + + int dataSetObjSize = 1; + int batchSizePerExecutor = 4; + int numSplits = 3; + int averagingFrequency = 3; + int totalExamples = numExecutors() * batchSizePerExecutor * numSplits * averagingFrequency; + DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, totalExamples, false); + int i = 0; + while (iter.hasNext()) { + File nextFile = new File(tempDirF, i + ".bin"); + DataSet ds = iter.next(); + ds.save(nextFile); + i++; + } + +// System.out.println("Saved to: " + tempDirF.getAbsolutePath()); + + + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10) + .activation(Activation.SOFTMAX).build()) + .build(); + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .workerPrefetchNumBatches(5).batchSizePerWorker(batchSizePerExecutor) + .averagingFrequency(averagingFrequency).repartionData(Repartition.Always) + .build()); + sparkNet.setCollectTrainingStats(true); + + + //List files: + Configuration config = new Configuration(); + FileSystem hdfs = FileSystem.get(tempDir.toUri(), config); + RemoteIterator fileIter = + hdfs.listFiles(new org.apache.hadoop.fs.Path(tempDir.toString()), false); + + List paths = new ArrayList<>(); + while (fileIter.hasNext()) { + String path = fileIter.next().getPath().toString(); + paths.add(path); + } + + INDArray paramsBefore = sparkNet.getNetwork().params().dup(); + JavaRDD pathRdd = sc.parallelize(paths); + sparkNet.fitPaths(pathRdd); + + INDArray paramsAfter = sparkNet.getNetwork().params().dup(); + assertNotEquals(paramsBefore, paramsAfter); + + Thread.sleep(200); + SparkTrainingStats stats = sparkNet.getSparkTrainingStats(); + + //Expect +// System.out.println(stats.statsAsString()); + stats.statsAsString(); + assertEquals(numSplits, stats.getValue("ParameterAveragingMasterRepartitionTimesMs").size()); + + List list = stats.getValue("ParameterAveragingWorkerFitTimesMs"); + assertEquals(numSplits * numExecutors() * averagingFrequency, list.size()); + for (EventStats es : list) { + ExampleCountEventStats e = (ExampleCountEventStats) es; + assertTrue(batchSizePerExecutor * averagingFrequency >= e.getTotalExampleCount()); + } + + + sparkNet.getTrainingMaster().deleteTempFiles(sc); + } + + + @Test + public void testFitViaStringPathsCompGraph() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + Path tempDir = new File(testDir, "DL4J-testFitViaStringPathsCG").toPath(); + Path tempDir2 = new File(testDir, "DL4J-testFitViaStringPathsCG-MDS").toPath(); + File tempDirF = tempDir.toFile(); + File tempDirF2 = tempDir2.toFile(); + tempDirF.deleteOnExit(); + tempDirF2.deleteOnExit(); + + int dataSetObjSize = 4; + int batchSizePerExecutor = 8; + DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, 128, false); + int i = 0; + while (iter.hasNext()) { + File nextFile = new File(tempDirF, i + ".bin"); + File nextFile2 = new File(tempDirF2, i + ".bin"); + DataSet ds = iter.next(); + MultiDataSet mds = new MultiDataSet(ds.getFeatures(), ds.getLabels()); + ds.save(nextFile); + mds.save(nextFile2); + i++; + } + +// System.out.println("Saved to: " + tempDirF.getAbsolutePath()); +// System.out.println("Saved to: " + tempDirF2.getAbsolutePath()); + + + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .graphBuilder().addInputs("in") + .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50) + .activation(Activation.TANH).build(), "in") + .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10) + .activation(Activation.SOFTMAX).build(), + "0") + .setOutputs("1").build(); + + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf, + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .workerPrefetchNumBatches(5).workerPrefetchNumBatches(0) + .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(1) + .repartionData(Repartition.Always).build()); + sparkNet.setCollectTrainingStats(true); + + + //List files: + Configuration config = new Configuration(); + FileSystem hdfs = FileSystem.get(tempDir.toUri(), config); + RemoteIterator fileIter = + hdfs.listFiles(new org.apache.hadoop.fs.Path(tempDir.toString()), false); + + List paths = new ArrayList<>(); + while (fileIter.hasNext()) { + String path = fileIter.next().getPath().toString(); + paths.add(path); + } + + INDArray paramsBefore = sparkNet.getNetwork().params().dup(); + JavaRDD pathRdd = sc.parallelize(paths); + sparkNet.fitPaths(pathRdd); + + INDArray paramsAfter = sparkNet.getNetwork().params().dup(); + assertNotEquals(paramsBefore, paramsAfter); + + SparkTrainingStats stats = sparkNet.getSparkTrainingStats(); +// System.out.println(stats.statsAsString()); + stats.statsAsString(); + + //Same thing, buf for MultiDataSet objects: + config = new Configuration(); + hdfs = FileSystem.get(tempDir2.toUri(), config); + fileIter = hdfs.listFiles(new org.apache.hadoop.fs.Path(tempDir2.toString()), false); + + paths = new ArrayList<>(); + while (fileIter.hasNext()) { + String path = fileIter.next().getPath().toString(); + paths.add(path); + } + + paramsBefore = sparkNet.getNetwork().params().dup(); + pathRdd = sc.parallelize(paths); + sparkNet.fitPathsMultiDataSet(pathRdd); + + paramsAfter = sparkNet.getNetwork().params().dup(); + assertNotEquals(paramsBefore, paramsAfter); + + stats = sparkNet.getSparkTrainingStats(); +// System.out.println(stats.statsAsString()); + stats.statsAsString(); + } + + + @Test + //@Ignore("AB 2019/05/23 - Failing on CI only - passing locally. Possible precision or threading issue") + public void testSeedRepeatability() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new RmsProp()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .weightInit(WeightInit.XAVIER).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(4).nOut(4) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(4).nOut(3).activation(Activation.SOFTMAX) + .build()) + .build(); + + Nd4j.getRandom().setSeed(12345); + MultiLayerNetwork n1 = new MultiLayerNetwork(conf); + n1.init(); + + Nd4j.getRandom().setSeed(12345); + MultiLayerNetwork n2 = new MultiLayerNetwork(conf); + n2.init(); + + Nd4j.getRandom().setSeed(12345); + MultiLayerNetwork n3 = new MultiLayerNetwork(conf); + n3.init(); + + SparkDl4jMultiLayer sparkNet1 = new SparkDl4jMultiLayer(sc, n1, + new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5) + .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always) + .rngSeed(12345).build()); + + Thread.sleep(100); //Training master IDs are only unique if they are created at least 1 ms apart... + + SparkDl4jMultiLayer sparkNet2 = new SparkDl4jMultiLayer(sc, n2, + new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5) + .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always) + .rngSeed(12345).build()); + + Thread.sleep(100); + + SparkDl4jMultiLayer sparkNet3 = new SparkDl4jMultiLayer(sc, n3, + new ParameterAveragingTrainingMaster.Builder(1).workerPrefetchNumBatches(5) + .batchSizePerWorker(5).averagingFrequency(1).repartionData(Repartition.Always) + .rngSeed(98765).build()); + + List data = new ArrayList<>(); + DataSetIterator iter = new IrisDataSetIterator(1, 150); + while (iter.hasNext()) + data.add(iter.next()); + + JavaRDD rdd = sc.parallelize(data); + + + sparkNet1.fit(rdd); + sparkNet2.fit(rdd); + sparkNet3.fit(rdd); + + + INDArray p1 = sparkNet1.getNetwork().params(); + INDArray p2 = sparkNet2.getNetwork().params(); + INDArray p3 = sparkNet3.getNetwork().params(); + + sparkNet1.getTrainingMaster().deleteTempFiles(sc); + sparkNet2.getTrainingMaster().deleteTempFiles(sc); + sparkNet3.getTrainingMaster().deleteTempFiles(sc); + + boolean eq1 = p1.equalsWithEps(p2, 0.01); + boolean eq2 = p1.equalsWithEps(p3, 0.01); + assertTrue(eq1, "Model 1 and 2 params should be equal"); + assertFalse(eq2, "Model 1 and 3 params shoud be different"); + } + + + @Test + public void testIterationCounts() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + int dataSetObjSize = 5; + int batchSizePerExecutor = 25; + List list = new ArrayList<>(); + int minibatchesPerWorkerPerEpoch = 10; + DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, + batchSizePerExecutor * numExecutors() * minibatchesPerWorkerPerEpoch, false); + while (iter.hasNext()) { + list.add(iter.next()); + } + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10) + .activation(Activation.SOFTMAX).build()) + .build(); + + for (int avgFreq : new int[] {1, 5, 10}) { +// System.out.println("--- Avg freq " + avgFreq + " ---"); + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf.clone(), + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(avgFreq) + .repartionData(Repartition.Always).build()); + + sparkNet.setListeners(new ScoreIterationListener(5)); + + + + JavaRDD rdd = sc.parallelize(list); + + assertEquals(0, sparkNet.getNetwork().getLayerWiseConfigurations().getIterationCount()); + sparkNet.fit(rdd); + assertEquals(minibatchesPerWorkerPerEpoch, + sparkNet.getNetwork().getLayerWiseConfigurations().getIterationCount()); + sparkNet.fit(rdd); + assertEquals(2 * minibatchesPerWorkerPerEpoch, + sparkNet.getNetwork().getLayerWiseConfigurations().getIterationCount()); + + sparkNet.getTrainingMaster().deleteTempFiles(sc); + } + } + + @Test + public void testIterationCountsGraph() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + int dataSetObjSize = 5; + int batchSizePerExecutor = 25; + List list = new ArrayList<>(); + int minibatchesPerWorkerPerEpoch = 10; + DataSetIterator iter = new MnistDataSetIterator(dataSetObjSize, + batchSizePerExecutor * numExecutors() * minibatchesPerWorkerPerEpoch, false); + while (iter.hasNext()) { + list.add(iter.next()); + } + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp()) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) + .graphBuilder().addInputs("in") + .addLayer("0", new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(50) + .activation(Activation.TANH).build(), "in") + .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(50).nOut(10) + .activation(Activation.SOFTMAX).build(), + "0") + .setOutputs("1").build(); + + for (int avgFreq : new int[] {1, 5, 10}) { +// System.out.println("--- Avg freq " + avgFreq + " ---"); + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf.clone(), + new ParameterAveragingTrainingMaster.Builder(numExecutors(), dataSetObjSize) + .batchSizePerWorker(batchSizePerExecutor).averagingFrequency(avgFreq) + .repartionData(Repartition.Always).build()); + + sparkNet.setListeners(new ScoreIterationListener(5)); + + JavaRDD rdd = sc.parallelize(list); + + assertEquals(0, sparkNet.getNetwork().getConfiguration().getIterationCount()); + sparkNet.fit(rdd); + assertEquals(minibatchesPerWorkerPerEpoch, sparkNet.getNetwork().getConfiguration().getIterationCount()); + sparkNet.fit(rdd); + assertEquals(2 * minibatchesPerWorkerPerEpoch, + sparkNet.getNetwork().getConfiguration().getIterationCount()); + + sparkNet.getTrainingMaster().deleteTempFiles(sc); + } + } + + + @Test + //@Ignore //Ignored 2019/04/09 - low priority: https://github.com/eclipse/deeplearning4j/issues/6656 + public void testVaePretrainSimple() { + //Simple sanity check on pretraining + int nIn = 8; + + Nd4j.getRandom().setSeed(12345); + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new RmsProp()) + .weightInit(WeightInit.XAVIER).list() + .layer(0, new VariationalAutoencoder.Builder().nIn(8).nOut(10).encoderLayerSizes(12) + .decoderLayerSizes(13).reconstructionDistribution( + new GaussianReconstructionDistribution(Activation.IDENTITY)) + .build()) + .build(); + + //Do training on Spark with one executor, for 3 separate minibatches + int rddDataSetNumExamples = 10; + int totalAveragings = 5; + int averagingFrequency = 3; + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples) + .averagingFrequency(averagingFrequency).batchSizePerWorker(rddDataSetNumExamples) + .saveUpdater(true).workerPrefetchNumBatches(0).build(); + Nd4j.getRandom().setSeed(12345); + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf.clone(), tm); + + List trainData = new ArrayList<>(); + int nDataSets = numExecutors() * totalAveragings * averagingFrequency; + for (int i = 0; i < nDataSets; i++) { + trainData.add(new DataSet(Nd4j.rand(rddDataSetNumExamples, nIn), null)); + } + + JavaRDD data = sc.parallelize(trainData); + + sparkNet.fit(data); + } + + @Test + //@Ignore //Ignored 2019/04/09 - low priority: https://github.com/eclipse/deeplearning4j/issues/6656 + public void testVaePretrainSimpleCG() { + //Simple sanity check on pretraining + int nIn = 8; + + Nd4j.getRandom().setSeed(12345); + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new RmsProp()) + .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in") + .addLayer("0", new VariationalAutoencoder.Builder().nIn(8).nOut(10).encoderLayerSizes(12) + .decoderLayerSizes(13).reconstructionDistribution( + new GaussianReconstructionDistribution(Activation.IDENTITY)) + .build(), "in") + .setOutputs("0").build(); + + //Do training on Spark with one executor, for 3 separate minibatches + int rddDataSetNumExamples = 10; + int totalAveragings = 5; + int averagingFrequency = 3; + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples) + .averagingFrequency(averagingFrequency).batchSizePerWorker(rddDataSetNumExamples) + .saveUpdater(true).workerPrefetchNumBatches(0).build(); + Nd4j.getRandom().setSeed(12345); + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf.clone(), tm); + + List trainData = new ArrayList<>(); + int nDataSets = numExecutors() * totalAveragings * averagingFrequency; + for (int i = 0; i < nDataSets; i++) { + trainData.add(new DataSet(Nd4j.rand(rddDataSetNumExamples, nIn), null)); + } + + JavaRDD data = sc.parallelize(trainData); + + sparkNet.fit(data); + } + + + @Test + public void testROC() { + + int nArrays = 100; + int minibatch = 64; + int steps = 20; + int nIn = 5; + int nOut = 2; + int layerSize = 10; + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(layerSize).build()) + .layer(1, new OutputLayer.Builder().nIn(layerSize).nOut(nOut) + .activation(Activation.SOFTMAX).lossFunction( + LossFunctions.LossFunction.MCXENT) + .build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + + Nd4j.getRandom().setSeed(12345); + Random r = new Random(12345); + + ROC local = new ROC(steps); + List dsList = new ArrayList<>(); + for (int i = 0; i < nArrays; i++) { + INDArray features = Nd4j.rand(minibatch, nIn); + + INDArray p = net.output(features); + + INDArray l = Nd4j.zeros(minibatch, 2); + for (int j = 0; j < minibatch; j++) { + l.putScalar(j, r.nextInt(2), 1.0); + } + + local.eval(l, p); + + dsList.add(new DataSet(features, l)); + } + + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, net, null); + JavaRDD rdd = sc.parallelize(dsList); + + ROC sparkROC = sparkNet.evaluateROC(rdd, steps, 32); + + assertEquals(sparkROC.calculateAUC(), sparkROC.calculateAUC(), 1e-6); + + assertEquals(local.getRocCurve(), sparkROC.getRocCurve()); + } + + + @Test + public void testROCMultiClass() { + + int nArrays = 100; + int minibatch = 64; + int steps = 20; + int nIn = 5; + int nOut = 3; + int layerSize = 10; + + MultiLayerConfiguration conf = + new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list() + .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(layerSize).build()) + .layer(1, new OutputLayer.Builder().nIn(layerSize).nOut(nOut) + .activation(Activation.SOFTMAX).lossFunction( + LossFunctions.LossFunction.MCXENT) + .build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + + Nd4j.getRandom().setSeed(12345); + Random r = new Random(12345); + + ROCMultiClass local = new ROCMultiClass(steps); + List dsList = new ArrayList<>(); + for (int i = 0; i < nArrays; i++) { + INDArray features = Nd4j.rand(minibatch, nIn); + + INDArray p = net.output(features); + + INDArray l = Nd4j.zeros(minibatch, nOut); + for (int j = 0; j < minibatch; j++) { + l.putScalar(j, r.nextInt(nOut), 1.0); + } + + local.eval(l, p); + + dsList.add(new DataSet(features, l)); + } + + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, net, null); + JavaRDD rdd = sc.parallelize(dsList); + + ROCMultiClass sparkROC = sparkNet.evaluateROCMultiClass(rdd, steps, 32); + + for (int i = 0; i < nOut; i++) { + assertEquals(sparkROC.calculateAUC(i), sparkROC.calculateAUC(i), 1e-6); + + assertEquals(local.getRocCurve(i), sparkROC.getRocCurve(i)); + } + } + + + @Test + @Timeout(120) + public void testEpochCounter() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new OutputLayer.Builder().nIn(4).nOut(3).build()) + .build(); + + ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() + .graphBuilder() + .addInputs("in") + .addLayer("out", new OutputLayer.Builder().nIn(4).nOut(3).build(), "in") + .setOutputs("out") + .build(); + + DataSetIterator iter = new IrisDataSetIterator(1, 50); + + List l = new ArrayList<>(); + while(iter.hasNext()){ + l.add(iter.next()); + } + + JavaRDD rdd = sc.parallelize(l); + + + int rddDataSetNumExamples = 1; + int averagingFrequency = 2; + int batch = 2; + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples) + .averagingFrequency(averagingFrequency).batchSizePerWorker(batch) + .saveUpdater(true).workerPrefetchNumBatches(0).build(); + Nd4j.getRandom().setSeed(12345); + + + SparkDl4jMultiLayer sn1 = new SparkDl4jMultiLayer(sc, conf.clone(), tm); + SparkComputationGraph sn2 = new SparkComputationGraph(sc, conf2.clone(), tm); + + + for(int i=0; i<3; i++ ){ + assertEquals(i, sn1.getNetwork().getLayerWiseConfigurations().getEpochCount()); + assertEquals(i, sn2.getNetwork().getConfiguration().getEpochCount()); + sn1.fit(rdd); + sn2.fit(rdd); + assertEquals(i+1, sn1.getNetwork().getLayerWiseConfigurations().getEpochCount()); + assertEquals(i+1, sn2.getNetwork().getConfiguration().getEpochCount()); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupportTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupportTest.java new file mode 100644 index 000000000..0fdeaaabf --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupportTest.java @@ -0,0 +1,76 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.paramavg.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author Ede Meijer + */ +public class ExportSupportTest { + private static final String FS_CONF = "spark.hadoop.fs.defaultFS"; + + @Test + public void testLocalSupported() throws IOException { + assertSupported(new SparkConf().setMaster("local").set(FS_CONF, "file:///")); + assertSupported(new SparkConf().setMaster("local[2]").set(FS_CONF, "file:///")); + assertSupported(new SparkConf().setMaster("local[64]").set(FS_CONF, "file:///")); + assertSupported(new SparkConf().setMaster("local[*]").set(FS_CONF, "file:///")); + + assertSupported(new SparkConf().setMaster("local").set(FS_CONF, "hdfs://localhost:9000")); + } + + @Test + public void testClusterWithRemoteFSSupported() throws IOException, URISyntaxException { + assertSupported("spark://localhost:7077", FileSystem.get(new URI("hdfs://localhost:9000"), new Configuration()), + true); + } + + @Test + public void testClusterWithLocalFSNotSupported() throws IOException, URISyntaxException { + assertSupported("spark://localhost:7077", FileSystem.get(new URI("file:///home/test"), new Configuration()), + false); + } + + private void assertSupported(SparkConf conf) throws IOException { + JavaSparkContext sc = new JavaSparkContext(conf.setAppName("Test").set("spark.driver.host", "localhost")); + try { + assertTrue(ExportSupport.exportSupported(sc)); + } finally { + sc.stop(); + } + } + + private void assertSupported(String master, FileSystem fs, boolean supported) throws IOException { + assertEquals(supported, ExportSupport.exportSupported(master, fs)); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/stats/TestTrainingStatsCollection.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/stats/TestTrainingStatsCollection.java new file mode 100644 index 000000000..f4939e369 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/impl/stats/TestTrainingStatsCollection.java @@ -0,0 +1,301 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.impl.stats; + +import com.sun.jna.Platform; +import org.apache.commons.io.FilenameUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.api.Repartition; +import org.deeplearning4j.spark.api.stats.CommonSparkTrainingStats; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingMasterStats; +import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingWorkerStats; +import org.deeplearning4j.spark.stats.EventStats; +import org.deeplearning4j.spark.stats.StatsUtils; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; + +public class TestTrainingStatsCollection extends BaseSparkTest { + + @Test + public void testStatsCollection() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + int nWorkers = numExecutors(); + + JavaSparkContext sc = getContext(); + + try { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) + .layer(1, new OutputLayer.Builder().nIn(10).nOut(10).build()) + .build(); + + int miniBatchSizePerWorker = 10; + int averagingFrequency = 5; + int numberOfAveragings = 3; + + int totalExamples = nWorkers * miniBatchSizePerWorker * averagingFrequency * numberOfAveragings; + + Nd4j.getRandom().setSeed(12345); + List list = new ArrayList<>(); + for (int i = 0; i < totalExamples; i++) { + INDArray f = Nd4j.rand(1, 10); + INDArray l = Nd4j.rand(1, 10); + DataSet ds = new DataSet(f, l); + list.add(ds); + } + + JavaRDD rdd = sc.parallelize(list); + rdd.repartition(4); + + ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(nWorkers, 1) + .averagingFrequency(averagingFrequency).batchSizePerWorker(miniBatchSizePerWorker) + .saveUpdater(true).workerPrefetchNumBatches(0).repartionData(Repartition.Always).build(); + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, tm); + sparkNet.setCollectTrainingStats(true); + sparkNet.fit(rdd); + + + //Collect the expected keys: + List expectedStatNames = new ArrayList<>(); + Class[] classes = new Class[] {CommonSparkTrainingStats.class, + ParameterAveragingTrainingMasterStats.class, ParameterAveragingTrainingWorkerStats.class}; + String[] fieldNames = new String[] {"columnNames", "columnNames", "columnNames"}; + for (int i = 0; i < classes.length; i++) { + Field field = classes[i].getDeclaredField(fieldNames[i]); + field.setAccessible(true); + Object f = field.get(null); + Collection c = (Collection) f; + expectedStatNames.addAll(c); + } + +// System.out.println(expectedStatNames); + + + SparkTrainingStats stats = sparkNet.getSparkTrainingStats(); + Set actualKeySet = stats.getKeySet(); + assertEquals(expectedStatNames.size(), actualKeySet.size()); + for (String s : stats.getKeySet()) { + assertTrue(expectedStatNames.contains(s)); + assertNotNull(stats.getValue(s)); + } + + String statsAsString = stats.statsAsString(); +// System.out.println(statsAsString); + assertEquals(actualKeySet.size(), statsAsString.split("\n").length); //One line per stat + + + //Go through nested stats + //First: master stats + assertTrue(stats instanceof ParameterAveragingTrainingMasterStats); + ParameterAveragingTrainingMasterStats masterStats = (ParameterAveragingTrainingMasterStats) stats; + + List exportTimeStats = masterStats.getParameterAveragingMasterExportTimesMs(); + assertEquals(1, exportTimeStats.size()); + assertDurationGreaterZero(exportTimeStats); + assertNonNullFields(exportTimeStats); + assertExpectedNumberMachineIdsJvmIdsThreadIds(exportTimeStats, 1, 1, 1); + + List countRddTime = masterStats.getParameterAveragingMasterCountRddSizeTimesMs(); + assertEquals(1, countRddTime.size()); //occurs once per fit + assertDurationGreaterEqZero(countRddTime); + assertNonNullFields(countRddTime); + assertExpectedNumberMachineIdsJvmIdsThreadIds(countRddTime, 1, 1, 1); //should occur only in master once + + List broadcastCreateTime = masterStats.getParameterAveragingMasterBroadcastCreateTimesMs(); + assertEquals(numberOfAveragings, broadcastCreateTime.size()); + assertDurationGreaterEqZero(broadcastCreateTime); + assertNonNullFields(broadcastCreateTime); + assertExpectedNumberMachineIdsJvmIdsThreadIds(broadcastCreateTime, 1, 1, 1); //only 1 thread for master + + List fitTimes = masterStats.getParameterAveragingMasterFitTimesMs(); + assertEquals(1, fitTimes.size()); //i.e., number of times fit(JavaRDD) was called + assertDurationGreaterZero(fitTimes); + assertNonNullFields(fitTimes); + assertExpectedNumberMachineIdsJvmIdsThreadIds(fitTimes, 1, 1, 1); //only 1 thread for master + + List splitTimes = masterStats.getParameterAveragingMasterSplitTimesMs(); + assertEquals(1, splitTimes.size()); //Splitting of the data set is executed once only (i.e., one fit(JavaRDD) call) + assertDurationGreaterEqZero(splitTimes); + assertNonNullFields(splitTimes); + assertExpectedNumberMachineIdsJvmIdsThreadIds(splitTimes, 1, 1, 1); //only 1 thread for master + + List aggregateTimesMs = masterStats.getParamaterAveragingMasterAggregateTimesMs(); + assertEquals(numberOfAveragings, aggregateTimesMs.size()); + assertDurationGreaterEqZero(aggregateTimesMs); + assertNonNullFields(aggregateTimesMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(aggregateTimesMs, 1, 1, 1); //only 1 thread for master + + List processParamsTimesMs = + masterStats.getParameterAveragingMasterProcessParamsUpdaterTimesMs(); + assertEquals(numberOfAveragings, processParamsTimesMs.size()); + assertDurationGreaterEqZero(processParamsTimesMs); + assertNonNullFields(processParamsTimesMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(processParamsTimesMs, 1, 1, 1); //only 1 thread for master + + List repartitionTimesMs = masterStats.getParameterAveragingMasterRepartitionTimesMs(); + assertEquals(numberOfAveragings, repartitionTimesMs.size()); + assertDurationGreaterEqZero(repartitionTimesMs); + assertNonNullFields(repartitionTimesMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(repartitionTimesMs, 1, 1, 1); //only 1 thread for master + + //Second: Common spark training stats + SparkTrainingStats commonStats = masterStats.getNestedTrainingStats(); + assertNotNull(commonStats); + assertTrue(commonStats instanceof CommonSparkTrainingStats); + CommonSparkTrainingStats cStats = (CommonSparkTrainingStats) commonStats; + List workerFlatMapTotalTimeMs = cStats.getWorkerFlatMapTotalTimeMs(); + assertEquals(numberOfAveragings * nWorkers, workerFlatMapTotalTimeMs.size()); + assertDurationGreaterZero(workerFlatMapTotalTimeMs); + assertNonNullFields(workerFlatMapTotalTimeMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(workerFlatMapTotalTimeMs, 1, 1, nWorkers); + + List workerFlatMapGetInitialModelTimeMs = cStats.getWorkerFlatMapGetInitialModelTimeMs(); + assertEquals(numberOfAveragings * nWorkers, workerFlatMapGetInitialModelTimeMs.size()); + assertDurationGreaterEqZero(workerFlatMapGetInitialModelTimeMs); + assertNonNullFields(workerFlatMapGetInitialModelTimeMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(workerFlatMapGetInitialModelTimeMs, 1, 1, nWorkers); + + List workerFlatMapDataSetGetTimesMs = cStats.getWorkerFlatMapDataSetGetTimesMs(); + int numMinibatchesProcessed = workerFlatMapDataSetGetTimesMs.size(); + int expectedNumMinibatchesProcessed = numberOfAveragings * nWorkers * averagingFrequency; //1 for every time we get a data set + + //Sometimes random split is just bad - some executors might miss out on getting the expected amount of data + assertTrue(numMinibatchesProcessed >= expectedNumMinibatchesProcessed - 5); + + List workerFlatMapProcessMiniBatchTimesMs = cStats.getWorkerFlatMapProcessMiniBatchTimesMs(); + assertTrue(workerFlatMapProcessMiniBatchTimesMs.size() >= numberOfAveragings * nWorkers * averagingFrequency + - 5); + assertDurationGreaterEqZero(workerFlatMapProcessMiniBatchTimesMs); + assertNonNullFields(workerFlatMapDataSetGetTimesMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(workerFlatMapDataSetGetTimesMs, 1, 1, nWorkers); + + //Third: ParameterAveragingTrainingWorker stats + SparkTrainingStats paramAvgStats = cStats.getNestedTrainingStats(); + assertNotNull(paramAvgStats); + assertTrue(paramAvgStats instanceof ParameterAveragingTrainingWorkerStats); + + ParameterAveragingTrainingWorkerStats pStats = (ParameterAveragingTrainingWorkerStats) paramAvgStats; + List parameterAveragingWorkerBroadcastGetValueTimeMs = + pStats.getParameterAveragingWorkerBroadcastGetValueTimeMs(); + assertEquals(numberOfAveragings * nWorkers, parameterAveragingWorkerBroadcastGetValueTimeMs.size()); + assertDurationGreaterEqZero(parameterAveragingWorkerBroadcastGetValueTimeMs); + assertNonNullFields(parameterAveragingWorkerBroadcastGetValueTimeMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(parameterAveragingWorkerBroadcastGetValueTimeMs, 1, 1, + nWorkers); + + List parameterAveragingWorkerInitTimeMs = pStats.getParameterAveragingWorkerInitTimeMs(); + assertEquals(numberOfAveragings * nWorkers, parameterAveragingWorkerInitTimeMs.size()); + assertDurationGreaterEqZero(parameterAveragingWorkerInitTimeMs); + assertNonNullFields(parameterAveragingWorkerInitTimeMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(parameterAveragingWorkerInitTimeMs, 1, 1, nWorkers); + + List parameterAveragingWorkerFitTimesMs = pStats.getParameterAveragingWorkerFitTimesMs(); + assertTrue(parameterAveragingWorkerFitTimesMs.size() >= numberOfAveragings * nWorkers * averagingFrequency + - 5); + assertDurationGreaterEqZero(parameterAveragingWorkerFitTimesMs); + assertNonNullFields(parameterAveragingWorkerFitTimesMs); + assertExpectedNumberMachineIdsJvmIdsThreadIds(parameterAveragingWorkerFitTimesMs, 1, 1, nWorkers); + + assertNull(pStats.getNestedTrainingStats()); + + + //Finally: try exporting stats + String tempDir = System.getProperty("java.io.tmpdir"); + String outDir = FilenameUtils.concat(tempDir, "dl4j_testTrainingStatsCollection"); + stats.exportStatFiles(outDir, sc.sc()); + + String htmlPlotsPath = FilenameUtils.concat(outDir, "AnalysisPlots.html"); + StatsUtils.exportStatsAsHtml(stats, htmlPlotsPath, sc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + StatsUtils.exportStatsAsHTML(stats, baos); + baos.close(); + byte[] bytes = baos.toByteArray(); + String str = new String(bytes, "UTF-8"); + // System.out.println(str); + } finally { + sc.stop(); + } + } + + private static void assertDurationGreaterEqZero(List array) { + for (EventStats e : array) + assertTrue(e.getDurationMs() >= 0); + } + + private static void assertDurationGreaterZero(List array) { + for (EventStats e : array) + assertTrue(e.getDurationMs() > 0); + } + + private static void assertNonNullFields(List array) { + for (EventStats e : array) { + assertNotNull(e.getMachineID()); + assertNotNull(e.getJvmID()); + assertNotNull(e.getDurationMs()); + assertFalse(e.getMachineID().isEmpty()); + assertFalse(e.getJvmID().isEmpty()); + assertTrue(e.getThreadID() > 0); + } + } + + private static void assertExpectedNumberMachineIdsJvmIdsThreadIds(List events, int expNMachineIDs, + int expNumJvmIds, int expNumThreadIds) { + Set machineIDs = new HashSet<>(); + Set jvmIDs = new HashSet<>(); + Set threadIDs = new HashSet<>(); + for (EventStats e : events) { + machineIDs.add(e.getMachineID()); + jvmIDs.add(e.getJvmID()); + threadIDs.add(e.getThreadID()); + } + assertTrue(machineIDs.size() == expNMachineIDs); + assertTrue(jvmIDs.size() == expNumJvmIds); + assertTrue(threadIDs.size() == expNumThreadIds); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/time/TestTimeSource.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/time/TestTimeSource.java new file mode 100644 index 000000000..85a73aab4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/time/TestTimeSource.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.time; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestTimeSource { + + @Test + public void testTimeSourceNTP() throws Exception { + TimeSource timeSource = TimeSourceProvider.getInstance(); + assertTrue(timeSource instanceof NTPTimeSource); + + for (int i = 0; i < 10; i++) { + long systemTime = System.currentTimeMillis(); + long ntpTime = timeSource.currentTimeMillis(); + long offset = ntpTime - systemTime; +// System.out.println("System: " + systemTime + "\tNTPTimeSource: " + ntpTime + "\tOffset: " + offset); + Thread.sleep(500); + } + } + + @Test + public void testTimeSourceSystem() throws Exception { + TimeSource timeSource = TimeSourceProvider.getInstance("org.deeplearning4j.spark.time.SystemClockTimeSource"); + assertTrue(timeSource instanceof SystemClockTimeSource); + + for (int i = 0; i < 10; i++) { + long systemTime = System.currentTimeMillis(); + long ntpTime = timeSource.currentTimeMillis(); + long offset = ntpTime - systemTime; +// System.out.println("System: " + systemTime + "\tSystemClockTimeSource: " + ntpTime + "\tOffset: " + offset); + assertEquals(systemTime, ntpTime, 2); //Should be exact, but we might randomly tick over between one ms and the next + Thread.sleep(500); + } + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/ui/TestListeners.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/ui/TestListeners.java new file mode 100644 index 000000000..6f79d7595 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/ui/TestListeners.java @@ -0,0 +1,127 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.ui; + +import com.sun.jna.Platform; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StatsStorage; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.nn.api.OptimizationAlgorithm; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.deeplearning4j.ui.model.stats.StatsListener; +import org.deeplearning4j.ui.model.storage.mapdb.MapDBStatsStorage; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestListeners extends BaseSparkTest { + + @Test + public void testStatsCollection() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + JavaSparkContext sc = getContext(); + int nExecutors = numExecutors(); + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123) + .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() + .layer(0, new DenseLayer.Builder().nIn(4).nOut(100).weightInit(WeightInit.XAVIER) + .activation(Activation.RELU).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(100).nOut(3) + .activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER) + .build()) + .build(); + + + + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + + TrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(1).batchSizePerWorker(5).averagingFrequency(6) + .build(); + + SparkDl4jMultiLayer net = new SparkDl4jMultiLayer(sc, conf, tm); + StatsStorage ss = new MapDBStatsStorage(); //In-memory + + net.setListeners(ss, Collections.singletonList(new StatsListener(null))); + + List list = new IrisDataSetIterator(120, 150).next().asList(); + //120 examples, 4 executors, 30 examples per executor -> 6 updates of size 5 per executor + + JavaRDD rdd = sc.parallelize(list); + + net.fit(rdd); + + List sessions = ss.listSessionIDs(); +// System.out.println("Sessions: " + sessions); + assertEquals(1, sessions.size()); + + String sid = sessions.get(0); + + List typeIDs = ss.listTypeIDsForSession(sid); + List workers = ss.listWorkerIDsForSession(sid); + +// System.out.println(sid + "\t" + typeIDs + "\t" + workers); + + List lastUpdates = ss.getLatestUpdateAllWorkers(sid, StatsListener.TYPE_ID); +// System.out.println(lastUpdates); + +// System.out.println("Static info:"); + for (String wid : workers) { + Persistable staticInfo = ss.getStaticInfo(sid, StatsListener.TYPE_ID, wid); +// System.out.println(sid + "\t" + wid); + } + + assertEquals(1, typeIDs.size()); + assertEquals(numExecutors(), workers.size()); + String firstWorker = workers.get(0); + String firstWorkerSubstring = workers.get(0).substring(0, firstWorker.length() - 1); + for (String wid : workers) { + String widSubstring = wid.substring(0, wid.length() - 1); + assertEquals(firstWorkerSubstring, widSubstring); + + String counterVal = wid.substring(wid.length() - 1, wid.length()); + int cv = Integer.parseInt(counterVal); + assertTrue(0 <= cv && cv < numExecutors()); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/MLLIbUtilTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/MLLIbUtilTest.java new file mode 100644 index 000000000..ef7c0788f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/MLLIbUtilTest.java @@ -0,0 +1,95 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util; + + +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.mllib.linalg.Matrices; +import org.apache.spark.mllib.linalg.Matrix; +import org.apache.spark.mllib.regression.LabeledPoint; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.spark.BaseSparkTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MLLIbUtilTest extends BaseSparkTest { + private static final Logger log = LoggerFactory.getLogger(MLLIbUtilTest.class); + + @Test + public void testMlLibTest() { + DataSet dataSet = new IrisDataSetIterator(150, 150).next(); + List list = dataSet.asList(); + JavaRDD data = sc.parallelize(list); + JavaRDD mllLibData = MLLibUtil.fromDataSet(sc, data); + } + + + @Test + public void testINDtoMLMatrix() { + INDArray matIND = Nd4j.rand(23, 100); + + Matrix matMl = MLLibUtil.toMatrix(matIND); + + assertTrue(matrixEquals(matMl, matIND, 0.01)); + } + + @Test + public void testMltoINDMatrix() { + Matrix matMl = Matrices.randn(23, 100, new Random(3949955)); + + INDArray matIND = MLLibUtil.toMatrix(matMl); + log.info("matrix shape: {}", Arrays.toString(matIND.shapeInfoDataBuffer().asInt())); + + assertTrue(matrixEquals(matMl, matIND, 0.01)); + } + + private boolean matrixEquals(Matrix mlMatrix, INDArray indMatrix, Double eps) { + final int mlRows = mlMatrix.numRows(); + final int mlCols = mlMatrix.numCols(); + final int indRows = indMatrix.rows(); + final int indCols = indMatrix.columns(); + + if (mlRows != indRows) + return false; + if (mlCols != indCols) + return false; + + for (int i = 0; i < mlRows; i++) { + for (int j = 0; j < mlCols; j++) { + double delta = Math.abs(mlMatrix.apply(i, j) - indMatrix.getDouble(i, j)); + if (delta > eps) + return false; + } + } + return true; + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/TestRepartitioning.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/TestRepartitioning.java new file mode 100644 index 000000000..c83282547 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/TestRepartitioning.java @@ -0,0 +1,289 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util; + +import com.sun.jna.Platform; +import org.apache.spark.Partitioner; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.api.Repartition; +import org.deeplearning4j.spark.api.RepartitionStrategy; +import org.deeplearning4j.spark.impl.common.CountPartitionsFunction; +import org.deeplearning4j.spark.impl.repartitioner.DefaultRepartitioner; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestRepartitioning extends BaseSparkTest { + + @Override + public long getTimeoutMilliseconds() { + return isIntegrationTests() ? 240000 : 60000; + } + + @Test + public void testRepartitioning() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + List list = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + list.add(String.valueOf(i)); + } + + JavaRDD rdd = sc.parallelize(list); + rdd = rdd.repartition(200); + + JavaRDD rdd2 = SparkUtils.repartitionBalanceIfRequired(rdd, Repartition.Always, 100, 10); + assertFalse(rdd == rdd2); //Should be different objects due to repartitioning + + assertEquals(10, rdd2.partitions().size()); + for (int i = 0; i < 10; i++) { + List partition = rdd2.collectPartitions(new int[] {i})[0]; +// System.out.println("Partition " + i + " size: " + partition.size()); + assertEquals(100, partition.size()); //Should be exactly 100, for the util method (but NOT spark .repartition) + } + } + + @Test + public void testRepartitioning2() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + int[] ns; + if(isIntegrationTests()){ + ns = new int[]{320, 321, 25600, 25601, 25615}; + } else { + ns = new int[]{320, 2561}; + } + + for (int n : ns) { + + List list = new ArrayList<>(); + for (int i = 0; i < n; i++) { + list.add(String.valueOf(i)); + } + + JavaRDD rdd = sc.parallelize(list); + rdd.repartition(65); + + int totalDataSetObjectCount = n; + int dataSetObjectsPerSplit = 8 * 4 * 10; + int valuesPerPartition = 10; + int nPartitions = 32; + + JavaRDD[] splits = org.deeplearning4j.spark.util.SparkUtils.balancedRandomSplit( + totalDataSetObjectCount, dataSetObjectsPerSplit, rdd, new Random().nextLong()); + + List counts = new ArrayList<>(); + List>> partitionCountList = new ArrayList<>(); + // System.out.println("------------------------"); + // System.out.println("Partitions Counts:"); + for (JavaRDD split : splits) { + JavaRDD repartitioned = SparkUtils.repartition(split, Repartition.Always, + RepartitionStrategy.Balanced, valuesPerPartition, nPartitions); + List> partitionCounts = repartitioned + .mapPartitionsWithIndex(new CountPartitionsFunction(), true).collect(); + // System.out.println(partitionCounts); + partitionCountList.add(partitionCounts); + counts.add((int) split.count()); + } + + // System.out.println(counts.size()); + // System.out.println(counts); + + + int expNumPartitionsWithMore = totalDataSetObjectCount % nPartitions; + int actNumPartitionsWithMore = 0; + for (List> l : partitionCountList) { + assertEquals(nPartitions, l.size()); + + for (Tuple2 t2 : l) { + int partitionSize = t2._2(); + if (partitionSize > valuesPerPartition) + actNumPartitionsWithMore++; + } + } + + assertEquals(expNumPartitionsWithMore, actNumPartitionsWithMore); + } + } + + @Test + public void testRepartitioning3(){ + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + //Initial partitions (idx, count) - [(0,29), (1,29), (2,29), (3,34), (4,34), (5,35), (6,34)] + + List ints = new ArrayList<>(); + for( int i=0; i<224; i++ ){ + ints.add(i); + } + + JavaRDD rdd = sc.parallelize(ints); + JavaPairRDD pRdd = SparkUtils.indexedRDD(rdd); + JavaPairRDD initial = pRdd.partitionBy(new Partitioner() { + @Override + public int getPartition(Object key) { + int i = (Integer)key; + if(i < 29){ + return 0; + } else if(i < 29+29){ + return 1; + } else if(i < 29+29+29){ + return 2; + } else if(i < 29+29+29+34){ + return 3; + } else if(i < 29+29+29+34+34){ + return 4; + } else if(i < 29+29+29+34+34+35){ + return 5; + } else { + return 6; + } + } + @Override + public int numPartitions() { + return 7; + } + }); + + List> partitionCounts = initial.values().mapPartitionsWithIndex(new CountPartitionsFunction(), true).collect(); + +// System.out.println(partitionCounts); + + List> initialExpected = Arrays.asList( + new Tuple2<>(0,29), + new Tuple2<>(1,29), + new Tuple2<>(2,29), + new Tuple2<>(3,34), + new Tuple2<>(4,34), + new Tuple2<>(5,35), + new Tuple2<>(6,34)); + Assertions.assertEquals(initialExpected, partitionCounts); + + + JavaRDD afterRepartition = SparkUtils.repartitionBalanceIfRequired(initial.values(), Repartition.Always, 2, 112); + List> partitionCountsAfter = afterRepartition.mapPartitionsWithIndex(new CountPartitionsFunction(), true).collect(); +// System.out.println(partitionCountsAfter); + + for(Tuple2 t2 : partitionCountsAfter){ + assertEquals(2, (int)t2._2()); + } + } + + @Test + public void testRepartitioning4() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + List ints = new ArrayList<>(); + for( int i = 0; i < 7040; i++) { + ints.add(i); + } + + JavaRDD rdd = sc.parallelize(ints); + + JavaRDD afterRepartition = new DefaultRepartitioner().repartition(rdd, 1, 32); + List> partitionCountsAfter = afterRepartition.mapPartitionsWithIndex(new CountPartitionsFunction(), true).collect(); + + int min = Integer.MAX_VALUE; + int max = Integer.MIN_VALUE; + int minIdx = 0; + int maxIdx = 0; + for(Tuple2 t2 : partitionCountsAfter){ + min = Math.min(min, t2._2()); + max = Math.max(max, t2._2()); + if(min == t2._2()){ + minIdx = t2._1(); + } + if(max == t2._2()){ + maxIdx = t2._1(); + } + } + +// System.out.println("min: " + min + "\t@\t" + minIdx); +// System.out.println("max: " + max + "\t@\t" + maxIdx); + + assertEquals(1, min); + assertEquals(2, max); + } + + + @Test + public void testRepartitioningApprox() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + List list = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + list.add(String.valueOf(i)); + } + + JavaRDD rdd = sc.parallelize(list); + rdd = rdd.repartition(200); + + JavaRDD rdd2 = SparkUtils.repartitionApproximateBalance(rdd, Repartition.Always, 10); + assertFalse(rdd == rdd2); //Should be different objects due to repartitioning + + assertEquals(10, rdd2.partitions().size()); + + for (int i = 0; i < 10; i++) { + List partition = rdd2.collectPartitions(new int[] {i})[0]; +// System.out.println("Partition " + i + " size: " + partition.size()); + assertTrue(partition.size() >= 90 && partition.size() <= 110); + } + } + + @Test + public void testRepartitioningApproxReverse() { + List list = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + list.add(String.valueOf(i)); + } + + // initial # of partitions = cores, probably < 100 + JavaRDD rdd = sc.parallelize(list); + + JavaRDD rdd2 = SparkUtils.repartitionApproximateBalance(rdd, Repartition.Always, 100); + assertFalse(rdd == rdd2); //Should be different objects due to repartitioning + + assertEquals(100, rdd2.partitions().size()); + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/TestValidation.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/TestValidation.java new file mode 100644 index 000000000..21ba9fc23 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/java/org/deeplearning4j/spark/util/TestValidation.java @@ -0,0 +1,204 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.util; + +import com.sun.jna.Platform; +import org.apache.commons.io.FileUtils; +import org.deeplearning4j.spark.BaseSparkTest; +import org.deeplearning4j.spark.util.data.SparkDataValidation; +import org.deeplearning4j.spark.util.data.ValidationResult; + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.File; +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestValidation extends BaseSparkTest { + + @TempDir + public File folder; + + @Test + public void testDataSetValidation() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + File f = folder; + + for( int i = 0; i < 3; i++ ) { + DataSet ds = new DataSet(Nd4j.create(1,10), Nd4j.create(1,10)); + ds.save(new File(f, i + ".bin")); + } + + ValidationResult r = SparkDataValidation.validateDataSets(sc, f.toURI().toString()); + ValidationResult exp = ValidationResult.builder() + .countTotal(3) + .countTotalValid(3) + .build(); + assertEquals(exp, r); + + //Add a DataSet that is corrupt (can't be loaded) + File f3 = new File(f, "3.bin"); + FileUtils.writeStringToFile(f3, "This isn't a DataSet!"); + r = SparkDataValidation.validateDataSets(sc, f.toURI().toString()); + exp = ValidationResult.builder() + .countTotal(4) + .countTotalValid(3) + .countTotalInvalid(1) + .countLoadingFailure(1) + .build(); + assertEquals(exp, r); + f3.delete(); + + + //Add a DataSet with missing features: + new DataSet(null, Nd4j.create(1,10)).save(f3); + + r = SparkDataValidation.validateDataSets(sc, f.toURI().toString()); + exp = ValidationResult.builder() + .countTotal(4) + .countTotalValid(3) + .countTotalInvalid(1) + .countMissingFeatures(1) + .build(); + assertEquals(exp, r); + + r = SparkDataValidation.deleteInvalidDataSets(sc, f.toURI().toString()); + exp.setCountInvalidDeleted(1); + assertEquals(exp, r); + assertFalse(f3.exists()); + for( int i=0; i<3; i++ ){ + assertTrue(new File(f,i + ".bin").exists()); + } + + //Add DataSet with incorrect labels shape: + new DataSet(Nd4j.create(1,10), Nd4j.create(1,20)).save(f3); + r = SparkDataValidation.validateDataSets(sc, f.toURI().toString(), new int[]{-1,10}, new int[]{-1,10}); + exp = ValidationResult.builder() + .countTotal(4) + .countTotalValid(3) + .countTotalInvalid(1) + .countInvalidLabels(1) + .build(); + + assertEquals(exp, r); + } + + @Test + public void testMultiDataSetValidation() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + File f = folder; + + for( int i = 0; i < 3; i++ ) { + MultiDataSet ds = new MultiDataSet(Nd4j.create(1,10), Nd4j.create(1,10)); + ds.save(new File(f, i + ".bin")); + } + + ValidationResult r = SparkDataValidation.validateMultiDataSets(sc, f.toURI().toString()); + ValidationResult exp = ValidationResult.builder() + .countTotal(3) + .countTotalValid(3) + .build(); + assertEquals(exp, r); + + //Add a MultiDataSet that is corrupt (can't be loaded) + File f3 = new File(f, "3.bin"); + FileUtils.writeStringToFile(f3, "This isn't a MultiDataSet!"); + r = SparkDataValidation.validateMultiDataSets(sc, f.toURI().toString()); + exp = ValidationResult.builder() + .countTotal(4) + .countTotalValid(3) + .countTotalInvalid(1) + .countLoadingFailure(1) + .build(); + assertEquals(exp, r); + f3.delete(); + + + //Add a MultiDataSet with missing features: + new MultiDataSet(null, Nd4j.create(1,10)).save(f3); + + r = SparkDataValidation.validateMultiDataSets(sc, f.toURI().toString()); + exp = ValidationResult.builder() + .countTotal(4) + .countTotalValid(3) + .countTotalInvalid(1) + .countMissingFeatures(1) + .build(); + assertEquals(exp, r); + + r = SparkDataValidation.deleteInvalidMultiDataSets(sc, f.toURI().toString()); + exp.setCountInvalidDeleted(1); + assertEquals(exp, r); + assertFalse(f3.exists()); + for( int i=0; i<3; i++ ){ + assertTrue(new File(f,i + ".bin").exists()); + } + + //Add MultiDataSet with incorrect labels shape: + new MultiDataSet(Nd4j.create(1,10), Nd4j.create(1,20)).save(f3); + r = SparkDataValidation.validateMultiDataSets(sc, f.toURI().toString(), Arrays.asList(new int[]{-1,10}), + Arrays.asList(new int[]{-1,10})); + exp = ValidationResult.builder() + .countTotal(4) + .countTotalValid(3) + .countTotalInvalid(1) + .countInvalidLabels(1) + .build(); + f3.delete(); + assertEquals(exp, r); + + //Add a MultiDataSet with incorrect number of feature arrays: + new MultiDataSet(new INDArray[]{Nd4j.create(1,10), Nd4j.create(1,10)}, + new INDArray[]{Nd4j.create(1,10)}).save(f3); + r = SparkDataValidation.validateMultiDataSets(sc, f.toURI().toString(), Arrays.asList(new int[]{-1,10}), + Arrays.asList(new int[]{-1,10})); + exp = ValidationResult.builder() + .countTotal(4) + .countTotalValid(3) + .countTotalInvalid(1) + .countInvalidFeatures(1) + .build(); + assertEquals(exp, r); + + + r = SparkDataValidation.deleteInvalidMultiDataSets(sc, f.toURI().toString(), Arrays.asList(new int[]{-1,10}), + Arrays.asList(new int[]{-1,10})); + exp.setCountInvalidDeleted(1); + assertEquals(exp, r); + assertFalse(f3.exists()); + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/resources/log4j.properties b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/resources/log4j.properties new file mode 100644 index 000000000..e0dc1ce63 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/resources/log4j.properties @@ -0,0 +1,35 @@ +# +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +log4j.rootLogger=ERROR, Console +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n + +log4j.appender.org.springframework=DEBUG +log4j.appender.org.deeplearning4j=DEBUG +log4j.appender.org.nd4j=DEBUG + +log4j.logger.org.springframework=INFO +log4j.logger.org.deeplearning4j=DEBUG +log4j.logger.org.nd4j=DEBUG +log4j.logger.org.apache.spark=WARN + + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/resources/logback.xml b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/resources/logback.xml new file mode 100644 index 000000000..aef9b5e2e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-core/src/test/resources/logback.xml @@ -0,0 +1,57 @@ + + + + + + + + logs/application.log + + %logger{15} - %message%n%xException{5} + + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/build.gradle b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/build.gradle new file mode 100644 index 000000000..fd9d2d5e8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/build.gradle @@ -0,0 +1,42 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +ext { + scalaVersion = rootProject.ext.scalaVersion +} + +dependencies { + implementation projects.cavisDnn.cavisDnnNlp + implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkCore + implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisDnn.cavisDnnApi + compileOnly "org.apache.spark:spark-core_${scalaVersion}" + + implementation "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}" + implementation "com.google.guava:guava" + + testImplementation projects.cavisDnn.cavisDnnCommonTests + testImplementation "com.sun.jna:jna:3.0.9" + testCompileOnly "org.apache.spark:spark-core_${scalaVersion}" +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/pom.xml b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/pom.xml new file mode 100644 index 000000000..e20fc1ea9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/pom.xml @@ -0,0 +1,82 @@ + + + + + + 4.0.0 + + + net.brutex.ai + spark_2.12 + 1.0.0-SNAPSHOT + + + dl4j-spark-nlp_2.12 + + dl4j-spark-nlp + + + 3.4.2 + + + + + net.brutex.ai + deeplearning4j-nlp + ${project.version} + + + net.brutex.ai + dl4j-spark_2.12 + ${project.version} + + + net.brutex.ai + datavec-spark_2.12 + ${project.version} + + + org.apache.spark + spark-core_2.12 + ${spark.version} + provided + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + 2.6.7.1 + + + net.brutex.ai + deeplearning4j-common-tests + ${project.version} + test + + + + com.google.guava + guava + ${guava.jre.version} + + + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/FirstIterationFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/FirstIterationFunction.java new file mode 100644 index 000000000..d8e6f235d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/FirstIterationFunction.java @@ -0,0 +1,266 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; +import scala.Tuple2; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicLong; + +public class FirstIterationFunction implements + FlatMapFunction, Long>>, Entry> { + + private int ithIteration = 1; + private int vectorLength; + private boolean useAdaGrad; + private int batchSize = 0; + private double negative; + private int window; + private double alpha; + private double minAlpha; + private long totalWordCount; + private long seed; + private int maxExp; + private double[] expTable; + private int iterations; + private Map indexSyn0VecMap; + private Map pointSyn1VecMap; + private AtomicLong nextRandom = new AtomicLong(5); + + private volatile VocabCache vocab; + private volatile NegativeHolder negativeHolder; + private AtomicLong cid = new AtomicLong(0); + private AtomicLong aff = new AtomicLong(0); + + + + public FirstIterationFunction(Broadcast> word2vecVarMapBroadcast, + Broadcast expTableBroadcast, Broadcast> vocabCacheBroadcast) { + + Map word2vecVarMap = word2vecVarMapBroadcast.getValue(); + this.expTable = expTableBroadcast.getValue(); + this.vectorLength = (int) word2vecVarMap.get("vectorLength"); + this.useAdaGrad = (boolean) word2vecVarMap.get("useAdaGrad"); + this.negative = (double) word2vecVarMap.get("negative"); + this.window = (int) word2vecVarMap.get("window"); + this.alpha = (double) word2vecVarMap.get("alpha"); + this.minAlpha = (double) word2vecVarMap.get("minAlpha"); + this.totalWordCount = (long) word2vecVarMap.get("totalWordCount"); + this.seed = (long) word2vecVarMap.get("seed"); + this.maxExp = (int) word2vecVarMap.get("maxExp"); + this.iterations = (int) word2vecVarMap.get("iterations"); + this.batchSize = (int) word2vecVarMap.get("batchSize"); + this.indexSyn0VecMap = new HashMap<>(); + this.pointSyn1VecMap = new HashMap<>(); + this.vocab = vocabCacheBroadcast.getValue(); + + if (this.vocab == null) + throw new RuntimeException("VocabCache is null"); + + if (negative > 0) { + negativeHolder = NegativeHolder.getInstance(); + negativeHolder.initHolder(vocab, expTable, this.vectorLength); + } + } + + + + @Override + public Iterator> call(Iterator, Long>> pairIter) { + while (pairIter.hasNext()) { + List, Long>> batch = new ArrayList<>(); + while (pairIter.hasNext() && batch.size() < batchSize) { + Tuple2, Long> pair = pairIter.next(); + List vocabWordsList = pair._1(); + Long sentenceCumSumCount = pair._2(); + batch.add(Pair.of(vocabWordsList, sentenceCumSumCount)); + } + + for (int i = 0; i < iterations; i++) { + //System.out.println("Training sentence: " + vocabWordsList); + for (Pair, Long> pair : batch) { + List vocabWordsList = pair.getKey(); + Long sentenceCumSumCount = pair.getValue(); + double currentSentenceAlpha = Math.max(minAlpha, + alpha - (alpha - minAlpha) * (sentenceCumSumCount / (double) totalWordCount)); + trainSentence(vocabWordsList, currentSentenceAlpha); + } + } + } + return indexSyn0VecMap.entrySet().iterator(); + } + + + public void trainSentence(List vocabWordsList, double currentSentenceAlpha) { + + if (vocabWordsList != null && !vocabWordsList.isEmpty()) { + for (int ithWordInSentence = 0; ithWordInSentence < vocabWordsList.size(); ithWordInSentence++) { + // Random value ranging from 0 to window size + nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11)); + int b = (int) (long) this.nextRandom.get() % window; + VocabWord currentWord = vocabWordsList.get(ithWordInSentence); + if (currentWord != null) { + skipGram(ithWordInSentence, vocabWordsList, b, currentSentenceAlpha); + } + } + } + } + + public void skipGram(int ithWordInSentence, List vocabWordsList, int b, double currentSentenceAlpha) { + + VocabWord currentWord = vocabWordsList.get(ithWordInSentence); + if (currentWord != null && !vocabWordsList.isEmpty()) { + int end = window * 2 + 1 - b; + for (int a = b; a < end; a++) { + if (a != window) { + int c = ithWordInSentence - window + a; + if (c >= 0 && c < vocabWordsList.size()) { + VocabWord lastWord = vocabWordsList.get(c); + iterateSample(currentWord, lastWord, currentSentenceAlpha); + } + } + } + } + } + + public void iterateSample(VocabWord w1, VocabWord w2, double currentSentenceAlpha) { + + + if (w1 == null || w2 == null || w2.getIndex() < 0 || w2.getIndex() == w1.getIndex()) + return; + final int currentWordIndex = w2.getIndex(); + + // error for current word and context + INDArray neu1e = Nd4j.create(vectorLength); + + // First iteration Syn0 is random numbers + INDArray l1 = null; + if (indexSyn0VecMap.containsKey(vocab.elementAtIndex(currentWordIndex))) { + l1 = indexSyn0VecMap.get(vocab.elementAtIndex(currentWordIndex)); + } else { + l1 = getRandomSyn0Vec(vectorLength, (long) currentWordIndex); + } + + // + for (int i = 0; i < w1.getCodeLength(); i++) { + int code = w1.getCodes().get(i); + int point = w1.getPoints().get(i); + if (point < 0) + throw new IllegalStateException("Illegal point " + point); + // Point to + INDArray syn1; + if (pointSyn1VecMap.containsKey(point)) { + syn1 = pointSyn1VecMap.get(point); + } else { + syn1 = Nd4j.zeros(1, vectorLength); // 1 row of vector length of zeros + pointSyn1VecMap.put(point, syn1); + } + + // Dot product of Syn0 and Syn1 vecs + double dot = Nd4j.getBlasWrapper().level1().dot(vectorLength, 1.0, l1, syn1); + + if (dot < -maxExp || dot >= maxExp) + continue; + + int idx = (int) ((dot + maxExp) * ((double) expTable.length / maxExp / 2.0)); + + if (idx >= expTable.length) + continue; + + //score + double f = expTable[idx]; + //gradient + double g = (1 - code - f) * (useAdaGrad ? w1.getGradient(i, currentSentenceAlpha, currentSentenceAlpha) + : currentSentenceAlpha); + + + Nd4j.getBlasWrapper().level1().axpy(vectorLength, g, syn1, neu1e); + Nd4j.getBlasWrapper().level1().axpy(vectorLength, g, l1, syn1); + } + + int target = w1.getIndex(); + int label; + //negative sampling + if (negative > 0) + for (int d = 0; d < negative + 1; d++) { + if (d == 0) + label = 1; + else { + nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11)); + + int idx = Math.abs((int) (nextRandom.get() >> 16) % (int) negativeHolder.getTable().length()); + + target = negativeHolder.getTable().getInt(idx); + if (target <= 0) + target = (int) nextRandom.get() % (vocab.numWords() - 1) + 1; + + if (target == w1.getIndex()) + continue; + label = 0; + } + + if (target >= negativeHolder.getSyn1Neg().rows() || target < 0) + continue; + + double f = Nd4j.getBlasWrapper().dot(l1, negativeHolder.getSyn1Neg().slice(target)); + double g; + if (f > maxExp) + g = useAdaGrad ? w1.getGradient(target, (label - 1), alpha) : (label - 1) * alpha; + else if (f < -maxExp) + g = label * (useAdaGrad ? w1.getGradient(target, alpha, alpha) : alpha); + else { + int idx = (int) ((f + maxExp) * (expTable.length / maxExp / 2)); + if (idx >= expTable.length) + continue; + + g = useAdaGrad ? w1.getGradient(target, label - expTable[idx], alpha) + : (label - expTable[idx]) * alpha; + } + + Nd4j.getBlasWrapper().level1().axpy(vectorLength, g, negativeHolder.getSyn1Neg().slice(target), neu1e); + + Nd4j.getBlasWrapper().level1().axpy(vectorLength, g, l1, negativeHolder.getSyn1Neg().slice(target)); + } + + + // Updated the Syn0 vector based on gradient. Syn0 is not random anymore. + Nd4j.getBlasWrapper().level1().axpy(vectorLength, 1.0f, neu1e, l1); + + VocabWord word = vocab.elementAtIndex(currentWordIndex); + indexSyn0VecMap.put(word, l1); + } + + private INDArray getRandomSyn0Vec(int vectorLength, long lseed) { + /* + we use wordIndex as part of seed here, to guarantee that during word syn0 initialization on dwo distinct nodes, initial weights will be the same for the same word + */ + return Nd4j.rand( new int[] {1, vectorLength}, lseed * seed).subi(0.5).divi(vectorLength); + } +} + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/MapToPairFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/MapToPairFunction.java new file mode 100644 index 000000000..ccd8f7b0f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/MapToPairFunction.java @@ -0,0 +1,39 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.spark.api.java.function.Function; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.primitives.Pair; + +import java.util.Map; + +/** + * @author jeffreytang + */ +public class MapToPairFunction implements Function, Pair> { + + @Override + public Pair call(Map.Entry pair) { + return new Pair<>(pair.getKey(), pair.getValue()); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/NegativeHolder.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/NegativeHolder.java new file mode 100644 index 000000000..5b788562b --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/NegativeHolder.java @@ -0,0 +1,89 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import lombok.Getter; +import lombok.NonNull; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicBoolean; + +public class NegativeHolder implements Serializable { + private static NegativeHolder ourInstance = new NegativeHolder(); + + public static NegativeHolder getInstance() { + return ourInstance; + } + + @Getter + private volatile INDArray syn1Neg; + @Getter + private volatile INDArray table; + + private transient AtomicBoolean wasInit = new AtomicBoolean(false); + private transient VocabCache vocab; + + private NegativeHolder() { + + } + + public synchronized void initHolder(@NonNull VocabCache vocabCache, double[] expTable, int layerSize) { + if (!wasInit.get()) { + this.vocab = vocabCache; + this.syn1Neg = Nd4j.zeros(vocabCache.numWords(), layerSize); + makeTable(Math.max(expTable.length, 100000), 0.75); + wasInit.set(true); + } + } + + protected void makeTable(int tableSize, double power) { + int vocabSize = vocab.numWords(); + table = Nd4j.create(DataType.FLOAT, tableSize); + double trainWordsPow = 0.0; + for (String word : vocab.words()) { + trainWordsPow += Math.pow(vocab.wordFrequency(word), power); + } + + int wordIdx = 0; + String word = vocab.wordAtIndex(wordIdx); + double d1 = Math.pow(vocab.wordFrequency(word), power) / trainWordsPow; + for (int i = 0; i < tableSize; i++) { + table.putScalar(i, wordIdx); + double mul = i * 1.0 / (double) tableSize; + if (mul > d1) { + if (wordIdx < vocabSize - 1) + wordIdx++; + word = vocab.wordAtIndex(wordIdx); + String wordAtIndex = vocab.wordAtIndex(wordIdx); + if (word == null) + continue; + d1 += Math.pow(vocab.wordFrequency(wordAtIndex), power) / trainWordsPow; + } + } + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/SecondIterationFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/SecondIterationFunction.java new file mode 100644 index 000000000..205d54ae0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/SecondIterationFunction.java @@ -0,0 +1,273 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; +import scala.Tuple2; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicLong; + +public class SecondIterationFunction implements FlatMapFunction, Long>>, Entry> { + + private int ithIteration = 1; + private int vectorLength; + private boolean useAdaGrad; + private int batchSize = 0; + private double negative; + private int window; + private double alpha; + private double minAlpha; + private long totalWordCount; + private long seed; + private int maxExp; + private double[] expTable; + private int iterations; + + private AtomicLong nextRandom = new AtomicLong(5); + + private volatile VocabCache vocab; + private transient volatile NegativeHolder negativeHolder; + private transient volatile VocabHolder vocabHolder; + private AtomicLong cid = new AtomicLong(0); + private AtomicLong aff = new AtomicLong(0); + + + + public SecondIterationFunction(Broadcast> word2vecVarMapBroadcast, + Broadcast expTableBroadcast, Broadcast> vocabCacheBroadcast) { + + Map word2vecVarMap = word2vecVarMapBroadcast.getValue(); + this.expTable = expTableBroadcast.getValue(); + this.vectorLength = (int) word2vecVarMap.get("vectorLength"); + this.useAdaGrad = (boolean) word2vecVarMap.get("useAdaGrad"); + this.negative = (double) word2vecVarMap.get("negative"); + this.window = (int) word2vecVarMap.get("window"); + this.alpha = (double) word2vecVarMap.get("alpha"); + this.minAlpha = (double) word2vecVarMap.get("minAlpha"); + this.totalWordCount = (long) word2vecVarMap.get("totalWordCount"); + this.seed = (long) word2vecVarMap.get("seed"); + this.maxExp = (int) word2vecVarMap.get("maxExp"); + this.iterations = (int) word2vecVarMap.get("iterations"); + this.batchSize = (int) word2vecVarMap.get("batchSize"); + + // this.indexSyn0VecMap = new HashMap<>(); + // this.pointSyn1VecMap = new HashMap<>(); + + this.vocab = vocabCacheBroadcast.getValue(); + + + if (this.vocab == null) + throw new RuntimeException("VocabCache is null"); + + + } + + + + @Override + public Iterator> call(Iterator, Long>> pairIter) { + this.vocabHolder = VocabHolder.getInstance(); + this.vocabHolder.setSeed(seed, vectorLength); + + if (negative > 0) { + negativeHolder = NegativeHolder.getInstance(); + negativeHolder.initHolder(vocab, expTable, this.vectorLength); + } + + while (pairIter.hasNext()) { + List, Long>> batch = new ArrayList<>(); + while (pairIter.hasNext() && batch.size() < batchSize) { + Tuple2, Long> pair = pairIter.next(); + List vocabWordsList = pair._1(); + Long sentenceCumSumCount = pair._2(); + batch.add(Pair.of(vocabWordsList, sentenceCumSumCount)); + } + + for (int i = 0; i < iterations; i++) { + //System.out.println("Training sentence: " + vocabWordsList); + for (Pair, Long> pair : batch) { + List vocabWordsList = pair.getKey(); + Long sentenceCumSumCount = pair.getValue(); + double currentSentenceAlpha = Math.max(minAlpha, + alpha - (alpha - minAlpha) * (sentenceCumSumCount / (double) totalWordCount)); + trainSentence(vocabWordsList, currentSentenceAlpha); + } + } + } + return vocabHolder.getSplit(vocab).iterator(); + } + + + public void trainSentence(List vocabWordsList, double currentSentenceAlpha) { + + if (vocabWordsList != null && !vocabWordsList.isEmpty()) { + for (int ithWordInSentence = 0; ithWordInSentence < vocabWordsList.size(); ithWordInSentence++) { + // Random value ranging from 0 to window size + nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11)); + int b = (int) (long) this.nextRandom.get() % window; + VocabWord currentWord = vocabWordsList.get(ithWordInSentence); + if (currentWord != null) { + skipGram(ithWordInSentence, vocabWordsList, b, currentSentenceAlpha); + } + } + } + } + + public void skipGram(int ithWordInSentence, List vocabWordsList, int b, double currentSentenceAlpha) { + + VocabWord currentWord = vocabWordsList.get(ithWordInSentence); + if (currentWord != null && !vocabWordsList.isEmpty()) { + int end = window * 2 + 1 - b; + for (int a = b; a < end; a++) { + if (a != window) { + int c = ithWordInSentence - window + a; + if (c >= 0 && c < vocabWordsList.size()) { + VocabWord lastWord = vocabWordsList.get(c); + iterateSample(currentWord, lastWord, currentSentenceAlpha); + } + } + } + } + } + + public void iterateSample(VocabWord w1, VocabWord w2, double currentSentenceAlpha) { + + + if (w1 == null || w2 == null || w2.getIndex() < 0 || w2.getIndex() == w1.getIndex()) + return; + final int currentWordIndex = w2.getIndex(); + + // error for current word and context + INDArray neu1e = Nd4j.create(vectorLength); + + // First iteration Syn0 is random numbers + INDArray l1 = vocabHolder.getSyn0Vector(currentWordIndex, vocab); + + + // + for (int i = 0; i < w1.getCodeLength(); i++) { + int code = w1.getCodes().get(i); + int point = w1.getPoints().get(i); + if (point < 0) + throw new IllegalStateException("Illegal point " + point); + // Point to + INDArray syn1 = vocabHolder.getSyn1Vector(point); + /* + if (pointSyn1VecMap.containsKey(point)) { + syn1 = pointSyn1VecMap.get(point); + } else { + syn1 = Nd4j.zeros(1, vectorLength); // 1 row of vector length of zeros + pointSyn1VecMap.put(point, syn1); + } + */ + + // Dot product of Syn0 and Syn1 vecs + double dot = Nd4j.getBlasWrapper().level1().dot(vectorLength, 1.0, l1, syn1); + + if (dot < -maxExp || dot >= maxExp) + continue; + + int idx = (int) ((dot + maxExp) * ((double) expTable.length / maxExp / 2.0)); + + if (idx >= expTable.length) + continue; + + //score + double f = expTable[idx]; + //gradient + double g = (1 - code - f) * (useAdaGrad ? w1.getGradient(i, currentSentenceAlpha, currentSentenceAlpha) + : currentSentenceAlpha); + + + Nd4j.getBlasWrapper().level1().axpy(vectorLength, g, syn1, neu1e); + Nd4j.getBlasWrapper().level1().axpy(vectorLength, g, l1, syn1); + } + + int target = w1.getIndex(); + int label; + //negative sampling + if (negative > 0) + for (int d = 0; d < negative + 1; d++) { + if (d == 0) + label = 1; + else { + nextRandom.set(Math.abs(nextRandom.get() * 25214903917L + 11)); + + int idx = (int) Math.abs((int) (nextRandom.get() >> 16) % negativeHolder.getTable().length()); + + target = negativeHolder.getTable().getInt(idx); + if (target <= 0) + target = (int) nextRandom.get() % (vocab.numWords() - 1) + 1; + + if (target == w1.getIndex()) + continue; + label = 0; + } + + if (target >= negativeHolder.getSyn1Neg().rows() || target < 0) + continue; + + double f = Nd4j.getBlasWrapper().dot(l1, negativeHolder.getSyn1Neg().slice(target)); + double g; + if (f > maxExp) + g = useAdaGrad ? w1.getGradient(target, (label - 1), alpha) : (label - 1) * alpha; + else if (f < -maxExp) + g = label * (useAdaGrad ? w1.getGradient(target, alpha, alpha) : alpha); + else { + int idx = (int) ((f + maxExp) * (expTable.length / maxExp / 2)); + if (idx >= expTable.length) + continue; + + g = useAdaGrad ? w1.getGradient(target, label - expTable[idx], alpha) + : (label - expTable[idx]) * alpha; + } + + Nd4j.getBlasWrapper().level1().axpy(vectorLength, g, negativeHolder.getSyn1Neg().slice(target), neu1e); + + Nd4j.getBlasWrapper().level1().axpy(vectorLength, g, l1, negativeHolder.getSyn1Neg().slice(target)); + } + + + // Updated the Syn0 vector based on gradient. Syn0 is not random anymore. + Nd4j.getBlasWrapper().level1().axpy(vectorLength, 1.0f, neu1e, l1); + + //VocabWord word = vocab.elementAtIndex(currentWordIndex); + //indexSyn0VecMap.put(word, l1); + } + + private INDArray getRandomSyn0Vec(int vectorLength, long lseed) { + /* + we use wordIndex as part of seed here, to guarantee that during word syn0 initialization on dwo distinct nodes, initial weights will be the same for the same word + */ + return Nd4j.rand(new int[] {1, vectorLength}, lseed * seed).subi(0.5).divi(vectorLength); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/SentenceBatch.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/SentenceBatch.java new file mode 100644 index 000000000..66b9299f4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/SentenceBatch.java @@ -0,0 +1,206 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.spark.api.java.function.Function; +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Triple; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author Adam Gibson + */ +@Deprecated +public class SentenceBatch implements Function { + + private AtomicLong nextRandom = new AtomicLong(5); + // private static Logger log = LoggerFactory.getLogger(SentenceBatch.class); + + + @Override + public Word2VecChange call(Word2VecFuncCall sentence) throws Exception { + Word2VecParam param = sentence.getParam().getValue(); + List> changed = new ArrayList<>(); + double alpha = Math.max(param.getMinAlpha(), + param.getAlpha() * (1 - (1.0 * sentence.getWordsSeen() / (double) param.getTotalWords()))); + + trainSentence(param, sentence.getSentence(), alpha, changed); + return new Word2VecChange(changed, param); + } + + + /** + * Train on a list of vocab words + * @param sentence the list of vocab words to train on + */ + public void trainSentence(Word2VecParam param, final List sentence, double alpha, + List> changed) { + if (sentence != null && !sentence.isEmpty()) { + for (int i = 0; i < sentence.size(); i++) { + VocabWord vocabWord = sentence.get(i); + if (vocabWord != null && vocabWord.getWord().endsWith("STOP")) { + nextRandom.set(nextRandom.get() * 25214903917L + 11); + skipGram(param, i, sentence, (int) nextRandom.get() % param.getWindow(), alpha, changed); + } + } + } + } + + + /** + * Train via skip gram + * @param i the current word + * @param sentence the sentence to train on + * @param b + * @param alpha the learning rate + */ + public void skipGram(Word2VecParam param, int i, List sentence, int b, double alpha, + List> changed) { + + final VocabWord word = sentence.get(i); + int window = param.getWindow(); + if (word != null && !sentence.isEmpty()) { + int end = window * 2 + 1 - b; + for (int a = b; a < end; a++) { + if (a != window) { + int c = i - window + a; + if (c >= 0 && c < sentence.size()) { + VocabWord lastWord = sentence.get(c); + iterateSample(param, word, lastWord, alpha, changed); + } + } + } + } + } + + + + /** + * Iterate on the given 2 vocab words + * + * @param w1 the first word to iterate on + * @param w2 the second word to iterate on + */ + public void iterateSample(Word2VecParam param, VocabWord w1, VocabWord w2, double alpha, + List> changed) { + if (w2 == null || w2.getIndex() < 0 || w1.getIndex() == w2.getIndex() || w1.getWord().equals("STOP") + || w2.getWord().equals("STOP") || w1.getWord().equals("UNK") || w2.getWord().equals("UNK")) + return; + int vectorLength = param.getVectorLength(); + InMemoryLookupTable weights = param.getWeights(); + boolean useAdaGrad = param.isUseAdaGrad(); + double negative = param.getNegative(); + INDArray table = param.getTable(); + double[] expTable = param.getExpTable().getValue(); + double MAX_EXP = 6; + int numWords = param.getNumWords(); + //current word vector + INDArray l1 = weights.vector(w2.getWord()); + + + //error for current word and context + INDArray neu1e = Nd4j.create(vectorLength); + + for (int i = 0; i < w1.getCodeLength(); i++) { + int code = w1.getCodes().get(i); + int point = w1.getPoints().get(i); + + INDArray syn1 = weights.getSyn1().slice(point); + + double dot = Nd4j.getBlasWrapper().level1().dot(syn1.length(), 1.0, l1, syn1); + + if (dot < -MAX_EXP || dot >= MAX_EXP) + continue; + + int idx = (int) ((dot + MAX_EXP) * ((double) expTable.length / MAX_EXP / 2.0)); + + //score + double f = expTable[idx]; + //gradient + double g = (1 - code - f) * (useAdaGrad ? w1.getGradient(i, alpha, alpha) : alpha); + + + Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, syn1, neu1e); + Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, l1, syn1); + + + changed.add(new Triple<>(point, w1.getIndex(), -1)); + + } + + + changed.add(new Triple<>(w1.getIndex(), w2.getIndex(), -1)); + //negative sampling + if (negative > 0) { + int target = w1.getIndex(); + int label; + INDArray syn1Neg = weights.getSyn1Neg().slice(target); + + for (int d = 0; d < negative + 1; d++) { + if (d == 0) { + + label = 1; + } else { + nextRandom.set(nextRandom.get() * 25214903917L + 11); + target = table.getInt((int) (nextRandom.get() >> 16) % (int) table.length()); + if (target == 0) + target = (int) nextRandom.get() % (numWords - 1) + 1; + if (target == w1.getIndex()) + continue; + label = 0; + } + + double f = Nd4j.getBlasWrapper().dot(l1, syn1Neg); + double g; + if (f > MAX_EXP) + g = useAdaGrad ? w1.getGradient(target, (label - 1), alpha) : (label - 1) * alpha; + else if (f < -MAX_EXP) + g = label * (useAdaGrad ? w1.getGradient(target, alpha, alpha) : alpha); + else + g = useAdaGrad ? w1 + .getGradient(target, + label - expTable[(int) ((f + MAX_EXP) + * (expTable.length / MAX_EXP / 2))], + alpha) + : (label - expTable[(int) ((f + MAX_EXP) * (expTable.length / MAX_EXP / 2))]) + * alpha; + Nd4j.getBlasWrapper().level1().axpy(l1.length(), g, neu1e, l1); + + Nd4j.getBlasWrapper().level1().axpy(l1.length(), g, syn1Neg, l1); + + changed.add(new Triple<>(-1, -1, label)); + + } + } + + + Nd4j.getBlasWrapper().level1().axpy(l1.length(), 1.0f, neu1e, l1); + + + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/VocabHolder.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/VocabHolder.java new file mode 100644 index 000000000..1a983c68d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/VocabHolder.java @@ -0,0 +1,109 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.io.Serializable; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class VocabHolder implements Serializable { + private static VocabHolder ourInstance = new VocabHolder(); + + private Map indexSyn0VecMap = new ConcurrentHashMap<>(); + private Map pointSyn1VecMap = new ConcurrentHashMap<>(); + private HashSet workers = new LinkedHashSet<>(); + + private AtomicLong seed = new AtomicLong(0); + private AtomicInteger vectorLength = new AtomicInteger(0); + + public static VocabHolder getInstance() { + return ourInstance; + } + + private VocabHolder() {} + + public void setSeed(long seed, int vectorLength) { + this.seed.set(seed); + this.vectorLength.set(vectorLength); + } + + public INDArray getSyn0Vector(Integer wordIndex, VocabCache vocabCache) { + if (!workers.contains(Thread.currentThread().getId())) + workers.add(Thread.currentThread().getId()); + + VocabWord word = vocabCache.elementAtIndex(wordIndex); + + if (!indexSyn0VecMap.containsKey(word)) { + synchronized (this) { + if (!indexSyn0VecMap.containsKey(word)) { + indexSyn0VecMap.put(word, getRandomSyn0Vec(vectorLength.get(), wordIndex)); + } + } + } + + return indexSyn0VecMap.get(word); + } + + public INDArray getSyn1Vector(Integer point) { + + if (!pointSyn1VecMap.containsKey(point)) { + synchronized (this) { + if (!pointSyn1VecMap.containsKey(point)) { + pointSyn1VecMap.put(point, Nd4j.zeros(1, vectorLength.get())); + } + } + } + + return pointSyn1VecMap.get(point); + } + + private INDArray getRandomSyn0Vec(int vectorLength, long lseed) { + /* + we use wordIndex as part of seed here, to guarantee that during word syn0 initialization on dwo distinct nodes, initial weights will be the same for the same word + */ + return Nd4j.rand(new int[] {1, vectorLength}, lseed * seed.get()).subi(0.5).divi(vectorLength); + } + + public Iterable> getSplit(VocabCache vocabCache) { + Set> set = new HashSet<>(); + int cnt = 0; + for (Map.Entry entry : indexSyn0VecMap.entrySet()) { + set.add(entry); + cnt++; + if (cnt > 10) + break; + } + + System.out.println("Returning set: " + set.size()); + + return set; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2Vec.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2Vec.java new file mode 100644 index 000000000..b5146f74d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2Vec.java @@ -0,0 +1,592 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import lombok.Getter; +import lombok.NonNull; +import org.apache.commons.math3.util.FastMath; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration; +import org.deeplearning4j.models.embeddings.wordvectors.WordVectorsImpl; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.deeplearning4j.spark.text.functions.CountCumSum; +import org.deeplearning4j.spark.text.functions.TextPipeline; +import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; +import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.heartbeat.reports.Environment; +import org.nd4j.linalg.heartbeat.reports.Event; +import org.nd4j.linalg.heartbeat.utils.EnvironmentUtils; +import org.nd4j.common.primitives.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class Word2Vec extends WordVectorsImpl implements Serializable { + + private INDArray trainedSyn1; + private static Logger log = LoggerFactory.getLogger(Word2Vec.class); + private int MAX_EXP = 6; + @Getter + private double[] expTable; + @Getter + protected VectorsConfiguration configuration; + + // Input by user only via setters + private int nGrams = 1; + private String tokenizer = "org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory"; + private String tokenPreprocessor = "org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor"; + private boolean removeStop = false; + private long seed = 42L; + private boolean useUnknown = false; + + // Constructor to take InMemoryLookupCache table from an already trained model + protected Word2Vec(INDArray trainedSyn1) { + this.trainedSyn1 = trainedSyn1; + this.expTable = initExpTable(); + } + + protected Word2Vec() { + this.expTable = initExpTable(); + } + + protected double[] initExpTable() { + double[] expTable = new double[100000]; + for (int i = 0; i < expTable.length; i++) { + double tmp = FastMath.exp((i / (double) expTable.length * 2 - 1) * MAX_EXP); + expTable[i] = tmp / (tmp + 1.0); + } + return expTable; + } + + public Map getTokenizerVarMap() { + return new HashMap() { + { + put("numWords", minWordFrequency); + put("nGrams", nGrams); + put("tokenizer", tokenizer); + put("tokenPreprocessor", tokenPreprocessor); + put("removeStop", removeStop); + put("stopWords", stopWords); + put("useUnk", useUnknown); + put("vectorsConfiguration", configuration); + } + }; + } + + public Map getWord2vecVarMap() { + return new HashMap() { + { + put("vectorLength", layerSize); + put("useAdaGrad", useAdeGrad); + put("negative", negative); + put("window", window); + put("alpha", learningRate.get()); + put("minAlpha", minLearningRate); + put("iterations", numIterations); + put("seed", seed); + put("maxExp", MAX_EXP); + put("batchSize", batchSize); + } + }; + } + + /** + * Training word2vec model on a given text corpus + * + * @param corpusRDD training corpus + * @throws Exception + */ + public void train(JavaRDD corpusRDD) throws Exception { + log.info("Start training ..."); + + if (workers > 0) + corpusRDD.repartition(workers); + + // SparkContext + final JavaSparkContext sc = new JavaSparkContext(corpusRDD.context()); + + // Pre-defined variables + Map tokenizerVarMap = getTokenizerVarMap(); + Map word2vecVarMap = getWord2vecVarMap(); + + // Variables to fill in train + final JavaRDD sentenceWordsCountRDD; + final JavaRDD> vocabWordListRDD; + final JavaPairRDD, Long> vocabWordListSentenceCumSumRDD; + final VocabCache vocabCache; + final JavaRDD sentenceCumSumCountRDD; + int maxRep = 1; + + // Start Training // + ////////////////////////////////////// + log.info("Tokenization and building VocabCache ..."); + // Processing every sentence and make a VocabCache which gets fed into a LookupCache + Broadcast> broadcastTokenizerVarMap = sc.broadcast(tokenizerVarMap); + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + pipeline.buildVocabWordListRDD(); + + // Get total word count and put into word2vec variable map + word2vecVarMap.put("totalWordCount", pipeline.getTotalWordCount()); + + // 2 RDDs: (vocab words list) and (sentence Count).Already cached + sentenceWordsCountRDD = pipeline.getSentenceCountRDD(); + vocabWordListRDD = pipeline.getVocabWordListRDD(); + + // Get vocabCache and broad-casted vocabCache + Broadcast> vocabCacheBroadcast = pipeline.getBroadCastVocabCache(); + vocabCache = vocabCacheBroadcast.getValue(); + + log.info("Vocab size: {}", vocabCache.numWords()); + + ////////////////////////////////////// + log.info("Building Huffman Tree ..."); + // Building Huffman Tree would update the code and point in each of the vocabWord in vocabCache + /* + We don't need to build tree here, since it was built earlier, at TextPipeline.buildVocabCache() call. + + Huffman huffman = new Huffman(vocabCache.vocabWords()); + huffman.build(); + huffman.applyIndexes(vocabCache); + */ + ////////////////////////////////////// + log.info("Calculating cumulative sum of sentence counts ..."); + sentenceCumSumCountRDD = new CountCumSum(sentenceWordsCountRDD).buildCumSum(); + + ////////////////////////////////////// + log.info("Mapping to RDD(vocabWordList, cumulative sentence count) ..."); + vocabWordListSentenceCumSumRDD = + vocabWordListRDD.zip(sentenceCumSumCountRDD).setName("vocabWordListSentenceCumSumRDD"); + + ///////////////////////////////////// + log.info("Broadcasting word2vec variables to workers ..."); + Broadcast> word2vecVarMapBroadcast = sc.broadcast(word2vecVarMap); + Broadcast expTableBroadcast = sc.broadcast(expTable); + + + + ///////////////////////////////////// + log.info("Training word2vec sentences ..."); + FlatMapFunction firstIterFunc = + new FirstIterationFunction(word2vecVarMapBroadcast, expTableBroadcast, vocabCacheBroadcast); + @SuppressWarnings("unchecked") + JavaRDD> indexSyn0UpdateEntryRDD = + vocabWordListSentenceCumSumRDD.mapPartitions(firstIterFunc).map(new MapToPairFunction()); + + // Get all the syn0 updates into a list in driver + List> syn0UpdateEntries = indexSyn0UpdateEntryRDD.collect(); + + // Instantiate syn0 + INDArray syn0 = Nd4j.zeros(vocabCache.numWords(), layerSize); + + // Updating syn0 first pass: just add vectors obtained from different nodes + log.info("Averaging results..."); + Map updates = new HashMap<>(); + Map updaters = new HashMap<>(); + for (Pair syn0UpdateEntry : syn0UpdateEntries) { + syn0.getRow(syn0UpdateEntry.getFirst().getIndex()).addi(syn0UpdateEntry.getSecond()); + + // for proper averaging we need to divide resulting sums later, by the number of additions + if (updates.containsKey(syn0UpdateEntry.getFirst())) { + updates.get(syn0UpdateEntry.getFirst()).incrementAndGet(); + } else + updates.put(syn0UpdateEntry.getFirst(), new AtomicInteger(1)); + + if (!updaters.containsKey(syn0UpdateEntry.getFirst().getVocabId())) { + updaters.put(syn0UpdateEntry.getFirst().getVocabId(), syn0UpdateEntry.getFirst().getAffinityId()); + } + } + + // Updating syn0 second pass: average obtained vectors + for (Map.Entry entry : updates.entrySet()) { + if (entry.getValue().get() > 1) { + if (entry.getValue().get() > maxRep) + maxRep = entry.getValue().get(); + syn0.getRow(entry.getKey().getIndex()).divi(entry.getValue().get()); + } + } + + long totals = 0; + + log.info("Finished calculations..."); + + + vocab = vocabCache; + InMemoryLookupTable inMemoryLookupTable = new InMemoryLookupTable(); + Environment env = EnvironmentUtils.buildEnvironment(); + env.setNumCores(maxRep); + env.setAvailableMemory(totals); + update(env, Event.SPARK); + inMemoryLookupTable.setVocab(vocabCache); + inMemoryLookupTable.setVectorLength(layerSize); + inMemoryLookupTable.setSyn0(syn0); + lookupTable = inMemoryLookupTable; + modelUtils.init(lookupTable); + } + + + + public static class Builder { + protected int nGrams = 1; + protected int numIterations = 1; + protected int minWordFrequency = 1; + protected int numEpochs = 1; + protected double learningRate = 0.025; + protected double minLearningRate = 0.001; + protected int windowSize = 5; + protected double negative = 0; + protected double sampling = 1e-5; + protected long seed = 42L; + protected boolean useAdaGrad = false; + protected TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory(); + protected VectorsConfiguration configuration = new VectorsConfiguration(); + protected int layerSize; + protected List stopWords = new ArrayList<>(); + protected int batchSize = 100; + protected boolean useUnk = false; + private String tokenizer = ""; + private String tokenPreprocessor = ""; + private int workers = 0; + + /** + * Creates Builder instance with default parameters set. + */ + public Builder() { + this(new VectorsConfiguration()); + } + + /** + * Uses VectorsConfiguration bean to initialize Word2Vec model parameters + * + * @param configuration + */ + public Builder(VectorsConfiguration configuration) { + this.configuration = configuration; + this.numIterations = configuration.getIterations(); + this.numEpochs = configuration.getEpochs(); + this.minLearningRate = configuration.getMinLearningRate(); + this.learningRate = configuration.getLearningRate(); + this.sampling = configuration.getSampling(); + this.negative = configuration.getNegative(); + this.minWordFrequency = configuration.getMinWordFrequency(); + this.seed = configuration.getSeed(); + // this.stopWords = configuration.get + + // TODO: investigate this + //this.hugeModelExpected = configuration.isHugeModelExpected(); + + this.batchSize = configuration.getBatchSize(); + this.layerSize = configuration.getLayersSize(); + + // this.learningRateDecayWords = configuration.getLearningRateDecayWords(); + this.useAdaGrad = configuration.isUseAdaGrad(); + this.windowSize = configuration.getWindow(); + + if (configuration.getStopList() != null) + this.stopWords.addAll(configuration.getStopList()); + } + + /** + * Specifies window size + * + * @param windowSize + * @return + */ + public Builder windowSize(int windowSize) { + this.windowSize = windowSize; + return this; + } + + /** + * Specifies negative sampling + * @param negative + * @return + */ + public Builder negative(int negative) { + this.negative = negative; + return this; + } + + /** + * Specifies subsamplng value + * + * @param sampling + * @return + */ + public Builder sampling(double sampling) { + this.sampling = sampling; + return this; + } + + /** + * This method specifies initial learning rate for model + * + * @param lr + * @return + */ + public Builder learningRate(double lr) { + this.learningRate = lr; + return this; + } + + /** + * This method specifies bottom threshold for learning rate decay + * + * @param mlr + * @return + */ + public Builder minLearningRate(double mlr) { + this.minLearningRate = mlr; + return this; + } + + /** + * This method specifies number of iterations over batch on each node + * + * @param numIterations + * @return + */ + public Builder iterations(int numIterations) { + this.numIterations = numIterations; + return this; + } + + /** + * This method specifies number of epochs done over whole corpus + * + * PLEASE NOTE: NOT IMPLEMENTED + * + * @param numEpochs + * @return + */ + public Builder epochs(int numEpochs) { + // TODO: implement epochs imitation for spark w2v + this.numEpochs = numEpochs; + return this; + } + + /** + * This method specifies minimum word frequency threshold. All words below this threshold will be ignored. + * + * @param minWordFrequency + * @return + */ + public Builder minWordFrequency(int minWordFrequency) { + this.minWordFrequency = minWordFrequency; + return this; + } + + /** + * This method specifies, if adaptive gradients should be used during model training + * + * @param reallyUse + * @return + */ + public Builder useAdaGrad(boolean reallyUse) { + this.useAdaGrad = reallyUse; + return this; + } + + /** + * Specifies random seed to be used during weights initialization; + * + * @param seed + * @return + */ + public Builder seed(long seed) { + this.seed = seed; + return this; + } + + /** + * Specifies TokenizerFactory to be used for tokenization + * + * PLEASE NOTE: You can't use anonymous implementation here + * + * @param factory + * @return + */ + public Builder tokenizerFactory(@NonNull TokenizerFactory factory) { + this.tokenizer = factory.getClass().getCanonicalName(); + + if (factory.getTokenPreProcessor() != null) { + this.tokenPreprocessor = factory.getTokenPreProcessor().getClass().getCanonicalName(); + } else { + this.tokenPreprocessor = ""; + } + + return this; + } + + /** + * Specifies TokenizerFactory class to be used for tokenization + * + * + * @param tokenizer class name for tokenizerFactory + * @return + */ + public Builder tokenizerFactory(@NonNull String tokenizer) { + this.tokenizer = tokenizer; + return this; + } + + /** + * Specifies TokenPreProcessor class to be used during tokenization + * + * + * @param tokenPreprocessor class name for tokenPreProcessor + * @return + */ + public Builder tokenPreprocessor(@NonNull String tokenPreprocessor) { + this.tokenPreprocessor = tokenPreprocessor; + return this; + } + + /** + * Specify number of workers for training process. + * This value will be used to repartition RDD. + * + * PLEASE NOTE: Recommended value is number of vCPU available within your spark cluster. + * + * @param workers + * @return + */ + public Builder workers(int workers) { + this.workers = workers; + return this; + } + + /** + * Specifies output vector's dimensions + * + * @param layerSize + * @return + */ + public Builder layerSize(int layerSize) { + this.layerSize = layerSize; + return this; + } + + /** + * Specifies N of n-Grams :) + * + * @param nGrams + * @return + */ + public Builder setNGrams(int nGrams) { + this.nGrams = nGrams; + return this; + } + + /** + * This method defines list of stop-words, that are to be ignored during vocab building and training + * + * @param stopWords + * @return + */ + public Builder stopWords(@NonNull List stopWords) { + for (String word : stopWords) { + if (!this.stopWords.contains(word)) + this.stopWords.add(word); + } + return this; + } + + /** + * Specifies the size of mini-batch, used in single iteration during training + * + * @param batchSize + * @return + */ + public Builder batchSize(int batchSize) { + this.batchSize = batchSize; + return this; + } + + /** + * Specifies, if UNK word should be used instead of words that are absent in vocab + * + * @param reallyUse + * @return + */ + public Builder useUnknown(boolean reallyUse) { + this.useUnk = reallyUse; + return this; + } + + public Word2Vec build() { + Word2Vec ret = new Word2Vec(); + + this.configuration.setLearningRate(this.learningRate); + this.configuration.setLayersSize(layerSize); + this.configuration.setWindow(windowSize); + this.configuration.setMinWordFrequency(minWordFrequency); + this.configuration.setIterations(numIterations); + this.configuration.setSeed(seed); + this.configuration.setMinLearningRate(minLearningRate); + this.configuration.setSampling(this.sampling); + this.configuration.setUseAdaGrad(useAdaGrad); + this.configuration.setNegative(negative); + this.configuration.setEpochs(this.numEpochs); + this.configuration.setBatchSize(this.batchSize); + this.configuration.setStopList(this.stopWords); + + ret.workers = this.workers; + ret.nGrams = this.nGrams; + + ret.configuration = this.configuration; + + ret.numEpochs = this.numEpochs; + ret.numIterations = this.numIterations; + ret.minWordFrequency = this.minWordFrequency; + ret.learningRate.set(this.learningRate); + ret.minLearningRate = this.minLearningRate; + ret.sampling = this.sampling; + ret.negative = this.negative; + ret.layerSize = this.layerSize; + ret.window = this.windowSize; + ret.useAdeGrad = this.useAdaGrad; + ret.stopWords = this.stopWords; + ret.batchSize = this.batchSize; + ret.useUnknown = this.useUnk; + + ret.tokenizer = this.tokenizer; + ret.tokenPreprocessor = this.tokenPreprocessor; + + return ret; + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecChange.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecChange.java new file mode 100644 index 000000000..5ce201f0c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecChange.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Triple; + +import java.io.Serializable; +import java.util.*; + +/** + * @author Adam Gibson + */ +@Deprecated +public class Word2VecChange implements Serializable { + private Map> changes = new HashMap<>(); + + public Word2VecChange(List> counterMap, Word2VecParam param) { + Iterator> iter = counterMap.iterator(); + while (iter.hasNext()) { + Triple next = iter.next(); + Integer point = next.getFirst(); + Integer index = next.getSecond(); + + Set changes = this.changes.get(point); + if (changes == null) { + changes = new HashSet<>(); + this.changes.put(point, changes); + } + + changes.add(param.getWeights().getSyn1().slice(index)); + + } + } + + /** + * Take the changes and apply them + * to the given table + * @param table the memory lookup table + * to apply the changes to + */ + public void apply(InMemoryLookupTable table) { + for (Map.Entry> entry : changes.entrySet()) { + Set changes = entry.getValue(); + INDArray toChange = table.getSyn0().slice(entry.getKey()); + for (INDArray syn1 : changes) + Nd4j.getBlasWrapper().level1().axpy(toChange.length(), 1, syn1, toChange); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecFuncCall.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecFuncCall.java new file mode 100644 index 000000000..b7a41ea58 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecFuncCall.java @@ -0,0 +1,64 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.word2vec.VocabWord; + +import java.io.Serializable; +import java.util.List; + +@Deprecated +public class Word2VecFuncCall implements Serializable { + private Broadcast param; + private Long wordsSeen; + private List sentence; + + public Word2VecFuncCall(Broadcast param, Long wordsSeen, List sentence) { + this.param = param; + this.wordsSeen = wordsSeen; + this.sentence = sentence; + } + + public Broadcast getParam() { + return param; + } + + public void setParam(Broadcast param) { + this.param = param; + } + + public Long getWordsSeen() { + return wordsSeen; + } + + public void setWordsSeen(Long wordsSeen) { + this.wordsSeen = wordsSeen; + } + + public List getSentence() { + return sentence; + } + + public void setSentence(List sentence) { + this.sentence = sentence; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecParam.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecParam.java new file mode 100644 index 000000000..1e7f81133 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecParam.java @@ -0,0 +1,302 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author Adam Gibson + */ +@Deprecated +public class Word2VecParam implements Serializable { + + private boolean useAdaGrad = false; + private double negative = 5; + private int numWords = 1; + private INDArray table; + private int window = 5; + private AtomicLong nextRandom = new AtomicLong(5); + private double alpha = 0.025; + private double minAlpha = 1e-2; + private int totalWords = 1; + private static transient final Logger log = LoggerFactory.getLogger(Word2VecPerformer.class); + private int lastChecked = 0; + private Broadcast wordCount; + private InMemoryLookupTable weights; + private int vectorLength; + private Broadcast expTable; + private AtomicLong wordsSeen = new AtomicLong(0); + private AtomicLong lastWords = new AtomicLong(0); + + public Word2VecParam(boolean useAdaGrad, double negative, int numWords, INDArray table, int window, + AtomicLong nextRandom, double alpha, double minAlpha, int totalWords, int lastChecked, + Broadcast wordCount, InMemoryLookupTable weights, int vectorLength, + Broadcast expTable) { + this.useAdaGrad = useAdaGrad; + this.negative = negative; + this.numWords = numWords; + this.table = table; + this.window = window; + this.nextRandom = nextRandom; + this.alpha = alpha; + this.minAlpha = minAlpha; + this.totalWords = totalWords; + this.lastChecked = lastChecked; + this.wordCount = wordCount; + this.weights = weights; + this.vectorLength = vectorLength; + this.expTable = expTable; + } + + public AtomicLong getLastWords() { + return lastWords; + } + + public void setLastWords(AtomicLong lastWords) { + this.lastWords = lastWords; + } + + public AtomicLong getWordsSeen() { + return wordsSeen; + } + + public void setWordsSeen(AtomicLong wordsSeen) { + this.wordsSeen = wordsSeen; + } + + public Broadcast getExpTable() { + return expTable; + } + + public void setExpTable(Broadcast expTable) { + this.expTable = expTable; + } + + public boolean isUseAdaGrad() { + return useAdaGrad; + } + + public void setUseAdaGrad(boolean useAdaGrad) { + this.useAdaGrad = useAdaGrad; + } + + public double getNegative() { + return negative; + } + + public void setNegative(double negative) { + this.negative = negative; + } + + public int getNumWords() { + return numWords; + } + + public void setNumWords(int numWords) { + this.numWords = numWords; + } + + public INDArray getTable() { + return table; + } + + public void setTable(INDArray table) { + this.table = table; + } + + public int getWindow() { + return window; + } + + public void setWindow(int window) { + this.window = window; + } + + public AtomicLong getNextRandom() { + return nextRandom; + } + + public void setNextRandom(AtomicLong nextRandom) { + this.nextRandom = nextRandom; + } + + public double getAlpha() { + return alpha; + } + + public void setAlpha(double alpha) { + this.alpha = alpha; + } + + public double getMinAlpha() { + return minAlpha; + } + + public void setMinAlpha(double minAlpha) { + this.minAlpha = minAlpha; + } + + public int getTotalWords() { + return totalWords; + } + + public void setTotalWords(int totalWords) { + this.totalWords = totalWords; + } + + public static Logger getLog() { + return log; + } + + public int getLastChecked() { + return lastChecked; + } + + public void setLastChecked(int lastChecked) { + this.lastChecked = lastChecked; + } + + public Broadcast getWordCount() { + return wordCount; + } + + public void setWordCount(Broadcast wordCount) { + this.wordCount = wordCount; + } + + public InMemoryLookupTable getWeights() { + return weights; + } + + public void setWeights(InMemoryLookupTable weights) { + this.weights = weights; + } + + + + public int getVectorLength() { + return vectorLength; + } + + public void setVectorLength(int vectorLength) { + this.vectorLength = vectorLength; + } + + public static class Builder { + private boolean useAdaGrad = true; + private double negative = 0; + private int numWords = 1; + private INDArray table; + private int window = 5; + private AtomicLong nextRandom; + private double alpha = 0.025; + private double minAlpha = 0.01; + private int totalWords; + private int lastChecked; + private Broadcast wordCount; + private InMemoryLookupTable weights; + private int vectorLength = 300; + private Broadcast expTable; + + public Builder expTable(Broadcast expTable) { + this.expTable = expTable; + return this; + } + + + public Builder useAdaGrad(boolean useAdaGrad) { + this.useAdaGrad = useAdaGrad; + return this; + } + + public Builder negative(double negative) { + this.negative = negative; + return this; + } + + public Builder numWords(int numWords) { + this.numWords = numWords; + return this; + } + + public Builder table(INDArray table) { + this.table = table; + return this; + } + + public Builder window(int window) { + this.window = window; + return this; + } + + public Builder setNextRandom(AtomicLong nextRandom) { + this.nextRandom = nextRandom; + return this; + } + + public Builder setAlpha(double alpha) { + this.alpha = alpha; + return this; + } + + public Builder setMinAlpha(double minAlpha) { + this.minAlpha = minAlpha; + return this; + } + + public Builder totalWords(int totalWords) { + this.totalWords = totalWords; + return this; + } + + public Builder lastChecked(int lastChecked) { + this.lastChecked = lastChecked; + return this; + } + + public Builder wordCount(Broadcast wordCount) { + this.wordCount = wordCount; + return this; + } + + public Builder weights(InMemoryLookupTable weights) { + this.weights = weights; + return this; + } + + public Builder setVectorLength(int vectorLength) { + this.vectorLength = vectorLength; + return this; + } + + public Word2VecParam build() { + return new Word2VecParam(useAdaGrad, negative, numWords, table, window, nextRandom, alpha, minAlpha, + totalWords, lastChecked, wordCount, weights, vectorLength, expTable); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecPerformer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecPerformer.java new file mode 100644 index 000000000..3b65a353d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecPerformer.java @@ -0,0 +1,260 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.commons.math3.util.FastMath; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.VoidFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +@Deprecated +public class Word2VecPerformer implements VoidFunction, AtomicLong>> { + + private static double MAX_EXP = 6; + private boolean useAdaGrad = false; + private double negative = 5; + private int numWords = 1; + private INDArray table; + private int window = 5; + private AtomicLong nextRandom = new AtomicLong(5); + private double alpha = 0.025; + private double minAlpha = 1e-2; + private int totalWords = 1; + private static transient final Logger log = LoggerFactory.getLogger(Word2VecPerformer.class); + private int lastChecked = 0; + private Broadcast wordCount; + private InMemoryLookupTable weights; + private double[] expTable = new double[1000]; + private int vectorLength; + + + public Word2VecPerformer(SparkConf sc, Broadcast wordCount, InMemoryLookupTable weights) { + this.weights = weights; + this.wordCount = wordCount; + setup(sc); + } + + public void setup(SparkConf conf) { + useAdaGrad = conf.getBoolean(Word2VecVariables.ADAGRAD, false); + negative = conf.getDouble(Word2VecVariables.NEGATIVE, 5); + numWords = conf.getInt(Word2VecVariables.NUM_WORDS, 1); + window = conf.getInt(Word2VecVariables.WINDOW, 5); + alpha = conf.getDouble(Word2VecVariables.ALPHA, 0.025f); + minAlpha = conf.getDouble(Word2VecVariables.MIN_ALPHA, 1e-2f); + totalWords = conf.getInt(Word2VecVariables.NUM_WORDS, 1); + vectorLength = conf.getInt(Word2VecVariables.VECTOR_LENGTH, 100); + initExpTable(); + + if (negative > 0 && conf.contains(Word2VecVariables.TABLE)) { + ByteArrayInputStream bis = new ByteArrayInputStream(conf.get(Word2VecVariables.TABLE).getBytes()); + DataInputStream dis = new DataInputStream(bis); + table = Nd4j.read(dis); + } + + } + + + + /** + * Train on a list of vocab words + * @param sentence the list of vocab words to train on + */ + public void trainSentence(final List sentence, double alpha) { + if (sentence != null && !sentence.isEmpty()) { + for (int i = 0; i < sentence.size(); i++) { + if (!sentence.get(i).getWord().endsWith("STOP")) { + nextRandom.set(nextRandom.get() * 25214903917L + 11); + skipGram(i, sentence, (int) nextRandom.get() % window, alpha); + } + } + } + + } + + + /** + * Train via skip gram + * @param i + * @param sentence + */ + public void skipGram(int i, List sentence, int b, double alpha) { + + final VocabWord word = sentence.get(i); + if (word != null && !sentence.isEmpty()) { + int end = window * 2 + 1 - b; + for (int a = b; a < end; a++) { + if (a != window) { + int c = i - window + a; + if (c >= 0 && c < sentence.size()) { + VocabWord lastWord = sentence.get(c); + iterateSample(word, lastWord, alpha); + } + } + } + } + } + + + + /** + * Iterate on the given 2 vocab words + * + * @param w1 the first word to iterate on + * @param w2 the second word to iterate on + */ + public void iterateSample(VocabWord w1, VocabWord w2, double alpha) { + if (w2 == null || w2.getIndex() < 0) + return; + + //current word vector + INDArray l1 = weights.vector(w2.getWord()); + + + //error for current word and context + INDArray neu1e = Nd4j.create(vectorLength); + + for (int i = 0; i < w1.getCodeLength(); i++) { + int code = w1.getCodes().get(i); + int point = w1.getPoints().get(i); + + INDArray syn1 = weights.getSyn1().slice(point); + + double dot = Nd4j.getBlasWrapper().dot(l1, syn1); + + if (dot >= -MAX_EXP && dot < MAX_EXP) { + + int idx = (int) ((dot + MAX_EXP) * ((double) expTable.length / MAX_EXP / 2.0)); + if (idx >= expTable.length) + continue; + + //score + double f = expTable[idx]; + //gradient + double g = (1 - code - f) * (useAdaGrad ? w1.getGradient(i, alpha, this.alpha) : alpha); + + Nd4j.getBlasWrapper().level1().axpy(l1.length(), g, syn1, neu1e); + Nd4j.getBlasWrapper().level1().axpy(l1.length(), g, l1, syn1); + } + + + } + + + //negative sampling + if (negative > 0) { + int target = w1.getIndex(); + int label; + INDArray syn1Neg = weights.getSyn1Neg().slice(target); + + for (int d = 0; d < negative + 1; d++) { + if (d == 0) { + + label = 1; + } else { + nextRandom.set(nextRandom.get() * 25214903917L + 11); + + target = table.getInt((int) (nextRandom.get() >> 16) % (int) table.length()); + if (target == 0) + target = (int) nextRandom.get() % (numWords - 1) + 1; + if (target == w1.getIndex()) + continue; + label = 0; + } + + double f = Nd4j.getBlasWrapper().dot(l1, syn1Neg); + double g; + if (f > MAX_EXP) + g = useAdaGrad ? w1.getGradient(target, (label - 1), this.alpha) : (label - 1) * alpha; + else if (f < -MAX_EXP) + g = label * (useAdaGrad ? w1.getGradient(target, alpha, this.alpha) : alpha); + else + g = useAdaGrad ? w1 + .getGradient(target, + label - expTable[(int) ((f + MAX_EXP) + * (expTable.length / MAX_EXP / 2))], + this.alpha) + : (label - expTable[(int) ((f + MAX_EXP) * (expTable.length / MAX_EXP / 2))]) + * alpha; + if (syn1Neg.data().dataType() == DataType.DOUBLE) + Nd4j.getBlasWrapper().axpy(g, neu1e, l1); + else + Nd4j.getBlasWrapper().axpy((float) g, neu1e, l1); + + if (syn1Neg.data().dataType() == DataType.DOUBLE) + Nd4j.getBlasWrapper().axpy(g, syn1Neg, l1); + else + Nd4j.getBlasWrapper().axpy((float) g, syn1Neg, l1); + } + } + + if (neu1e.data().dataType() == DataType.DOUBLE) + Nd4j.getBlasWrapper().axpy(1.0, neu1e, l1); + + else + Nd4j.getBlasWrapper().axpy(1.0f, neu1e, l1); + + } + + private void initExpTable() { + for (int i = 0; i < expTable.length; i++) { + double tmp = FastMath.exp((i / (double) expTable.length * 2 - 1) * MAX_EXP); + expTable[i] = tmp / (tmp + 1.0); + } + } + + + @Override + public void call(Pair, AtomicLong> pair) throws Exception { + double numWordsSoFar = wordCount.getValue().doubleValue(); + + List sentence = pair.getFirst(); + double alpha2 = Math.max(minAlpha, alpha * (1 - (1.0 * numWordsSoFar / (double) totalWords))); + int totalNewWords = 0; + trainSentence(sentence, alpha2); + totalNewWords += sentence.size(); + + + + double newWords = totalNewWords + numWordsSoFar; + double diff = Math.abs(newWords - lastChecked); + if (diff >= 10000) { + lastChecked = (int) newWords; + log.info("Words so far " + newWords + " out of " + totalWords); + } + + pair.getSecond().getAndAdd((long) totalNewWords); + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecPerformerVoid.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecPerformerVoid.java new file mode 100644 index 000000000..7bb7c44d8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecPerformerVoid.java @@ -0,0 +1,410 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.commons.math3.util.FastMath; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.VoidFunction; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +@Deprecated +public class Word2VecPerformerVoid implements VoidFunction, AtomicLong>> { + + + public final static String NAME_SPACE = "org.deeplearning4j.scaleout.perform.models.word2vec"; + public final static String VECTOR_LENGTH = NAME_SPACE + ".length"; + public final static String ADAGRAD = NAME_SPACE + ".adagrad"; + public final static String NEGATIVE = NAME_SPACE + ".negative"; + public final static String NUM_WORDS = NAME_SPACE + ".numwords"; + public final static String TABLE = NAME_SPACE + ".table"; + public final static String WINDOW = NAME_SPACE + ".window"; + public final static String ALPHA = NAME_SPACE + ".alpha"; + public final static String MIN_ALPHA = NAME_SPACE + ".minalpha"; + public final static String ITERATIONS = NAME_SPACE + ".iterations"; + + private static double MAX_EXP = 6; + private boolean useAdaGrad = false; + private double negative = 5; + private int numWords = 1; + private INDArray table; + private int window = 5; + private AtomicLong nextRandom = new AtomicLong(5); + private double alpha = 0.025; + private double minAlpha = 1e-2; + private int totalWords = 1; + private int iterations = 5; + private static transient final Logger log = LoggerFactory.getLogger(Word2VecPerformerVoid.class); + private int lastChecked = 0; + private Broadcast wordCount; + private InMemoryLookupTable weights; + private double[] expTable = new double[1000]; + private int vectorLength; + + + public Word2VecPerformerVoid(SparkConf sc, Broadcast wordCount, InMemoryLookupTable weights) { + this.weights = weights; + this.wordCount = wordCount; + setup(sc); + } + + public void setup(SparkConf conf) { + useAdaGrad = conf.getBoolean(ADAGRAD, false); + negative = conf.getDouble(NEGATIVE, 5); + numWords = conf.getInt(NUM_WORDS, 1); + window = conf.getInt(WINDOW, 5); + alpha = conf.getDouble(ALPHA, 0.025f); + minAlpha = conf.getDouble(MIN_ALPHA, 1e-2f); + totalWords = conf.getInt(NUM_WORDS, 1); + iterations = conf.getInt(ITERATIONS, 5); + vectorLength = conf.getInt(VECTOR_LENGTH, 100); + + initExpTable(); + + if (negative > 0 && conf.contains(TABLE)) { + ByteArrayInputStream bis = new ByteArrayInputStream(conf.get(TABLE).getBytes()); + DataInputStream dis = new DataInputStream(bis); + table = Nd4j.read(dis); + } + } + + + public int getVectorLength() { + return vectorLength; + } + + public void setVectorLength(int vectorLength) { + this.vectorLength = vectorLength; + } + + public double[] getExpTable() { + return expTable; + } + + public void setExpTable(double[] expTable) { + this.expTable = expTable; + } + + public InMemoryLookupTable getWeights() { + return weights; + } + + public void setWeights(InMemoryLookupTable weights) { + this.weights = weights; + } + + public Broadcast getWordCount() { + return wordCount; + } + + public void setWordCount(Broadcast wordCount) { + this.wordCount = wordCount; + } + + public int getLastChecked() { + return lastChecked; + } + + public void setLastChecked(int lastChecked) { + this.lastChecked = lastChecked; + } + + public static Logger getLog() { + return log; + } + + public int getIterations() { + return iterations; + } + + public void setIterations(int iterations) { + this.iterations = iterations; + } + + public int getTotalWords() { + return totalWords; + } + + public void setTotalWords(int totalWords) { + this.totalWords = totalWords; + } + + public double getMinAlpha() { + return minAlpha; + } + + public void setMinAlpha(double minAlpha) { + this.minAlpha = minAlpha; + } + + public double getAlpha() { + return alpha; + } + + public void setAlpha(double alpha) { + this.alpha = alpha; + } + + public AtomicLong getNextRandom() { + return nextRandom; + } + + public void setNextRandom(AtomicLong nextRandom) { + this.nextRandom = nextRandom; + } + + public int getWindow() { + return window; + } + + public void setWindow(int window) { + this.window = window; + } + + public INDArray getTable() { + return table; + } + + public void setTable(INDArray table) { + this.table = table; + } + + public int getNumWords() { + return numWords; + } + + public void setNumWords(int numWords) { + this.numWords = numWords; + } + + public double getNegative() { + return negative; + } + + public void setNegative(double negative) { + this.negative = negative; + } + + public boolean isUseAdaGrad() { + return useAdaGrad; + } + + public void setUseAdaGrad(boolean useAdaGrad) { + this.useAdaGrad = useAdaGrad; + } + + public static double getMAX_EXP() { + return MAX_EXP; + } + + public static void setMAX_EXP(double MAX_EXP) { + Word2VecPerformerVoid.MAX_EXP = MAX_EXP; + } + + /** + * Train on a list of vocab words + * @param sentence the list of vocab words to train on + */ + public void trainSentence(final List sentence, double alpha) { + if (sentence != null && !sentence.isEmpty()) { + for (int i = 0; i < sentence.size(); i++) { + if (!sentence.get(i).getWord().endsWith("STOP")) { + nextRandom.set(nextRandom.get() * 25214903917L + 11); + skipGram(i, sentence, (int) nextRandom.get() % window, alpha); + } + } + } + + } + + + /** + * Train via skip gram + * @param i + * @param sentence + */ + public void skipGram(int i, List sentence, int b, double alpha) { + + final VocabWord word = sentence.get(i); + if (word != null && !sentence.isEmpty()) { + int end = window * 2 + 1 - b; + for (int a = b; a < end; a++) { + if (a != window) { + int c = i - window + a; + if (c >= 0 && c < sentence.size()) { + VocabWord lastWord = sentence.get(c); + iterateSample(word, lastWord, alpha); + } + } + } + } + } + + + + /** + * Iterate on the given 2 vocab words + * + * @param w1 the first word to iterate on + * @param w2 the second word to iterate on + */ + public void iterateSample(VocabWord w1, VocabWord w2, double alpha) { + if (w2 == null || w2.getIndex() < 0) + return; + + //current word vector + INDArray l1 = weights.vector(w2.getWord()); + + + //error for current word and context + INDArray neu1e = Nd4j.create(vectorLength); + + for (int i = 0; i < w1.getCodeLength(); i++) { + int code = w1.getCodes().get(i); + int point = w1.getPoints().get(i); + + INDArray syn1 = weights.getSyn1().slice(point); + + double dot = Nd4j.getBlasWrapper().dot(l1, syn1); + + if (dot >= -MAX_EXP && dot < MAX_EXP) { + + int idx = (int) ((dot + MAX_EXP) * ((double) expTable.length / MAX_EXP / 2.0)); + if (idx >= expTable.length) + continue; + + //score + double f = expTable[idx]; + //gradient + double g = (1 - code - f) * (useAdaGrad ? w1.getGradient(i, alpha, this.alpha) : alpha); + + + if (neu1e.data().dataType() == DataType.DOUBLE) { + Nd4j.getBlasWrapper().axpy(g, syn1, neu1e); + Nd4j.getBlasWrapper().axpy(g, l1, syn1); + } else { + Nd4j.getBlasWrapper().axpy((float) g, syn1, neu1e); + Nd4j.getBlasWrapper().axpy((float) g, l1, syn1); + } + } + + + } + + + //negative sampling + if (negative > 0) { + int target = w1.getIndex(); + int label; + INDArray syn1Neg = weights.getSyn1Neg().slice(target); + + for (int d = 0; d < negative + 1; d++) { + if (d == 0) { + + label = 1; + } else { + nextRandom.set(nextRandom.get() * 25214903917L + 11); + target = table.getInt((int) (nextRandom.get() >> 16) % (int) table.length()); + if (target == 0) + target = (int) nextRandom.get() % (numWords - 1) + 1; + if (target == w1.getIndex()) + continue; + label = 0; + } + + double f = Nd4j.getBlasWrapper().dot(l1, syn1Neg); + double g; + if (f > MAX_EXP) + g = useAdaGrad ? w1.getGradient(target, (label - 1), this.alpha) : (label - 1) * alpha; + else if (f < -MAX_EXP) + g = label * (useAdaGrad ? w1.getGradient(target, alpha, this.alpha) : alpha); + else + g = useAdaGrad ? w1 + .getGradient(target, + label - expTable[(int) ((f + MAX_EXP) + * (expTable.length / MAX_EXP / 2))], + this.alpha) + : (label - expTable[(int) ((f + MAX_EXP) * (expTable.length / MAX_EXP / 2))]) + * alpha; + if (syn1Neg.data().dataType() == DataType.DOUBLE) + Nd4j.getBlasWrapper().axpy(g, neu1e, l1); + else + Nd4j.getBlasWrapper().axpy((float) g, neu1e, l1); + + if (syn1Neg.data().dataType() == DataType.DOUBLE) + Nd4j.getBlasWrapper().axpy(g, syn1Neg, l1); + else + Nd4j.getBlasWrapper().axpy((float) g, syn1Neg, l1); + } + } + + if (neu1e.data().dataType() == DataType.DOUBLE) + Nd4j.getBlasWrapper().axpy(1.0, neu1e, l1); + + else + Nd4j.getBlasWrapper().axpy(1.0f, neu1e, l1); + + } + + private void initExpTable() { + for (int i = 0; i < expTable.length; i++) { + double tmp = FastMath.exp((i / (double) expTable.length * 2 - 1) * MAX_EXP); + expTable[i] = tmp / (tmp + 1.0); + } + } + + + @Override + public void call(Pair, AtomicLong> pair) throws Exception { + double numWordsSoFar = wordCount.getValue().doubleValue(); + + List sentence = pair.getFirst(); + double alpha2 = Math.max(minAlpha, alpha * (1 - (1.0 * numWordsSoFar / (double) totalWords))); + int totalNewWords = 0; + trainSentence(sentence, alpha2); + totalNewWords += sentence.size(); + + + + double newWords = totalNewWords + numWordsSoFar; + double diff = Math.abs(newWords - lastChecked); + if (diff >= 10000) { + lastChecked = (int) newWords; + log.info("Words so far " + newWords + " out of " + totalWords); + } + + pair.getSecond().getAndAdd((long) totalNewWords); + } + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecSetup.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecSetup.java new file mode 100644 index 000000000..677fb3738 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecSetup.java @@ -0,0 +1,42 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.spark.api.java.function.Function; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.word2vec.VocabWord; +import scala.Tuple2; + +import java.util.List; + +@Deprecated +public class Word2VecSetup implements Function, Long>, Word2VecFuncCall> { + private Broadcast param; + + public Word2VecSetup(Broadcast param) { + this.param = param; + } + + @Override + public Word2VecFuncCall call(Tuple2, Long> listLongTuple2) throws Exception { + return new Word2VecFuncCall(param, listLongTuple2._2(), listLongTuple2._1()); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecVariables.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecVariables.java new file mode 100644 index 000000000..6adcc7d1f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecVariables.java @@ -0,0 +1,98 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import org.apache.spark.SparkConf; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author jeffreytang + */ +@Deprecated +public class Word2VecVariables { + + public final static String NAME_SPACE = "org.deeplearning4j.scaleout.perform.models.word2vec"; + public final static String VECTOR_LENGTH = NAME_SPACE + ".length"; + public final static String ADAGRAD = NAME_SPACE + ".adagrad"; + public final static String NEGATIVE = NAME_SPACE + ".negative"; + public final static String NUM_WORDS = NAME_SPACE + ".numwords"; + public final static String TABLE = NAME_SPACE + ".table"; + public final static String WINDOW = NAME_SPACE + ".window"; + public final static String ALPHA = NAME_SPACE + ".alpha"; + public final static String MIN_ALPHA = NAME_SPACE + ".minalpha"; + public final static String ITERATIONS = NAME_SPACE + ".iterations"; + public final static String N_GRAMS = NAME_SPACE + ".ngrams"; + public final static String TOKENIZER = NAME_SPACE + ".tokenizer"; + public final static String TOKEN_PREPROCESSOR = NAME_SPACE + ".preprocessor"; + public final static String REMOVE_STOPWORDS = NAME_SPACE + ".removestopwords"; + public final static String SEED = NAME_SPACE + ".SEED"; + + public final static Map defaultVals = new HashMap() { + { + put(VECTOR_LENGTH, 100); + put(ADAGRAD, false); + put(NEGATIVE, 5); + put(NUM_WORDS, 1); + // TABLE would be a string of byte of the ndarray used for -ve sampling + put(WINDOW, 5); + put(ALPHA, 0.025); + put(MIN_ALPHA, 1e-2); + put(ITERATIONS, 1); + put(N_GRAMS, 1); + put(TOKENIZER, "org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory"); + put(TOKEN_PREPROCESSOR, "org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor"); + put(REMOVE_STOPWORDS, false); + put(SEED, 42L); + } + }; + + private Word2VecVariables() {} + + @SuppressWarnings("unchecked") + public static T getDefault(String variableName) { + return (T) defaultVals.get(variableName); + } + + @SuppressWarnings("unchecked") + public static T assignVar(String variableName, SparkConf conf, Class clazz) throws Exception { + Object ret; + if (clazz.equals(Integer.class)) { + ret = conf.getInt(variableName, (Integer) getDefault(variableName)); + + } else if (clazz.equals(Double.class)) { + ret = conf.getDouble(variableName, (Double) getDefault(variableName)); + + } else if (clazz.equals(Boolean.class)) { + ret = conf.getBoolean(variableName, (Boolean) getDefault(variableName)); + + } else if (clazz.equals(String.class)) { + ret = conf.get(variableName, (String) getDefault(variableName)); + + } else if (clazz.equals(Long.class)) { + ret = conf.getLong(variableName, (Long) getDefault(variableName)); + } else { + throw new Exception("Variable Type not supported. Only boolean, int, double and String supported."); + } + return (T) ret; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/MaxPerPartitionAccumulator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/MaxPerPartitionAccumulator.java new file mode 100644 index 000000000..36af95bd2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/MaxPerPartitionAccumulator.java @@ -0,0 +1,47 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.accumulators; + +import org.apache.spark.util.CollectionAccumulator; +import org.nd4j.common.primitives.Counter; + +/** + * @author jeffreytang + */ +public class MaxPerPartitionAccumulator extends CollectionAccumulator> { + + public Counter addInPlace(Counter c1, Counter c2) { + c1.incrementAll(c2); + return c1; + } + + public Counter zero(Counter initialCounter) { + return new Counter<>(); + } + + public Counter addAccumulator(Counter c1, Counter c2) { + if (c1 == null) { + return new Counter<>(); + } + addInPlace(c1, c2); + return c1; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/WordFreqAccumulator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/WordFreqAccumulator.java new file mode 100644 index 000000000..6cd1e62cd --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/WordFreqAccumulator.java @@ -0,0 +1,47 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.accumulators; + +import org.apache.spark.util.CollectionAccumulator; +import org.nd4j.common.primitives.Counter; + +/** + * @author jeffreytang + */ +public class WordFreqAccumulator extends CollectionAccumulator> { + + public Counter addInPlace(Counter c1, Counter c2) { + c1.incrementAll(c2); + return c1; + } + + public Counter zero(Counter initialCounter) { + return new Counter<>(); + } + + public Counter addAccumulator(Counter c1, Counter c2) { + if (c1 == null) { + return new Counter<>(); + } + addInPlace(c1, c2); + return c1; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/CountCumSum.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/CountCumSum.java new file mode 100644 index 000000000..4b757ec5f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/CountCumSum.java @@ -0,0 +1,97 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.util.CollectionAccumulator; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.spark.text.accumulators.MaxPerPartitionAccumulator; +import org.nd4j.common.primitives.Counter; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author jeffreytang + */ +@SuppressWarnings("unchecked") +public class CountCumSum { + + // Starting variables + private JavaSparkContext sc; + private JavaRDD sentenceCountRDD; + + // Variables to fill in as we go + private JavaRDD foldWithinPartitionRDD; + private Broadcast> broadcastedMaxPerPartitionCounter; + private JavaRDD cumSumRDD; + + // Constructor + public CountCumSum(JavaRDD sentenceCountRDD) { + this.sentenceCountRDD = sentenceCountRDD; + this.sc = new JavaSparkContext(sentenceCountRDD.context()); + } + + // Getter + public JavaRDD getCumSumRDD() { + if (cumSumRDD != null) { + return cumSumRDD; + } else { + throw new IllegalAccessError("Cumulative Sum list not defined. Call buildCumSum() first."); + } + } + + // For each equivalent for partitions + public void actionForMapPartition(JavaRDD rdd) { + // Action to fill the accumulator + rdd.foreachPartition(new MapPerPartitionVoidFunction()); + } + + // Do cum sum within the partition + public void cumSumWithinPartition() { + + // Accumulator to get the max of the cumulative sum in each partition + final CollectionAccumulator> maxPerPartitionAcc = + sc.sc().collectionAccumulator("MaxPerPartitionAccumulator"); + // Partition mapping to fold within partition + foldWithinPartitionRDD = sentenceCountRDD + .mapPartitionsWithIndex(new FoldWithinPartitionFunction(maxPerPartitionAcc), true).cache(); + actionForMapPartition(foldWithinPartitionRDD); + + // Broadcast the counter (partition index : sum of count) to all workers + broadcastedMaxPerPartitionCounter = sc.broadcast(maxPerPartitionAcc.value().get(0)); + } + + public void cumSumBetweenPartition() { + + cumSumRDD = foldWithinPartitionRDD + .mapPartitionsWithIndex(new FoldBetweenPartitionFunction(broadcastedMaxPerPartitionCounter), + true) + .setName("cumSumRDD").cache(); + foldWithinPartitionRDD.unpersist(); + } + + public JavaRDD buildCumSum() { + cumSumWithinPartition(); + cumSumBetweenPartition(); + return getCumSumRDD(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldBetweenPartitionFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldBetweenPartitionFunction.java new file mode 100644 index 000000000..f332c1f92 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldBetweenPartitionFunction.java @@ -0,0 +1,62 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.api.java.function.Function2; +import org.apache.spark.broadcast.Broadcast; +import org.nd4j.common.primitives.Counter; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author jeffreytang + */ +public class FoldBetweenPartitionFunction implements Function2, Iterator> { + private Broadcast> broadcastedMaxPerPartitionCounter; + + public FoldBetweenPartitionFunction(Broadcast> broadcastedMaxPerPartitionCounter) { + this.broadcastedMaxPerPartitionCounter = broadcastedMaxPerPartitionCounter; + } + + @Override + public Iterator call(Integer ind, Iterator partition) throws Exception { + int sumToAdd = 0; + Counter maxPerPartitionCounterInScope = broadcastedMaxPerPartitionCounter.value(); + + // Add the sum of counts of all the partition with an index lower than the current one + if (ind != 0) { + for (int i = 0; i < ind; i++) { + sumToAdd += maxPerPartitionCounterInScope.getCount(i); + } + } + + // Add the sum of counts to each element of the partition + List itemsAddedToList = new ArrayList<>(); + while (partition.hasNext()) { + itemsAddedToList.add(partition.next().get() + sumToAdd); + } + + return itemsAddedToList.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldWithinPartitionFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldWithinPartitionFunction.java new file mode 100644 index 000000000..38910c623 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldWithinPartitionFunction.java @@ -0,0 +1,74 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.util.CollectionAccumulator; +import org.apache.spark.api.java.function.Function2; +import org.nd4j.common.primitives.Counter; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author jeffreytang + */ +public class FoldWithinPartitionFunction implements Function2, Iterator> { + + public FoldWithinPartitionFunction(CollectionAccumulator> maxPartitionAcc) { + this.maxPerPartitionAcc = maxPartitionAcc; + } + + private CollectionAccumulator> maxPerPartitionAcc; + + + @Override + public Iterator call(Integer ind, Iterator partition) throws Exception { + + List foldedItemList = new ArrayList() { + { + add(new AtomicLong(0L)); + } + }; + + // Recurrent state implementation of cum sum + int foldedItemListSize = 1; + while (partition.hasNext()) { + long curPartitionItem = partition.next().get(); + int lastFoldedIndex = foldedItemListSize - 1; + long lastFoldedItem = foldedItemList.get(lastFoldedIndex).get(); + AtomicLong sumLastCurrent = new AtomicLong(curPartitionItem + lastFoldedItem); + + foldedItemList.set(lastFoldedIndex, sumLastCurrent); + foldedItemList.add(sumLastCurrent); + foldedItemListSize += 1; + } + + // Update Accumulator + long maxFoldedItem = foldedItemList.remove(foldedItemListSize - 1).get(); + Counter partitionIndex2maxItemCounter = new Counter<>(); + partitionIndex2maxItemCounter.incrementCount(ind, maxFoldedItem); + maxPerPartitionAcc.add(partitionIndex2maxItemCounter); + + return foldedItemList.iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/GetSentenceCountFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/GetSentenceCountFunction.java new file mode 100644 index 000000000..e6b2c239d --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/GetSentenceCountFunction.java @@ -0,0 +1,38 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.api.java.function.Function; +import org.nd4j.common.primitives.Pair; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author jeffreytang + */ +public class GetSentenceCountFunction implements Function, AtomicLong>, AtomicLong> { + + @Override + public AtomicLong call(Pair, AtomicLong> pair) throws Exception { + return pair.getSecond(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/MapPerPartitionVoidFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/MapPerPartitionVoidFunction.java new file mode 100644 index 000000000..87eb55ada --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/MapPerPartitionVoidFunction.java @@ -0,0 +1,35 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.api.java.function.VoidFunction; + +import java.util.Iterator; + +/** + * @author jeffreytang + */ +public class MapPerPartitionVoidFunction implements VoidFunction> { + + @Override + public void call(Iterator integerIterator) throws Exception {} +} + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/ReduceSentenceCount.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/ReduceSentenceCount.java new file mode 100644 index 000000000..34dbb5538 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/ReduceSentenceCount.java @@ -0,0 +1,34 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.api.java.function.Function2; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author jeffreytang + */ +public class ReduceSentenceCount implements Function2 { + public AtomicLong call(AtomicLong a, AtomicLong b) { + return new AtomicLong(a.get() + b.get()); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TextPipeline.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TextPipeline.java new file mode 100644 index 000000000..5fb7b0fbc --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TextPipeline.java @@ -0,0 +1,262 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.util.CollectionAccumulator; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration; +import org.deeplearning4j.models.word2vec.Huffman; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; +import org.deeplearning4j.spark.text.accumulators.WordFreqAccumulator; +import org.nd4j.common.primitives.AtomicDouble; +import org.nd4j.common.primitives.Counter; +import org.nd4j.common.primitives.Pair; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicLong; + +@SuppressWarnings("unchecked") +public class TextPipeline { + //params + private JavaRDD corpusRDD; + private int numWords; + private int nGrams; + private String tokenizer; + private String tokenizerPreprocessor; + private List stopWords = new ArrayList<>(); + //Setup + private JavaSparkContext sc; + private CollectionAccumulator> wordFreqAcc; + private Broadcast> stopWordBroadCast; + // Return values + private JavaRDD, AtomicLong>> sentenceWordsCountRDD; + private VocabCache vocabCache = new AbstractCache<>(); + private Broadcast> vocabCacheBroadcast; + private JavaRDD> vocabWordListRDD; + private JavaRDD sentenceCountRDD; + private long totalWordCount; + private boolean useUnk; + private VectorsConfiguration configuration; + + // Empty Constructor + public TextPipeline() {} + + // Constructor + public TextPipeline(JavaRDD corpusRDD, Broadcast> broadcasTokenizerVarMap) + throws Exception { + setRDDVarMap(corpusRDD, broadcasTokenizerVarMap); + // Setup all Spark variables + setup(); + } + + public void setRDDVarMap(JavaRDD corpusRDD, Broadcast> broadcasTokenizerVarMap) { + Map tokenizerVarMap = broadcasTokenizerVarMap.getValue(); + this.corpusRDD = corpusRDD; + this.numWords = (int) tokenizerVarMap.get("numWords"); + // TokenizerFunction Settings + this.nGrams = (int) tokenizerVarMap.get("nGrams"); + this.tokenizer = (String) tokenizerVarMap.get("tokenizer"); + this.tokenizerPreprocessor = (String) tokenizerVarMap.get("tokenPreprocessor"); + this.useUnk = (boolean) tokenizerVarMap.get("useUnk"); + this.configuration = (VectorsConfiguration) tokenizerVarMap.get("vectorsConfiguration"); + // Remove Stop words + // if ((boolean) tokenizerVarMap.get("removeStop")) { + stopWords = (List) tokenizerVarMap.get("stopWords"); + // } + } + + private void setup() { + // Set up accumulators and broadcast stopwords + this.sc = new JavaSparkContext(corpusRDD.context()); + this.wordFreqAcc = sc.sc().collectionAccumulator("WordFreqAccumulator"); //(new Counter(), new WordFreqAccumulator()); + this.stopWordBroadCast = sc.broadcast(stopWords); + } + + public JavaRDD> tokenize() { + if (corpusRDD == null) { + throw new IllegalStateException("corpusRDD not assigned. Define TextPipeline with corpusRDD assigned."); + } + return corpusRDD.map(new TokenizerFunction(tokenizer, tokenizerPreprocessor, nGrams)); + } + + public JavaRDD, AtomicLong>> updateAndReturnAccumulatorVal(JavaRDD> tokenizedRDD) { + // Update the 2 accumulators + UpdateWordFreqAccumulatorFunction accumulatorClassFunction = + new UpdateWordFreqAccumulatorFunction(stopWordBroadCast, wordFreqAcc); + JavaRDD, AtomicLong>> sentenceWordsCountRDD = tokenizedRDD.map(accumulatorClassFunction); + + // Loop through each element to update accumulator. Count does the same job (verified). + sentenceWordsCountRDD.count(); + + return sentenceWordsCountRDD; + } + + private String filterMinWord(String stringToken, double tokenCount) { + return (tokenCount < numWords) ? configuration.getUNK() : stringToken; + } + + private void addTokenToVocabCache(String stringToken, Float tokenCount) { + // Making string token into actual token if not already an actual token (vocabWord) + VocabWord actualToken; + if (vocabCache.hasToken(stringToken)) { + actualToken = vocabCache.tokenFor(stringToken); + actualToken.increaseElementFrequency(tokenCount.intValue()); + } else { + actualToken = new VocabWord(tokenCount, stringToken); + } + + // Set the index of the actual token (vocabWord) + // Put vocabWord into vocabs in InMemoryVocabCache + boolean vocabContainsWord = vocabCache.containsWord(stringToken); + if (!vocabContainsWord) { + int idx = vocabCache.numWords(); + + vocabCache.addToken(actualToken); + actualToken.setIndex(idx); + vocabCache.putVocabWord(stringToken); + } + } + + public void filterMinWordAddVocab(Counter wordFreq) { + + if (wordFreq.isEmpty()) { + throw new IllegalStateException( + "IllegalStateException: wordFreqCounter has nothing. Check accumulator updating"); + } + + for (Entry entry : wordFreq.entrySet()) { + String stringToken = entry.getKey(); + double tokenCount = entry.getValue().doubleValue(); + + // Turn words below min count to UNK + stringToken = filterMinWord(stringToken, tokenCount); + if (!useUnk && stringToken.equals("UNK")) { + // Turn tokens to vocab and add to vocab cache + } else + addTokenToVocabCache(stringToken, entry.getValue().floatValue()); + } + } + + public void buildVocabCache() { + + // Tokenize + JavaRDD> tokenizedRDD = tokenize(); + + // Update accumulator values and map to an RDD of sentence counts + sentenceWordsCountRDD = updateAndReturnAccumulatorVal(tokenizedRDD).cache(); + + // Get value from accumulator + Counter wordFreqCounter = wordFreqAcc.value().get(0); + + // Filter out low count words and add to vocab cache object and feed into LookupCache + filterMinWordAddVocab(wordFreqCounter); + + // huffman tree should be built BEFORE vocab broadcast + Huffman huffman = new Huffman(vocabCache.vocabWords()); + huffman.build(); + huffman.applyIndexes(vocabCache); + + // At this point the vocab cache is built. Broadcast vocab cache + vocabCacheBroadcast = sc.broadcast(vocabCache); + + } + + public void buildVocabWordListRDD() { + + if (sentenceWordsCountRDD == null) + throw new IllegalStateException("SentenceWordCountRDD must be defined first. Run buildLookupCache first."); + + vocabWordListRDD = sentenceWordsCountRDD.map(new WordsListToVocabWordsFunction(vocabCacheBroadcast)) + .setName("vocabWordListRDD").cache(); + sentenceCountRDD = + sentenceWordsCountRDD.map(new GetSentenceCountFunction()).setName("sentenceCountRDD").cache(); + // Actions to fill vocabWordListRDD and sentenceCountRDD + vocabWordListRDD.count(); + totalWordCount = sentenceCountRDD.reduce(new ReduceSentenceCount()).get(); + + // Release sentenceWordsCountRDD from cache + sentenceWordsCountRDD.unpersist(); + } + + // Getters + public CollectionAccumulator> getWordFreqAcc() { + if (wordFreqAcc != null) { + return wordFreqAcc; + } else { + throw new IllegalStateException("IllegalStateException: wordFreqAcc not set at TextPipline."); + } + } + + public Broadcast> getBroadCastVocabCache() throws IllegalStateException { + if (vocabCache.numWords() > 0) { + return vocabCacheBroadcast; + } else { + throw new IllegalStateException("IllegalStateException: VocabCache not set at TextPipline."); + } + } + + public VocabCache getVocabCache() throws IllegalStateException { + if (vocabCache != null && vocabCache.numWords() > 0) { + return vocabCache; + } else { + throw new IllegalStateException("IllegalStateException: VocabCache not set at TextPipline."); + } + } + + public JavaRDD, AtomicLong>> getSentenceWordsCountRDD() { + if (sentenceWordsCountRDD != null) { + return sentenceWordsCountRDD; + } else { + throw new IllegalStateException("IllegalStateException: sentenceWordsCountRDD not set at TextPipline."); + } + } + + public JavaRDD> getVocabWordListRDD() throws IllegalStateException { + if (vocabWordListRDD != null) { + return vocabWordListRDD; + } else { + throw new IllegalStateException("IllegalStateException: vocabWordListRDD not set at TextPipline."); + } + } + + public JavaRDD getSentenceCountRDD() throws IllegalStateException { + if (sentenceCountRDD != null) { + return sentenceCountRDD; + } else { + throw new IllegalStateException("IllegalStateException: sentenceCountRDD not set at TextPipline."); + } + } + + public Long getTotalWordCount() { + if (totalWordCount != 0L) { + return totalWordCount; + } else { + throw new IllegalStateException("IllegalStateException: totalWordCount not set at TextPipline."); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TokenizerFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TokenizerFunction.java new file mode 100644 index 000000000..75b855695 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TokenizerFunction.java @@ -0,0 +1,79 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.api.java.function.Function; +import org.deeplearning4j.common.config.DL4JClassLoading; +import org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess; +import org.deeplearning4j.text.tokenization.tokenizerfactory.NGramTokenizerFactory; +import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; + +import java.util.Collections; +import java.util.List; + +@SuppressWarnings("unchecked") +@Slf4j +public class TokenizerFunction implements Function> { + private String tokenizerFactoryClazz; + private String tokenizerPreprocessorClazz; + private transient TokenizerFactory tokenizerFactory; + private int nGrams = 1; + + public TokenizerFunction(String tokenizer, String tokenizerPreprocessor, int nGrams) { + this.tokenizerFactoryClazz = tokenizer; + this.tokenizerPreprocessorClazz = tokenizerPreprocessor; + this.nGrams = nGrams; + } + + @Override + public List call(String str) { + if (tokenizerFactory == null) { + tokenizerFactory = getTokenizerFactory(); + } + + if (str.isEmpty()) { + return Collections.singletonList(""); + } + + return tokenizerFactory.create(str).getTokens(); + } + + private TokenizerFactory getTokenizerFactory() { + TokenPreProcess tokenPreProcessInst = null; + + if (StringUtils.isNotEmpty(tokenizerPreprocessorClazz)) { + tokenPreProcessInst = DL4JClassLoading.createNewInstance(tokenizerPreprocessorClazz); + } + + tokenizerFactory = DL4JClassLoading.createNewInstance(tokenizerFactoryClazz); + + if (tokenPreProcessInst != null) + tokenizerFactory.setTokenPreProcessor(tokenPreProcessInst); + if (nGrams > 1) { + tokenizerFactory = new NGramTokenizerFactory(tokenizerFactory, nGrams, nGrams); + } + + return tokenizerFactory; + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/UpdateWordFreqAccumulatorFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/UpdateWordFreqAccumulatorFunction.java new file mode 100644 index 000000000..312677c98 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/UpdateWordFreqAccumulatorFunction.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.util.CollectionAccumulator; +import org.apache.spark.api.java.function.Function; +import org.apache.spark.broadcast.Broadcast; +import org.nd4j.common.primitives.Counter; +import org.nd4j.common.primitives.Pair; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author Jeffrey Tang + */ +public class UpdateWordFreqAccumulatorFunction implements Function, Pair, AtomicLong>> { + + private Broadcast> stopWords; + private CollectionAccumulator> wordFreqAcc; + + public UpdateWordFreqAccumulatorFunction(Broadcast> stopWords, + CollectionAccumulator> wordFreqAcc) { + this.wordFreqAcc = wordFreqAcc; + this.stopWords = stopWords; + } + + // Function to add to word freq counter and total count of words + @Override + public Pair, AtomicLong> call(List lstOfWords) throws Exception { + List stops = stopWords.getValue(); + Counter counter = new Counter<>(); + + for (String w : lstOfWords) { + if (w.isEmpty()) + continue; + + if (!stops.isEmpty()) { + if (stops.contains(w)) { + counter.incrementCount("STOP", 1.0f); + } else { + counter.incrementCount(w, 1.0f); + } + } else { + counter.incrementCount(w, 1.0f); + } + } + wordFreqAcc.add(counter); + AtomicLong lstOfWordsSize = new AtomicLong(lstOfWords.size()); + return new Pair<>(lstOfWords, lstOfWordsSize); + } +} + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/WordsListToVocabWordsFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/WordsListToVocabWordsFunction.java new file mode 100644 index 000000000..c20c498e0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/WordsListToVocabWordsFunction.java @@ -0,0 +1,63 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text.functions; + +import org.apache.spark.api.java.function.Function; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.nd4j.common.primitives.Pair; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author jeffreytang + */ +public class WordsListToVocabWordsFunction implements Function, AtomicLong>, List> { + + Broadcast> vocabCacheBroadcast; + + public WordsListToVocabWordsFunction(Broadcast> vocabCacheBroadcast) { + this.vocabCacheBroadcast = vocabCacheBroadcast; + } + + @Override + public List call(Pair, AtomicLong> pair) throws Exception { + List wordsList = pair.getFirst(); + List vocabWordsList = new ArrayList<>(); + VocabCache vocabCache = vocabCacheBroadcast.getValue(); + for (String s : wordsList) { + if (vocabCache.containsWord(s)) { + VocabWord word = vocabCache.wordFor(s); + + vocabWordsList.add(word); + } else if (vocabCache.containsWord("UNK")) { + VocabWord word = vocabCache.wordFor("UNK"); + + vocabWordsList.add(word); + } + } + return vocabWordsList; + } +} + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecTest.java new file mode 100644 index 000000000..4859b91a6 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecTest.java @@ -0,0 +1,221 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.models.embeddings.word2vec; + +import com.sun.jna.Platform; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; + + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.common.io.ClassPathResource; +import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; +import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; +import org.deeplearning4j.models.embeddings.reader.impl.FlatModelUtils; +import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; +import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.LowCasePreProcessor; +import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; +import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.File; +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.*; + +//@Ignore +public class Word2VecTest { + + @TempDir + public File testDir; + + @Test + public void testConcepts() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + // These are all default values for word2vec + SparkConf sparkConf = new SparkConf().setMaster("local[8]") + .set("spark.driver.host", "localhost") + .setAppName("sparktest"); + + // Set SparkContext + JavaSparkContext sc = new JavaSparkContext(sparkConf); + + // Path of data part-00000 + String dataPath = new ClassPathResource("big/raw_sentences.txt").getFile().getAbsolutePath(); + // dataPath = "/ext/Temp/part-00000"; + // String dataPath = new ClassPathResource("spark_word2vec_test.txt").getFile().getAbsolutePath(); + + // Read in data + JavaRDD corpus = sc.textFile(dataPath); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + Word2Vec word2Vec = new Word2Vec.Builder().setNGrams(1) + // .setTokenizer("org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory") + // .setTokenPreprocessor("org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor") + // .setRemoveStop(false) + .tokenizerFactory(t).seed(42L).negative(10).useAdaGrad(false).layerSize(150).windowSize(5) + .learningRate(0.025).minLearningRate(0.0001).iterations(1).batchSize(100).minWordFrequency(5) + .stopWords(Arrays.asList("three")).useUnknown(true).build(); + + word2Vec.train(corpus); + + //word2Vec.setModelUtils(new FlatModelUtils()); + + System.out.println("UNK: " + word2Vec.getWordVectorMatrix("UNK")); + + InMemoryLookupTable table = (InMemoryLookupTable) word2Vec.lookupTable(); + + double sim = word2Vec.similarity("day", "night"); + System.out.println("day/night similarity: " + sim); + /* + System.out.println("Hornjo: " + word2Vec.getWordVectorMatrix("hornjoserbsce")); + System.out.println("carro: " + word2Vec.getWordVectorMatrix("carro")); + + Collection portu = word2Vec.wordsNearest("carro", 10); + printWords("carro", portu, word2Vec); + + portu = word2Vec.wordsNearest("davi", 10); + printWords("davi", portu, word2Vec); + + System.out.println("---------------------------------------"); + */ + + Collection words = word2Vec.wordsNearest("day", 10); + printWords("day", words, word2Vec); + + assertTrue(words.contains("night")); + assertTrue(words.contains("week")); + assertTrue(words.contains("year")); + + sim = word2Vec.similarity("two", "four"); + System.out.println("two/four similarity: " + sim); + + words = word2Vec.wordsNearest("two", 10); + printWords("two", words, word2Vec); + + // three should be absent due to stopWords + assertFalse(words.contains("three")); + + assertTrue(words.contains("five")); + assertTrue(words.contains("four")); + + sc.stop(); + + + // test serialization + File tempFile = new File(testDir, "temp" + System.currentTimeMillis() + ".tmp"); + + int idx1 = word2Vec.vocab().wordFor("day").getIndex(); + + INDArray array1 = word2Vec.getWordVectorMatrix("day").dup(); + + VocabWord word1 = word2Vec.vocab().elementAtIndex(0); + + WordVectorSerializer.writeWordVectors(word2Vec.getLookupTable(), tempFile); + + WordVectors vectors = WordVectorSerializer.loadTxtVectors(tempFile); + + VocabWord word2 = ((VocabCache) vectors.vocab()).elementAtIndex(0); + VocabWord wordIT = ((VocabCache) vectors.vocab()).wordFor("it"); + int idx2 = vectors.vocab().wordFor("day").getIndex(); + + INDArray array2 = vectors.getWordVectorMatrix("day").dup(); + + System.out.println("word 'i': " + word2); + System.out.println("word 'it': " + wordIT); + + assertEquals(idx1, idx2); + assertEquals(word1, word2); + assertEquals(array1, array2); + } + + //@Ignore + @Test + public void testSparkW2VonBiggerCorpus() throws Exception { + SparkConf sparkConf = new SparkConf().setMaster("local[8]").setAppName("sparktest") + .set("spark.driver.host", "localhost") + .set("spark.driver.maxResultSize", "4g").set("spark.driver.memory", "8g") + .set("spark.executor.memory", "8g"); + + // Set SparkContext + JavaSparkContext sc = new JavaSparkContext(sparkConf); + + // Path of data part-00000 + //String dataPath = Resources.asFile("big/raw_sentences.txt").getAbsolutePath(); + // String dataPath = "/ext/Temp/SampleRussianCorpus.txt"; + String dataPath = new ClassPathResource("spark_word2vec_test.txt").getFile().getAbsolutePath(); + + // Read in data + JavaRDD corpus = sc.textFile(dataPath); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new LowCasePreProcessor()); + + Word2Vec word2Vec = new Word2Vec.Builder().setNGrams(1) + // .setTokenizer("org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory") + // .setTokenPreprocessor("org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor") + // .setRemoveStop(false) + .tokenizerFactory(t).seed(42L).negative(3).useAdaGrad(false).layerSize(100).windowSize(5) + .learningRate(0.025).minLearningRate(0.0001).iterations(1).batchSize(100).minWordFrequency(5) + .useUnknown(true).build(); + + word2Vec.train(corpus); + + + sc.stop(); + + WordVectorSerializer.writeWordVectors(word2Vec.getLookupTable(), "/ext/Temp/sparkRuModel.txt"); + } + + @Test + //@Ignore + public void testPortugeseW2V() throws Exception { + WordVectors word2Vec = WordVectorSerializer.loadTxtVectors(new File("/ext/Temp/para.txt")); + word2Vec.setModelUtils(new FlatModelUtils()); + + Collection portu = word2Vec.wordsNearest("carro", 10); + printWords("carro", portu, word2Vec); + + portu = word2Vec.wordsNearest("davi", 10); + printWords("davi", portu, word2Vec); + } + + private static void printWords(String target, Collection list, WordVectors vec) { + System.out.println("Words close to [" + target + "]:"); + for (String word : list) { + double sim = vec.similarity(target, word); + System.out.print("'" + word + "': [" + sim + "], "); + } + System.out.print("\n"); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java new file mode 100644 index 000000000..d998ddde4 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java @@ -0,0 +1,98 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.spark.models.embeddings.word2vec.Word2VecVariables; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import java.io.Serializable; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.Map; + +public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable { + protected transient JavaSparkContext sc; + + @Override + public long getTimeoutMilliseconds() { + return 120000L; + } + + @BeforeEach + public void before() throws Exception { + sc = getContext(); + } + + @AfterEach + public void after() { + if(sc != null) { + sc.close(); + } + sc = null; + } + + /** + * + * @return + */ + public JavaSparkContext getContext() { + if (sc != null) + return sc; + + //Ensure SPARK_USER environment variable is set for Spark tests + String u = System.getenv("SPARK_USER"); + Map env = System.getenv(); + if(u == null || u.isEmpty()) { + try { + Class[] classes = Collections.class.getDeclaredClasses(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map map = (Map) obj; + String user = System.getProperty("user.name"); + if (user == null || user.isEmpty()) + user = "user"; + map.put("SPARK_USER", user); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + // set to test mode + SparkConf sparkConf = new SparkConf().setMaster("local[4]").set("spark.driver.host", "localhost") + .setAppName("sparktest") + .set(Word2VecVariables.NUM_WORDS, String.valueOf(1)); + + + sc = new JavaSparkContext(sparkConf); + return sc; + + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TestFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TestFunction.java new file mode 100644 index 000000000..618bf0ac7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TestFunction.java @@ -0,0 +1,51 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text; + +import org.apache.spark.api.java.function.Function; + +import java.util.List; + +public class TestFunction implements Function { + public TestFunction(List lst) { + this.lst = lst; + } + + public List getLst() { + return lst; + } + + public int getA() { + return a; + } + + private List lst; + private int a; + + + @Override + public Integer call(Integer i) { + lst.add(i); + a = 1000; + return i + 1; + } +} + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java new file mode 100644 index 000000000..7e4a4944e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java @@ -0,0 +1,527 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.text; + +import com.sun.jna.Platform; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.models.word2vec.Huffman; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.wordstore.VocabCache; +import org.deeplearning4j.spark.models.embeddings.word2vec.FirstIterationFunction; +import org.deeplearning4j.spark.models.embeddings.word2vec.MapToPairFunction; +import org.deeplearning4j.spark.models.embeddings.word2vec.Word2Vec; +import org.deeplearning4j.spark.text.functions.CountCumSum; +import org.deeplearning4j.spark.text.functions.TextPipeline; +import org.deeplearning4j.text.stopwords.StopWords; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.primitives.Counter; +import org.nd4j.common.primitives.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author Jeffrey Tang + */ +public class TextPipelineTest extends BaseSparkTest { + + private List sentenceList; + private SparkConf conf; + private Word2Vec word2vec; + private Word2Vec word2vecNoStop; + + private static final Logger log = LoggerFactory.getLogger(TextPipeline.class); + + public JavaRDD getCorpusRDD(JavaSparkContext sc) { + return sc.parallelize(sentenceList, 2); + } + + @BeforeEach + public void before() throws Exception { + conf = new SparkConf().setMaster("local[4]").setAppName("sparktest").set("spark.driver.host", "localhost"); + + // All the avaliable options. These are default values + word2vec = new Word2Vec.Builder().minWordFrequency(1).setNGrams(1) + .tokenizerFactory( + "org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory") + .tokenPreprocessor( + "org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor") + // .setRemoveStop(true) + .stopWords(StopWords.getStopWords()).seed(42L).negative(0).useAdaGrad(false).layerSize(100) + .windowSize(5).learningRate(0.025).minLearningRate(0.0001).iterations(1).build(); + + word2vecNoStop = new Word2Vec.Builder().minWordFrequency(1).setNGrams(1) + .tokenizerFactory( + "org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory") + .tokenPreprocessor( + "org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor") + .seed(42L).negative(0).useAdaGrad(false).layerSize(100).windowSize(5).learningRate(0.025) + .minLearningRate(0.0001).iterations(1).build(); + + sentenceList = Arrays.asList("This is a strange strange world.", "Flowers are red."); + } + + + @Test + public void testTokenizer() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + JavaRDD> tokenizedRDD = pipeline.tokenize(); + + assertEquals(2, tokenizedRDD.count()); + + assertEquals(Arrays.asList("this", "is", "a", "strange", "strange", "world"), tokenizedRDD.first()); + + sc.stop(); + } + + @Test + public void testWordFreqAccIdentifyStopWords() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + JavaRDD> tokenizedRDD = pipeline.tokenize(); + JavaRDD, AtomicLong>> sentenceWordsCountRDD = + pipeline.updateAndReturnAccumulatorVal(tokenizedRDD); + + Counter wordFreqCounter = pipeline.getWordFreqAcc().value().get(0); + assertEquals(wordFreqCounter.getCount("STOP"), 4, 0); + assertEquals(wordFreqCounter.getCount("strange"), 2, 0); + assertEquals(wordFreqCounter.getCount("flowers"), 1, 0); + assertEquals(wordFreqCounter.getCount("world"), 1, 0); + assertEquals(wordFreqCounter.getCount("red"), 1, 0); + + List, AtomicLong>> ret = sentenceWordsCountRDD.collect(); + assertEquals(ret.get(0).getFirst(), Arrays.asList("this", "is", "a", "strange", "strange", "world")); + assertEquals(ret.get(1).getFirst(), Arrays.asList("flowers", "are", "red")); + assertEquals(ret.get(0).getSecond().get(), 6); + assertEquals(ret.get(1).getSecond().get(), 3); + + + sc.stop(); + } + + @Test + public void testWordFreqAccNotIdentifyingStopWords() throws Exception { + + JavaSparkContext sc = getContext(); + // word2vec.setRemoveStop(false); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vecNoStop.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + JavaRDD> tokenizedRDD = pipeline.tokenize(); + pipeline.updateAndReturnAccumulatorVal(tokenizedRDD); + + Counter wordFreqCounter = pipeline.getWordFreqAcc().value().get(0); + assertEquals(wordFreqCounter.getCount("is"), 1, 0); + assertEquals(wordFreqCounter.getCount("this"), 1, 0); + assertEquals(wordFreqCounter.getCount("are"), 1, 0); + assertEquals(wordFreqCounter.getCount("a"), 1, 0); + assertEquals(wordFreqCounter.getCount("strange"), 2, 0); + assertEquals(wordFreqCounter.getCount("flowers"), 1, 0); + assertEquals(wordFreqCounter.getCount("world"), 1, 0); + assertEquals(wordFreqCounter.getCount("red"), 1, 0); + + sc.stop(); + } + + @Test + public void testWordFreqAccIdentifyingStopWords() throws Exception { + + JavaSparkContext sc = getContext(); + // word2vec.setRemoveStop(false); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + JavaRDD> tokenizedRDD = pipeline.tokenize(); + pipeline.updateAndReturnAccumulatorVal(tokenizedRDD); + + Counter wordFreqCounter = pipeline.getWordFreqAcc().value().get(0); + assertEquals(wordFreqCounter.getCount("is"), 0, 0); + assertEquals(wordFreqCounter.getCount("this"), 0, 0); + assertEquals(wordFreqCounter.getCount("are"), 0, 0); + assertEquals(wordFreqCounter.getCount("a"), 0, 0); + assertEquals(wordFreqCounter.getCount("STOP"), 4, 0); + assertEquals(wordFreqCounter.getCount("strange"), 2, 0); + assertEquals(wordFreqCounter.getCount("flowers"), 1, 0); + assertEquals(wordFreqCounter.getCount("world"), 1, 0); + assertEquals(wordFreqCounter.getCount("red"), 1, 0); + + sc.stop(); + } + + @Test + public void testFilterMinWordAddVocab() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + JavaRDD> tokenizedRDD = pipeline.tokenize(); + pipeline.updateAndReturnAccumulatorVal(tokenizedRDD); + Counter wordFreqCounter = pipeline.getWordFreqAcc().value().get(0); + + pipeline.filterMinWordAddVocab(wordFreqCounter); + VocabCache vocabCache = pipeline.getVocabCache(); + + assertTrue(vocabCache != null); + + VocabWord redVocab = vocabCache.tokenFor("red"); + VocabWord flowerVocab = vocabCache.tokenFor("flowers"); + VocabWord worldVocab = vocabCache.tokenFor("world"); + VocabWord strangeVocab = vocabCache.tokenFor("strange"); + + + assertEquals(redVocab.getWord(), "red"); + assertEquals(redVocab.getElementFrequency(), 1, 0); + + assertEquals(flowerVocab.getWord(), "flowers"); + assertEquals(flowerVocab.getElementFrequency(), 1, 0); + + assertEquals(worldVocab.getWord(), "world"); + assertEquals(worldVocab.getElementFrequency(), 1, 0); + + assertEquals(strangeVocab.getWord(), "strange"); + assertEquals(strangeVocab.getElementFrequency(), 2, 0); + + sc.stop(); + } + + @Test + public void testBuildVocabCache() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + VocabCache vocabCache = pipeline.getVocabCache(); + + assertTrue(vocabCache != null); + + log.info("VocabWords: " + vocabCache.words()); + assertEquals(5, vocabCache.numWords()); + + + VocabWord redVocab = vocabCache.tokenFor("red"); + VocabWord flowerVocab = vocabCache.tokenFor("flowers"); + VocabWord worldVocab = vocabCache.tokenFor("world"); + VocabWord strangeVocab = vocabCache.tokenFor("strange"); + + log.info("Red word: " + redVocab); + log.info("Flower word: " + flowerVocab); + log.info("World word: " + worldVocab); + log.info("Strange word: " + strangeVocab); + + assertEquals(redVocab.getWord(), "red"); + assertEquals(redVocab.getElementFrequency(), 1, 0); + + assertEquals(flowerVocab.getWord(), "flowers"); + assertEquals(flowerVocab.getElementFrequency(), 1, 0); + + assertEquals(worldVocab.getWord(), "world"); + assertEquals(worldVocab.getElementFrequency(), 1, 0); + + assertEquals(strangeVocab.getWord(), "strange"); + assertEquals(strangeVocab.getElementFrequency(), 2, 0); + + sc.stop(); + } + + @Test + public void testBuildVocabWordListRDD() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + pipeline.buildVocabWordListRDD(); + JavaRDD sentenceCountRDD = pipeline.getSentenceCountRDD(); + JavaRDD> vocabWordListRDD = pipeline.getVocabWordListRDD(); + List> vocabWordList = vocabWordListRDD.collect(); + List firstSentenceVocabList = vocabWordList.get(0); + List secondSentenceVocabList = vocabWordList.get(1); + + System.out.println(Arrays.deepToString(firstSentenceVocabList.toArray())); + + List firstSentenceTokenList = new ArrayList<>(); + List secondSentenceTokenList = new ArrayList<>(); + for (VocabWord v : firstSentenceVocabList) { + if (v != null) { + firstSentenceTokenList.add(v.getWord()); + } + } + for (VocabWord v : secondSentenceVocabList) { + if (v != null) { + secondSentenceTokenList.add(v.getWord()); + } + } + + assertEquals(pipeline.getTotalWordCount(), 9, 0); + assertEquals(sentenceCountRDD.collect().get(0).get(), 6); + assertEquals(sentenceCountRDD.collect().get(1).get(), 3); + assertTrue(firstSentenceTokenList.containsAll(Arrays.asList("strange", "strange", "world"))); + assertTrue(secondSentenceTokenList.containsAll(Arrays.asList("flowers", "red"))); + + sc.stop(); + } + + @Test + public void testHuffman() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + + VocabCache vocabCache = pipeline.getVocabCache(); + + Huffman huffman = new Huffman(vocabCache.vocabWords()); + huffman.build(); + huffman.applyIndexes(vocabCache); + + Collection vocabWords = vocabCache.vocabWords(); + System.out.println("Huffman Test:"); + for (VocabWord vocabWord : vocabWords) { + System.out.println("Word: " + vocabWord); + System.out.println(vocabWord.getCodes()); + System.out.println(vocabWord.getPoints()); + } + + sc.stop(); + } + + @Test //@Ignore //AB 2020/04/20 https://github.com/eclipse/deeplearning4j/issues/8849 + public void testCountCumSum() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + pipeline.buildVocabWordListRDD(); + JavaRDD sentenceCountRDD = pipeline.getSentenceCountRDD(); + + CountCumSum countCumSum = new CountCumSum(sentenceCountRDD); + JavaRDD sentenceCountCumSumRDD = countCumSum.buildCumSum(); + List sentenceCountCumSumList = sentenceCountCumSumRDD.collect(); + assertTrue(sentenceCountCumSumList.get(0) == 6L); + assertTrue(sentenceCountCumSumList.get(1) == 9L); + + sc.stop(); + } + + /** + * This test checked generations retrieved using stopWords + * + * @throws Exception + */ + @Test //@Ignore //AB 2020/04/19 https://github.com/eclipse/deeplearning4j/issues/8849 + public void testZipFunction1() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + // word2vec.setRemoveStop(false); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + pipeline.buildVocabWordListRDD(); + JavaRDD sentenceCountRDD = pipeline.getSentenceCountRDD(); + JavaRDD> vocabWordListRDD = pipeline.getVocabWordListRDD(); + + CountCumSum countCumSum = new CountCumSum(sentenceCountRDD); + JavaRDD sentenceCountCumSumRDD = countCumSum.buildCumSum(); + + JavaPairRDD, Long> vocabWordListSentenceCumSumRDD = + vocabWordListRDD.zip(sentenceCountCumSumRDD); + List, Long>> lst = vocabWordListSentenceCumSumRDD.collect(); + + List vocabWordsList1 = lst.get(0)._1(); + Long cumSumSize1 = lst.get(0)._2(); + assertEquals(3, vocabWordsList1.size()); + assertEquals(vocabWordsList1.get(0).getWord(), "strange"); + assertEquals(vocabWordsList1.get(1).getWord(), "strange"); + assertEquals(vocabWordsList1.get(2).getWord(), "world"); + assertEquals(cumSumSize1, 6L, 0); + + List vocabWordsList2 = lst.get(1)._1(); + Long cumSumSize2 = lst.get(1)._2(); + assertEquals(2, vocabWordsList2.size()); + assertEquals(vocabWordsList2.get(0).getWord(), "flowers"); + assertEquals(vocabWordsList2.get(1).getWord(), "red"); + assertEquals(cumSumSize2, 9L, 0); + + sc.stop(); + } + + @Test //@Ignore //AB 2020/04/19 https://github.com/eclipse/deeplearning4j/issues/8849 + public void testZipFunction2() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + // word2vec.setRemoveStop(false); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vecNoStop.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + pipeline.buildVocabWordListRDD(); + JavaRDD sentenceCountRDD = pipeline.getSentenceCountRDD(); + JavaRDD> vocabWordListRDD = pipeline.getVocabWordListRDD(); + + CountCumSum countCumSum = new CountCumSum(sentenceCountRDD); + JavaRDD sentenceCountCumSumRDD = countCumSum.buildCumSum(); + + JavaPairRDD, Long> vocabWordListSentenceCumSumRDD = + vocabWordListRDD.zip(sentenceCountCumSumRDD); + List, Long>> lst = vocabWordListSentenceCumSumRDD.collect(); + + List vocabWordsList1 = lst.get(0)._1(); + Long cumSumSize1 = lst.get(0)._2(); + assertEquals(6, vocabWordsList1.size()); + assertEquals(vocabWordsList1.get(0).getWord(), "this"); + assertEquals(vocabWordsList1.get(1).getWord(), "is"); + assertEquals(vocabWordsList1.get(2).getWord(), "a"); + assertEquals(vocabWordsList1.get(3).getWord(), "strange"); + assertEquals(vocabWordsList1.get(4).getWord(), "strange"); + assertEquals(vocabWordsList1.get(5).getWord(), "world"); + assertEquals(cumSumSize1, 6L, 0); + + List vocabWordsList2 = lst.get(1)._1(); + Long cumSumSize2 = lst.get(1)._2(); + assertEquals(vocabWordsList2.size(), 3); + assertEquals(vocabWordsList2.get(0).getWord(), "flowers"); + assertEquals(vocabWordsList2.get(1).getWord(), "are"); + assertEquals(vocabWordsList2.get(2).getWord(), "red"); + assertEquals(cumSumSize2, 9L, 0); + + sc.stop(); + } + + @Test + public void testFirstIteration() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + // word2vec.setRemoveStop(false); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + pipeline.buildVocabWordListRDD(); + VocabCache vocabCache = pipeline.getVocabCache(); + /* Huffman huffman = new Huffman(vocabCache.vocabWords()); + huffman.build(); + huffman.applyIndexes(vocabCache); + */ + VocabWord token = vocabCache.tokenFor("strange"); + VocabWord word = vocabCache.wordFor("strange"); + log.info("Strange token: " + token); + log.info("Strange word: " + word); + + // Get total word count and put into word2vec variable map + Map word2vecVarMap = word2vec.getWord2vecVarMap(); + word2vecVarMap.put("totalWordCount", pipeline.getTotalWordCount()); + double[] expTable = word2vec.getExpTable(); + + JavaRDD sentenceCountRDD = pipeline.getSentenceCountRDD(); + JavaRDD> vocabWordListRDD = pipeline.getVocabWordListRDD(); + + CountCumSum countCumSum = new CountCumSum(sentenceCountRDD); + JavaRDD sentenceCountCumSumRDD = countCumSum.buildCumSum(); + + JavaPairRDD, Long> vocabWordListSentenceCumSumRDD = + vocabWordListRDD.zip(sentenceCountCumSumRDD); + + Broadcast> word2vecVarMapBroadcast = sc.broadcast(word2vecVarMap); + Broadcast expTableBroadcast = sc.broadcast(expTable); + + Iterator, Long>> iterator = vocabWordListSentenceCumSumRDD.collect().iterator(); + + FirstIterationFunction firstIterationFunction = new FirstIterationFunction( + word2vecVarMapBroadcast, expTableBroadcast, pipeline.getBroadCastVocabCache()); + + Iterator> ret = firstIterationFunction.call(iterator); + assertTrue(ret.hasNext()); + } + + @Test + public void testSyn0AfterFirstIteration() throws Exception { + JavaSparkContext sc = getContext(); + JavaRDD corpusRDD = getCorpusRDD(sc); + // word2vec.setRemoveStop(false); + Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); + + TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); + pipeline.buildVocabCache(); + pipeline.buildVocabWordListRDD(); + VocabCache vocabCache = pipeline.getVocabCache(); + Huffman huffman = new Huffman(vocabCache.vocabWords()); + huffman.build(); + + // Get total word count and put into word2vec variable map + Map word2vecVarMap = word2vec.getWord2vecVarMap(); + word2vecVarMap.put("totalWordCount", pipeline.getTotalWordCount()); + double[] expTable = word2vec.getExpTable(); + + JavaRDD sentenceCountRDD = pipeline.getSentenceCountRDD(); + JavaRDD> vocabWordListRDD = pipeline.getVocabWordListRDD(); + + CountCumSum countCumSum = new CountCumSum(sentenceCountRDD); + JavaRDD sentenceCountCumSumRDD = countCumSum.buildCumSum(); + + JavaPairRDD, Long> vocabWordListSentenceCumSumRDD = + vocabWordListRDD.zip(sentenceCountCumSumRDD); + + Broadcast> word2vecVarMapBroadcast = sc.broadcast(word2vecVarMap); + Broadcast expTableBroadcast = sc.broadcast(expTable); + + FirstIterationFunction firstIterationFunction = new FirstIterationFunction(word2vecVarMapBroadcast, + expTableBroadcast, pipeline.getBroadCastVocabCache()); + JavaRDD> pointSyn0Vec = vocabWordListSentenceCumSumRDD + .mapPartitions(firstIterationFunction).map(new MapToPairFunction()); + } + +} + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/resources/log4j.properties b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/resources/log4j.properties new file mode 100644 index 000000000..e0dc1ce63 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/resources/log4j.properties @@ -0,0 +1,35 @@ +# +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +log4j.rootLogger=ERROR, Console +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n + +log4j.appender.org.springframework=DEBUG +log4j.appender.org.deeplearning4j=DEBUG +log4j.appender.org.nd4j=DEBUG + +log4j.logger.org.springframework=INFO +log4j.logger.org.deeplearning4j=DEBUG +log4j.logger.org.nd4j=DEBUG +log4j.logger.org.apache.spark=WARN + + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/resources/logback.xml b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/resources/logback.xml new file mode 100644 index 000000000..aef9b5e2e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-nlp/src/test/resources/logback.xml @@ -0,0 +1,57 @@ + + + + + + + + logs/application.log + + %logger{15} - %message%n%xException{5} + + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/build.gradle b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/build.gradle new file mode 100644 index 000000000..ddcbf2914 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/build.gradle @@ -0,0 +1,54 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +ext { + buildTarget = rootProject.ext.buildTarget + scalaVersion = rootProject.ext.scalaVersion +} + +apply from: "${rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisDnn.cavisDnnSpark.cavisDnnSparkCore + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnParallelwrapper + implementation projects.cavisDnn.cavisDnnModelimport + implementation projects.cavisDatavec.cavisDatavecSpark.cavisDatavecSparkCore + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerNode + implementation projects.cavisNative.cavisNativeBlas + + compileOnly "org.apache.spark:spark-core_${scalaVersion}" + testCompileOnly "org.apache.spark:spark-core_${scalaVersion}" + implementation "org.reactivestreams:reactive-streams:1.0.3" + implementation "io.reactivex.rxjava2:rxjava:2.2.21" + implementation "org.bytedeco:javacpp" + implementation group: "org.bytedeco", name: "hdf5" + implementation group: "org.bytedeco", name: "hdf5", classifier: buildTarget + implementation "com.sun.jna:jna:3.0.9" + + testImplementation projects.cavisDnn.cavisDnnCommonTests + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/ParameterServerSubscriber.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/ParameterServerSubscriber.java new file mode 100644 index 000000000..08db1a386 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/ParameterServerSubscriber.java @@ -0,0 +1,26 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver; + +public class ParameterServerSubscriber { + + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/ParameterServerTrainingHook.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/ParameterServerTrainingHook.java new file mode 100644 index 000000000..402560c73 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/ParameterServerTrainingHook.java @@ -0,0 +1,81 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver; + +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.spark.api.TrainingHook; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +/** + * Training hook for the + * parameter server + * @author Adam Gibson + */ +public class ParameterServerTrainingHook implements TrainingHook { + /** + * A hook method for pre update. + * + * @param minibatch the inibatch + * that was used for the update + * @param model themodel that was update + */ + @Override + public void preUpdate(DataSet minibatch, Model model) { + //pull + } + + /** + * A hook method for post update + * + * @param minibatch the minibatch + * that was usd for the update + * @param model the model that was updated + */ + @Override + public void postUpdate(DataSet minibatch, Model model) { + //push + } + + /** + * A hook method for pre update. + * + * @param minibatch the inibatch + * that was used for the update + * @param model themodel that was update + */ + @Override + public void preUpdate(MultiDataSet minibatch, Model model) { + //pull + } + + /** + * A hook method for post update + * + * @param minibatch the minibatch + * that was usd for the update + * @param model the model that was updated + */ + @Override + public void postUpdate(MultiDataSet minibatch, Model model) { + //push + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunction.java new file mode 100644 index 000000000..a1bc43c6c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunction.java @@ -0,0 +1,138 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.accumulation; + +import org.apache.spark.api.java.function.Function2; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithmReducer; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public class SharedTrainingAccumulationFunction implements + Function2 { + + @Override + public SharedTrainingAccumulationTuple call(SharedTrainingAccumulationTuple tuple1, + SharedTrainingAccumulationTuple tuple2) throws Exception { + // if one of tuples is null - return other one + if (tuple1 == null) + return tuple2; + else if (tuple2 == null) + return tuple1; + + double score = 0.0; + INDArray stateView = null; + int aggregationsCount = 0; + if (tuple1.getUpdaterStateArray() != null && tuple2.getUpdaterStateArray() != null) { + // we have multiple state views here. average them + stateView = tuple1.getUpdaterStateArray().addi(tuple2.getUpdaterStateArray()); + } else if (tuple1.getUpdaterStateArray() != null || tuple2.getUpdaterStateArray() != null) { + // only one of state views exists. just use it + stateView = tuple1.getUpdaterStateArray() != null ? tuple1.getUpdaterStateArray() + : tuple2.getUpdaterStateArray(); + } + + // we assume that aggregationsCount field is set only for entries that hold updaters state + aggregationsCount = tuple1.getAggregationsCount() + tuple2.getAggregationsCount(); + score = tuple1.getScoreSum() + tuple2.getScoreSum(); + + // aggregating spark stats + SparkTrainingStats stats = tuple1.getSparkTrainingStats(); + if (tuple2.getSparkTrainingStats() != null) { + if (stats == null) + stats = tuple2.getSparkTrainingStats(); + else + stats.addOtherTrainingStats(tuple2.getSparkTrainingStats()); + } + + Nd4j.getExecutioner().commit(); + + Collection listenerMetaData = tuple1.getListenerMetaData(); + if (listenerMetaData == null) + listenerMetaData = tuple2.getListenerMetaData(); + else { + Collection newMeta = tuple2.getListenerMetaData(); + if (newMeta != null) + listenerMetaData.addAll(newMeta); + } + + Collection listenerStaticInfo = tuple1.getListenerStaticInfo(); + if (listenerStaticInfo == null) + listenerStaticInfo = tuple2.getListenerStaticInfo(); + else { + Collection newStatic = tuple2.getListenerStaticInfo(); + if (newStatic != null) + listenerStaticInfo.addAll(newStatic); + } + + Collection listenerUpdates = tuple1.getListenerUpdates(); + if (listenerUpdates == null) + listenerUpdates = tuple2.getListenerUpdates(); + else { + Collection listenerUpdates2 = tuple2.getListenerUpdates(); + if (listenerUpdates2 != null) + listenerUpdates.addAll(listenerUpdates2); + } + + Map minibatchesPerExecutor = new HashMap<>(); + if(tuple1.getMinibatchesPerExecutor() != null) { + for (Map.Entry e : tuple1.getMinibatchesPerExecutor().entrySet()){ + minibatchesPerExecutor.put(e.getKey(), e.getValue()); + } + } + if(tuple2.getMinibatchesPerExecutor() != null){ + for (Map.Entry e : tuple2.getMinibatchesPerExecutor().entrySet()){ + if(minibatchesPerExecutor.containsKey(e.getKey())){ + minibatchesPerExecutor.put(e.getKey(), minibatchesPerExecutor.get(e.getKey()) + e.getValue()); + } else { + minibatchesPerExecutor.put(e.getKey(), e.getValue()); + } + } + } + + ThresholdAlgorithmReducer thresholdAlgorithmReducer = null; + if(tuple1.getThresholdAlgorithmReducer() != null){ + thresholdAlgorithmReducer = tuple1.getThresholdAlgorithmReducer(); + } + if(tuple2.getThresholdAlgorithmReducer() != null){ + if(thresholdAlgorithmReducer == null){ + thresholdAlgorithmReducer = tuple2.getThresholdAlgorithmReducer(); + } else { + //Merge threshold algorithm reducers + thresholdAlgorithmReducer = thresholdAlgorithmReducer.merge(tuple2.getThresholdAlgorithmReducer()); + } + } + + return SharedTrainingAccumulationTuple.builder().scoreSum(score).updaterStateArray(stateView) + .aggregationsCount(aggregationsCount).sparkTrainingStats(stats) + .listenerMetaData(listenerMetaData).listenerUpdates(listenerUpdates) + .listenerStaticInfo(listenerStaticInfo) + .minibatchesPerExecutor(minibatchesPerExecutor) + .thresholdAlgorithmReducer(thresholdAlgorithmReducer) + .build(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationTuple.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationTuple.java new file mode 100644 index 000000000..4fa9d77ce --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationTuple.java @@ -0,0 +1,51 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.accumulation; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithmReducer; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Map; + +@AllArgsConstructor +@Data +@NoArgsConstructor +@Builder +public class SharedTrainingAccumulationTuple implements Serializable { + private INDArray updaterStateArray; + private double scoreSum; + private int aggregationsCount; + private SparkTrainingStats sparkTrainingStats; + private Collection listenerMetaData; + private Collection listenerStaticInfo; + private Collection listenerUpdates; + private Map minibatchesPerExecutor; + private ThresholdAlgorithmReducer thresholdAlgorithmReducer; +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunction.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunction.java new file mode 100644 index 000000000..356be12b2 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunction.java @@ -0,0 +1,146 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.accumulation; + +import org.apache.spark.api.java.function.Function2; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithmReducer; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingResult; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public class SharedTrainingAggregateFunction implements + Function2 { + + @Override + public SharedTrainingAccumulationTuple call(SharedTrainingAccumulationTuple tuple, SharedTrainingResult result) + throws Exception { + if (tuple == null) { + ThresholdAlgorithmReducer tar = null; + if(result.getThresholdAlgorithm() != null){ + tar = result.getThresholdAlgorithm().newReducer(); + tar.add(result.getThresholdAlgorithm()); + } + + return SharedTrainingAccumulationTuple.builder().updaterStateArray(result.getUpdaterStateArray()) + .scoreSum(result.getScoreSum()).listenerStaticInfo(result.getListenerStaticInfo()) + .listenerUpdates(result.getListenerUpdates()).listenerMetaData(result.getListenerMetaData()) + .sparkTrainingStats(result.getSparkTrainingStats()) + .aggregationsCount(result.getAggregationsCount()) + .minibatchesPerExecutor(result.getMinibatchesPerExecutor()) + .thresholdAlgorithmReducer(tar) + .build(); + } + + + INDArray updaterStateSum = null; + int aggregationsCount = 0; + double score = 0.0; + if (tuple.getUpdaterStateArray() != null) { + if (result.getUpdaterStateArray() != null) { + updaterStateSum = tuple.getUpdaterStateArray().addi(result.getUpdaterStateArray()); + aggregationsCount = tuple.getAggregationsCount() + 1; + score = tuple.getScoreSum() + result.getScoreSum(); + } + } else { + if (result.getUpdaterStateArray() != null) { + updaterStateSum = result.getUpdaterStateArray(); + aggregationsCount = 1; + score = result.getScoreSum(); + } + } + + SparkTrainingStats stats = tuple.getSparkTrainingStats(); + if (result.getSparkTrainingStats() != null) { + if (stats == null) + stats = result.getSparkTrainingStats(); + else + stats.addOtherTrainingStats(result.getSparkTrainingStats()); + } + + Nd4j.getExecutioner().commit(); + + Collection listenerMetaData = tuple.getListenerMetaData(); + if (listenerMetaData == null) + listenerMetaData = result.getListenerMetaData(); + else { + Collection newMeta = result.getListenerMetaData(); + if (newMeta != null) + listenerMetaData.addAll(newMeta); + } + + Collection listenerStaticInfo = tuple.getListenerStaticInfo(); + if (listenerStaticInfo == null) + listenerStaticInfo = result.getListenerStaticInfo(); + else { + Collection newStatic = result.getListenerStaticInfo(); + if (newStatic != null) + listenerStaticInfo.addAll(newStatic); + } + + Collection listenerUpdates = tuple.getListenerUpdates(); + if (listenerUpdates == null) + listenerUpdates = result.getListenerUpdates(); + else { + Collection listenerUpdates2 = result.getListenerUpdates(); + if (listenerUpdates2 != null) + listenerUpdates.addAll(listenerUpdates2); + } + + Map minibatchesPerExecutor = new HashMap<>(); + if(tuple.getMinibatchesPerExecutor() != null) { + for (Map.Entry e : tuple.getMinibatchesPerExecutor().entrySet()){ + minibatchesPerExecutor.put(e.getKey(), e.getValue()); + } + } + if(result.getMinibatchesPerExecutor() != null){ + for (Map.Entry e : result.getMinibatchesPerExecutor().entrySet()){ + if(minibatchesPerExecutor.containsKey(e.getKey())){ + minibatchesPerExecutor.put(e.getKey(), minibatchesPerExecutor.get(e.getKey()) + e.getValue()); + } else { + minibatchesPerExecutor.put(e.getKey(), e.getValue()); + } + } + } + + ThresholdAlgorithmReducer thresholdAlgorithmReducer = tuple.getThresholdAlgorithmReducer(); + if(thresholdAlgorithmReducer == null && result.getThresholdAlgorithm() != null){ + thresholdAlgorithmReducer = result.getThresholdAlgorithm().newReducer(); + } + if(thresholdAlgorithmReducer != null){ + thresholdAlgorithmReducer.add(result.getThresholdAlgorithm()); + } + + return SharedTrainingAccumulationTuple.builder().scoreSum(score).updaterStateArray(updaterStateSum) + .aggregationsCount(aggregationsCount).sparkTrainingStats(stats) + .listenerMetaData(listenerMetaData).listenerUpdates(listenerUpdates) + .listenerStaticInfo(listenerStaticInfo) + .minibatchesPerExecutor(minibatchesPerExecutor) + .thresholdAlgorithmReducer(thresholdAlgorithmReducer) + .build(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/DataSetDeserializationCallback.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/DataSetDeserializationCallback.java new file mode 100644 index 000000000..ae4c7bdd5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/DataSetDeserializationCallback.java @@ -0,0 +1,41 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.callbacks; + +import org.apache.spark.input.PortableDataStream; +import org.nd4j.linalg.dataset.DataSet; + +import java.io.DataInputStream; + +public class DataSetDeserializationCallback implements PortableDataStreamCallback { + + @Override + public DataSet compute(PortableDataStream pds) { + try (DataInputStream is = pds.open()) { + // TODO: do something better here + org.nd4j.linalg.dataset.DataSet ds = new org.nd4j.linalg.dataset.DataSet(); + ds.load(is); + return ds; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/MultiDataSetDeserializationCallback.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/MultiDataSetDeserializationCallback.java new file mode 100644 index 000000000..e82b7b755 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/MultiDataSetDeserializationCallback.java @@ -0,0 +1,41 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.callbacks; + +import org.apache.spark.input.PortableDataStream; +import org.nd4j.linalg.dataset.MultiDataSet; + +import java.io.DataInputStream; + +public class MultiDataSetDeserializationCallback implements PortableDataStreamMDSCallback { + + @Override + public MultiDataSet compute(PortableDataStream pds) { + try (DataInputStream is = pds.open()) { + // TODO: do something better here + MultiDataSet ds = new MultiDataSet(); + ds.load(is); + return ds; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/PortableDataStreamCallback.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/PortableDataStreamCallback.java new file mode 100644 index 000000000..97b51ac06 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/PortableDataStreamCallback.java @@ -0,0 +1,34 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.callbacks; + +import org.apache.spark.input.PortableDataStream; +import org.nd4j.linalg.dataset.DataSet; + +public interface PortableDataStreamCallback { + + /** + * This method should do something, and return DataSet after all + * @param pds + * @return + */ + DataSet compute(PortableDataStream pds); +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/PortableDataStreamMDSCallback.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/PortableDataStreamMDSCallback.java new file mode 100644 index 000000000..4df33863a --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/callbacks/PortableDataStreamMDSCallback.java @@ -0,0 +1,34 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.callbacks; + +import org.apache.spark.input.PortableDataStream; +import org.nd4j.linalg.dataset.MultiDataSet; + +public interface PortableDataStreamMDSCallback { + + /** + * This method should do something, and return DataSet after all + * @param pds + * @return + */ + MultiDataSet compute(PortableDataStream pds); +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/conf/SharedTrainingConfiguration.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/conf/SharedTrainingConfiguration.java new file mode 100644 index 000000000..d179a269c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/conf/SharedTrainingConfiguration.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.conf; + +import lombok.*; +import org.deeplearning4j.nn.conf.WorkspaceMode; +import org.deeplearning4j.optimize.solvers.accumulation.MessageHandler; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ResidualPostProcessor; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; +import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; + +import java.io.Serializable; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class SharedTrainingConfiguration implements Serializable { + protected VoidConfiguration voidConfiguration; + + @Builder.Default + protected WorkspaceMode workspaceMode = WorkspaceMode.ENABLED; + @Builder.Default + protected int prefetchSize = 2; + @Builder.Default + protected boolean epochReset = false; + @Builder.Default + protected int numberOfWorkersPerNode = -1; + @Builder.Default + protected long debugLongerIterations = 0L; + @Builder.Default + protected boolean encodingDebugMode = false; + + /** + * This value **overrides** bufferSize calculations for gradients accumulator + */ + @Builder.Default + protected int bufferSize = 0; + + protected ThresholdAlgorithm thresholdAlgorithm; + protected ResidualPostProcessor residualPostProcessor; + protected String messageHandlerClass; + + + + public void setMessageHandlerClass(@NonNull String messageHandlerClass) { + this.messageHandlerClass = messageHandlerClass; + } + + public void setMessageHandlerClass(@NonNull MessageHandler handler) { + this.messageHandlerClass = handler.getClass().getCanonicalName(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapDataSet.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapDataSet.java new file mode 100644 index 000000000..eb1924c09 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapDataSet.java @@ -0,0 +1,66 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.functions; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.parameterserver.pw.SharedTrainingWrapper; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingResult; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingWorker; +import org.nd4j.linalg.dataset.DataSet; + +import java.util.Collections; +import java.util.Iterator; + +public class SharedFlatMapDataSet implements FlatMapFunction, R> { + + private final SharedTrainingWorker worker; + + public SharedFlatMapDataSet(TrainingWorker worker) { + // we're not going to have anything but Shared classes here ever + this.worker = (SharedTrainingWorker) worker; + } + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + //Under some limited circumstances, we might have an empty partition. In this case, we should return immediately + if(!dataSetIterator.hasNext()){ + return Collections.emptyIterator(); + } + + /* + That's the place where we do our stuff. Here's the plan: + 1) we pass given iterator to VirtualDataSetIterator, which acts as holder for them + 2) Virtual iterator will provide load balancing between available devices + 3) we'll lock out here + */ + + // iterator should be silently attached to VirtualDataSetIterator, and used appropriately + SharedTrainingWrapper.getInstance(worker.getInstanceId()).attachDS(dataSetIterator); + + // first callee will become master, others will obey and die + // all threads in this executor will be blocked here until training finished + SharedTrainingResult result = SharedTrainingWrapper.getInstance(worker.getInstanceId()).run(worker); + + return Collections.singletonList((R) result).iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapMultiDataSet.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapMultiDataSet.java new file mode 100644 index 000000000..5d5672a5b --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapMultiDataSet.java @@ -0,0 +1,65 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.functions; + +import org.apache.spark.api.java.function.FlatMapFunction; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.parameterserver.pw.SharedTrainingWrapper; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingResult; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingWorker; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.util.Collections; +import java.util.Iterator; + +public class SharedFlatMapMultiDataSet implements FlatMapFunction, R> { + + private final SharedTrainingWorker worker; + + public SharedFlatMapMultiDataSet(TrainingWorker worker) { + // we're not going to have anything but Shared classes here ever + this.worker = (SharedTrainingWorker) worker; + } + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + //Under some limited circumstances, we might have an empty partition. In this case, we should return immediately + if(!dataSetIterator.hasNext()){ + return Collections.emptyIterator(); + } + /* + That's the place where we do our stuff. Here's the plan: + 1) we pass given iterator to VirtualDataSetIterator, which acts as holder for them + 2) Virtual iterator will provide load balancing between available devices + 3) we'll lock out here + */ + + // iterator should be silently attached to VirtualDataSetIterator, and used appropriately + SharedTrainingWrapper.getInstance(worker.getInstanceId()).attachMDS(dataSetIterator); + + // first callee will become master, others will obey and die + // all threads in this executor will be blocked here until training finished + SharedTrainingResult result = SharedTrainingWrapper.getInstance(worker.getInstanceId()).run(worker); + + return Collections.singletonList((R) result).iterator(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapPaths.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapPaths.java new file mode 100644 index 000000000..270d2d8ee --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapPaths.java @@ -0,0 +1,94 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.functions; + +import org.apache.commons.io.LineIterator; +import org.apache.hadoop.conf.Configuration; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.iterator.PathSparkDataSetIterator; +import org.deeplearning4j.spark.parameterserver.pw.SharedTrainingWrapper; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingResult; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingWorker; + +import java.io.*; +import java.nio.file.Files; +import java.util.Collections; +import java.util.Iterator; + +public class SharedFlatMapPaths implements FlatMapFunction, R> { + + public static File toTempFile(Iterator dataSetIterator) throws IOException { + File f = Files.createTempFile("SharedFlatMapPaths",".txt").toFile(); + f.deleteOnExit(); + try(BufferedWriter bw = new BufferedWriter(new FileWriter(f))){ + while(dataSetIterator.hasNext()){ + bw.write(dataSetIterator.next()); + bw.write("\n"); + } + } + return f; + } + + public static Configuration defaultConfig; + + protected final SharedTrainingWorker worker; + protected final DataSetLoader loader; + protected final Broadcast hadoopConfig; + + public SharedFlatMapPaths(TrainingWorker worker, DataSetLoader loader, Broadcast hadoopConfig) { + // we're not going to have anything but Shared classes here ever + this.worker = (SharedTrainingWorker) worker; + this.loader = loader; + this.hadoopConfig = hadoopConfig; + } + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + //Under some limited circumstances, we might have an empty partition. In this case, we should return immediately + if(!dataSetIterator.hasNext()){ + return Collections.emptyIterator(); + } + // here we'll be converting out Strings coming out of iterator to DataSets + // PathSparkDataSetIterator does that for us + //For better fault tolerance, we'll pull all paths to a local file. This way, if the Iterator is backed + // by a remote source that later goes down, we won't fail (as long as the source is still available) + File f = SharedFlatMapPaths.toTempFile(dataSetIterator); + + LineIterator lineIter = new LineIterator(new FileReader(f)); //Buffered reader added automatically + try { + // iterator should be silently attached to VirtualDataSetIterator, and used appropriately + SharedTrainingWrapper.getInstance(worker.getInstanceId()).attachDS(new PathSparkDataSetIterator(lineIter, loader, hadoopConfig)); + + // first callee will become master, others will obey and die + SharedTrainingResult result = SharedTrainingWrapper.getInstance(worker.getInstanceId()).run(worker); + + return Collections.singletonList((R) result).iterator(); + } finally { + lineIter.close(); + f.delete(); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapPathsMDS.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapPathsMDS.java new file mode 100644 index 000000000..9a9454128 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/functions/SharedFlatMapPathsMDS.java @@ -0,0 +1,79 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.functions; + +import org.apache.commons.io.LineIterator; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.datavec.spark.util.SerializableHadoopConfig; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.iterator.PathSparkMultiDataSetIterator; +import org.deeplearning4j.spark.parameterserver.pw.SharedTrainingWrapper; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingResult; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingWorker; + +import java.io.File; +import java.io.FileReader; +import java.util.Collections; +import java.util.Iterator; + +public class SharedFlatMapPathsMDS implements FlatMapFunction, R> { + + protected final SharedTrainingWorker worker; + protected final MultiDataSetLoader loader; + protected final Broadcast hadoopConfig; + + public SharedFlatMapPathsMDS(TrainingWorker worker, MultiDataSetLoader loader, Broadcast hadoopConfig) { + // we're not going to have anything but Shared classes here ever + this.worker = (SharedTrainingWorker) worker; + this.loader = loader; + this.hadoopConfig = hadoopConfig; + } + + @Override + public Iterator call(Iterator dataSetIterator) throws Exception { + //Under some limited circumstances, we might have an empty partition. In this case, we should return immediately + if(!dataSetIterator.hasNext()){ + return Collections.emptyIterator(); + } + // here we'll be converting out Strings coming out of iterator to DataSets + // PathSparkDataSetIterator does that for us + //For better fault tolerance, we'll pull all paths to a local file. This way, if the Iterator is backed + // by a remote source that later goes down, we won't fail (as long as the source is still available) + File f = SharedFlatMapPaths.toTempFile(dataSetIterator); + + LineIterator lineIter = new LineIterator(new FileReader(f)); //Buffered reader added automatically + try { + // iterator should be silently attached to VirtualDataSetIterator, and used appropriately + SharedTrainingWrapper.getInstance(worker.getInstanceId()).attachMDS(new PathSparkMultiDataSetIterator(lineIter, loader, hadoopConfig)); + + // first callee will become master, others will obey and die + SharedTrainingResult result = SharedTrainingWrapper.getInstance(worker.getInstanceId()).run(worker); + + return Collections.singletonList((R) result).iterator(); + } finally { + lineIter.close(); + f.delete(); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/MultiPdsIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/MultiPdsIterator.java new file mode 100644 index 000000000..feab94776 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/MultiPdsIterator.java @@ -0,0 +1,60 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.iterators; + +import lombok.NonNull; +import org.apache.spark.input.PortableDataStream; +import org.deeplearning4j.spark.parameterserver.callbacks.PortableDataStreamMDSCallback; +import org.nd4j.linalg.dataset.api.MultiDataSet; + +import java.util.Iterator; +import java.util.function.Consumer; + +public class MultiPdsIterator implements Iterator { + protected final Iterator iterator; + protected final PortableDataStreamMDSCallback callback; + + public MultiPdsIterator(@NonNull Iterator pds, + @NonNull PortableDataStreamMDSCallback callback) { + this.iterator = pds; + this.callback = callback; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public MultiDataSet next() { + return callback.compute(iterator.next()); + } + + @Override + public void remove() { + // no-op + } + + @Override + public void forEachRemaining(Consumer action) { + throw new UnsupportedOperationException(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/PdsIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/PdsIterator.java new file mode 100644 index 000000000..f8841831c --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/PdsIterator.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.iterators; + +import lombok.NonNull; +import org.apache.spark.input.PortableDataStream; +import org.deeplearning4j.spark.parameterserver.callbacks.PortableDataStreamCallback; +import org.nd4j.linalg.dataset.DataSet; + +import java.util.Iterator; +import java.util.function.Consumer; + +public class PdsIterator implements Iterator { + protected final Iterator iterator; + protected final PortableDataStreamCallback callback; + + public PdsIterator(@NonNull Iterator pds, @NonNull PortableDataStreamCallback callback) { + this.iterator = pds; + this.callback = callback; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public DataSet next() { + return callback.compute(iterator.next()); + } + + @Override + public void remove() { + // no-op + } + + @Override + public void forEachRemaining(Consumer action) { + throw new UnsupportedOperationException(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIterator.java new file mode 100644 index 000000000..002602acc --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIterator.java @@ -0,0 +1,149 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.iterators; + +import lombok.NonNull; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.DataSetPreProcessor; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +public class VirtualDataSetIterator implements DataSetIterator { + + /** + * Basic idea here is simple: this DataSetIterator will take in multiple lazy Iterator, + * and will push them is round-robin manner to ParallelWrapper workers + */ + + protected final List> iterators; + protected final AtomicInteger position; + + public VirtualDataSetIterator(@NonNull List> iterators) { + this.iterators = iterators; + this.position = new AtomicInteger(0); + } + + /* + + // TODO: to be implemented + + @Override + public void attachThread(int producer) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasNextFor() { + return false; + } + + @Override + public boolean hasNextFor(int consumer) { + return false; + } + + @Override + public DataSet nextFor(int consumer) { + return null; + } + + @Override + public DataSet nextFor() { + return null; + } + + */ + @Override + public boolean resetSupported() { + // we're NOT supporting reset() here + return false; + } + + @Override + public boolean asyncSupported() { + return true; + } + + @Override + public void setPreProcessor(DataSetPreProcessor preProcessor) { + + } + + @Override + public boolean hasNext() { + // just checking if that's not the last iterator, or if that's the last one - check if it has something + return position.get() < iterators.size() - 1 + || (position.get() < iterators.size() && iterators.get(position.get()).hasNext()); + } + + @Override + public DataSet next() { + // TODO: this solution isn't ideal, it assumes non-empty iterators all the time. Would be nice to do something here + if (!iterators.get(position.get()).hasNext()) + position.getAndIncrement(); + + return iterators.get(position.get()).next(); + } + + @Override + public void remove() { + // no-op + } + + @Override + public void reset() { + throw new UnsupportedOperationException(); + } + + @Override + public DataSetPreProcessor getPreProcessor() { + // we probably don't need this thing here + return null; + } + + @Override + public int batch() { + throw new UnsupportedOperationException(); + } + + @Override + public int inputColumns() { + throw new UnsupportedOperationException(); + } + + @Override + public int totalOutcomes() { + throw new UnsupportedOperationException(); + } + + @Override + public DataSet next(int num) { + throw new UnsupportedOperationException(); + } + + @Override + public List getLabels() { + return null; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIterator.java new file mode 100644 index 000000000..9419e4d82 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIterator.java @@ -0,0 +1,76 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.iterators; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; +import java.util.function.Consumer; + +@Slf4j +public class VirtualIterator extends java.util.Observable implements Iterator { + // TODO: use AsyncIterator here? + protected Iterator iterator; + protected AtomicBoolean state = new AtomicBoolean(true); + + public VirtualIterator(@NonNull Iterator iterator) { + this.iterator = iterator; + } + + + @Override + public boolean hasNext() { + boolean u = iterator.hasNext(); + state.compareAndSet(true, u); + if (!state.get()) { + this.setChanged(); + notifyObservers(); + } + return u; + } + + @Override + public E next() { + return iterator.next(); + } + + @Override + public void remove() { + // no-op, we don't need this call implemented + } + + @Override + public void forEachRemaining(Consumer action) { + iterator.forEachRemaining(action); + state.compareAndSet(true, false); + } + + /** + * This method blocks until underlying Iterator is depleted + */ + public void blockUntilDepleted() { + while (state.get()) + LockSupport.parkNanos(1000L); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualMultiDataSetIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualMultiDataSetIterator.java new file mode 100644 index 000000000..55cd5625a --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualMultiDataSetIterator.java @@ -0,0 +1,118 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.iterators; + +import lombok.NonNull; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.MultiDataSetPreProcessor; +import org.nd4j.linalg.dataset.api.iterator.ParallelMultiDataSetIterator; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +public class VirtualMultiDataSetIterator implements ParallelMultiDataSetIterator { + + protected final List> iterators; + protected final AtomicInteger position; + + public VirtualMultiDataSetIterator(@NonNull List> iterators) { + this.iterators = iterators; + this.position = new AtomicInteger(0); + } + + @Override + public MultiDataSet next(int num) { + return next(); + } + + @Override + public void setPreProcessor(MultiDataSetPreProcessor preProcessor) { + + } + + @Override + public MultiDataSetPreProcessor getPreProcessor() { + return null; + } + + @Override + public boolean resetSupported() { + return false; + } + + @Override + public boolean asyncSupported() { + return true; + } + + @Override + public void reset() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasNext() { + // just checking if that's not the last iterator, or if that's the last one - check if it has something + boolean ret = position.get() < iterators.size() - 1 + || (position.get() < iterators.size() && iterators.get(position.get()).hasNext()); + return ret; + } + + @Override + public MultiDataSet next() { + // TODO: this solution isn't ideal, it assumes non-empty iterators all the time. Would be nice to do something here + if (!iterators.get(position.get()).hasNext()) + position.getAndIncrement(); + + return iterators.get(position.get()).next(); + } + + @Override + public void remove() { + // no-op + } + + @Override + public void attachThread(int producer) { + + } + + @Override + public boolean hasNextFor() { + return false; + } + + @Override + public boolean hasNextFor(int consumer) { + return false; + } + + @Override + public MultiDataSet nextFor(int consumer) { + return null; + } + + @Override + public MultiDataSet nextFor() { + return null; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/ElephasModelImport.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/ElephasModelImport.java new file mode 100644 index 000000000..43e2401f1 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/ElephasModelImport.java @@ -0,0 +1,177 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.modelimport.elephas; + +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.modelimport.keras.Hdf5Archive; +import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; +import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; +import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException; +import org.deeplearning4j.nn.modelimport.keras.utils.KerasModelUtils; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.threshold.FixedThresholdAlgorithm; +import org.deeplearning4j.spark.api.RDDTrainingApproach; +import org.deeplearning4j.spark.api.Repartition; +import org.deeplearning4j.spark.api.RepartitionStrategy; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.deeplearning4j.spark.impl.repartitioner.DefaultRepartitioner; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingMaster; +import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; + +import java.io.IOException; +import java.util.Map; + +/** + * Reads HDF5-persisted Elephas models stored with `model.save()` for both underlying + * `Sequential` and `Model` Keras models + * + * @author Max Pumperla + * + */ +public class ElephasModelImport { + + private static final String DISTRIBUTED_CONFIG = "distributed_config"; + private static final RDDTrainingApproach APPROACH = RDDTrainingApproach.Export; + + /** + * Load Elephas model stored using model.save(...) in case that the underlying Keras + * model is a functional `Model` instance, which corresponds to a DL4J SparkComputationGraph. + * + * @param sparkContext Java SparkContext + * @param modelHdf5Filename Path to HDF5 archive storing Elephas Model + * @return SparkComputationGraph Spark computation graph + * + * @throws IOException IO exception + * @throws InvalidKerasConfigurationException Invalid Keras config + * @throws UnsupportedKerasConfigurationException Unsupported Keras config + * @see SparkComputationGraph + */ + public static SparkComputationGraph importElephasModelAndWeights(JavaSparkContext sparkContext, + String modelHdf5Filename) + throws IOException, UnsupportedKerasConfigurationException, InvalidKerasConfigurationException { + ComputationGraph model = KerasModelImport.importKerasModelAndWeights(modelHdf5Filename, true); + + Map distributedProperties = distributedTrainingMap(modelHdf5Filename); + TrainingMaster tm = getTrainingMaster(distributedProperties); + + return new SparkComputationGraph(sparkContext, model, tm); + } + + /** + * Load Elephas model stored using model.save(...) in case that the underlying Keras + * model is a functional `Sequential` instance, which corresponds to a DL4J SparkDl4jMultiLayer. + * + * @param sparkContext Java SparkContext + * @param modelHdf5Filename Path to HDF5 archive storing Elephas model + * @return SparkDl4jMultiLayer Spark computation graph + * + * @throws IOException IO exception + * @throws InvalidKerasConfigurationException Invalid Keras config + * @throws UnsupportedKerasConfigurationException Unsupported Keras config + * @see SparkDl4jMultiLayer + */ + public static SparkDl4jMultiLayer importElephasSequentialModelAndWeights(JavaSparkContext sparkContext, + String modelHdf5Filename) + throws IOException, UnsupportedKerasConfigurationException, InvalidKerasConfigurationException { + MultiLayerNetwork model = KerasModelImport.importKerasSequentialModelAndWeights( + modelHdf5Filename, true); + + Map distributedProperties = distributedTrainingMap(modelHdf5Filename); + TrainingMaster tm = getTrainingMaster(distributedProperties); + + return new SparkDl4jMultiLayer(sparkContext, model, tm); + } + + private static Map distributedTrainingMap(String modelHdf5Filename) + throws UnsupportedKerasConfigurationException, IOException { + Hdf5Archive archive = new Hdf5Archive(modelHdf5Filename); + String initialModelJson = archive.readAttributeAsJson(DISTRIBUTED_CONFIG); + return KerasModelUtils.parseJsonString(initialModelJson); + } + + private static TrainingMaster getTrainingMaster(Map distributedProperties) + throws InvalidKerasConfigurationException { + Map innerConfig = (Map) distributedProperties.get("config"); + + Integer numWorkers = (Integer) innerConfig.get("num_workers"); + int batchSize = (int) innerConfig.get("batch_size"); + + String mode = "synchronous"; + if (innerConfig.containsKey("mode")) { + mode = (String) innerConfig.get("mode"); + } else { + throw new InvalidKerasConfigurationException("Couldn't find mode field."); + } + + // TODO: Create InvalidElephasConfigurationException + boolean collectStats = false; + if (innerConfig.containsKey("collect_stats")) + collectStats = (boolean) innerConfig.get("collect_stats"); + + int numBatchesPrefetch = 0; + if (innerConfig.containsKey("num_batches_prefetch")) + numBatchesPrefetch = (int) innerConfig.get("num_batches_prefetch"); + + + TrainingMaster tm; + if (mode.equals("synchronous")) { + int averagingFrequency = 5; + if (innerConfig.containsKey("averaging_frequency")) + averagingFrequency = (int) innerConfig.get("averaging_frequency"); + + tm = new ParameterAveragingTrainingMaster.Builder(numWorkers, batchSize) + .collectTrainingStats(collectStats) + .batchSizePerWorker(batchSize) + .averagingFrequency(averagingFrequency) + .workerPrefetchNumBatches(numBatchesPrefetch) + .aggregationDepth(2) // we leave this as default + .repartionData(Repartition.Always) + .rddTrainingApproach(APPROACH) + .repartitionStrategy(RepartitionStrategy.Balanced) + .saveUpdater(false) + .build(); + } else if (mode.equals("asynchronous")){ + double updateThreshold = 1e-3; + if (innerConfig.containsKey("update_threshold")) + updateThreshold = (double) innerConfig.get("update_threshold"); + ThresholdAlgorithm thresholdAlgorithm = new FixedThresholdAlgorithm(updateThreshold); + + VoidConfiguration voidConfiguration = VoidConfiguration.builder() + .build(); + tm = new SharedTrainingMaster.Builder(voidConfiguration, batchSize) + .thresholdAlgorithm(thresholdAlgorithm) + .batchSizePerWorker(batchSize) + .collectTrainingStats(collectStats) + .workerPrefetchNumBatches(numBatchesPrefetch) + .rddTrainingApproach(APPROACH) + .repartitioner(new DefaultRepartitioner()) + .build(); + } else { + throw new InvalidKerasConfigurationException("Unknown mode " + mode); + } + return tm; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/SilentTrainingDriver.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/SilentTrainingDriver.java new file mode 100644 index 000000000..64d83910f --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/SilentTrainingDriver.java @@ -0,0 +1,241 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v1; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.exception.DL4JInvalidConfigException; +import org.deeplearning4j.optimize.api.StepFunction; +import org.deeplearning4j.optimize.solvers.accumulation.FancyBlockingQueue; +import org.deeplearning4j.optimize.solvers.accumulation.GradientsAccumulator; +import org.deeplearning4j.optimize.solvers.accumulation.IndexedTail; +import org.deeplearning4j.spark.parameterserver.networking.v1.messages.SilentUpdatesMessage; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.compression.ThresholdCompression; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; +import org.nd4j.parameterserver.distributed.logic.Storage; +import org.nd4j.parameterserver.distributed.logic.completion.Clipboard; +import org.nd4j.parameterserver.distributed.messages.VoidAggregation; +import org.nd4j.parameterserver.distributed.training.TrainingDriver; +import org.nd4j.parameterserver.distributed.transport.Transport; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +@Deprecated +public class SilentTrainingDriver implements TrainingDriver { + protected transient INDArray params; + protected transient INDArray updates; + protected transient StepFunction stepFunction; + + protected transient GradientsAccumulator accumulator; + + protected transient VoidConfiguration voidConfiguration; + protected transient Transport transport; + protected transient AtomicLong updatesCount; + protected transient AtomicBoolean hasSomething; + + protected transient AtomicBoolean bypassMode = new AtomicBoolean(false); + + protected transient AtomicLong denseCounter = new AtomicLong(0); + protected transient AtomicLong sparseCounter = new AtomicLong(0); + + /* + We use this buffer to provide double buffering for incoming messages. + So we store incoming messages right here, and apply them as time comes + */ + protected transient IndexedTail updatesBuffer; + + // these 2 are not used here + protected transient Storage storage; + protected transient Clipboard clipboard; + + + public SilentTrainingDriver(@NonNull GradientsAccumulator accumulator) { + log.info("Creating TrainingDriver for worker..."); + this.accumulator = accumulator; + this.updatesCount = new AtomicLong(0); + + // TODO: make this configurable + this.updatesBuffer = new IndexedTail(1); + + // FBQ will guarantee that all workers using given queue will be applying the same updates in the same order + this.accumulator.setExternalSource(updatesBuffer); + } + + public SilentTrainingDriver(@NonNull INDArray params, @NonNull StepFunction stepFunction) { + log.info("Creating TrainingDriver for master..."); + log.info("Params at Master BEFORE: {}", params.meanNumber().doubleValue()); + this.params = params; + this.stepFunction = stepFunction; + this.updatesCount = new AtomicLong(0); + + this.hasSomething = new AtomicBoolean(false); + + // updates are always the same size as params + try (MemoryWorkspace wsO = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) { + this.updates = Nd4j.create(params.shape(), params.ordering()); + } + } + + /** + * This method is viable only at Spark Workers, Master node will always have empty buffer here by design + * @return + */ + public IndexedTail getUpdatesBuffer() { + return updatesBuffer; + } + + @Override + public void init(@NonNull VoidConfiguration voidConfiguration, @NonNull Transport transport, Storage storage, + Clipboard clipboard) { + this.voidConfiguration = voidConfiguration; + this.transport = transport; + } + + public void bypassMode(boolean reallyBypass) { + bypassMode.set(reallyBypass); + + // if TrainingDriver is temporary disabled - remove existing messages from queue + if (reallyBypass) { + //updatesBuffer.clear(); + } + } + + @Override + public void startTraining(SilentUpdatesMessage message) { + /* + this method will be invoked on master, and will do 2 things: + 1) silently update params via given StepFunction + 2) propagate this message to everyone + + on workers, it just enqueues updates into the FancyBlockingQueue + */ + // if accumulator is defined, we're working at Worker level, so it's not our problem what happens inside + if (accumulator != null) { + if (message.getOriginatorId() == transport.getOwnOriginatorId()) { + //log.info("Skipping since originators match"); + return; + } ; + + /* + we're just putting messages here. if thread gets blocked - messages won't be arriving, + enforcing periodic messages retransmission from other nodes, so we should be all fine + */ + + try { + if (!bypassMode.get()) { + updatesBuffer.put(message.getUpdates()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + //accumulator.receiveUpdate(message.getUpdates()); + } else if (params != null && stepFunction != null) { + // master invokes everything, since that's Silent Worker approach: we want master to be always up-to-date + synchronized (this) { + // threshold decoder is inplace & fast + int encoding = message.getUpdates().data().getInt(3); + if (encoding == ThresholdCompression.FLEXIBLE_ENCODING) { + Nd4j.getExecutioner().thresholdDecode(message.getUpdates(), updates); + sparseCounter.incrementAndGet(); + } else if (encoding == ThresholdCompression.BITMAP_ENCODING) { + Nd4j.getExecutioner().bitmapDecode(message.getUpdates(), updates); + denseCounter.incrementAndGet(); + } else + throw new DL4JInvalidConfigException("Unknown compression header received: " + encoding); + + /* + if ((sparseCounter.get() + denseCounter.get()) % 100 == 0) { + log.info("Sparse/Dense ratio: {}", String.format("%.2f", (sparseCounter.get() +1) / (double) (denseCounter.get() + 1))); + } + */ + + + // this simple flag shows that we have something not applied, will be used at finishTraining() method + hasSomething.set(true); + + // we apply updates every X iterations, and we don't really need X to be small here + if (updatesCount.incrementAndGet() % Math.max(transport.numberOfKnownClients(), 5) == 0) { + stepFunction.step(params, updates); + + // once accumulated updates are applied - reset storage, and wait for other messsages + Nd4j.getMemoryManager().memset(updates); + hasSomething.set(false); + } + } + + // we should echo this message to everyone but this shard, but only if there's > 1 shard/client available + if (transport.numberOfKnownClients() > 1) { + //log.info("Resending message, skipping {}", message.getOriginatorId()); + transport.sendMessageToAllClients(message, message.getOriginatorId(), transport.getOwnOriginatorId()); + } // else log.info("No known Clients so far"); + } else + throw new DL4JInvalidConfigException("Neither GradientsAccumulator or StepFunction is defined!"); + } + + @Override + public void pickTraining(SilentUpdatesMessage message) { + throw new UnsupportedOperationException(); + } + + @Override + public void aggregationFinished(VoidAggregation aggregation) { + throw new UnsupportedOperationException(); + } + + /** + * This method is used on Master only, applies buffered updates to params + * + * @param originatorId + * @param taskId + */ + @Override + public void finishTraining(long originatorId, long taskId) { + // on Master thread we'll be applying final gradients + + if (params != null && stepFunction != null) { + if (hasSomething.get()) { + stepFunction.step(params, updates); + //Nd4j.getMemoryManager().memset(updates); + updates.assign(0.0); + } + } + + } + + @Override + public void addCompletionHook(long originatorId, long frameId, long messageId) { + // no-op + throw new UnsupportedOperationException(); + } + + @Override + public String targetMessageClass() { + return SilentUpdatesMessage.class.getSimpleName(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/WiredEncodingHandler.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/WiredEncodingHandler.java new file mode 100644 index 000000000..2ae4767af --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/WiredEncodingHandler.java @@ -0,0 +1,73 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v1; + +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.optimize.solvers.accumulation.EncodingHandler; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ResidualPostProcessor; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; +import org.deeplearning4j.spark.parameterserver.networking.v1.messages.SilentUpdatesMessage; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.parameterserver.distributed.VoidParameterServer; + +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +@Deprecated +public class WiredEncodingHandler extends EncodingHandler { + protected AtomicLong updatesCounter = new AtomicLong(0); + + /** + * This method builds new WiredEncodingHandler instance + * + * @param thresholdAlgorithm threshold algorithm to use + * @param boundary + */ + public WiredEncodingHandler(ThresholdAlgorithm thresholdAlgorithm, ResidualPostProcessor residualPostProcessor, Integer boundary, boolean encodingDebugMode) { + super(thresholdAlgorithm, residualPostProcessor, boundary, encodingDebugMode); + } + + /** + * This method sends given message to all registered recipients + * + * @param message + */ + @Override + protected void sendMessage(INDArray message, int iterationNumber, int epochNumber) { + // here we'll send our stuff to other executores over the wire + // and let's pray for udp broadcast availability + + // Send this message away + // FIXME: do something with unsafe duplication, which is bad and used ONLY for local spark + try (MemoryWorkspace wsO = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) { + long updateId = updatesCounter.getAndIncrement(); + + VoidParameterServer.getInstance().execDistributedImmediately( + new SilentUpdatesMessage(message.unsafeDuplication(), updateId)); + } + + + // heere we update local queue + super.sendMessage(message, iterationNumber, epochNumber); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentIntroductoryConfirmation.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentIntroductoryConfirmation.java new file mode 100644 index 000000000..09f398eb9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentIntroductoryConfirmation.java @@ -0,0 +1,32 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v1.messages; + +import org.nd4j.parameterserver.distributed.messages.BaseVoidMessage; + +public class SilentIntroductoryConfirmation extends BaseVoidMessage { + @Override + public void processMessage() { + /* + we just want to get clearance before training starts here + */ + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentIntroductoryMessage.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentIntroductoryMessage.java new file mode 100644 index 000000000..4631bb141 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentIntroductoryMessage.java @@ -0,0 +1,60 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v1.messages; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.parameterserver.distributed.messages.BaseVoidMessage; +import org.nd4j.parameterserver.distributed.messages.DistributedMessage; + +@Slf4j +public class SilentIntroductoryMessage extends BaseVoidMessage implements DistributedMessage { + protected String localIp; + protected int port; + + protected SilentIntroductoryMessage() { + // + } + + public SilentIntroductoryMessage(@NonNull String localIP, int port) { + this.localIp = localIP; + this.port = port; + } + + @Override + public void processMessage() { + /* + basically we just want to send our IP, and get our new shardIndex in return. haha. bad idea obviously, but still... + + or, we can skip direct addressing here, use passive addressing instead, like in client mode? + */ + + log.info("Adding client {}:{}", localIp, port); + //transport.addShard(localIp, port); + transport.addClient(localIp, port); + } + + @Override + public boolean isBlockingMessage() { + // this is blocking message, we want to get reply back before going further + return true; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentUpdatesMessage.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentUpdatesMessage.java new file mode 100644 index 000000000..94b4fb766 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v1/messages/SilentUpdatesMessage.java @@ -0,0 +1,89 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v1.messages; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; +import org.nd4j.parameterserver.distributed.enums.NodeRole; +import org.nd4j.parameterserver.distributed.logic.Storage; +import org.nd4j.parameterserver.distributed.logic.completion.Clipboard; +import org.nd4j.parameterserver.distributed.messages.BaseVoidMessage; +import org.nd4j.parameterserver.distributed.messages.RequestMessage; +import org.nd4j.parameterserver.distributed.messages.TrainingMessage; +import org.nd4j.parameterserver.distributed.training.TrainingDriver; +import org.nd4j.parameterserver.distributed.transport.Transport; + +@Slf4j +public class SilentUpdatesMessage extends BaseVoidMessage implements TrainingMessage, RequestMessage { + + @Getter + protected long updateId; + @Getter + protected INDArray updates; + protected long frameId; + + protected SilentUpdatesMessage() { + // just for ser/de + } + + public SilentUpdatesMessage(INDArray encodedUpdates, long updateId) { + this.updates = encodedUpdates; + this.updateId = updateId; + } + + + @Override + public void attachContext(VoidConfiguration voidConfiguration, TrainingDriver trainer, + Clipboard clipboard, Transport transport, Storage storage, NodeRole role, short shardIndex) { + this.voidConfiguration = voidConfiguration; + this.trainer = trainer; + this.transport = transport; + } + + @Override + public void processMessage() { + // basically no-op? + TrainingDriver tr = (TrainingDriver) trainer; + tr.startTraining(this); + } + + @Override + public byte getCounter() { + return 0; + } + + @Override + public long getFrameId() { + return frameId; + } + + @Override + public void setFrameId(long frameId) { + this.frameId = frameId; + } + + @Override + public boolean isJoinSupported() { + return false; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/ModelParamsConsumer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/ModelParamsConsumer.java new file mode 100644 index 000000000..49e07c5e8 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/ModelParamsConsumer.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v2; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.function.Supplier; +import org.nd4j.common.primitives.Atomic; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +@Slf4j +public class ModelParamsConsumer implements Subscriber, Supplier { + protected transient final Atomic params = new Atomic<>(); + + @Override + public void onSubscribe(Subscription subscription) { + // no-op + } + + @Override + public synchronized void onNext(@NonNull INDArray array) { + log.info("Storing params for future use..."); + if (array != null) + params.set(array); + } + + @Override + public void onError(Throwable throwable) { + throw new RuntimeException(throwable); + } + + @Override + public void onComplete() { + // no-op + } + + @Override + public INDArray get() { + return params.get(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/UpdaterParamsConsumer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/UpdaterParamsConsumer.java new file mode 100644 index 000000000..a2f074cba --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/UpdaterParamsConsumer.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v2; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.function.Supplier; +import org.nd4j.common.primitives.Atomic; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +@Slf4j +public class UpdaterParamsConsumer implements Subscriber, Supplier { + protected transient final Atomic params = new Atomic<>(); + + @Override + public void onSubscribe(Subscription subscription) { + // no-op + } + + @Override + public synchronized void onNext(INDArray array) { + if (array != null) + params.set(array); + } + + @Override + public void onError(Throwable throwable) { + throw new RuntimeException(throwable); + } + + @Override + public void onComplete() { + // no-op + } + + @Override + public INDArray get() { + return params.get(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/UpdatesConsumer.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/UpdatesConsumer.java new file mode 100644 index 000000000..22b358e41 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/UpdatesConsumer.java @@ -0,0 +1,183 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v2; + +import io.reactivex.functions.Consumer; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.exception.DL4JInvalidConfigException; +import org.deeplearning4j.optimize.api.StepFunction; +import org.deeplearning4j.optimize.solvers.accumulation.FancyBlockingQueue; +import org.deeplearning4j.optimize.solvers.accumulation.GradientsAccumulator; +import org.deeplearning4j.optimize.solvers.accumulation.IndexedTail; +import org.deeplearning4j.optimize.solvers.accumulation.SmartFancyBlockingQueue; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.compression.ThresholdCompression; +import org.nd4j.linalg.exception.ND4JIllegalStateException; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.parameterserver.distributed.v2.transport.UpdatesHandler; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Slf4j +public class UpdatesConsumer implements UpdatesHandler { + protected int numWorkers; + + protected transient INDArray params; + protected transient INDArray updates; + protected transient StepFunction stepFunction; + + protected transient GradientsAccumulator accumulator; + + protected transient final AtomicLong updatesCount = new AtomicLong(0); + protected transient final AtomicBoolean hasSomething = new AtomicBoolean(false); + protected transient final AtomicBoolean bypassMode = new AtomicBoolean(false); + protected transient final AtomicLong denseCounter = new AtomicLong(0); + protected transient final AtomicLong sparseCounter = new AtomicLong(0); + + // make this stuff configurable + protected transient IndexedTail updatesBuffer; + + @Override + public void onSubscribe(Subscription subscription) { + // no-op + } + + /** + * This + * @param reallBypass + */ + public void bypassMode(boolean reallBypass) { + bypassMode.set(reallBypass); + } + + /** + * + * @return + */ + public boolean isBypassMod() { + return bypassMode.get(); + } + + public IndexedTail getUpdatesQueue() { + if (updatesBuffer == null && accumulator != null) { + synchronized (this) { + if (updatesBuffer == null) { + updatesBuffer = new IndexedTail(numWorkers, true, params.shape()); + } + } + } + + return updatesBuffer; + } + + @Override + public void onNext(INDArray array) { + if (updatesBuffer == null && accumulator != null) { + synchronized (this) { + if (updatesBuffer == null) { + updatesBuffer = new IndexedTail(numWorkers, true, params.shape()); + } + } + } + + if (!bypassMode.get()) { + if (accumulator != null) { + // this means consumer runs on worker node + + try { + // we're just storing update into buffer, and it'll be consumed by GradientsAccumulator on next cycle + //log.info("Putting update to the queue, current size: [{}]", updatesBuffer.size()); + updatesBuffer.put(array); + } catch (Exception e) { + log.error("",e); + throw new RuntimeException(e); + } + } else if (params != null && stepFunction != null) { + synchronized (this) { + // threshold decoder is inplace & fast + int encoding = array.data().getInt(3); + if (encoding == ThresholdCompression.FLEXIBLE_ENCODING) { + Nd4j.getExecutioner().thresholdDecode(array, updates); + sparseCounter.incrementAndGet(); + } else if (encoding == ThresholdCompression.BITMAP_ENCODING) { + Nd4j.getExecutioner().bitmapDecode(array, updates); + denseCounter.incrementAndGet(); + } else + throw new DL4JInvalidConfigException("Unknown compression header received: " + encoding); + + + // this simple flag shows that we have something not applied, will be used at finishTraining() method + hasSomething.set(true); + + // we apply updates every X iterations, and we don't really need X to be small here + if (updatesCount.incrementAndGet() % 32 == 0) { + flush(); + } + } + } else + throw new ND4JIllegalStateException("Accumulator & StepFunction is null at the same time"); + } + } + + public void flush() { + synchronized (this) { + if (params != null && updates != null && hasSomething.get()) { + stepFunction.step(params, updates); + Nd4j.getExecutioner().commit(); + + log.debug("Applying updates. Current ratio: [{}]; Sparse: [{}]; Dense: [{}];", (double) sparseCounter.get() / denseCounter.get(), sparseCounter.get(), denseCounter.get()); + + // once accumulated updates are applied - reset storage, and wait for other messsages + Nd4j.getMemoryManager().memset(updates); + hasSomething.set(false); + } + } + } + + @Override + public void onError(Throwable throwable) { + throw new RuntimeException(throwable); + } + + @Override + public void onComplete() { + // no-op + } + + @Override + public INDArray getParametersArray() { + synchronized (this) { + return params.dup(params.ordering()); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/WiredEncodingHandler.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/WiredEncodingHandler.java new file mode 100644 index 000000000..2f65919f0 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/networking/v2/WiredEncodingHandler.java @@ -0,0 +1,72 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.networking.v2; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.deeplearning4j.optimize.solvers.accumulation.EncodingHandler; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ResidualPostProcessor; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.parameterserver.distributed.v2.ModelParameterServer; + +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +public class WiredEncodingHandler extends EncodingHandler { + protected AtomicLong updatesCounter = new AtomicLong(0); + + /** + * This method builds new WiredEncodingHandler instance + * + * @param thresholdAlgorithm The threshold algorithm to use + */ + public WiredEncodingHandler(ThresholdAlgorithm thresholdAlgorithm, ResidualPostProcessor residualPostProcessor, Integer boundary, boolean encodingDebugMode) { + super(thresholdAlgorithm, residualPostProcessor, boundary, encodingDebugMode); + } + + /** + * This method sends given message to all registered recipients + * + * @param message + */ + @Override + protected void sendMessage(@NonNull INDArray message, int iterationNumber, int epochNumber) { + // here we'll send our stuff to other executores over the wire + // and let's pray for udp broadcast availability + + // Send this message away + // FIXME: do something with unsafe duplication, which is bad and used ONLY for local spark + try (MemoryWorkspace wsO = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) { + long updateId = updatesCounter.getAndIncrement(); + + val m = message.unsafeDuplication(); + ModelParameterServer.getInstance().sendUpdate(m, iterationNumber, epochNumber); + } + + + // heere we update local queue + super.sendMessage(message, iterationNumber, epochNumber); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/pw/SharedTrainingWrapper.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/pw/SharedTrainingWrapper.java new file mode 100644 index 000000000..f3f2cee80 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/pw/SharedTrainingWrapper.java @@ -0,0 +1,579 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.pw; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.bytedeco.javacpp.Loader; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.listener.RoutingIterationListener; +import org.deeplearning4j.common.config.DL4JEnvironmentVars; +import org.deeplearning4j.exception.DL4JInvalidConfigException; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.api.Updater; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.updater.BaseMultiLayerUpdater; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.optimize.listeners.SleepyTrainingListener; +import org.deeplearning4j.optimize.solvers.accumulation.EncodedGradientsAccumulator; +import org.deeplearning4j.optimize.solvers.accumulation.EncodingHandler; +import org.deeplearning4j.parallelism.ParallelWrapper; +import org.deeplearning4j.spark.parameterserver.conf.SharedTrainingConfiguration; +import org.deeplearning4j.spark.parameterserver.iterators.VirtualDataSetIterator; +import org.deeplearning4j.spark.parameterserver.iterators.VirtualIterator; +import org.deeplearning4j.spark.parameterserver.iterators.VirtualMultiDataSetIterator; +import org.deeplearning4j.spark.parameterserver.networking.v2.ModelParamsConsumer; +import org.deeplearning4j.spark.parameterserver.networking.v2.UpdaterParamsConsumer; +import org.deeplearning4j.spark.parameterserver.networking.v2.UpdatesConsumer; +import org.deeplearning4j.spark.parameterserver.networking.v2.WiredEncodingHandler; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingResult; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingWorker; +import org.deeplearning4j.spark.parameterserver.util.BlockingObserver; +import org.deeplearning4j.spark.parameterserver.util.CountingIterator; +import org.deeplearning4j.spark.util.SparkUtils; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; +import org.nd4j.parameterserver.distributed.enums.TransportType; +import org.nd4j.parameterserver.distributed.util.NetworkOrganizer; +import org.nd4j.parameterserver.distributed.v2.ModelParameterServer; +import org.nd4j.parameterserver.distributed.v2.transport.UpdaterParametersProvider; +import org.nd4j.parameterserver.distributed.v2.transport.impl.AeronUdpTransport; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +public class SharedTrainingWrapper { + private static SharedTrainingWrapper INSTANCE = new SharedTrainingWrapper(); + private static AtomicLong LAST_INSTANCE_ID = new AtomicLong(Long.MIN_VALUE); + protected ParallelWrapper wrapper; + protected VirtualDataSetIterator iteratorDS; + protected VirtualMultiDataSetIterator iteratorMDS; + + protected List> iteratorsDS; + protected List> iteratorsMDS; + + + protected AtomicBoolean isFirst = new AtomicBoolean(false); + protected AtomicBoolean exceptionEncountered = new AtomicBoolean(false); + protected Throwable exception; + + protected ThreadLocal iteratorDataSetCount = new ThreadLocal<>(); //Using AtomicInteger because it's mutable, not because it's atomic + protected ThreadLocal observer = new ThreadLocal<>(); + protected EncodedGradientsAccumulator accumulator; + protected Model originalModel; + + protected UpdatesConsumer consumer; + + protected SharedTrainingWrapper() { + init(); + } + + protected void init() { + // instantiate some stuff here + iteratorsDS = new CopyOnWriteArrayList<>(); + iteratorsMDS = new CopyOnWriteArrayList<>(); + + // now we're creating DataSetIterators, to feed ParallelWrapper + iteratorDS = new VirtualDataSetIterator(iteratorsDS); + iteratorMDS = new VirtualMultiDataSetIterator(iteratorsMDS); + } + + public static synchronized SharedTrainingWrapper getInstance(long id) { + if(LAST_INSTANCE_ID.get() != Long.MIN_VALUE && LAST_INSTANCE_ID.get() != id){ + log.debug("Shutting down existing SharedTrainingWrapper instances; resetting state - previous instance ID {}," + + " new instance ID {}", LAST_INSTANCE_ID.get(), id); + if(INSTANCE.wrapper != null){ + INSTANCE.wrapper.shutdown(); + INSTANCE.wrapper = null; + } + INSTANCE.iteratorsDS.clear(); + INSTANCE.iteratorsMDS.clear(); + INSTANCE.exceptionEncountered.set(false); + INSTANCE.iteratorDataSetCount = new ThreadLocal<>(); + INSTANCE.accumulator = null; + INSTANCE.originalModel = null; + INSTANCE.consumer = null; + LAST_INSTANCE_ID.set(id); + } + + if(LAST_INSTANCE_ID.get() == Long.MIN_VALUE){ + LAST_INSTANCE_ID.set(id); + } + + return INSTANCE; + } + + /** + * This method registers given Iterable in VirtualDataSetIterator + * + * @param iterator + */ + public void attachDS(Iterator iterator) { + log.debug("Attaching thread..."); + + //Count the number of minibatches - used for reporting/debugging purposes + if(iteratorDataSetCount.get() == null) + iteratorDataSetCount.set(new AtomicInteger(0)); + AtomicInteger count = iteratorDataSetCount.get(); + count.set(0); + + // we're creating our Observable wrapper + VirtualIterator wrapped = new VirtualIterator<>(new CountingIterator<>(iterator, count)); + + // and creating Observer which will be used to monitor progress within iterator + BlockingObserver obs = new BlockingObserver(exceptionEncountered); + wrapped.addObserver(obs); + + // putting that "somewhere" + iteratorsDS.add(wrapped); + + // storing observer into ThreadLocal, since we're going to use that later + observer.set(obs); + } + + /** + * This method registers given Iterable in VirtualMultiDataSetIterator + * + * @param iterator + */ + public void attachMDS(Iterator iterator) { + log.debug("Attaching thread..."); + + //Count the number of minibatches - used for reporting/debugging purposes + if(iteratorDataSetCount.get() == null) + iteratorDataSetCount.set(new AtomicInteger(0)); + AtomicInteger count = iteratorDataSetCount.get(); + count.set(0); + + // we're creating our Observable wrapper + VirtualIterator wrapped = new VirtualIterator<>(new CountingIterator<>(iterator, count)); + + // and creating Observer which will be used to monitor progress within iterator + BlockingObserver obs = new BlockingObserver(exceptionEncountered); + wrapped.addObserver(obs); + + // putting that "somewhere" + iteratorsMDS.add(wrapped); + + // storing observer into ThreadLocal, since we're going to use that later + observer.set(obs); + } + + public SharedTrainingResult run(SharedTrainingWorker worker) { + /* + first call instantiates pw, messenger etc, and gets in charge here. + */ + if (isFirst.compareAndSet(false, true)) { + //Reset past exception encountered in case we're doing correct fit after incorrect... + exceptionEncountered.set(false); + exception = null; + + SharedTrainingConfiguration trainingConfiguration = worker.getBroadcastConfiguration().getValue(); + VoidConfiguration voidConfiguration = worker.getBroadcastConfiguration().getValue().getVoidConfiguration(); + + Model model = null; + + /* + Plan is simple here: if there's defined field in SharedTrainingConfiguration - use that. + If no - try to guess something + */ + int numDevices = Nd4j.getAffinityManager().getNumberOfDevices(); + + int numCores = Loader.totalCores(); + + /** + * Logic here is simple: + * 1) If user had specified number of workers per node - use that value + * 2) If not, and there's > 1 devices in system (as in Multi-GPU system) - use numberOfDevices as number of workers + * 3) otherwise, let's assume that's regular multi-core node, so we'll use 1..6 workers, depending on number of cores/4 + */ + int numWorkers = trainingConfiguration.getNumberOfWorkersPerNode() > 0 + ? trainingConfiguration.getNumberOfWorkersPerNode() + : numDevices > 1 ? numDevices : Math.min(6, Math.max(1, numCores / 4)); + + if (numDevices > 1 && numWorkers > numDevices) + log.warn("WARNING! Using more workers then number of available computational devices!"); + + + + // now we're attaching VoidParameterServer to GradientsAccumulator, but doing that only once + if (wrapper == null) { + log.debug("Starting ParallelWrapper at thread {}", Thread.currentThread().getId()); + + model = worker.getInitialModel(); + if (model == null) { + model = worker.getInitialModelGraph(); + } + + if (model == null) + throw new DL4JInvalidConfigException("No model was defined for training"); + + List listeners = worker.getListeners(); + if(listeners != null){ + model.setListeners(listeners); + StatsStorageRouter r = worker.getRouter(); + if(r != null){ + for(TrainingListener l : listeners){ + if(l instanceof RoutingIterationListener){ + ((RoutingIterationListener) l).setStorageRouter(r); + } + } + } + } + + val handler = new WiredEncodingHandler(trainingConfiguration.getThresholdAlgorithm(), trainingConfiguration.getResidualPostProcessor(), null, trainingConfiguration.isEncodingDebugMode()); + + // TODO: if there will be no code difference - use the same class instead of 2 different classes + val modelParamsSupplier = new ModelParamsConsumer(); + val updateParamsSupplier = new UpdaterParamsConsumer(); + + // this accumulator will provide sharing gradients over network, via WiredEncodedHandler. But we create it only once + if (accumulator == null) { + /** + * We know, that updates are guaranteed to have MAX size of params / 16. So, here we go. + * I.e. for model with 100m params, that's 400m of floats (or 800m of doubles) + * The worst case for us is bitmap encoding, that takes 2 bits to encode each gradient value + * + * so, for float in worst case we'll have (100m / 16) int elements. So, our buffer size will be 6.25m * queueSize * 4 bytes per int + */ + + int queueSize = numWorkers * 2; + + val bufferSize = trainingConfiguration.getBufferSize() > 0 ? trainingConfiguration.getBufferSize() + : EncodedGradientsAccumulator.getOptimalBufferSize(model, numWorkers, 2); + + accumulator = new EncodedGradientsAccumulator.Builder(numWorkers).messageHandler(handler) + .thresholdAlgorithm(trainingConfiguration.getThresholdAlgorithm()) + .residualPostProcessor(trainingConfiguration.getResidualPostProcessor()) + .memoryParameters(bufferSize, queueSize) + .encodingDebugMode(trainingConfiguration.isEncodingDebugMode()) + .build(); + + // we should introduce ourselves to controller + // FIXME: if localIP is null - use original ip discovery available in VoidParameterServer + String localIP = null; + + // picking IP address based on network mask + if (localIP == null && voidConfiguration.getNetworkMask() != null) { + NetworkOrganizer organizer = new NetworkOrganizer(voidConfiguration.getNetworkMask()); + localIP = organizer.getMatchingAddress(); + } + + // last resort here... + if (localIP == null) + localIP = System.getenv(DL4JEnvironmentVars.DL4J_VOID_IP); + + // set it to localhost, and hope for BroadcastTransport used + if (localIP == null) { + localIP = "127.0.0.1"; + log.warn("Can't get IP address to start VoidParameterServer client. Using localhost instead"); + } + + log.debug("Checking for ModelParameterServer existence"); + + // we're saving reference to original model + originalModel = model; + + // if we're running in spark localhost mode - we don't want double initialization + if (!ModelParameterServer.getInstance().isInitialized()) { + log.info("Initializing transport [{}:{}] with root as [{}:{}]...", localIP, voidConfiguration.getPortSupplier().getPort(), + voidConfiguration.getControllerAddress(), voidConfiguration.getUnicastControllerPort()); + // FIXME: implement support for Custom transport implementation + + val transport = voidConfiguration.getTransportType() == TransportType.ROUTED_UDP ? new AeronUdpTransport(localIP, voidConfiguration.getPortSupplier().getPort(), + voidConfiguration.getControllerAddress(), voidConfiguration.getUnicastControllerPort(), voidConfiguration) : null; + + if (transport == null) + throw new DL4JInvalidConfigException( + "No Transport implementation was defined for this training session!"); + + consumer = UpdatesConsumer.builder() + .numWorkers(numWorkers) + .accumulator(accumulator) + .params(model.params()) + .build(); + + accumulator.setExternalSource(consumer.getUpdatesQueue()); + + log.debug("Configuring transport..."); + // pass values right away + ModelParameterServer.getInstance().configure(voidConfiguration, transport, new UpdaterParametersProvider() { + @Override + public INDArray getUpdaterParameters() { + log.info("Serving updater parameters..."); + Updater updater = null; + if (originalModel instanceof MultiLayerNetwork) { + updater = ((MultiLayerNetwork) originalModel).getUpdater(); + } else if (originalModel instanceof ComputationGraph) { + updater = ((ComputationGraph) originalModel).getUpdater(); + } + + if (updater != null) { + if (updater instanceof BaseMultiLayerUpdater) { + return ((BaseMultiLayerUpdater) updater).getStateViewArrayCopy(); + } else { + log.error("Updater doesn't implement getStateViewArrayCopy()"); + return null; + } + } else { + log.warn("No Updater in the model"); + return null; + } + }; + }); + + ModelParameterServer.getInstance().addUpdatesSubscriber(consumer); + ModelParameterServer.getInstance().addModelParamsSubscriber(modelParamsSupplier); + ModelParameterServer.getInstance().addUpdaterParamsSubscriber(updateParamsSupplier); + } + + log.debug("Starting ModelParameterServer..."); + // after initialization finished, we're ok to actually start training + ModelParameterServer.getInstance().launch(); + + // waiting for introduction. probably no-op in 99.9999% cases + while (!ModelParameterServer.getInstance().getTransport().isIntroduced()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + // propagate iteration/epoch numbers + if (originalModel instanceof MultiLayerNetwork) { + ((MultiLayerNetwork) model).setIterationCount(ModelParameterServer.getInstance().getStartPosition().getFirst()); + ((MultiLayerNetwork) model).setEpochCount(ModelParameterServer.getInstance().getStartPosition().getSecond()); + } else if (originalModel instanceof ComputationGraph) { + ((ComputationGraph) model).getConfiguration().setIterationCount(ModelParameterServer.getInstance().getStartPosition().getFirst()); + ((ComputationGraph) model).getConfiguration().setEpochCount(ModelParameterServer.getInstance().getStartPosition().getSecond()); + } + + // if we're going to extend iteratation for debugging purposes - let's do that here + if (trainingConfiguration.getDebugLongerIterations() > 0) { + log.warn("Adding SleepyListener: {} ms", trainingConfiguration.getDebugLongerIterations()); + model.addListeners(SleepyTrainingListener.builder() + .timerIteration(trainingConfiguration.getDebugLongerIterations()).build()); + } + + // :) + accumulator.markExternalUpdates(true); + + // we're launching PW only if number of workers is more then 1 + if (numWorkers > 1) { + //log.info("Params at PW: {mean: [{}]; stdev: [{}]}", originalModel.params().meanNumber().doubleValue(), originalModel.params().stdNumber().doubleValue()); + + wrapper = new ParallelWrapper.Builder<>(originalModel) + .workers(numWorkers) + .workspaceMode(trainingConfiguration.getWorkspaceMode()) + .trainingMode(ParallelWrapper.TrainingMode.CUSTOM) + .gradientsAccumulator(accumulator) + .prefetchBuffer(trainingConfiguration.getPrefetchSize()) + .modelParamsSupplier(modelParamsSupplier) + .updaterParamsSupplier(updateParamsSupplier) + .thresholdAlgorithm(trainingConfiguration.getThresholdAlgorithm()) + .residualPostProcessor(trainingConfiguration.getResidualPostProcessor()) + .build(); + wrapper.setExceptionEncountered(exceptionEncountered); + } else { + log.debug("Using standalone model instead..."); + + // since there'll be only one consumer, we don't need complex sync logic anymore + accumulator.fallbackToSingleConsumerMode(true); + accumulator.touch(); + + // checking if there were updated params received (i.e. if that's failover routine + val mParams = modelParamsSupplier.get(); + if (mParams != null) { + log.info("Updating model params to the most recent ones..."); + originalModel.params().assign(mParams); + } + + // ok. attaching accumulator to model + if (model instanceof ComputationGraph) { + ((ComputationGraph) originalModel).getConfiguration() + .setTrainingWorkspaceMode(trainingConfiguration.getWorkspaceMode()); + ((ComputationGraph) originalModel).setGradientsAccumulator(accumulator); + } else if (model instanceof MultiLayerNetwork) { + ((MultiLayerNetwork) originalModel).getLayerWiseConfigurations() + .setTrainingWorkspaceMode(trainingConfiguration.getWorkspaceMode()); + ((MultiLayerNetwork) originalModel).setGradientsAccumulator(accumulator); + } + } + } + + // TODO: optionally we might be waiting until we have >1 splits delivered + + + if (consumer != null) + consumer.bypassMode(false); + + // now we're just calling for fit + if(iteratorDS == null && iteratorMDS == null) + throw new DL4JInvalidConfigException("No iterators were defined for training"); + + try { + boolean dsNext; + boolean mdsNext; + while((dsNext = iteratorDS != null && iteratorDS.hasNext()) || (mdsNext = iteratorMDS != null && iteratorMDS.hasNext())) { + //Loop as a guard against concurrent modifications and RCs + + if (wrapper != null) { + if (dsNext) + wrapper.fit(iteratorDS); + else + wrapper.fit(iteratorMDS); + } else { + // if wrapper is null, we're fitting standalone model then + if (dsNext) { + if (model instanceof ComputationGraph) { + ((ComputationGraph) originalModel).fit(iteratorDS); + } else if (model instanceof MultiLayerNetwork) { + ((MultiLayerNetwork) originalModel).fit(iteratorDS); + } + } else { + if (model instanceof ComputationGraph) { + ((ComputationGraph) originalModel).fit(iteratorMDS); + } else if (model instanceof MultiLayerNetwork) { + ((MultiLayerNetwork) originalModel).fit(iteratorMDS); + } + } + } + + if(consumer != null) + consumer.getUpdatesQueue().purge(); + } + } catch (Throwable t){ + log.warn("Exception encountered during fit operation", t); + exceptionEncountered.set(true); + exception = t; + } + + + // conditionally shutdown & reset ParallelWrapper + EncodedGradientsAccumulator accum; + if(wrapper != null){ + accum = (EncodedGradientsAccumulator) wrapper.getGradientsAccumulator(); //Store before possible shutdown for below + } else { + accum = accumulator; + } + if (trainingConfiguration.isEpochReset()) { + wrapper.shutdown(); + wrapper = null; + } + + // reset iterators too + init(); + + // and accumulator, to reset its states + accumulator.reset(); + + // current TrainingDriver won't be receiving any updates beyond this point + if (consumer != null) + consumer.bypassMode(true); + + + isFirst.set(false); + + log.info("Master thread done..."); + + INDArray updaterState = null; + if (model instanceof ComputationGraph) { + updaterState = ((ComputationGraph) originalModel).getUpdater().getUpdaterStateViewArray(); + } else if (model instanceof MultiLayerNetwork) { + updaterState = ((MultiLayerNetwork) originalModel).getUpdater().getStateViewArray(); + } + + //Get threshold algorithm instances from each thread, and average them - they may have state that needs + // to be averaged and persisted, to avoid starting threshold adaption from scratch + val mh = (EncodingHandler) accum.getHandler(); + val taAveraged = mh.getAverageThresholdAlgorithm(); + + // FIXME: fill stats here + val result = SharedTrainingResult.builder().aggregationsCount(1).scoreSum(originalModel.score()) + .updaterStateArray(updaterState).listenerMetaData(new ArrayList<>()) + .listenerStaticInfo(new ArrayList<>()).listenerUpdates(new ArrayList<>()) + .minibatchesPerExecutor(Collections.singletonMap(SparkUtils.getSparkExecutorId(), iteratorDataSetCount.get().get())) + .thresholdAlgorithm(taAveraged) + .build(); + + // releasing Context here +// Nd4j.getMemoryManager().releaseCurrentContext(); + + return result; + } else { + // blocking call right here, all non-master threads will be blocked here + try { + observer.get().waitTillDone(); + //observer.get().wait(); + + log.info("Feeder [{}] thread done...", Thread.currentThread().getName()); + + if(exceptionEncountered.get()){ + //Propagate exception + Throwable t; + if(wrapper == null || exception != null) { + t = exception; + } else { + t = wrapper.getException(); + } + + throw new RuntimeException("Training failed due to exception in ParallelWrapper fit operation", t); + } + + // nothing to do here, just give away empty result (other than iterator count) + return SharedTrainingResult.builder().minibatchesPerExecutor(Collections.singletonMap(SparkUtils.getSparkExecutorId(), iteratorDataSetCount.get().get())).build(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // FIXME: we don't really need to throw it again, it's here only for debugging purposes + throw new RuntimeException(e); + } + } + } + + public void passDataSet(DataSet dataSet) { + // we're going to save this dataset into VirtualDataSetIterator + } + + public void passDataSet(MultiDataSet dataSet) { + // we're going to save this dataset into VirtualMultiDataSetIterator + } + + + public void blockUntilFinished() throws InterruptedException { + if (observer.get() != null) + observer.get().wait(); + else + throw new IllegalStateException("This method can't be called before iterators initialization"); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/ArrayDescriptor.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/ArrayDescriptor.java new file mode 100644 index 000000000..f0abb2efa --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/ArrayDescriptor.java @@ -0,0 +1,92 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.python; + +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.javacpp.FloatPointer; +import org.bytedeco.javacpp.Pointer; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.nativeblas.NativeOps; +import org.nd4j.nativeblas.NativeOpsHolder; + + +public class ArrayDescriptor implements java.io.Serializable{ + + private long address; + private long[] shape; + private long[] stride; + DataType type; + char ordering; + private static NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); + + public ArrayDescriptor(INDArray array) throws Exception{ + this(array.data().address(), array.shape(), array.stride(), array.data().dataType(), array.ordering()); + if (array.isEmpty()){ + throw new UnsupportedOperationException("Empty arrays are not supported"); + } + } + + public ArrayDescriptor(long address, long[] shape, long[] stride, DataType type, char ordering){ + this.address = address; + this.shape = shape; + this.stride = stride; + this.type = type; + this.ordering = ordering; + } + public long getAddress(){ + return address; + } + + public long[] getShape(){ + return shape; + } + + public long[] getStride(){ + return stride; + } + + public DataType getType(){ + return type; + } + + public char getOrdering(){ + return ordering; + } + + private long size(){ + long s = 1; + for (long d: shape){ + s *= d; + } + return s; + } + + public INDArray getArray() { + Pointer ptr = nativeOps.pointerForAddress(address); + ptr = ptr.limit(size()); + DataBuffer buff = Nd4j.createBuffer(ptr, size(), type); + return Nd4j.create(buff, shape, stride, 0, ordering, type); + } + +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/DataSetDescriptor.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/DataSetDescriptor.java new file mode 100644 index 000000000..66de004e5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/DataSetDescriptor.java @@ -0,0 +1,99 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.python; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; + + +public class DataSetDescriptor implements java.io.Serializable{ + private ArrayDescriptor features, labels; + private ArrayDescriptor featuresMask; + private ArrayDescriptor labelsMask; + private boolean preProcessed; + + public DataSetDescriptor(ArrayDescriptor features, ArrayDescriptor labels, ArrayDescriptor featuresMask, ArrayDescriptor labelsMask){ + this.features = features; + this.labels = labels; + this.featuresMask = featuresMask; + this.labelsMask = labelsMask; + } + + public DataSetDescriptor(DataSet ds)throws Exception{ + features = new ArrayDescriptor(ds.getFeatures()); + labels = new ArrayDescriptor(ds.getLabels()); + INDArray featuresMask = ds.getFeaturesMaskArray(); + if (featuresMask == null){ + this.featuresMask = null; + } + else{ + this.featuresMask = new ArrayDescriptor(featuresMask); + } + INDArray labelsMask = ds.getLabelsMaskArray(); + if (labelsMask == null){ + this.labelsMask = null; + } + else{ + this.labelsMask = new ArrayDescriptor(labelsMask); + } + + preProcessed = ds.isPreProcessed(); + } + + public DataSet getDataSet(){ + INDArray features = this.features.getArray(); + INDArray labels = this.labels.getArray(); + INDArray featuresMask; + INDArray labelsMask; + if (this.featuresMask == null){ + featuresMask = null; + } + else{ + featuresMask = this.featuresMask.getArray(); + } + if (this.labelsMask == null){ + labelsMask = null; + } + else{ + labelsMask = this.labelsMask.getArray(); + } + DataSet ds = new DataSet(features, labels, featuresMask, labelsMask); + if(preProcessed) { + ds.markAsPreProcessed(); + } + return ds; + } + + public ArrayDescriptor getFeatures() { + return features; + } + + public ArrayDescriptor getLabels() { + return labels; + } + + public ArrayDescriptor getFeaturesMask() { + return featuresMask; + } + + public ArrayDescriptor getLabelsMask() { + return labelsMask; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/Utils.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/Utils.java new file mode 100644 index 000000000..ef90e6181 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/python/Utils.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.python; + +import org.apache.spark.api.java.JavaRDD; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; + +import javax.xml.crypto.Data; + +public class Utils { + + private static ArrayDescriptor getArrayDescriptor(INDArray arr) throws Exception{ + return new ArrayDescriptor(arr); + } + + private static INDArray getArray(ArrayDescriptor arrDesc){ + return arrDesc.getArray(); + } + + private static DataSetDescriptor getDataSetDescriptor(DataSet ds)throws Exception{ + return new DataSetDescriptor(ds); + } + + private static DataSet getDataSet(DataSetDescriptor dsDesc){ + return dsDesc.getDataSet(); + } + public static JavaRDD getArrayDescriptorRDD(JavaRDD indarrayRDD){ + return indarrayRDD.map(Utils::getArrayDescriptor); + } + + public static JavaRDD getArrayRDD(JavaRDD arrayDescriptorRDD){ + return arrayDescriptorRDD.map(ArrayDescriptor::getArray); + } + + public static JavaRDD getDatasetDescriptorRDD(JavaRDD dsRDD){ + return dsRDD.map(Utils::getDataSetDescriptor); + } + + public static JavaRDD getDataSetRDD(JavaRDD dsDescriptorRDD){ + return dsDescriptorRDD.map(Utils::getDataSet); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java new file mode 100644 index 000000000..c55b3268a --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java @@ -0,0 +1,1271 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.training; + +import lombok.Data; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.lang3.RandomUtils; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaRDDLike; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.storage.StorageLevel; +import org.datavec.spark.util.BroadcastHadoopConfigHolder; +import org.deeplearning4j.core.loader.DataSetLoader; +import org.deeplearning4j.core.loader.MultiDataSetLoader; +import org.deeplearning4j.core.loader.impl.SerializedDataSetLoader; +import org.deeplearning4j.core.loader.impl.SerializedMultiDataSetLoader; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.common.config.DL4JEnvironmentVars; +import org.deeplearning4j.exception.DL4JInvalidConfigException; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ResidualPostProcessor; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.residual.ResidualClippingPostProcessor; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.threshold.AdaptiveThresholdAlgorithm; +import org.deeplearning4j.spark.api.*; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.api.worker.NetBroadcastTuple; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.BaseTrainingMaster; +import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingMasterStats; +import org.deeplearning4j.spark.impl.repartitioner.DefaultRepartitioner; +import org.deeplearning4j.spark.parameterserver.accumulation.SharedTrainingAccumulationFunction; +import org.deeplearning4j.spark.parameterserver.accumulation.SharedTrainingAccumulationTuple; +import org.deeplearning4j.spark.parameterserver.accumulation.SharedTrainingAggregateFunction; +import org.deeplearning4j.spark.parameterserver.conf.SharedTrainingConfiguration; +import org.deeplearning4j.spark.parameterserver.functions.SharedFlatMapDataSet; +import org.deeplearning4j.spark.parameterserver.functions.SharedFlatMapMultiDataSet; +import org.deeplearning4j.spark.parameterserver.functions.SharedFlatMapPaths; +import org.deeplearning4j.spark.parameterserver.functions.SharedFlatMapPathsMDS; +import org.deeplearning4j.spark.parameterserver.networking.v1.SilentTrainingDriver; +import org.deeplearning4j.spark.parameterserver.networking.v2.UpdatesConsumer; +import org.deeplearning4j.spark.util.SparkUtils; +import org.deeplearning4j.core.util.UIDProvider; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.config.ND4JEnvironmentVars; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; +import org.nd4j.parameterserver.distributed.enums.ExecutionMode; +import org.nd4j.parameterserver.distributed.enums.NodeRole; +import org.nd4j.parameterserver.distributed.enums.TransportType; +import org.nd4j.parameterserver.distributed.util.NetworkOrganizer; +import org.nd4j.parameterserver.distributed.v2.ModelParameterServer; +import org.nd4j.parameterserver.distributed.v2.transport.Transport; +import org.nd4j.parameterserver.distributed.v2.transport.impl.AeronUdpTransport; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * SharedTrainingMaster implements distributed training of neural networks using a compressed quantized gradient + * (update) sharing implementation based on the Strom 2015 paper "Scalable Distributed DNN Training Using Commodity + * GPU Cloud Computing": https://s3-us-west-2.amazonaws.com/amazon.jobs-public-documents/strom_interspeech2015.pdf. + * The Deeplearning4j implementation makes a number of modifications, such as having the option to use a + * parameter-server based implementation for fault tolerance and execution where multicast networking support + * is not available. + */ +@Slf4j +@Data + +public class SharedTrainingMaster extends BaseTrainingMaster + implements TrainingMaster { + //Static counter/id fields used to determine which training master last set up the singleton param servers, etc + protected static final AtomicInteger INSTANCE_COUNTER = new AtomicInteger(); + protected static final AtomicInteger LAST_TRAINING_INSTANCE = new AtomicInteger(-1); + + protected List trainingHooks; + protected VoidConfiguration voidConfiguration; + + protected Integer numWorkers; + protected Integer numWorkersPerNode; + protected int workerPrefetchBatches; + protected RDDTrainingApproach rddTrainingApproach; + protected StorageLevel storageLevel; + protected Repartitioner repartitioner; + + protected boolean collectTrainingStats; + protected int rddDataSetNumExamples; + protected long debugLongerIterations = 0L; + protected boolean logMinibatchesPerWorker = false; + protected boolean encodingDebugMode = false; + + protected ThresholdAlgorithm thresholdAlgorithm; + protected ResidualPostProcessor residualPostProcessor; + + protected Repartition repartition; + protected RepartitionStrategy repartitionStrategy; + + protected ParameterAveragingTrainingMasterStats.ParameterAveragingTrainingMasterStatsHelper stats; + + protected Random rng; + + protected AtomicBoolean isFirstRun; + + // better ignore + protected final transient int instanceId; + protected transient Broadcast broadcastModel; + protected transient Broadcast broadcastConfiguration; + protected transient Transport transport; + protected transient SilentTrainingDriver trainingDriver; + + protected transient UpdatesConsumer updatesConsumer; + + protected boolean setupDone; + + protected SharedTrainingMaster() { + // just a stub for ser/de + instanceId = INSTANCE_COUNTER.getAndIncrement(); + } + + public SharedTrainingMaster(@NonNull VoidConfiguration voidConfiguration, Integer numWorkers, + RDDTrainingApproach rddTrainingApproach, StorageLevel storageLevel, boolean collectTrainingStats, + RepartitionStrategy repartitionStrategy, Repartition repartition, + ThresholdAlgorithm thresholdAlgorithm, ResidualPostProcessor residualPostProcessor, + int rddDataSetNumExamples, + int batchSizePerWorker, long debugLongerIterations, int numWorkersPerNode, int workerPrefetchBatches, + Repartitioner repartitioner, Boolean workerTogglePeriodicGC, Integer workerPeriodicGCFrequency, + boolean encodingDebugMode) { + this.voidConfiguration = voidConfiguration; + this.numWorkers = numWorkers; + this.thresholdAlgorithm = thresholdAlgorithm; + this.residualPostProcessor = residualPostProcessor; + this.rddTrainingApproach = rddTrainingApproach; + this.repartitionStrategy = repartitionStrategy; + this.repartition = repartition; + this.storageLevel = storageLevel; + this.collectTrainingStats = collectTrainingStats; + this.isFirstRun = new AtomicBoolean(false); + this.batchSizePerWorker = batchSizePerWorker; + this.rddDataSetNumExamples = rddDataSetNumExamples; + this.debugLongerIterations = debugLongerIterations; + this.numWorkersPerNode = numWorkersPerNode; + this.workerPrefetchBatches = workerPrefetchBatches; + this.repartitioner = repartitioner; + this.workerTogglePeriodicGC = workerTogglePeriodicGC; + this.workerPeriodicGCFrequency = workerPeriodicGCFrequency; + this.encodingDebugMode = encodingDebugMode; + + + if (collectTrainingStats) + stats = new ParameterAveragingTrainingMasterStats.ParameterAveragingTrainingMasterStatsHelper(); + + + String jvmuid = UIDProvider.getJVMUID(); + this.trainingMasterUID = + System.currentTimeMillis() + "_" + (jvmuid.length() <= 8 ? jvmuid : jvmuid.substring(0, 8)); + instanceId = INSTANCE_COUNTER.getAndIncrement(); + } + + @Override + public void removeHook(TrainingHook trainingHook) { + if (trainingHooks != null) + trainingHooks.remove(trainingHook); + } + + @Override + public void addHook(@NonNull TrainingHook trainingHook) { + if (trainingHooks == null) + trainingHooks = new ArrayList<>(); + + trainingHooks.add(trainingHook); + } + + @Override + public String toJson() { + ObjectMapper om = getJsonMapper(); + + try { + return om.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException("Error producing JSON representation for ParameterAveragingTrainingMaster", e); + } + } + + @Override + public String toYaml() { + ObjectMapper om = getYamlMapper(); + + try { + return om.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException("Error producing YAML representation for ParameterAveragingTrainingMaster", e); + } + } + + /** + * Create a SharedTrainingMaster instance by deserializing a JSON string that has been serialized with + * {@link #toJson()} + * + * @param jsonStr SharedTrainingMaster configuration serialized as JSON + */ + public static SharedTrainingMaster fromJson(String jsonStr) { + ObjectMapper om = getJsonMapper(); + try { + return om.readValue(jsonStr, SharedTrainingMaster.class); + } catch (IOException e) { + throw new RuntimeException("Could not parse JSON", e); + } + } + + /** + * Create a SharedTrainingMaster instance by deserializing a YAML string that has been serialized with + * {@link #toYaml()} + * + * @param yamlStr SharedTrainingMaster configuration serialized as YAML + */ + public static SharedTrainingMaster fromYaml(String yamlStr) { + ObjectMapper om = getYamlMapper(); + try { + return om.readValue(yamlStr, SharedTrainingMaster.class); + } catch (IOException e) { + throw new RuntimeException("Could not parse YAML", e); + } + } + + @Override + public SharedTrainingWorker getWorkerInstance(SparkDl4jMultiLayer network) { + /* + Here we're going create our worker, which will be passed into corresponding FlatMapFunction + */ + NetBroadcastTuple tuple = new NetBroadcastTuple(network.getNetwork().getLayerWiseConfigurations(), + network.getNetwork().params(), network.getNetwork().getUpdater().getStateViewArray()); + + voidConfiguration.setUnicastControllerPort(voidConfiguration.getPortSupplier().getPort()); + + SharedTrainingConfiguration configuration = SharedTrainingConfiguration.builder() + .thresholdAlgorithm(thresholdAlgorithm) + .residualPostProcessor(residualPostProcessor) + .voidConfiguration(voidConfiguration) + .debugLongerIterations(debugLongerIterations) + .numberOfWorkersPerNode(numWorkersPerNode) + .encodingDebugMode(encodingDebugMode).build(); + + if (collectTrainingStats) + stats.logBroadcastStart(); + + if (broadcastModel == null) + broadcastModel = network.getSparkContext().broadcast(tuple); + + if (broadcastConfiguration == null) + broadcastConfiguration = network.getSparkContext().broadcast(configuration); + + if (collectTrainingStats) + stats.logBroadcastEnd(); + + SharedTrainingWorker worker = new SharedTrainingWorker(instanceId, broadcastModel, broadcastConfiguration, listeners, + statsStorage, workerTogglePeriodicGC, workerPeriodicGCFrequency); + + return worker; + } + + @Override + public SharedTrainingWorker getWorkerInstance(SparkComputationGraph graph) { + NetBroadcastTuple tuple = new NetBroadcastTuple(graph.getNetwork().getConfiguration(), + graph.getNetwork().params(), graph.getNetwork().getUpdater().getStateViewArray()); + + SharedTrainingConfiguration configuration = SharedTrainingConfiguration.builder() + .thresholdAlgorithm(thresholdAlgorithm) + .residualPostProcessor(residualPostProcessor) + .voidConfiguration(voidConfiguration).debugLongerIterations(debugLongerIterations) + .numberOfWorkersPerNode(numWorkersPerNode) + .prefetchSize(workerPrefetchBatches) + .encodingDebugMode(encodingDebugMode) + .build(); + + if (collectTrainingStats) + stats.logBroadcastStart(); + + if (broadcastModel == null) + broadcastModel = graph.getSparkContext().broadcast(tuple); + + if (broadcastConfiguration == null) + broadcastConfiguration = graph.getSparkContext().broadcast(configuration); + + if (collectTrainingStats) + stats.logBroadcastEnd(); + + SharedTrainingWorker worker = new SharedTrainingWorker(instanceId, broadcastModel, broadcastConfiguration, listeners, + statsStorage, workerTogglePeriodicGC, workerPeriodicGCFrequency); + + return worker; + } + + protected int numObjectsEachWorker(int numExamplesEachRddObject) { + return batchSizePerWorker / numExamplesEachRddObject; + } + + protected > long getTotalDataSetObjectCount( + JavaRDDLike trainingData) { + if (collectTrainingStats) + stats.logCountStart(); + + long totalDataSetObjectCount = trainingData.count(); + + if (collectTrainingStats) + stats.logCountEnd(); + + return totalDataSetObjectCount; + } + + protected void executeTrainingDirect(SparkDl4jMultiLayer network, JavaRDD trainingData) { + if (collectTrainingStats) + stats.logFitStart(); + + //For "vanilla" parameter averaging training, we need to split the full data set into batches of size N, such that we can process the specified + // number of minibatches between averagings + //But to do that, wee need to know: (a) the number of examples, and (b) the number of workers + if (storageLevel != null) + trainingData.persist(storageLevel); + + long totalDataSetObjectCount = getTotalDataSetObjectCount(trainingData); + + // since this is real distributed training, we don't need to split data + doIteration(network, trainingData, 1, 1); + + if (collectTrainingStats) + stats.logFitEnd((int) totalDataSetObjectCount); + } + + protected void executeTrainingDirectMDS(SparkComputationGraph network, JavaRDD trainingData) { + if (collectTrainingStats) + stats.logFitStart(); + + //For "vanilla" parameter averaging training, we need to split the full data set into batches of size N, such that we can process the specified + // number of minibatches between averagings + //But to do that, wee need to know: (a) the number of examples, and (b) the number of workers + if (storageLevel != null) + trainingData.persist(storageLevel); + + long totalDataSetObjectCount = getTotalDataSetObjectCount(trainingData); + + // since this is real distributed training, we don't need to split data + doIterationMDS(network, trainingData, 1, 1); + + if (collectTrainingStats) + stats.logFitEnd((int) totalDataSetObjectCount); + } + + protected void executeTrainingDirect(SparkComputationGraph network, JavaRDD trainingData) { + if (collectTrainingStats) + stats.logFitStart(); + + //For "vanilla" parameter averaging training, we need to split the full data set into batches of size N, such that we can process the specified + // number of minibatches between averagings + //But to do that, wee need to know: (a) the number of examples, and (b) the number of workers + if (storageLevel != null) + trainingData.persist(storageLevel); + + long totalDataSetObjectCount = getTotalDataSetObjectCount(trainingData); + + // since this is real distributed training, we don't need to split data + doIteration(network, trainingData, 1, 1); + + if (collectTrainingStats) + stats.logFitEnd((int) totalDataSetObjectCount); + } + + + @Override + public void executeTrainingPaths(SparkDl4jMultiLayer network, SparkComputationGraph graph, JavaRDD trainingDataPaths, + DataSetLoader dsLoader, MultiDataSetLoader mdsLoader) { + prepareNetworkAndStuff(network, graph); + executeTrainingPathsHelper(network, graph, trainingDataPaths, dsLoader, mdsLoader, rddDataSetNumExamples); + } + + protected void executeTrainingPathsHelper(SparkDl4jMultiLayer network, SparkComputationGraph graph, JavaRDD trainingDataPaths, + DataSetLoader dsLoader, MultiDataSetLoader mdsLoader, int dataSetObjectsNumExamples) { + + if (numWorkers == null) { + if(network != null){ + numWorkers = network.getSparkContext().defaultParallelism(); + } else { + numWorkers = graph.getSparkContext().defaultParallelism(); + } + } + + if (collectTrainingStats) + stats.logFitStart(); + + if (storageLevelStreams != null) + trainingDataPaths.persist(storageLevelStreams); + + long totalDataSetObjectCount = getTotalDataSetObjectCount(trainingDataPaths); + + doIterationPaths(network, graph, trainingDataPaths, 1, 1, dsLoader, mdsLoader, dataSetObjectsNumExamples); + + if (collectTrainingStats) + stats.logFitEnd((int) totalDataSetObjectCount); + } + + protected void prepareNetworkAndStuff(SparkDl4jMultiLayer network, SparkComputationGraph graph) { + if (network == null && graph == null) + throw new IllegalStateException("Both MLN & CG are undefined"); + + //Get the port for communicating with the master/driver - and add it to the configuration for use from each machine + //Note that each machine will allocate their own port for inbound communications according to what the PortSupplier + //returns on each worker machine. + voidConfiguration.setUnicastControllerPort(voidConfiguration.getPortSupplier().getPort()); + + // if streamId has default value - generate random one + if (voidConfiguration.getStreamId() < 1) + voidConfiguration.setStreamId(RandomUtils.nextInt(119, Integer.MAX_VALUE - 1)); + + // first of all, we're instantiating ParameterServer shard here\ + if (numWorkers == null) + numWorkers = network != null ? network.getSparkContext().defaultParallelism() + : graph.getSparkContext().defaultParallelism(); + + // set current box as controller, if field is unset - switch to next step + if (voidConfiguration.getControllerAddress() == null) { + try { + val e = System.getenv("SPARK_PUBLIC_DNS"); + log.info("Trying {SPARK_PUBLIC_DNS}: [{}]", e); + if (e != null) { + String sparkIp = InetAddress.getByName(e).getHostAddress(); + voidConfiguration.setControllerAddress(sparkIp); + } + } catch (UnknownHostException e) { + } + } + + // next step - is to get ip address that matches specific network mask + if (voidConfiguration.getControllerAddress() == null && voidConfiguration.getNetworkMask() != null) { + NetworkOrganizer organizer = new NetworkOrganizer(voidConfiguration.getNetworkMask()); + val s = organizer.getMatchingAddress(); + log.info("Trying auto-detected address: [{}]", s); + + voidConfiguration.setControllerAddress(s); + } + + if (voidConfiguration.getControllerAddress() == null) { + String envVar = System.getenv(DL4JEnvironmentVars.DL4J_VOID_IP); + if(envVar != null && !envVar.isEmpty()) { + voidConfiguration.setControllerAddress(envVar); + } + } + + if (voidConfiguration.getControllerAddress() == null) + throw new DL4JInvalidConfigException( + "Can't get Spark Master local address. Please specify it manually using VoidConfiguration.setControllerAddress(String) method or VoidConfiguration.setNetworkMask(String) method"); + + // we're forcing proper defaults + log.info("Setting controller address to {}:{}", voidConfiguration.getControllerAddress(), + voidConfiguration.getUnicastControllerPort()); + voidConfiguration.setShardAddresses(voidConfiguration.getControllerAddress()); + voidConfiguration.setNumberOfShards(1); + + if (network != null) + network.getNetwork().init(); + else + graph.getNetwork().init(); + + // this instance will be SilentWorker - it'll accept and apply messages, but won't contribute to training. And we init it only once + if (isFirstRun.compareAndSet(false, true) || LAST_TRAINING_INSTANCE.get() != instanceId) { + if(LAST_TRAINING_INSTANCE.get() >= 0 && LAST_TRAINING_INSTANCE.get() != instanceId){ + log.debug("Detected changed training instance - setting up new parameter server - old instance {}, new instance {}", + LAST_TRAINING_INSTANCE, instanceId); + + ModelParameterServer.getInstance().shutdown(); + try{ //TODO is this required? + Thread.sleep(3000); + } catch (Exception e){ + throw new RuntimeException(e); + } + } + + val transport = voidConfiguration.getTransportType() == TransportType.ROUTED_UDP + ? new AeronUdpTransport(voidConfiguration.getControllerAddress(), voidConfiguration.getUnicastControllerPort(), voidConfiguration) + : null; + + if (transport == null) + throw new DL4JInvalidConfigException("No Transport implementation was defined for this training session!"); + + val params = network != null ? network.getNetwork().params() : graph.getNetwork().params(); + + updatesConsumer = UpdatesConsumer.builder() + .params(params) + .updates(Nd4j.create(params.shape(), params.ordering())) + .stepFunction(network != null ? network.getNetwork().getOptimizer().getStepFunction() : graph.getNetwork().getOptimizer().getStepFunction()) + .build(); + + // apply configuration + ModelParameterServer.getInstance().configure(voidConfiguration, transport, true); + + // and attach our consumer + ModelParameterServer.getInstance().addUpdatesSubscriber(updatesConsumer); + + + // and start actual server + if (!ModelParameterServer.getInstance().isInitialized()) + ModelParameterServer.getInstance().launch(); + + LAST_TRAINING_INSTANCE.set(instanceId); + } + + setupDone = true; + } + + protected void finalizeTraining() { + /* + Here we basically want to do few things: + 1) update statistics, if any + 2) finalize updates of silent worker + 3) pull back gradients, maybe? + */ + + // applying non-applied updates, if any :) + if (trainingDriver != null) { + trainingDriver.finishTraining(0L, 0L); + } + + // the same, but v2 impl + if (updatesConsumer != null) + updatesConsumer.flush(); + } + + @Override + public void executeTraining(SparkDl4jMultiLayer network, JavaRDD trainingData) { + /* + This method (and other similar methods) is basically one of our entry points, here we'll spawn our training process: + 1) broadcast everything needed: initial model params, updaters state, conf. Useful for uptraining + 2) shuffle, if needed + 3) repartition, if needed + 4) EXECUTE SILENT WORKER + 5) invoke training function via mapPartitions + 6) wait till finished + 7) do something with final model, i.e. export it somewhere :) + */ + + prepareNetworkAndStuff(network, null); + + // at this moment we have coordinator server up (master works as coordinator) + if (rddTrainingApproach == RDDTrainingApproach.Direct) { + executeTrainingDirect(network, trainingData); + } else if (rddTrainingApproach == RDDTrainingApproach.Export) { + //Export data if required (or, use cached export) + JavaRDD paths = exportIfRequired(network.getSparkContext(), trainingData); + executeTrainingPathsHelper(network, null, paths, new SerializedDataSetLoader(), null, batchSizePerWorker); + } else + throw new DL4JInvalidConfigException( + "Unknown RDDtrainingApproach [" + rddTrainingApproach + "] was specified!"); + } + + @Override + public void executeTraining(SparkComputationGraph graph, JavaRDD trainingData) { + prepareNetworkAndStuff(null, graph); + + // at this moment we have coordinator server up (master works as coordinator) + if (rddTrainingApproach == RDDTrainingApproach.Direct) { + executeTrainingDirect(graph, trainingData); + } else if (rddTrainingApproach == RDDTrainingApproach.Export) { + //Export data if required (or, use cached export) + JavaRDD paths = exportIfRequired(graph.getSparkContext(), trainingData); + executeTrainingPathsHelper(null, graph, paths, new SerializedDataSetLoader(), null, batchSizePerWorker); + } else + throw new DL4JInvalidConfigException( + "Unknown RDDtrainingApproach [" + rddTrainingApproach + "] was specified!"); + } + + @Override + public void executeTrainingMDS(SparkComputationGraph graph, JavaRDD trainingData) { + prepareNetworkAndStuff(null, graph); + + // at this moment we have coordinator server up (master works as coordinator) + if (rddTrainingApproach == RDDTrainingApproach.Direct) { + executeTrainingDirectMDS(graph, trainingData); + } else if (rddTrainingApproach == RDDTrainingApproach.Export) { + //Export data if required (or, use cached export) + JavaRDD paths = exportIfRequiredMDS(graph.getSparkContext(), trainingData); + executeTrainingPathsHelper(null, graph, paths, null, new SerializedMultiDataSetLoader(), batchSizePerWorker); + } else + throw new DL4JInvalidConfigException( + "Unknown RDDtrainingApproach [" + rddTrainingApproach + "] was specified!"); + } + + @Override + public void setCollectTrainingStats(boolean collectTrainingStats) { + this.collectTrainingStats = collectTrainingStats; + } + + @Override + public boolean getIsCollectTrainingStats() { + return collectTrainingStats; + } + + @Override + public SparkTrainingStats getTrainingStats() { + return null; + } + + @Override + public void setListeners(Collection listeners) { + setListeners(null, listeners); + } + + @Override + public void setListeners(StatsStorageRouter router, Collection listeners) { + this.statsStorage = router; + this.listeners = (listeners == null ? null : new ArrayList<>(listeners)); + } + + + protected void processResults(SparkDl4jMultiLayer network, SparkComputationGraph graph, + JavaRDD results) { + Preconditions.checkState(network != null || graph != null, "Both MLN & CG are null"); + Preconditions.checkState(setupDone, "Setup was not completed before trying to process results"); + + + + if (collectTrainingStats) + stats.logAggregateStartTime(); + + SharedTrainingAccumulationTuple finalResult = results.treeAggregate(null, new SharedTrainingAggregateFunction(), + new SharedTrainingAccumulationFunction(), 4); + SparkTrainingStats aggregatedStats = finalResult.getSparkTrainingStats(); + if (collectTrainingStats) + stats.logAggregationEndTime(); + + //finalizeTraining has to be *after* training has completed, otherwise the RDD (via tree aggregate) + finalizeTraining(); + + + if (collectTrainingStats) + stats.logProcessParamsUpdaterStart(); + + if (finalResult.getUpdaterStateArray() != null) { + + if (finalResult.getAggregationsCount() > 1) { + finalResult.getUpdaterStateArray().divi(finalResult.getAggregationsCount()); + } + + if (network != null) { + if (network.getNetwork().getUpdater() != null + && network.getNetwork().getUpdater().getStateViewArray() != null) + network.getNetwork().getUpdater().getStateViewArray().assign(finalResult.getUpdaterStateArray()); + } else { + if (graph.getNetwork().getUpdater() != null + && graph.getNetwork().getUpdater().getStateViewArray() != null) + graph.getNetwork().getUpdater().getStateViewArray().assign(finalResult.getUpdaterStateArray()); + } + } + + + double score = finalResult.getScoreSum() / Math.max(1, finalResult.getAggregationsCount()); + + if (network != null) { + network.getNetwork().setScore(score); + } else { + graph.getNetwork().setScore(score); + } + + if (collectTrainingStats) + stats.logProcessParamsUpdaterEnd(); + + + if (collectTrainingStats) { + stats.logProcessParamsUpdaterEnd(); + stats.addWorkerStats(aggregatedStats); + } + + if (statsStorage != null) { + Collection meta = finalResult.getListenerMetaData(); + if (meta != null && !meta.isEmpty()) { + statsStorage.putStorageMetaData(meta); + } + + Collection staticInfo = finalResult.getListenerStaticInfo(); + if (staticInfo != null && !staticInfo.isEmpty()) { + statsStorage.putStaticInfo(staticInfo); + } + + Collection updates = finalResult.getListenerUpdates(); + if (updates != null && !updates.isEmpty()) { + statsStorage.putUpdate(updates); + } + } + + if (logMinibatchesPerWorker){ + if(finalResult.getMinibatchesPerExecutor() != null){ + List l = new ArrayList<>(finalResult.getMinibatchesPerExecutor().keySet()); + Collections.sort(l); + Map linkedMap = new LinkedHashMap<>(); + for(String s : l){ + linkedMap.put(s, finalResult.getMinibatchesPerExecutor().get(s)); + } + log.info("Number of minibatches processed per JVM/executor: {}", linkedMap); + } + } + + if(finalResult.getThresholdAlgorithmReducer() != null){ + //Store the final threshold algorithm after aggregation + //Some threshold algorithms contain state/history, used to adapt the threshold algorithm + //The idea is we want to keep this history/state for next epoch, rather than simply throwing it away + // and starting the threshold adaption process from scratch on each epoch + ThresholdAlgorithm ta = finalResult.getThresholdAlgorithmReducer().getFinalResult(); + this.thresholdAlgorithm = ta; + } + + Nd4j.getExecutioner().commit(); + } + + protected void doIteration(SparkDl4jMultiLayer network, JavaRDD split, int splitNum, int numSplits) { + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, thresholdAlgorithm={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, thresholdAlgorithm, numWorkers); + + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + JavaRDD splitData = split; + + if (collectTrainingStats) + stats.logRepartitionStart(); + + if(repartitioner != null){ + log.info("Repartitioning training data using repartitioner: {}", repartitioner); + int minPerWorker = Math.max(1, batchSizePerWorker/rddDataSetNumExamples); + splitData = repartitioner.repartition(splitData, minPerWorker, numWorkers); + } else { + log.info("Repartitioning training data using SparkUtils repartitioner"); + splitData = SparkUtils.repartitionEqually(splitData, repartition, numWorkers); + } + int nPartitions = splitData.partitions().size(); + + if (collectTrainingStats && repartition != Repartition.Never) + stats.logRepartitionEnd(); + + + FlatMapFunction, SharedTrainingResult> function = + new SharedFlatMapDataSet<>(getWorkerInstance(network)); + + JavaRDD result = splitData.mapPartitions(function); + + processResults(network, null, result); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + protected void doIterationMDS(SparkComputationGraph network, JavaRDD split, int splitNum, + int numSplits) { + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, thresholdAlgorithm={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, thresholdAlgorithm, numWorkers); + + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + JavaRDD splitData = split; + + if (collectTrainingStats) + stats.logRepartitionStart(); + + if(repartitioner != null){ + log.info("Repartitioning training data using repartitioner: {}", repartitioner); + int minPerWorker = Math.max(1, batchSizePerWorker/rddDataSetNumExamples); + splitData = repartitioner.repartition(splitData, minPerWorker, numWorkers); + } else { + log.info("Repartitioning training data using SparkUtils repartitioner"); + splitData = SparkUtils.repartitionEqually(splitData, repartition, numWorkers); + } + int nPartitions = splitData.partitions().size(); + + if (collectTrainingStats && repartition != Repartition.Never) + stats.logRepartitionEnd(); + + + FlatMapFunction, SharedTrainingResult> function = + new SharedFlatMapMultiDataSet<>(getWorkerInstance(network)); + + JavaRDD result = splitData.mapPartitions(function); + + processResults(null, network, result); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + protected void doIteration(SparkComputationGraph network, JavaRDD data, int splitNum, int numSplits) { + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, thresholdAlgorithm={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, thresholdAlgorithm, numWorkers); + + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + if (collectTrainingStats) + stats.logRepartitionStart(); + + if(repartitioner != null){ + log.info("Repartitioning training data using repartitioner: {}", repartitioner); + int minPerWorker = Math.max(1, batchSizePerWorker/rddDataSetNumExamples); + data = repartitioner.repartition(data, minPerWorker, numWorkers); + } else { + log.info("Repartitioning training data using SparkUtils repartitioner"); + data = SparkUtils.repartitionEqually(data, repartition, numWorkers); + } + int nPartitions = data.partitions().size(); + + if (collectTrainingStats && repartition != Repartition.Never) + stats.logRepartitionEnd(); + + + FlatMapFunction, SharedTrainingResult> function = + new SharedFlatMapDataSet<>(getWorkerInstance(network)); + + JavaRDD result = data.mapPartitions(function); + + processResults(null, network, result); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + protected void doIterationPaths(SparkDl4jMultiLayer network, SparkComputationGraph graph, JavaRDD data, + int splitNum, int numSplits, DataSetLoader dsLoader, MultiDataSetLoader mdsLoader, int dataSetObjectNumExamples) { + if (network == null && graph == null) + throw new DL4JInvalidConfigException("Both MLN & CompGraph are NULL"); + + log.info("Starting training of split {} of {}. workerMiniBatchSize={}, thresholdAlgorithm={}, Configured for {} workers", + splitNum, numSplits, batchSizePerWorker, thresholdAlgorithm, numWorkers); + + if (collectTrainingStats) + stats.logMapPartitionsStart(); + + if (collectTrainingStats) + stats.logRepartitionStart(); + + if(repartitioner != null){ + log.info("Repartitioning training data using repartitioner: {}", repartitioner); + int minPerWorker = Math.max(1, batchSizePerWorker/dataSetObjectNumExamples); + data = repartitioner.repartition(data, minPerWorker, numWorkers); + } else { + log.info("Repartitioning training data using SparkUtils repartitioner"); + data = SparkUtils.repartitionEqually(data, repartition, numWorkers); + } + + int nPartitions = data.partitions().size(); + if (collectTrainingStats && repartition != Repartition.Never) + stats.logRepartitionEnd(); + + JavaSparkContext sc = (network != null ? network.getSparkContext() : graph.getSparkContext()); + FlatMapFunction, SharedTrainingResult> function; + if(dsLoader != null){ + function = new SharedFlatMapPaths<>( + network != null ? getWorkerInstance(network) : getWorkerInstance(graph), dsLoader, BroadcastHadoopConfigHolder.get(sc)); + } else { + function = new SharedFlatMapPathsMDS<>( + network != null ? getWorkerInstance(network) : getWorkerInstance(graph), mdsLoader, BroadcastHadoopConfigHolder.get(sc)); + } + + + JavaRDD result = data.mapPartitions(function); + + processResults(network, graph, result); + + if (collectTrainingStats) + stats.logMapPartitionsEnd(nPartitions); + } + + + public static class Builder { + protected ThresholdAlgorithm thresholdAlgorithm = new AdaptiveThresholdAlgorithm(); + protected ResidualPostProcessor residualPostProcessor = new ResidualClippingPostProcessor(5.0, 5); + protected int rddDataSetNumExamples = 1; + @Deprecated + protected Repartition repartition = Repartition.Always; + @Deprecated + protected RepartitionStrategy repartitionStrategy = RepartitionStrategy.Balanced; + protected StorageLevel storageLevel = StorageLevel.MEMORY_ONLY_SER(); + protected VoidConfiguration voidConfiguration; + protected RDDTrainingApproach rddTrainingApproach = RDDTrainingApproach.Export; + protected long rngSeed; + protected String exportDirectory = null; + protected Integer numWorkers; + protected boolean collectTrainingStats; + protected Transport transport; + protected int batchSize; + protected long debugLongerIterations = 0L; + protected int numWorkersPerNode = -1; + protected int workerPrefetchNumBatches = 2; + protected Repartitioner repartitioner = new DefaultRepartitioner(); + protected Boolean workerTogglePeriodicGC = new Boolean(true); + protected Integer workerPeriodicGCFrequency = new Integer(5000); + protected boolean encodingDebugMode = false; + + /** + * Create a SharedTrainingMaster with defaults other than the RDD number of examples + * @param rddDataSetNumExamples When fitting from an {@code RDD} how many examples are in each dataset? + */ + public Builder(int rddDataSetNumExamples) { + this(new AdaptiveThresholdAlgorithm(), rddDataSetNumExamples); + } + + /** + * Create a SharedTrainingMaster with defaults other than the RDD number of examples + * @param voidConfiguration Configuration bean for the SharedTrainingMaster parameter server + * @param rddDataSetNumExamples When fitting from an {@code RDD} how many examples are in each dataset? + */ + public Builder(@NonNull VoidConfiguration voidConfiguration, int rddDataSetNumExamples) { + this(voidConfiguration, new AdaptiveThresholdAlgorithm(), rddDataSetNumExamples); + } + + /** + * Create a SharedTrainingMaster with defaults other than the RDD number of examples + * @param thresholdAlgorithm Threshold algorithm for the sparse update encoding + * @param rddDataSetNumExamples When fitting from an {@code RDD} how many examples are in each dataset? + */ + public Builder(ThresholdAlgorithm thresholdAlgorithm, int rddDataSetNumExamples) { + this(VoidConfiguration.builder().executionMode(ExecutionMode.MANAGED).forcedRole(NodeRole.SHARD) + // we're setting controller to Spark Master, if it's null - that's ok for now. + .controllerAddress(System.getenv("SPARK_PUBLIC_DNS")).build(), thresholdAlgorithm, + rddDataSetNumExamples); + } + + /** + * @param voidConfiguration Configuration bean for the SharedTrainingMaster parameter server + * @param numWorkers No longer used/required + * @param threshold Encoding threshold + * @param rddDataSetNumExamples When fitting from an {@code RDD} how many examples are in each dataset? + * @deprecated This constructor is deprecated - use {@link #Builder(VoidConfiguration, int)} or {@link #Builder(VoidConfiguration, ThresholdAlgorithm, int)} + */ + @Deprecated + public Builder(@NonNull VoidConfiguration voidConfiguration, Integer numWorkers, double threshold, int rddDataSetNumExamples) { + this(voidConfiguration, new AdaptiveThresholdAlgorithm(threshold), rddDataSetNumExamples); + } + + /** + * @param voidConfiguration Configuration bean for the SharedTrainingMaster parameter server + * @param thresholdAlgorithm Update sharing threshold algorithm + * @param rddDataSetNumExamples + */ + public Builder(@NonNull VoidConfiguration voidConfiguration, ThresholdAlgorithm thresholdAlgorithm, int rddDataSetNumExamples) { + this.thresholdAlgorithm = thresholdAlgorithm; + this.voidConfiguration = voidConfiguration; + this.rddDataSetNumExamples = rddDataSetNumExamples; + + // we're enforcing managed mode in all cases here + this.voidConfiguration.setExecutionMode(ExecutionMode.MANAGED); + } + + public Builder(@NonNull VoidConfiguration voidConfiguration, Integer numWorkers, ThresholdAlgorithm thresholdAlgorithm, int rddDataSetNumExamples) { + this.thresholdAlgorithm = thresholdAlgorithm; + this.voidConfiguration = voidConfiguration; + this.rddDataSetNumExamples = rddDataSetNumExamples; + this.numWorkers = numWorkers; + + // we're enforcing managed mode in all cases here + this.voidConfiguration.setExecutionMode(ExecutionMode.MANAGED); + } + + /** + * Enable/disable collection of training statistics + * @param enable Enable + * @return + */ + public Builder collectTrainingStats(boolean enable) { + this.collectTrainingStats = enable; + return this; + } + + /** + * This parameter defines when repartition is applied (if applied). + * @param repartition Repartition setting + * @deprecated Use {@link #repartitioner(Repartitioner)} + */ + @Deprecated + public Builder repartitionData(Repartition repartition) { + this.repartition = repartition; + return this; + } + + /** + * Used in conjunction with {@link #repartitionData(Repartition)} (which defines when repartitioning should be + * conducted), repartitionStrategy defines how the repartitioning should be done. See {@link RepartitionStrategy} + * for details + * + * @param repartitionStrategy Repartitioning strategy to use + * @deprecated Use {@link #repartitioner(Repartitioner)} + */ + @Deprecated + public Builder repartitionStrategy(RepartitionStrategy repartitionStrategy) { + this.repartitionStrategy = repartitionStrategy; + return this; + } + + /** + * Set the storage level for {@code RDD}s.
+ * Default: StorageLevel.MEMORY_ONLY_SER() - i.e., store in memory, in serialized form
+ * To use no RDD persistence, use {@code null}
+ * Note that this only has effect when {@code RDDTrainingApproach.Direct} is used (which is not the default), + * and when fitting from an {@code RDD}. + *

+ * Note: Spark's StorageLevel.MEMORY_ONLY() and StorageLevel.MEMORY_AND_DISK() can be problematic when + * it comes to off-heap data (which DL4J/ND4J uses extensively). Spark does not account for off-heap memory + * when deciding if/when to drop blocks to ensure enough free memory; consequently, for DataSet RDDs that are + * larger than the total amount of (off-heap) memory, this can lead to OOM issues. Put another way: Spark counts + * the on-heap size of DataSet and INDArray objects only (which is negligible) resulting in a significant + * underestimate of the true DataSet object sizes. More DataSets are thus kept in memory than we can really afford.
+ *
+ * Note also that fitting directly from an {@code RDD} is discouraged - it is better to export your + * prepared data once and call (for example} {@code SparkDl4jMultiLayer.fit(String savedDataDirectory)}. + * See DL4J's Spark website documentation for details.
+ * + * @param storageLevel Storage level to use for DataSet RDDs + */ + public Builder storageLevel(StorageLevel storageLevel) { + this.storageLevel = storageLevel; + return this; + } + + /** + * The approach to use when training on a {@code RDD} or {@code RDD}. + * Default: {@link RDDTrainingApproach#Export}, which exports data to a temporary directory first.
+ * The default cluster temporary directory is used, though can be configured using {@link #exportDirectory(String)} + * Note also that fitting directly from an {@code RDD} is discouraged - it is better to export your + * prepared data once and call (for example} {@code SparkDl4jMultiLayer.fit(String savedDataDirectory)}. + * See DL4J's Spark website documentation for details.
+ * + * @param rddTrainingApproach Training approach to use when training from a {@code RDD} or {@code RDD} + */ + public Builder rddTrainingApproach(RDDTrainingApproach rddTrainingApproach) { + this.rddTrainingApproach = rddTrainingApproach; + return this; + } + + /** + * When {@link #rddTrainingApproach(RDDTrainingApproach)} is set to {@link RDDTrainingApproach#Export} (as it is by default) + * the data is exported to a temporary directory first. + *

+ * Default: null. -> use {hadoop.tmp.dir}/dl4j/. In this case, data is exported to {hadoop.tmp.dir}/dl4j/SOME_UNIQUE_ID/
+ * If you specify a directory, the directory {exportDirectory}/SOME_UNIQUE_ID/ will be used instead. + * + * @param exportDirectory Base directory to export data + */ + public Builder exportDirectory(String exportDirectory) { + this.exportDirectory = exportDirectory; + return this; + } + + /** + * Random number generator seed, used mainly for enforcing repeatable splitting/repartitioning on RDDs + * Default: no seed set (i.e., random seed) + * + * @param rngSeed RNG seed + */ + public Builder rngSeed(long rngSeed) { + this.rngSeed = rngSeed; + return this; + } + + /** + * @deprecated Use {@link #thresholdAlgorithm(ThresholdAlgorithm)} with (for example) {@link AdaptiveThresholdAlgorithm} + */ + @Deprecated + public Builder updatesThreshold(double updatesThreshold){ + return thresholdAlgorithm(new AdaptiveThresholdAlgorithm(updatesThreshold)); + } + + /** + * Algorithm to use to determine the threshold for updates encoding. Lower values might improve convergence, but + * increase amount of network communication
+ * Values that are too low may also impact network convergence. If convergence problems are observed, try increasing + * or decreasing this by a factor of 10 - say 1e-4 and 1e-2.
+ * For technical details, see the paper + * Scalable Distributed DNN Training Using Commodity GPU Cloud Computing
+ * See also {@link ThresholdAlgorithm}

+ * Default: {@link AdaptiveThresholdAlgorithm} with default parameters + * @param thresholdAlgorithm Threshold algorithm to use to determine encoding threshold + */ + public Builder thresholdAlgorithm(ThresholdAlgorithm thresholdAlgorithm){ + this.thresholdAlgorithm = thresholdAlgorithm; + return this; + } + + /** + * Residual post processor. See {@link ResidualPostProcessor} for details. + * + * Default: {@code new ResidualClippingPostProcessor(5.0, 5)} - i.e., a {@link ResidualClippingPostProcessor} + * that clips the residual to +/- 5x current threshold, every 5 iterations. + * + * @param residualPostProcessor Residual post processor to use + */ + public Builder residualPostProcessor(ResidualPostProcessor residualPostProcessor){ + this.residualPostProcessor = residualPostProcessor; + return this; + } + + /** + * Minibatch size to use when training workers. In principle, the source data (i.e., {@code RDD} etc) + * can have a different number of examples in each {@code DataSet} than we want to use when training. + * i.e., we can split or combine DataSets if required. + * + * @param batchSize Minibatch size to use when fitting each worker + */ + public Builder batchSizePerWorker(int batchSize) { + this.batchSize = batchSize; + return this; + } + + /** + * This method allows to configure number of network training threads per cluster node.
+ * Default value: -1, which defines automated number of workers selection, based on hardware present in system + * (i.e., number of GPUs, if training on a GPU enabled system). + *
+ * When training on GPUs, you should use 1 worker per GPU (which is the default). For CPUs, 1 worker per + * node is usually preferred, though multi-CPU (i.e., multiple physical CPUs) or CPUs with large core counts + * may have better throughput (i.e., more examples per second) when increasing the number of workers, + * at the expense of more memory consumed. Note that if you increase the number of workers on a CPU system, + * you should set the number of OpenMP threads using the {@code OMP_NUM_THREADS} property - see + * {@link ND4JEnvironmentVars#OMP_NUM_THREADS} for more details. + * For example, a machine with 32 physical cores could use 4 workers with {@code OMP_NUM_THREADS=8} + * + * @param numWorkers Number of workers on each node. + */ + public Builder workersPerNode(int numWorkers) { + if (numWorkers < 1) + numWorkers = -1; + + this.numWorkersPerNode = numWorkers; + return this; + } + + /** + * This method allows you to artificially extend iteration time using Thread.sleep() for a given time. + * + * PLEASE NOTE: Never use that option in production environment. It's suited for debugging purposes only. + * + * @param timeMs + * @return + */ + @Deprecated + public Builder debugLongerIterations(long timeMs) { + if (timeMs < 0) + timeMs = 0L; + this.debugLongerIterations = timeMs; + return this; + } + + /** + * Optional method: Transport implementation to be used as TransportType.CUSTOM for VoidParameterAveraging method
+ * Generally not used by users + * + * @param transport Transport to use + * @return + */ + public Builder transport(Transport transport) { + this.transport = transport; + return this; + } + + /** + * Number of minibatches to asynchronously prefetch on each worker when training. Default: 2, which is usually suitable + * in most cases. Increasing this might help in some cases of ETL (data loading) bottlenecks, at the expense + * of greater memory consumption + * @param prefetchNumBatches Number of batches to prefetch + */ + public Builder workerPrefetchNumBatches(int prefetchNumBatches){ + this.workerPrefetchNumBatches = prefetchNumBatches; + return this; + } + + /** + * Repartitioner to use to repartition data before fitting.
+ * DL4J performs a MapPartitions operation for training, hence how the data is partitioned can matter a lot for + * performance - too few partitions (or very imbalanced partitions can result in poor cluster utilization, due to + * some workers being idle. A larger number of smaller partitions can help to avoid so-called "end-of-epoch" + * effects where training can only complete once the last/slowest worker finishes it's partition.
+ * Default repartitioner is {@link DefaultRepartitioner}, which repartitions equally up to a maximum of 5000 + * partitions, and is usually suitable for most purposes. In the worst case, the "end of epoch" effect + * when using the partitioner should be limited to a maximum of the amount of time required to process a single partition. + * + * @param repartitioner Repartitioner to use + */ + public Builder repartitioner(Repartitioner repartitioner){ + this.repartitioner = repartitioner; + return this; + } + + /** + * Used to disable the periodic garbage collection calls on the workers.
+ * Equivalent to {@code Nd4j.getMemoryManager().togglePeriodicGc(workerTogglePeriodicGC);}
+ * Pass false to disable periodic GC on the workers or true (equivalent to the default, or not setting it) to keep it enabled. + * + * @param workerTogglePeriodicGC Worker periodic garbage collection setting + */ + public Builder workerTogglePeriodicGC(boolean workerTogglePeriodicGC){ + this.workerTogglePeriodicGC = workerTogglePeriodicGC; + return this; + } + + /** + * Used to set the periodic garbage collection frequency on the workers.
+ * Equivalent to calling {@code Nd4j.getMemoryManager().setAutoGcWindow(workerPeriodicGCFrequency);} on each worker
+ * Does not have any effect if {@link #workerTogglePeriodicGC(boolean)} is set to false + * + * @param workerPeriodicGCFrequency The periodic GC frequency to use on the workers + */ + public Builder workerPeriodicGCFrequency(int workerPeriodicGCFrequency){ + this.workerPeriodicGCFrequency = workerPeriodicGCFrequency; + return this; + } + + /** + * Enable debug mode for threshold encoding. When enabled, various statistics for the threshold and the residual + * will be calculated and logged on each worker (at info log level).
+ * This information can be used to check if the encoding threshold is too big (for example, virtually all updates + * are much smaller than the threshold) or too big (majority of updates are much larger than the threshold).
+ * encodingDebugMode is disabled by default.
+ * IMPORTANT: enabling this has a performance overhead, and should not be enabled unless the debug information is actually required.
+ * + * @param enabled True to enable + */ + public Builder encodingDebugMode(boolean enabled){ + this.encodingDebugMode = enabled; + return this; + } + + public SharedTrainingMaster build() { + SharedTrainingMaster master = new SharedTrainingMaster(voidConfiguration, numWorkers, rddTrainingApproach, + storageLevel, collectTrainingStats, repartitionStrategy, repartition, + thresholdAlgorithm, residualPostProcessor, rddDataSetNumExamples, batchSize, + debugLongerIterations, numWorkersPerNode, workerPrefetchNumBatches, repartitioner, workerTogglePeriodicGC, + workerPeriodicGCFrequency, encodingDebugMode); + if (transport != null) + master.transport = this.transport; + + return master; + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingResult.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingResult.java new file mode 100644 index 000000000..7fb974001 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingResult.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.training; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.deeplearning4j.core.storage.Persistable; +import org.deeplearning4j.core.storage.StorageMetaData; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; +import org.deeplearning4j.spark.api.TrainingResult; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.impl.paramavg.BaseTrainingResult; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Map; + +@Data +@AllArgsConstructor +@Builder +@NoArgsConstructor +public class SharedTrainingResult extends BaseTrainingResult implements TrainingResult, Serializable { + private INDArray updaterStateArray; + private double scoreSum; + private int aggregationsCount; + private SparkTrainingStats sparkTrainingStats; + private Collection listenerMetaData; + private Collection listenerStaticInfo; + private Collection listenerUpdates; + private Map minibatchesPerExecutor; + private ThresholdAlgorithm thresholdAlgorithm; + + + @Override + public void setStats(SparkTrainingStats sparkTrainingStats) { + setSparkTrainingStats(sparkTrainingStats); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingWorker.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingWorker.java new file mode 100644 index 000000000..f64660ed5 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingWorker.java @@ -0,0 +1,203 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.training; + +import lombok.Getter; +import org.apache.spark.broadcast.Broadcast; +import org.deeplearning4j.core.storage.StatsStorageRouter; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.spark.api.TrainingHook; +import org.deeplearning4j.spark.api.TrainingWorker; +import org.deeplearning4j.spark.api.WorkerConfiguration; +import org.deeplearning4j.spark.api.stats.SparkTrainingStats; +import org.deeplearning4j.spark.api.worker.NetBroadcastTuple; +import org.deeplearning4j.spark.impl.paramavg.BaseTrainingWorker; +import org.deeplearning4j.spark.parameterserver.conf.SharedTrainingConfiguration; +import org.nd4j.linalg.dataset.api.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.common.primitives.Pair; + +import java.util.List; + +@Getter +public class SharedTrainingWorker extends BaseTrainingWorker + implements TrainingWorker { + + private final long instanceId; + private final Broadcast broadcastModel; + private final Broadcast broadcastConfiguration; + private final List listeners; + private final StatsStorageRouter router; + private final Boolean workerTogglePeriodicGC; + private final Integer workerPeriodicGCFrequency; + + public SharedTrainingWorker(long instanceId, Broadcast broadcastModel, + Broadcast broadcastConfiguration, + List listeners, StatsStorageRouter router, Boolean workerTogglePeriodicGC, + Integer workerPeriodicGCFrequency) { + this.instanceId = instanceId; + // our initial model is stored here. + this.broadcastModel = broadcastModel; + this.broadcastConfiguration = broadcastConfiguration; + this.listeners = listeners; + this.router = router; + this.workerTogglePeriodicGC = workerTogglePeriodicGC; + this.workerPeriodicGCFrequency = workerPeriodicGCFrequency; + } + + @Override + public void removeHook(TrainingHook trainingHook) { + throw new UnsupportedOperationException(); + } + + @Override + public void addHook(TrainingHook trainingHook) { + throw new UnsupportedOperationException(); + } + + @Override + public MultiLayerNetwork getInitialModel() { + if(workerTogglePeriodicGC != null) + Nd4j.getMemoryManager().togglePeriodicGc(workerTogglePeriodicGC); + if(workerPeriodicGCFrequency != null) + Nd4j.getMemoryManager().setAutoGcWindow(workerPeriodicGCFrequency); + + // This method will be called ONLY once, in master thread + //Before getting NetBroadcastTuple, to ensure it always gets mapped to device 0 + Nd4j.getAffinityManager().unsafeSetDevice(0); + + NetBroadcastTuple tuple = broadcastModel.getValue(); + if (tuple.getConfiguration() != null) { + MultiLayerConfiguration conf = tuple.getConfiguration(); + MultiLayerNetwork network = new MultiLayerNetwork(conf); + network.init(); + + if (tuple.getParameters() != null) + network.setParams(tuple.getParameters()); + + // we can assign properly, without + if (tuple.getUpdaterState() != null) + network.getUpdater().getStateViewArray().assign(tuple.getUpdaterState()); + + return network; + } else + return null; + } + + @Override + public ComputationGraph getInitialModelGraph() { + //Before getting NetBroadcastTuple, to ensure it always gets mapped to device 0 + Nd4j.getAffinityManager().unsafeSetDevice(0); + NetBroadcastTuple tuple = broadcastModel.getValue(); + if (tuple.getGraphConfiguration() != null) { + ComputationGraphConfiguration conf = tuple.getGraphConfiguration(); + ComputationGraph network = new ComputationGraph(conf); + network.init(); + + if (tuple.getParameters() != null) + network.setParams(tuple.getParameters()); + + if (tuple.getUpdaterState() != null) + network.getUpdater().getUpdaterStateViewArray().assign(tuple.getUpdaterState()); + + return network; + } else + return null; + } + + @Override + public SharedTrainingResult processMinibatch(DataSet dataSet, MultiLayerNetwork network, boolean isLast) { + /* + We're not really going to use this method for training. + Partitions will be mapped to ParallelWorker threads dynamically, wrt thread/device affinity. + So plan is simple: we're going to use individual partitions to feed main worker + */ + throw new UnsupportedOperationException(); + } + + @Override + public SharedTrainingResult processMinibatch(DataSet dataSet, ComputationGraph graph, boolean isLast) { + throw new UnsupportedOperationException(); + } + + @Override + public SharedTrainingResult processMinibatch(MultiDataSet dataSet, ComputationGraph graph, boolean isLast) { + throw new UnsupportedOperationException(); + } + + @Override + public Pair processMinibatchWithStats(DataSet dataSet, + MultiLayerNetwork network, boolean isLast) { + throw new UnsupportedOperationException(); + } + + @Override + public Pair processMinibatchWithStats(DataSet dataSet, + ComputationGraph graph, boolean isLast) { + throw new UnsupportedOperationException(); + } + + @Override + public Pair processMinibatchWithStats(MultiDataSet dataSet, + ComputationGraph graph, boolean isLast) { + throw new UnsupportedOperationException(); + } + + @Override + public SharedTrainingResult getFinalResult(MultiLayerNetwork network) { + throw new UnsupportedOperationException(); + } + + @Override + public SharedTrainingResult getFinalResult(ComputationGraph network) { + throw new UnsupportedOperationException(); + } + + @Override + public SharedTrainingResult getFinalResultNoData() { + throw new UnsupportedOperationException(); + } + + @Override + public Pair getFinalResultNoDataWithStats() { + throw new UnsupportedOperationException(); + } + + @Override + public Pair getFinalResultWithStats(MultiLayerNetwork network) { + throw new UnsupportedOperationException(); + } + + @Override + public Pair getFinalResultWithStats(ComputationGraph graph) { + throw new UnsupportedOperationException(); + } + + @Override + public WorkerConfiguration getDataConfiguration() { + throw new UnsupportedOperationException(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/util/BlockingObserver.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/util/BlockingObserver.java new file mode 100644 index 000000000..02be4f0a7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/util/BlockingObserver.java @@ -0,0 +1,56 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.util; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.Observable; +import java.util.Observer; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +@Data +public class BlockingObserver implements Observer { + protected AtomicBoolean state = new AtomicBoolean(false); + protected AtomicBoolean exception; + + public BlockingObserver(AtomicBoolean exception){ + this.exception = exception; + } + + @Override + public void update(Observable o, Object arg) { + state.set(true); + //notify(); + } + + /** + * This method blocks until state is set to True + */ + public void waitTillDone() throws InterruptedException { + while (!exception.get() && !state.get()) { + //LockSupport.parkNanos(1000L); + // we don't really need uber precision here, sleep is ok + Thread.sleep(5); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/util/CountingIterator.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/util/CountingIterator.java new file mode 100644 index 000000000..1b494b46a --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/util/CountingIterator.java @@ -0,0 +1,44 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.util; + +import lombok.AllArgsConstructor; + +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicInteger; + +@AllArgsConstructor +public class CountingIterator implements Iterator { + + private final Iterator iter; + private final AtomicInteger counter; + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public T next() { + counter.getAndIncrement(); + return iter.next(); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java new file mode 100644 index 000000000..d110e41bd --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java @@ -0,0 +1,140 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + + +public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable { + protected transient JavaSparkContext sc; + protected transient INDArray labels; + protected transient INDArray input; + protected transient INDArray rowSums; + protected transient int nRows = 200; + protected transient int nIn = 4; + protected transient int nOut = 3; + protected transient DataSet data; + protected transient JavaRDD sparkData; + + + @Override + public long getTimeoutMilliseconds() { + return 120000L; + } + + @BeforeEach + public void before() { + + sc = getContext(); + Random r = new Random(12345); + labels = Nd4j.create(nRows, nOut); + input = Nd4j.rand(nRows, nIn); + rowSums = input.sum(1); + input.diviColumnVector(rowSums); + + for (int i = 0; i < nRows; i++) { + int x1 = r.nextInt(nOut); + labels.putScalar(new int[] {i, x1}, 1.0); + } + + sparkData = getBasicSparkDataSet(nRows, input, labels); + } + + @AfterEach + public void after() { + sc.close(); + sc = null; + } + + /** + * + * @return + */ + public JavaSparkContext getContext() { + if (sc != null) + return sc; + // set to test mode + SparkConf sparkConf = new SparkConf().setMaster("local[" + numExecutors() + "]") + .set("spark.driver.host", "localhost").setAppName("sparktest"); + + + sc = new JavaSparkContext(sparkConf); + + return sc; + } + + protected JavaRDD getBasicSparkDataSet(int nRows, INDArray input, INDArray labels) { + List list = new ArrayList<>(); + for (int i = 0; i < nRows; i++) { + INDArray inRow = input.getRow(i, true).dup(); + INDArray outRow = labels.getRow(i, true).dup(); + + DataSet ds = new DataSet(inRow, outRow); + list.add(ds); + } + list.iterator(); + + data = DataSet.merge(list); + return sc.parallelize(list); + } + + + protected SparkDl4jMultiLayer getBasicNetwork() { + return new SparkDl4jMultiLayer(sc, getBasicConf(), + new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0)); + } + + protected int numExecutors() { + return 4; + } + + protected MultiLayerConfiguration getBasicConf() { + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123) + .updater(new Nesterovs(0.1, 0.9)).list() + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(nIn).nOut(3) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(3).nOut(nOut) + .activation(Activation.SOFTMAX).build()) + .build(); + return conf; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunctionTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunctionTest.java new file mode 100644 index 000000000..5b62cc038 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunctionTest.java @@ -0,0 +1,64 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.accumulation; + + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + + + +public class SharedTrainingAccumulationFunctionTest { + + @Test + public void testAccumulation1() throws Exception { + + INDArray updates1 = Nd4j.create(1000).assign(1.0); + INDArray updates2 = Nd4j.create(1000).assign(2.0); + INDArray expUpdates = Nd4j.create(1000).assign(3.0); + + SharedTrainingAccumulationTuple tuple1 = SharedTrainingAccumulationTuple.builder().updaterStateArray(updates1) + .scoreSum(1.0).aggregationsCount(1).build(); + + SharedTrainingAccumulationTuple tuple2 = SharedTrainingAccumulationTuple.builder().updaterStateArray(updates2) + .scoreSum(2.0).aggregationsCount(1).build(); + + SharedTrainingAccumulationFunction accumulationFunction = new SharedTrainingAccumulationFunction(); + + SharedTrainingAccumulationTuple tupleE = accumulationFunction.call(null, tuple1); + + // testing null + tuple accumulation + Assertions.assertEquals(1, tupleE.getAggregationsCount()); + Assertions.assertEquals(1.0, tupleE.getScoreSum(), 0.01); + Assertions.assertEquals(updates1, tupleE.getUpdaterStateArray()); + + + // testing tuple + tuple accumulation + SharedTrainingAccumulationTuple tupleResult = accumulationFunction.call(tuple1, tuple2); + Assertions.assertEquals(2, tupleResult.getAggregationsCount()); + Assertions.assertEquals(3.0, tupleResult.getScoreSum(), 0.01); + Assertions.assertEquals(expUpdates, tupleResult.getUpdaterStateArray()); + + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunctionTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunctionTest.java new file mode 100644 index 000000000..25ef434bd --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunctionTest.java @@ -0,0 +1,68 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.accumulation; + +import com.sun.jna.Platform; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingResult; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SharedTrainingAggregateFunctionTest { + @BeforeEach + public void setUp() throws Exception { + // + } + + @Test + public void testAggregate1() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + INDArray updates1 = Nd4j.create(1000).assign(1.0); + INDArray updates2 = Nd4j.create(1000).assign(2.0); + INDArray expUpdates = Nd4j.create(1000).assign(3.0); + + SharedTrainingResult result1 = SharedTrainingResult.builder().updaterStateArray(updates1).aggregationsCount(1) + .scoreSum(1.0).build(); + + SharedTrainingResult result2 = SharedTrainingResult.builder().updaterStateArray(updates2).aggregationsCount(1) + .scoreSum(2.0).build(); + + // testing null + result + SharedTrainingAggregateFunction aggregateFunction = new SharedTrainingAggregateFunction(); + SharedTrainingAccumulationTuple tuple1 = aggregateFunction.call(null, result1); + + + // testing tuple + result + SharedTrainingAccumulationTuple tuple2 = aggregateFunction.call(tuple1, result2); + + + // testing final result + assertEquals(2, tuple2.getAggregationsCount()); + assertEquals(3.0, tuple2.getScoreSum(), 0.001); + assertEquals(expUpdates, tuple2.getUpdaterStateArray()); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIteratorTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIteratorTest.java new file mode 100644 index 000000000..b837efe5e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIteratorTest.java @@ -0,0 +1,79 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.iterators; + +import com.sun.jna.Platform; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class VirtualDataSetIteratorTest { + @BeforeEach + public void setUp() throws Exception {} + + + @Test + public void testSimple1() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + List> iterators = new ArrayList<>(); + + List first = new ArrayList<>(); + List second = new ArrayList<>(); + + for (int i = 0; i < 100; i++) { + INDArray features = Nd4j.create(100).assign(i); + INDArray labels = Nd4j.create(10).assign(i); + DataSet ds = new DataSet(features, labels); + + if (i < 25) + first.add(ds); + else + second.add(ds); + } + + iterators.add(first.iterator()); + iterators.add(second.iterator()); + + VirtualDataSetIterator vdsi = new VirtualDataSetIterator(iterators); + int cnt = 0; + while (vdsi.hasNext()) { + DataSet ds = vdsi.next(); + + assertEquals((double) cnt, ds.getFeatures().meanNumber().doubleValue(), 0.0001); + assertEquals((double) cnt, ds.getLabels().meanNumber().doubleValue(), 0.0001); + + cnt++; + } + + assertEquals(100, cnt); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIteratorTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIteratorTest.java new file mode 100644 index 000000000..4e56b575a --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIteratorTest.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.iterators; + +import com.sun.jna.Platform; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class VirtualIteratorTest { + @BeforeEach + public void setUp() throws Exception { + // + } + + @Test + public void testIteration1() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + List integers = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + integers.add(i); + } + + VirtualIterator virt = new VirtualIterator<>(integers.iterator()); + + int cnt = 0; + while (virt.hasNext()) { + Integer n = virt.next(); + assertEquals(cnt, n.intValue()); + cnt++; + } + + + assertEquals(100, cnt); + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/TestElephasImport.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/TestElephasImport.java new file mode 100644 index 000000000..16429f41e --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/TestElephasImport.java @@ -0,0 +1,124 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.modelimport.elephas; + +import com.sun.jna.Platform; +import org.apache.spark.api.java.JavaSparkContext; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; +import org.deeplearning4j.spark.parameterserver.BaseSparkTest; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingMaster; +import org.junit.jupiter.api.Test; +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +import static java.io.File.createTempFile; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestElephasImport extends BaseSparkTest { + + @Test + public void testElephasSequentialImport() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + String modelPath = "modelimport/elephas/elephas_sequential.h5"; + SparkDl4jMultiLayer model = importElephasSequential(sc, modelPath); + // System.out.println(model.getNetwork().summary()); + assertTrue(model.getTrainingMaster() instanceof ParameterAveragingTrainingMaster); + } + + @Test + public void testElephasSequentialImportAsync() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + String modelPath = "modelimport/elephas/elephas_sequential_async.h5"; + SparkDl4jMultiLayer model = importElephasSequential(sc, modelPath); + // System.out.println(model.getNetwork().summary()); + assertTrue(model.getTrainingMaster() instanceof SharedTrainingMaster); + } + + private SparkDl4jMultiLayer importElephasSequential(JavaSparkContext sc, String modelPath) throws Exception { + + ClassPathResource modelResource = + new ClassPathResource(modelPath, + TestElephasImport.class.getClassLoader()); + File modelFile = createTempFile("tempModel", "h5"); + Files.copy(modelResource.getInputStream(), modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + SparkDl4jMultiLayer model = ElephasModelImport.importElephasSequentialModelAndWeights(sc, modelFile.getAbsolutePath()); + return model; + } + + + @Test + public void testElephasModelImport() throws Exception { + + String modelPath = "modelimport/elephas/elephas_model.h5"; + SparkComputationGraph model = importElephasModel(sc, modelPath); + // System.out.println(model.getNetwork().summary()); + assertTrue(model.getTrainingMaster() instanceof ParameterAveragingTrainingMaster); + } + + @Test + public void testElephasJavaAveragingModelImport() throws Exception { + + String modelPath = "modelimport/elephas/java_param_averaging_model.h5"; + SparkComputationGraph model = importElephasModel(sc, modelPath); + // System.out.println(model.getNetwork().summary()); + assert model.getTrainingMaster() instanceof ParameterAveragingTrainingMaster; + } + + @Test + public void testElephasJavaSharingModelImport() throws Exception { + + String modelPath = "modelimport/elephas/java_param_sharing_model.h5"; + SparkComputationGraph model = importElephasModel(sc, modelPath); + // System.out.println(model.getNetwork().summary()); + assert model.getTrainingMaster() instanceof SharedTrainingMaster; + } + + @Test + public void testElephasModelImportAsync() throws Exception { + + String modelPath = "modelimport/elephas/elephas_model_async.h5"; + SparkComputationGraph model = importElephasModel(sc, modelPath); + // System.out.println(model.getNetwork().summary()); + assertTrue(model.getTrainingMaster() instanceof SharedTrainingMaster); + } + + private SparkComputationGraph importElephasModel(JavaSparkContext sc, String modelPath) throws Exception { + + ClassPathResource modelResource = + new ClassPathResource(modelPath, + TestElephasImport.class.getClassLoader()); + File modelFile = createTempFile("tempModel", "h5"); + Files.copy(modelResource.getInputStream(), modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + SparkComputationGraph model = ElephasModelImport.importElephasModelAndWeights(sc, modelFile.getAbsolutePath()); + return model; + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java new file mode 100644 index 000000000..31cd119d7 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java @@ -0,0 +1,409 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.parameterserver.train; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.api.java.JavaRDD; +import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.api.BaseTrainingListener; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.ThresholdAlgorithm; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.threshold.AdaptiveThresholdAlgorithm; +import org.deeplearning4j.optimize.solvers.accumulation.encoding.threshold.FixedThresholdAlgorithm; +import org.deeplearning4j.spark.api.RDDTrainingApproach; +import org.deeplearning4j.spark.api.TrainingMaster; +import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; +import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.parameterserver.BaseSparkTest; +import org.deeplearning4j.spark.parameterserver.training.SharedTrainingMaster; + + +import org.junit.jupiter.api.Test; + +import org.junit.jupiter.api.io.TempDir; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.linalg.learning.config.AMSGrad; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.linalg.ops.transforms.Transforms; +import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; +import org.nd4j.parameterserver.distributed.v2.enums.MeshBuildMode; + +import java.io.File; +import java.io.Serializable; +import java.net.Inet4Address; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +////@Ignore("AB 2019/05/21 - Failing - Issue #7657") +public class GradientSharingTrainingTest extends BaseSparkTest { + + @TempDir + public File testDir; + + @Override + public long getTimeoutMilliseconds() { + return 180000L; + } + + @Test + public void trainSanityCheck() throws Exception { + + for(boolean mds : new boolean[]{false, true}) { + INDArray last = null; + INDArray lastDup = null; + for (String s : new String[]{"paths", "direct", "export"}) { + System.out.println("--------------------------------------------------------------------------------------------------------------"); + log.info("Starting: {} - {}", s, (mds ? "MultiDataSet" : "DataSet")); + boolean isPaths = "paths".equals(s); + + RDDTrainingApproach rddTrainingApproach; + switch (s) { + case "direct": + rddTrainingApproach = RDDTrainingApproach.Direct; + break; + case "export": + rddTrainingApproach = RDDTrainingApproach.Export; + break; + case "paths": + rddTrainingApproach = RDDTrainingApproach.Direct; //Actualy not used for fitPaths + break; + default: + throw new RuntimeException(); + } + + File temp = testDir; + + + //TODO this probably won't work everywhere... + String controller = Inet4Address.getLocalHost().getHostAddress(); + String networkMask = controller.substring(0, controller.lastIndexOf('.')) + ".0" + "/16"; + + VoidConfiguration voidConfiguration = VoidConfiguration.builder() + .unicastPort(40123) // Should be open for IN/OUT communications on all Spark nodes + .networkMask(networkMask) // Local network mask + .controllerAddress(controller) + .meshBuildMode(MeshBuildMode.PLAIN) // everyone is connected to the master + .build(); + TrainingMaster tm = new SharedTrainingMaster.Builder(voidConfiguration, 2, new AdaptiveThresholdAlgorithm(1e-3), 16) + .rngSeed(12345) + .collectTrainingStats(false) + .batchSizePerWorker(16) // Minibatch size for each worker + .workersPerNode(2) // Workers per node + .rddTrainingApproach(rddTrainingApproach) + .exportDirectory("file:///" + temp.getAbsolutePath().replaceAll("\\\\", "/")) + .build(); + + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345) + .updater(new AMSGrad(0.1)) + .graphBuilder() + .addInputs("in") + .layer("out", new OutputLayer.Builder().nIn(784).nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in") + .setOutputs("out") + .build(); + + + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf, tm); + sparkNet.setCollectTrainingStats(tm.getIsCollectTrainingStats()); + +// System.out.println(Arrays.toString(sparkNet.getNetwork().params().get(NDArrayIndex.point(0), NDArrayIndex.interval(0, 256)).dup().data().asFloat())); + File f = testDir; + DataSetIterator iter = new MnistDataSetIterator(16, true, 12345); + int count = 0; + List paths = new ArrayList<>(); + List ds = new ArrayList<>(); + while (iter.hasNext() && count++ < 8) { + DataSet d = iter.next(); + if (isPaths) { + File out = new File(f, count + ".bin"); + if(mds){ + d.toMultiDataSet().save(out); + } else { + d.save(out); + } + String path = "file:///" + out.getAbsolutePath().replaceAll("\\\\", "/"); + paths.add(path); + } + ds.add(d); + } + + int numIter = 1; + double[] acc = new double[numIter + 1]; + for (int i = 0; i < numIter; i++) { + //Check accuracy before: + DataSetIterator testIter = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(32, false, 12345), 10); + Evaluation eBefore = sparkNet.getNetwork().evaluate(testIter); + + INDArray paramsBefore = sparkNet.getNetwork().params().dup(); + ComputationGraph after; + if(mds) { + //Fitting from MultiDataSet + List mdsList = new ArrayList<>(); + for(DataSet d : ds){ + mdsList.add(d.toMultiDataSet()); + } + switch (s) { + case "direct": + case "export": + JavaRDD dsRDD = sc.parallelize(mdsList); + after = sparkNet.fitMultiDataSet(dsRDD); + break; + case "paths": + JavaRDD pathRdd = sc.parallelize(paths); + after = sparkNet.fitPathsMultiDataSet(pathRdd); + break; + default: + throw new RuntimeException(); + } + } else { + //Fitting from DataSet + switch (s) { + case "direct": + case "export": + JavaRDD dsRDD = sc.parallelize(ds); + after = sparkNet.fit(dsRDD); + break; + case "paths": + JavaRDD pathRdd = sc.parallelize(paths); + after = sparkNet.fitPaths(pathRdd); + break; + default: + throw new RuntimeException(); + } + } + + INDArray paramsAfter = after.params(); +// System.out.println(Arrays.toString(paramsBefore.get(NDArrayIndex.point(0), NDArrayIndex.interval(0, 256)).dup().data().asFloat())); +// System.out.println(Arrays.toString(paramsAfter.get(NDArrayIndex.point(0), NDArrayIndex.interval(0, 256)).dup().data().asFloat())); +// System.out.println(Arrays.toString( +// Transforms.abs(paramsAfter.sub(paramsBefore)).get(NDArrayIndex.point(0), NDArrayIndex.interval(0, 256)).dup().data().asFloat())); + assertNotEquals(paramsBefore, paramsAfter); + + + testIter = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(32, false, 12345), 10); + Evaluation eAfter = after.evaluate(testIter); + + double accAfter = eAfter.accuracy(); + double accBefore = eBefore.accuracy(); + assertTrue(accAfter >= accBefore + 0.005, "after: " + accAfter + ", before=" + accBefore); + + if (i == 0) { + acc[0] = eBefore.accuracy(); + } + acc[i + 1] = eAfter.accuracy(); + } + log.info("Accuracies: {}", Arrays.toString(acc)); + last = sparkNet.getNetwork().params(); + lastDup = last.dup(); + } + } + } + + + @Test //@Ignore //AB https://github.com/eclipse/deeplearning4j/issues/8985 + public void differentNetsTrainingTest() throws Exception { + int batch = 3; + + File temp = testDir; + DataSet ds = new IrisDataSetIterator(150, 150).next(); + List list = ds.asList(); + Collections.shuffle(list, new Random(12345)); + int pos = 0; + int dsCount = 0; + while (pos < list.size()) { + List l2 = new ArrayList<>(); + for (int i = 0; i < 3 && pos < list.size(); i++) { + l2.add(list.get(pos++)); + } + DataSet d = DataSet.merge(l2); + File f = new File(temp, dsCount++ + ".bin"); + d.save(f); + } + + INDArray last = null; + INDArray lastDup = null; + for (int i = 0; i < 2; i++) { + System.out.println("||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||"); + log.info("Starting: {}", i); + + MultiLayerConfiguration conf; + if (i == 0) { + conf = new NeuralNetConfiguration.Builder() + .weightInit(WeightInit.XAVIER) + .seed(12345) + .list() + .layer(new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + } else { + conf = new NeuralNetConfiguration.Builder() + .weightInit(WeightInit.XAVIER) + .seed(12345) + .list() + .layer(new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()) + .layer(new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()) + .build(); + } + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + + //TODO this probably won't work everywhere... + String controller = Inet4Address.getLocalHost().getHostAddress(); + String networkMask = controller.substring(0, controller.lastIndexOf('.')) + ".0" + "/16"; + + VoidConfiguration voidConfiguration = VoidConfiguration.builder() + .unicastPort(40123) // Should be open for IN/OUT communications on all Spark nodes + .networkMask(networkMask) // Local network mask + .controllerAddress(controller) + .build(); + TrainingMaster tm = new SharedTrainingMaster.Builder(voidConfiguration, 2, new FixedThresholdAlgorithm(1e-4), batch) + .rngSeed(12345) + .collectTrainingStats(false) + .batchSizePerWorker(batch) // Minibatch size for each worker + .workersPerNode(2) // Workers per node + .build(); + + + SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, net, tm); + + //System.out.println(Arrays.toString(sparkNet.getNetwork().params().get(NDArrayIndex.point(0), NDArrayIndex.interval(0, 256)).dup().data().asFloat())); + + String fitPath = "file:///" + temp.getAbsolutePath().replaceAll("\\\\", "/"); + INDArray paramsBefore = net.params().dup(); + for( int j=0; j<3; j++ ) { + sparkNet.fit(fitPath); + } + + INDArray paramsAfter = net.params(); + assertNotEquals(paramsBefore, paramsAfter); + + //Also check we don't have any issues + if(i == 0) { + last = sparkNet.getNetwork().params(); + lastDup = last.dup(); + } else { + assertEquals(lastDup, last); + } + } + } + + + @Test //@Ignore + public void testEpochUpdating() throws Exception { + //Ensure that epoch counter is incremented properly on the workers + + File temp = testDir; + + //TODO this probably won't work everywhere... + String controller = Inet4Address.getLocalHost().getHostAddress(); + String networkMask = controller.substring(0, controller.lastIndexOf('.')) + ".0" + "/16"; + + VoidConfiguration voidConfiguration = VoidConfiguration.builder() + .unicastPort(40123) // Should be open for IN/OUT communications on all Spark nodes + .networkMask(networkMask) // Local network mask + .controllerAddress(controller) + .meshBuildMode(MeshBuildMode.PLAIN) // everyone is connected to the master + .build(); + SharedTrainingMaster tm = new SharedTrainingMaster.Builder(voidConfiguration, 2, new AdaptiveThresholdAlgorithm(1e-3), 16) + .rngSeed(12345) + .collectTrainingStats(false) + .batchSizePerWorker(16) // Minibatch size for each worker + .workersPerNode(2) // Workers per node + .exportDirectory("file:///" + temp.getAbsolutePath().replaceAll("\\\\", "/")) + .build(); + + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .seed(12345) + .updater(new AMSGrad(0.001)) + .graphBuilder() + .addInputs("in") + .layer("out", new OutputLayer.Builder().nIn(784).nOut(10).activation(Activation.SOFTMAX) + .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in") + .setOutputs("out") + .build(); + + + SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf, tm); + sparkNet.setListeners(new TestListener()); + + DataSetIterator iter = new MnistDataSetIterator(16, true, 12345); + int count = 0; + List paths = new ArrayList<>(); + List ds = new ArrayList<>(); + File f = testDir; + while (iter.hasNext() && count++ < 8) { + DataSet d = iter.next(); + File out = new File(f, count + ".bin"); + d.save(out); + String path = "file:///" + out.getAbsolutePath().replaceAll("\\\\", "/"); + paths.add(path); + ds.add(d); + } + + JavaRDD pathRdd = sc.parallelize(paths); + for( int i=0; i<3; i++ ) { + ThresholdAlgorithm ta = tm.getThresholdAlgorithm(); + sparkNet.fitPaths(pathRdd); + //Check also that threshold algorithm was updated/averaged + ThresholdAlgorithm taAfter = tm.getThresholdAlgorithm(); + assertTrue(ta != taAfter, "Threshold algorithm should have been updated with different instance after averaging"); + AdaptiveThresholdAlgorithm ataAfter = (AdaptiveThresholdAlgorithm) taAfter; + assertFalse(Double.isNaN(ataAfter.getLastSparsity())); + assertFalse(Double.isNaN(ataAfter.getLastThreshold())); + } + + Set expectedEpochs = new HashSet<>(Arrays.asList(0, 1, 2)); + assertEquals(expectedEpochs, TestListener.epochs); + } + + private static class TestListener extends BaseTrainingListener implements Serializable { + private static final Set iterations = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private static final Set epochs = Collections.newSetFromMap(new ConcurrentHashMap<>()); + @Override + public void iterationDone(Model model, int iteration, int epoch) { + iterations.add(iteration); + epochs.add(epoch); + } + } +} diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/resources/log4j.properties b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/resources/log4j.properties new file mode 100644 index 000000000..64c5034c9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/resources/log4j.properties @@ -0,0 +1,35 @@ +# +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +log4j.rootLogger=ERROR, Console +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n + +log4j.appender.org.springframework=DEBUG +log4j.appender.org.deeplearning4j=INFO +log4j.appender.org.nd4j=INFO + +log4j.logger.org.springframework=INFO +log4j.logger.org.deeplearning4j=INFO +log4j.logger.org.nd4j=INFO +log4j.logger.org.apache.spark=WARN + + diff --git a/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/resources/logback.xml b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/resources/logback.xml new file mode 100644 index 000000000..c269334de --- /dev/null +++ b/cavis-dnn/cavis-dnn-spark/cavis-dnn-spark-parameterserver/src/test/resources/logback.xml @@ -0,0 +1,57 @@ + + + + + + + + logs/application.log + + %logger{15} - %message%n%xException{5} + + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + + diff --git a/cavis-dnn/cavis-dnn-tsne/build.gradle b/cavis-dnn/cavis-dnn-tsne/build.gradle new file mode 100644 index 000000000..d9f73f331 --- /dev/null +++ b/cavis-dnn/cavis-dnn-tsne/build.gradle @@ -0,0 +1,32 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisDnn.cavisDnnNnParent.cavisDnnNnCore + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnApi + implementation "com.google.guava:guava" + implementation "org.slf4j:slf4j-api" + implementation "org.apache.commons:commons-math3" + testImplementation projects.cavisDnn.cavisDnnCommonTests +} \ No newline at end of file diff --git a/cavis-dnn/cavis-dnn-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java b/cavis-dnn/cavis-dnn-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java new file mode 100644 index 000000000..35122d29d --- /dev/null +++ b/cavis-dnn/cavis-dnn-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java @@ -0,0 +1,1063 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.plot; + + +import com.google.common.util.concurrent.AtomicDouble; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.sptree.DataPoint; +import org.deeplearning4j.clustering.sptree.SpTree; +import org.deeplearning4j.clustering.vptree.VPTree; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.WorkspaceMode; +import org.deeplearning4j.nn.gradient.DefaultGradient; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.deeplearning4j.optimize.api.ConvexOptimizer; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; +import org.nd4j.linalg.api.memory.enums.*; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.BooleanIndexing; +import org.nd4j.linalg.indexing.conditions.Conditions; +import org.nd4j.linalg.learning.legacy.AdaGrad; +import org.nd4j.common.primitives.Pair; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.*; + +import static org.nd4j.linalg.factory.Nd4j.*; +import static org.nd4j.linalg.ops.transforms.Transforms.pow; +import static org.nd4j.linalg.ops.transforms.Transforms.sign; + + +/** + * Barnes hut algorithm for TSNE, uses a dual tree approximation approach. + * Work based on: + * http://lvdmaaten.github.io/tsne/ + * For hight dimensions, it's recommended to reduce the dimension up to 50 using another method (PCA or other) + * @author Adam Gibson + */ +@Slf4j +@Data +public class BarnesHutTsne implements Model { + + + public final static String workspaceCache = "LOOP_CACHE"; + public final static String workspaceExternal = "LOOP_EXTERNAL"; + + + protected int maxIter = 1000; + protected double realMin = Nd4j.EPS_THRESHOLD; + protected double initialMomentum = 0.5; + protected double finalMomentum = 0.8; + protected double minGain = 1e-2; + protected double momentum = initialMomentum; + protected int switchMomentumIteration = 250; + protected boolean normalize = true; + protected boolean usePca = false; + protected int stopLyingIteration = 250; + protected double tolerance = 1e-5; + protected double learningRate = 500; + protected AdaGrad adaGrad; + protected boolean useAdaGrad = true; + protected double perplexity = 30; + //protected INDArray gains,yIncs; + protected INDArray Y; + private int N; + private double theta; + private INDArray rows; + private INDArray cols; + private INDArray vals; + private String simiarlityFunction = "cosinesimilarity"; + private boolean invert = true; + private INDArray x; + private int numDimensions = 0; + public final static String Y_GRAD = "yIncs"; + private SpTree tree; + private INDArray gains; + @Setter + private INDArray yIncs; + private int vpTreeWorkers; + protected transient TrainingListener trainingListener; + protected WorkspaceMode workspaceMode; + private Initializer initializer; + + protected final static WorkspaceConfiguration workspaceConfigurationExternal = WorkspaceConfiguration.builder() + .initialSize(0).overallocationLimit(0.3).policyLearning(LearningPolicy.FIRST_LOOP) + .policyReset(ResetPolicy.BLOCK_LEFT).policySpill(SpillPolicy.REALLOCATE) + .policyAllocation(AllocationPolicy.OVERALLOCATE).build(); + + protected WorkspaceConfiguration workspaceConfigurationFeedForward = WorkspaceConfiguration.builder().initialSize(0) + .overallocationLimit(0.2).policyReset(ResetPolicy.BLOCK_LEFT) + .policyLearning(LearningPolicy.OVER_TIME).policySpill(SpillPolicy.REALLOCATE) + .policyAllocation(AllocationPolicy.OVERALLOCATE).build(); + + public final static WorkspaceConfiguration workspaceConfigurationCache = WorkspaceConfiguration.builder() + .overallocationLimit(0.2).policyReset(ResetPolicy.BLOCK_LEFT).cyclesBeforeInitialization(3) + .policyMirroring(MirroringPolicy.FULL).policySpill(SpillPolicy.REALLOCATE) + .policyLearning(LearningPolicy.OVER_TIME).build(); + + + public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter, + double realMin, double initialMomentum, double finalMomentum, double momentum, + int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance, + double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener, + double minGain,int vpTreeWorkers) { + this(numDimensions, simiarlityFunction, theta, invert, maxIter, realMin, initialMomentum, finalMomentum, + momentum, switchMomentumIteration, normalize, stopLyingIteration, tolerance, learningRate, + useAdaGrad, perplexity, TrainingListener, minGain, vpTreeWorkers, WorkspaceMode.NONE, null); + } + + public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter, + double realMin, double initialMomentum, double finalMomentum, double momentum, + int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance, + double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener, + double minGain,int vpTreeWorkers, WorkspaceMode workspaceMode, INDArray staticInput) { + this.maxIter = maxIter; + this.realMin = realMin; + this.initialMomentum = initialMomentum; + this.finalMomentum = finalMomentum; + this.momentum = momentum; + this.normalize = normalize; + this.useAdaGrad = useAdaGrad; + this.stopLyingIteration = stopLyingIteration; + this.learningRate = learningRate; + this.switchMomentumIteration = switchMomentumIteration; + this.tolerance = tolerance; + this.perplexity = perplexity; + this.minGain = minGain; + this.numDimensions = numDimensions; + this.simiarlityFunction = simiarlityFunction; + this.theta = theta; + this.trainingListener = TrainingListener; + this.invert = invert; + this.vpTreeWorkers = vpTreeWorkers; + this.workspaceMode = workspaceMode; + if(this.workspaceMode == null) + this.workspaceMode = WorkspaceMode.NONE; + initializer = (staticInput != null) ? new Initializer(staticInput) : new Initializer(); + } + + + public String getSimiarlityFunction() { + return simiarlityFunction; + } + + public void setSimiarlityFunction(String simiarlityFunction) { + this.simiarlityFunction = simiarlityFunction; + } + + public boolean isInvert() { + return invert; + } + + public void setInvert(boolean invert) { + this.invert = invert; + } + + public double getTheta() { + return theta; + } + + public double getPerplexity() { + return perplexity; + } + + public int getNumDimensions() { + return numDimensions; + } + + public void setNumDimensions(int numDimensions) { + this.numDimensions = numDimensions; + } + + /** + * Convert data to probability + * co-occurrences (aka calculating the kernel) + * @param d the data to convert + * @param perplexity the perplexity of the model + * @return the probabilities of co-occurrence + */ + public INDArray computeGaussianPerplexity(final INDArray d, double perplexity) { + N = d.rows(); + + final int k = (int) (3 * perplexity); + if (N - 1 < 3 * perplexity) + throw new IllegalStateException("Perplexity " + perplexity + "is too large for number of samples " + N); + + + rows = zeros(DataType.INT, 1, N + 1); + cols = zeros(DataType.INT, 1, N * k); + vals = zeros(d.dataType(), N * k); + + for (int n = 0; n < N; n++) + rows.putScalar(n + 1, rows.getDouble(n) + k); + + final double enthropy = Math.log(perplexity); + VPTree tree = new VPTree(d, simiarlityFunction, vpTreeWorkers,invert); + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + log.info("Calculating probabilities of data similarities..."); + for (int i = 0; i < N; i++) { + if (i % 500 == 0) + log.info("Handled " + i + " records"); + + double betaMin = -Double.MAX_VALUE; + double betaMax = Double.MAX_VALUE; + List results = new ArrayList<>(); + List distances = new ArrayList<>(); + tree.search(d.getRow(i), k + 1, results, distances, false, true); + double betas = 1.0; + + if(results.size() == 0){ + throw new IllegalStateException("Search returned no values for vector " + i + + " - similarity \"" + simiarlityFunction + "\" may not be defined (for example, vector is" + + " all zeros with cosine similarity)"); + } + + Double[] dists = new Double[distances.size()]; + distances.toArray(dists); + INDArray cArr = Nd4j.createFromArray(dists).castTo(d.dataType()); //VPTree.buildFromData(results); + + INDArray currP = null; + int tries = 0; + boolean found = false; + //binary search + while (!found && tries < 200) { + Pair pair = computeGaussianKernel(cArr, betas, k); + currP = pair.getFirst(); + double hDiff = pair.getSecond() - enthropy; + + if (hDiff < tolerance && -hDiff < tolerance) + found = true; + else { + if (hDiff > 0) { + betaMin = betas; + + if (betaMax == Double.MAX_VALUE || betaMax == -Double.MAX_VALUE) + betas *= 2; + else + betas = (betas + betaMax) / 2.0; + } else { + betaMax = betas; + if (betaMin == -Double.MAX_VALUE || betaMin == Double.MAX_VALUE) + betas /= 2.0; + else + betas = (betas + betaMin) / 2.0; + } + + tries++; + } + } + + currP.divi(currP.sumNumber().doubleValue() + Double.MIN_VALUE); + INDArray indices = Nd4j.create(1, k + 1); + for (int j = 0; j < indices.length(); j++) { + if (j >= results.size()) + break; + indices.putScalar(j, results.get(j).getIndex()); + } + + for (int l = 0; l < k; l++) { + cols.putScalar(rows.getInt(i) + l, indices.getDouble(l + 1)); + vals.putScalar(rows.getInt(i) + l, currP.getDouble(l)); + } + } + } + return vals; + } + + @Override + public INDArray input() { + return x; + } + + @Override + public ConvexOptimizer getOptimizer() { + return null; + } + + @Override + public INDArray getParam(String param) { + return null; + } + + @Override + public void addListeners(TrainingListener... listener) { + // no-op + } + + @Override + public Map paramTable() { + return null; + } + + @Override + public Map paramTable(boolean backprapParamsOnly) { + return null; + } + + @Override + public void setParamTable(Map paramTable) { + + } + + @Override + public void setParam(String key, INDArray val) { + + } + + @Override + public void clear() {} + + @Override + public void applyConstraints(int iteration, int epoch) { + //No op + } + + /* compute the gradient given the current solution, the probabilities and the constant */ + protected Pair gradient(INDArray p) { + throw new UnsupportedOperationException(); + } + + + @Data + @AllArgsConstructor + static class SymResult { + INDArray rows; + INDArray cols; + INDArray vals; + } + + /** + * Symmetrize the value matrix + * @param rowP + * @param colP + * @param valP + * @return + */ + public SymResult symmetrized(INDArray rowP, INDArray colP, INDArray valP) { + INDArray rowCounts = Nd4j.create(DataType.INT, N); + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + for (int n = 0; n < N; n++) { + int begin = rowP.getInt(n); + int end = rowP.getInt(n + 1); + for (int i = begin; i < end; i++) { + boolean present = false; + for (int m = rowP.getInt(colP.getInt(i)); m < rowP.getInt(colP.getInt(i) + 1); m++) + if (colP.getInt(m) == n) { + present = true; + } + + if (present) + rowCounts.putScalar(n, rowCounts.getInt(n) + 1); + + else { + rowCounts.putScalar(n, rowCounts.getInt(n) + 1); + rowCounts.putScalar(colP.getInt(i), rowCounts.getInt(colP.getInt(i)) + 1); + } + } + } + + int numElements = rowCounts.sumNumber().intValue(); + INDArray offset = Nd4j.create(DataType.INT, N); + INDArray symRowP = Nd4j.zeros(DataType.INT, N + 1); + INDArray symColP = Nd4j.create(DataType.INT, numElements); + INDArray symValP = Nd4j.create(valP.dataType(), numElements); + + for (int n = 0; n < N; n++) + symRowP.putScalar(n + 1, symRowP.getInt(n) + rowCounts.getInt(n)); + + for (int n = 0; n < N; n++) { + for (int i = rowP.getInt(n); i < rowP.getInt(n + 1); i++) { + boolean present = false; + for (int m = rowP.getInt(colP.getInt(i)); m < rowP.getInt(colP.getInt(i)+1); m++) { + if (colP.getInt(m) == n) { + present = true; + if (n <= colP.getInt(i)) { + // make sure we do not add elements twice + symColP.putScalar(symRowP.getInt(n) + offset.getInt(n), colP.getInt(i)); + symColP.putScalar(symRowP.getInt(colP.getInt(i)) + offset.getInt(colP.getInt(i)), n); + symValP.putScalar(symRowP.getInt(n) + offset.getInt(n), + valP.getDouble(i) + valP.getDouble(m)); + symValP.putScalar(symRowP.getInt(colP.getInt(i)) + offset.getInt(colP.getInt(i)), + valP.getDouble(i) + valP.getDouble(m)); + } + } + } + + // If (colP[i], n) is not present, there is no addition involved + if (!present) { + int colPI = colP.getInt(i); + symColP.putScalar(symRowP.getInt(n) + offset.getInt(n), colPI); + symColP.putScalar(symRowP.getInt(colP.getInt(i)) + offset.getInt(colPI), n); + symValP.putScalar(symRowP.getInt(n) + offset.getInt(n), valP.getDouble(i)); + symValP.putScalar(symRowP.getInt(colPI) + offset.getInt(colPI), valP.getDouble(i)); + } + + // Update offsets + if (!present || (present && n <= colP.getInt(i))) { + offset.putScalar(n, offset.getInt(n) + 1); + int colPI = colP.getInt(i); + if (colPI != n) + offset.putScalar(colPI, offset.getInt(colPI) + 1); + } + } + } + + // Divide the result by two + symValP.divi(2.0D); + return new SymResult(symRowP, symColP, symValP); + + } + + + } + + /** + * Computes a gaussian kernel + * given a vector of squared distance distances + * + * @param distances + * @param beta + * @return + */ + public Pair computeGaussianKernel(INDArray distances, double beta, int k) { + // Compute Gaussian kernel row + INDArray currP = Nd4j.create(distances.dataType(), k); + for (int m = 0; m < k; m++) { + currP.putScalar(m, Math.exp(-beta * distances.getDouble(m + 1))); + } + + double sum = currP.sumNumber().doubleValue() + Double.MIN_VALUE; + double h = 0.0; + for (int m = 0; m < k; m++) + h += beta * (distances.getDouble(m + 1) * currP.getDouble(m)); + + h = (h / sum) + Math.log(sum); + + return new Pair<>(currP, h); + } + + + /** + * Init the model + */ + @Override + public void init() { + + } + + /** + * Set the trainingListeners for the ComputationGraph (and all layers in the network) + * + * @param listeners + */ + @Override + public void setListeners(Collection listeners) { + + } + + /** + * Set the trainingListeners for the ComputationGraph (and all layers in the network) + * + * @param listeners + */ + @Override + public void setListeners(TrainingListener... listeners) { + + } + + private int calculateOutputLength() { + int ret = 0; + + INDArray rowCounts = Nd4j.create(N); + for (int n = 0; n < N; n++) { + int begin = rows.getInt(n); + int end = rows.getInt(n + 1); + for (int i = begin; i < end; i++) { + boolean present = false; + for (int m = rows.getInt(cols.getInt(i)); m < rows.getInt(cols.getInt(i) + 1); m++) { + if (cols.getInt(m) == n) { + present = true; + } + } + if (present) + rowCounts.putScalar(n, rowCounts.getDouble(n) + 1); + + else { + rowCounts.putScalar(n, rowCounts.getDouble(n) + 1); + rowCounts.putScalar(cols.getInt(i), rowCounts.getDouble(cols.getInt(i)) + 1); + } + } + } + ret = rowCounts.sum(Integer.MAX_VALUE).getInt(0); + return ret; + } + + public class Initializer { + + private INDArray staticData; + + public Initializer() {} + + public Initializer(INDArray input) { + this.staticData = input; + } + + public INDArray initData() { + if (staticData != null) + return staticData.dup(); + return randn(x.dataType(), x.rows(), numDimensions).muli(1e-3f); + } + } + + public static void zeroMean(INDArray input) { + INDArray means = input.mean(0); + input.subiRowVector(means); + } + + @Override + public void fit() { + if (theta == 0.0) { + log.debug("theta == 0, using decomposed version, might be slow"); + Tsne decomposedTsne = new Tsne(maxIter, realMin, initialMomentum, finalMomentum, minGain, momentum, + switchMomentumIteration, normalize, usePca, stopLyingIteration, tolerance, learningRate, + useAdaGrad, perplexity); + Y = decomposedTsne.calculate(x, numDimensions, perplexity); + } else { + //output + if (Y == null) { + Y = initializer.initData(); + } + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + x.divi(x.maxNumber()); + + computeGaussianPerplexity(x, perplexity); + /*INDArray outRows = Nd4j.create(new int[]{rows.rows(), rows.columns()}, DataType.INT); + BarnesHutSymmetrize op = new BarnesHutSymmetrize(rows, cols, vals, N, outRows); + Nd4j.getExecutioner().exec(op); + INDArray output = op.getSymmetrizedValues(); + INDArray outCols = op.getSymmetrizedCols(); + vals = output.divi(vals.sum(Integer.MAX_VALUE)); + rows = outRows; + cols = outCols;*/ + + SymResult result = symmetrized(rows, cols, vals); + vals = result.vals.divi(result.vals.sumNumber().doubleValue()); + rows = result.rows; + cols = result.cols; + //lie about gradient + vals.muli(12); + for (int i = 0; i < maxIter; i++) { + step(vals, i); + zeroMean(Y); + if (i == switchMomentumIteration) + momentum = finalMomentum; + if (i == stopLyingIteration) + vals.divi(12); + + + if (trainingListener != null) { + trainingListener.iterationDone(this, i, 0); + } + } + } + } + } + + @Override + public void update(Gradient gradient) { + } + + /** + * An individual iteration + * @param p the probabilities that certain points + * are near each other + * @param i the iteration (primarily for debugging purposes) + */ + public void step(INDArray p, int i) { + update(gradient().getGradientFor(Y_GRAD), Y_GRAD); + } + + static double sign_tsne(double x) { return (x == .0 ? .0 : (x < .0 ? -1.0 : 1.0)); } + + + @Override + public void update(INDArray gradient, String paramType) { + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + INDArray yGrads = gradient; +; if (gains == null) + gains = Y.ulike().assign(1.0); + + //Nd4j.getExecutioner().exec(new BarnesHutGains(gains, gains, yGrads, yIncs)); + // Copied from Reference + for (int i = 0; i < yGrads.rows(); ++i) { + for (int j = 0; j < yGrads.columns(); ++j) { + if (sign_tsne(yGrads.getDouble(i,j)) == sign_tsne(yIncs.getDouble(i,j))) { + gains.putScalar(new int[]{i,j}, gains.getDouble(i,j)*0.8); + } + else { + gains.putScalar(new int[]{i,j}, gains.getDouble(i,j)+0.2); + } + } + } + BooleanIndexing.replaceWhere(gains, minGain, Conditions.lessThan(minGain)); + + Y.addi(yIncs); + INDArray gradChange = gains.mul(yGrads); + + if (useAdaGrad) { + if (adaGrad == null) { + adaGrad = new AdaGrad(gradient.shape(), learningRate); + adaGrad.setStateViewArray(Nd4j.zeros(gradient.shape()).reshape(1, gradChange.length()), + gradChange.shape(), gradient.ordering(), true); + } + + gradChange = adaGrad.getGradient(gradChange, 0); + + } else { + gradChange.muli(learningRate); + } + yIncs.muli(momentum).subi(gradChange); + } + } + + + /** + * Save the model as a file with a csv format, adding the label as the last column. + * @param labels + * @param path the path to write + * @throws IOException + */ + public void saveAsFile(List labels, String path) throws IOException { + try (BufferedWriter write = new BufferedWriter(new FileWriter(new File(path)))) { + for (int i = 0; i < Y.rows(); i++) { + if (i >= labels.size()) + break; + String word = labels.get(i); + if (word == null) + continue; + StringBuilder sb = new StringBuilder(); + INDArray wordVector = Y.getRow(i); + for (int j = 0; j < wordVector.length(); j++) { + sb.append(wordVector.getDouble(j)); + if (j < wordVector.length() - 1) + sb.append(","); + } + + sb.append(","); + sb.append(word); + sb.append("\n"); + write.write(sb.toString()); + + } + write.flush(); + } + } + + public void saveAsFile(String path) throws IOException { + try (BufferedWriter write = new BufferedWriter(new FileWriter(new File(path)))) { + for (int i = 0; i < Y.rows(); i++) { + StringBuilder sb = new StringBuilder(); + INDArray wordVector = Y.getRow(i); + for (int j = 0; j < wordVector.length(); j++) { + sb.append(wordVector.getDouble(j)); + if (j < wordVector.length() - 1) + sb.append(","); + } + sb.append("\n"); + write.write(sb.toString()); + } + write.flush(); + } + } + /** + * Plot tsne + * + * @param matrix the matrix to plot + * @param nDims the number + * @param labels + * @param path the path to write + * @throws IOException + * @deprecated use {@link #fit(INDArray)} and {@link #saveAsFile(List, String)} instead. + */ + @Deprecated + public void plot(INDArray matrix, int nDims, List labels, String path) throws IOException { + fit(matrix, nDims); + saveAsFile(labels, path); + } + + + @Override + public double score() { + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + + // Get estimate of normalization term + INDArray buff = Nd4j.create(numDimensions); + AtomicDouble sum_Q = new AtomicDouble(0.0); + for (int n = 0; n < N; n++) + tree.computeNonEdgeForces(n, theta, buff, sum_Q); + + // Loop over all edges to compute t-SNE error + double C = .0; + INDArray linear = Y; + for (int n = 0; n < N; n++) { + int begin = rows.getInt(n); + int end = rows.getInt(n + 1); + int ind1 = n; + for (int i = begin; i < end; i++) { + int ind2 = cols.getInt(i); + linear.slice(ind1).subi(linear.slice(ind2), buff); + + double Q = pow(buff, 2).sumNumber().doubleValue(); + Q = (1.0 / (1.0 + Q)) / sum_Q.doubleValue(); + C += vals.getDouble(i) * Math.log(vals.getDouble(i) + Nd4j.EPS_THRESHOLD) + / (Q + Nd4j.EPS_THRESHOLD); + } + } + + return C; + + } + + } + + @Override + public void computeGradientAndScore(LayerWorkspaceMgr workspaceMgr) { + + } + + @Override + public INDArray params() { + return null; + } + + @Override + public long numParams() { + return 0; + } + + @Override + public long numParams(boolean backwards) { + return 0; + } + + @Override + public void setParams(INDArray params) { + + } + + @Override + public void setParamsViewArray(INDArray params) { + throw new UnsupportedOperationException(); + } + + @Override + public INDArray getGradientsViewArray() { + throw new UnsupportedOperationException(); + } + + @Override + public void setBackpropGradientsViewArray(INDArray gradients) { + throw new UnsupportedOperationException(); + } + + + public void fit(INDArray data) { + this.x = data; + fit(); + } + + @Override + public void fit(INDArray data, LayerWorkspaceMgr workspaceMgr){ + fit(data); + } + + /** + * Change the dimensions with + * + * @deprecated Use {@link #fit(INDArray)} + */ + @Deprecated + public void fit(INDArray data, int nDims) { + this.x = data; + this.numDimensions = nDims; + fit(); + } + + @Override + public Gradient gradient() { + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + + if (yIncs == null) + yIncs = Y.like(); + if (gains == null) + gains = Y.ulike().assign(1.0D); + + AtomicDouble sumQ = new AtomicDouble(0); + /* Calculate gradient based on barnes hut approximation with positive and negative forces */ + INDArray posF = Y.like(); + INDArray negF = Y.like(); + + tree = new SpTree(Y); + + tree.computeEdgeForces(rows, cols, vals, N, posF); + for (int n = 0; n < N; n++) { + INDArray temp = negF.slice(n); + tree.computeNonEdgeForces(n, theta, temp, sumQ); + } + INDArray dC = posF.subi(negF.divi(sumQ)); + + Gradient ret = new DefaultGradient(); + ret.gradientForVariable().put(Y_GRAD, dC); + return ret; + } + } + + @Override + public Pair gradientAndScore() { + return new Pair<>(gradient(), score()); + } + + @Override + public int batchSize() { + return 0; + } + + @Override + public NeuralNetConfiguration conf() { + return null; + } + + @Override + public void setConf(NeuralNetConfiguration conf) { + + } + + /** + * Return the matrix reduce to the NDim. + */ + public INDArray getData() { + return Y; + } + + public void setData(INDArray data) { + this.Y = data; + } + + // TODO: find better solution for test + public void setN(int N) { + this.N = N; + } + + public static class Builder { + private int maxIter = 1000; + private double realMin = 1e-12f; + private double initialMomentum = 5e-1f; + private double finalMomentum = 8e-1f; + private double momentum = 5e-1f; + private int switchMomentumIteration = 100; + private boolean normalize = true; + private int stopLyingIteration = 100; + private double tolerance = 1e-5f; + private double learningRate = 1e-1f; + private boolean useAdaGrad = false; + private double perplexity = 30; + private double minGain = 1e-2f; + private double theta = 0.5; + private boolean invert = true; + private int numDim = 2; + private String similarityFunction = Distance.EUCLIDEAN.toString(); + private int vpTreeWorkers = 1; + protected WorkspaceMode workspaceMode = WorkspaceMode.NONE; + + private INDArray staticInput; + + public Builder vpTreeWorkers(int vpTreeWorkers) { + this.vpTreeWorkers = vpTreeWorkers; + return this; + } + + public Builder staticInit(INDArray staticInput) { + this.staticInput = staticInput; + return this; + } + + public Builder minGain(double minGain) { + this.minGain = minGain; + return this; + } + + public Builder perplexity(double perplexity) { + this.perplexity = perplexity; + return this; + } + + public Builder useAdaGrad(boolean useAdaGrad) { + this.useAdaGrad = useAdaGrad; + return this; + } + + public Builder learningRate(double learningRate) { + this.learningRate = learningRate; + return this; + } + + + public Builder tolerance(double tolerance) { + this.tolerance = tolerance; + return this; + } + + public Builder stopLyingIteration(int stopLyingIteration) { + this.stopLyingIteration = stopLyingIteration; + return this; + } + + public Builder normalize(boolean normalize) { + this.normalize = normalize; + return this; + } + + public Builder setMaxIter(int maxIter) { + this.maxIter = maxIter; + return this; + } + + public Builder setRealMin(double realMin) { + this.realMin = realMin; + return this; + } + + public Builder setInitialMomentum(double initialMomentum) { + this.initialMomentum = initialMomentum; + return this; + } + + public Builder setFinalMomentum(double finalMomentum) { + this.finalMomentum = finalMomentum; + return this; + } + + public Builder setMomentum(double momentum) { + this.momentum = momentum; + return this; + } + + public Builder setSwitchMomentumIteration(int switchMomentumIteration) { + this.switchMomentumIteration = switchMomentumIteration; + return this; + } + + + public Builder similarityFunction(String similarityFunction) { + this.similarityFunction = similarityFunction; + return this; + } + + public Builder invertDistanceMetric(boolean invert) { + this.invert = invert; + return this; + } + + public Builder theta(double theta) { + this.theta = theta; + return this; + } + + public Builder numDimension(int numDim) { + this.numDim = numDim; + return this; + } + + public Builder workspaceMode(WorkspaceMode workspaceMode){ + this.workspaceMode = workspaceMode; + return this; + } + + public BarnesHutTsne build() { + return new BarnesHutTsne(numDim, similarityFunction, theta, invert, maxIter, realMin, initialMomentum, + finalMomentum, momentum, switchMomentumIteration, normalize, stopLyingIteration, tolerance, + learningRate, useAdaGrad, perplexity, null, minGain, vpTreeWorkers, workspaceMode, staticInput); + } + + } + + + @Override + public void close(){ + //No-op + } +} diff --git a/cavis-dnn/cavis-dnn-tsne/src/main/java/org/deeplearning4j/plot/Tsne.java b/cavis-dnn/cavis-dnn-tsne/src/main/java/org/deeplearning4j/plot/Tsne.java new file mode 100644 index 000000000..ce092eba9 --- /dev/null +++ b/cavis-dnn/cavis-dnn-tsne/src/main/java/org/deeplearning4j/plot/Tsne.java @@ -0,0 +1,436 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.plot; + +import com.google.common.primitives.Ints; +import org.apache.commons.math3.util.FastMath; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dimensionalityreduction.PCA; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.BooleanIndexing; +import org.nd4j.linalg.indexing.INDArrayIndex; +import org.nd4j.linalg.indexing.SpecifiedIndex; +import org.nd4j.linalg.indexing.conditions.Conditions; +import org.nd4j.linalg.learning.legacy.AdaGrad; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.util.ArrayUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.nd4j.linalg.factory.Nd4j.*; +import static org.nd4j.linalg.ops.transforms.Transforms.*; + +/** + * dl4j port of original t-sne algorithm described/implemented by van der Maaten and Hinton + * + * + * @author raver119@gmail.com + * @author Adam Gibson + */ +public class Tsne { + protected int maxIter = 1000; + protected double realMin = Nd4j.EPS_THRESHOLD; + protected double initialMomentum = 0.5; + protected double finalMomentum = 0.8; + protected double minGain = 1e-2; + protected double momentum = initialMomentum; + protected int switchMomentumIteration = 100; + protected boolean normalize = true; + protected boolean usePca = false; + protected int stopLyingIteration = 250; + protected double tolerance = 1e-5; + protected double learningRate = 500; + protected AdaGrad adaGrad; + protected boolean useAdaGrad = true; + protected double perplexity = 30; + //protected INDArray gains,yIncs; + protected INDArray Y; + + protected static final Logger logger = LoggerFactory.getLogger(Tsne.class); + + + public Tsne(final int maxIter, final double realMin, final double initialMomentum, final double finalMomentum, + final double minGain, final double momentum, final int switchMomentumIteration, + final boolean normalize, final boolean usePca, final int stopLyingIteration, final double tolerance, + final double learningRate, final boolean useAdaGrad, final double perplexity) { + this.maxIter = maxIter; + this.realMin = realMin; + this.initialMomentum = initialMomentum; + this.finalMomentum = finalMomentum; + this.minGain = minGain; + this.momentum = momentum; + this.switchMomentumIteration = switchMomentumIteration; + this.normalize = normalize; + this.usePca = usePca; + this.stopLyingIteration = stopLyingIteration; + this.tolerance = tolerance; + this.learningRate = learningRate; + this.useAdaGrad = useAdaGrad; + this.perplexity = perplexity; + this.init(); + } + + protected void init() { + + } + + public INDArray calculate(INDArray X, int targetDimensions, double perplexity) { + // pca hook + if (usePca) { + X = PCA.pca(X, Math.min(50, X.columns()), normalize); + } else if (normalize) { + X.subi(X.min(Integer.MAX_VALUE)); + X = X.divi(X.max(Integer.MAX_VALUE)); + X = X.subiRowVector(X.mean(0)); + } + + + int n = X.rows(); + // FIXME: this is wrong, another distribution required here + Y = Nd4j.randn(X.dataType(), X.rows(), targetDimensions); + INDArray dY = Nd4j.zeros(n, targetDimensions); + INDArray iY = Nd4j.zeros(n, targetDimensions); + INDArray gains = Nd4j.ones(n, targetDimensions); + + boolean stopLying = false; + logger.debug("Y:Shape is = " + Arrays.toString(Y.shape())); + + // compute P-values + INDArray P = x2p(X, tolerance, perplexity); + + // do training + for (int i = 0; i < maxIter; i++) { + INDArray sumY = pow(Y, 2).sum(1).transpose(); + + //Student-t distribution + //also un normalized q + // also known as num in original implementation + INDArray qu = Y.mmul(Y.transpose()).muli(-2).addiRowVector(sumY).transpose().addiRowVector(sumY).addi(1) + .rdivi(1); + + // doAlongDiagonal(qu,new Zero()); + + INDArray Q = qu.div(qu.sumNumber().doubleValue()); + BooleanIndexing.replaceWhere(Q, 1e-12, Conditions.lessThan(1e-12)); + + INDArray PQ = P.sub(Q).muli(qu); + + logger.debug("PQ shape is: " + Arrays.toString(PQ.shape())); + logger.debug("PQ.sum(1) shape is: " + Arrays.toString(PQ.sum(1).shape())); + + dY = diag(PQ.sum(1)).subi(PQ).mmul(Y).muli(4); + + + if (i < switchMomentumIteration) { + momentum = initialMomentum; + } else { + momentum = finalMomentum; + } + + gains = gains.add(.2).muli(dY.cond(Conditions.greaterThan(0)).neq(iY.cond(Conditions.greaterThan(0)))) + .addi(gains.mul(0.8).muli(dY.cond(Conditions.greaterThan(0)) + .eq(iY.cond(Conditions.greaterThan(0))))); + + BooleanIndexing.replaceWhere(gains, minGain, Conditions.lessThan(minGain)); + + INDArray gradChange = gains.mul(dY); + + gradChange.muli(learningRate); + + iY.muli(momentum).subi(gradChange); + + double cost = P.mul(log(P.div(Q), false)).sumNumber().doubleValue(); + logger.info("Iteration [" + i + "] error is: [" + cost + "]"); + + Y.addi(iY); + // Y.addi(iY).subiRowVector(Y.mean(0)); + INDArray tiled = Nd4j.tile(Y.mean(0), new int[] {Y.rows(), 1}); + Y.subi(tiled); + + if (!stopLying && (i > maxIter / 2 || i >= stopLyingIteration)) { + P.divi(4); + stopLying = true; + } + } + return Y; + } + + public INDArray diag(INDArray ds) { + boolean isLong = ds.rows() > ds.columns(); + INDArray sliceZero = ds.slice(0); + int dim = Math.max(ds.columns(), ds.rows()); + INDArray result = Nd4j.create(dim, dim); + for (int i = 0; i < dim; i++) { + INDArray sliceSrc = ds.slice(i); + INDArray sliceDst = result.slice(i); + for (int j = 0; j < dim; j++) { + if (i == j) { + if (isLong) + sliceDst.putScalar(j, sliceSrc.getDouble(0)); + else + sliceDst.putScalar(j, sliceZero.getDouble(i)); + } + } + } + + return result; + } + + public void plot(INDArray matrix, int nDims, List labels, String path) throws IOException { + + calculate(matrix, nDims, perplexity); + + BufferedWriter write = new BufferedWriter(new FileWriter(new File(path), true)); + + for (int i = 0; i < Y.rows(); i++) { + if (i >= labels.size()) + break; + String word = labels.get(i); + if (word == null) + continue; + StringBuilder sb = new StringBuilder(); + INDArray wordVector = Y.getRow(i); + for (int j = 0; j < wordVector.length(); j++) { + sb.append(wordVector.getDouble(j)); + if (j < wordVector.length() - 1) + sb.append(","); + } + + sb.append(","); + sb.append(word); + sb.append(" "); + + sb.append("\n"); + write.write(sb.toString()); + + } + + write.flush(); + write.close(); + } + + /** + * Computes a gaussian kernel + * given a vector of squared distance distances + * + * @param d the data + * @param beta + * @return + */ + public Pair hBeta(INDArray d, double beta) { + INDArray P = exp(d.neg().muli(beta)); + double sumP = P.sumNumber().doubleValue(); + double logSumP = FastMath.log(sumP); + Double H = logSumP + ((beta * (d.mul(P).sumNumber().doubleValue())) / sumP); + P.divi(sumP); + return new Pair<>(H, P); + } + + /** + * This method build probabilities for given source data + * + * @param X + * @param tolerance + * @param perplexity + * @return + */ + private INDArray x2p(final INDArray X, double tolerance, double perplexity) { + int n = X.rows(); + final INDArray p = zeros(n, n); + final INDArray beta = ones(n, 1); + final double logU = Math.log(perplexity); + + INDArray sumX = pow(X, 2).sum(1); + + logger.debug("sumX shape: " + Arrays.toString(sumX.shape())); + + INDArray times = X.mmul(X.transpose()).muli(-2); + + logger.debug("times shape: " + Arrays.toString(times.shape())); + + INDArray prodSum = times.transpose().addiColumnVector(sumX); + + logger.debug("prodSum shape: " + Arrays.toString(prodSum.shape())); + + INDArray D = X.mmul(X.transpose()).mul(-2) // thats times + .transpose().addColumnVector(sumX) // thats prodSum + .addRowVector(sumX.transpose()); // thats D + + logger.info("Calculating probabilities of data similarities..."); + logger.debug("Tolerance: " + tolerance); + for (int i = 0; i < n; i++) { + if (i % 500 == 0 && i > 0) + logger.info("Handled [" + i + "] records out of [" + n + "]"); + + double betaMin = Double.NEGATIVE_INFINITY; + double betaMax = Double.POSITIVE_INFINITY; + int[] vals = Ints.concat(ArrayUtil.range(0, i), ArrayUtil.range(i + 1, n)); + INDArrayIndex[] range = new INDArrayIndex[] {new SpecifiedIndex(vals)}; + + INDArray row = D.slice(i).get(range); + Pair pair = hBeta(row, beta.getDouble(i)); + //INDArray hDiff = pair.getFirst().sub(logU); + double hDiff = pair.getFirst() - logU; + int tries = 0; + + //while hdiff > tolerance + while (Math.abs(hDiff) > tolerance && tries < 50) { + //if hdiff > 0 + if (hDiff > 0) { + betaMin = beta.getDouble(i); + if (Double.isInfinite(betaMax)) + beta.putScalar(i, beta.getDouble(i) * 2.0); + else + beta.putScalar(i, (beta.getDouble(i) + betaMax) / 2.0); + } else { + betaMax = beta.getDouble(i); + if (Double.isInfinite(betaMin)) + beta.putScalar(i, beta.getDouble(i) / 2.0); + else + beta.putScalar(i, (beta.getDouble(i) + betaMin) / 2.0); + } + + pair = hBeta(row, beta.getDouble(i)); + hDiff = pair.getFirst() - logU; + tries++; + } + p.slice(i).put(range, pair.getSecond()); + } + + + //dont need data in memory after + logger.info("Mean value of sigma " + sqrt(beta.rdiv(1)).mean(Integer.MAX_VALUE)); + BooleanIndexing.replaceWhere(p, 1e-12, Conditions.isNan()); + + //set 0 along the diagonal + INDArray permute = p.transpose(); + + INDArray pOut = p.add(permute); + + pOut.divi(pOut.sumNumber().doubleValue() + 1e-6); + + pOut.muli(4); + + BooleanIndexing.replaceWhere(pOut, 1e-12, Conditions.lessThan(1e-12)); + //ensure no nans + + return pOut; + } + + + public static class Builder { + protected int maxIter = 1000; + protected double realMin = 1e-12f; + protected double initialMomentum = 5e-1f; + protected double finalMomentum = 8e-1f; + protected double momentum = 5e-1f; + protected int switchMomentumIteration = 100; + protected boolean normalize = true; + protected boolean usePca = false; + protected int stopLyingIteration = 100; + protected double tolerance = 1e-5f; + protected double learningRate = 1e-1f; + protected boolean useAdaGrad = false; + protected double perplexity = 30; + protected double minGain = 1e-1f; + + + public Builder minGain(double minGain) { + this.minGain = minGain; + return this; + } + + public Builder perplexity(double perplexity) { + this.perplexity = perplexity; + return this; + } + + public Builder useAdaGrad(boolean useAdaGrad) { + this.useAdaGrad = useAdaGrad; + return this; + } + + public Builder learningRate(double learningRate) { + this.learningRate = learningRate; + return this; + } + + + public Builder tolerance(double tolerance) { + this.tolerance = tolerance; + return this; + } + + public Builder stopLyingIteration(int stopLyingIteration) { + this.stopLyingIteration = stopLyingIteration; + return this; + } + + public Builder usePca(boolean usePca) { + this.usePca = usePca; + return this; + } + + public Builder normalize(boolean normalize) { + this.normalize = normalize; + return this; + } + + public Builder setMaxIter(int maxIter) { + this.maxIter = maxIter; + return this; + } + + public Builder setRealMin(double realMin) { + this.realMin = realMin; + return this; + } + + public Builder setInitialMomentum(double initialMomentum) { + this.initialMomentum = initialMomentum; + return this; + } + + public Builder setFinalMomentum(double finalMomentum) { + this.finalMomentum = finalMomentum; + return this; + } + + public Builder setMomentum(double momentum) { + this.momentum = momentum; + return this; + } + + public Builder setSwitchMomentumIteration(int switchMomentumIteration) { + this.switchMomentumIteration = switchMomentumIteration; + return this; + } + + public Tsne build() { + return new Tsne(maxIter, realMin, initialMomentum, finalMomentum, minGain, momentum, + switchMomentumIteration, normalize, usePca, stopLyingIteration, tolerance, learningRate, + useAdaGrad, perplexity); + } + } +} diff --git a/cavis-dnn/cavis-dnn-tsne/src/test/java/org/deeplearning4j/plot/Test6058.java b/cavis-dnn/cavis-dnn-tsne/src/test/java/org/deeplearning4j/plot/Test6058.java new file mode 100644 index 000000000..de88c6851 --- /dev/null +++ b/cavis-dnn/cavis-dnn-tsne/src/test/java/org/deeplearning4j/plot/Test6058.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.plot; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class Test6058 extends BaseDL4JTest { + + @Test + public void test() throws Exception { + //All zero input -> cosine similarity isn't defined + //https://github.com/deeplearning4j/deeplearning4j/issues/6058 + val iterations = 10; + val cacheList = new ArrayList(); + + int nWords = 100; + for(int i=0; i cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words +// +// //STEP 2: Turn text input into a list of words +// log.info("Load & Vectorize data...."); +// File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile(); //Open the file +// //Get the data of all unique word vectors +// Pair vectors = WordVectorSerializer.loadTxt(wordFile); +// VocabCache cache = vectors.getSecond(); +// INDArray weights = vectors.getFirst().getSyn0(); //seperate weights of unique words into their own list +// +// for(int i = 0; i < cache.numWords(); i++) //seperate strings of words into their own list +// cacheList.add(cache.wordAtIndex(i)); +// +// //STEP 3: build a dual-tree tsne to use later +// log.info("Build model...."); +// BarnesHutTsne tsne = new BarnesHutTsne.Builder() +// .setMaxIter(iterations).theta(0.5) +// .normalize(false) +// .learningRate(500) +// .useAdaGrad(false) +// .workspaceMode(wsm) +// .build(); +// +// //STEP 4: establish the tsne values and save them to a file +// log.info("Store TSNE Coordinates for Plotting...."); +// String outputFile = "target/archive-tmp/tsne-standard-coords.csv"; +// (new File(outputFile)).getParentFile().mkdirs(); +// +// tsne.fit(weights); +// tsne.saveAsFile(cacheList, outputFile); +// +// +// } +// } +// +//} diff --git a/cavis-full/build.gradle b/cavis-full/build.gradle new file mode 100644 index 000000000..a986d6671 --- /dev/null +++ b/cavis-full/build.gradle @@ -0,0 +1,87 @@ +plugins { + id 'java-library' + id 'maven-publish' +} + +configurations.archives.artifacts.with { archives -> + archives.each { + println(it.name) + } +} + +dependencies { + //Todo clean this + api platform(project(":cavis-common-platform")) + api "org.bytedeco:javacpp" + api "com.fasterxml.jackson.datatype:jackson-datatype-joda:2.10.5" + //api group: "org.bytedeco", name: "javacpp", classifier: "linux-x64_86" + + rootProject.getAllprojects().each { Project sproj -> + if(!sproj.name.equals(name) && !sproj.name.equals("cavis-common-platform") + && !sproj.name.equals("Cavis") + && !sproj.name.equals("cavis-datavec") + && !sproj.name.equals("cavis-dnn") + && !sproj.name.equals("cavis-native") + && !sproj.name.equals("cavis-nd4j") + && !sproj.name.equals("cavis-ui") + && !sproj.name.equals("cavis-zoo")) { + //compileOnly project(""+sproj.path) + api sproj + if(! sproj.configurations.empty) { + //compileOnly project(sproj.getPath()) + + /* + sproj.configurations.each {Configuration conf -> + conf.dependencies.each {Dependency dep -> + compileOnly dep + } + } + + */ + } + } + } + + +} + +/* +tasks.getByName("jar") { + + manifest { + attributes 'Main-Class': 'net.brutex.ai.Dummy' + } + zip64=true + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + from { + configurations.compileClasspath.collect { File f -> + if (f.exists()) { + f.isDirectory() ? f : zipTree(f) + } + } + + configurations.runtimeClasspath.collect { File f -> + if (f.exists()) { + f.isDirectory() ? f : zipTree(f) + } + } + + + } +} +/* + +/* +artifacts { + archives customFatJar +} +*/ + +publishing { + publications { + mavenJava(MavenPublication) { + // artifact customFatJar +// from components.java + } + } +} diff --git a/cavis-native/build.gradle b/cavis-native/build.gradle new file mode 100644 index 000000000..1519fe9d4 --- /dev/null +++ b/cavis-native/build.gradle @@ -0,0 +1,27 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +subprojects { + group = "net.brutex.cavis-native" + apply plugin: "java-library" + apply plugin: "maven-publish" + apply plugin: "signing" +} \ No newline at end of file diff --git a/cavis-native/cavis-native-blas/build.gradle b/cavis-native/cavis-native-blas/build.gradle new file mode 100644 index 000000000..49feeb8a2 --- /dev/null +++ b/cavis-native/cavis-native-blas/build.gradle @@ -0,0 +1,33 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +plugins { + id 'java-library' + id 'maven-publish' + id 'signing' +} + +dependencies { + implementation 'org.bytedeco:javacpp' + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCommon + implementation "org.slf4j:slf4j-api" +} \ No newline at end of file diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/BaseNativeNDArrayFactory.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/BaseNativeNDArrayFactory.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/BaseNativeNDArrayFactory.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/BaseNativeNDArrayFactory.java index 007440720..2a544489a 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/BaseNativeNDArrayFactory.java +++ b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/BaseNativeNDArrayFactory.java @@ -1,21 +1,22 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.nativeblas; @@ -27,13 +28,13 @@ import org.bytedeco.javacpp.indexer.*; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.concurrency.AffinityManager; +import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.performance.PerformanceTracker; import org.nd4j.linalg.api.shape.Shape; import org.nd4j.linalg.api.shape.options.ArrayOptionsHelper; import org.nd4j.linalg.factory.BaseNDArrayFactory; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.api.memory.MemcpyDirection; import java.io.File; import java.io.FileInputStream; @@ -64,7 +65,7 @@ public abstract class BaseNativeNDArrayFactory extends BaseNDArrayFactory { @Override public Pointer convertToNumpy(INDArray array) { - val size = new LongPointer(1); + LongPointer size = new LongPointer(1); Pointer header = NativeOpsHolder .getInstance().getDeviceNativeOps() .numpyHeaderForNd4j( @@ -138,7 +139,7 @@ public abstract class BaseNativeNDArrayFactory extends BaseNDArrayFactory { dataPointer.capacity(dataBufferElementSize * Shape.length(shapeBuffer)); val jvmShapeInfo = shapeBuffer.asLong(); - val dtype = ArrayOptionsHelper.dataType(jvmShapeInfo); + DataType dtype = ArrayOptionsHelper.dataType(jvmShapeInfo); switch (dtype) { case BOOL: { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/LongPointerWrapper.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/LongPointerWrapper.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/LongPointerWrapper.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/LongPointerWrapper.java index 62f0b0667..55bda7a6f 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/LongPointerWrapper.java +++ b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/LongPointerWrapper.java @@ -24,7 +24,7 @@ import org.bytedeco.javacpp.LongPointer; import org.bytedeco.javacpp.Pointer; /** - * Wrapper for DoublePointer -> LongPointer + * Wrapper for DoublePointer -> LongPointer */ public class LongPointerWrapper extends LongPointer { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeLapack.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeLapack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeLapack.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeLapack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOps.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOps.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOps.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOps.java index 426b31407..33a0bce73 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOps.java +++ b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOps.java @@ -64,8 +64,6 @@ public interface NativeOps { * @param extraParams * @param result * @param resultShapeInfoBuffer - * @param dimension - * @param dimensionLength */ void execIndexReduce(PointerPointer extraPointers, int opNum, @@ -88,8 +86,10 @@ public interface NativeOps { * @param yShapeInfo * @param result * @param resultShapeInfo - * @param dimension - * @param dimensionLength + * + * + * + */ void execBroadcast(PointerPointer extraPointers, int opNum, @@ -125,7 +125,7 @@ public interface NativeOps { /** * @param opNum - * @param dx + * @param x * @param xShapeInfo * @param y * @param yShapeInfo @@ -323,8 +323,7 @@ public interface NativeOps { * @param yShapeInfo * @param result * @param resultShapeInfoBuffer - * @param dimension - * @param dimensionLength + * */ void execReduce3Tad(PointerPointer extraPointers, int opNum, @@ -445,8 +444,7 @@ public interface NativeOps { * @param extraParams * @param result * @param resultShapeInfoBuffer - * @param dimension - * @param dimensionLength + */ void execSummaryStatsTad(PointerPointer extraPointers, int opNum, @@ -468,7 +466,7 @@ public interface NativeOps { /** * @param extraPointers * @param opNum - * @param dx + * @param x * @param xShapeInfo * @param result * @param resultShapeInfo @@ -535,8 +533,7 @@ public interface NativeOps { * @param zShapeInfo * @param scalars * @param extraParams - * @param dimension - * @param dimensionLength + */ void execScalarTad(PointerPointer extraPointers, int opNum, diff --git a/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOpsGPUInfoProvider.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOpsGPUInfoProvider.java new file mode 100644 index 000000000..1a8d3950b --- /dev/null +++ b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOpsGPUInfoProvider.java @@ -0,0 +1,61 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.nd4j.nativeblas; + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.systeminfo.GPUInfo; +import org.nd4j.systeminfo.GPUInfoProvider; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class NativeOpsGPUInfoProvider implements GPUInfoProvider { + + @Override + public List getGPUs() { + NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); + + List gpus = new ArrayList<>(); + + + int nDevices = nativeOps.getAvailableDevices(); + if (nDevices > 0) { + for (int i = 0; i < nDevices; i++) { + try { + String name = nativeOps.getDeviceName(i); + long total = nativeOps.getDeviceTotalMemory(i); + long free = nativeOps.getDeviceFreeMemory(i); + int major = nativeOps.getDeviceMajor(i); + int minor = nativeOps.getDeviceMinor(i); + + gpus.add(new GPUInfo(name, total, free, major, minor)); + } catch (Exception e) { + log.info("Can't add GPU", e); + } + } + } + + return gpus; + } + +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java index a1eda8e66..8907f0c92 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java +++ b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java @@ -20,8 +20,8 @@ package org.nd4j.nativeblas; -import java.util.Properties; -import lombok.Getter; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import org.bytedeco.javacpp.Loader; import org.nd4j.common.config.ND4JClassLoading; import org.nd4j.common.config.ND4JEnvironmentVars; @@ -29,14 +29,15 @@ import org.nd4j.common.config.ND4JSystemProperties; import org.nd4j.common.io.ReflectionUtils; import org.nd4j.context.Nd4jContext; import org.nd4j.linalg.factory.Nd4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import java.util.Properties; + +@Slf4j public class NativeOpsHolder { - private static Logger log = LoggerFactory.getLogger(NativeOpsHolder.class); private static final NativeOpsHolder INSTANCE = new NativeOpsHolder(); - @Getter + private final NativeOps deviceNativeOps; public static int getCores(int totals) { @@ -119,6 +120,10 @@ public class NativeOpsHolder { } } + public NativeOps getDeviceNativeOps() { + return deviceNativeOps; + } + public static NativeOpsHolder getInstance() { return INSTANCE; } diff --git a/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/Nd4jBlas.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/Nd4jBlas.java new file mode 100644 index 000000000..f1e2864bf --- /dev/null +++ b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/Nd4jBlas.java @@ -0,0 +1,80 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.nd4j.nativeblas; + + +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; +import org.bytedeco.javacpp.Loader; +import org.nd4j.common.config.ND4JEnvironmentVars; +import org.nd4j.common.config.ND4JSystemProperties; +import org.nd4j.linalg.api.blas.Blas; + + +@Slf4j +public abstract class Nd4jBlas implements Blas { + + + public Nd4jBlas() { + int numThreads; + String skipper = System.getenv(ND4JEnvironmentVars.ND4J_SKIP_BLAS_THREADS); + if (skipper == null || skipper.isEmpty()) { + String numThreadsString = System.getenv(ND4JEnvironmentVars.OMP_NUM_THREADS); + if (numThreadsString != null && !numThreadsString.isEmpty()) { + numThreads = Integer.parseInt(numThreadsString); + setMaxThreads(numThreads); + } else { + int cores = Loader.totalCores(); + int chips = Loader.totalChips(); + if (cores > 0 && chips > 0) + numThreads = Math.max(1, cores / chips); + else + numThreads = NativeOpsHolder.getCores(Runtime.getRuntime().availableProcessors()); + setMaxThreads(numThreads); + } + + String logInit = System.getProperty(ND4JSystemProperties.LOG_INITIALIZATION); + if(logOpenMPBlasThreads() && (logInit == null || logInit.isEmpty() || Boolean.parseBoolean(logInit))) { + log.info("Number of threads used for OpenMP BLAS: {}", getMaxThreads()); + } + } + } + + /** + * Returns the BLAS library vendor + * + * @return the BLAS library vendor + */ + @Override + public Vendor getBlasVendor() { + int vendor = getBlasVendorId(); + boolean isUnknowVendor = ((vendor > Vendor.values().length - 1) || (vendor <= 0)); + if (isUnknowVendor) { + return Vendor.UNKNOWN; + } + return Vendor.values()[vendor]; + } + + public boolean logOpenMPBlasThreads(){ + return true; + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueConstantDataBuffer.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueConstantDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueConstantDataBuffer.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueConstantDataBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueConstantShapeBuffer.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueConstantShapeBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueConstantShapeBuffer.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueConstantShapeBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueContext.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueContext.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueContext.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueContext.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueDataBuffer.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueDataBuffer.java similarity index 87% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueDataBuffer.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueDataBuffer.java index b4a20aae8..f8cff5803 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueDataBuffer.java +++ b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueDataBuffer.java @@ -1,26 +1,28 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.nativeblas; import lombok.NonNull; +import lombok.extern.log4j.Log4j2; import lombok.extern.slf4j.Slf4j; import org.bytedeco.javacpp.Pointer; import org.nd4j.linalg.api.buffer.DataType; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueLaunchContext.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueLaunchContext.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueLaunchContext.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueLaunchContext.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueRandomGenerator.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueRandomGenerator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueRandomGenerator.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueRandomGenerator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueResultWrapper.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueResultWrapper.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueResultWrapper.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueResultWrapper.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueShapeList.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueShapeList.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueShapeList.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueShapeList.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueTadPack.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueTadPack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueTadPack.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueTadPack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueVariable.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueVariable.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueVariable.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueVariable.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueVariablesSet.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueVariablesSet.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/OpaqueVariablesSet.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/OpaqueVariablesSet.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/PointerPointerWrapper.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/PointerPointerWrapper.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/PointerPointerWrapper.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/PointerPointerWrapper.java index 47775c399..580affc90 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/PointerPointerWrapper.java +++ b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/PointerPointerWrapper.java @@ -24,7 +24,7 @@ import org.bytedeco.javacpp.LongPointer; import org.bytedeco.javacpp.PointerPointer; /** - * Wrapper for DoublePointer -> LongPointer + * Wrapper for DoublePointer -> LongPointer */ public class PointerPointerWrapper extends PointerPointer { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/ResultWrapperAbstraction.java b/cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/ResultWrapperAbstraction.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/ResultWrapperAbstraction.java rename to cavis-native/cavis-native-blas/src/main/java/org/nd4j/nativeblas/ResultWrapperAbstraction.java diff --git a/cavis-native/cavis-native-common/build.gradle b/cavis-native/cavis-native-common/build.gradle new file mode 100644 index 000000000..11544c9f0 --- /dev/null +++ b/cavis-native/cavis-native-common/build.gradle @@ -0,0 +1,16 @@ +plugins { + id 'java-library' +} + +dependencies { + implementation "org.bytedeco:javacpp" + implementation "org.slf4j:slf4j-api" + implementation "commons-io:commons-io" + implementation "com.google.flatbuffers:flatbuffers-java" + + implementation project(":cavis-dnn:cavis-dnn-api") + implementation project(":cavis-dnn:cavis-dnn-common") + implementation project(":cavis-native:cavis-native-blas") + +} + diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/autodiff/execution/NativeGraphExecutioner.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/autodiff/execution/NativeGraphExecutioner.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/autodiff/execution/NativeGraphExecutioner.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/autodiff/execution/NativeGraphExecutioner.java index fc8b2022b..ebd17f8cb 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/autodiff/execution/NativeGraphExecutioner.java +++ b/cavis-native/cavis-native-common/src/main/java/org/nd4j/autodiff/execution/NativeGraphExecutioner.java @@ -20,6 +20,7 @@ package org.nd4j.autodiff.execution; +import lombok.extern.log4j.Log4j2; import lombok.extern.slf4j.Slf4j; import org.bytedeco.javacpp.BytePointer; import org.nd4j.autodiff.execution.conf.ExecutionMode; diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/compression/impl/AbstractCompressor.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/compression/impl/AbstractCompressor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/compression/impl/AbstractCompressor.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/compression/impl/AbstractCompressor.java index 485d55ae3..0ada9ae43 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/compression/impl/AbstractCompressor.java +++ b/cavis-native/cavis-native-common/src/main/java/org/nd4j/compression/impl/AbstractCompressor.java @@ -25,6 +25,7 @@ import lombok.val; import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.FloatPointer; import org.bytedeco.javacpp.Pointer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.buffer.DataTypeEx; @@ -32,7 +33,6 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.compression.CompressedDataBuffer; import org.nd4j.linalg.compression.NDArrayCompressor; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.util.ArrayUtil; @Slf4j public abstract class AbstractCompressor implements NDArrayCompressor { diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/compression/impl/Gzip.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/compression/impl/Gzip.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/compression/impl/Gzip.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/compression/impl/Gzip.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/compression/impl/NoOp.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/compression/impl/NoOp.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/compression/impl/NoOp.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/compression/impl/NoOp.java index 925652133..bd21a9a5e 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/compression/impl/NoOp.java +++ b/cavis-native/cavis-native-common/src/main/java/org/nd4j/compression/impl/NoOp.java @@ -26,12 +26,12 @@ import org.bytedeco.javacpp.Pointer; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.buffer.DataTypeEx; +import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.api.ops.performance.PerformanceTracker; import org.nd4j.linalg.compression.CompressedDataBuffer; import org.nd4j.linalg.compression.CompressionDescriptor; import org.nd4j.linalg.compression.CompressionType; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.api.memory.MemcpyDirection; public class NoOp extends AbstractCompressor { /** diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/rng/NativeRandom.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/rng/NativeRandom.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/rng/NativeRandom.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/rng/NativeRandom.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/rng/deallocator/GarbageStateReference.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/rng/deallocator/GarbageStateReference.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/rng/deallocator/GarbageStateReference.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/rng/deallocator/GarbageStateReference.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/rng/deallocator/NativePack.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/rng/deallocator/NativePack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/rng/deallocator/NativePack.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/rng/deallocator/NativePack.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/rng/deallocator/NativeRandomDeallocator.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/rng/deallocator/NativeRandomDeallocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/rng/deallocator/NativeRandomDeallocator.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/rng/deallocator/NativeRandomDeallocator.java diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/storage/CompressedRamStorage.java b/cavis-native/cavis-native-common/src/main/java/org/nd4j/storage/CompressedRamStorage.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/storage/CompressedRamStorage.java rename to cavis-native/cavis-native-common/src/main/java/org/nd4j/storage/CompressedRamStorage.java diff --git a/cavis-native/cavis-native-common/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor b/cavis-native/cavis-native-common/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor new file mode 100644 index 000000000..52f95daca --- /dev/null +++ b/cavis-native/cavis-native-common/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor @@ -0,0 +1,28 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + +#org.nd4j.compression.impl.Float8 +#org.nd4j.compression.impl.Float16 +org.nd4j.compression.impl.Gzip +#org.nd4j.compression.impl.Int8 +#org.nd4j.compression.impl.Int16 +org.nd4j.compression.impl.NoOp +#org.nd4j.compression.impl.Uint8 \ No newline at end of file diff --git a/cavis-native/cavis-native-cpu/build.gradle b/cavis-native/cavis-native-cpu/build.gradle new file mode 100644 index 000000000..a435e16f8 --- /dev/null +++ b/cavis-native/cavis-native-cpu/build.gradle @@ -0,0 +1,32 @@ +plugins { + id 'java-library' + id 'maven-publish' + id 'signing' +} + +ext { + buildTarget = rootProject.ext.buildTarget +} + +dependencies { + implementation projects.cavisNative.cavisNativeBlas + implementation projects.cavisNative.cavisNativeCommon + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCommon + + implementation (projects.cavisNative.cavisNativeLib) { + capabilities { + it.requireCapability group: "net.brutex.cavis-native", name:"cavis-native-lib-cpu-support" + } + } + + implementation "org.bytedeco:javacpp" + implementation group:"org.bytedeco", name:"javacpp", classifier:"${buildTarget}" + implementation "org.bytedeco:openblas" + implementation group:"org.bytedeco", name:"openblas", classifier:"${buildTarget}" + + implementation "com.google.flatbuffers:flatbuffers-java" + implementation "org.slf4j:slf4j-api" + implementation "org.apache.commons:commons-math3" + +} diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/BlasWrapper.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/BlasWrapper.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/BlasWrapper.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/BlasWrapper.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuAffinityManager.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuAffinityManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuAffinityManager.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuAffinityManager.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuBackend.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuBackend.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuBackend.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuBackend.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuEnvironment.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuEnvironment.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuEnvironment.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuEnvironment.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuMemoryManager.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuMemoryManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuMemoryManager.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuMemoryManager.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java index 1f847c6a7..7de40dbdb 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java +++ b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuNDArrayFactory.java @@ -526,8 +526,7 @@ public class CpuNDArrayFactory extends BaseNativeNDArrayFactory { public INDArray toFlattened(char order, Collection matrices) { Preconditions.checkArgument(matrices.size() > 0, "toFlattened expects > 0 operands"); - return Nd4j.exec(new Flatten(order, matrices.toArray(new INDArray[matrices.size()])))[0] - .castTo(matrices.iterator().next().dataType()); + return Nd4j.exec(new Flatten(order, matrices.toArray(new INDArray[matrices.size()])))[0]; } @Override diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuTADManager.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuTADManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuTADManager.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/CpuTADManager.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/DirectShapeInfoProvider.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/DirectShapeInfoProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/DirectShapeInfoProvider.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/DirectShapeInfoProvider.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/NDArray.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/NDArray.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/NDArray.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/NDArray.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuBlas.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuBlas.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuBlas.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuBlas.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLapack.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLapack.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLapack.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLapack.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel1.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel1.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel1.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel1.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel2.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel2.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel2.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel2.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel3.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel3.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel3.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/blas/CpuLevel3.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BFloat16Buffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BFloat16Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BFloat16Buffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BFloat16Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BoolBuffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BoolBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BoolBuffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BoolBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/CpuDeallocator.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/CpuDeallocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/CpuDeallocator.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/CpuDeallocator.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DefaultDataBufferFactory.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DefaultDataBufferFactory.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DefaultDataBufferFactory.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DefaultDataBufferFactory.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DoubleBuffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DoubleBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DoubleBuffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DoubleBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/FloatBuffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/FloatBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/FloatBuffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/FloatBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/HalfBuffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/HalfBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/HalfBuffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/HalfBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Int16Buffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Int16Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Int16Buffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Int16Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Int8Buffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Int8Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Int8Buffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Int8Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/IntBuffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/IntBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/IntBuffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/IntBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/LongBuffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/LongBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/LongBuffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/LongBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt16Buffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt16Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt16Buffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt16Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt32Buffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt32Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt32Buffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt32Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt64Buffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt64Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt64Buffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt64Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt8Buffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt8Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt8Buffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/UInt8Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Utf8Buffer.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Utf8Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Utf8Buffer.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Utf8Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/cache/ConstantBuffersCache.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/cache/ConstantBuffersCache.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/cache/ConstantBuffersCache.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/cache/ConstantBuffersCache.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/compression/CpuFlexibleThreshold.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/compression/CpuFlexibleThreshold.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/compression/CpuFlexibleThreshold.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/compression/CpuFlexibleThreshold.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/compression/CpuThreshold.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/compression/CpuThreshold.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/compression/CpuThreshold.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/compression/CpuThreshold.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/CpuOpContext.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/CpuOpContext.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/CpuOpContext.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/CpuOpContext.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/CpuOpContextDeallocator.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/CpuOpContextDeallocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/CpuOpContextDeallocator.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/CpuOpContextDeallocator.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/NativeOpExecutioner.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/NativeOpExecutioner.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/NativeOpExecutioner.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/NativeOpExecutioner.java index b0bb2ae08..52ab50235 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/NativeOpExecutioner.java +++ b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/ops/NativeOpExecutioner.java @@ -75,9 +75,9 @@ import java.util.*; @Slf4j public class NativeOpExecutioner extends DefaultOpExecutioner { - private NativeOps loop = NativeOpsHolder.getInstance().getDeviceNativeOps(); + private final NativeOpsHolder holder = NativeOpsHolder.getInstance(); + private NativeOps loop = holder.getDeviceNativeOps(); private ConstantHandler constantHandler = Nd4j.getConstantHandler(); - @Getter private CpuTADManager tadManager = new CpuTADManager(); //thread locals for custom op inputs and outputs to prevent allocations @@ -1272,20 +1272,18 @@ public class NativeOpExecutioner extends DefaultOpExecutioner { val zb = z == null ? null : ((BaseCpuDataBuffer) z.data()).getOpaqueDataBuffer(); if (x != null && y != null && z != null) { - DataBuffer dataBuffer = op.extraArgsDataBuff(z.dataType()); // triple arg call loop.execRandom3(null, op.opNum(), rng.getStatePointer(), // rng state ptr xb, (LongPointer) x.shapeInfoDataBuffer().addressPointer(), null, yb, (LongPointer) y.shapeInfoDataBuffer().addressPointer(), null, zb, (LongPointer) z.shapeInfoDataBuffer().addressPointer(), null, - dataBuffer != null ? dataBuffer.addressPointer() : null); + op.extraArgsDataBuff(z.dataType()).addressPointer()); } else if (x != null && z != null) { - DataBuffer dataBuffer = op.extraArgsDataBuff(z.dataType()); //double arg call loop.execRandom2(null, op.opNum(), rng.getStatePointer(), // rng state ptr xb, (LongPointer) x.shapeInfoDataBuffer().addressPointer(), null, zb, (LongPointer) z.shapeInfoDataBuffer().addressPointer(), null, - dataBuffer != null ? dataBuffer.addressPointer() : null); + op.extraArgsDataBuff(z.dataType()).addressPointer()); } else { // single arg call loop.execRandom(null, op.opNum(), rng.getStatePointer(), // rng state ptr @@ -1301,11 +1299,6 @@ public class NativeOpExecutioner extends DefaultOpExecutioner { return z; } - @Override - public TADManager getTADManager() { - return tadManager; - } - /** * This class holds memory chunks required for single specific Aggregate op. * Can be used together with ThreadLocal variables @@ -2011,6 +2004,11 @@ public class NativeOpExecutioner extends DefaultOpExecutioner { .build(); } + @Override + public TADManager getTADManager() { + return this.tadManager; + } + @Override public DataBuffer createShapeInfo(long[] shape, long[] stride, long elementWiseStride, char order, DataType dtype, boolean empty) { val dbf = loop.shapeBuffer(shape.length, new LongPointer(shape), new LongPointer(stride), dtype.toInt(), order, elementWiseStride, empty); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/rng/CpuNativeRandom.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/rng/CpuNativeRandom.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/rng/CpuNativeRandom.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/rng/CpuNativeRandom.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspace.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspace.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspace.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspace.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspaceDeallocator.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspaceDeallocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspaceDeallocator.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspaceDeallocator.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspaceManager.java b/cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspaceManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspaceManager.java rename to cavis-native/cavis-native-cpu/src/main/java/org/nd4j/linalg/cpu/nativecpu/workspace/CpuWorkspaceManager.java diff --git a/cavis-native/cavis-native-cpu/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor b/cavis-native/cavis-native-cpu/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor new file mode 100644 index 000000000..cb9fb036e --- /dev/null +++ b/cavis-native/cavis-native-cpu/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor @@ -0,0 +1,23 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + + +org.nd4j.linalg.cpu.nativecpu.compression.CpuThreshold \ No newline at end of file diff --git a/cavis-native/cavis-native-cpu/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend b/cavis-native/cavis-native-cpu/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend new file mode 100644 index 000000000..6cbfb7bf0 --- /dev/null +++ b/cavis-native/cavis-native-cpu/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend @@ -0,0 +1,23 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + + +org.nd4j.linalg.cpu.nativecpu.CpuBackend \ No newline at end of file diff --git a/cavis-native/cavis-native-cpu/src/main/resources/nd4j-native.properties b/cavis-native/cavis-native-cpu/src/main/resources/nd4j-native.properties new file mode 100644 index 000000000..f2f5c69cc --- /dev/null +++ b/cavis-native/cavis-native-cpu/src/main/resources/nd4j-native.properties @@ -0,0 +1,37 @@ +# +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +real.class.double = org.nd4j.linalg.cpu.NDArray +shapeinfoprovider = org.nd4j.linalg.cpu.nativecpu.DirectShapeInfoProvider +constantsprovider = org.nd4j.linalg.cpu.nativecpu.cache.ConstantBuffersCache +affinitymanager = org.nd4j.linalg.cpu.nativecpu.CpuAffinityManager +memorymanager = org.nd4j.linalg.cpu.nativecpu.CpuMemoryManager +dtype = float +blas.ops = org.nd4j.linalg.cpu.nativecpu.BlasWrapper +opexec = org.nd4j.linalg.cpu.nativecpu.ops.NativeOpExecutioner +native.ops= org.nd4j.nativeblas.Nd4jCpu +ndarrayfactory.class = org.nd4j.linalg.cpu.nativecpu.CpuNDArrayFactory +ndarray.order = c +resourcemanager_state = false +databufferfactory = org.nd4j.linalg.cpu.nativecpu.buffer.DefaultDataBufferFactory +workspacemanager = org.nd4j.linalg.cpu.nativecpu.workspace.CpuWorkspaceManager +alloc = javacpp +opexec.mode = native +random=org.nd4j.linalg.cpu.nativecpu.rng.CpuNativeRandom diff --git a/cavis-native/cavis-native-jcublas/build.gradle b/cavis-native/cavis-native-jcublas/build.gradle new file mode 100644 index 000000000..b9b3c37e4 --- /dev/null +++ b/cavis-native/cavis-native-jcublas/build.gradle @@ -0,0 +1,37 @@ +plugins { + id 'java-library' + id 'maven-publish' + id 'signing' +} + +ext { + buildTarget = rootProject.ext.buildTarget +} + +dependencies { + implementation platform(projects.cavisCommonPlatform) + + implementation project(":cavis-native:cavis-native-blas") + + implementation group: "org.bytedeco", name: "cuda" + implementation group: "org.bytedeco", name: "cuda", classifier: buildTarget + implementation group: "org.bytedeco", name: "cuda", classifier: "${buildTarget}-redist" + + implementation group: "org.bytedeco", name: "javacpp" + implementation group: "org.bytedeco", name: "javacpp", classifier: buildTarget + + implementation(project(path: ":cavis-native:cavis-native-lib")) { + capabilities { + it.requireCapability("net.brutex.cavis-native:cavis-native-lib-cuda-support:1.0.0-SNAPSHOT") + } + } + implementation project(":cavis-native:cavis-native-common") + implementation project(":cavis-dnn:cavis-dnn-api") + implementation project(":cavis-dnn:cavis-dnn-common") + + implementation "com.google.guava:guava" + implementation "com.google.flatbuffers:flatbuffers-java" + implementation "org.slf4j:slf4j-api" + implementation "org.apache.commons:commons-lang3" +} + diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/Allocator.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/Allocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/Allocator.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/Allocator.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/AtomicState.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/AtomicState.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/AtomicState.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/AtomicState.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/DeviceAllocationsTracker.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/DeviceAllocationsTracker.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/DeviceAllocationsTracker.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/DeviceAllocationsTracker.java index ca2c54529..907fd8103 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/DeviceAllocationsTracker.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/DeviceAllocationsTracker.java @@ -21,10 +21,10 @@ package org.nd4j.jita.allocator.concurrency; import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import org.nd4j.jita.conf.Configuration; import org.nd4j.linalg.factory.Nd4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -36,6 +36,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; * * @author raver119@gmail.com */ +@Slf4j public class DeviceAllocationsTracker { private Configuration configuration; @@ -47,8 +48,6 @@ public class DeviceAllocationsTracker { private final Map reservedSpace = new ConcurrentHashMap<>(); - private static Logger log = LoggerFactory.getLogger(DeviceAllocationsTracker.class); - public DeviceAllocationsTracker(@NonNull Configuration configuration) { this.configuration = configuration; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/Lock.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/Lock.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/Lock.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/Lock.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/RRWLock.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/RRWLock.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/concurrency/RRWLock.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/concurrency/RRWLock.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/AccessState.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/AccessState.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/AccessState.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/AccessState.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/Aggressiveness.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/Aggressiveness.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/Aggressiveness.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/Aggressiveness.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/AllocationStatus.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/AllocationStatus.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/AllocationStatus.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/AllocationStatus.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/CudaConstants.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/CudaConstants.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/CudaConstants.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/CudaConstants.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/SyncState.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/SyncState.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/enums/SyncState.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/enums/SyncState.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/garbage/GarbageBufferReference.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/garbage/GarbageBufferReference.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/garbage/GarbageBufferReference.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/garbage/GarbageBufferReference.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/garbage/GarbageResourceReference.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/garbage/GarbageResourceReference.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/garbage/GarbageResourceReference.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/garbage/GarbageResourceReference.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AllocationPoint.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/AllocationPoint.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AllocationPoint.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/AllocationPoint.java index 1b702eff1..e07bbf544 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AllocationPoint.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/AllocationPoint.java @@ -23,6 +23,8 @@ package org.nd4j.jita.allocator.impl; import lombok.Getter; import lombok.NonNull; import lombok.Setter; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.bytedeco.javacpp.Pointer; import org.nd4j.jita.allocator.enums.AllocationStatus; @@ -37,8 +39,7 @@ import org.nd4j.linalg.jcublas.context.CudaContext; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; import org.nd4j.nativeblas.OpaqueDataBuffer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import java.util.concurrent.atomic.AtomicBoolean; @@ -49,8 +50,8 @@ import java.util.concurrent.atomic.AtomicBoolean; * @author raver119@gmail.com */ // DO NOT EVER MAKE THIS CLASS SERIALIZABLE. +@Slf4j public class AllocationPoint { - private static Logger log = LoggerFactory.getLogger(AllocationPoint.class); @Getter private OpaqueDataBuffer ptrDataBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AllocationShape.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/AllocationShape.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AllocationShape.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/AllocationShape.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java index 8b95febe7..21ba561f8 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java @@ -22,6 +22,8 @@ package org.nd4j.jita.allocator.impl; import lombok.Getter; import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.bytedeco.javacpp.Pointer; import org.nd4j.jita.allocator.Allocator; @@ -44,8 +46,7 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer; import org.nd4j.linalg.jcublas.context.CudaContext; import org.nd4j.nativeblas.NativeOpsHolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -80,6 +81,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; * * @author raver119@gmail.com */ +@Slf4j public class AtomicAllocator implements Allocator { private static final AtomicAllocator INSTANCE = new AtomicAllocator(); @@ -94,8 +96,6 @@ public class AtomicAllocator implements Allocator { // we have single tracking point for allocation points, since we're not going to cycle through it any time soon private Map allocationsMap = new ConcurrentHashMap<>(); - private static Logger log = LoggerFactory.getLogger(AtomicAllocator.class); - /* locks for internal resources */ diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/CudaDeallocator.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/CudaDeallocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/CudaDeallocator.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/CudaDeallocator.java index 12b2c7263..72848face 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/CudaDeallocator.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/CudaDeallocator.java @@ -20,8 +20,8 @@ package org.nd4j.jita.allocator.impl; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer; import org.nd4j.linalg.api.memory.Deallocator; +import org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer; import org.nd4j.nativeblas.NativeOpsHolder; import org.nd4j.nativeblas.OpaqueDataBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/MemoryTracker.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/MemoryTracker.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/MemoryTracker.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/MemoryTracker.java index e371643ed..4db458499 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/MemoryTracker.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/MemoryTracker.java @@ -18,15 +18,15 @@ package org.nd4j.jita.allocator.impl; -import java.util.*; -import java.util.concurrent.atomic.AtomicLong; - import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.nd4j.jita.allocator.pointers.CudaPointer; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.nativeblas.NativeOpsHolder; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + @Slf4j public class MemoryTracker { diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/NestedPoint.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/NestedPoint.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/NestedPoint.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/impl/NestedPoint.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/CudaPointer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/CudaPointer.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/CudaPointer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/CudaPointer.java index 73fca7aab..f52abee97 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/CudaPointer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/CudaPointer.java @@ -20,9 +20,8 @@ package org.nd4j.jita.allocator.pointers; +import lombok.extern.slf4j.Slf4j; import org.bytedeco.javacpp.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * This class is simple logic-less holder for pointers derived from CUDA. @@ -33,11 +32,9 @@ import org.slf4j.LoggerFactory; * * @author raver119@gmail.com */ +@Slf4j public class CudaPointer extends Pointer { - private static Logger logger = LoggerFactory.getLogger(CudaPointer.class); - - public CudaPointer(Pointer pointer) { this.address = pointer.address(); this.capacity = pointer.capacity(); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/PointersPair.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/PointersPair.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/PointersPair.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/PointersPair.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/CUcontext.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/CUcontext.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/CUcontext.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/CUcontext.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cublasHandle_t.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cublasHandle_t.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cublasHandle_t.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cublasHandle_t.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaEvent_t.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaEvent_t.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaEvent_t.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaEvent_t.java index 46a459704..ec1073761 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaEvent_t.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaEvent_t.java @@ -27,7 +27,6 @@ import org.bytedeco.javacpp.Pointer; import org.nd4j.jita.allocator.pointers.CudaPointer; import org.nd4j.linalg.exception.ND4JException; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; /** diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaStream_t.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaStream_t.java similarity index 92% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaStream_t.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaStream_t.java index 34216a7e1..5e67ca603 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaStream_t.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cudaStream_t.java @@ -24,8 +24,6 @@ import lombok.NonNull; import lombok.val; import org.bytedeco.javacpp.Pointer; import org.nd4j.jita.allocator.pointers.CudaPointer; -import org.nd4j.linalg.exception.ND4JException; -import org.nd4j.linalg.factory.Nd4j; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; @@ -42,7 +40,7 @@ public class cudaStream_t extends CudaPointer { NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); int res = nativeOps.streamSynchronize(this); - val ec = nativeOps.lastErrorCode(); + int ec = nativeOps.lastErrorCode(); if (ec != 0) throw new RuntimeException(nativeOps.lastErrorMessage() + "; Error code: " + ec); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cusolverDnHandle_t.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cusolverDnHandle_t.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cusolverDnHandle_t.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/pointers/cuda/cusolverDnHandle_t.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/tad/BasicTADManager.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/tad/BasicTADManager.java similarity index 94% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/tad/BasicTADManager.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/tad/BasicTADManager.java index bc6a9d7f9..16c75072b 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/tad/BasicTADManager.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/tad/BasicTADManager.java @@ -20,16 +20,16 @@ package org.nd4j.jita.allocator.tad; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Pair; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.cache.TADManager; +import org.nd4j.linalg.factory.Nd4j; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; @@ -37,9 +37,9 @@ import java.util.concurrent.atomic.AtomicLong; /** * @author raver119@gmail.com */ +@Slf4j public class BasicTADManager implements TADManager { protected NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); - private static Logger logger = LoggerFactory.getLogger(BasicTADManager.class); protected AtomicLong bytes = new AtomicLong(0); @Override diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/tad/DeviceTADManager.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/tad/DeviceTADManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/tad/DeviceTADManager.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/tad/DeviceTADManager.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/RateTimer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/RateTimer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/RateTimer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/RateTimer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/Ring.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/Ring.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/Ring.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/Ring.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/TimeProvider.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/TimeProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/TimeProvider.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/TimeProvider.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/impl/BinaryTimer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/impl/BinaryTimer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/impl/BinaryTimer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/impl/BinaryTimer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/impl/SimpleTimer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/impl/SimpleTimer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/impl/SimpleTimer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/impl/SimpleTimer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/providers/MillisecondsProvider.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/providers/MillisecondsProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/providers/MillisecondsProvider.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/providers/MillisecondsProvider.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/providers/NanosecondsProvider.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/providers/NanosecondsProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/providers/NanosecondsProvider.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/providers/NanosecondsProvider.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/providers/OperativeProvider.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/providers/OperativeProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/providers/OperativeProvider.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/providers/OperativeProvider.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/rings/LockedRing.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/rings/LockedRing.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/time/rings/LockedRing.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/time/rings/LockedRing.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/utils/AllocationUtils.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/utils/AllocationUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/utils/AllocationUtils.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/allocator/utils/AllocationUtils.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/balance/Balancer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/balance/Balancer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/balance/Balancer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/balance/Balancer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java index 15f2cdba6..5e90a12a6 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java @@ -21,6 +21,8 @@ package org.nd4j.jita.concurrency; import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.nd4j.jita.allocator.impl.AllocationPoint; import org.nd4j.jita.allocator.impl.AtomicAllocator; @@ -31,8 +33,7 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer; import org.nd4j.nativeblas.NativeOpsHolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -44,9 +45,8 @@ import java.util.concurrent.atomic.AtomicInteger; * * @author raver119@gmail.com */ +@Slf4j public class CudaAffinityManager extends BasicAffinityManager { - private static Logger logger = LoggerFactory.getLogger(CudaAffinityManager.class); - private Map affinityMap = new ConcurrentHashMap<>(); private AtomicInteger devPtr = new AtomicInteger(0); private ThreadLocal affiliated = new ThreadLocal<>(); @@ -111,11 +111,11 @@ public class CudaAffinityManager extends BasicAffinityManager { val t = Thread.currentThread(); val n = t.getId() == threadId ? t.getName() : "N/A"; - logger.debug("Mapping thread [{} - {}] to device [{}], out of [{}] devices...", threadId, n, device, CudaEnvironment.getInstance().getConfiguration().getAvailableDevices().size()); + log.debug("Mapping thread [{} - {}] to device [{}], out of [{}] devices...", threadId, n, device, CudaEnvironment.getInstance().getConfiguration().getAvailableDevices().size()); } } else { device = CudaEnvironment.getInstance().getConfiguration().getAvailableDevices().get(0); - logger.debug("Single device is forced, mapping to device [{}]", device); + log.debug("Single device is forced, mapping to device [{}]", device); } return device; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/EventsProvider.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/concurrency/EventsProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/EventsProvider.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/concurrency/EventsProvider.java diff --git a/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/conf/Configuration.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/conf/Configuration.java new file mode 100644 index 000000000..f25c90698 --- /dev/null +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/conf/Configuration.java @@ -0,0 +1,810 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.jita.conf; + +import lombok.Getter; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.config.ND4JEnvironmentVars; +import org.nd4j.jita.allocator.enums.Aggressiveness; +import org.nd4j.jita.allocator.enums.AllocationStatus; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.nativeblas.NativeOpsHolder; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author raver119@gmail.com + */ +@Slf4j +public class Configuration implements Serializable { + + public enum ExecutionModel { + SEQUENTIAL, ASYNCHRONOUS, OPTIMIZED, + } + + public enum AllocationModel { + DIRECT, CACHE_HOST, CACHE_ALL, + } + + public enum MemoryModel { + IMMEDIATE, DELAYED + } + + @Getter + @Deprecated //Only SEQUENTIAL is supported + private ExecutionModel executionModel = ExecutionModel.SEQUENTIAL; + + @Getter + private AllocationModel allocationModel = AllocationModel.CACHE_ALL; + + @Getter + private AllocationStatus firstMemory = AllocationStatus.DEVICE; + + @Getter + private MemoryModel memoryModel = MemoryModel.IMMEDIATE; + + @Getter + private boolean debug = false; + + @Getter + private boolean verbose = false; + + @Getter + private boolean fillDashboard = false; + + private boolean forceSingleGPU = false; + + @Getter + private long noGcWindowMs = 100; + + /** + * Keep this value between 0.01 and 0.95 please + */ + @Getter + private double maximumDeviceMemoryUsed = 0.85; + + /** + * Minimal number of activations for relocation threshold + */ + @Getter + private int minimumRelocationThreshold = 5; + + /** + * Minimal guaranteed TTL for memory chunk + */ + @Getter + private long minimumTTLMilliseconds = 10 * 1000L; + + /** + * Number of buckets/garbage collectors for host memory + */ + @Getter + private int numberOfGcThreads = 6; + + /** + * Deallocation aggressiveness + */ + @Deprecated + @Getter + private Aggressiveness hostDeallocAggressiveness = Aggressiveness.REASONABLE; + + @Deprecated + @Getter + private Aggressiveness gpuDeallocAggressiveness = Aggressiveness.REASONABLE; + + /** + * Allocation aggressiveness + */ + @Deprecated + @Getter + private Aggressiveness gpuAllocAggressiveness = Aggressiveness.REASONABLE; + + + /** + * Maximum allocated per-device memory, in bytes + */ + @Getter + private long maximumDeviceAllocation = 4 * 1024 * 1024 * 1024L; + + + /** + * Maximum allocatable zero-copy/pinned/pageable memory + */ + @Getter + private long maximumZeroAllocation = Runtime.getRuntime().maxMemory() + (500 * 1024 * 1024L); + + /** + * True if allowed, false if relocation required + */ + @Getter + private boolean crossDeviceAccessAllowed = true; + + /** + * True, if allowed, false otherwise + */ + @Getter + private boolean zeroCopyFallbackAllowed = false; + + /** + * Maximum length of single memory chunk + */ + @Getter + private long maximumSingleHostAllocation = Long.MAX_VALUE; + + @Getter + private long maximumSingleDeviceAllocation = 1024 * 1024 * 1024L; + + @Getter + private List availableDevices = new ArrayList<>(); + + @Getter + private List bannedDevices = new ArrayList<>(); + + @Getter + private int maximumGridSize = 4096; + + @Getter + private int maximumBlockSize = 256; + + @Getter + private int minimumBlockSize = 32; + + @Getter + private long maximumHostCache = 1024 * 1024 * 1024L; + + @Getter + private long maximumDeviceCache = 512L * 1024L * 1024L; + + @Getter + private boolean usePreallocation = false; + + @Getter + private int preallocationCalls = 10; + + @Getter + private long maximumHostCacheableLength = 100663296; + + @Getter + private long maximumDeviceCacheableLength = 16L * 1024L * 1024L; + + @Getter + private int commandQueueLength = 3; + + @Getter + private int commandLanesNumber = 4; + + @Getter + private int debugTriggered = 0; + + @Getter + private int poolSize = 32; + + private final AtomicBoolean initialized = new AtomicBoolean(false); + + public boolean isInitialized() { + return initialized.get(); + } + + public void setInitialized() { + this.initialized.compareAndSet(false, true); + } + + + private void parseEnvironmentVariables() { + + // Do not call System.getenv(): Accessing all variables requires higher security privileges + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_BLOCK_SIZE) != null) { + try { + int var = Integer.parseInt(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_BLOCK_SIZE)); + setMaximumBlockSize(var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_MAX_BLOCK_SIZE, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_BLOCK_SIZE)); + } + } + + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MIN_BLOCK_SIZE) != null) { + try { + int var = Integer.parseInt(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MIN_BLOCK_SIZE)); + setMinimumBlockSize(var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_MIN_BLOCK_SIZE, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MIN_BLOCK_SIZE)); + } + } + + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_GRID_SIZE) != null) { + try { + int var = Integer.parseInt(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_GRID_SIZE)); + setMaximumGridSize(var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_MAX_GRID_SIZE, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_GRID_SIZE)); + } + } + + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_CONTEXTS) != null) { + try { + int var = Integer.parseInt(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_CONTEXTS)); + setPoolSize(var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_MAX_CONTEXTS, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_CONTEXTS)); + } + } + + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_FORCE_SINGLE_GPU) != null) { + try { + boolean var = Boolean.parseBoolean(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_FORCE_SINGLE_GPU)); + allowMultiGPU(!var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_FORCE_SINGLE_GPU, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_FORCE_SINGLE_GPU)); + } + } + + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_USE_PREALLOCATION) != null) { + try { + boolean var = Boolean.parseBoolean(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_USE_PREALLOCATION)); + allowPreallocation(var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_USE_PREALLOCATION, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_USE_PREALLOCATION)); + } + } + + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_DEVICE_CACHE) != null) { + try { + long var = Long.parseLong(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_DEVICE_CACHE)); + setMaximumDeviceCache(var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_MAX_DEVICE_CACHE, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_DEVICE_CACHE)); + } + } + + + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_HOST_CACHE) != null) { + try { + long var = Long.parseLong(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_HOST_CACHE)); + setMaximumHostCache(var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_MAX_HOST_CACHE, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_HOST_CACHE)); + } + } + + if (System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_DEVICE_ALLOCATION) != null) { + try { + long var = Long.parseLong(System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_DEVICE_ALLOCATION)); + setMaximumSingleDeviceAllocation(var); + } catch (Exception e) { + log.error("Can't parse {}: [{}]", ND4JEnvironmentVars.ND4J_CUDA_MAX_DEVICE_ALLOCATION, System.getenv(ND4JEnvironmentVars.ND4J_CUDA_MAX_DEVICE_ALLOCATION)); + } + } + + } + + /** + * This method enables/disables + * + * @param reallyEnable + * @return + */ + public Configuration enableDashboard(boolean reallyEnable) { + fillDashboard = reallyEnable; + return this; + } + + /** + * Per-device resources pool size. Streams, utility memory + * + * @param poolSize + * @return + */ + public Configuration setPoolSize(int poolSize) { + if (poolSize < 8) + throw new IllegalStateException("poolSize can't be lower then 8"); + this.poolSize = poolSize; + return this; + } + + public Configuration triggerDebug(int code) { + this.debugTriggered = code; + return this; + } + + public Configuration setMinimumRelocationThreshold(int threshold) { + this.maximumDeviceAllocation = Math.max(2, threshold); + + return this; + } + + /** + * This method allows you to specify maximum memory cache for host memory + * + * @param maxCache + * @return + */ + public Configuration setMaximumHostCache(long maxCache) { + this.maximumHostCache = maxCache; + return this; + } + + /** + * This method allows you to specify maximum memory cache per device + * + * @param maxCache + * @return + */ + public Configuration setMaximumDeviceCache(long maxCache) { + this.maximumDeviceCache = maxCache; + return this; + } + + /** + * This method allows you to specify max per-device memory use. + * + * PLEASE NOTE: Accepted value range is 0.01 > x < 0.95 + * + * @param percentage + */ + public Configuration setMaximumDeviceMemoryUsed(double percentage) { + if (percentage < 0.02 || percentage > 0.95) { + this.maximumDeviceMemoryUsed = 0.85; + } else + this.maximumDeviceMemoryUsed = percentage; + + return this; + } + + public Configuration() { + parseEnvironmentVariables(); + } + + + void updateDevice() { + int cnt = Nd4j.getAffinityManager().getNumberOfDevices(); + + if (cnt == 0) + throw new RuntimeException("No CUDA devices were found in system"); + + for (int i = 0; i < cnt; i++) { + availableDevices.add(i); + } + } + + + + /** + * This method checks, if GPU subsystem supports cross-device P2P access over PCIe. + * + * PLEASE NOTE: This method also returns TRUE if system has only one device. This is done to guarantee reallocation avoidance within same device. + * + * @return + */ + public boolean isP2PSupported() { + return NativeOpsHolder.getInstance().getDeviceNativeOps().isP2PAvailable(); + } + + /** + * This method allows you to ban specific device. + * + * PLEASE NOTE: This method + * + * @param deviceId + * @return + */ + public Configuration banDevice(@NonNull Integer deviceId) { + if (!availableDevices.contains(deviceId)) + return this; + + if (!bannedDevices.contains(deviceId)) { + bannedDevices.add(deviceId); + } + + availableDevices.remove(deviceId); + + return this; + } + + /** + * This method forces specific device to be used. All other devices present in system will be ignored. + * + * @param deviceId + * @return + */ + public Configuration useDevice(@NonNull Integer deviceId) { + return useDevices(deviceId); + } + + /** + * This method forces specific devices to be used. All other devices present in system will be ignored. + * + * @param devices + * @return + */ + public Configuration useDevices(@NonNull int... devices) { + List usableDevices = new ArrayList<>(); + for (int device : devices) { + if (!availableDevices.contains(device)) { + log.warn("Non-existent device [{}] requested, ignoring...", device); + } else { + if (!usableDevices.contains(device)) + usableDevices.add(device); + } + + } + + if (usableDevices.size() > 0) { + availableDevices.clear(); + availableDevices.addAll(usableDevices); + } + + return this; + } + + /** + * This method allows you to set maximum host allocation. However, it's recommended to leave it as default: Xmx + something. + * + * @param max amount of memory in bytes + */ + public Configuration setMaximumZeroAllocation(long max) { + long xmx = Runtime.getRuntime().maxMemory(); + if (max < xmx) + log.warn("Setting maximum memory below -Xmx value can cause problems"); + + if (max <= 0) + throw new IllegalStateException("You can't set maximum host memory <= 0"); + + maximumZeroAllocation = max; + + return this; + } + + /** + * This method allows you to set maximum device allocation. It's recommended to keep it equal to MaximumZeroAllocation + * @param max + */ + public Configuration setMaximumDeviceAllocation(long max) { + if (max < 0) + throw new IllegalStateException("You can't set maximum device memory < 0"); + + return this; + } + + /** + * This method allows to specify maximum single allocation on host. + * + * Default value: Long.MAX_VALUE + * + * @param max + * @return + */ + public Configuration setMaximumSingleHostAllocation(long max) { + this.maximumSingleHostAllocation = max; + + return this; + } + + /** + * This method allows to specify maximum single allocation on device. + * + * Default value: Long.MAX_VALUE + * + * @param max + * @return + */ + public Configuration setMaximumSingleDeviceAllocation(long max) { + this.maximumSingleDeviceAllocation = max; + + return this; + } + + /** + * This method allows to specify max gridDim for kernel launches. + * + * Default value: 128 + * + * @param gridDim + * @return + */ + public Configuration setMaximumGridSize(int gridDim) { + if (gridDim <= 7 || gridDim > 8192) + throw new IllegalStateException("Please keep gridDim in range [8...8192]"); + + this.maximumGridSize = gridDim; + + return this; + } + + /** + * This methos allows to specify max blockSize for kernel launches + * + * Default value: -1 (that means pick value automatically, device occupancy dependent) + * + * @param blockDim + * @return + */ + public Configuration setMaximumBlockSize(int blockDim) { + if (blockDim < 32 || blockDim > 768) + throw new IllegalStateException("Please keep blockDim in range [32...768]"); + + + this.maximumBlockSize = blockDim; + + return this; + } + + public Configuration setMinimumBlockSize(int blockDim) { + if (blockDim < 32 || blockDim > 768) + throw new IllegalStateException("Please keep blockDim in range [32...768]"); + + + this.minimumBlockSize = blockDim; + + return this; + } + + /** + * With debug enabled all CUDA launches will become synchronous, with forced stream synchronizations after calls. + * + * Default value: false; + * + * @return + */ + public Configuration enableDebug(boolean debug) { + this.debug = debug; + return this; + } + + public Configuration setVerbose(boolean verbose) { + this.verbose = verbose; + return this; + } + + /** + * Enables/disables P2P memory access for multi-gpu + * + * @param reallyAllow + * @return + */ + public Configuration allowCrossDeviceAccess(boolean reallyAllow) { + this.crossDeviceAccessAllowed = reallyAllow; + + return this; + } + + /** + * This method allows to specify execution model for matrix/blas operations + * + * SEQUENTIAL: Issue commands in order Java compiler sees them. + * ASYNCHRONOUS: Issue commands asynchronously, if that's possible. + * OPTIMIZED: Not implemented yet. Equals to asynchronous for now. + * + * Default value: SEQUENTIAL + * + * @param executionModel + * @return + * @deprecated Only ExecutionModel.SEQUENTIAL is supported + */ + @Deprecated + public Configuration setExecutionModel(@NonNull ExecutionModel executionModel) { + if(executionModel != ExecutionModel.SEQUENTIAL){ + throw new IllegalArgumentException("Only ExecutionModel.SEQUENTIAL is supported"); + } + this.executionModel = ExecutionModel.SEQUENTIAL; + return this; + } + + /** + * This method allows to specify allocation model for memory. + * + * DIRECT: Do not cache anything, release memory as soon as it's not used. + * CACHE_HOST: Cache host memory only, Device memory (if any) will use DIRECT mode. + * CACHE_ALL: All memory will be cached. + * + * Defailt value: CACHE_ALL + * + * @param allocationModel + * @return + */ + public Configuration setAllocationModel(@NonNull AllocationModel allocationModel) { + this.allocationModel = allocationModel; + + return this; + } + + /** + * This method allows to specify initial memory to be used within system. + * HOST: all data is located on host memory initially, and gets into DEVICE, if used frequent enough + * DEVICE: all memory is located on device. + * DELAYED: memory allocated on HOST first, and on first use gets moved to DEVICE + * + * PLEASE NOTE: For device memory all data still retains on host side as well. + * + * Default value: DEVICE + * @param initialMemory + * @return + */ + public Configuration setFirstMemory(@NonNull AllocationStatus initialMemory) { + if (initialMemory != AllocationStatus.DEVICE && initialMemory != AllocationStatus.HOST + && initialMemory != AllocationStatus.DELAYED) + throw new IllegalStateException("First memory should be either [HOST], [DEVICE] or [DELAYED]"); + + this.firstMemory = initialMemory; + + return this; + } + + /** + * NOT IMPLEMENTED YET + * @param reallyAllow + * @return + */ + public Configuration allowFallbackFromDevice(boolean reallyAllow) { + this.zeroCopyFallbackAllowed = reallyAllow; + return this; + } + + /** + * This method allows you to set number of threads that'll handle memory releases on native side. + * + * Default value: 4 + * @return + */ + public Configuration setNumberOfGcThreads(int numThreads) { + if (numThreads <= 0 || numThreads > 20) + throw new IllegalStateException("Please, use something in range of [1..20] as number of GC threads"); + + if (!isInitialized()) + this.numberOfGcThreads = numThreads; + + return this; + } + + /** + * This method allows to specify maximum length of single memory chunk that's allowed to be cached. + * Please note: -1 value totally disables limits here. + * + * Default value: 96 MB + * @param maxLen + * @return + */ + public Configuration setMaximumHostCacheableLength(long maxLen) { + this.maximumHostCacheableLength = maxLen; + + return this; + } + + /** + * This method allows to specify maximum length of single memory chunk that's allowed to be cached. + * Please note: -1 value totally disables limits here. + * + * Default value: 96 MB + * @param maxLen + * @return + */ + public Configuration setMaximumDeviceCacheableLength(long maxLen) { + this.maximumDeviceCacheableLength = maxLen; + + return this; + } + + /** + * If set to true, each non-cached allocation request will cause few additional allocations, + * + * Default value: true + * + * @param reallyAllow + * @return + */ + public Configuration allowPreallocation(boolean reallyAllow) { + this.usePreallocation = reallyAllow; + + return this; + } + + /** + * This method allows to specify number of preallocation calls done by cache subsystem in parallel, to serve later requests. + * + * Default value: 25 + * + * @param numCalls + * @return + */ + public Configuration setPreallocationCalls(int numCalls) { + if (numCalls < 0 || numCalls > 100) + throw new IllegalStateException("Please use preallocation calls in range of [1..100]"); + this.preallocationCalls = numCalls; + + return this; + } + + /** + * This method allows you to specify command queue length, as primary argument for asynchronous execution controller + * + * Default value: 3 + * + * @param length + * @return + */ + public Configuration setCommandQueueLength(int length) { + if (length <= 0) + throw new IllegalStateException("Command queue length can't be <= 0"); + this.commandQueueLength = length; + + return this; + } + + /** + * This option specifies minimal time gap between two subsequent System.gc() calls + * Set to 0 to disable this option. + * + * @param windowMs + * @return + */ + public Configuration setNoGcWindowMs(long windowMs) { + if (windowMs < 1) + throw new IllegalStateException("No-GC window should have positive value"); + + this.noGcWindowMs = windowMs; + return this; + } + + /** + * This method allows you to specify maximum number of probable parallel cuda processes + * + * Default value: 4 + * + * PLEASE NOTE: This parameter has effect only for ASYNCHRONOUS execution model + * + * @param length + * @return + */ + public Configuration setCommandLanesNumber(int length) { + if (length < 1) + throw new IllegalStateException("Command Lanes number can't be < 1"); + if (length > 8) + length = 8; + this.commandLanesNumber = length; + + return this; + } + + public boolean isForcedSingleGPU() { + return forceSingleGPU; + } + + /** + * This method allows you to enable or disable multi-GPU mode. + * + * PLEASE NOTE: This is NOT magic method, that will automatically scale your application performance. + * + * @param reallyAllow + * @return + */ + public Configuration allowMultiGPU(boolean reallyAllow) { + forceSingleGPU = !reallyAllow; + return this; + } + + public Configuration setMemoryModel(@NonNull MemoryModel model) { + memoryModel = model; + return this; + } +} diff --git a/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/conf/CudaEnvironment.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/conf/CudaEnvironment.java new file mode 100644 index 000000000..69f600cd5 --- /dev/null +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/conf/CudaEnvironment.java @@ -0,0 +1,81 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.jita.conf; + +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.nativeblas.NativeOpsHolder; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + * The cuda environment contains information + * for a given {@link Configuration} + * singleton. + * + * @author raver119@gmail.com + */ +public class CudaEnvironment { + private static final CudaEnvironment INSTANCE = new CudaEnvironment(); + private static volatile Configuration configuration; + private static Map arch = new ConcurrentHashMap<>(); + + private CudaEnvironment() { + configuration = new Configuration(); + + } + + public static CudaEnvironment getInstance() { + return INSTANCE; + } + + /** + * Get the {@link Configuration} + * for the environment + * @return + */ + public Configuration getConfiguration() { + return configuration; + } + + /** + * Get the current device architecture + * @return the major/minor version of + * the current device + */ + public int getCurrentDeviceArchitecture() { + int deviceId = Nd4j.getAffinityManager().getDeviceForCurrentThread(); + if (!arch.containsKey(deviceId)) { + int major = NativeOpsHolder.getInstance().getDeviceNativeOps().getDeviceMajor(deviceId); + int minor = NativeOpsHolder.getInstance().getDeviceNativeOps().getDeviceMinor(deviceId); + Integer cc = Integer.parseInt(new String("" + major + minor)); + arch.put(deviceId, cc); + return cc; + } + + return arch.get(deviceId); + } + + public void notifyConfigurationApplied() { + configuration.updateDevice(); + } +} diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/conf/DeviceInformation.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/conf/DeviceInformation.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/conf/DeviceInformation.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/conf/DeviceInformation.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ConstantProtector.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ConstantProtector.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ConstantProtector.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ConstantProtector.java index b6c5dd03d..635b4d3dd 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ConstantProtector.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ConstantProtector.java @@ -20,9 +20,9 @@ package org.nd4j.jita.constant; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; import org.nd4j.common.primitives.Pair; import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.shape.LongShapeDescriptor; import org.nd4j.linalg.api.shape.ShapeDescriptor; import org.nd4j.linalg.factory.Nd4j; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/CudaConstantHandler.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/CudaConstantHandler.java similarity index 95% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/CudaConstantHandler.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/CudaConstantHandler.java index 1a616a3bb..cf329176b 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/CudaConstantHandler.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/CudaConstantHandler.java @@ -20,20 +20,21 @@ package org.nd4j.jita.constant; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.cache.BasicConstantHandler; import org.nd4j.linalg.cache.ConstantHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * ConstantHandler implementation for CUDA backend. * * @author raver119@gmail.com */ +@Slf4j public class CudaConstantHandler extends BasicConstantHandler { - private static Logger logger = LoggerFactory.getLogger(CudaConstantHandler.class); + protected static final ConstantHandler wrappedHandler = ProtectedCudaConstantHandler.getInstance(); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ProtectedCudaConstantHandler.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ProtectedCudaConstantHandler.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ProtectedCudaConstantHandler.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ProtectedCudaConstantHandler.java index e5be68a75..239fa6a8e 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ProtectedCudaConstantHandler.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ProtectedCudaConstantHandler.java @@ -20,8 +20,11 @@ package org.nd4j.jita.constant; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.bytedeco.javacpp.Pointer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.jita.allocator.enums.AllocationStatus; import org.nd4j.jita.allocator.impl.AllocationPoint; import org.nd4j.jita.allocator.impl.AtomicAllocator; @@ -31,6 +34,7 @@ import org.nd4j.jita.flow.FlowController; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.AllocationsTracker; +import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.api.memory.enums.AllocationKind; import org.nd4j.linalg.api.ops.performance.PerformanceTracker; import org.nd4j.linalg.cache.ArrayDescriptor; @@ -38,12 +42,7 @@ import org.nd4j.linalg.cache.ConstantHandler; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.buffer.*; -import org.nd4j.linalg.api.memory.MemcpyDirection; -import org.nd4j.common.util.ArrayUtil; import org.nd4j.nativeblas.NativeOpsHolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import lombok.extern.slf4j.Slf4j; import java.util.HashMap; import java.util.Map; @@ -68,8 +67,6 @@ public class ProtectedCudaConstantHandler implements ConstantHandler { protected static final ConstantProtector protector = ConstantProtector.getInstance(); - private static Logger logger = LoggerFactory.getLogger(ProtectedCudaConstantHandler.class); - private static final int MAX_CONSTANT_LENGTH = 49152; private static final int MAX_BUFFER_LENGTH = 272; @@ -93,7 +90,7 @@ public class ProtectedCudaConstantHandler implements ConstantHandler { protector.purgeProtector(); resetHappened = true; - logger.info("Resetting Constants..."); + log.info("Resetting Constants..."); for (Integer device : constantOffsets.keySet()) { constantOffsets.get(device).set(0); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ProtectedCudaShapeInfoProvider.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ProtectedCudaShapeInfoProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ProtectedCudaShapeInfoProvider.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ProtectedCudaShapeInfoProvider.java index b232f67d6..e225e68c8 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/constant/ProtectedCudaShapeInfoProvider.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/constant/ProtectedCudaShapeInfoProvider.java @@ -21,14 +21,14 @@ package org.nd4j.jita.constant; import lombok.extern.slf4j.Slf4j; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.shape.LongShapeDescriptor; -import org.nd4j.linalg.api.shape.options.ArrayOptionsHelper; -import org.nd4j.linalg.api.shape.options.ArrayType; import org.nd4j.common.primitives.Pair; import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.BaseShapeInfoProvider; +import org.nd4j.linalg.api.shape.LongShapeDescriptor; +import org.nd4j.linalg.api.shape.options.ArrayOptionsHelper; +import org.nd4j.linalg.api.shape.options.ArrayType; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicLong; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/FlowController.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/flow/FlowController.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/FlowController.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/flow/FlowController.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/GridFlowController.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/flow/impl/GridFlowController.java similarity index 92% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/GridFlowController.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/flow/impl/GridFlowController.java index 7dd2291e5..6993cdde7 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/GridFlowController.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/flow/impl/GridFlowController.java @@ -20,11 +20,10 @@ package org.nd4j.jita.flow.impl; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import org.nd4j.jita.allocator.impl.AllocationPoint; -import org.nd4j.linalg.api.ops.executioner.GridExecutioner; import org.nd4j.linalg.factory.Nd4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -34,10 +33,9 @@ import org.slf4j.LoggerFactory; * * @author raver119@gmail.com */ +@Slf4j public class GridFlowController extends SynchronousFlowController { - private static Logger logger = LoggerFactory.getLogger(GridFlowController.class); - /** * This method makes sure HOST memory contains latest data from GPU * diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java index a97d836ed..2d45334d5 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java @@ -23,6 +23,8 @@ package org.nd4j.jita.flow.impl; import lombok.Getter; import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.nd4j.jita.allocator.Allocator; import org.nd4j.jita.allocator.enums.AllocationStatus; @@ -40,14 +42,13 @@ import org.nd4j.linalg.jcublas.JCublasNDArray; import org.nd4j.linalg.jcublas.context.CudaContext; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + /** * @author raver119@gmail.com */ +@Slf4j public class SynchronousFlowController implements FlowController { - private static Logger log = LoggerFactory.getLogger(SynchronousFlowController.class); private volatile Allocator allocator; protected NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); protected Configuration configuration = CudaEnvironment.getInstance().getConfiguration(); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/MemoryHandler.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/handler/MemoryHandler.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/MemoryHandler.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/handler/MemoryHandler.java index 2a82f06e5..6310b267f 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/MemoryHandler.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/handler/MemoryHandler.java @@ -20,7 +20,7 @@ package org.nd4j.jita.handler; -import org.nd4j.shade.guava.collect.Table; +import com.google.common.collect.Table; import org.bytedeco.javacpp.Pointer; import org.nd4j.jita.allocator.Allocator; import org.nd4j.jita.allocator.enums.AllocationStatus; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java index feb648962..abc3aa5f0 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java @@ -20,16 +20,15 @@ package org.nd4j.jita.handler.impl; -import lombok.var; -import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.nativeblas.OpaqueLaunchContext; -import org.nd4j.shade.guava.collect.HashBasedTable; -import org.nd4j.shade.guava.collect.Table; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.lang3.RandomUtils; import org.bytedeco.javacpp.Pointer; +import org.nd4j.common.base.Preconditions; import org.nd4j.jita.allocator.Allocator; import org.nd4j.jita.allocator.concurrency.DeviceAllocationsTracker; import org.nd4j.jita.allocator.enums.AllocationStatus; @@ -50,6 +49,7 @@ import org.nd4j.jita.handler.MemoryHandler; import org.nd4j.jita.memory.MemoryProvider; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.concurrency.AffinityManager; +import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.performance.PerformanceTracker; @@ -57,12 +57,11 @@ import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer; import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.profiler.OpProfiler; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.nd4j.nativeblas.OpaqueLaunchContext; + import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -80,11 +79,10 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; * * @author raver119@gmail.com */ +@Slf4j public class CudaZeroHandler implements MemoryHandler { private static Configuration configuration = CudaEnvironment.getInstance().getConfiguration(); - private static Logger log = LoggerFactory.getLogger(CudaZeroHandler.class); - // simple counter to track allocated host-memory protected final AtomicLong zeroUseCounter = new AtomicLong(0); @@ -340,11 +338,11 @@ public class CudaZeroHandler implements MemoryHandler { CudaContext tContext = null; if (dstBuffer.isConstant()) { - org.bytedeco.javacpp.Pointer dstPointer = new CudaPointer(point.getHostPointer().address() + dstOffset, 0L); - org.bytedeco.javacpp.Pointer srcPointerJ = new CudaPointer(srcPointer, length); + Pointer dstPointer = new CudaPointer(point.getHostPointer().address() + dstOffset, 0L); + Pointer srcPointerJ = new CudaPointer(srcPointer, length); val profD = PerformanceTracker.getInstance().helperStartTransaction(); - org.bytedeco.javacpp.Pointer.memcpy(dstPointer, srcPointerJ, length); + Pointer.memcpy(dstPointer, srcPointerJ, length); PerformanceTracker.getInstance().helperRegisterTransaction(point.getDeviceId(), profD, point.getNumberOfBytes(), MemcpyDirection.HOST_TO_HOST); point.tickHostRead(); @@ -355,7 +353,7 @@ public class CudaZeroHandler implements MemoryHandler { if (tContext == null) tContext = flowController.prepareAction(point); - var prof = PerformanceTracker.getInstance().helperStartTransaction(); + long prof = PerformanceTracker.getInstance().helperStartTransaction(); flowController.commitTransfer(tContext.getSpecialStream()); @@ -532,7 +530,7 @@ public class CudaZeroHandler implements MemoryHandler { * @return */ @Override - public org.bytedeco.javacpp.Pointer getDevicePointer(DataBuffer buffer, CudaContext context) { + public Pointer getDevicePointer(DataBuffer buffer, CudaContext context) { // TODO: It would be awesome to get rid of typecasting here AllocationPoint dstPoint = ((BaseCudaDataBuffer) buffer).getAllocationPoint(); @@ -588,7 +586,7 @@ public class CudaZeroHandler implements MemoryHandler { * @return */ @Override - public org.bytedeco.javacpp.Pointer getHostPointer(DataBuffer buffer) { + public Pointer getHostPointer(DataBuffer buffer) { AllocationPoint dstPoint = ((BaseCudaDataBuffer) buffer).getAllocationPoint(); // return pointer with offset if needed. length is specified for constructor compatibility purposes @@ -1031,7 +1029,7 @@ public class CudaZeroHandler implements MemoryHandler { * @return */ public CudaContext getCudaContext() { - var ctx = tlContext.get(); + CudaContext ctx = tlContext.get(); if (ctx == null) { val lc = nativeOps.defaultLaunchContext(); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/memory/CudaMemoryManager.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/memory/CudaMemoryManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/memory/CudaMemoryManager.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/memory/CudaMemoryManager.java index f47eb38f9..d0477085d 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/memory/CudaMemoryManager.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/memory/CudaMemoryManager.java @@ -29,15 +29,15 @@ import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.jita.conf.CudaEnvironment; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.memory.AllocationsTracker; +import org.nd4j.linalg.api.memory.BasicMemoryManager; import org.nd4j.linalg.api.memory.enums.AllocationKind; +import org.nd4j.linalg.api.memory.enums.MemoryKind; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.compression.CompressedDataBuffer; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer; import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.linalg.api.memory.BasicMemoryManager; -import org.nd4j.linalg.api.memory.enums.MemoryKind; import org.nd4j.nativeblas.NativeOpsHolder; import java.util.Map; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/memory/MemoryProvider.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/memory/MemoryProvider.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/memory/MemoryProvider.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/memory/MemoryProvider.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java index f40102b6c..62a02fd12 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/workspace/CudaWorkspace.java @@ -26,20 +26,21 @@ import lombok.val; import org.bytedeco.javacpp.Pointer; import org.nd4j.jita.allocator.impl.AllocationShape; import org.nd4j.jita.allocator.impl.AtomicAllocator; +import org.nd4j.jita.allocator.impl.MemoryTracker; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.AllocationsTracker; +import org.nd4j.linalg.api.memory.Deallocator; +import org.nd4j.linalg.api.memory.abstracts.Nd4jWorkspace; import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; import org.nd4j.linalg.api.memory.enums.*; import org.nd4j.linalg.api.memory.pointers.PagedPointer; import org.nd4j.linalg.api.memory.pointers.PointersPair; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.api.memory.abstracts.Nd4jWorkspace; import org.nd4j.nativeblas.NativeOpsHolder; -import org.nd4j.linalg.api.memory.Deallocator; + import java.util.List; import java.util.Queue; -import org.nd4j.jita.allocator.impl.MemoryTracker; /** diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceDeallocator.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceDeallocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceDeallocator.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceDeallocator.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceManager.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceManager.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceManager.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceManager.java index 2e3b4d453..d0cb84ad6 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceManager.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/jita/workspace/CudaWorkspaceManager.java @@ -22,11 +22,11 @@ package org.nd4j.jita.workspace; import lombok.NonNull; import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.api.memory.abstracts.DummyWorkspace; import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; import org.nd4j.linalg.api.memory.enums.DebugMode; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.api.memory.abstracts.DummyWorkspace; import org.nd4j.linalg.api.memory.provider.BasicWorkspaceManager; +import org.nd4j.linalg.factory.Nd4j; /** * @author raver119@gmail.com diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/CachedShapeInfoProvider.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/CachedShapeInfoProvider.java similarity index 93% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/CachedShapeInfoProvider.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/CachedShapeInfoProvider.java index 3fb92b838..cfd670c09 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/CachedShapeInfoProvider.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/CachedShapeInfoProvider.java @@ -20,20 +20,21 @@ package org.nd4j.linalg.jcublas; -import org.nd4j.linalg.api.buffer.DataType; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import org.nd4j.common.primitives.Pair; import org.nd4j.jita.constant.ProtectedCudaShapeInfoProvider; import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.BaseShapeInfoProvider; import org.nd4j.linalg.api.ndarray.ShapeInfoProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + /** * @author raver119@gmail.com */ +@Slf4j public class CachedShapeInfoProvider extends BaseShapeInfoProvider { - private static Logger logger = LoggerFactory.getLogger(CachedShapeInfoProvider.class); protected ShapeInfoProvider provider = ProtectedCudaShapeInfoProvider.getInstance(); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/CublasPointer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/CublasPointer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/CublasPointer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/CublasPointer.java index 9cd800ba6..911348239 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/CublasPointer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/CublasPointer.java @@ -21,12 +21,12 @@ package org.nd4j.linalg.jcublas; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.bytedeco.javacpp.Pointer; import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.jcublas.buffer.JCudaBuffer; import org.nd4j.linalg.jcublas.context.CudaContext; -import lombok.extern.slf4j.Slf4j; /** * Wraps the allocation diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasBackend.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasBackend.java similarity index 79% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasBackend.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasBackend.java index f0407bd2b..40fa8c5bb 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasBackend.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasBackend.java @@ -1,21 +1,22 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas; @@ -23,15 +24,16 @@ package org.nd4j.linalg.jcublas; import lombok.extern.slf4j.Slf4j; import org.bytedeco.javacpp.Loader; import org.nd4j.common.config.ND4JSystemProperties; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.common.io.Resource; import org.nd4j.linalg.api.environment.Nd4jEnvironment; import org.nd4j.linalg.factory.Environment; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.factory.Nd4jBackend; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.io.Resource; -import org.nd4j.nativeblas.CudaEnvironment; -import org.nd4j.nativeblas.Nd4jCuda; +import org.nd4j.nativeblas.cuda.CudaEnvironment; import org.nd4j.nativeblas.NativeOpsHolder; +import org.nd4j.nativeblas.Nd4jCuda; + import java.util.List; import java.util.Map; import java.util.Properties; @@ -80,7 +82,7 @@ public class JCublasBackend extends Nd4jBackend { @Override public int getPriority() { - return BACKEND_PRIORITY_GPU; + return Nd4jBackend.BACKEND_PRIORITY_GPU; } @Override diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java index 04dfb10cb..e581faa72 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java @@ -32,6 +32,7 @@ import org.nd4j.jita.allocator.impl.AllocationPoint; import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.ndarray.BaseNDArray; import org.nd4j.linalg.api.ndarray.BaseNDArrayProxy; @@ -44,7 +45,6 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.buffer.CudaLongDataBuffer; import org.nd4j.linalg.jcublas.buffer.CudaUtf8Buffer; import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.workspace.WorkspaceUtils; import org.nd4j.nativeblas.NativeOpsHolder; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArrayFactory.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArrayFactory.java similarity index 90% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArrayFactory.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArrayFactory.java index 8a1369856..88c08ceaa 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArrayFactory.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArrayFactory.java @@ -1,61 +1,66 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas; +import lombok.extern.log4j.Log4j2; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.buffer.DataTypeEx; -import org.nd4j.linalg.api.memory.enums.MemoryKind; -import org.nd4j.linalg.api.ops.custom.Flatten; -import org.nd4j.linalg.api.ops.impl.shape.Concat; -import org.nd4j.linalg.api.ops.performance.PerformanceTracker; -import org.nd4j.linalg.api.shape.options.ArrayOptionsHelper; -import org.nd4j.linalg.api.shape.options.ArrayType; -import org.nd4j.linalg.compression.CompressionUtils; -import org.nd4j.linalg.jcublas.buffer.*; -import org.nd4j.linalg.api.memory.MemcpyDirection; -import org.nd4j.common.primitives.Pair; import org.bytedeco.javacpp.*; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.jita.allocator.enums.CudaConstants; import org.nd4j.jita.allocator.impl.AllocationPoint; import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.jita.allocator.pointers.CudaPointer; import org.nd4j.jita.conf.CudaEnvironment; import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.buffer.DataTypeEx; +import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.api.memory.enums.MemoryKind; import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.custom.Flatten; import org.nd4j.linalg.api.ops.executioner.GridExecutioner; +import org.nd4j.linalg.api.ops.impl.shape.Concat; +import org.nd4j.linalg.api.ops.performance.PerformanceTracker; import org.nd4j.linalg.api.shape.Shape; +import org.nd4j.linalg.api.shape.options.ArrayOptionsHelper; +import org.nd4j.linalg.api.shape.options.ArrayType; import org.nd4j.linalg.cache.TADManager; import org.nd4j.linalg.compression.CompressedDataBuffer; import org.nd4j.linalg.compression.CompressionDescriptor; import org.nd4j.linalg.compression.CompressionType; +import org.nd4j.linalg.compression.CompressionUtils; import org.nd4j.linalg.exception.ND4JIllegalStateException; +import org.nd4j.linalg.factory.NDArrayFactory; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.blas.*; +import org.nd4j.linalg.jcublas.buffer.*; import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.common.util.ArrayUtil; -import org.nd4j.nativeblas.*; +import org.nd4j.nativeblas.BaseNativeNDArrayFactory; +import org.nd4j.nativeblas.LongPointerWrapper; +import org.nd4j.nativeblas.PointerPointerWrapper; import java.util.*; @@ -69,7 +74,8 @@ import java.util.*; public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { - public JCublasNDArrayFactory() { } + public JCublasNDArrayFactory() { + } public JCublasNDArrayFactory(DataType dtype, Character order) { super(dtype, order); @@ -168,7 +174,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { @Override public INDArray create(DataBuffer data, long rows, long columns, int[] stride, long offset) { // FIXME: int cast - return new JCublasNDArray(data, new long[] {rows, columns}, ArrayUtil.toLongArray(stride), Nd4j.order(), data.dataType()); + return new JCublasNDArray(data, new long[]{rows, columns}, ArrayUtil.toLongArray(stride), Nd4j.order(), data.dataType()); } @Override @@ -193,7 +199,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { @Override public INDArray create(float[] data, long rows, long columns, int[] stride, long offset, char ordering) { - return new JCublasNDArray(data, new long[] {rows, columns}, ArrayUtil.toLongArray(stride), offset, ordering); + return new JCublasNDArray(data, new long[]{rows, columns}, ArrayUtil.toLongArray(stride), offset, ordering); } @Override @@ -276,7 +282,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { */ @Override public INDArray create(List list, int[] shape) { - if (order == FORTRAN) + if (order == NDArrayFactory.FORTRAN) return new JCublasNDArray(list, shape, ArrayUtil.calcStridesFortran(shape)); else return new JCublasNDArray(list, shape); @@ -405,9 +411,9 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { ((BaseCudaDataBuffer) ret.data()).lazyAllocateHostPointer(); nativeOps.specialConcat(null, dimension, toConcat.length, dataPointers, shapeInfoPointers, - ret.data().addressPointer(), - (LongPointer) ret.shapeInfoDataBuffer().addressPointer(), - null, null); + ret.data().addressPointer(), + (LongPointer) ret.shapeInfoDataBuffer().addressPointer(), + null, null); if (nativeOps.lastErrorCode() != 0) throw new RuntimeException(nativeOps.lastErrorMessage()); @@ -431,7 +437,6 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { } - /** * This method produces concatenated array, that consist from tensors, fetched from source array, against some dimension and specified indexes * @@ -468,9 +473,9 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { if (source.rank() == 1) { shape = new long[]{indexes.length}; } else if (sourceDimension == 1) - shape = new long[] {indexes.length, source.shape()[sourceDimension]}; + shape = new long[]{indexes.length, source.shape()[sourceDimension]}; else if (sourceDimension == 0) - shape = new long[] {source.shape()[sourceDimension], indexes.length}; + shape = new long[]{source.shape()[sourceDimension], indexes.length}; else throw new UnsupportedOperationException("2D input is expected"); @@ -490,17 +495,17 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { if (source.rank() == 1) { shape = new long[]{indexes.length}; } else if (sourceDimension == 1) - shape = new long[] {indexes.length, source.shape()[sourceDimension]}; + shape = new long[]{indexes.length, source.shape()[sourceDimension]}; else if (sourceDimension == 0) - shape = new long[] {source.shape()[sourceDimension], indexes.length}; + shape = new long[]{source.shape()[sourceDimension], indexes.length}; else throw new UnsupportedOperationException("2D input is expected"); INDArray ret = destination; - if(ret == null){ + if (ret == null) { ret = Nd4j.createUninitialized(source.dataType(), shape, order); } else { - if(!Arrays.equals(shape, destination.shape())){ + if (!Arrays.equals(shape, destination.shape())) { throw new IllegalStateException("Cannot pull rows into destination array: expected destination array of" + " shape " + Arrays.toString(shape) + " but got destination array of shape " + Arrays.toString(destination.shape())); } @@ -524,8 +529,8 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { TADManager tadManager = Nd4j.getExecutioner().getTADManager(); - Pair tadBuffers = tadManager.getTADOnlyShapeInfo(source, new int[] {sourceDimension}); - Pair zTadBuffers = tadManager.getTADOnlyShapeInfo(ret, new int[] {sourceDimension}); + Pair tadBuffers = tadManager.getTADOnlyShapeInfo(source, new int[]{sourceDimension}); + Pair zTadBuffers = tadManager.getTADOnlyShapeInfo(ret, new int[]{sourceDimension}); Pointer tadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); Pointer zTadShapeInfo = AtomicAllocator.getInstance().getPointer(zTadBuffers.getFirst(), context); @@ -597,7 +602,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { PointerPointer x = new PointerPointer(AtomicAllocator.getInstance().getPointer(tempX, context)); - nativeOps.accumulate(extras, null, (LongPointer) arrays[0].shapeInfoDataBuffer().addressPointer(), x, null, null, (LongPointer) allocator.getHostPointer(target.shapeInfoDataBuffer()) , z, (LongPointer) allocator.getPointer(target.shapeInfoDataBuffer()), arrays.length, len); + nativeOps.accumulate(extras, null, (LongPointer) arrays[0].shapeInfoDataBuffer().addressPointer(), x, null, null, (LongPointer) allocator.getHostPointer(target.shapeInfoDataBuffer()), z, (LongPointer) allocator.getPointer(target.shapeInfoDataBuffer()), arrays.length, len); if (nativeOps.lastErrorCode() != 0) throw new RuntimeException(nativeOps.lastErrorMessage()); @@ -614,7 +619,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { val dataPointers = new PointerPointer(arrays.length); val extras = new PointerPointer(null, // not used - context.getOldStream(), AtomicAllocator.getInstance().getDeviceIdPointer(), new CudaPointer(1) ); + context.getOldStream(), AtomicAllocator.getInstance().getDeviceIdPointer(), new CudaPointer(1)); for (int i = 0; i < arrays.length; i++) { Nd4j.getCompressor().autoDecompress(arrays[i]); @@ -663,7 +668,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { if (arrays.length == 1) { //Edge case - average 1 array - no op - if(target == null){ + if (target == null) { return null; } return target.assign(arrays[0]); @@ -712,7 +717,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { x, null, null, - (LongPointer) (target == null ? null : target.shapeInfoDataBuffer().addressPointer()), + (LongPointer) (target == null ? null : target.shapeInfoDataBuffer().addressPointer()), target == null ? null : z, null, arrays.length, @@ -735,7 +740,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { val dataPointers = new PointerPointer(arrays.length); val extras = new PointerPointer(null, // not used - context.getOldStream(), AtomicAllocator.getInstance().getDeviceIdPointer(), new CudaPointer(1) ); + context.getOldStream(), AtomicAllocator.getInstance().getDeviceIdPointer(), new CudaPointer(1)); for (int i = 0; i < arrays.length; i++) { Nd4j.getCompressor().autoDecompress(arrays[i]); @@ -760,7 +765,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { null, null, target == null ? null : target.data().addressPointer(), - (LongPointer) (target == null ? null : target.shapeInfoDataBuffer().addressPointer()), + (LongPointer) (target == null ? null : target.shapeInfoDataBuffer().addressPointer()), null, null, arrays.length, @@ -824,7 +829,6 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { * * @param array the ndarray to shuffle * @param dimension the dimension to do the shuffle - * @return */ @Override public void shuffle(INDArray array, Random rnd, int... dimension) { @@ -835,9 +839,8 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { * Symmetric in place shuffle of an ndarray * along a specified set of dimensions. Each array in list should have it's own dimension at the same index of dimensions array * - * @param arrays the ndarrays to shuffle + * @param arrays the ndarrays to shuffle * @param dimensions the dimensions to do the shuffle - * @return */ @Override public void shuffle(List arrays, Random rnd, List dimensions) { @@ -879,7 +882,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { val shuffleMap = allocator.getPointer(shuffle, context); val extras = new PointerPointer(null, // not used - context.getOldStream(), allocator.getDeviceIdPointer()); + context.getOldStream(), allocator.getDeviceIdPointer()); long[] hPointers = new long[arrays.size()]; @@ -933,16 +936,16 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { AtomicAllocator.getInstance().memcpyBlocking(tempOffsets, new LongPointer(tadOffsets), xPointers.length * 8, 0); nativeOps.shuffle(extras, - null, - hosthost, - new PointerPointer(allocator.getPointer(tempX, context)), - new PointerPointer(allocator.getPointer(tempShapes, context)), - null, - null, - new PointerPointer(allocator.getPointer(tempX, context)), - new PointerPointer(allocator.getPointer(tempShapes, context)), arrays.size(), - (IntPointer) shuffleMap, new PointerPointer(allocator.getPointer(tempTAD, context)), - new PointerPointer(allocator.getPointer(tempOffsets, context))); + null, + hosthost, + new PointerPointer(allocator.getPointer(tempX, context)), + new PointerPointer(allocator.getPointer(tempShapes, context)), + null, + null, + new PointerPointer(allocator.getPointer(tempX, context)), + new PointerPointer(allocator.getPointer(tempShapes, context)), arrays.size(), + (IntPointer) shuffleMap, new PointerPointer(allocator.getPointer(tempTAD, context)), + new PointerPointer(allocator.getPointer(tempOffsets, context))); if (nativeOps.lastErrorCode() != 0) throw new RuntimeException(nativeOps.lastErrorMessage()); @@ -966,8 +969,8 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { * Symmetric in place shuffle of an ndarray * along a specified set of dimensions. All arrays * - * @param sourceArrays the ndarray to shuffle - * @param dimension the dimension to do the shuffle + * @param sourceArrays the ndarray to shuffle + * @param dimension the dimension to do the shuffle * @return */ @Override @@ -1084,7 +1087,6 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { } - @Override public void convertDataEx(DataTypeEx typeSrc, Pointer source, DataTypeEx typeDst, Pointer target, long length) { val stream = AtomicAllocator.getInstance().getDeviceContext().getOldStream(); @@ -1203,7 +1205,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { // we were compressing something into temporary buffer if (target instanceof CompressedDataBuffer) { - nativeOps.memcpyAsync(target.addressPointer(), dstPtr, target.capacity(), CudaConstants.cudaMemcpyHostToHost, stream); + nativeOps.memcpyAsync(target.addressPointer(), dstPtr, target.capacity(), CudaConstants.cudaMemcpyHostToHost, stream); if (Nd4j.getWorkspaceManager().anyWorkspaceActiveForCurrentThread()) { // no-op, workspace was used @@ -1288,7 +1290,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { } - int numTads = (int)(tensor.length() / tadLength); + int numTads = (int) (tensor.length() / tadLength); INDArray[] result = new INDArray[numTads]; long[] xPointers = new long[numTads]; @@ -1314,18 +1316,18 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { nativeOps.tear(extraz, - x, (LongPointer) tensor.shapeInfoDataBuffer().addressPointer(), (LongPointer) AtomicAllocator.getInstance().getPointer(tensor.shapeInfoDataBuffer(), context), - new PointerPointer(AtomicAllocator.getInstance().getPointer(tempX, context)), - (LongPointer) AtomicAllocator.getInstance().getPointer(result[0].shapeInfoDataBuffer(), context), - (LongPointer) AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context), - new LongPointerWrapper(AtomicAllocator.getInstance().getPointer(tadBuffers.getSecond(), context)) - ); + x, (LongPointer) tensor.shapeInfoDataBuffer().addressPointer(), (LongPointer) AtomicAllocator.getInstance().getPointer(tensor.shapeInfoDataBuffer(), context), + new PointerPointer(AtomicAllocator.getInstance().getPointer(tempX, context)), + (LongPointer) AtomicAllocator.getInstance().getPointer(result[0].shapeInfoDataBuffer(), context), + (LongPointer) AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context), + new LongPointerWrapper(AtomicAllocator.getInstance().getPointer(tadBuffers.getSecond(), context)) + ); if (nativeOps.lastErrorCode() != 0) throw new RuntimeException(nativeOps.lastErrorMessage()); AtomicAllocator.getInstance().getFlowController().registerActionAllWrite(context, result); - AtomicAllocator.getInstance().getFlowController().registerAction(context,null, result); + AtomicAllocator.getInstance().getFlowController().registerAction(context, null, result); return result; } @@ -1372,12 +1374,12 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { nativeOps.sort(extraz, - null, - (LongPointer) x.shapeInfoDataBuffer().addressPointer(), - AtomicAllocator.getInstance().getPointer(tmpX, context), - (LongPointer) AtomicAllocator.getInstance().getPointer(tmpX.shapeInfoDataBuffer(), context), - descending - ); + null, + (LongPointer) x.shapeInfoDataBuffer().addressPointer(), + AtomicAllocator.getInstance().getPointer(tmpX, context), + (LongPointer) AtomicAllocator.getInstance().getPointer(tmpX.shapeInfoDataBuffer(), context), + descending + ); if (nativeOps.lastErrorCode() != 0) throw new RuntimeException(nativeOps.lastErrorMessage()); @@ -1389,7 +1391,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { @Override public INDArray empty(DataType type) { - long extras = ArrayOptionsHelper.setOptionBit(0L, ArrayType.EMPTY); + long extras = ArrayOptionsHelper.setOptionBit(0L, ArrayType.EMPTY); extras = ArrayOptionsHelper.setOptionBit(extras, type); val shape = Nd4j.getShapeInfoProvider().createShapeInformation(new long[0], new long[0], 1, 'c', extras); return new JCublasNDArray(null, (CudaLongDataBuffer) shape.getFirst(), shape.getSecond()); @@ -1418,16 +1420,16 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { nativeOps.sortTad(extraz, - null, - (LongPointer) x.shapeInfoDataBuffer().addressPointer(), - AtomicAllocator.getInstance().getPointer(x, context), - (LongPointer) AtomicAllocator.getInstance().getPointer(x.shapeInfoDataBuffer(), context), - (IntPointer) dimensionPointer, - dimension.length, - (LongPointer) AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context), - new LongPointerWrapper(AtomicAllocator.getInstance().getPointer(tadBuffers.getSecond(), context)), - descending - ); + null, + (LongPointer) x.shapeInfoDataBuffer().addressPointer(), + AtomicAllocator.getInstance().getPointer(x, context), + (LongPointer) AtomicAllocator.getInstance().getPointer(x.shapeInfoDataBuffer(), context), + (IntPointer) dimensionPointer, + dimension.length, + (LongPointer) AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context), + new LongPointerWrapper(AtomicAllocator.getInstance().getPointer(tadBuffers.getSecond(), context)), + descending + ); if (nativeOps.lastErrorCode() != 0) throw new RuntimeException(nativeOps.lastErrorMessage()); @@ -1445,7 +1447,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { @Override public INDArray create(float[] data, long[] shape, long[] stride, char order, DataType dataType) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, order, dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, order, dataType); } @Override @@ -1455,42 +1457,42 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { @Override public INDArray create(double[] data, long[] shape, long[] stride, DataType dataType, MemoryWorkspace workspace) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType, workspace), shape, stride, Nd4j.order(), dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType, workspace), shape, stride, Nd4j.order(), dataType); } @Override public INDArray create(float[] data, long[] shape, long[] stride, DataType dataType, MemoryWorkspace workspace) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType, workspace), shape, stride, Nd4j.order(), dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType, workspace), shape, stride, Nd4j.order(), dataType); } @Override public INDArray create(long[] data, long[] shape, long[] stride, DataType dataType, MemoryWorkspace workspace) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); } @Override public INDArray create(int[] data, long[] shape, long[] stride, DataType dataType, MemoryWorkspace workspace) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); } @Override public INDArray create(short[] data, long[] shape, long[] stride, DataType dataType, MemoryWorkspace workspace) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); } @Override public INDArray create(byte[] data, long[] shape, long[] stride, DataType dataType, MemoryWorkspace workspace) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); } @Override public INDArray create(boolean[] data, long[] shape, long[] stride, DataType dataType, MemoryWorkspace workspace) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType), shape, stride, Nd4j.order(), dataType); } @Override public INDArray create(double[] data, long[] shape, long[] stride, char order, DataType dataType, MemoryWorkspace workspace) { - return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType, workspace), shape, stride, order, dataType); + return new JCublasNDArray(Nd4j.createTypedBuffer(data, dataType, workspace), shape, stride, order, dataType); } @Override @@ -1510,7 +1512,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { @Override public INDArray create(long rows, long columns, long[] stride, long offset) { - return create(new long[] {rows, columns}, stride, offset, Nd4j.order()); + return create(new long[]{rows, columns}, stride, offset, Nd4j.order()); } @Override @@ -1605,7 +1607,7 @@ public class JCublasNDArrayFactory extends BaseNativeNDArrayFactory { @Override public INDArray create(DataType dataType, long[] shape, long[] paddings, long[] paddingOffsets, char ordering, - MemoryWorkspace workspace) { + MemoryWorkspace workspace) { return new JCublasNDArray(dataType, shape, paddings, paddingOffsets, ordering, workspace); } } diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasWrapper.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasWrapper.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasWrapper.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/JCublasWrapper.java diff --git a/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/CudaBlas.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/CudaBlas.java new file mode 100644 index 000000000..37c7d5d86 --- /dev/null +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/CudaBlas.java @@ -0,0 +1,145 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.nd4j.linalg.jcublas.blas; + +import org.bytedeco.cuda.global.cublas; +import org.nd4j.nativeblas.Nd4jBlas; + +/** + * Implementation of Nd4jBlas for cuBLAS + * + * @author saudet + */ +public class CudaBlas extends Nd4jBlas { + + static int convertStatus(int status) { + switch (status) { + case 0: + return cublas.CUBLAS_STATUS_SUCCESS; + case 1: + return cublas.CUBLAS_STATUS_NOT_INITIALIZED; + case 3: + return cublas.CUBLAS_STATUS_ALLOC_FAILED; + case 7: + return cublas.CUBLAS_STATUS_INVALID_VALUE; + case 8: + return cublas.CUBLAS_STATUS_ARCH_MISMATCH; + case 11: + return cublas.CUBLAS_STATUS_MAPPING_ERROR; + case 13: + return cublas.CUBLAS_STATUS_EXECUTION_FAILED; + case 14: + return cublas.CUBLAS_STATUS_INTERNAL_ERROR; + case 15: + return cublas.CUBLAS_STATUS_NOT_SUPPORTED; + case 16: + return cublas.CUBLAS_STATUS_LICENSE_ERROR; + default: + return cublas.CUBLAS_STATUS_SUCCESS; + } + } + + static int convertUplo(int fillMode) { + switch (fillMode) { + case 0: + return cublas.CUBLAS_FILL_MODE_LOWER; + case 1: + return cublas.CUBLAS_FILL_MODE_UPPER; + default: + return cublas.CUBLAS_FILL_MODE_LOWER; + } + } + + static int convertDiag(int diag) { + switch (diag) { + case 0: + return cublas.CUBLAS_DIAG_NON_UNIT; + case 1: + return cublas.CUBLAS_DIAG_UNIT; + default: + return cublas.CUBLAS_DIAG_NON_UNIT; + } + } + + static int convertTranspose(int op) { + switch (op) { + case 78: + return cublas.CUBLAS_OP_N; + case 84: + return cublas.CUBLAS_OP_T; + case 67: + return cublas.CUBLAS_OP_C; + default: + return cublas.CUBLAS_OP_N; + } + } + + static int convertPointerMode(int pointerMode) { + switch (pointerMode) { + case 0: + return cublas.CUBLAS_POINTER_MODE_HOST; + case 1: + return cublas.CUBLAS_POINTER_MODE_DEVICE; + default: + return cublas.CUBLAS_POINTER_MODE_HOST; + } + } + + static int convertSideMode(int sideMode) { + switch (sideMode) { + case 0: + return cublas.CUBLAS_SIDE_LEFT; + case 1: + return cublas.CUBLAS_SIDE_RIGHT; + default: + return cublas.CUBLAS_SIDE_LEFT; + } + } + + @Override + public void setMaxThreads(int num) { + // no-op + } + + @Override + public int getMaxThreads() { + // 0 - cuBLAS + return 0; + } + + /** + * Returns the BLAS library vendor id + * + * 1 - CUBLAS + * + * @return the BLAS library vendor id + */ + @Override + public int getBlasVendorId() { + return 1; + } + + @Override + public boolean logOpenMPBlasThreads() { + return false; + } +} diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLapack.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLapack.java similarity index 89% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLapack.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLapack.java index 52dea6dea..912d2c388 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLapack.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLapack.java @@ -1,27 +1,31 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas.blas; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.bytedeco.cuda.cudart.CUstream_st; +import org.bytedeco.cuda.cusolver.cusolverDnContext; + import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.FloatPointer; import org.bytedeco.javacpp.IntPointer; @@ -45,10 +49,9 @@ import org.nd4j.linalg.jcublas.context.CudaContext; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cusolver.*; +import org.bytedeco.cuda.global.cublas; +import org.bytedeco.cuda.global.cusolver; -import static org.bytedeco.cuda.global.cublas.*; import static org.bytedeco.cuda.global.cusolver.*; /** @@ -85,7 +88,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - int result = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + int result = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new BlasException("solverSetStream failed"); @@ -100,7 +103,7 @@ public class JcublasLapack extends BaseLapack { (IntPointer) worksizeBuffer.addressPointer() // we intentionally use host pointer here ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSgetrf_bufferSize failed", stat); } @@ -117,7 +120,7 @@ public class JcublasLapack extends BaseLapack { // we do sync to make sure getrf is finished //ctx.syncOldStream(); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSgetrf failed", stat); } } @@ -152,7 +155,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - int result = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + int result = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new BlasException("solverSetStream failed"); @@ -167,7 +170,7 @@ public class JcublasLapack extends BaseLapack { (IntPointer) worksizeBuffer.addressPointer() // we intentionally use host pointer here ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnDgetrf_bufferSize failed", stat); } int worksize = worksizeBuffer.getInt(0); @@ -181,7 +184,7 @@ public class JcublasLapack extends BaseLapack { new CudaPointer(allocator.getPointer(IPIV, ctx)).asIntPointer(), new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer()); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSgetrf failed", stat); } } @@ -224,7 +227,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - int result = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + int result = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new IllegalStateException("solverSetStream failed"); @@ -242,7 +245,7 @@ public class JcublasLapack extends BaseLapack { ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSgeqrf_bufferSize failed", stat); } int worksize = worksizeBuffer.getInt(0); @@ -257,7 +260,7 @@ public class JcublasLapack extends BaseLapack { worksize, new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer() ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSgeqrf failed", stat); } @@ -295,7 +298,7 @@ public class JcublasLapack extends BaseLapack { worksize, new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer() ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSorgqr failed", stat); } } @@ -340,7 +343,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - int result = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + int result = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new BlasException("solverSetStream failed"); @@ -357,7 +360,7 @@ public class JcublasLapack extends BaseLapack { (IntPointer) worksizeBuffer.addressPointer() // we intentionally use host pointer here ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnDgeqrf_bufferSize failed", stat); } int worksize = worksizeBuffer.getInt(0); @@ -372,7 +375,7 @@ public class JcublasLapack extends BaseLapack { worksize, new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer() ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnDgeqrf failed", stat); } @@ -409,7 +412,7 @@ public class JcublasLapack extends BaseLapack { worksize, new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer() ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnDorgqr failed", stat); } } @@ -431,7 +434,7 @@ public class JcublasLapack extends BaseLapack { public void spotrf(byte _uplo, int N, INDArray A, INDArray INFO) { INDArray a = A; - int uplo = _uplo == 'L' ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER; + int uplo = _uplo == 'L' ? cublas.CUBLAS_FILL_MODE_LOWER : cublas.CUBLAS_FILL_MODE_UPPER; if (A.dataType() != DataType.FLOAT) log.warn("FLOAT potrf called for " + A.dataType()); @@ -451,7 +454,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - int result = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + int result = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new BlasException("solverSetStream failed"); @@ -467,7 +470,7 @@ public class JcublasLapack extends BaseLapack { (IntPointer) worksizeBuffer.addressPointer() // we intentionally use host pointer here ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSpotrf_bufferSize failed", stat); } @@ -483,7 +486,7 @@ public class JcublasLapack extends BaseLapack { new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer() ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSpotrf failed", stat); } } @@ -493,7 +496,7 @@ public class JcublasLapack extends BaseLapack { if (a != A) A.assign(a); - if (uplo == CUBLAS_FILL_MODE_UPPER ) { + if (uplo == cublas.CUBLAS_FILL_MODE_UPPER ) { A.assign(A.transpose()); INDArrayIndex ix[] = new INDArrayIndex[2]; for (int i = 1; i < Math.min(A.rows(), A.columns()); i++) { @@ -517,7 +520,7 @@ public class JcublasLapack extends BaseLapack { public void dpotrf(byte _uplo, int N, INDArray A, INDArray INFO) { INDArray a = A; - int uplo = _uplo == 'L' ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER; + int uplo = _uplo == 'L' ? cublas.CUBLAS_FILL_MODE_LOWER : cublas.CUBLAS_FILL_MODE_UPPER; if (A.dataType() != DataType.DOUBLE) log.warn("DOUBLE potrf called for " + A.dataType()); @@ -537,7 +540,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - int result = cusolverDnSetStream(solverDn, new CUstream_st(ctx.getCublasStream())); + int result = cusolver.cusolverDnSetStream(solverDn, new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new BlasException("solverSetStream failed"); @@ -553,7 +556,7 @@ public class JcublasLapack extends BaseLapack { (IntPointer) worksizeBuffer.addressPointer() // we intentionally use host pointer here ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnDpotrf_bufferSize failed", stat); } @@ -569,7 +572,7 @@ public class JcublasLapack extends BaseLapack { new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer() ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnDpotrf failed", stat); } } @@ -579,7 +582,7 @@ public class JcublasLapack extends BaseLapack { if (a != A) A.assign(a); - if (uplo == CUBLAS_FILL_MODE_UPPER ) { + if (uplo == cublas.CUBLAS_FILL_MODE_UPPER ) { A.assign(A.transpose()); INDArrayIndex ix[] = new INDArrayIndex[2]; for (int i = 1; i < Math.min(A.rows(), A.columns()); i++) { @@ -670,7 +673,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - int result = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + int result = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new BlasException("solverSetStream failed"); @@ -683,7 +686,7 @@ public class JcublasLapack extends BaseLapack { int stat = cusolverDnSgesvd_bufferSize(solverDn, M, N, (IntPointer) worksizeBuffer.addressPointer() // we intentionally use host pointer here ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSgesvd_bufferSize failed", stat); } int worksize = worksizeBuffer.getInt(0); @@ -699,7 +702,7 @@ public class JcublasLapack extends BaseLapack { new CudaPointer(workspace).asFloatPointer(), worksize, new CudaPointer(allocator.getPointer(rwork, ctx)).asFloatPointer(), new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer()); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSgesvd failed", stat); } } @@ -780,7 +783,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - int result = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + int result = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new BlasException("solverSetStream failed"); @@ -794,7 +797,7 @@ public class JcublasLapack extends BaseLapack { int stat = cusolverDnSgesvd_bufferSize(solverDn, M, N, (IntPointer) worksizeBuffer.addressPointer() // we intentionally use host pointer here ); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnSgesvd_bufferSize failed", stat); } int worksize = worksizeBuffer.getInt(0); @@ -812,7 +815,7 @@ public class JcublasLapack extends BaseLapack { new CudaPointer(allocator.getPointer(rwork, ctx)).asDoublePointer(), new CudaPointer(allocator.getPointer(INFO, ctx)).asIntPointer()); - if (stat != CUSOLVER_STATUS_SUCCESS) { + if (stat != cusolver.CUSOLVER_STATUS_SUCCESS) { throw new BlasException("cusolverDnDgesvd failed" + stat); } } @@ -844,8 +847,8 @@ public class JcublasLapack extends BaseLapack { int status = -1; - int jobz = _jobz == 'V' ? CUSOLVER_EIG_MODE_VECTOR : CUSOLVER_EIG_MODE_NOVECTOR; - int uplo = _uplo == 'L' ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER; + int jobz = _jobz == 'V' ? cusolver.CUSOLVER_EIG_MODE_VECTOR : cusolver.CUSOLVER_EIG_MODE_NOVECTOR; + int uplo = _uplo == 'L' ? cublas.CUBLAS_FILL_MODE_LOWER : cublas.CUBLAS_FILL_MODE_UPPER; INDArray a = A; @@ -869,7 +872,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - status = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + status = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (status == 0) { // transfer the INDArray into GPU memory CublasPointer xAPointer = new CublasPointer(a, ctx); @@ -885,7 +888,7 @@ public class JcublasLapack extends BaseLapack { (FloatPointer) xRPointer.getDevicePointer(), (IntPointer) worksizeBuffer.addressPointer()); - if (status == CUSOLVER_STATUS_SUCCESS) { + if (status == cusolver.CUSOLVER_STATUS_SUCCESS) { int worksize = worksizeBuffer.getInt(0); // allocate memory for the workspace, the non-converging row buffer and a return code @@ -921,8 +924,8 @@ public class JcublasLapack extends BaseLapack { public int dsyev(char _jobz, char _uplo, int N, INDArray A, INDArray R) { int status = -1; - int jobz = _jobz == 'V' ? CUSOLVER_EIG_MODE_VECTOR : CUSOLVER_EIG_MODE_NOVECTOR; - int uplo = _uplo == 'L' ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER; + int jobz = _jobz == 'V' ? cusolver.CUSOLVER_EIG_MODE_VECTOR : cusolver.CUSOLVER_EIG_MODE_NOVECTOR; + int uplo = _uplo == 'L' ? cublas.CUBLAS_FILL_MODE_LOWER : cublas.CUBLAS_FILL_MODE_UPPER; INDArray a = A; @@ -947,7 +950,7 @@ public class JcublasLapack extends BaseLapack { // synchronized on the solver synchronized (handle) { - status = cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); + status = cusolver.cusolverDnSetStream(new cusolverDnContext(handle), new CUstream_st(ctx.getCublasStream())); if (status == 0) { // transfer the INDArray into GPU memory CublasPointer xAPointer = new CublasPointer(a, ctx); @@ -963,7 +966,7 @@ public class JcublasLapack extends BaseLapack { (DoublePointer) xRPointer.getDevicePointer(), (IntPointer) worksizeBuffer.addressPointer()); - if (status == CUSOLVER_STATUS_SUCCESS) { + if (status == cusolver.CUSOLVER_STATUS_SUCCESS) { int worksize = worksizeBuffer.getInt(0); // allocate memory for the workspace, the non-converging row buffer and a return code diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel1.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel1.java similarity index 88% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel1.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel1.java index e1be2f502..e20a9f1d4 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel1.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel1.java @@ -1,27 +1,35 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas.blas; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.bytedeco.javacpp.*; +import org.bytedeco.cuda.cublas.cublasContext; +import org.bytedeco.cuda.cudart.CUstream_st; +import org.bytedeco.cuda.global.cublas; +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.javacpp.FloatPointer; +import org.bytedeco.javacpp.IntPointer; import org.nd4j.common.base.Preconditions; import org.nd4j.jita.allocator.Allocator; import org.nd4j.jita.allocator.impl.AtomicAllocator; @@ -42,22 +50,18 @@ import org.nd4j.linalg.jcublas.ops.executioner.CudaExecutioner; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; import org.nd4j.nativeblas.Nd4jBlas; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cublas.*; import static org.bytedeco.cuda.global.cublas.*; /** * @author Adam Gibson */ +@Slf4j public class JcublasLevel1 extends BaseLevel1 { private Allocator allocator = AtomicAllocator.getInstance(); private Nd4jBlas nd4jBlas = (Nd4jBlas) Nd4j.factory().blas(); private NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); - private static Logger logger = LoggerFactory.getLogger(JcublasLevel1.class); @Override protected float sdsdot(long N, float alpha, INDArray X, int incX, INDArray Y, int incY) { @@ -106,7 +110,7 @@ public class JcublasLevel1 extends BaseLevel1 { val cctx = new cublasContext(handle); synchronized (handle) { - long result = cublasSetStream_v2(cctx, new CUstream_st(ctx.getCublasStream())); + long result = cublas.cublasSetStream_v2(cctx, new CUstream_st(ctx.getCublasStream())); if (result != 0) throw new IllegalStateException("cublasSetStream failed"); @@ -149,7 +153,7 @@ public class JcublasLevel1 extends BaseLevel1 { val handle = ctx.getCublasHandle(); synchronized (handle) { val cctx = new cublasContext(handle); - cublasSetStream_v2(cctx, new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(cctx, new CUstream_st(ctx.getCublasStream())); val resultPointer = new DoublePointer(0.0); cublasDdot_v2(cctx, (int) N, (DoublePointer) xCPointer.getDevicePointer(), incX, @@ -181,7 +185,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); FloatPointer resultPointer = new FloatPointer(0.0f); cublasSnrm2_v2(new cublasContext(handle), (int) N, (FloatPointer) cAPointer.getDevicePointer(), incX, @@ -239,7 +243,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); DoublePointer resultPointer = new DoublePointer(0.0f); cublasDnrm2_v2(new cublasContext(handle), (int) N, (DoublePointer) cAPointer.getDevicePointer(), incX, @@ -280,7 +284,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); IntPointer resultPointer = new IntPointer(new int[] {0}); cublasIsamax_v2(new cublasContext(handle), (int) N, (FloatPointer) xCPointer.getDevicePointer(), incX, @@ -310,7 +314,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); IntPointer resultPointer = new IntPointer(new int[] {0}); cublasIdamax_v2(new cublasContext(handle), (int) N, (DoublePointer) xCPointer.getDevicePointer(), incX, @@ -341,7 +345,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasSswap_v2(new cublasContext(handle), (int) N, (FloatPointer) xCPointer.getDevicePointer(), incX, (FloatPointer) yCPointer.getDevicePointer(), incY); @@ -365,7 +369,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasScopy_v2(new cublasContext(handle), (int) N, (FloatPointer) xCPointer.getDevicePointer(), incX, (FloatPointer) yCPointer.getDevicePointer(), incY); @@ -428,7 +432,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDswap_v2(new cublasContext(handle), (int) N, (DoublePointer) xCPointer.getDevicePointer(), incX, (DoublePointer) yCPointer.getDevicePointer(), incY); @@ -450,7 +454,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDcopy_v2(new cublasContext(handle), (int) N, (DoublePointer) xCPointer.getDevicePointer(), incX, (DoublePointer) yCPointer.getDevicePointer(), incY); @@ -544,7 +548,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasSscal_v2(new cublasContext(handle),(int) N, new FloatPointer(alpha), (FloatPointer) xCPointer.getDevicePointer(), incX); @@ -567,7 +571,7 @@ public class JcublasLevel1 extends BaseLevel1 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDscal_v2(new cublasContext(handle), (int) N, new DoublePointer(alpha), (DoublePointer) xCPointer.getDevicePointer(), incX); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel2.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel2.java similarity index 88% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel2.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel2.java index 8ba0c77b5..ef6a5a567 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel2.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel2.java @@ -1,25 +1,31 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas.blas; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; +import org.bytedeco.cuda.cublas.cublasContext; +import org.bytedeco.cuda.cudart.CUstream_st; +import org.bytedeco.cuda.global.cublas; import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.FloatPointer; import org.nd4j.jita.allocator.Allocator; @@ -34,11 +40,6 @@ import org.nd4j.linalg.jcublas.context.CudaContext; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; import org.nd4j.nativeblas.Nd4jBlas; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cublas.*; import static org.bytedeco.cuda.global.cublas.*; import static org.nd4j.linalg.jcublas.blas.CudaBlas.convertTranspose; @@ -46,11 +47,11 @@ import static org.nd4j.linalg.jcublas.blas.CudaBlas.convertTranspose; /** * @author Adam Gibson */ +@Slf4j public class JcublasLevel2 extends BaseLevel2 { private Allocator allocator = AtomicAllocator.getInstance(); private Nd4jBlas nd4jBlas = (Nd4jBlas) Nd4j.factory().blas(); private NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); - private static Logger logger = LoggerFactory.getLogger(JcublasLevel2.class); @Override protected void sgemv(char order, char TransA, int M, int N, float alpha, INDArray A, int lda, INDArray X, int incX, @@ -66,7 +67,7 @@ public class JcublasLevel2 extends BaseLevel2 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasSgemv_v2(new cublasContext(handle), convertTranspose(TransA), M, N, new FloatPointer(alpha), (FloatPointer) cAPointer.getDevicePointer(), lda, @@ -138,7 +139,7 @@ public class JcublasLevel2 extends BaseLevel2 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDgemv_v2(new cublasContext(handle), convertTranspose(TransA), M, N, new DoublePointer(alpha), (DoublePointer) cAPointer.getDevicePointer(), lda, diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel3.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel3.java similarity index 79% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel3.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel3.java index 8299dbb6a..69338ed7b 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel3.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/blas/JcublasLevel3.java @@ -1,28 +1,35 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas.blas; +import lombok.extern.log4j.Log4j2; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.bytedeco.cuda.cublas.cublasContext; +import org.bytedeco.cuda.cudart.CUstream_st; +import org.bytedeco.cuda.cudart.__half; +import org.bytedeco.cuda.global.cublas; +import org.bytedeco.cuda.global.cudart; import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.FloatPointer; import org.bytedeco.javacpp.ShortPointer; @@ -41,14 +48,25 @@ import org.nd4j.linalg.jcublas.context.CudaContext; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; import org.nd4j.nativeblas.Nd4jBlas; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cublas.*; -import static org.bytedeco.cuda.global.cudart.*; -import static org.bytedeco.cuda.global.cublas.*; -import static org.nd4j.linalg.jcublas.blas.CudaBlas.*; +import static org.bytedeco.cuda.global.cublas.cublasDgemm_v2; +import static org.bytedeco.cuda.global.cublas.cublasDsymm_v2; +import static org.bytedeco.cuda.global.cublas.cublasDsyr2k_v2; +import static org.bytedeco.cuda.global.cublas.cublasDsyrk_v2; +import static org.bytedeco.cuda.global.cublas.cublasDtrmm_v2; +import static org.bytedeco.cuda.global.cublas.cublasDtrsm_v2; +import static org.bytedeco.cuda.global.cublas.cublasHgemm; +import static org.bytedeco.cuda.global.cublas.cublasSetStream_v2; +import static org.bytedeco.cuda.global.cublas.cublasSgemmEx; +import static org.bytedeco.cuda.global.cublas.cublasSgemm_v2; +import static org.bytedeco.cuda.global.cublas.cublasSsymm_v2; +import static org.bytedeco.cuda.global.cublas.cublasSsyrk_v2; +import static org.bytedeco.cuda.global.cublas.cublasStrsm_v2; +import static org.bytedeco.cuda.global.cudart.CUDA_VERSION; +import static org.nd4j.linalg.jcublas.blas.CudaBlas.convertDiag; +import static org.nd4j.linalg.jcublas.blas.CudaBlas.convertSideMode; +import static org.nd4j.linalg.jcublas.blas.CudaBlas.convertTranspose; +import static org.nd4j.linalg.jcublas.blas.CudaBlas.convertUplo; /** * Level 3 implementation of matrix matrix operations @@ -60,7 +78,6 @@ public class JcublasLevel3 extends BaseLevel3 { private Allocator allocator = AtomicAllocator.getInstance(); private Nd4jBlas nd4jBlas = (Nd4jBlas) Nd4j.factory().blas(); private NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); - private static Logger logger = LoggerFactory.getLogger(JcublasLevel3.class); @Override protected void hgemm(char Order, char TransA, char TransB, int M, int N, int K, float alpha, INDArray A, int lda, @@ -78,18 +95,18 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); int arch = CudaEnvironment.getInstance().getCurrentDeviceArchitecture(); - if ((CUDA_VERSION >= 8000 && (arch == 53 || arch == 60 || arch >= 70)) || (CUDA_VERSION >= 8000 && CUDA_VERSION < 9020)) { + if ((cudart.CUDA_VERSION >= 8000 && (arch == 53 || arch == 60 || arch >= 70)) || (cudart.CUDA_VERSION >= 8000 && cudart.CUDA_VERSION < 9020)) { // on these selected archs we run with cublasHgemm __half alphaHalf = new __half(); __half betaHalf = new __half(); new ShortPointer(alphaHalf).put((short) HalfIndexer.fromFloat(alpha)); new ShortPointer(betaHalf).put((short) HalfIndexer.fromFloat(beta)); - cublasHgemm(new cublasContext(handle), convertTranspose(TransA), convertTranspose(TransB), M, N, K, + cublas.cublasHgemm(new cublasContext(handle), convertTranspose(TransA), convertTranspose(TransB), M, N, K, alphaHalf, new __half(cAPointer.getDevicePointer()), lda, new __half(cBPointer.getDevicePointer()), ldb, betaHalf, new __half(cCPointer.getDevicePointer()), ldc); @@ -137,7 +154,7 @@ public class JcublasLevel3 extends BaseLevel3 { val handle = ctx.getCublasHandle(); synchronized (handle) { //log.info("Handle: {}; Stream: {}", handle.address(), ctx.getCublasStream().address()); - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasSgemm_v2(new cublasContext(handle), convertTranspose(TransA), convertTranspose(TransB), M, N, K, new FloatPointer(alpha), (FloatPointer) cAPointer.getDevicePointer(), lda, @@ -166,7 +183,7 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasSsymm_v2(new cublasContext(handle), convertSideMode(Side), convertUplo(Uplo), M, N, new FloatPointer(alpha), (FloatPointer) aPointer.getDevicePointer(), lda, @@ -191,7 +208,7 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasSsyrk_v2(new cublasContext(handle), convertUplo(Uplo), convertTranspose(Trans), N, K, new FloatPointer(alpha), (FloatPointer) aPointer.getDevicePointer(), lda, @@ -228,7 +245,7 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasStrsm_v2(new cublasContext(handle), convertSideMode(Side), convertUplo(Uplo), convertTranspose(TransA), convertDiag(Diag), M, N, new FloatPointer(alpha), @@ -258,7 +275,7 @@ public class JcublasLevel3 extends BaseLevel3 { val handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDgemm_v2(new cublasContext(handle), convertTranspose(TransA), convertTranspose(TransB), M, N, K, new DoublePointer(alpha), (DoublePointer) cAPointer.getDevicePointer(), lda, @@ -285,7 +302,7 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDsymm_v2(new cublasContext(handle), convertSideMode(Side), convertUplo(Uplo), M, N, new DoublePointer(alpha), (DoublePointer) aPointer.getDevicePointer(), lda, @@ -310,7 +327,7 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDsyrk_v2(new cublasContext(handle), convertUplo(Uplo), Trans, N, K, new DoublePointer(alpha), (DoublePointer) aPointer.getDevicePointer(), lda, new DoublePointer(beta), @@ -335,7 +352,7 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDsyr2k_v2(new cublasContext(handle), convertUplo(Uplo), Trans, N, K, new DoublePointer(alpha), (DoublePointer) aPointer.getDevicePointer(), lda, @@ -360,7 +377,7 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDtrmm_v2(new cublasContext(handle), convertSideMode(Side), convertUplo(Uplo), convertTranspose(TransA), convertDiag(Diag), M, N, new DoublePointer(alpha), @@ -386,7 +403,7 @@ public class JcublasLevel3 extends BaseLevel3 { cublasHandle_t handle = ctx.getCublasHandle(); synchronized (handle) { - cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); + cublas.cublasSetStream_v2(new cublasContext(handle), new CUstream_st(ctx.getCublasStream())); cublasDtrsm_v2(new cublasContext(handle), convertSideMode(Side), convertUplo(Uplo), convertTranspose(TransA), convertDiag(Diag), M, N, new DoublePointer(alpha), diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/AddressRetriever.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/AddressRetriever.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/AddressRetriever.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/AddressRetriever.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java index b81a20efb..3f97c6818 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/BaseCudaDataBuffer.java @@ -1,31 +1,35 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas.buffer; import lombok.Getter; import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.indexer.*; import org.nd4j.common.base.Preconditions; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.jita.allocator.enums.CudaConstants; import org.nd4j.jita.allocator.impl.AllocationPoint; import org.nd4j.jita.allocator.impl.AllocationShape; @@ -38,6 +42,7 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.buffer.util.DataTypeUtil; import org.nd4j.linalg.api.memory.Deallocatable; import org.nd4j.linalg.api.memory.Deallocator; +import org.nd4j.linalg.api.memory.MemcpyDirection; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.memory.enums.MemoryKind; import org.nd4j.linalg.api.memory.enums.MirroringPolicy; @@ -45,16 +50,15 @@ import org.nd4j.linalg.api.memory.pointers.PagedPointer; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.performance.PerformanceTracker; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.api.memory.MemcpyDirection; -import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.util.LongUtils; import org.nd4j.nativeblas.NativeOpsHolder; import org.nd4j.nativeblas.OpaqueDataBuffer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.*; -import java.nio.*; +import java.nio.ByteBuffer; +import java.nio.DoubleBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; import java.util.Collection; /** @@ -72,6 +76,7 @@ import java.util.Collection; * @author Adam Gibson * @author raver119@gmail.com */ +@Slf4j public abstract class BaseCudaDataBuffer extends BaseDataBuffer implements JCudaBuffer, Deallocatable { protected OpaqueDataBuffer ptrDataBuffer; @@ -80,7 +85,7 @@ public abstract class BaseCudaDataBuffer extends BaseDataBuffer implements JCuda private static AtomicAllocator allocator = AtomicAllocator.getInstance(); - private static Logger log = LoggerFactory.getLogger(BaseCudaDataBuffer.class); + protected DataType globalType = DataTypeUtil.getDtypeFromContext(); @@ -1327,14 +1332,14 @@ public abstract class BaseCudaDataBuffer extends BaseDataBuffer implements JCuda super.write(dos); } - private void writeObject(java.io.ObjectOutputStream stream) throws IOException { + private void writeObject(ObjectOutputStream stream) throws IOException { lazyAllocateHostPointer(); allocator.synchronizeHostData(this); stream.defaultWriteObject(); write(stream); } - private void readObject(java.io.ObjectInputStream stream) throws IOException, ClassNotFoundException { + private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { doReadObject(stream); } @@ -1384,12 +1389,12 @@ public abstract class BaseCudaDataBuffer extends BaseDataBuffer implements JCuda case LONG: pointer = new LongPointer(length()); setIndexer(LongIndexer.create((LongPointer) pointer)); - type = DataType.LONG; + type = LONG; break; case INT: pointer = new IntPointer(length()); setIndexer(IntIndexer.create((IntPointer) pointer)); - type = DataType.INT; + type = INT; break; case DOUBLE: pointer = new DoublePointer(length()); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBfloat16DataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBfloat16DataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBfloat16DataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBfloat16DataBuffer.java index 7eddde576..e5a4c20f4 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBfloat16DataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBfloat16DataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBoolDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBoolDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBoolDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBoolDataBuffer.java index a1b6e7f0c..a02e15e94 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBoolDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaBoolDataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaByteDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaByteDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaByteDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaByteDataBuffer.java index f6d5ac379..bd9f9d3c7 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaByteDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaByteDataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaDoubleDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaDoubleDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaDoubleDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaDoubleDataBuffer.java index 6a9ef4b49..626caee96 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaDoubleDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaDoubleDataBuffer.java @@ -22,12 +22,12 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.jita.allocator.impl.AllocationShape; import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaFloatDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaFloatDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaFloatDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaFloatDataBuffer.java index 720f7d88d..e6bbd123f 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaFloatDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaFloatDataBuffer.java @@ -20,13 +20,13 @@ package org.nd4j.linalg.jcublas.buffer; +import lombok.extern.slf4j.Slf4j; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; -import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaHalfDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaHalfDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaHalfDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaHalfDataBuffer.java index 45521f138..d0a22b70a 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaHalfDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaHalfDataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaIntDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaIntDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaIntDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaIntDataBuffer.java index 4408d6de8..508caa881 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaIntDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaIntDataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaLongDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaLongDataBuffer.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaLongDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaLongDataBuffer.java index d433e21c8..58374d8e0 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaLongDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaLongDataBuffer.java @@ -32,7 +32,6 @@ import org.nd4j.jita.allocator.pointers.CudaPointer; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.nativeblas.NativeOpsHolder; import org.nd4j.nativeblas.OpaqueDataBuffer; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaShortDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaShortDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaShortDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaShortDataBuffer.java index 3bba37f49..aa390d248 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaShortDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaShortDataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUByteDataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUByteDataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUByteDataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUByteDataBuffer.java index a5b6be52a..979a06802 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUByteDataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUByteDataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt16DataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt16DataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt16DataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt16DataBuffer.java index df877815d..eb2342fbc 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt16DataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt16DataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt32DataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt32DataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt32DataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt32DataBuffer.java index 092bd37fd..17b604d7d 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt32DataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt32DataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt64DataBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt64DataBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt64DataBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt64DataBuffer.java index 454109b2f..1e2710f2e 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt64DataBuffer.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUInt64DataBuffer.java @@ -22,10 +22,10 @@ package org.nd4j.linalg.jcublas.buffer; import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.indexer.Indexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUtf8Buffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUtf8Buffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUtf8Buffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/CudaUtf8Buffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/DevicePointerInfo.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/DevicePointerInfo.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/DevicePointerInfo.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/DevicePointerInfo.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/JCudaBuffer.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/JCudaBuffer.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/JCudaBuffer.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/JCudaBuffer.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java similarity index 96% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java index 9a0f82d81..09a404b15 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java @@ -1,21 +1,22 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas.buffer.factory; @@ -25,14 +26,17 @@ import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.FloatPointer; import org.bytedeco.javacpp.IntPointer; import org.bytedeco.javacpp.Pointer; -import org.bytedeco.javacpp.indexer.*; +import org.bytedeco.javacpp.indexer.DoubleIndexer; +import org.bytedeco.javacpp.indexer.FloatIndexer; +import org.bytedeco.javacpp.indexer.Indexer; +import org.bytedeco.javacpp.indexer.IntIndexer; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.buffer.factory.DataBufferFactory; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.jcublas.buffer.*; -import org.nd4j.common.util.ArrayUtil; import java.nio.ByteBuffer; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/context/ContextHolder.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/context/ContextHolder.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/context/ContextHolder.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/context/ContextHolder.java index 7ecbaceb9..0d6d03ee8 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/context/ContextHolder.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/context/ContextHolder.java @@ -21,9 +21,9 @@ package org.nd4j.linalg.jcublas.context; import lombok.Data; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import org.nd4j.common.io.ClassPathResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; @@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * @author Adam Gibson */ @Data +@Slf4j public class ContextHolder { private Map threadNameToDeviceNumber = new ConcurrentHashMap<>(); @@ -54,7 +55,6 @@ public class ContextHolder { public final static String DEVICES_TO_BAN = "org.nd4j.linalg.jcuda.jcublas.ban_devices"; private static AtomicBoolean deviceSetup = new AtomicBoolean(false); private boolean confCalled = false; - private static Logger log = LoggerFactory.getLogger(ContextHolder.class); private AtomicBoolean shutdown = new AtomicBoolean(false); // holder for memory strategies override @@ -185,7 +185,8 @@ public class ContextHolder { */ } catch (Exception e) { - log.warn("Unable to initialize cuda", e); + //log.("Unable to initialize cuda", e); + log.error("Unable to initialize cuda", e); } /* diff --git a/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/context/CudaContext.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/context/CudaContext.java new file mode 100644 index 000000000..7be9e08c1 --- /dev/null +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/context/CudaContext.java @@ -0,0 +1,112 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.nd4j.linalg.jcublas.context; + +import lombok.*; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.PointerPointer; +import org.nd4j.jita.allocator.pointers.cuda.cublasHandle_t; +import org.nd4j.jita.allocator.pointers.cuda.cudaStream_t; +import org.nd4j.jita.allocator.pointers.cuda.cusolverDnHandle_t; +import org.nd4j.linalg.exception.ND4JIllegalStateException; +import org.nd4j.nativeblas.NativeOps; +import org.nd4j.nativeblas.NativeOpsHolder; + +/** + * A higher level class for handling + * the different primitives around the cuda apis + * This being: + * streams (both old and new) as well as + * the cublas handles. + * + * + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class CudaContext { + + // execution stream + private cudaStream_t oldStream; + + // memcpy stream + private cudaStream_t specialStream; + + // exactly what it says + private cublasHandle_t cublasHandle; + private cusolverDnHandle_t solverHandle; + + // temporary buffers, exactly 1 per thread + private Pointer bufferReduction; + private Pointer bufferAllocation; + private Pointer bufferScalar; + + // legacy. to be removed. + private Pointer bufferSpecial; + + @Builder.Default + private int deviceId = -1; + + private transient final static NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); + + @Override + public String toString() { + return "CudaContext{" + + "bufferReduction=" + bufferReduction + + ", bufferScalar=" + bufferScalar + + ", deviceId=" + deviceId + + '}'; + } + + /** + * Synchronizes + * on the old stream + */ + public void syncOldStream() { + if (nativeOps.streamSynchronize(oldStream) == 0) + throw new ND4JIllegalStateException("CUDA stream synchronization failed"); + } + + public void syncSpecialStream() { + if (nativeOps.streamSynchronize(specialStream) == 0) + throw new ND4JIllegalStateException("CUDA special stream synchronization failed"); + } + + public Pointer getCublasStream() { + // FIXME: can we cache this please + val lptr = new PointerPointer(this.getOldStream()); + return lptr.get(0); + } + + public cublasHandle_t getCublasHandle() { + // FIXME: can we cache this please + val lptr = new PointerPointer(cublasHandle); + return new cublasHandle_t(lptr.get(0)); + } + + public cusolverDnHandle_t getSolverHandle() { + // FIXME: can we cache this please + val lptr = new PointerPointer(solverHandle); + return new cusolverDnHandle_t(lptr.get(0)); + } +} diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java similarity index 93% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java index 58b6fcb2b..2cc5077e4 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java @@ -1,34 +1,36 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas.ops.executioner; -import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; -import lombok.var; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.indexer.LongIndexer; import org.nd4j.common.base.Preconditions; +import org.nd4j.common.primitives.AtomicBoolean; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.jita.allocator.pointers.CudaPointer; import org.nd4j.jita.allocator.tad.DeviceTADManager; @@ -64,9 +66,6 @@ import org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer; import org.nd4j.linalg.jcublas.buffer.CudaLongDataBuffer; import org.nd4j.linalg.jcublas.buffer.CudaUtf8Buffer; import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.common.primitives.AtomicBoolean; -import org.nd4j.common.primitives.Pair; -import org.nd4j.common.util.ArrayUtil; import org.nd4j.nativeblas.*; import java.util.*; @@ -74,7 +73,6 @@ import java.util.*; /** * JCuda executioner. - *

* Runs ops directly on the gpu * * If requested Op doesn't exist within GPU context, DefaultOpExecutioner will be used, with arrays/buffers updated after that. @@ -89,7 +87,6 @@ public class CudaExecutioner extends DefaultOpExecutioner { // private static final Allocator allocator = AtomicAllocator.getInstance(); - @Getter protected static TADManager tadManager = new DeviceTADManager(); protected ThreadLocal extraz = new ThreadLocal<>(); protected volatile transient Properties properties; @@ -233,14 +230,14 @@ public class CudaExecutioner extends DefaultOpExecutioner { throw new ND4JIllegalStateException("Op target dimension " + Arrays.toString(dimension) + " contains element that higher then rank of op.X: [" + op.x().rank() + "]"); - val context = AtomicAllocator.getInstance().getDeviceContext(); + CudaContext context = AtomicAllocator.getInstance().getDeviceContext(); if (CudaEnvironment.getInstance().getConfiguration().isDebug()) lastOp.set(op.opName()); - val hostXShapeInfo = op.x() == null ? null : AddressRetriever.retrieveHostPointer(op.x().shapeInfoDataBuffer()); - val hostYShapeInfo = op.y() == null ? null : AddressRetriever.retrieveHostPointer(op.y().shapeInfoDataBuffer()); - val hostZShapeInfo = op.z() == null ? null : AddressRetriever.retrieveHostPointer(op.z().shapeInfoDataBuffer()); + Pointer hostXShapeInfo = op.x() == null ? null : AddressRetriever.retrieveHostPointer(op.x().shapeInfoDataBuffer()); + Pointer hostYShapeInfo = op.y() == null ? null : AddressRetriever.retrieveHostPointer(op.y().shapeInfoDataBuffer()); + Pointer hostZShapeInfo = op.z() == null ? null : AddressRetriever.retrieveHostPointer(op.z().shapeInfoDataBuffer()); Pair tadBuffers = tadManager.getTADOnlyShapeInfo(op.x(), dimension); @@ -290,7 +287,7 @@ public class CudaExecutioner extends DefaultOpExecutioner { } } else { // TAD vs full array code branch - val fakeOffsets = Nd4j.getConstantHandler().getConstantBuffer(new int[] {0, 0}, DataType.LONG); + DataBuffer fakeOffsets = Nd4j.getConstantHandler().getConstantBuffer(new int[] {0, 0}, DataType.LONG); yDevTadOffsets = fakeOffsets == null ? null : AtomicAllocator.getInstance().getPointer(fakeOffsets, context); yDevTadShapeInfo = AtomicAllocator.getInstance().getPointer(op.y().shapeInfoDataBuffer(), context); @@ -558,13 +555,13 @@ public class CudaExecutioner extends DefaultOpExecutioner { if (CudaEnvironment.getInstance().getConfiguration().isDebug()) lastOp.set(op.opName()); - val context = AtomicAllocator.getInstance().getDeviceContext(); + CudaContext context = AtomicAllocator.getInstance().getDeviceContext(); - val hostXShapeInfo = + Pointer hostXShapeInfo = op.x() == null ? null : AddressRetriever.retrieveHostPointer(op.x().shapeInfoDataBuffer()); - val hostYShapeInfo = + Pointer hostYShapeInfo = op.y() == null ? null : AddressRetriever.retrieveHostPointer(op.y().shapeInfoDataBuffer()); - val hostZShapeInfo = + Pointer hostZShapeInfo = op.z() == null ? null : AddressRetriever.retrieveHostPointer(op.z().shapeInfoDataBuffer()); val xShapeInfo = AtomicAllocator.getInstance().getPointer(op.x().shapeInfoDataBuffer(), context); @@ -572,11 +569,11 @@ public class CudaExecutioner extends DefaultOpExecutioner { Pair tadBuffers = tadManager.getTADOnlyShapeInfo(op.x(), dimension); - val hostTadShapeInfo = AddressRetriever.retrieveHostPointer(tadBuffers.getFirst()); - val devTadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); + Pointer hostTadShapeInfo = AddressRetriever.retrieveHostPointer(tadBuffers.getFirst()); + Pointer devTadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); - val offsets = tadBuffers.getSecond(); - val devTadOffsets = offsets == null ? null : AtomicAllocator.getInstance().getPointer(offsets, context); + DataBuffer offsets = tadBuffers.getSecond(); + Pointer devTadOffsets = offsets == null ? null : AtomicAllocator.getInstance().getPointer(offsets, context); PointerPointer xShapeInfoHostPointer = extraz.get().put( AddressRetriever.retrieveHostPointer(op.x().shapeInfoDataBuffer()), context.getOldStream(), @@ -693,17 +690,17 @@ public class CudaExecutioner extends DefaultOpExecutioner { Pointer xShapeInfo = AtomicAllocator.getInstance().getPointer(x.shapeInfoDataBuffer(), context); - val hostXShapeInfo = + Pointer hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); - val hostYShapeInfo = + Pointer hostYShapeInfo = y == null ? null : AddressRetriever.retrieveHostPointer(y.shapeInfoDataBuffer()); - val hostZShapeInfo = + Pointer hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); val tadBuffers = tadManager.getTADOnlyShapeInfo(x, op.getDimension()); - val hostTadShapeInfo = AddressRetriever.retrieveHostPointer(tadBuffers.getFirst()); - val devTadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); + Pointer hostTadShapeInfo = AddressRetriever.retrieveHostPointer(tadBuffers.getFirst()); + Pointer devTadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); val offsets = tadBuffers.getSecond(); val devTadOffsets = AtomicAllocator.getInstance().getPointer(offsets, context); @@ -817,14 +814,14 @@ public class CudaExecutioner extends DefaultOpExecutioner { if (dimension[i] >= x.rank() && dimension[i] != Integer.MAX_VALUE) throw new ND4JIllegalStateException("Op target dimension " + Arrays.toString(dimension) + " contains element that higher then rank of op.X: [" + x.rank() + "]"); - val context = AtomicAllocator.getInstance().getDeviceContext(); + CudaContext context = AtomicAllocator.getInstance().getDeviceContext(); Pointer xShapeInfo = AtomicAllocator.getInstance().getPointer(x.shapeInfoDataBuffer(), context); Pointer extraArgs = op.extraArgs() != null ? AtomicAllocator.getInstance().getPointer(op.extraArgsDataBuff(x.dataType()), context) : null; - val hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); - val hostYShapeInfo = y == null ? null : AddressRetriever.retrieveHostPointer(y.shapeInfoDataBuffer()); - val hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); + Pointer hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); + Pointer hostYShapeInfo = y == null ? null : AddressRetriever.retrieveHostPointer(y.shapeInfoDataBuffer()); + Pointer hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); int fdimension[] = dimension; if (fdimension == null) @@ -837,7 +834,7 @@ public class CudaExecutioner extends DefaultOpExecutioner { DataBuffer offsets = tadBuffers.getSecond(); Pointer devTadOffsets = offsets == null ? null : AtomicAllocator.getInstance().getPointer(offsets, context); - val zShapeInfo = AtomicAllocator.getInstance().getPointer(z.shapeInfoDataBuffer(), context); + Pointer zShapeInfo = AtomicAllocator.getInstance().getPointer(z.shapeInfoDataBuffer(), context); val xb = x == null ? null : ((BaseCudaDataBuffer) x.data()).getOpaqueDataBuffer(); val zb = z == null ? null : ((BaseCudaDataBuffer) z.data()).getOpaqueDataBuffer(); @@ -939,11 +936,11 @@ public class CudaExecutioner extends DefaultOpExecutioner { val tadBuffers = x.isEmpty() ? Pair.makePair(x.data(), null) : tadManager.getTADOnlyShapeInfo(x, dimension); - val hostTadShapeInfo = AddressRetriever.retrieveHostPointer(tadBuffers.getFirst()); - val devTadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); + Pointer hostTadShapeInfo = AddressRetriever.retrieveHostPointer(tadBuffers.getFirst()); + Pointer devTadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); - val offsets = x.isEmpty() ? null : tadBuffers.getSecond(); - val devTadOffsets = offsets == null ? null : AtomicAllocator.getInstance().getPointer(offsets, context); + DataBuffer offsets = x.isEmpty() ? null : tadBuffers.getSecond(); + Pointer devTadOffsets = offsets == null ? null : AtomicAllocator.getInstance().getPointer((DataBuffer) offsets, context); Pointer xShapeInfo = AtomicAllocator.getInstance().getPointer(x.shapeInfoDataBuffer(), context); @@ -984,12 +981,12 @@ public class CudaExecutioner extends DefaultOpExecutioner { + " but has datatype " + z.dataType() + " and shape " + Arrays.toString(z.shape())); } - val eb = op.extraArgsDataBuff(z.dataType() == DataType.BOOL || op.getOpType() == Op.Type.REDUCE_LONG ? x.dataType() : z.dataType()); + DataBuffer eb = op.extraArgsDataBuff(z.dataType() == DataType.BOOL || op.getOpType() == Op.Type.REDUCE_LONG ? x.dataType() : z.dataType()); Pointer extraArgs = op.extraArgs() != null ? AtomicAllocator.getInstance().getPointer(eb, context) : null; - val hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); - val hostYShapeInfo = y == null ? null : AddressRetriever.retrieveHostPointer(y.shapeInfoDataBuffer()); - val hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); + Pointer hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); + Pointer hostYShapeInfo = y == null ? null : AddressRetriever.retrieveHostPointer(y.shapeInfoDataBuffer()); + Pointer hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); val xShapeInfoHostPointer = extraz.get().put( AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()), context.getOldStream(), @@ -999,9 +996,9 @@ public class CudaExecutioner extends DefaultOpExecutioner { val yTadBuffers = y == null ? null : tadManager.getTADOnlyShapeInfo(y, dimension); - val yDevTadShapeInfo = y == null ? null : AtomicAllocator.getInstance().getPointer(yTadBuffers.getFirst(), context); - val yOffsets = y == null ? null : yTadBuffers.getSecond(); - val yDevTadOffsets = yOffsets == null ? null : AtomicAllocator.getInstance().getPointer(yOffsets, context); + Pointer yDevTadShapeInfo = y == null ? null : AtomicAllocator.getInstance().getPointer((DataBuffer) ((Pair) yTadBuffers).getFirst(), context); + DataBuffer yOffsets = y == null ? null : yTadBuffers.getSecond(); + Pointer yDevTadOffsets = yOffsets == null ? null : AtomicAllocator.getInstance().getPointer(yOffsets, context); if (y != null) { xShapeInfoHostPointer.put(12, yDevTadShapeInfo); @@ -1135,31 +1132,31 @@ public class CudaExecutioner extends DefaultOpExecutioner { if (dimension != null && dimension.length > 1) Arrays.sort(dimension); - val context = AtomicAllocator.getInstance().getDeviceContext(); + CudaContext context = AtomicAllocator.getInstance().getDeviceContext(); if (CudaEnvironment.getInstance().getConfiguration().isDebug()) lastOp.set(op.opName()); - val hostXShapeInfo = op.x() == null ? null : AddressRetriever.retrieveHostPointer(op.x().shapeInfoDataBuffer()); - val hostYShapeInfo = op.y() == null ? null : AddressRetriever.retrieveHostPointer(op.y().shapeInfoDataBuffer()); - val hostZShapeInfo = op.z() == null ? null : AddressRetriever.retrieveHostPointer(op.z().shapeInfoDataBuffer()); + Pointer hostXShapeInfo = op.x() == null ? null : AddressRetriever.retrieveHostPointer(op.x().shapeInfoDataBuffer()); + Pointer hostYShapeInfo = op.y() == null ? null : AddressRetriever.retrieveHostPointer(op.y().shapeInfoDataBuffer()); + Pointer hostZShapeInfo = op.z() == null ? null : AddressRetriever.retrieveHostPointer(op.z().shapeInfoDataBuffer()); - val xShapeInfo = AtomicAllocator.getInstance().getPointer(op.x().shapeInfoDataBuffer(), context); - val yShapeInfo = AtomicAllocator.getInstance().getPointer(op.y().shapeInfoDataBuffer(), context); - val zShapeInfo = AtomicAllocator.getInstance().getPointer(op.z().shapeInfoDataBuffer(), context); + Pointer xShapeInfo = AtomicAllocator.getInstance().getPointer(op.x().shapeInfoDataBuffer(), context); + Pointer yShapeInfo = AtomicAllocator.getInstance().getPointer(op.y().shapeInfoDataBuffer(), context); + Pointer zShapeInfo = AtomicAllocator.getInstance().getPointer(op.z().shapeInfoDataBuffer(), context); - val tadBuffers = tadManager.getTADOnlyShapeInfo(op.x(), dimension); + Pair tadBuffers = tadManager.getTADOnlyShapeInfo(op.x(), dimension); - val hostTadShapeInfo = AddressRetriever.retrieveHostPointer(tadBuffers.getFirst()); - val devTadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); + Pointer hostTadShapeInfo = AddressRetriever.retrieveHostPointer(tadBuffers.getFirst()); + Pointer devTadShapeInfo = AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context); - val offsets = tadBuffers.getSecond(); - val devTadOffsets = AtomicAllocator.getInstance().getPointer(offsets, context); + DataBuffer offsets = tadBuffers.getSecond(); + Pointer devTadOffsets = AtomicAllocator.getInstance().getPointer(offsets, context); Pointer devTadShapeInfoZ = null; Pointer devTadOffsetsZ = null; - val tadBuffersZ = tadManager.getTADOnlyShapeInfo(op.z(), dimension); + Pair tadBuffersZ = tadManager.getTADOnlyShapeInfo(op.z(), dimension); devTadShapeInfoZ = AtomicAllocator.getInstance().getPointer(tadBuffersZ.getFirst(), context); devTadOffsetsZ = AtomicAllocator.getInstance().getPointer(tadBuffersZ.getSecond(), context); @@ -1261,11 +1258,11 @@ public class CudaExecutioner extends DefaultOpExecutioner { return null; } - val context = AtomicAllocator.getInstance().getDeviceContext(); + CudaContext context = AtomicAllocator.getInstance().getDeviceContext(); - val hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); - val hostYShapeInfo = op.scalar() == null ? null : AddressRetriever.retrieveHostPointer(op.scalar().shapeInfoDataBuffer()); - val hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); + Pointer hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); + Pointer hostYShapeInfo = op.scalar() == null ? null : AddressRetriever.retrieveHostPointer(op.scalar().shapeInfoDataBuffer()); + Pointer hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); Pointer xShapeInfo = AtomicAllocator.getInstance().getPointer(x.shapeInfoDataBuffer(), context); Pointer extraArgs = op.extraArgs() != null ? AtomicAllocator.getInstance().getPointer(op.extraArgsDataBuff(op.getOpType() == Op.Type.SCALAR_BOOL ? x.dataType() : z.dataType()), context) : null; @@ -1342,8 +1339,8 @@ public class CudaExecutioner extends DefaultOpExecutioner { Pointer retHostShape = null; int dimension[] = null; - val hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); - var hostYShapeInfo = y == null ? null : AddressRetriever.retrieveHostPointer(y.shapeInfoDataBuffer()); + Pointer hostXShapeInfo = x == null ? null : AddressRetriever.retrieveHostPointer(x.shapeInfoDataBuffer()); + Pointer hostYShapeInfo = y == null ? null : AddressRetriever.retrieveHostPointer(y.shapeInfoDataBuffer()); if (z == null) { @@ -1352,8 +1349,8 @@ public class CudaExecutioner extends DefaultOpExecutioner { z = ret; } - var extraArgs = op.extraArgs() != null ? allocator.getPointer(op.extraArgsDataBuff(op.getOpType() == Op.Type.TRANSFORM_BOOL || op.getOpType() == Op.Type.PAIRWISE_BOOL ? x.dataType() : z.dataType()), context) : null; - val hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); + Pointer extraArgs = op.extraArgs() != null ? allocator.getPointer(op.extraArgsDataBuff(op.getOpType() == Op.Type.TRANSFORM_BOOL || op.getOpType() == Op.Type.PAIRWISE_BOOL ? x.dataType() : z.dataType()), context) : null; + Pointer hostZShapeInfo = z == null ? null : AddressRetriever.retrieveHostPointer(z.shapeInfoDataBuffer()); Pointer hostTadShapeInfo = null; Pointer devTadShapeInfo = null; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaGridExecutioner.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaGridExecutioner.java similarity index 97% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaGridExecutioner.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaGridExecutioner.java index 6b5f9b175..2787e7282 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaGridExecutioner.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaGridExecutioner.java @@ -1,21 +1,22 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ package org.nd4j.linalg.jcublas.ops.executioner; @@ -23,14 +24,16 @@ package org.nd4j.linalg.jcublas.ops.executioner; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.bytedeco.javacpp.Pointer; import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ops.impl.summarystats.Variance; import org.nd4j.common.primitives.Pair; -import org.bytedeco.javacpp.*; +import org.nd4j.common.util.ArrayUtil; import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.ops.*; import org.nd4j.linalg.api.ops.aggregates.Aggregate; @@ -42,15 +45,13 @@ import org.nd4j.linalg.api.ops.impl.meta.PostulateMetaOp; import org.nd4j.linalg.api.ops.impl.meta.PredicateMetaOp; import org.nd4j.linalg.api.ops.impl.scalar.ScalarMax; import org.nd4j.linalg.api.ops.impl.scalar.ScalarMin; +import org.nd4j.linalg.api.ops.impl.summarystats.Variance; import org.nd4j.linalg.api.rng.Random; import org.nd4j.linalg.api.shape.Shape; import org.nd4j.linalg.exception.ND4JIllegalStateException; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.context.CudaContext; import org.nd4j.linalg.jcublas.ops.executioner.aggregates.AggregateDescriptor; -import org.nd4j.common.util.ArrayUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; @@ -64,6 +65,7 @@ import java.util.concurrent.atomic.AtomicLong; * @author raver119@gmail.com */ @Deprecated +@Slf4j public class CudaGridExecutioner extends CudaExecutioner implements GridExecutioner { protected enum MetaType { NOT_APPLICABLE, @@ -89,8 +91,6 @@ public class CudaGridExecutioner extends CudaExecutioner implements GridExecutio private List> aggregates = new ArrayList<>(); - private static Logger logger = LoggerFactory.getLogger(CudaGridExecutioner.class); - private AtomicBoolean experimental = new AtomicBoolean(false); public CudaGridExecutioner() { @@ -169,7 +169,7 @@ public class CudaGridExecutioner extends CudaExecutioner implements GridExecutio } protected boolean compareDevicePointers(INDArray array, Op op) { - val context = (CudaContext) AtomicAllocator.getInstance().getDeviceContext(); + CudaContext context = AtomicAllocator.getInstance().getDeviceContext(); val pointer = AtomicAllocator.getInstance().getPointer(array, context); @@ -427,7 +427,7 @@ public class CudaGridExecutioner extends CudaExecutioner implements GridExecutio } else { // Experimental native compilation required for full MIMD support if (experimental.get()) { - logger.info("Experimental hook"); + log.info("Experimental hook"); if (last.getOp() instanceof ScalarOp || last.getOp() instanceof TransformOp) { /* Predicate logic is simple: diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java similarity index 98% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java index 98bd1fb60..c14b9c7eb 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java @@ -23,10 +23,10 @@ package org.nd4j.linalg.jcublas.ops.executioner; import lombok.NonNull; import lombok.val; import org.bytedeco.javacpp.*; +import org.nd4j.common.primitives.Pair; import org.nd4j.jita.allocator.impl.AtomicAllocator; import org.nd4j.jita.allocator.pointers.cuda.cudaStream_t; import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.concurrency.AffinityManager; import org.nd4j.linalg.api.memory.Deallocatable; import org.nd4j.linalg.api.memory.Deallocator; import org.nd4j.linalg.api.ndarray.INDArray; @@ -35,8 +35,6 @@ import org.nd4j.linalg.api.ops.ExecutionMode; import org.nd4j.linalg.api.ops.OpContext; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer; -import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.common.primitives.Pair; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; import org.nd4j.nativeblas.OpaqueContext; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContextDeallocator.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContextDeallocator.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContextDeallocator.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContextDeallocator.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/aggregates/AggregateDescriptor.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/aggregates/AggregateDescriptor.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/aggregates/AggregateDescriptor.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/aggregates/AggregateDescriptor.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/rng/CudaNativeRandom.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/rng/CudaNativeRandom.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/rng/CudaNativeRandom.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/rng/CudaNativeRandom.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/util/CudaArgs.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/util/CudaArgs.java similarity index 99% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/util/CudaArgs.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/util/CudaArgs.java index a520c842e..e3e32d03f 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/util/CudaArgs.java +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/util/CudaArgs.java @@ -21,8 +21,8 @@ package org.nd4j.linalg.jcublas.util; -import org.nd4j.shade.guava.collect.ArrayListMultimap; -import org.nd4j.shade.guava.collect.Multimap; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; import lombok.AllArgsConstructor; import lombok.Data; import org.nd4j.linalg.api.ndarray.INDArray; diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/util/FFTUtils.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/util/FFTUtils.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/util/FFTUtils.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/util/FFTUtils.java diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/util/OpUtil.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/util/OpUtil.java similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/util/OpUtil.java rename to cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/linalg/jcublas/util/OpUtil.java diff --git a/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/nativeblas/cuda/CudaEnvironment.java b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/nativeblas/cuda/CudaEnvironment.java new file mode 100644 index 000000000..1d525a856 --- /dev/null +++ b/cavis-native/cavis-native-jcublas/src/main/java/org/nd4j/nativeblas/cuda/CudaEnvironment.java @@ -0,0 +1,200 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +package org.nd4j.nativeblas.cuda; + +import org.nd4j.linalg.factory.Environment; +import org.nd4j.nativeblas.Nd4jCuda; + +/** + * CUDA backend implementation of {@link Environment} + * + * @author Alex Black + */ +public class CudaEnvironment implements Environment { + + + private static final CudaEnvironment INSTANCE = new CudaEnvironment(Nd4jCuda.Environment.getInstance()); + + private final Nd4jCuda.Environment e; + + public static CudaEnvironment getInstance(){ + return INSTANCE; + } + + protected CudaEnvironment(Nd4jCuda.Environment environment){ + this.e = environment; + } + + @Override + public int blasMajorVersion() { + return e.blasMajorVersion(); + } + + @Override + public int blasMinorVersion() { + return e.blasMinorVersion(); + } + + @Override + public int blasPatchVersion() { + return e.blasMajorVersion(); + } + + @Override + public boolean isVerbose() { + return e.isVerbose(); + } + + @Override + public void setVerbose(boolean reallyVerbose) { + e.setVerbose(reallyVerbose); + } + + @Override + public boolean isDebug() { + return e.isDebug(); + } + + @Override + public boolean isProfiling() { + return e.isProfiling(); + } + + @Override + public boolean isDetectingLeaks() { + return e.isDetectingLeaks(); + } + + @Override + public boolean isDebugAndVerbose() { + return e.isDebugAndVerbose(); + } + + @Override + public void setDebug(boolean reallyDebug) { + e.setDebug(reallyDebug); + } + + @Override + public void setProfiling(boolean reallyProfile) { + e.setProfiling(reallyProfile); + } + + @Override + public void setLeaksDetector(boolean reallyDetect) { + e.setLeaksDetector(reallyDetect); + } + + @Override + public boolean helpersAllowed() { + return e.helpersAllowed(); + } + + @Override + public void allowHelpers(boolean reallyAllow) { + e.allowHelpers(reallyAllow); + } + + @Override + public int tadThreshold() { + return e.tadThreshold(); + } + + @Override + public void setTadThreshold(int threshold) { + e.setTadThreshold(threshold); + } + + @Override + public int elementwiseThreshold() { + return e.elementwiseThreshold(); + } + + @Override + public void setElementwiseThreshold(int threshold) { + e.setElementwiseThreshold(threshold); + } + + @Override + public int maxThreads() { + return e.maxThreads(); + } + + @Override + public void setMaxThreads(int max) { + e.setMaxThreads(max); + } + + @Override + public int maxMasterThreads() { + return e.maxMasterThreads(); + } + + @Override + public void setMaxMasterThreads(int max) { + e.setMaxMasterThreads(max); + } + + @Override + public void setMaxPrimaryMemory(long maxBytes) { + e.setMaxPrimaryMemory(maxBytes); + } + + @Override + public void setMaxSpecialMemory(long maxBytes) { + e.setMaxSpecialyMemory(maxBytes); + } + + @Override + public void setMaxDeviceMemory(long maxBytes) { + e.setMaxDeviceMemory(maxBytes); + } + + @Override + public boolean isCPU() { + return e.isCPU(); + } + + @Override + public void setGroupLimit(int group, long numBytes) { + e.setGroupLimit(group, numBytes); + } + + @Override + public void setDeviceLimit(int deviceId, long numBytes) { + e.setDeviceLimit(deviceId, numBytes); + } + + @Override + public long getGroupLimit(int group) { + return e.getGroupLimit(group); + } + + @Override + public long getDeviceLimit(int deviceId) { + return e.getDeviceLimit(deviceId); + } + + @Override + public long getDeviceCouner(int deviceId) { + return e.getDeviceCounter(deviceId); + } +} diff --git a/cavis-native/cavis-native-jcublas/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor b/cavis-native/cavis-native-jcublas/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor new file mode 100644 index 000000000..e6d4fa5e3 --- /dev/null +++ b/cavis-native/cavis-native-jcublas/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor @@ -0,0 +1,21 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + diff --git a/cavis-native/cavis-native-jcublas/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend b/cavis-native/cavis-native-jcublas/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend new file mode 100644 index 000000000..b48833c16 --- /dev/null +++ b/cavis-native/cavis-native-jcublas/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend @@ -0,0 +1,24 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + + + +org.nd4j.linalg.jcublas.JCublasBackend \ No newline at end of file diff --git a/cavis-native/cavis-native-jcublas/src/main/resources/cudafunctions.properties b/cavis-native/cavis-native-jcublas/src/main/resources/cudafunctions.properties new file mode 100644 index 000000000..561cff55e --- /dev/null +++ b/cavis-native/cavis-native-jcublas/src/main/resources/cudafunctions.properties @@ -0,0 +1,24 @@ +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +org.nd4j.linalg.jcuda.jcublas.functions=broadcast,indexReduce,reduce,reduce3,transform,pairWiseTransform,scalar +org.nd4j.linalg.jcuda.jcublas.threads =128 +org.nd4j.linalg.jcuda.jcublas.blocks = 512 +org.nd4j.linalg.jcuda.jcublas.sharedmem = 1024 +org.nd4j.linalg.jcuda.jcublas.ban_devices=-1 diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/resources/nd4j-jcublas.properties b/cavis-native/cavis-native-jcublas/src/main/resources/nd4j-jcublas.properties similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/resources/nd4j-jcublas.properties rename to cavis-native/cavis-native-jcublas/src/main/resources/nd4j-jcublas.properties diff --git a/cavis-native/cavis-native-lib/CMakeLists.txt b/cavis-native/cavis-native-lib/CMakeLists.txt new file mode 100644 index 000000000..3795e7bd0 --- /dev/null +++ b/cavis-native/cavis-native-lib/CMakeLists.txt @@ -0,0 +1,439 @@ +cmake_minimum_required(VERSION 3.20) +#set(CMAKE_GNUtoMS ON) #https://gitlab.kitware.com/cmake/cmake/-/issues/19171 + +project(libnd4j) +set(CMAKE_VERBOSE_MAKEFILE ON) + + +set (CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}") +message("CMAKE MODULE PATH IS ${CMAKE_MODULE_PATH}") + +#ensure we create lib files +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS OFF) + + +option(SD_NATIVE "Optimize for build machine (might not work on others)" OFF) +option(SD_CHECK_VECTORIZATION "checks for vectorization" OFF) +option(SD_BUILD_TESTS "Build tests" OFF) +option(SD_STATIC_LIB "Build static library" OFF) +option(SD_SHARED_LIB "Build shared library" ON) +option(SD_SANITIZE "Enable Address Sanitizer" ON) +option(SD_EXPERIMENTAL "Enable experimental features" OFf) + +option(FLATBUFFERS_BUILD_FLATC "Enable the build of the flatbuffers compiler" OFF) +set(FLATBUFFERS_BUILD_FLATC "OFF" CACHE STRING "Hack to disable flatc build" FORCE) + +set(CMAKE_CXX_STANDARD 14) + +#/////////////////////////////////////////////////////////////////////////////// +# genCompilation: Generates cpp, cu files +# INPUT: +# $FILE_ITEM template-configuration that utilizes libnd4j type, macros helpers +# defined inside { include/types/types.h, include/system/type_boilerplate.h} +# OUTPUT: +# $CUSTOMOPS_GENERIC_SOURCES generated files will be added into this List +#//////////////////////////////////////////////////////////////////////////////// +# A simple template-configuration file example: +# // hints and defines what types will be generated +# #cmakedefine LIBND4J_TYPE_GEN +# #cmakedefine FLOAT_TYPE_GEN +# // below if defines blocks are needed for correctly handling multiple types +# #if defined(LIBND4J_TYPE_GEN) +# BUILD_DOUBLE_TEMPLATE(template void someFunc, (arg_list,..), +# LIBND4J_TYPES_@FL_TYPE_INDEX@, INDEXING_TYPES); +# #endif +# #if defined(FLOAT_TYPE_GEN) +# BUILD_SINGLE_TEMPLATE(template class SomeClass,, FLOAT_TYPES_@FL_TYPE_INDEX@); +# #endif +#//////////////////////////////////////////////////////////////////////////////// + +set_property(GLOBAL PROPERTY JOB_POOLS one_jobs=1 two_jobs=2) + + + + +function(genCompilation FILE_ITEM) + get_filename_component(FILE_ITEM_WE ${FL_ITEM} NAME_WE) + + set(EXTENSION "cpp") + + if(FL_ITEM MATCHES "cu.in$") + set(EXTENSION "cu") + endif() + + file(READ ${FL_ITEM} CONTENT_FL) + #check content for types + + #set all to false + set (FLOAT_TYPE_GEN 0) + set (INT_TYPE_GEN 0) + set (LIBND4J_TYPE_GEN 0) + set (PAIRWISE_TYPE_GEN 0) + set (RANGE_STOP -1) + + string(REGEX MATCHALL "#cmakedefine[ \t]+[^_]+_TYPE_GEN" TYPE_MATCHES ${CONTENT_FL}) + + foreach(TYPEX ${TYPE_MATCHES}) + set(STOP -1) + if(TYPEX MATCHES "INT_TYPE_GEN$") + set (INT_TYPE_GEN 1) + set(STOP 7) + endif() + if(TYPEX MATCHES "LIBND4J_TYPE_GEN$") + set (LIBND4J_TYPE_GEN 1) + set(STOP 9) + endif() + if(TYPEX MATCHES "FLOAT_TYPE_GEN$") + set (FLOAT_TYPE_GEN 1) + set(STOP 3) + endif() + if(TYPEX MATCHES "PAIRWISE_TYPE_GEN$") + set (PAIRWISE_TYPE_GEN 1) + set(STOP 12) + endif() + if(STOP GREATER RANGE_STOP) + set(RANGE_STOP ${STOP}) + endif() + + endforeach() + + if(RANGE_STOP GREATER -1) + foreach(FL_TYPE_INDEX RANGE 0 ${RANGE_STOP}) + # set OFF if the index is above + if(FL_TYPE_INDEX GREATER 3) + set (FLOAT_TYPE_GEN 0) + endif() + if(FL_TYPE_INDEX GREATER 7) + set (INT_TYPE_GEN 0) + endif() + if(FL_TYPE_INDEX GREATER 9) + set (LIBND4J_TYPE_GEN 0) + endif() + set(GENERATED_SOURCE "${CMAKE_BINARY_DIR}/compilation_units/${FILE_ITEM_WE}_${FL_TYPE_INDEX}.${EXTENSION}") + configure_file( "${FL_ITEM}" "${GENERATED_SOURCE}" @ONLY) + LIST(APPEND CUSTOMOPS_GENERIC_SOURCES ${GENERATED_SOURCE} ) + endforeach() + endif() + + set(CUSTOMOPS_GENERIC_SOURCES ${CUSTOMOPS_GENERIC_SOURCES} PARENT_SCOPE) +endfunction() + + +if (SD_CUDA) + #enable_language(CUDA) + find_package(CUDAToolkit 11.2 REQUIRED) + message(STATUS "CUDAToolkit_VERSION: ${CUDAToolkit_VERSION}") + message(STATUS "CUDAToolkit_VERSION_MAJOR: ${CUDAToolkit_VERSION_MAJOR}") + message(STATUS "CUDAToolkit_VERSION_MINOR: ${CUDAToolkit_VERSION_MINOR}") + message(STATUS "CUDAToolkit_VERSION_PATCH: ${CUDAToolkit_VERSION_PATCH}") + message(STATUS "CUDAToolkit_BIN_DIR: ${CUDAToolkit_BIN_DIR}") + message(STATUS "CUDAToolkit_INCLUDE_DIRS: ${CUDAToolkit_INCLUDE_DIRS}") + message(STATUS "CUDAToolkit_LIBRARY_DIR: ${CUDAToolkit_LIBRARY_DIR}") + message(STATUS "CUDAToolkit_NVCC_EXECUTABLE ${CUDAToolkit_NVCC_EXECUTABLE}") + + set(DEFAULT_ENGINE "samediff::ENGINE_CUDA") +else() + set(DEFAULT_ENGINE "samediff::ENGINE_CPU") +endif() + +# MSVC runtime lib can be either "MultiThreaded" or "MultiThreadedDLL", /MT and /MD respectively +#set(MSVC_RT_LIB "MultiThreadedDLL") + +set(SD_X86_BUILD false) + +if (NOT SD_IOS_BUILD AND NOT SD_ANDROID_BUILD AND NOT ${SD_ARCH} MATCHES "power*" AND NOT ${SD_ARCH} MATCHES "arm*") + set(SD_X86_BUILD true) +endif() + +# -fsanitize=address +# -fsanitize=leak +if (SD_ANDROID_BUILD) + set_property(GLOBAL PROPERTY JOB_POOLS one_job=1 two_jobs=2) + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else -D_RELEASE=true") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else") +elseif (APPLE) + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else -D__APPLE_OS__=true -D_RELEASE=true") + set(CMAKE_CXX_FLAGS_DEBUG " -O0 -g -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else -D__APPLE_OS__=true") +elseif(WIN32) + set(SD_X86_BUILD true) + if (SD_CUDA) + set(CMAKE_CXX_FLAGS_RELEASE "-D_RELEASE=true") + set(CMAKE_CXX_FLAGS_DEBUG " /FS /EHsc") + else() + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fPIC -D_RELEASE=true") + set(CMAKE_CXX_FLAGS_DEBUG " -g -O2 -fPIC") + endif() +else() + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fPIC -D_RELEASE=true") + set(CMAKE_CXX_FLAGS_DEBUG " -g -O0 -fPIC") + + if (SD_CPU AND SD_SANITIZE) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address") + endif() +endif() + +if(SD_NATIVE) + IF(${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64*") + set(SD_X86_BUILD false) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=native") + ELSEIF(NOT CMKAE_CXX_COMPILER_ID STREQUAL "MSVC") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + ENDIF() +endif() + + +if(NOT SD_CUDA) + # we need this definition to avoid global memory use within mkldnn + add_definitions(-DDNNL_ENABLE_CONCURRENT_EXEC=true) + + # there's a chance, we have no BLAS provided externally + if ("${OPENBLAS_PATH}" STREQUAL "") + #we don't want OpenBLAS on Apple + if (NOT APPLE) + # note: this is not a typo + set(BLA_VENDOR "OpenBLAS") + endif() + + # look around for system blas instead, see: https://cmake.org/cmake/help/latest/module/FindBLAS.html + find_package(BLAS REQUIRED) + if (BLAS_FOUND) + message("Found external BLAS implementation: ${BLAS_LIBRARIES} ") + add_definitions(-D__EXTERNAL_BLAS__=true) + endif() + else() + # if we have externally provided OPENBLAS_PATH - let's use it + set(HAVE_OPENBLAS 1) + message("Setting openblas") + include_directories(${OPENBLAS_PATH}/include/) + link_directories(${OPENBLAS_PATH} ${OPENBLAS_PATH}/lib/) + set(OPENBLAS_LIBRARIES openblas) + endif() + + # building cpu_features + if (SD_X86_BUILD) + add_definitions(-DCPU_FEATURES=true) + set(BUILD_PIC "ON" CACHE STRING "Hack to enforce fPIC mode" FORCE) + configure_file(./CMakeLists.txt.cpu_features.in cpu_features-download/CMakeLists.txt) + message("CMAKE_COMMAND: ${CMAKE_COMMAND}") + execute_process(COMMAND ${CMAKE_COMMAND} -DBUILD_PIC=ON -G "${CMAKE_GENERATOR}" . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/cpu_features-download ) + + if(result) + message(FATAL_ERROR "CMake step for cpu_features failed: ${result}") + endif() + execute_process(COMMAND ${CMAKE_COMMAND} --build . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/cpu_features-download ) + if(result) + message(FATAL_ERROR "Build step for cpu_features failed: ${result}") + endif() + + add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/cpu_features-src + ${CMAKE_CURRENT_BINARY_DIR}/cpu_features-build + EXCLUDE_FROM_ALL) + set(CPUF_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cpu_features-src) + include_directories(${CPUF_SOURCE_DIR}/include) + set(CPU_FEATURES cpu_features) + endif() +endif() + + +#arm-compute entry +if(${HELPERS_armcompute}) + find_package(ARMCOMPUTE REQUIRED) + execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE ld_version) + if ("${ld_version}" MATCHES "GNU gold") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold ") + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + add_link_options("-Wl,--long-plt") + endif() + endif() + + if(ARMCOMPUTE_FOUND) + message("Found ARMCOMPUTE: ${ARMCOMPUTE_LIBRARIES}") + set(HAVE_ARMCOMPUTE 1) + # Add preprocessor definition for ARM Compute NEON + add_definitions(-DARMCOMPUTENEON_ENABLED) + include_directories(${ARMCOMPUTE_INCLUDE}) + message("----${ARMCOMPUTE_INCLUDE}---") + endif() + + +endif() + + + +# new mkl-dnn entry + +if (${HELPERS_mkldnn}) + message("Going to pull & build mkldnn") + set(HAVE_MKLDNN 1) + set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "Hack to enforce static mode" FORCE) + + configure_file(./CMakeLists.txt.mkldnn.in mkldnn-download/CMakeLists.txt) + execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-download ) + if(result) + message(FATAL_ERROR "CMake step for mkldnn failed: ${result}") + endif() + execute_process(COMMAND ${CMAKE_COMMAND} --build . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-download ) + if(result) + message(FATAL_ERROR "Build step for mkldnn failed: ${result}") + endif() + + add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/mkldnn-src + ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-build + EXCLUDE_FROM_ALL) + + set(mkldnn_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-build) + set(mkldnn_EXT_DIR ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-src) + set(MKLDNN_PATH "${mkldnn_SOURCE_DIR}") + include_directories(${mkldnn_SOURCE_DIR}/include ${mkldnn_EXT_DIR}/include ${mkldnn_SOURCE_DIR}) + set(MKLDNN dnnl) +endif() + + +if (${HELPERS_cudnn}) + if (NOT SD_CUDA) + message(FATAL_ERROR "Can't build cuDNN on non-CUDA platform") + endif() + + set(CUDNN_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA cuDNN") + + SET(CUDNN_LIBNAME "cudnn") + find_path(CUDNN_INCLUDE_DIR cudnn.h + HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} + PATH_SUFFIXES cuda/include include) + + find_library(CUDNN_LIBRARY ${CUDNN_LIBNAME} + HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} + PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) + + #find_library(CULIBOS_LIBRARY ${CULIBOS_LIBNAME} + # HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} + # PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) + + + if (CUDNN_LIBRARY) + set(HAVE_CUDNN true) + set(CUDNN ${CUDNN_LIBRARY}) + else() + message(FATAL_ERROR "Unable to find cuDNN") + endif() +endif() + +# Download and unpack flatbuffers at configure time +configure_file(CMakeLists.txt.flatbuffers.in flatbuffers-download/CMakeLists.txt) +execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-download ) +if(result) + message(FATAL_ERROR "CMake step for flatbuffers failed: ${result}") +endif() +execute_process(COMMAND ${CMAKE_COMMAND} --build . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-download ) +if(result) +# message(FATAL_ERROR "Build step for flatbuffers failed: ${result}") +endif() + +# Add flatbuffers directly to our build. +add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-src + ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-build + EXCLUDE_FROM_ALL) + +set(HAVE_FLATBUFFERS 1) +set(FLATBUFFERS_PATH ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-src) +include_directories(${FLATBUFFERS_PATH}/include) + + + +configure_file(src/main/include/config.h.in src/main/include/config.h) +include_directories(${CMAKE_CURRENT_BINARY_DIR}/src/main/include/) + + +#include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/main/include) +add_subdirectory(src/main/cpp/blas output) +if(SD_BUILD_TESTS) + # tests are always compiled with all ops included + set(SD_ALL_OPS true) + set(SD_BUILD_MINIFIER true) + add_subdirectory(tests_cpu) +endif() + + +if (MSVC_DEV) + set(SD_BUILD_MINIFIER false) +endif () + +set (CMAKE_INSTALL_PREFIX $ENV{ND4J_HOME}/bruai4j-native/bruai4j-native-common/src/main/resources) + +# Set package information +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Native operations for nd4j.") +set(CPACK_PACKAGE_RELEASE 1) +set(CPACK_PACKAGE_CONTACT "agibsonccc ") +set(CPACK_PACKAGE_VENDOR "Brutex Network") +set(CPACK_SETDESTDIR "false") +set(CPACK_PACKAGING_INSTALL_PREFIX "/usr/local/lib") +set(CPACK_PACKAGE_NAME "libnd4j") +set(CPACK_PACKAGE_VERSION_MAJOR "0") +set(CPACK_PACKAGE_VERSION_MINOR "8") +set(CPACK_PACKAGE_VERSION_PATCH "0") +set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") +set(CPACK_PACKAGE_INSTALL_DIRECTORY "libnd4j") +set(CPACK_RESOURCE_FILE_README "${CMAKE_CURRENT_SOURCE_DIR}/README.md") + +# Determine distribution and release — may require redhat-lsb-core installed on CentOS / RH +execute_process(COMMAND lsb_release -si OUTPUT_VARIABLE DISTRIBUTION OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND lsb_release -sc OUTPUT_VARIABLE RELEASE OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND uname -i OUTPUT_VARIABLE ARCHITECTURE) + +# Set package name and type (deb vs rpm) +if(DISTRIBUTION STREQUAL "Ubuntu") + + # Set Ubuntu-specific information (see http://www.cmake.org/Wiki/CMake:CPackPackageGenerators) + if(ARCHITECTURE MATCHES ".*x86_64.*") + set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64") + else() + set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "i386") + endif() + set(CPACK_DEBIAN_PACKAGE_MAINTAINER "raver119") + set(CPACK_DEBIAN_PACKAGE_SECTION "devel") + set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "cuda") + # For Ubuntu <= 12, libatlas3gf-base, liblapack3gf + # Build deps: libatlas3-base liblapack3 libopenblas-dev libatlas-dev liblapack-dev gcc-5 g++-5 + set(CPACK_DEBIAN_PACKAGE_DEPENDS "") + set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/eclipse/deeplearning4j") + set(CPACK_GENERATOR "DEB") + set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION}-${RELEASE}_${CPACK_DEBIAN_PACKAGE_ARCHITECTURE}) + set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/cmake/postinst;${CMAKE_CURRENT_SOURCE_DIR}/cmake/postrm;" ) + +elseif(DISTRIBUTION STREQUAL "CentOS") + + # Set Fedora-specific information (see http://www.cmake.org/Wiki/CMake:CPackPackageGenerators) + execute_process(COMMAND lsb_release -sr OUTPUT_VARIABLE RELEASE OUTPUT_STRIP_TRAILING_WHITESPACE) + if(ARCHITECTURE MATCHES ".*x86_64.*") + set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64") + else() + set(CPACK_RPM_PACKAGE_ARCHITECTURE "i686") + endif() + set(CPACK_PACKAGE_CONTACT "agibsonccc") + set(CPACK_RPM_PACKAGE_GROUP "Development/Tools") + set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") + set(CPACK_RPM_PACKAGE_SUGGESTS "cuda") + # Build deps: atlas blas lapack cmake3 devtoolset-4-gcc devtoolset-4-gcc-c++ + set(CPACK_RPM_PACKAGE_REQUIRES "") + set(CPACK_RPM_PACKAGE_URL "https://github.com/eclipse/deeplearning4j/libnd4j") + set(CPACK_GENERATOR "RPM") + set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}.fc${RELEASE}.${CPACK_RPM_PACKAGE_ARCHITECTURE}) + set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/postinst") + set(CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/postrm") + set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/usr/local/lib") + +endif() + +include(CPack) diff --git a/cavis-native/cavis-native-lib/CMakeLists.txt.cpu_features.in b/cavis-native/cavis-native-lib/CMakeLists.txt.cpu_features.in new file mode 100644 index 000000000..f2f491aed --- /dev/null +++ b/cavis-native/cavis-native-lib/CMakeLists.txt.cpu_features.in @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.6) + +project(cpu_features-download NONE) + +include(ExternalProject) +ExternalProject_Add(cpu_features + GIT_REPOSITORY https://github.com/google/cpu_features.git + GIT_TAG v0.4.1 + GIT_SHALLOW true + SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/cpu_features-src" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/cpu_features-build" + CONFIGURE_COMMAND "" + CMAKE_ARGS "-DBUILD_PIC=ON" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/cavis-native/cavis-native-lib/CMakeLists.txt.flatbuffers.in b/cavis-native/cavis-native-lib/CMakeLists.txt.flatbuffers.in new file mode 100644 index 000000000..2a44dbc7c --- /dev/null +++ b/cavis-native/cavis-native-lib/CMakeLists.txt.flatbuffers.in @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.6) + +project(flatbuffers-download NONE) + +include(ExternalProject) +ExternalProject_Add(flatbuffers + GIT_REPOSITORY https://github.com/google/flatbuffers.git + GIT_TAG v1.10.0 + GIT_SHALLOW true + SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-src" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-build" + CONFIGURE_COMMAND "" + CMAKE_ARGS "-DFLATBUFFERS_BUILD_FLATC=ON " + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/libnd4j/CMakeLists.txt.mkldnn.in b/cavis-native/cavis-native-lib/CMakeLists.txt.mkldnn.in similarity index 75% rename from libnd4j/CMakeLists.txt.mkldnn.in rename to cavis-native/cavis-native-lib/CMakeLists.txt.mkldnn.in index 6b7b3163d..1cdd8aa7c 100644 --- a/libnd4j/CMakeLists.txt.mkldnn.in +++ b/cavis-native/cavis-native-lib/CMakeLists.txt.mkldnn.in @@ -1,16 +1,21 @@ -cmake_minimum_required(VERSION 2.8.2) +cmake_minimum_required(VERSION 3.6) project(mkldnn-download NONE) +#war #v1.4 -G \"Unix Makefiles\" +#ver v2.2.3 + include(ExternalProject) ExternalProject_Add(mkldnn GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git - GIT_TAG v1.4 + GIT_TAG v1.8.1 + GIT_SHALLOW true SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/mkldnn-src" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/mkldnn-build" CONFIGURE_COMMAND "" - CMAKE_ARGS -DDNNL_USE_MKL=ML -DDNNL_LIBRARY_TYPE=STATIC -G \"Unix Makefiles\" + CMAKE_ARGS -DDNNL_USE_MKL=ML -DDNNL_LIBRARY_TYPE=STATIC BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" ) + diff --git a/libnd4j/README.md b/cavis-native/cavis-native-lib/README.md similarity index 100% rename from libnd4j/README.md rename to cavis-native/cavis-native-lib/README.md diff --git a/cavis-native/cavis-native-lib/build.gradle b/cavis-native/cavis-native-lib/build.gradle new file mode 100644 index 000000000..043b8b8d8 --- /dev/null +++ b/cavis-native/cavis-native-lib/build.gradle @@ -0,0 +1,553 @@ +import org.gradle.api.publish.maven.internal.publisher.MavenRemotePublisher +import org.gradle.language.nativeplatform.internal.Dimensions + +buildscript { +/**************************************************************************** + * Establish Visual Studio configuration environment for Windows native builds + * NOTE: vsconfig.gradle path is relative to each GPL project module + ****************************************************************************/ + apply from: "../../vsconfig.gradle" + apply from: "../../chooseBackend.gradle" + + ext { + + host_cores = Runtime.getRuntime().availableProcessors() + + buildHelper = "-mingw" + javacppPlatform = osdetector.classifier + + println "Building on ${host_cores} CPU cores." + println "JavaCPP target plattform is ${javacppPlatform}" + + + if (project.hasProperty("CAVIS_AVX_EXTENSION")) { + avxExtension = project.getProperty("CAVIS_AVX_EXTENSION").toLowerCase() + println "Bulding with ${avxExtension}" + } else { + avxExtension = "avx2" + logger.quiet("No AVX CPU extension selected (avx2|avx512). Building with default 'avx2'") + } + + javacppPlatformExtension = "-${avxExtension}".toString() + + getBuildPlatform = { String chip, Task tsk -> + def pf ="" + if(chip.equals("cuda")) { + pf = osdetector.classifier + } else { + if(osdetector.os.equals("windows")) { + pf = "${osdetector.classifier}-mingw" + } else { + pf = "${osdetector.classifier}" + } + } + logger.info("Setting properties for task '{}' to '{}'", tsk.getName(), pf) + return pf + } + + + } + + + dependencies { + classpath platform(project(":cavis-common-platform")) + classpath group: "org.bytedeco", name: "openblas" + classpath group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}" + classpath group: "org.bytedeco", name:"mkl-dnn" + classpath group: "org.bytedeco", name:"mkl-dnn", classifier: "${javacppPlatform}" + classpath group: "org.bytedeco", name: "javacpp" + classpath group: "org.bytedeco", name: "javacpp", classifier: "${javacppPlatform}" + } + + +} + + +plugins { + id 'java-library' + id 'org.bytedeco.gradle-javacpp-build' version "1.5.6" + id 'maven-publish' + id 'signing' +} + +chipList.each {thisChip -> + sourceSets.register("${thisChip}Support") { + java { + srcDirs = ['src/main/java', "${buildDir}/generated/sources/javacpp/${thisChip}//${javacppPlatform}${javacppPlatformExtension}/"] + include "org/nd4j/nativeblas/${thisChip}/Nd4j${thisChip.capitalize()}Helper.java" + include "org/nd4j/nativeblas/${thisChip}/Nd4j${thisChip.capitalize()}Presets.java" + include "org/nd4j/nativeblas/Nd4j${thisChip.capitalize()}.java" + } + it.compiledBy("javacpp${thisChip.capitalize()}SupportBuildCommand", + "javacpp${thisChip.capitalize()}SupportBuildCompiler") + } +} + + +if(osdetector.os.startsWith("windows")) { + sourceSets { + main { + java { + srcDirs = ['src/main/java'] + include 'org/nd4j/nativeblas/Dummy.java' + } + } + } +} + + +java { + chipList.each {thisChip -> + registerFeature("${thisChip}Support") { + usingSourceSet(sourceSets.findByName("${thisChip}Support")) + //withJavadocJar() + //withSourcesJar() + } + } +} + +/* +configurations.each(s -> { + println "Configurations: " + s.name + " " + s.artifacts.each( x -> + { println x.getFile().getName()}) +}) +*/ + +dependencies { + api platform(project(':cavis-common-platform')) + + + api "org.bytedeco:javacpp" + + if(withCuda()) { + cudaSupportImplementation platform(project(':cavis-common-platform')) + cudaSupportImplementation project(":cavis-dnn:cavis-dnn-api") + cudaSupportImplementation project(":cavis-dnn:cavis-dnn-common") + cudaSupportImplementation project(":cavis-native:cavis-native-blas") + cudaSupportImplementation project(":cavis-native:cavis-native-common") + cudaSupportImplementation "commons-io:commons-io" + cudaSupportImplementation group: "org.bytedeco", name: "openblas" + cudaSupportImplementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}" + cudaSupportImplementation group: "org.bytedeco", name: "cuda" + cudaSupportImplementation group: "org.bytedeco", name: "cuda", classifier: "${javacppPlatform}" + cudaSupportImplementation "org.apache.logging.log4j:log4j-core:2.17.0" + cudaSupportImplementation "com.google.guava:guava:14.0.1" + cudaSupportImplementation "org.apache.commons:commons-lang3" + cudaSupportImplementation "org.apache.commons:commons-math3" + cudaSupportImplementation "com.google.flatbuffers:flatbuffers-java" + cudaSupportImplementation 'javax.mail:javax.mail-api:1.6.2' + } + + if(withCpu()) { + cpuSupportImplementation platform(project(':cavis-common-platform')) + cpuSupportImplementation project(":cavis-dnn:cavis-dnn-api") + cpuSupportImplementation project(":cavis-dnn:cavis-dnn-common") + cpuSupportImplementation project(":cavis-native:cavis-native-blas") + cpuSupportImplementation project(":cavis-native:cavis-native-common") + cpuSupportImplementation "commons-io:commons-io" + cpuSupportImplementation group: "org.bytedeco", name: "openblas" + cpuSupportImplementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}" + cpuSupportImplementation group: "org.bytedeco", name: "opencv" + cpuSupportImplementation group: "org.bytedeco", name: "opencv", classifier: "${javacppPlatform}" + cpuSupportImplementation "org.apache.logging.log4j:log4j-core:2.17.0" + cpuSupportImplementation "com.google.guava:guava:14.0.1" + cpuSupportImplementation "org.apache.commons:commons-lang3" + cpuSupportImplementation "org.apache.commons:commons-math3" + cpuSupportImplementation "com.google.flatbuffers:flatbuffers-java" + cpuSupportImplementation 'javax.mail:javax.mail-api:1.6.2' + } + + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCommon + implementation project(":cavis-native:cavis-native-blas") + implementation project(":cavis-native:cavis-native-common") + implementation "commons-io:commons-io" + implementation "org.bytedeco:openblas" + implementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}" + implementation "org.apache.logging.log4j:log4j-core" + implementation "com.google.guava:guava:14.0.1" + implementation "org.apache.commons:commons-lang3" + implementation "org.apache.commons:commons-math3" + implementation "com.google.flatbuffers:flatbuffers-java" + + //javacppPlatform project(":cavis-native:cavis-native-blas") +} + + +clean { + doFirst { + delete "${projectDir}/build" + delete "${projectDir}/src/main/include/config.h" + chipList.each { + delete "${projectDir}/blasbuild/${it}" + } + } +} + +task deepClean(type: Delete) { + dependsOn clean + doFirst { + delete "$projectDir}/blasbuild" + } +} + + +tasks.withType(org.bytedeco.gradle.javacpp.BuildTask) { + buildResource = [ "/org/bytedeco/openblas/${javacppPlatform}/", + "/org/bytedeco/mkldnn/${javacppPlatform}/"] + + includeResource = ["/org/bytedeco/openblas/${javacppPlatform}/include/"] + + linkResource = ["/org/bytedeco/openblas/${javacppPlatform}/", + "/org/bytedeco/openblas/${javacppPlatform}/lib/"] + + //buildPath = [ org.bytedeco.javacpp.Loader.getCacheDir() ] + + + +} + + +// Disable the standard javacpp generated tasks and use own +// versions below. This allows to build for each variant +[javacppBuildParser, javacppBuildCommand, javacppCompileJava, javacppBuildCompiler].each { + it.enabled false; +} + +chipList.each { thisChip -> + + // 1) + //Run the C++ compile first + tasks.register("javacpp${thisChip.capitalize()}SupportBuildCommand", org.bytedeco.gradle.javacpp.BuildTask) { + if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) { + enabled = false + } + properties = getBuildPlatform( thisChip, it ) + + + includePath = ["${projectDir}/src/main/cpp/blas/", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"] + linkPath = ["${projectDir}/blasbuild/${thisChip}/${avxExtension}/output"] + //No idea why this is here, but it looks like even for the javacppBuildCommand task, + //there is a javacpp Loader actively determining platform etc. + classOrPackageNames = ["org.nd4j.nativeblas.${thisChip}.Nd4j${thisChip.capitalize()}Presets"] + workingDirectory = projectDir + //if the classpath is not set here, the javacpp classloader starts to look around + //everywhere and causes java.io.IOExceptions: because files is being used by another process + classPath = [:] + classPath += ["${buildDir}/classes/java/${thisChip}Support/"] + //classPath += ["${buildDir}/classes/java/main/"] + + /* Get VCVARS in case we want to build CUDA + * MinGW64 g++ on MSYS is used otherwise */ + if (thisChip.equals('cuda') && osdetector.os.startsWith("win") && !VISUAL_STUDIO_INSTALL_DIR.isEmpty()) { + def proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && set"].execute() + it.environmentVariables = it.environmentVariables ?: [:] + def lines = proc.text.split("\\r?\\n") + for (def line in lines) { + if (line.contains("=")) { + def parts = line.split("=") + it.environmentVariables.put(parts[0], parts[1]) + } + } + } + + if (thisChip.equals('cuda') && osdetector.os.startsWith("windows")) { //cuDNN requires CUDA + it.buildCommand = ['sh', 'buildnativeoperations.sh', + '-V', + '--build-type', 'release', + '--chip', thisChip, + '--plattform', 'x86_64', + '--chip-extension', avxExtension, + '-j', "${host_cores}", + // '--helper', 'mkldnn', + '--helper', 'cudnn'] + } else if (thisChip.equals('cuda') && osdetector.os.startsWith("linux")) { //cuDNN requires CUDA + it.buildCommand = ['bash', 'buildnativeoperations.sh', + '-V', + '--build-type', 'release', + '--chip', thisChip, + '--plattform', 'x86_64', + '--chip-extension', avxExtension, + '-j', "${host_cores}", + // '--helper', 'mkldnn', + '--helper', 'cudnn'] + } else { + it.buildCommand = ['bash', 'buildnativeoperations.sh', + '-V', + '--build-type', 'release', + '--chip', thisChip, + '--plattform', 'x86_64', + '--chip-extension', avxExtension, + '-j', "${host_cores}", + '--helper', 'mkldnn'] + } + } + + + //Create a task to (pre)compile the java presets (required for javacppBuildParser) + tasks.register("compile${thisChip.capitalize()}Support", JavaCompile) { + def thisSS = sourceSets.findByName("${thisChip}Support") + it.source = thisSS.allSource + it.classpath = thisSS.compileClasspath + it.destinationDirectory = file("${buildDir}/classes/java/${thisChip}Support/") + } + + //Run the parser on the InfoMap in Nd4j$ChipPresets and listed header files in @Platform + //Generates Nd4jCpu.java and/ or Nd4jCuda.java Java JNI code + tasks.register("javacpp${thisChip.capitalize()}SupportBuildParser", org.bytedeco.gradle.javacpp.BuildTask) { + if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) { + enabled = false + } + dependsOn "compile${thisChip.capitalize()}Support" + + includePath = ["${projectDir}/src/main/cpp/blas/", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"] + + + + classOrPackageNames = ["org.nd4j.nativeblas.${thisChip}.Nd4j${thisChip.capitalize()}Presets"] + outputDirectory = file("${buildDir}/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/") + + + classPath = sourceSets.getByName("${thisChip}Support").getRuntimeClasspath() + classPath += ["${buildDir}/classes/java/${thisChip}Support/"] + } + + + // Generates jnijavacpp.cpp and jniNativeLibrary.cpp, compiles and links it + tasks.register("javacpp${thisChip.capitalize()}SupportBuildCompiler", org.bytedeco.gradle.javacpp.BuildTask) { + if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) { + enabled = false + } + def thisTask = (org.bytedeco.gradle.javacpp.BuildTask) it + thisTask.dependsOn = ["javacpp${thisChip.capitalize()}SupportBuildParser"] + + thisTask.linkPath = ["${projectDir}/blasbuild/${thisChip}/${avxExtension}/output"] + thisTask.includePath = ["${projectDir}/src/main/cpp/blas/", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include", + "${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"] + + thisTask.properties = getBuildPlatform( thisChip, thisTask ) + + if(thisChip.equals('cuda') && osdetector.os.startsWith("win") && !VISUAL_STUDIO_INSTALL_DIR.isEmpty()) { + def proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && where.exe cl.exe"].execute() + def outp = proc.text + def cl = outp.replace("\\", "\\\\").trim() + def currentCompiler = "" + doFirst{ + currentCompiler = System.getProperty("org.bytedeco.javacpp.platform.compiler") + System.setProperty("org.bytedeco.javacpp.platform.compiler", cl) + logger.quiet("Task ${thisTask.name} overrides compiler '${currentCompiler}' with '${cl}'.") + } + doLast { + //restore compiler + System.setProperty("org.bytedeco.javacpp.platform.compiler", currentCompiler ?: "") + }//System.setProperty("org.bytedeco.javacpp.platform.compiler", cl) + //System.setProperty("org.bytedeco.javacpp.platform.compiler.cpp11", cl) + + proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && set"].execute() + thisTask.environmentVariables = thisTask.environmentVariables ?: [:] + def lines = proc.text.split("\\r?\\n") + for (def line in lines) { + if (line.contains("=")) { + def parts = line.split("=") + thisTask.environmentVariables.put(parts[0], parts[1]) + } + } + + } else { + //System.setProperty("org.bytedeco.javacpp.platform.compiler", "g++") + } + + + thisTask.buildPath = ["$buildDir/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/"] + thisTask.copyLibs = true + thisTask.deleteJniFiles(false) + outputName = "jnind4j${thisChip}" + thisTask.outputDirectory = file("$buildDir/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/") + thisTask.classOrPackageNames= ["org.nd4j.nativeblas.Nd4j${thisChip.capitalize()}"] + + thisTask.configDirectory = file("${buildDir}/classes/java/${thisChip}Support/META-INF/native-image/${javacppPlatform}") + + //Need to set the classpath, so that external jars from the dependency list are resolved by the ClassLoader as well + thisTask.classPath = [:] + thisTask.classPath = ["${buildDir}/classes/java/${thisChip}Support"] + thisTask.classPath += sourceSets.findByName("${thisChip}Support").runtimeClasspath + //sourceSets.findByName("${thisChip}Support").runtimeClasspath.each{ s -> + // thisTask.classPath += s + //} + } + + // Generates jnijavacpp.cpp and jniNativeLibrary.cpp, compiles and links it + tasks.getByName("${thisChip}SupportJar") { Jar thisTask -> + dependsOn "javacpp${thisChip.capitalize()}SupportBuildCompiler" + dependsOn "javacpp${thisChip.capitalize()}SupportBuildCommand" + + //it.from sourceSets.getByName("${thisChip}Support").getOutput() + def spec = copySpec { + from(tasks.getByName("javacpp${thisChip.capitalize()}SupportBuildCompiler")) { + exclude { f -> + def exclude = f.file.isDirectory() + if(exclude) { + logger.info("${thisTask.name}: excluding '${f}'") + } else { + logger.info("${thisTask.name}: including '${f}'") + } + return exclude + } + into "${javacppPlatform}/" //we need it in a platform, that javacpp Loader understands + } + from(sourceSets.getByName("${thisChip}Support").getOutput()) { + + } + duplicatesStrategy DuplicatesStrategy.EXCLUDE + } + + thisTask.with spec + thisTask.archiveClassifier = "${javacppPlatform}${javacppPlatformExtension}-${thisChip}" + } + + //tasks.getByName("${thisChip}SupportJar").dependsOn("javacpp${thisChip.capitalize()}SupportJar") + + +} + +//Before we can compile the whole java part, we +//need to generate the Nd4jXXX.java files first +chipList.each { thisChip -> + tasks.findByName("compile${thisChip.capitalize()}SupportJava").each { t -> + t.dependsOn "javacpp${thisChip.capitalize()}SupportBuildParser" + } +} + + + +tasks.withType(JavaCompile) { + // options.setCompilerArgs(Arrays.asList("-Xlint:unchecked")) +} + +tasks.withType(Javadoc) { + options.addStringOption('Xdoclint:none', '-quiet') +} + +jar { + manifest { + attributes 'Class-Path': configurations.runtimeClasspath.collect { it.getName() }.join(' '), + 'Implementation-Title': 'Brutex AI - Native Components', + 'Implementation-Vendor': 'Brutex Network', + 'Implementation-Version': archiveVersion, + 'Specification-Title': 'Brutex AI - Native Components', + 'Specification-Vendor': 'Brutex Network', + 'Specification-Version': archiveVersion + } + //archiveClassifier = "${javacppPlatform}${javacppPlatformExtension}-${chip}" +} + +javadoc { + dependsOn "javacppPomProperties" + failOnError = false + //options.links = ['http://bytedeco.org/javacpp/apidocs'] + options.addStringOption('Xdoclint:none', '-quiet') + //options.JFlags = ["-Xdoclint:none"] +} + + + + + +if(! osdetector.os.startsWith("windows")) { + tasks.getByName("publish") { + enabled = false + } + tasks.getByName("generatePomFileForMavenJavaPublication") { + enabled = false + } + tasks.getByName("publishMavenJavaPublicationToLocalRemoteRepository") { + enabled = false + } + chipList.each {thisChip -> + artifacts { + archives tasks.getByName("${thisChip}SupportJar") + } + } + + chipList.each { thisChip -> + publishing { + publications { + mavenJava(MavenPublication) { + artifact tasks.getByName("${thisChip}SupportJar") + } + } + } + } +} + + +if( osdetector.os.startsWith("windows")) { + + FileCollection collection = layout.files { file("build/libs/").listFiles() } + + //collection.collect { relativePath(it) }.sort().each { println it } + + publishing { + publications { + mavenJava(MavenPublication) { + artifact jar + collection.collect {File fi -> + if( fi.name.contains('linux-x86_64-avx2-cpu')) { + logger.quiet("Adding artifact ${fi.name} to publication.") + artifact source: fi, classifier: 'linux-x86_64-avx2-cpu', extension: 'jar' + } + } + + } + } + } +} + +/* +def pomClosure = { + name = 'Brutex AI - Native Components' + delegate.description = 'Underlying native components for the Brutex AI deeplearning framework for Java' + url = 'https://ai.brutex.net' + licenses { + license { + name = 'Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0' + distribution = 'repo' + } + } + developers { + developer { + id = 'irnbrux' + name = 'Brian Rosenberger' + email = 'bru@brutex.de' + } + } + scm { + url = 'https://brutex.net/svn/' + connection = 'scm:svn:https://brutex.net/svn/bruai4j/' + } +} +*/ + +//tasks.getByName("publishMavenJavaPublicationToOSSRHRepository") { MavenRemotePublisher pub -> + // logger.quiet(pub.dump()); +//} + +signing { + useGpgCmd() + if (!version.endsWith('SNAPSHOT')) { + sign publishing.publications.mavenJava + //sign publishing.publications.mavenJavacppPlatform + } +} \ No newline at end of file diff --git a/libnd4j/buildnativeoperations.sh b/cavis-native/cavis-native-lib/buildnativeoperations.sh old mode 100755 new mode 100644 similarity index 91% rename from libnd4j/buildnativeoperations.sh rename to cavis-native/cavis-native-lib/buildnativeoperations.sh index 5c13cd12f..a86b11f00 --- a/libnd4j/buildnativeoperations.sh +++ b/cavis-native/cavis-native-lib/buildnativeoperations.sh @@ -19,12 +19,15 @@ # ******************************************************************************/ # +#env + set -eu # cd to the directory containing this script DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$DIR" + setwindows_msys() { if [[ $KERNEL == *"windows"* ]]; then export CMAKE_COMMAND="$CMAKE_COMMAND -G \"MSYS Makefiles\"" @@ -52,6 +55,9 @@ fi } export CMAKE_COMMAND="cmake" +if [[ -f "/snap/bin/cmake" ]]; then + export CMAKE_COMMAND="/snap/bin/cmake" +fi if which cmake3 &> /dev/null; then export CMAKE_COMMAND="cmake3" fi @@ -79,6 +85,7 @@ CHIP_EXTENSION= CHIP_VERSION= EXPERIMENTAL= OPERATIONS= +INSTALL_DIR= CLEAN="false" MINIFIER="false" TESTS="false" @@ -94,7 +101,7 @@ value="${2:-}" #Build type (release/debug), packaging type, chip: cpu,cuda, lib type (static/dynamic) case $key in -h|--helper) - HELPER="$value" + HELPER="${HELPER},$value" shift # past argument ;; -o|-platform|--platform) @@ -175,6 +182,8 @@ if [[ $# -gt 0 ]]; then shift # past argument or value fi done + +INSTALL_DIR="$DIR/target/$CHIP/$CHIP_EXTENSION" HOST=$(uname -s | tr [A-Z] [a-z]) KERNEL=$HOST-$(uname -m | tr [A-Z] [a-z]) if [ "$(uname)" == "Darwin" ]; then @@ -349,11 +358,15 @@ case "$OS" in ;; linux*) + export CC="/usr/bin/gcc" + export CMAKE_C_CMAKE="/usr/bin/gcc" + export CXX="/usr/bin/g++" + export CMAKE_CXX_CMAKE="/usr/bin/g++" ;; macosx*) - export CC=clang - export CXX=clang++ + export CC="clang" + export CXX="clang++" PARALLEL="true" export CMAKE_COMMAND="$CMAKE_COMMAND -DCMAKE_MACOSX_RPATH=ON -DSD_APPLE_BUILD=true" ;; @@ -370,23 +383,14 @@ case "$OS" in else export CMAKE_COMMAND="cmake -G \"MSYS Makefiles\"" export MAKE_COMMAND="make" - export CC=/mingw64/bin/gcc - export CXX=/mingw64/bin/g++ + export CC="/mingw64/bin/gcc" + export CXX="/mingw64/bin/g++" PARALLEL="true" fi # Try some defaults for Visual Studio 2013 if user has not run vcvarsall.bat or something if [ -z "${VCINSTALLDIR:-}" ]; then echo "NEED TO SET DEFAULTS FOR VISUAL STUDIO, NO VCINSTALLDIR environment variable found" - export VisualStudioVersion=12.0 - export VSINSTALLDIR="C:\\Program Files (x86)\\Microsoft Visual Studio $VisualStudioVersion" - export VCINSTALLDIR="$VSINSTALLDIR\\VC" - export WindowsSdkDir="C:\\Program Files (x86)\\Windows Kits\\8.1" - export Platform=X64 - export INCLUDE="$VCINSTALLDIR\\INCLUDE;$WindowsSdkDir\\include\\shared;$WindowsSdkDir\\include\\um" - export LIB="$VCINSTALLDIR\\LIB\\amd64;$WindowsSdkDir\\lib\\winv6.3\\um\\x64" - export LIBPATH="$VCINSTALLDIR\\LIB\\amd64;$WindowsSdkDir\\References\\CommonConfiguration\\Neutral" - export PATH="$PATH:$VCINSTALLDIR\\BIN\\amd64:$WindowsSdkDir\\bin\\x64:$WindowsSdkDir\\bin\\x86" fi # Make sure we are using 64-bit MinGW-w64 export PATH=/mingw64/bin/:/mingw64/lib:$PATH @@ -430,9 +434,11 @@ fi if [ -z "$COMPUTE" ]; then if [ "$ARCH" == "x86-64" ]; then - COMPUTE="all" + COMPUTE="5.0 5.2 5.3 6.0 8.0" + #COMPUTE="all" else - COMPUTE="all" + COMPUTE="5.0 5.2 5.3 6.0" + #COMPUTE="all" fi fi @@ -534,6 +540,8 @@ fi [[ -z ${OPENBLAS_PATH:-} ]] && OPENBLAS_PATH="" OPENBLAS_PATH="${OPENBLAS_PATH//\\//}" +#environment variable BUILD_PATH is set by the JavaCPP Builder.class from the platform.buildpath property +#and includes ~/.javacpp/cache/... if [[ -n "${BUILD_PATH:-}" ]]; then PREVIFS="$IFS" IFS="$BUILD_PATH_SEPARATOR" @@ -558,8 +566,8 @@ mkbuilddir() { echo "Removing blasbuild" rm -Rf blasbuild fi - mkdir -p "blasbuild/$CHIP" - cd "blasbuild/$CHIP" + mkdir -p "blasbuild/$CHIP/$CHIP_EXTENSION" + cd "blasbuild/$CHIP/$CHIP_EXTENSION" } HELPERS="" @@ -582,7 +590,7 @@ else IFS=',' read -ra HLP <<< "$HELPER" for i in "${HLP[@]}"; do - HELPERS="${HELPERS} -DHELPERS_$i=true" + HELPERS="${HELPERS} -DHELPERS_$i:BOOL=true" done IFS=' ' fi @@ -600,12 +608,28 @@ echo OPERATIONS = "${OPERATIONS_ARG}" echo MINIFIER = "${MINIFIER_ARG}" echo TESTS = "${TESTS_ARG}" echo NAME = "${NAME_ARG}" +echo CMAKE_COMMAND = "${CMAKE_COMMAND}" +echo MAKE_COMMAND = "${MAKE_COMMAND}" echo OPENBLAS_PATH = "$OPENBLAS_PATH" echo CHECK_VECTORIZATION = "$CHECK_VECTORIZATION" echo HELPERS = "$HELPERS" +echo INSTALL_DIR = "$INSTALL_DIR" mkbuilddir -pwd -eval "$CMAKE_COMMAND" "$BLAS_ARG" "$ARCH_ARG" "$NAME_ARG" -DSD_CHECK_VECTORIZATION="${CHECK_VECTORIZATION}" "$HELPERS" "$SHARED_LIBS_ARG" "$MINIFIER_ARG" "$OPERATIONS_ARG" "$BUILD_TYPE" "$PACKAGING_ARG" "$EXPERIMENTAL_ARG" "$TESTS_ARG" "$CUDA_COMPUTE" -DOPENBLAS_PATH="$OPENBLAS_PATH" -DDEV=FALSE -DCMAKE_NEED_RESPONSE=YES -DMKL_MULTI_THREADED=TRUE ../.. + +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +env +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + + +RUN_COMMAND="$CMAKE_COMMAND $BLAS_ARG $ARCH_ARG $NAME_ARG $HELPERS $SHARED_LIBS_ARG $MINIFIER_ARG $OPERATIONS_ARG $BUILD_TYPE \ + $PACKAGING_ARG $EXPERIMENTAL_ARG $TESTS_ARG $CUDA_COMPUTE \ + -DOPENBLAS_PATH=\"$OPENBLAS_PATH\" -DDEV=FALSE -DCMAKE_NEED_RESPONSE=YES -DMKL_MULTI_THREADED=TRUE \ + -DSD_CHECK_VECTORIZATION=\"${CHECK_VECTORIZATION}\" \ + -DINSTALL_DIR=\"$INSTALL_DIR\" \ + ../../.." + +echo "Running '$RUN_COMMAND'" +eval $RUN_COMMAND if [ "$PARALLEL" == "true" ]; then MAKE_ARGUMENTS="$MAKE_ARGUMENTS -j $MAKEJ" @@ -622,7 +646,7 @@ fi exec 3>&1 eval "$MAKE_COMMAND" "$MAKE_ARGUMENTS" 2>&1 >&3 3>&- | python3 ../../auto_vectorization/auto_vect.py && cd ../../.. -exec 3>&- +exec 3>&- else eval "$MAKE_COMMAND" "$MAKE_ARGUMENTS" && cd ../../.. fi diff --git a/cavis-native/cavis-native-lib/native-nd4j-library.iml b/cavis-native/cavis-native-lib/native-nd4j-library.iml new file mode 100644 index 000000000..5c6fd7064 --- /dev/null +++ b/cavis-native/cavis-native-lib/native-nd4j-library.iml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cavis-native/cavis-native-lib/src/main/cpp/blas/CMakeLists.txt b/cavis-native/cavis-native-lib/src/main/cpp/blas/CMakeLists.txt new file mode 100644 index 000000000..981841fe4 --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/CMakeLists.txt @@ -0,0 +1,489 @@ +################################################################################ +# +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# See the NOTICE file distributed with this work for additional +# information regarding copyright ownership. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + +set(CMAKE_VERBOSE_MAKEFILE ON) + +if(LINUX) + link_directories(/usr/local/lib) + link_directories(/usr/lib) + link_directories(/lib) +endif() + +if(APPLE) + message("Using apple") + link_directories(/usr/local/lib) + link_directories(/usr/lib) + link_directories(/lib) +endif() + +if (SD_APPLE_BUILD) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSD_APPLE_BUILD=true -mmacosx-version-min=10.10") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSD_APPLE_BUILD=true -mmacosx-version-min=10.10") +endif() + +if (SD_ARM_BUILD) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSD_ARM_BUILD=true") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSD_ARM_BUILD=true") +endif() + +if (SD_ANDROID_BUILD) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSD_ANDROID_BUILD=true") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSD_ANDROID_BUILD=true") +endif() + +if (SD_IOS_BUILD) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSD_IOS_BUILD=true") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSD_IOS_BUILD=true") +endif() + +if(WIN32 AND NOT ANDROID) + message("Building for Windows") + get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wa,-mbig-obj") + endif() + foreach(dir ${dirs}) + message(STATUS "dir='${dir}'") + endforeach() + + # workaround for long command lines + SET(CMAKE_C_USE_RESPONSE_FILE_FOR_OBJECTS 1) + SET(CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS 1) + + SET(CMAKE_C_RESPONSE_FILE_LINK_FLAG "@") + SET(CMAKE_CXX_RESPONSE_FILE_LINK_FLAG "@") + + SET(CMAKE_NINJA_FORCE_RESPONSE_FILE 1 CACHE INTERNAL "") +endif() + +if ("${SD_ALL_OPS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSD_ALL_OPS=true") +else() + message("_OPS: ${SD_OPS_LIST}") + foreach(OP "${SD_OPS_LIST}") + message(STATUS "${OP}") + endforeach() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SD_OPS_LIST}") +endif() + +IF(${SD_ARCH} MATCHES "armv8") + message("Building ARM v8 (x64) binary...") + set(ARCH_TUNE "-march=${SD_ARCH}") +ELSEIF(${SD_ARCH} MATCHES "armv7") + message("Building ARM v7 binary...") + set(ARCH_TUNE "-march=${SD_ARCH} -mfpu=neon ") +ELSEIF(${SD_ARCH} MATCHES "power*") + message("Building Power binary...") + set(ARCH_TUNE "-mcpu=${SD_ARCH} -mtune=${SD_ARCH} -D__POWER") +ELSEIF("${SD_ARCH}" STREQUAL "x86-64") + message("Building x86_64 binary...") + set(ARCH_TYPE "generic") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DF_X64=true") +ELSE() + message("Building ${SD_ARCH} binary...") + set(ARCH_TYPE "${SD_ARCH}") +ENDIF() + +IF(${SD_EXTENSION} MATCHES "avx2") + message("Extension AVX2 enabled.") + set(ARCH_TUNE "${ARCH_TUNE} -mmmx -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx -mavx2 -mfma -mf16c -mprefetchwt1 -DSD_F16C=true -DF_AVX2=true") +ELSEIF(${SD_EXTENSION} MATCHES "avx512") + message("Extension AVX512 enabled.") + # we need to set flag here, that we can use hardware f16 conversion + tell that cpu features should be tracked + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmmx -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx -mavx2 -mfma -mf16c -mavx512f -mavx512vl -mavx512bw -mavx512dq -mavx512cd -mbmi -mbmi2 -mprefetchwt1 -mclflushopt -mxsavec -mxsaves -DSD_F16C=true -DF_AVX512=true") +ENDIF() + +if (NOT WIN32) + # we don't want this definition for msvc + set(ARCH_TUNE "-march=${SD_ARCH} -mtune=${ARCH_TYPE}") +endif() + +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang" AND SD_X86_BUILD) + # apple clang but not ios-arm + SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_TUNE}") +elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + # using Clang + SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_TUNE}") +elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") + # using Intel C++ + SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_TUNE} -O3 -fp-model fast") +elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + # using Visual Studio C++ + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_TUNE}") +elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + # using GCC + SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_TUNE} -fmax-errors=2 -fdiagnostics-show-caret ") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,$ORIGIN/") + + if (CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT(APPLE) AND NOT(WIN32)) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -rdynamic -Wl,-export-dynamic") + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -export-dynamic") + endif() +endif() + + +IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + include_directories("/usr/include") + include_directories("/usr/local/include") +ENDIF(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + +if(!SD_CUDA) + if(!SD_CPU) + set(SD_CUDA FALSE) + set(SD_CPU TRUE) + endif() +endif() + +#if MKLDNN is enabled - we're building mkldnn-powered helpers +if (HAVE_MKLDNN) + file(GLOB_RECURSE CUSTOMOPS_MKLDNN_SOURCES false ops/declarable/platform/mkldnn/*.cpp ops/declarable/platform/mkldnn/mkldnnUtils.h) +endif() + +if(HAVE_ARMCOMPUTE) + file(GLOB_RECURSE CUSTOMOPS_ARMCOMPUTE_SOURCES false ops/declarable/platform/armcompute/*.cpp ops/declarable/platform/armcompute/*.h) +endif() + +if(SD_CUDA) + message("Build cublas") + if(NOT DEFINED ${CMAKE_CUDA_ARCHITECTURES}) + set(CMAKE_CUDA_ARCHITECTURES 75) + endif() + message(STATUS "CUDA architectures set to ${CMAKE_CUDA_ARCHITECTURES}") + + find_package(CUDAToolkit) + enable_language(CUDA) + + set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CXX_STANDARD 14) + + add_definitions(-D__CUDABLAS__=true) + #Enable features prio C++17 + add_definitions(-D_HAS_AUTO_PTR_ETC=1) + + #This basically kills instrinsic activated through SD_F16C=true + #if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + # set (CMAKE_CXX_FLAGS "") + #endif() + + if (CUDAToolkit_FOUND) + include_directories(${CUDAToolkit_INCLUDE_DIRS}) + message("CUDA found!") + if ("${SD_EXPERIMENTAL}" STREQUAL "yes") + message("Experimental mode ENABLED") + set(CMAKE_CUDA_FLAGS " ${CMAKE_CUDA_FLAGS} -D__ND4J_EXPERIMENTAL__=true") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__ND4J_EXPERIMENTAL__=true") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__ND4J_EXPERIMENTAL__=true") + set(EXPM " -D__ND4J_EXPERIMENTAL__=true") + endif() + + + + # the only difference for debug mode here is host/device debug symbols + set(CMAKE_CUDA_FLAGS_DEBUG " -G -g") + + # we need -fPIC on Linux/GCC + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + message("Enabling fPIC for GNU compilers...") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=-fPIC") + endif() + + if(WIN32) + message("In windows, setting cublas library and cusolver library") + if(NOT DEFINED CUDA_cublas_LIBRARY) + set(CUDA_cublas_LIBRARY ${CUDA_HOME}/lib/x64/cublas.lib) + endif() + + if(NOT DEFINED CUDA_cusolver_LIBRARY) + set(CUDA_cusolver_LIBRARY ${CUDA_HOME}/lib/x64/cusolver.lib) + endif() + endif() + +# + #string( TOLOWER "${COMPUTE}" COMPUTE_CMP ) +# if ("${COMPUTE_CMP}" STREQUAL "all") +# CUDA_SELECT_NVCC_ARCH_FLAGS(CUDA_ARCH_FLAGS "Common") +# elseif("${COMPUTE_CMP}" STREQUAL "auto") +# CUDA_SELECT_NVCC_ARCH_FLAGS(CUDA_ARCH_FLAGS "Auto") +# elseif(COMPUTE_CMP MATCHES "^[0-9]+$") +# #matches USER COMPUTE old way + #set(CUDA_ARCH_FLAGS "-gencode arch=compute_${COMPUTE},code=sm_${COMPUTE} ") +# else() +# #matches numbers NAME | NUM.NUM | NUM.NUM(NUM.NUM) | NUM.NUM+PTX +# #NAME: Fermi Kepler Maxwell Kepler+Tegra Kepler+Tesla Maxwell+Tegra Pascal +# #NUM: 2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 et cetera +# CUDA_SELECT_NVCC_ARCH_FLAGS(CUDA_ARCH_FLAGS "${COMPUTE}") +# endif() + # list to spaces + #string (REPLACE ";" " " CUDA_ARCH_FLAGS "${CUDA_ARCH_FLAGS}") + + #set(CMAKE_CUDA_FLAGS " ${CMAKE_CUDA_FLAGS} -DCUDA_VERSION_MAJOR=${CUDA_VERSION_MAJOR} ${EXPM} -w --cudart=static --expt-extended-lambda -Xfatbin -compress-all ") + set(CMAKE_CUDA_ARCHITECTURES OFF) + #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --generate-code \"arch=compute_53,code=[compute_53,sm_53]\" " ) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --generate-code \"arch=compute_61,code=[compute_61,sm_61]\" " ) + #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --generate-code \"arch=compute_75,code=[compute_75,sm_75]\" " ) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --extended-lambda ") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr ") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DCUDA_VERSION_MAJOR=11 -w --cudart=static -Xfatbin -compress-all") + if(WIN32) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=/EHsc") + endif() + #set(GPU_ARCH) + + message("CMAKE_CUDA_FLAGS = ${CMAKE_CUDA_FLAGS}") + message("CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}") + message("CMAKE_CUDA_FLAGS_RELEASE = ${CMAKE_CUDA_FLAGS_RELEASE}") + message("CMAKE_CXX_FLAGS_RELEASE = ${CMAKE_CXX_FLAGS_RELEASE}") + message("CMAKE_CUDA_EXTENSIONS = ${CMAKE_CUDA_EXTENSIONS}") + message("CUDA_NVCC_FLAGS = ${CUDA_NVCC_FLAGS}") + message("CUDA_PROPAGATE_HOST_FLAGS = ${CUDA_PROPAGATE_HOST_FLAGS}") + message("CUDA_ARCH_FLAGS = ${CUDA_ARCH_FLAGS}") + + file(GLOB_RECURSE PERF_SOURCES false performance/*.cpp performance/*.h) + file(GLOB_RECURSE EXCEPTIONS_SOURCES false exceptions/*.cpp exceptions/*.h) + file(GLOB_RECURSE EXEC_SOURCES false execution/impl/*.cpp execution/*.cu execution/*.h) + file(GLOB_RECURSE TYPES_SOURCES false types/*.cpp types/*.h) + file(GLOB_RECURSE ARRAY_SOURCES false array/impl/*.cpp array/cuda/*.cu array/*.h) + file(GLOB_RECURSE MEMORY_SOURCES false memory/impl/*.cpp memory/cuda/*.cu memory/*.h) + file(GLOB_RECURSE GRAPH_SOURCES false graph/*.cpp graph/*.cu graph/*.h) + file(GLOB_RECURSE CUSTOMOPS_SOURCES false ops/declarable/generic/*.cpp) + file(GLOB_RECURSE CUSTOMOPS_HELPERS_SOURCES false ops/declarable/helpers/cuda/*.cu ops/declarable/helpers/impl/*.cpp) + file(GLOB_RECURSE OPS_SOURCES false ops/impl/*.cpp ops/declarable/impl/*.cpp ops/*.h) + file(GLOB_RECURSE HELPERS_SOURCES false build_info.cu helpers/impl/*.cpp helpers/*.cu helpers/*.cupp helpers/*.h) + file(GLOB_RECURSE INDEXING_SOURCES false indexing/*.cpp indexing/*.h) + file(GLOB_RECURSE LOOPS_SOURCES false ./loops/impl/*.cpp ./loops/*.h) + file(GLOB_RECURSE LEGACY_SOURCES false legacy/impl/*.cpp legacy/*.cu legacy/*.h) + file(GLOB_RECURSE LOOPS_SOURCES_CUDA false loops/cuda/*.cu) + + + file(GLOB_RECURSE COMPILATION_UNITS false loops/cuda/compilation_units/*.cu.in + ops/impl/compilation_units/*.cpp.in) + + foreach(FL_ITEM ${COMPILATION_UNITS}) + genCompilation(FL_ITEM) + endforeach() + + if (HAVE_CUDNN) + message("cuDNN included") + file(GLOB_RECURSE CUSTOMOPS_CUDNN_SOURCES false ops/declarable/platform/cudnn/*.cu) + else() + message("cuDNN not included") + endif() + + add_library(samediff_obj OBJECT ${LOOPS_SOURCES_CUDA} ${LEGACY_SOURCES} + ${CUSTOMOPS_HELPERS_SOURCES} ${HELPERS_SOURCES} ${EXEC_SOURCES} + ${LOOPS_SOURCES} ${ARRAY_SOURCES} ${TYPES_SOURCES} + ${MEMORY_SOURCES} ${GRAPH_SOURCES} ${CUSTOMOPS_SOURCES} ${INDEXING_SOURCES} ${EXCEPTIONS_SOURCES} ${OPS_SOURCES} ${PERF_SOURCES} ${CUSTOMOPS_CUDNN_SOURCES} ${CUSTOMOPS_MKLDNN_SOURCES} + ${CUSTOMOPS_ARMCOMPUTE_SOURCES} ${CUSTOMOPS_GENERIC_SOURCES}) + + include_directories(samediff_obj PUBLIC .) + + if (WIN32) + message("MSVC runtime for library: ${MSVC_RT_LIB}") + endif() + + # build shared library by default or when it's explicitly requested + if(NOT SD_STATIC_LIB OR SD_SHARED_LIB) + add_library(${SD_LIBRARY_NAME} SHARED $) + endif() + + if (SD_STATIC_LIB AND SD_SHARED_LIB) + # if both static and shared library are going to be built - static library will have special suffix + add_library(${SD_LIBRARY_NAME}static STATIC $) + set_property(TARGET ${SD_LIBRARY_NAME}static PROPERTY MSVC_RUNTIME_LIBRARY "${MSVC_RT_LIB}$<$:Debug>") + install(TARGETS ${SD_LIBRARY_NAME}static DESTINATION .) + elseif(SD_STATIC_LIB) + # if we only build static library - use this name + add_library(${SD_LIBRARY_NAME} STATIC $) + set_property(TARGET ${SD_LIBRARY_NAME} PROPERTY MSVC_RUNTIME_LIBRARY "${MSVC_RT_LIB}$<$:Debug>") + install(TARGETS ${SD_LIBRARY_NAME} DESTINATION .) + endif() + + # on windows we want to make sure we use MT or MD, but since we use it in one lib, we must use it everywhere to avoid conflicts + set_property(TARGET samediff_obj PROPERTY MSVC_RUNTIME_LIBRARY "${MSVC_RT_LIB}$<$:Debug>") + set_property(TARGET ${SD_LIBRARY_NAME} PROPERTY MSVC_RUNTIME_LIBRARY "${MSVC_RT_LIB}$<$:Debug>") + +# Done by nvcc as default on windows + if(WIN32) + message("CUDA on Windows: enabling /EHsc") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /bigobj") + endif() + + #target_link_libraries(${SD_LIBRARY_NAME} ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_cusolver_LIBRARY} ${CUDNN} ${MKLDNN}) + target_link_libraries(${SD_LIBRARY_NAME} CUDA::cudart CUDA::cublas CUDA::cusolver ${CUDNN} ${MKLDNN}) + + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/cuda/${SD_EXTENSION}) + install(TARGETS ${SD_LIBRARY_NAME} DESTINATION .) + endif(CUDAToolkit_FOUND) + + + + +elseif(SD_CPU) + + if ("${SD_EXPERIMENTAL}" STREQUAL "yes") + message("Experimental mode ENABLED") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__ND4J_EXPERIMENTAL__=true") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__ND4J_EXPERIMENTAL__=true") + endif() + + file(GLOB_RECURSE PERF_SOURCES false performance/*.cpp performance/*.h) + file(GLOB_RECURSE EXCEPTIONS_SOURCES false exceptions/*.cpp exceptions/*.h) + file(GLOB_RECURSE EXEC_SOURCES false execution/*.cpp execution/*.h) + file(GLOB_RECURSE TYPES_SOURCES false types/*.cpp types/*.h) + file(GLOB_RECURSE ARRAY_SOURCES false array/*.cpp array/*.h) + file(GLOB_RECURSE MEMORY_SOURCES false memory/*.cpp memory/*.h) + file(GLOB_RECURSE GRAPH_SOURCES false graph/*.cpp graph/*.h) + file(GLOB_RECURSE CUSTOMOPS_SOURCES false ops/declarable/generic/*.cpp) + file(GLOB_RECURSE CUSTOMOPS_GENERIC_SOURCES false ops/declarable/helpers/cpu/*.cpp ops/declarable/helpers/impl/*.cpp) + file(GLOB_RECURSE OPS_SOURCES false ops/impl/*.cpp ops/declarable/impl/*.cpp ops/*.h) + file(GLOB_RECURSE INDEXING_SOURCES false indexing/*.cpp indexing/*.h) + file(GLOB_RECURSE HELPERS_SOURCES false build_info.cpp helpers/*.cpp helpers/*.h) + file(GLOB_RECURSE LEGACY_SOURCES false legacy/impl/*.cpp legacy/cpu/*.cpp ./legacy/*.h system/*.h) + file(GLOB_RECURSE LOOPS_SOURCES false loops/*.cpp loops/*.h) + + + file(GLOB_RECURSE COMPILATION_UNITS false ops/declarable/helpers/cpu/compilation_units/*.cpp.in + loops/cpu/compilation_units/*.cpp.in helpers/cpu/loops/*.cpp.in + ops/impl/compilation_units/*.cpp.in) + + foreach(FL_ITEM ${COMPILATION_UNITS}) + genCompilation(FL_ITEM) + endforeach() + + if (SD_X86_BUILD) + # we disable platform optimizations for certains files for linux/macos + set_source_files_properties(cpu/NativeOps.cpp PROPERTIES COMPILE_FLAGS "-march=x86-64 -mtune=generic") + set_source_files_properties(../include/helpers/impl/OpTracker.cpp PROPERTIES COMPILE_FLAGS "-march=x86-64 -mtune=generic") + endif() + + + + if(SD_CHECK_VECTORIZATION) + set(VECT_FILES cpu/NativeOps.cpp ${OPS_SOURCES} ${HELPERS_SOURCES} ${CUSTOMOPS_GENERIC_SOURCES} ${LOOPS_SOURCES}) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + + if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0) + set(CHECK_VECT_FLAGS "-ftree-vectorize -fsave-optimization-record") + #to process fsave-optimization-record we will need our cython version code + message("Build Auto vectorization helpers") + execute_process(COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../auto_vectorization/cython_setup.py" "build_ext" "--inplace" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../auto_vectorization/" RESULT_VARIABLE ret) + message("build='${ret}'") + + #remove fail cases that gcc fails produce sometimes + file(GLOB_RECURSE FAILURE_CASES false loops/cpu/compilation_units/reduce3*.cpp) + #message("*****${FAILURE_CASES}") + foreach(FL_ITEM ${FAILURE_CASES}) + message("Removing failure cases ${FL_ITEM}") + list(REMOVE_ITEM VECT_FILES ${FL_ITEM}) + endforeach() + else() + set(CHECK_VECT_FLAGS "-ftree-vectorize -fopt-info-vec-optimized-missed") + endif() + message("CHECK VECTORIZATION ${CHECK_VECT_FLAGS}") + set_source_files_properties( ${VECT_FILES} PROPERTIES COMPILE_FLAGS "${CHECK_VECT_FLAGS}" ) + endif() + endif() + + message("Build native CPU BLAS") + add_definitions(-D__CPUBLAS__=true) + + add_library(samediff_obj OBJECT ${LEGACY_SOURCES} + ${LOOPS_SOURCES} ${HELPERS_SOURCES} ${EXEC_SOURCES} ${ARRAY_SOURCES} ${TYPES_SOURCES} + ${MEMORY_SOURCES} ${GRAPH_SOURCES} ${CUSTOMOPS_SOURCES} ${EXCEPTIONS_SOURCES} ${INDEXING_SOURCES} ${CUSTOMOPS_MKLDNN_SOURCES} + ${CUSTOMOPS_ARMCOMPUTE_SOURCES} ${CUSTOMOPS_GENERIC_SOURCES} ${OPS_SOURCES} ${PERF_SOURCES}) + #target_include_directories(samediff_obj PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + target_include_directories(samediff_obj PUBLIC ./) + + if(IOS) + message("Building static library for IOS ${SD_LIBRARY_NAME}") + add_library(${SD_LIBRARY_NAME} STATIC $) + else() + # build shared library by default or when it's explicitly requested + if(NOT SD_STATIC_LIB OR SD_SHARED_LIB) + message("Building a shared library for ${SD_LIBRARY_NAME}") + add_library(${SD_LIBRARY_NAME} SHARED $) + + #set_target_properties(${SD_LIBRARY_NAME} PROPERTIES IMPORT_SUFFIX ".lib") + #target_link_libraries(${SD_LIBRARY_NAME} $) + + if(ANDROID) + # See: https://www.scivision.dev/cmake-ninja-job-pool-limited-memory/ + # See: https://cmake.org/cmake/help/v3.0/command/cmake_host_system_information.html + # See: https://cmake.org/cmake/help/latest/prop_gbl/JOB_POOLS.html + cmake_host_system_information(RESULT _logical_cores QUERY NUMBER_OF_LOGICAL_CORES) + if(_logical_cores LESS 4) + set_target_properties(${SD_LIBRARY_NAME} PROPERTIES JOB_POOL_COMPILE one_jobs) + endif() + endif() + endif() + + if (SD_STATIC_LIB AND SD_SHARED_LIB) + # if both static and shared library are going to be built - static library will have special suffix + message("Adding a static library for ${SD_LIBRARY_NAME} as ${SD_LIBRARY_NAME}static") + add_library(${SD_LIBRARY_NAME}static STATIC $) + set_property(TARGET ${SD_LIBRARY_NAME}static PROPERTY MSVC_RUNTIME_LIBRARY "${MSVC_RT_LIB}$<$:Debug>") + install(TARGETS ${SD_LIBRARY_NAME}static DESTINATION .) + elseif(SD_STATIC_LIB) + # if we only build static library - use this name + message(Only building a static library for ${SD_LIBRARY_NAME}) + add_library(${SD_LIBRARY_NAME} STATIC $) + set_property(TARGET ${SD_LIBRARY_NAME} PROPERTY MSVC_RUNTIME_LIBRARY "${MSVC_RT_LIB}$<$:Debug>") + install(TARGETS ${SD_LIBRARY_NAME} DESTINATION .) + endif() + endif() + + # we're including {MKLDNN} here in case of building from sources. in future that'll replace {MKLDNN_LIBRARIES}. same applies to BLAS + if (NOT BLAS_LIBRARIES) + set(BLAS_LIBRARIES "") + endif() + get_cmake_property(_variableNames VARIABLES) + list (SORT _variableNames) + foreach (_variableName ${_variableNames}) + message(STATUS "${_variableName}=${${_variableName}}") + endforeach() + + #This breaks the build. Normally you want to run tests anyways. + if(NOT "$ENV{CLION_IDE}") + target_link_libraries(${SD_LIBRARY_NAME} ${MKLDNN} ${MKLDNN_LIBRARIES} ${ARMCOMPUTE_LIBRARIES} ${OPENBLAS_LIBRARIES} ${BLAS_LIBRARIES} ${CPU_FEATURES}) + endif() + + if ("${SD_ALL_OPS}" AND "${SD_BUILD_MINIFIER}") + message(STATUS "Building minifier...") + add_executable(minifier ../minifier/minifier.cpp ../minifier/graphopt.cpp) + target_link_libraries(minifier samediff_obj ${MKLDNN_LIBRARIES} ${ARMCOMPUTE_LIBRARIES} ${OPENBLAS_LIBRARIES} ${MKLDNN} ${BLAS_LIBRARIES} ${CPU_FEATURES}) + endif() + + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS 4.9) + message(FATAL_ERROR "You need at least GCC 4.9") + endif() + + # OpenMP works well pretty much only with GCC + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + find_package(OpenMP) + if (OPENMP_FOUND) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + endif() + endif() + + message("Installing ${SD_LIBRARY_NAME}") + install(TARGETS ${SD_LIBRARY_NAME} DESTINATION .) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/cpu/${SD_EXTENSION}/) + +endif() diff --git a/cavis-native/cavis-native-lib/src/main/cpp/blas/CMakeSettings.json b/cavis-native/cavis-native-lib/src/main/cpp/blas/CMakeSettings.json new file mode 100644 index 000000000..9204f06eb --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/CMakeSettings.json @@ -0,0 +1,15 @@ +{ + "configurations": [ + { + "name": "x64-Debug", + "generator": "Ninja", + "configurationType": "Debug", + "inheritEnvironments": [ "msvc_x64_x64" ], + "buildRoot": "${projectDir}\\out\\build\\${name}", + "installRoot": "${projectDir}\\out\\install\\${name}", + "cmakeCommandArgs": "", + "buildCommandArgs": "", + "ctestCommandArgs": "" + } + ] +} \ No newline at end of file diff --git a/libnd4j/include/array/ArrayOptions.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ArrayOptions.h similarity index 100% rename from libnd4j/include/array/ArrayOptions.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ArrayOptions.h diff --git a/libnd4j/include/array/ArrayType.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ArrayType.h similarity index 100% rename from libnd4j/include/array/ArrayType.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ArrayType.h diff --git a/libnd4j/include/array/ByteOrder.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ByteOrder.h similarity index 100% rename from libnd4j/include/array/ByteOrder.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ByteOrder.h diff --git a/libnd4j/include/array/ByteOrderUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ByteOrderUtils.h similarity index 100% rename from libnd4j/include/array/ByteOrderUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ByteOrderUtils.h diff --git a/libnd4j/include/array/ConstantDataBuffer.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantDataBuffer.h similarity index 100% rename from libnd4j/include/array/ConstantDataBuffer.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantDataBuffer.h diff --git a/libnd4j/include/array/ConstantDescriptor.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantDescriptor.h similarity index 100% rename from libnd4j/include/array/ConstantDescriptor.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantDescriptor.h diff --git a/libnd4j/include/array/ConstantHolder.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantHolder.h similarity index 100% rename from libnd4j/include/array/ConstantHolder.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantHolder.h diff --git a/libnd4j/include/array/ConstantOffsetsBuffer.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantOffsetsBuffer.h similarity index 100% rename from libnd4j/include/array/ConstantOffsetsBuffer.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantOffsetsBuffer.h diff --git a/libnd4j/include/array/ConstantShapeBuffer.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantShapeBuffer.h similarity index 100% rename from libnd4j/include/array/ConstantShapeBuffer.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ConstantShapeBuffer.h diff --git a/libnd4j/include/array/CudaPointerDeallocator.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/CudaPointerDeallocator.h similarity index 100% rename from libnd4j/include/array/CudaPointerDeallocator.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/CudaPointerDeallocator.h diff --git a/libnd4j/include/array/DataBuffer.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/DataBuffer.h similarity index 100% rename from libnd4j/include/array/DataBuffer.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/DataBuffer.h diff --git a/libnd4j/include/array/DataType.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/DataType.h similarity index 100% rename from libnd4j/include/array/DataType.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/DataType.h diff --git a/libnd4j/include/array/DataTypeConversions.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/DataTypeConversions.h similarity index 100% rename from libnd4j/include/array/DataTypeConversions.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/DataTypeConversions.h diff --git a/libnd4j/include/array/DataTypeUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/DataTypeUtils.h similarity index 100% rename from libnd4j/include/array/DataTypeUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/DataTypeUtils.h diff --git a/libnd4j/include/array/ExtraArguments.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ExtraArguments.h similarity index 100% rename from libnd4j/include/array/ExtraArguments.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ExtraArguments.h diff --git a/libnd4j/include/array/InteropDataBuffer.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/InteropDataBuffer.h similarity index 100% rename from libnd4j/include/array/InteropDataBuffer.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/InteropDataBuffer.h diff --git a/libnd4j/include/array/NDArray.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArray.h similarity index 100% rename from libnd4j/include/array/NDArray.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArray.h diff --git a/libnd4j/include/array/NDArray.hXX b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArray.hXX similarity index 100% rename from libnd4j/include/array/NDArray.hXX rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArray.hXX diff --git a/libnd4j/include/array/NDArrayFactory.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArrayFactory.h similarity index 100% rename from libnd4j/include/array/NDArrayFactory.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArrayFactory.h diff --git a/libnd4j/include/array/NDArrayLambda.hXX b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArrayLambda.hXX similarity index 100% rename from libnd4j/include/array/NDArrayLambda.hXX rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArrayLambda.hXX diff --git a/libnd4j/include/array/NDArrayList.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArrayList.h similarity index 100% rename from libnd4j/include/array/NDArrayList.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/NDArrayList.h diff --git a/libnd4j/include/array/PointerDeallocator.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/PointerDeallocator.h similarity index 100% rename from libnd4j/include/array/PointerDeallocator.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/PointerDeallocator.h diff --git a/libnd4j/include/array/PointerWrapper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/PointerWrapper.h similarity index 100% rename from libnd4j/include/array/PointerWrapper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/PointerWrapper.h diff --git a/libnd4j/include/array/PrimaryPointerDeallocator.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/PrimaryPointerDeallocator.h similarity index 100% rename from libnd4j/include/array/PrimaryPointerDeallocator.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/PrimaryPointerDeallocator.h diff --git a/libnd4j/include/array/ResultSet.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ResultSet.h similarity index 100% rename from libnd4j/include/array/ResultSet.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ResultSet.h diff --git a/libnd4j/include/array/ShapeDescriptor.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ShapeDescriptor.h similarity index 100% rename from libnd4j/include/array/ShapeDescriptor.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ShapeDescriptor.h diff --git a/libnd4j/include/array/ShapeList.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/ShapeList.h similarity index 100% rename from libnd4j/include/array/ShapeList.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/ShapeList.h diff --git a/libnd4j/include/array/SpaceType.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/SpaceType.h similarity index 100% rename from libnd4j/include/array/SpaceType.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/SpaceType.h diff --git a/libnd4j/include/array/SparseType.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/SparseType.h similarity index 100% rename from libnd4j/include/array/SparseType.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/SparseType.h diff --git a/libnd4j/include/array/TadDescriptor.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/TadDescriptor.h similarity index 100% rename from libnd4j/include/array/TadDescriptor.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/TadDescriptor.h diff --git a/libnd4j/include/array/TadPack.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/TadPack.h similarity index 100% rename from libnd4j/include/array/TadPack.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/TadPack.h diff --git a/libnd4j/include/array/cpu/DataBuffer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/cpu/DataBuffer.cpp similarity index 100% rename from libnd4j/include/array/cpu/DataBuffer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/cpu/DataBuffer.cpp diff --git a/libnd4j/include/array/cpu/NDArray.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/cpu/NDArray.cpp similarity index 100% rename from libnd4j/include/array/cpu/NDArray.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/cpu/NDArray.cpp diff --git a/libnd4j/include/array/cpu/NDArray.macro b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/cpu/NDArray.macro similarity index 100% rename from libnd4j/include/array/cpu/NDArray.macro rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/cpu/NDArray.macro diff --git a/libnd4j/include/array/cpu/NDArrayLambda.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/cpu/NDArrayLambda.hpp similarity index 100% rename from libnd4j/include/array/cpu/NDArrayLambda.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/cpu/NDArrayLambda.hpp diff --git a/libnd4j/include/array/cuda/CudaPointerDeallocator.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/cuda/CudaPointerDeallocator.cu similarity index 100% rename from libnd4j/include/array/cuda/CudaPointerDeallocator.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/cuda/CudaPointerDeallocator.cu diff --git a/libnd4j/include/array/cuda/DataBuffer.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/cuda/DataBuffer.cu similarity index 100% rename from libnd4j/include/array/cuda/DataBuffer.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/cuda/DataBuffer.cu diff --git a/libnd4j/include/array/cuda/NDArray.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/cuda/NDArray.cu similarity index 100% rename from libnd4j/include/array/cuda/NDArray.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/cuda/NDArray.cu diff --git a/libnd4j/include/array/impl/ByteOrderUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ByteOrderUtils.cpp similarity index 100% rename from libnd4j/include/array/impl/ByteOrderUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ByteOrderUtils.cpp diff --git a/libnd4j/include/array/impl/ConstantDataBuffer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantDataBuffer.cpp similarity index 100% rename from libnd4j/include/array/impl/ConstantDataBuffer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantDataBuffer.cpp diff --git a/libnd4j/include/array/impl/ConstantDescriptor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantDescriptor.cpp similarity index 100% rename from libnd4j/include/array/impl/ConstantDescriptor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantDescriptor.cpp diff --git a/libnd4j/include/array/impl/ConstantHolder.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantHolder.cpp similarity index 100% rename from libnd4j/include/array/impl/ConstantHolder.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantHolder.cpp diff --git a/libnd4j/include/array/impl/ConstantOffsetsBuffer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantOffsetsBuffer.cpp similarity index 100% rename from libnd4j/include/array/impl/ConstantOffsetsBuffer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantOffsetsBuffer.cpp diff --git a/libnd4j/include/array/impl/ConstantShapeBuffer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantShapeBuffer.cpp similarity index 100% rename from libnd4j/include/array/impl/ConstantShapeBuffer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ConstantShapeBuffer.cpp diff --git a/libnd4j/include/array/impl/DataBuffer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/DataBuffer.cpp similarity index 100% rename from libnd4j/include/array/impl/DataBuffer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/DataBuffer.cpp diff --git a/libnd4j/include/array/impl/DataTypeUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/DataTypeUtils.cpp similarity index 100% rename from libnd4j/include/array/impl/DataTypeUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/DataTypeUtils.cpp diff --git a/libnd4j/include/array/impl/ExtraArguments.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ExtraArguments.cpp similarity index 100% rename from libnd4j/include/array/impl/ExtraArguments.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ExtraArguments.cpp diff --git a/libnd4j/include/array/impl/InteropDataBuffer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/InteropDataBuffer.cpp similarity index 100% rename from libnd4j/include/array/impl/InteropDataBuffer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/InteropDataBuffer.cpp diff --git a/libnd4j/include/array/impl/NDArrayFactory.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/NDArrayFactory.cpp similarity index 100% rename from libnd4j/include/array/impl/NDArrayFactory.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/NDArrayFactory.cpp diff --git a/libnd4j/include/array/impl/NDArrayList.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/NDArrayList.cpp similarity index 100% rename from libnd4j/include/array/impl/NDArrayList.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/NDArrayList.cpp diff --git a/libnd4j/include/array/impl/PointerDeallocator.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/PointerDeallocator.cpp similarity index 100% rename from libnd4j/include/array/impl/PointerDeallocator.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/PointerDeallocator.cpp diff --git a/libnd4j/include/array/impl/PointerWrapper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/PointerWrapper.cpp similarity index 100% rename from libnd4j/include/array/impl/PointerWrapper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/PointerWrapper.cpp diff --git a/libnd4j/include/array/impl/PrimaryPointerDeallocator.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/PrimaryPointerDeallocator.cpp similarity index 100% rename from libnd4j/include/array/impl/PrimaryPointerDeallocator.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/PrimaryPointerDeallocator.cpp diff --git a/libnd4j/include/array/impl/ResultSet.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ResultSet.cpp similarity index 100% rename from libnd4j/include/array/impl/ResultSet.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ResultSet.cpp diff --git a/libnd4j/include/array/impl/ShapeDescriptor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ShapeDescriptor.cpp similarity index 100% rename from libnd4j/include/array/impl/ShapeDescriptor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ShapeDescriptor.cpp diff --git a/libnd4j/include/array/impl/ShapeList.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ShapeList.cpp similarity index 100% rename from libnd4j/include/array/impl/ShapeList.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/ShapeList.cpp diff --git a/libnd4j/include/array/impl/TadDescriptor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/TadDescriptor.cpp similarity index 100% rename from libnd4j/include/array/impl/TadDescriptor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/TadDescriptor.cpp diff --git a/libnd4j/include/array/impl/TadPack.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/TadPack.cpp similarity index 100% rename from libnd4j/include/array/impl/TadPack.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/array/impl/TadPack.cpp diff --git a/libnd4j/include/build_info.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/build_info.cpp similarity index 100% rename from libnd4j/include/build_info.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/build_info.cpp diff --git a/libnd4j/include/build_info.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/build_info.cu similarity index 100% rename from libnd4j/include/build_info.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/build_info.cu diff --git a/libnd4j/include/build_info.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/build_info.h similarity index 96% rename from libnd4j/include/build_info.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/build_info.h index 2fd2f5c9e..51622c9fb 100644 --- a/libnd4j/include/build_info.h +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/build_info.h @@ -20,7 +20,7 @@ #define LIBND4J_BUILD_INFO_H #ifdef _WIN32 -#define ND4J_EXPORT __declspec( dllexport ) +#define ND4J_EXPORT __declspec( dllexport ) #else #define ND4J_EXPORT #endif diff --git a/libnd4j/include/cblas.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/cblas.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/cblas.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/cblas.h diff --git a/libnd4j/include/cblas_enum_conversion.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/cblas_enum_conversion.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/cblas_enum_conversion.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/cblas_enum_conversion.h diff --git a/libnd4j/include/cnpy/LICENSE b/cavis-native/cavis-native-lib/src/main/cpp/blas/cnpy/LICENSE similarity index 100% rename from libnd4j/include/cnpy/LICENSE rename to cavis-native/cavis-native-lib/src/main/cpp/blas/cnpy/LICENSE diff --git a/libnd4j/include/cnpy/README b/cavis-native/cavis-native-lib/src/main/cpp/blas/cnpy/README similarity index 100% rename from libnd4j/include/cnpy/README rename to cavis-native/cavis-native-lib/src/main/cpp/blas/cnpy/README diff --git a/libnd4j/include/cnpy/cnpy.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/cnpy/cnpy.h similarity index 99% rename from libnd4j/include/cnpy/cnpy.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/cnpy/cnpy.h index c84623599..96e87a9c0 100644 --- a/libnd4j/include/cnpy/cnpy.h +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/cnpy/cnpy.h @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include #include diff --git a/libnd4j/include/exceptions/allocation_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/allocation_exception.h similarity index 100% rename from libnd4j/include/exceptions/allocation_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/allocation_exception.h diff --git a/libnd4j/include/exceptions/cuda_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/cuda_exception.h similarity index 100% rename from libnd4j/include/exceptions/cuda_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/cuda_exception.h diff --git a/libnd4j/include/exceptions/datatype_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/datatype_exception.h similarity index 100% rename from libnd4j/include/exceptions/datatype_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/datatype_exception.h diff --git a/libnd4j/include/exceptions/graph_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/graph_exception.h similarity index 100% rename from libnd4j/include/exceptions/graph_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/graph_exception.h diff --git a/libnd4j/include/exceptions/graph_execution_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/graph_execution_exception.h similarity index 100% rename from libnd4j/include/exceptions/graph_execution_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/graph_execution_exception.h diff --git a/libnd4j/include/exceptions/graph_exists_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/graph_exists_exception.h similarity index 100% rename from libnd4j/include/exceptions/graph_exists_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/graph_exists_exception.h diff --git a/libnd4j/include/exceptions/impl/allocation_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/allocation_exception.cpp similarity index 100% rename from libnd4j/include/exceptions/impl/allocation_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/allocation_exception.cpp diff --git a/libnd4j/include/exceptions/impl/cuda_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/cuda_exception.cpp similarity index 100% rename from libnd4j/include/exceptions/impl/cuda_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/cuda_exception.cpp diff --git a/libnd4j/include/exceptions/impl/datatype_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/datatype_exception.cpp similarity index 100% rename from libnd4j/include/exceptions/impl/datatype_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/datatype_exception.cpp diff --git a/libnd4j/include/exceptions/impl/graph_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/graph_exception.cpp similarity index 100% rename from libnd4j/include/exceptions/impl/graph_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/graph_exception.cpp diff --git a/libnd4j/include/exceptions/impl/graph_execution_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/graph_execution_exception.cpp similarity index 100% rename from libnd4j/include/exceptions/impl/graph_execution_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/graph_execution_exception.cpp diff --git a/libnd4j/include/exceptions/impl/graph_exists_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/graph_exists_exception.cpp similarity index 100% rename from libnd4j/include/exceptions/impl/graph_exists_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/graph_exists_exception.cpp diff --git a/libnd4j/include/exceptions/impl/no_results_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/no_results_exception.cpp similarity index 100% rename from libnd4j/include/exceptions/impl/no_results_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/no_results_exception.cpp diff --git a/libnd4j/include/exceptions/impl/unknown_graph_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/unknown_graph_exception.cpp similarity index 100% rename from libnd4j/include/exceptions/impl/unknown_graph_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/impl/unknown_graph_exception.cpp diff --git a/libnd4j/include/exceptions/no_results_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/no_results_exception.h similarity index 100% rename from libnd4j/include/exceptions/no_results_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/no_results_exception.h diff --git a/libnd4j/include/exceptions/unknown_graph_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/unknown_graph_exception.h similarity index 100% rename from libnd4j/include/exceptions/unknown_graph_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/exceptions/unknown_graph_exception.h diff --git a/libnd4j/include/execution/AffinityManager.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/AffinityManager.h similarity index 100% rename from libnd4j/include/execution/AffinityManager.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/AffinityManager.h diff --git a/libnd4j/include/execution/BlockingQueue.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/BlockingQueue.h similarity index 100% rename from libnd4j/include/execution/BlockingQueue.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/BlockingQueue.h diff --git a/libnd4j/include/execution/CallableInterface.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/CallableInterface.h similarity index 100% rename from libnd4j/include/execution/CallableInterface.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/CallableInterface.h diff --git a/libnd4j/include/execution/CallableWithArguments.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/CallableWithArguments.h similarity index 100% rename from libnd4j/include/execution/CallableWithArguments.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/CallableWithArguments.h diff --git a/libnd4j/include/execution/ContextBuffers.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/ContextBuffers.h similarity index 100% rename from libnd4j/include/execution/ContextBuffers.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/ContextBuffers.h diff --git a/libnd4j/include/execution/Engine.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/Engine.h similarity index 100% rename from libnd4j/include/execution/Engine.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/Engine.h diff --git a/libnd4j/include/execution/ErrorReference.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/ErrorReference.h similarity index 100% rename from libnd4j/include/execution/ErrorReference.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/ErrorReference.h diff --git a/libnd4j/include/execution/ExecutionMode.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/ExecutionMode.h similarity index 100% rename from libnd4j/include/execution/ExecutionMode.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/ExecutionMode.h diff --git a/libnd4j/include/execution/Executor.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/Executor.h similarity index 100% rename from libnd4j/include/execution/Executor.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/Executor.h diff --git a/libnd4j/include/execution/LaunchContext.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/LaunchContext.h similarity index 100% rename from libnd4j/include/execution/LaunchContext.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/LaunchContext.h diff --git a/libnd4j/include/execution/ThreadPool.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/ThreadPool.h similarity index 100% rename from libnd4j/include/execution/ThreadPool.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/ThreadPool.h diff --git a/libnd4j/include/execution/Threads.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/Threads.h similarity index 100% rename from libnd4j/include/execution/Threads.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/Threads.h diff --git a/libnd4j/include/execution/Ticket.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/Ticket.h similarity index 100% rename from libnd4j/include/execution/Ticket.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/Ticket.h diff --git a/libnd4j/include/execution/cpu/AffinityManager.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cpu/AffinityManager.cpp similarity index 100% rename from libnd4j/include/execution/cpu/AffinityManager.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cpu/AffinityManager.cpp diff --git a/libnd4j/include/execution/cpu/ContextBuffers.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cpu/ContextBuffers.cpp similarity index 100% rename from libnd4j/include/execution/cpu/ContextBuffers.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cpu/ContextBuffers.cpp diff --git a/libnd4j/include/execution/cpu/LaunchContext.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cpu/LaunchContext.cpp similarity index 100% rename from libnd4j/include/execution/cpu/LaunchContext.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cpu/LaunchContext.cpp diff --git a/libnd4j/include/execution/cuda/AffinityManager.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cuda/AffinityManager.cu similarity index 100% rename from libnd4j/include/execution/cuda/AffinityManager.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cuda/AffinityManager.cu diff --git a/libnd4j/include/execution/cuda/ContextBuffers.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cuda/ContextBuffers.cu similarity index 100% rename from libnd4j/include/execution/cuda/ContextBuffers.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cuda/ContextBuffers.cu diff --git a/libnd4j/include/execution/cuda/LaunchContext.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cuda/LaunchContext.cu similarity index 100% rename from libnd4j/include/execution/cuda/LaunchContext.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/cuda/LaunchContext.cu diff --git a/libnd4j/include/execution/impl/BlockingQueue.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/BlockingQueue.cpp similarity index 100% rename from libnd4j/include/execution/impl/BlockingQueue.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/BlockingQueue.cpp diff --git a/libnd4j/include/execution/impl/CallableInterface.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/CallableInterface.cpp similarity index 100% rename from libnd4j/include/execution/impl/CallableInterface.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/CallableInterface.cpp diff --git a/libnd4j/include/execution/impl/CallableWithArguments.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/CallableWithArguments.cpp similarity index 100% rename from libnd4j/include/execution/impl/CallableWithArguments.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/CallableWithArguments.cpp diff --git a/libnd4j/include/execution/impl/ErrorReference.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/ErrorReference.cpp similarity index 100% rename from libnd4j/include/execution/impl/ErrorReference.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/ErrorReference.cpp diff --git a/libnd4j/include/execution/impl/ThreadPool.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/ThreadPool.cpp similarity index 100% rename from libnd4j/include/execution/impl/ThreadPool.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/ThreadPool.cpp diff --git a/libnd4j/include/execution/impl/Threads.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/Threads.cpp similarity index 100% rename from libnd4j/include/execution/impl/Threads.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/Threads.cpp diff --git a/libnd4j/include/execution/impl/Ticket.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/Ticket.cpp similarity index 100% rename from libnd4j/include/execution/impl/Ticket.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/execution/impl/Ticket.cpp diff --git a/libnd4j/include/graph/ArgumentsList.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ArgumentsList.h similarity index 100% rename from libnd4j/include/graph/ArgumentsList.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ArgumentsList.h diff --git a/libnd4j/include/graph/Context.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Context.h similarity index 100% rename from libnd4j/include/graph/Context.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Context.h diff --git a/libnd4j/include/graph/ContextPrototype.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ContextPrototype.h similarity index 100% rename from libnd4j/include/graph/ContextPrototype.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ContextPrototype.h diff --git a/libnd4j/include/graph/ExecutionResult.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ExecutionResult.h similarity index 100% rename from libnd4j/include/graph/ExecutionResult.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ExecutionResult.h diff --git a/libnd4j/include/graph/ExecutorConfiguration.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ExecutorConfiguration.h similarity index 100% rename from libnd4j/include/graph/ExecutorConfiguration.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ExecutorConfiguration.h diff --git a/libnd4j/include/graph/FlatUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/FlatUtils.h similarity index 100% rename from libnd4j/include/graph/FlatUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/FlatUtils.h diff --git a/libnd4j/include/graph/FlowPath.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/FlowPath.h similarity index 100% rename from libnd4j/include/graph/FlowPath.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/FlowPath.h diff --git a/libnd4j/include/graph/FrameState.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/FrameState.h similarity index 100% rename from libnd4j/include/graph/FrameState.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/FrameState.h diff --git a/libnd4j/include/graph/Graph.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Graph.h similarity index 100% rename from libnd4j/include/graph/Graph.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Graph.h diff --git a/libnd4j/include/graph/GraphExecutioner.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/GraphExecutioner.h similarity index 100% rename from libnd4j/include/graph/GraphExecutioner.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/GraphExecutioner.h diff --git a/libnd4j/include/graph/GraphHolder.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/GraphHolder.h similarity index 100% rename from libnd4j/include/graph/GraphHolder.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/GraphHolder.h diff --git a/libnd4j/include/graph/GraphState.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/GraphState.h similarity index 100% rename from libnd4j/include/graph/GraphState.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/GraphState.h diff --git a/libnd4j/include/graph/GraphUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/GraphUtils.h similarity index 100% rename from libnd4j/include/graph/GraphUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/GraphUtils.h diff --git a/libnd4j/include/graph/InferenceRequest.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/InferenceRequest.h similarity index 100% rename from libnd4j/include/graph/InferenceRequest.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/InferenceRequest.h diff --git a/libnd4j/include/graph/Intervals.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Intervals.h similarity index 100% rename from libnd4j/include/graph/Intervals.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Intervals.h diff --git a/libnd4j/include/graph/Node.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Node.h similarity index 100% rename from libnd4j/include/graph/Node.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Node.h diff --git a/libnd4j/include/graph/NodeState.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/NodeState.h similarity index 100% rename from libnd4j/include/graph/NodeState.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/NodeState.h diff --git a/libnd4j/include/graph/RandomGenerator.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/RandomGenerator.h similarity index 100% rename from libnd4j/include/graph/RandomGenerator.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/RandomGenerator.h diff --git a/libnd4j/include/graph/RandomGenerator.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/RandomGenerator.hpp similarity index 100% rename from libnd4j/include/graph/RandomGenerator.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/RandomGenerator.hpp diff --git a/libnd4j/include/graph/ResultWrapper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ResultWrapper.h similarity index 100% rename from libnd4j/include/graph/ResultWrapper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/ResultWrapper.h diff --git a/libnd4j/include/graph/Scope.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Scope.h similarity index 100% rename from libnd4j/include/graph/Scope.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Scope.h diff --git a/libnd4j/include/graph/SessionLocalStorage.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/SessionLocalStorage.h similarity index 100% rename from libnd4j/include/graph/SessionLocalStorage.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/SessionLocalStorage.h diff --git a/libnd4j/include/graph/Stash.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Stash.h similarity index 100% rename from libnd4j/include/graph/Stash.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Stash.h diff --git a/libnd4j/include/graph/Status.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Status.h similarity index 100% rename from libnd4j/include/graph/Status.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Status.h diff --git a/libnd4j/include/graph/TimeHolder.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/TimeHolder.h similarity index 100% rename from libnd4j/include/graph/TimeHolder.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/TimeHolder.h diff --git a/libnd4j/include/graph/Variable.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Variable.h similarity index 100% rename from libnd4j/include/graph/Variable.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/Variable.h diff --git a/libnd4j/include/graph/VariableProxy.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/VariableProxy.h similarity index 100% rename from libnd4j/include/graph/VariableProxy.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/VariableProxy.h diff --git a/libnd4j/include/graph/VariableSpace.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/VariableSpace.h similarity index 100% rename from libnd4j/include/graph/VariableSpace.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/VariableSpace.h diff --git a/libnd4j/include/graph/VariableType.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/VariableType.h similarity index 100% rename from libnd4j/include/graph/VariableType.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/VariableType.h diff --git a/libnd4j/include/graph/VariablesSet.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/VariablesSet.h similarity index 100% rename from libnd4j/include/graph/VariablesSet.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/VariablesSet.h diff --git a/libnd4j/include/graph/exceptions/impl/unresolved_input_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/exceptions/impl/unresolved_input_exception.cpp similarity index 100% rename from libnd4j/include/graph/exceptions/impl/unresolved_input_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/exceptions/impl/unresolved_input_exception.cpp diff --git a/libnd4j/include/graph/exceptions/impl/unresolved_output_exception.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/exceptions/impl/unresolved_output_exception.cpp similarity index 100% rename from libnd4j/include/graph/exceptions/impl/unresolved_output_exception.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/exceptions/impl/unresolved_output_exception.cpp diff --git a/libnd4j/include/graph/exceptions/unresolved_input_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/exceptions/unresolved_input_exception.h similarity index 100% rename from libnd4j/include/graph/exceptions/unresolved_input_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/exceptions/unresolved_input_exception.h diff --git a/libnd4j/include/graph/exceptions/unresolved_output_exception.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/exceptions/unresolved_output_exception.h similarity index 100% rename from libnd4j/include/graph/exceptions/unresolved_output_exception.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/exceptions/unresolved_output_exception.h diff --git a/libnd4j/include/graph/execution/LogicConditional.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicConditional.h similarity index 100% rename from libnd4j/include/graph/execution/LogicConditional.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicConditional.h diff --git a/libnd4j/include/graph/execution/LogicEnter.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicEnter.h similarity index 100% rename from libnd4j/include/graph/execution/LogicEnter.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicEnter.h diff --git a/libnd4j/include/graph/execution/LogicExecutor.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicExecutor.h similarity index 100% rename from libnd4j/include/graph/execution/LogicExecutor.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicExecutor.h diff --git a/libnd4j/include/graph/execution/LogicExit.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicExit.h similarity index 100% rename from libnd4j/include/graph/execution/LogicExit.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicExit.h diff --git a/libnd4j/include/graph/execution/LogicExpose.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicExpose.h similarity index 100% rename from libnd4j/include/graph/execution/LogicExpose.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicExpose.h diff --git a/libnd4j/include/graph/execution/LogicLoopCond.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicLoopCond.h similarity index 100% rename from libnd4j/include/graph/execution/LogicLoopCond.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicLoopCond.h diff --git a/libnd4j/include/graph/execution/LogicMerge.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicMerge.h similarity index 100% rename from libnd4j/include/graph/execution/LogicMerge.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicMerge.h diff --git a/libnd4j/include/graph/execution/LogicNextIteration.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicNextIteration.h similarity index 100% rename from libnd4j/include/graph/execution/LogicNextIteration.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicNextIteration.h diff --git a/libnd4j/include/graph/execution/LogicReturn.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicReturn.h similarity index 100% rename from libnd4j/include/graph/execution/LogicReturn.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicReturn.h diff --git a/libnd4j/include/graph/execution/LogicScope.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicScope.h similarity index 100% rename from libnd4j/include/graph/execution/LogicScope.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicScope.h diff --git a/libnd4j/include/graph/execution/LogicSwitch.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicSwitch.h similarity index 100% rename from libnd4j/include/graph/execution/LogicSwitch.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicSwitch.h diff --git a/libnd4j/include/graph/execution/LogicWhile.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicWhile.h similarity index 100% rename from libnd4j/include/graph/execution/LogicWhile.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/LogicWhile.h diff --git a/libnd4j/include/graph/execution/impl/LogicConditional.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicConditional.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicConditional.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicConditional.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicEnter.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicEnter.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicEnter.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicEnter.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicExecutor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicExecutor.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicExecutor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicExecutor.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicExit.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicExit.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicExit.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicExit.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicExpose.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicExpose.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicExpose.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicExpose.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicLoopCond.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicLoopCond.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicLoopCond.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicLoopCond.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicMerge.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicMerge.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicMerge.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicMerge.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicNextIteration.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicNextIteration.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicNextIteration.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicNextIteration.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicReturn.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicReturn.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicReturn.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicReturn.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicScope.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicScope.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicScope.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicScope.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicSwitch.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicSwitch.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicSwitch.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicSwitch.cpp diff --git a/libnd4j/include/graph/execution/impl/LogicWhile.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicWhile.cpp similarity index 100% rename from libnd4j/include/graph/execution/impl/LogicWhile.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/execution/impl/LogicWhile.cpp diff --git a/libnd4j/include/graph/generated/array_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/array_generated.h similarity index 99% rename from libnd4j/include/graph/generated/array_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/array_generated.h index 5c4c0d7af..28c369e29 100644 --- a/libnd4j/include/graph/generated/array_generated.h +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/array_generated.h @@ -4,7 +4,7 @@ #ifndef FLATBUFFERS_GENERATED_ARRAY_ND4J_GRAPH_H_ #define FLATBUFFERS_GENERATED_ARRAY_ND4J_GRAPH_H_ -#include "flatbuffers/flatbuffers.h" +#include namespace sd { namespace graph { diff --git a/libnd4j/include/graph/generated/array_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/array_generated.js similarity index 100% rename from libnd4j/include/graph/generated/array_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/array_generated.js diff --git a/libnd4j/include/graph/generated/config_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/config_generated.h similarity index 100% rename from libnd4j/include/graph/generated/config_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/config_generated.h diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/config_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/config_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/config_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/config_generated.js diff --git a/libnd4j/include/graph/generated/graph.grpc.fb.cc b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/graph.grpc.fb.cc similarity index 100% rename from libnd4j/include/graph/generated/graph.grpc.fb.cc rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/graph.grpc.fb.cc diff --git a/libnd4j/include/graph/generated/graph.grpc.fb.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/graph.grpc.fb.h similarity index 100% rename from libnd4j/include/graph/generated/graph.grpc.fb.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/graph.grpc.fb.h diff --git a/libnd4j/include/graph/generated/graph_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/graph_generated.h similarity index 100% rename from libnd4j/include/graph/generated/graph_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/graph_generated.h diff --git a/libnd4j/include/graph/generated/graph_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/graph_generated.js similarity index 100% rename from libnd4j/include/graph/generated/graph_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/graph_generated.js diff --git a/libnd4j/include/graph/generated/nd4j/graph/__init__.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/__init__.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/__init__.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/__init__.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/ByteOrder.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ByteOrder.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/ByteOrder.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ByteOrder.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/ByteOrder.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ByteOrder.java similarity index 92% rename from libnd4j/include/graph/generated/nd4j/graph/ByteOrder.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ByteOrder.java index b0d703b61..a4466cd3e 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/ByteOrder.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ByteOrder.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class ByteOrder { private ByteOrder() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/ByteOrder.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ByteOrder.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/ByteOrder.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ByteOrder.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/DType.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/DType.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/DType.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/DType.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/DType.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/DType.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/DType.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/DType.java index c1b394ca7..8a316515a 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/DType.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/DType.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class DType { private DType() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/DType.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/DType.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/DType.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/DType.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/Direction.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/Direction.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/Direction.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/Direction.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/Direction.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/Direction.java similarity index 94% rename from libnd4j/include/graph/generated/nd4j/graph/Direction.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/Direction.java index dd53517f6..19b4abcb9 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/Direction.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/Direction.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class Direction { private Direction() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/Direction.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/Direction.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/Direction.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/Direction.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/ExecutionMode.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ExecutionMode.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/ExecutionMode.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ExecutionMode.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/ExecutionMode.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ExecutionMode.java similarity index 94% rename from libnd4j/include/graph/generated/nd4j/graph/ExecutionMode.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ExecutionMode.java index 5db2864ef..a73f95904 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/ExecutionMode.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ExecutionMode.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class ExecutionMode { private ExecutionMode() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/ExecutionMode.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ExecutionMode.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/ExecutionMode.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ExecutionMode.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatArray.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArray.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatArray.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArray.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatArray.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArray.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/FlatArray.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArray.java index 81c43d04a..cccf58f39 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatArray.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArray.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatArray.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArray.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatArray.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArray.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatArrayList.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArrayList.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatArrayList.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArrayList.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatArrayList.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArrayList.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/FlatArrayList.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArrayList.java index 74f1dbd29..d6c3a3eb1 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatArrayList.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArrayList.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatArrayList.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArrayList.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatArrayList.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatArrayList.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatConfiguration.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatConfiguration.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatConfiguration.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatConfiguration.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatConfiguration.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatConfiguration.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/FlatConfiguration.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatConfiguration.java index e104f49d6..d23d685e9 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatConfiguration.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatConfiguration.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatConfiguration.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatConfiguration.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatConfiguration.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatConfiguration.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatDropRequest.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatDropRequest.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatDropRequest.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatDropRequest.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatDropRequest.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatDropRequest.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/FlatDropRequest.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatDropRequest.java index 548722be3..bfee625d9 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatDropRequest.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatDropRequest.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatDropRequest.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatDropRequest.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatDropRequest.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatDropRequest.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatGraph.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatGraph.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatGraph.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatGraph.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatGraph.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatGraph.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/FlatGraph.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatGraph.java index 660d9e431..c7f0c611d 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatGraph.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatGraph.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatGraph.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatGraph.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatGraph.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatGraph.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatInferenceRequest.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatInferenceRequest.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatInferenceRequest.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatInferenceRequest.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatInferenceRequest.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatInferenceRequest.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/FlatInferenceRequest.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatInferenceRequest.java index fc907bcd8..1c8720ab7 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatInferenceRequest.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatInferenceRequest.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatInferenceRequest.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatInferenceRequest.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatInferenceRequest.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatInferenceRequest.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatNode.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatNode.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatNode.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatNode.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatNode.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatNode.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/FlatNode.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatNode.java index 2fe0a0ee9..a65786875 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatNode.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatNode.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatNode.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatNode.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatNode.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatNode.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatProperties.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatProperties.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatProperties.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatProperties.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatProperties.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatProperties.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/FlatProperties.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatProperties.java index 72df15cde..623fd8e2a 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatProperties.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatProperties.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatProperties.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatProperties.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatProperties.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatProperties.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatResponse.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResponse.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatResponse.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResponse.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatResponse.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResponse.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/FlatResponse.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResponse.java index 2fed88d5a..a81821f35 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatResponse.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResponse.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatResponse.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResponse.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatResponse.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResponse.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatResult.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResult.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatResult.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResult.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatResult.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResult.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/FlatResult.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResult.java index 8424e3ad2..252aff83a 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatResult.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResult.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatResult.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResult.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatResult.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatResult.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatTiming.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatTiming.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatTiming.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatTiming.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatTiming.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatTiming.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/FlatTiming.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatTiming.java index 926bf6811..cb27364a0 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatTiming.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatTiming.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatTiming.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatTiming.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatTiming.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatTiming.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatVariable.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatVariable.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatVariable.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatVariable.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatVariable.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatVariable.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/FlatVariable.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatVariable.java index d73c990bb..7d06dbbff 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FlatVariable.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatVariable.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FlatVariable.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatVariable.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FlatVariable.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FlatVariable.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/FrameIteration.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FrameIteration.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FrameIteration.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FrameIteration.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/FrameIteration.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FrameIteration.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/FrameIteration.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FrameIteration.java index 58690c018..c85345e5d 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/FrameIteration.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FrameIteration.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/FrameIteration.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FrameIteration.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/FrameIteration.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/FrameIteration.py diff --git a/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/GraphInferenceServerGrpc.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/GraphInferenceServerGrpc.java new file mode 100644 index 000000000..03c262c65 --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/GraphInferenceServerGrpc.java @@ -0,0 +1,551 @@ +//Generated by flatc compiler (version 1.10.0) +//If you make any local changes, they will be lost +//source: graph.fbs + +package org.nd4j.graph; + +import com.google.flatbuffers.grpc.FlatbuffersUtils; + +import java.nio.ByteBuffer; +import static io.grpc.MethodDescriptor.generateFullMethodName; +import static io.grpc.stub.ClientCalls.asyncBidiStreamingCall; +import static io.grpc.stub.ClientCalls.asyncClientStreamingCall; +import static io.grpc.stub.ClientCalls.asyncServerStreamingCall; +import static io.grpc.stub.ClientCalls.asyncUnaryCall; +import static io.grpc.stub.ClientCalls.blockingServerStreamingCall; +import static io.grpc.stub.ClientCalls.blockingUnaryCall; +import static io.grpc.stub.ClientCalls.futureUnaryCall; +import static io.grpc.stub.ServerCalls.asyncBidiStreamingCall; +import static io.grpc.stub.ServerCalls.asyncClientStreamingCall; +import static io.grpc.stub.ServerCalls.asyncServerStreamingCall; +import static io.grpc.stub.ServerCalls.asyncUnaryCall; +import static io.grpc.stub.ServerCalls.asyncUnimplementedStreamingCall; +import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall; + +/** + */ +@javax.annotation.Generated( + value = "by gRPC proto compiler", + comments = "Source: graph.fbs") +public final class GraphInferenceServerGrpc { + + private GraphInferenceServerGrpc() {} + + public static final String SERVICE_NAME = "nd4j.graph.GraphInferenceServer"; + + // Static method descriptors that strictly reflect the proto. + @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") + @java.lang.Deprecated // Use {@link #getRegisterGraphMethod()} instead. + public static final io.grpc.MethodDescriptor METHOD_REGISTER_GRAPH = getRegisterGraphMethod(); + + private static volatile io.grpc.MethodDescriptor getRegisterGraphMethod; + + private static volatile FlatbuffersUtils.FBExtactor extractorOfFlatGraph; + private static FlatbuffersUtils.FBExtactor getExtractorOfFlatGraph() { + if (extractorOfFlatGraph != null) return extractorOfFlatGraph; + synchronized (GraphInferenceServerGrpc.class) { + if (extractorOfFlatGraph != null) return extractorOfFlatGraph; + extractorOfFlatGraph = new FlatbuffersUtils.FBExtactor() { + public org.nd4j.graph.FlatGraph extract (ByteBuffer buffer) { + return org.nd4j.graph.FlatGraph.getRootAsFlatGraph(buffer); + } + }; + return extractorOfFlatGraph; + } + } + + private static volatile FlatbuffersUtils.FBExtactor extractorOfFlatResponse; + private static FlatbuffersUtils.FBExtactor getExtractorOfFlatResponse() { + if (extractorOfFlatResponse != null) return extractorOfFlatResponse; + synchronized (GraphInferenceServerGrpc.class) { + if (extractorOfFlatResponse != null) return extractorOfFlatResponse; + extractorOfFlatResponse = new FlatbuffersUtils.FBExtactor() { + public org.nd4j.graph.FlatResponse extract (ByteBuffer buffer) { + return org.nd4j.graph.FlatResponse.getRootAsFlatResponse(buffer); + } + }; + return extractorOfFlatResponse; + } + } + + @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") + public static io.grpc.MethodDescriptor getRegisterGraphMethod() { + io.grpc.MethodDescriptor getRegisterGraphMethod; + if ((getRegisterGraphMethod = GraphInferenceServerGrpc.getRegisterGraphMethod) == null) { + synchronized (GraphInferenceServerGrpc.class) { + if ((getRegisterGraphMethod = GraphInferenceServerGrpc.getRegisterGraphMethod) == null) { + GraphInferenceServerGrpc.getRegisterGraphMethod = getRegisterGraphMethod = + io.grpc.MethodDescriptor.newBuilder() + .setType(io.grpc.MethodDescriptor.MethodType.UNARY) + .setFullMethodName(generateFullMethodName( + "nd4j.graph.GraphInferenceServer", "RegisterGraph")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(FlatbuffersUtils.marshaller( + org.nd4j.graph.FlatGraph.class, getExtractorOfFlatGraph())) + .setResponseMarshaller(FlatbuffersUtils.marshaller( + org.nd4j.graph.FlatResponse.class, getExtractorOfFlatResponse())) + .setSchemaDescriptor(null) + .build(); + } + } + } + return getRegisterGraphMethod; + } + + @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") + @java.lang.Deprecated // Use {@link #getForgetGraphMethod()} instead. + public static final io.grpc.MethodDescriptor METHOD_FORGET_GRAPH = getForgetGraphMethod(); + + private static volatile io.grpc.MethodDescriptor getForgetGraphMethod; + + private static volatile FlatbuffersUtils.FBExtactor extractorOfFlatDropRequest; + private static FlatbuffersUtils.FBExtactor getExtractorOfFlatDropRequest() { + if (extractorOfFlatDropRequest != null) return extractorOfFlatDropRequest; + synchronized (GraphInferenceServerGrpc.class) { + if (extractorOfFlatDropRequest != null) return extractorOfFlatDropRequest; + extractorOfFlatDropRequest = new FlatbuffersUtils.FBExtactor() { + public org.nd4j.graph.FlatDropRequest extract (ByteBuffer buffer) { + return org.nd4j.graph.FlatDropRequest.getRootAsFlatDropRequest(buffer); + } + }; + return extractorOfFlatDropRequest; + } + } + + @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") + public static io.grpc.MethodDescriptor getForgetGraphMethod() { + io.grpc.MethodDescriptor getForgetGraphMethod; + if ((getForgetGraphMethod = GraphInferenceServerGrpc.getForgetGraphMethod) == null) { + synchronized (GraphInferenceServerGrpc.class) { + if ((getForgetGraphMethod = GraphInferenceServerGrpc.getForgetGraphMethod) == null) { + GraphInferenceServerGrpc.getForgetGraphMethod = getForgetGraphMethod = + io.grpc.MethodDescriptor.newBuilder() + .setType(io.grpc.MethodDescriptor.MethodType.UNARY) + .setFullMethodName(generateFullMethodName( + "nd4j.graph.GraphInferenceServer", "ForgetGraph")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(FlatbuffersUtils.marshaller( + org.nd4j.graph.FlatDropRequest.class, getExtractorOfFlatDropRequest())) + .setResponseMarshaller(FlatbuffersUtils.marshaller( + org.nd4j.graph.FlatResponse.class, getExtractorOfFlatResponse())) + .setSchemaDescriptor(null) + .build(); + } + } + } + return getForgetGraphMethod; + } + + @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") + @java.lang.Deprecated // Use {@link #getReplaceGraphMethod()} instead. + public static final io.grpc.MethodDescriptor METHOD_REPLACE_GRAPH = getReplaceGraphMethod(); + + private static volatile io.grpc.MethodDescriptor getReplaceGraphMethod; + + @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") + public static io.grpc.MethodDescriptor getReplaceGraphMethod() { + io.grpc.MethodDescriptor getReplaceGraphMethod; + if ((getReplaceGraphMethod = GraphInferenceServerGrpc.getReplaceGraphMethod) == null) { + synchronized (GraphInferenceServerGrpc.class) { + if ((getReplaceGraphMethod = GraphInferenceServerGrpc.getReplaceGraphMethod) == null) { + GraphInferenceServerGrpc.getReplaceGraphMethod = getReplaceGraphMethod = + io.grpc.MethodDescriptor.newBuilder() + .setType(io.grpc.MethodDescriptor.MethodType.UNARY) + .setFullMethodName(generateFullMethodName( + "nd4j.graph.GraphInferenceServer", "ReplaceGraph")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(FlatbuffersUtils.marshaller( + org.nd4j.graph.FlatGraph.class, getExtractorOfFlatGraph())) + .setResponseMarshaller(FlatbuffersUtils.marshaller( + org.nd4j.graph.FlatResponse.class, getExtractorOfFlatResponse())) + .setSchemaDescriptor(null) + .build(); + } + } + } + return getReplaceGraphMethod; + } + + @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") + @java.lang.Deprecated // Use {@link #getInferenceRequestMethod()} instead. + public static final io.grpc.MethodDescriptor METHOD_INFERENCE_REQUEST = getInferenceRequestMethod(); + + private static volatile io.grpc.MethodDescriptor getInferenceRequestMethod; + + private static volatile FlatbuffersUtils.FBExtactor extractorOfFlatInferenceRequest; + private static FlatbuffersUtils.FBExtactor getExtractorOfFlatInferenceRequest() { + if (extractorOfFlatInferenceRequest != null) return extractorOfFlatInferenceRequest; + synchronized (GraphInferenceServerGrpc.class) { + if (extractorOfFlatInferenceRequest != null) return extractorOfFlatInferenceRequest; + extractorOfFlatInferenceRequest = new FlatbuffersUtils.FBExtactor() { + public org.nd4j.graph.FlatInferenceRequest extract (ByteBuffer buffer) { + return org.nd4j.graph.FlatInferenceRequest.getRootAsFlatInferenceRequest(buffer); + } + }; + return extractorOfFlatInferenceRequest; + } + } + + private static volatile FlatbuffersUtils.FBExtactor extractorOfFlatResult; + private static FlatbuffersUtils.FBExtactor getExtractorOfFlatResult() { + if (extractorOfFlatResult != null) return extractorOfFlatResult; + synchronized (GraphInferenceServerGrpc.class) { + if (extractorOfFlatResult != null) return extractorOfFlatResult; + extractorOfFlatResult = new FlatbuffersUtils.FBExtactor() { + public org.nd4j.graph.FlatResult extract (ByteBuffer buffer) { + return org.nd4j.graph.FlatResult.getRootAsFlatResult(buffer); + } + }; + return extractorOfFlatResult; + } + } + + @io.grpc.ExperimentalApi("https://github.com/grpc/grpc-java/issues/1901") + public static io.grpc.MethodDescriptor getInferenceRequestMethod() { + io.grpc.MethodDescriptor getInferenceRequestMethod; + if ((getInferenceRequestMethod = GraphInferenceServerGrpc.getInferenceRequestMethod) == null) { + synchronized (GraphInferenceServerGrpc.class) { + if ((getInferenceRequestMethod = GraphInferenceServerGrpc.getInferenceRequestMethod) == null) { + GraphInferenceServerGrpc.getInferenceRequestMethod = getInferenceRequestMethod = + io.grpc.MethodDescriptor.newBuilder() + .setType(io.grpc.MethodDescriptor.MethodType.UNARY) + .setFullMethodName(generateFullMethodName( + "nd4j.graph.GraphInferenceServer", "InferenceRequest")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(FlatbuffersUtils.marshaller( + org.nd4j.graph.FlatInferenceRequest.class, getExtractorOfFlatInferenceRequest())) + .setResponseMarshaller(FlatbuffersUtils.marshaller( + org.nd4j.graph.FlatResult.class, getExtractorOfFlatResult())) + .setSchemaDescriptor(null) + .build(); + } + } + } + return getInferenceRequestMethod; + } + + /** + * Creates a new async stub that supports all call types for the service + */ + public static GraphInferenceServerStub newStub(io.grpc.Channel channel) { + return new GraphInferenceServerStub(channel); + } + + /** + * Creates a new blocking-style stub that supports unary and streaming output calls on the service + */ + public static GraphInferenceServerBlockingStub newBlockingStub( + io.grpc.Channel channel) { + return new GraphInferenceServerBlockingStub(channel); + } + + /** + * Creates a new ListenableFuture-style stub that supports unary calls on the service + */ + public static GraphInferenceServerFutureStub newFutureStub( + io.grpc.Channel channel) { + return new GraphInferenceServerFutureStub(channel); + } + + /** + */ + public static abstract class GraphInferenceServerImplBase implements io.grpc.BindableService { + + /** + */ + public void registerGraph(org.nd4j.graph.FlatGraph request, + io.grpc.stub.StreamObserver responseObserver) { + asyncUnimplementedUnaryCall(getRegisterGraphMethod(), responseObserver); + } + + /** + */ + public void forgetGraph(org.nd4j.graph.FlatDropRequest request, + io.grpc.stub.StreamObserver responseObserver) { + asyncUnimplementedUnaryCall(getForgetGraphMethod(), responseObserver); + } + + /** + */ + public void replaceGraph(org.nd4j.graph.FlatGraph request, + io.grpc.stub.StreamObserver responseObserver) { + asyncUnimplementedUnaryCall(getReplaceGraphMethod(), responseObserver); + } + + /** + */ + public void inferenceRequest(org.nd4j.graph.FlatInferenceRequest request, + io.grpc.stub.StreamObserver responseObserver) { + asyncUnimplementedUnaryCall(getInferenceRequestMethod(), responseObserver); + } + + @java.lang.Override public final io.grpc.ServerServiceDefinition bindService() { + return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor()) + .addMethod( + getRegisterGraphMethod(), + asyncUnaryCall( + new MethodHandlers< + org.nd4j.graph.FlatGraph, + org.nd4j.graph.FlatResponse>( + this, METHODID_REGISTER_GRAPH))) + .addMethod( + getForgetGraphMethod(), + asyncUnaryCall( + new MethodHandlers< + org.nd4j.graph.FlatDropRequest, + org.nd4j.graph.FlatResponse>( + this, METHODID_FORGET_GRAPH))) + .addMethod( + getReplaceGraphMethod(), + asyncUnaryCall( + new MethodHandlers< + org.nd4j.graph.FlatGraph, + org.nd4j.graph.FlatResponse>( + this, METHODID_REPLACE_GRAPH))) + .addMethod( + getInferenceRequestMethod(), + asyncUnaryCall( + new MethodHandlers< + org.nd4j.graph.FlatInferenceRequest, + org.nd4j.graph.FlatResult>( + this, METHODID_INFERENCE_REQUEST))) + .build(); + } + } + + /** + */ + public static final class GraphInferenceServerStub extends io.grpc.stub.AbstractStub { + private GraphInferenceServerStub(io.grpc.Channel channel) { + super(channel); + } + + private GraphInferenceServerStub(io.grpc.Channel channel, + io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @java.lang.Override + protected GraphInferenceServerStub build(io.grpc.Channel channel, + io.grpc.CallOptions callOptions) { + return new GraphInferenceServerStub(channel, callOptions); + } + + /** + */ + public void registerGraph(org.nd4j.graph.FlatGraph request, + io.grpc.stub.StreamObserver responseObserver) { + asyncUnaryCall( + getChannel().newCall(getRegisterGraphMethod(), getCallOptions()), request, responseObserver); + } + + /** + */ + public void forgetGraph(org.nd4j.graph.FlatDropRequest request, + io.grpc.stub.StreamObserver responseObserver) { + asyncUnaryCall( + getChannel().newCall(getForgetGraphMethod(), getCallOptions()), request, responseObserver); + } + + /** + */ + public void replaceGraph(org.nd4j.graph.FlatGraph request, + io.grpc.stub.StreamObserver responseObserver) { + asyncUnaryCall( + getChannel().newCall(getReplaceGraphMethod(), getCallOptions()), request, responseObserver); + } + + /** + */ + public void inferenceRequest(org.nd4j.graph.FlatInferenceRequest request, + io.grpc.stub.StreamObserver responseObserver) { + asyncUnaryCall( + getChannel().newCall(getInferenceRequestMethod(), getCallOptions()), request, responseObserver); + } + } + + /** + */ + public static final class GraphInferenceServerBlockingStub extends io.grpc.stub.AbstractStub { + private GraphInferenceServerBlockingStub(io.grpc.Channel channel) { + super(channel); + } + + private GraphInferenceServerBlockingStub(io.grpc.Channel channel, + io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @java.lang.Override + protected GraphInferenceServerBlockingStub build(io.grpc.Channel channel, + io.grpc.CallOptions callOptions) { + return new GraphInferenceServerBlockingStub(channel, callOptions); + } + + /** + */ + public org.nd4j.graph.FlatResponse registerGraph(org.nd4j.graph.FlatGraph request) { + return blockingUnaryCall( + getChannel(), getRegisterGraphMethod(), getCallOptions(), request); + } + + /** + */ + public org.nd4j.graph.FlatResponse forgetGraph(org.nd4j.graph.FlatDropRequest request) { + return blockingUnaryCall( + getChannel(), getForgetGraphMethod(), getCallOptions(), request); + } + + /** + */ + public org.nd4j.graph.FlatResponse replaceGraph(org.nd4j.graph.FlatGraph request) { + return blockingUnaryCall( + getChannel(), getReplaceGraphMethod(), getCallOptions(), request); + } + + /** + */ + public org.nd4j.graph.FlatResult inferenceRequest(org.nd4j.graph.FlatInferenceRequest request) { + return blockingUnaryCall( + getChannel(), getInferenceRequestMethod(), getCallOptions(), request); + } + } + + /** + */ + public static final class GraphInferenceServerFutureStub extends io.grpc.stub.AbstractStub { + private GraphInferenceServerFutureStub(io.grpc.Channel channel) { + super(channel); + } + + private GraphInferenceServerFutureStub(io.grpc.Channel channel, + io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @java.lang.Override + protected GraphInferenceServerFutureStub build(io.grpc.Channel channel, + io.grpc.CallOptions callOptions) { + return new GraphInferenceServerFutureStub(channel, callOptions); + } + + /** + */ + public com.google.common.util.concurrent.ListenableFuture registerGraph( + org.nd4j.graph.FlatGraph request) { + return futureUnaryCall( + getChannel().newCall(getRegisterGraphMethod(), getCallOptions()), request); + } + + /** + */ + public com.google.common.util.concurrent.ListenableFuture forgetGraph( + org.nd4j.graph.FlatDropRequest request) { + return futureUnaryCall( + getChannel().newCall(getForgetGraphMethod(), getCallOptions()), request); + } + + /** + */ + public com.google.common.util.concurrent.ListenableFuture replaceGraph( + org.nd4j.graph.FlatGraph request) { + return futureUnaryCall( + getChannel().newCall(getReplaceGraphMethod(), getCallOptions()), request); + } + + /** + */ + public com.google.common.util.concurrent.ListenableFuture inferenceRequest( + org.nd4j.graph.FlatInferenceRequest request) { + return futureUnaryCall( + getChannel().newCall(getInferenceRequestMethod(), getCallOptions()), request); + } + } + + private static final int METHODID_REGISTER_GRAPH = 0; + private static final int METHODID_FORGET_GRAPH = 1; + private static final int METHODID_REPLACE_GRAPH = 2; + private static final int METHODID_INFERENCE_REQUEST = 3; + + private static final class MethodHandlers implements + io.grpc.stub.ServerCalls.UnaryMethod, + io.grpc.stub.ServerCalls.ServerStreamingMethod, + io.grpc.stub.ServerCalls.ClientStreamingMethod, + io.grpc.stub.ServerCalls.BidiStreamingMethod { + private final GraphInferenceServerImplBase serviceImpl; + private final int methodId; + + MethodHandlers(GraphInferenceServerImplBase serviceImpl, int methodId) { + this.serviceImpl = serviceImpl; + this.methodId = methodId; + } + + @java.lang.Override + @java.lang.SuppressWarnings("unchecked") + public void invoke(Req request, io.grpc.stub.StreamObserver responseObserver) { + switch (methodId) { + case METHODID_REGISTER_GRAPH: + serviceImpl.registerGraph((org.nd4j.graph.FlatGraph) request, + (io.grpc.stub.StreamObserver) responseObserver); + break; + case METHODID_FORGET_GRAPH: + serviceImpl.forgetGraph((org.nd4j.graph.FlatDropRequest) request, + (io.grpc.stub.StreamObserver) responseObserver); + break; + case METHODID_REPLACE_GRAPH: + serviceImpl.replaceGraph((org.nd4j.graph.FlatGraph) request, + (io.grpc.stub.StreamObserver) responseObserver); + break; + case METHODID_INFERENCE_REQUEST: + serviceImpl.inferenceRequest((org.nd4j.graph.FlatInferenceRequest) request, + (io.grpc.stub.StreamObserver) responseObserver); + break; + default: + throw new AssertionError(); + } + } + + @java.lang.Override + @java.lang.SuppressWarnings("unchecked") + public io.grpc.stub.StreamObserver invoke( + io.grpc.stub.StreamObserver responseObserver) { + switch (methodId) { + default: + throw new AssertionError(); + } + } + } + + private static volatile io.grpc.ServiceDescriptor serviceDescriptor; + + public static io.grpc.ServiceDescriptor getServiceDescriptor() { + io.grpc.ServiceDescriptor result = serviceDescriptor; + if (result == null) { + synchronized (GraphInferenceServerGrpc.class) { + result = serviceDescriptor; + if (result == null) { + serviceDescriptor = result = io.grpc.ServiceDescriptor.newBuilder(SERVICE_NAME) + .setSchemaDescriptor(null) + .addMethod(getRegisterGraphMethod()) + .addMethod(getForgetGraphMethod()) + .addMethod(getReplaceGraphMethod()) + .addMethod(getInferenceRequestMethod()) + .build(); + } + } + } + return result; + } +} diff --git a/libnd4j/include/graph/generated/nd4j/graph/InputType.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/InputType.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/InputType.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/InputType.cs diff --git a/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/InputType.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/InputType.java new file mode 100644 index 000000000..e7f24596c --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/InputType.java @@ -0,0 +1,17 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +package org.nd4j.graph; + +public final class InputType { + private InputType() { } + public static final byte UNDEFINED = 0; + public static final byte NUMERIC = 1; + public static final byte STRINGULAR = 2; + public static final byte NUMERIC_SET = 3; + public static final byte STRINGULAR_SET = 4; + + public static final String[] names = { "UNDEFINED", "NUMERIC", "STRINGULAR", "NUMERIC_SET", "STRINGULAR_SET", }; + + public static String name(int e) { return names[e]; } +} + diff --git a/libnd4j/include/graph/generated/nd4j/graph/InputType.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/InputType.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/InputType.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/InputType.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/IntPair.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntPair.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/IntPair.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntPair.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/IntPair.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntPair.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/IntPair.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntPair.java index c988143e6..bc600c7e4 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/IntPair.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntPair.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/IntPair.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntPair.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/IntPair.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntPair.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/IntTriple.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntTriple.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/IntTriple.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntTriple.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/IntTriple.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntTriple.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/IntTriple.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntTriple.java index 8bc8961c8..d10f7eb24 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/IntTriple.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntTriple.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/IntTriple.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntTriple.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/IntTriple.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/IntTriple.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/LongPair.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongPair.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/LongPair.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongPair.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/LongPair.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongPair.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/LongPair.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongPair.java index e17c019f6..41a60da55 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/LongPair.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongPair.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/LongPair.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongPair.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/LongPair.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongPair.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/LongTriple.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongTriple.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/LongTriple.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongTriple.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/LongTriple.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongTriple.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/LongTriple.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongTriple.java index c35e27f4c..6b6cb8abd 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/LongTriple.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongTriple.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/LongTriple.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongTriple.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/LongTriple.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/LongTriple.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/OpClass.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpClass.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/OpClass.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpClass.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/OpClass.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpClass.java similarity index 95% rename from libnd4j/include/graph/generated/nd4j/graph/OpClass.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpClass.java index 996009041..6fb7a5329 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/OpClass.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpClass.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class OpClass { private OpClass() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/OpClass.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpClass.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/OpClass.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpClass.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/OpType.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpType.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/OpType.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpType.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/OpType.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpType.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/OpType.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpType.java index bac8509d8..124afa6ce 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/OpType.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpType.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class OpType { private OpType() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/OpType.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpType.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/OpType.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OpType.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/OutputMode.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OutputMode.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/OutputMode.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OutputMode.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/OutputMode.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OutputMode.java similarity index 95% rename from libnd4j/include/graph/generated/nd4j/graph/OutputMode.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OutputMode.java index 9413825e2..387ee55ac 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/OutputMode.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OutputMode.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class OutputMode { private OutputMode() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/OutputMode.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OutputMode.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/OutputMode.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/OutputMode.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/ProfilingMode.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ProfilingMode.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/ProfilingMode.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ProfilingMode.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/ProfilingMode.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ProfilingMode.java similarity index 94% rename from libnd4j/include/graph/generated/nd4j/graph/ProfilingMode.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ProfilingMode.java index 34e3e320f..87243a853 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/ProfilingMode.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ProfilingMode.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class ProfilingMode { private ProfilingMode() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/ProfilingMode.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ProfilingMode.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/ProfilingMode.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/ProfilingMode.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIAddName.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIAddName.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIAddName.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIAddName.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIAddName.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIAddName.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/UIAddName.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIAddName.java index 9caf5f0d7..23784a3c6 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIAddName.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIAddName.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIAddName.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIAddName.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIAddName.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIAddName.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEvent.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEvent.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIEvent.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEvent.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEvent.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEvent.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/UIEvent.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEvent.java index e586e8967..ddbafe7c5 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIEvent.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEvent.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEvent.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEvent.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIEvent.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEvent.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEventSubtype.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventSubtype.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIEventSubtype.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventSubtype.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEventSubtype.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventSubtype.java similarity index 97% rename from libnd4j/include/graph/generated/nd4j/graph/UIEventSubtype.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventSubtype.java index 98a9c5951..405af9b7c 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIEventSubtype.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventSubtype.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class UIEventSubtype { private UIEventSubtype() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEventSubtype.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventSubtype.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIEventSubtype.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventSubtype.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEventType.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventType.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIEventType.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventType.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEventType.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventType.java similarity index 96% rename from libnd4j/include/graph/generated/nd4j/graph/UIEventType.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventType.java index 0c5f38c2b..511ed2adf 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIEventType.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventType.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class UIEventType { private UIEventType() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIEventType.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventType.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIEventType.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIEventType.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIGraphStructure.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIGraphStructure.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIGraphStructure.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIGraphStructure.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIGraphStructure.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIGraphStructure.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/UIGraphStructure.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIGraphStructure.java index 65ff2d58b..d850ff3d7 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIGraphStructure.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIGraphStructure.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIGraphStructure.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIGraphStructure.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIGraphStructure.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIGraphStructure.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHardwareState.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHardwareState.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIHardwareState.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHardwareState.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHardwareState.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHardwareState.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/UIHardwareState.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHardwareState.java index 469f6357d..41c8c0ca2 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIHardwareState.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHardwareState.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHardwareState.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHardwareState.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIHardwareState.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHardwareState.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHistogram.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogram.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIHistogram.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogram.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHistogram.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogram.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/UIHistogram.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogram.java index eea513be4..a9f9003df 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIHistogram.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogram.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHistogram.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogram.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIHistogram.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogram.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHistogramType.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogramType.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIHistogramType.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogramType.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHistogramType.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogramType.java similarity index 94% rename from libnd4j/include/graph/generated/nd4j/graph/UIHistogramType.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogramType.java index eed543ca5..176f44124 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIHistogramType.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogramType.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class UIHistogramType { private UIHistogramType() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIHistogramType.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogramType.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIHistogramType.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIHistogramType.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIInfoType.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIInfoType.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIInfoType.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIInfoType.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIInfoType.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIInfoType.java similarity index 94% rename from libnd4j/include/graph/generated/nd4j/graph/UIInfoType.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIInfoType.java index a2792912c..e4bd259f0 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIInfoType.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIInfoType.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class UIInfoType { private UIInfoType() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIInfoType.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIInfoType.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIInfoType.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIInfoType.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIOp.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIOp.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIOp.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIOp.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIOp.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIOp.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/UIOp.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIOp.java index 5ca33cda5..bd211444f 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIOp.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIOp.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIOp.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIOp.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIOp.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIOp.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIStaticInfoRecord.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIStaticInfoRecord.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIStaticInfoRecord.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIStaticInfoRecord.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIStaticInfoRecord.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIStaticInfoRecord.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/UIStaticInfoRecord.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIStaticInfoRecord.java index 45dc5a961..7e1990ed5 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIStaticInfoRecord.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIStaticInfoRecord.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIStaticInfoRecord.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIStaticInfoRecord.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIStaticInfoRecord.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIStaticInfoRecord.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UISummaryStatistics.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISummaryStatistics.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UISummaryStatistics.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISummaryStatistics.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UISummaryStatistics.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISummaryStatistics.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/UISummaryStatistics.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISummaryStatistics.java index ddc9d776a..7d8122018 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UISummaryStatistics.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISummaryStatistics.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UISummaryStatistics.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISummaryStatistics.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UISummaryStatistics.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISummaryStatistics.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UISystemInfo.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISystemInfo.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UISystemInfo.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISystemInfo.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UISystemInfo.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISystemInfo.java similarity index 98% rename from libnd4j/include/graph/generated/nd4j/graph/UISystemInfo.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISystemInfo.java index 4ff62ab98..a8328463e 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UISystemInfo.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISystemInfo.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UISystemInfo.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISystemInfo.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UISystemInfo.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UISystemInfo.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIVariable.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIVariable.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIVariable.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIVariable.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIVariable.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIVariable.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/UIVariable.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIVariable.java index 97ffb8c24..3efaf50ad 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UIVariable.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIVariable.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UIVariable.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIVariable.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UIVariable.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UIVariable.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/UpdaterState.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UpdaterState.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UpdaterState.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UpdaterState.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/UpdaterState.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UpdaterState.java similarity index 99% rename from libnd4j/include/graph/generated/nd4j/graph/UpdaterState.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UpdaterState.java index 76868354c..2df31416f 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/UpdaterState.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UpdaterState.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; import java.nio.*; import java.lang.*; diff --git a/libnd4j/include/graph/generated/nd4j/graph/UpdaterState.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UpdaterState.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/UpdaterState.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/UpdaterState.py diff --git a/libnd4j/include/graph/generated/nd4j/graph/VarType.cs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/VarType.cs similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/VarType.cs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/VarType.cs diff --git a/libnd4j/include/graph/generated/nd4j/graph/VarType.java b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/VarType.java similarity index 94% rename from libnd4j/include/graph/generated/nd4j/graph/VarType.java rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/VarType.java index 14937cd76..96500d233 100644 --- a/libnd4j/include/graph/generated/nd4j/graph/VarType.java +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/VarType.java @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -package nd4j.graph; +package org.nd4j.graph; public final class VarType { private VarType() { } diff --git a/libnd4j/include/graph/generated/nd4j/graph/VarType.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/VarType.py similarity index 100% rename from libnd4j/include/graph/generated/nd4j/graph/VarType.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/VarType.py diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/ai/__init__.py b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/__init__.py similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/ai/__init__.py rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/nd4j/graph/__init__.py diff --git a/libnd4j/include/graph/generated/node_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/node_generated.h similarity index 100% rename from libnd4j/include/graph/generated/node_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/node_generated.h diff --git a/libnd4j/include/graph/generated/node_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/node_generated.js similarity index 100% rename from libnd4j/include/graph/generated/node_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/node_generated.js diff --git a/libnd4j/include/graph/generated/properties_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/properties_generated.h similarity index 100% rename from libnd4j/include/graph/generated/properties_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/properties_generated.h diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/properties_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/properties_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/properties_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/properties_generated.js diff --git a/libnd4j/include/graph/generated/request_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/request_generated.h similarity index 100% rename from libnd4j/include/graph/generated/request_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/request_generated.h diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/request_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/request_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/request_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/request_generated.js diff --git a/libnd4j/include/graph/generated/result_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/result_generated.h similarity index 100% rename from libnd4j/include/graph/generated/result_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/result_generated.h diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/result_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/result_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/result_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/result_generated.js diff --git a/libnd4j/include/graph/generated/uigraphevents_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/uigraphevents_generated.h similarity index 100% rename from libnd4j/include/graph/generated/uigraphevents_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/uigraphevents_generated.h diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/uigraphevents_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/uigraphevents_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/uigraphevents_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/uigraphevents_generated.js diff --git a/libnd4j/include/graph/generated/uigraphstatic_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/uigraphstatic_generated.h similarity index 100% rename from libnd4j/include/graph/generated/uigraphstatic_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/uigraphstatic_generated.h diff --git a/libnd4j/include/graph/generated/uigraphstatic_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/uigraphstatic_generated.js similarity index 100% rename from libnd4j/include/graph/generated/uigraphstatic_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/uigraphstatic_generated.js diff --git a/libnd4j/include/graph/generated/utils_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/utils_generated.h similarity index 100% rename from libnd4j/include/graph/generated/utils_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/utils_generated.h diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/utils_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/utils_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/utils_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/utils_generated.js diff --git a/libnd4j/include/graph/generated/variable_generated.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/variable_generated.h similarity index 100% rename from libnd4j/include/graph/generated/variable_generated.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/variable_generated.h diff --git a/libnd4j/include/graph/generated/variable_generated.js b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/variable_generated.js similarity index 100% rename from libnd4j/include/graph/generated/variable_generated.js rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/generated/variable_generated.js diff --git a/libnd4j/include/graph/impl/ArgumentsList.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ArgumentsList.cpp similarity index 100% rename from libnd4j/include/graph/impl/ArgumentsList.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ArgumentsList.cpp diff --git a/libnd4j/include/graph/impl/Context.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Context.cpp similarity index 100% rename from libnd4j/include/graph/impl/Context.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Context.cpp diff --git a/libnd4j/include/graph/impl/ContextPrototype.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ContextPrototype.cpp similarity index 100% rename from libnd4j/include/graph/impl/ContextPrototype.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ContextPrototype.cpp diff --git a/libnd4j/include/graph/impl/ExecutionResult.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ExecutionResult.cpp similarity index 100% rename from libnd4j/include/graph/impl/ExecutionResult.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ExecutionResult.cpp diff --git a/libnd4j/include/graph/impl/ExecutorConfiguration.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ExecutorConfiguration.cpp similarity index 100% rename from libnd4j/include/graph/impl/ExecutorConfiguration.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ExecutorConfiguration.cpp diff --git a/libnd4j/include/graph/impl/FlatUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/FlatUtils.cpp similarity index 100% rename from libnd4j/include/graph/impl/FlatUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/FlatUtils.cpp diff --git a/libnd4j/include/graph/impl/FlowPath.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/FlowPath.cpp similarity index 100% rename from libnd4j/include/graph/impl/FlowPath.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/FlowPath.cpp diff --git a/libnd4j/include/graph/impl/FrameState.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/FrameState.cpp similarity index 100% rename from libnd4j/include/graph/impl/FrameState.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/FrameState.cpp diff --git a/libnd4j/include/graph/impl/Graph.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Graph.cpp similarity index 100% rename from libnd4j/include/graph/impl/Graph.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Graph.cpp diff --git a/libnd4j/include/graph/impl/GraphExecutioner.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/GraphExecutioner.cpp similarity index 100% rename from libnd4j/include/graph/impl/GraphExecutioner.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/GraphExecutioner.cpp diff --git a/libnd4j/include/graph/impl/GraphHolder.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/GraphHolder.cpp similarity index 100% rename from libnd4j/include/graph/impl/GraphHolder.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/GraphHolder.cpp diff --git a/libnd4j/include/graph/impl/GraphState.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/GraphState.cpp similarity index 100% rename from libnd4j/include/graph/impl/GraphState.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/GraphState.cpp diff --git a/libnd4j/include/graph/impl/GraphUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/GraphUtils.cpp similarity index 100% rename from libnd4j/include/graph/impl/GraphUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/GraphUtils.cpp diff --git a/libnd4j/include/graph/impl/InferenceRequest.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/InferenceRequest.cpp similarity index 100% rename from libnd4j/include/graph/impl/InferenceRequest.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/InferenceRequest.cpp diff --git a/libnd4j/include/graph/impl/Intervals.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Intervals.cpp similarity index 100% rename from libnd4j/include/graph/impl/Intervals.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Intervals.cpp diff --git a/libnd4j/include/graph/impl/Node.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Node.cpp similarity index 100% rename from libnd4j/include/graph/impl/Node.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Node.cpp diff --git a/libnd4j/include/graph/impl/NodeState.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/NodeState.cpp similarity index 100% rename from libnd4j/include/graph/impl/NodeState.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/NodeState.cpp diff --git a/libnd4j/include/graph/impl/ResultWrapper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ResultWrapper.cpp similarity index 100% rename from libnd4j/include/graph/impl/ResultWrapper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/ResultWrapper.cpp diff --git a/libnd4j/include/graph/impl/Scope.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Scope.cpp similarity index 100% rename from libnd4j/include/graph/impl/Scope.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Scope.cpp diff --git a/libnd4j/include/graph/impl/SessionLocalStorage.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/SessionLocalStorage.cpp similarity index 100% rename from libnd4j/include/graph/impl/SessionLocalStorage.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/SessionLocalStorage.cpp diff --git a/libnd4j/include/graph/impl/Stash.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Stash.cpp similarity index 100% rename from libnd4j/include/graph/impl/Stash.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Stash.cpp diff --git a/libnd4j/include/graph/impl/TimeHolder.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/TimeHolder.cpp similarity index 100% rename from libnd4j/include/graph/impl/TimeHolder.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/TimeHolder.cpp diff --git a/libnd4j/include/graph/impl/Variable.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Variable.cpp similarity index 100% rename from libnd4j/include/graph/impl/Variable.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/Variable.cpp diff --git a/libnd4j/include/graph/impl/VariableProxy.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/VariableProxy.cpp similarity index 100% rename from libnd4j/include/graph/impl/VariableProxy.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/VariableProxy.cpp diff --git a/libnd4j/include/graph/impl/VariableSpace.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/VariableSpace.cpp similarity index 100% rename from libnd4j/include/graph/impl/VariableSpace.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/VariableSpace.cpp diff --git a/libnd4j/include/graph/impl/VariablesSet.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/VariablesSet.cpp similarity index 100% rename from libnd4j/include/graph/impl/VariablesSet.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/impl/VariablesSet.cpp diff --git a/libnd4j/include/graph/profiling/GraphProfile.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/GraphProfile.h similarity index 100% rename from libnd4j/include/graph/profiling/GraphProfile.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/GraphProfile.h diff --git a/libnd4j/include/graph/profiling/GraphProfilingHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/GraphProfilingHelper.h similarity index 100% rename from libnd4j/include/graph/profiling/GraphProfilingHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/GraphProfilingHelper.h diff --git a/libnd4j/include/graph/profiling/NodeProfile.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/NodeProfile.h similarity index 100% rename from libnd4j/include/graph/profiling/NodeProfile.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/NodeProfile.h diff --git a/libnd4j/include/graph/profiling/impl/GraphProfile.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/impl/GraphProfile.cpp similarity index 100% rename from libnd4j/include/graph/profiling/impl/GraphProfile.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/impl/GraphProfile.cpp diff --git a/libnd4j/include/graph/profiling/impl/GraphProfilingHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/impl/GraphProfilingHelper.cpp similarity index 100% rename from libnd4j/include/graph/profiling/impl/GraphProfilingHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/impl/GraphProfilingHelper.cpp diff --git a/libnd4j/include/graph/profiling/impl/NodeProfile.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/impl/NodeProfile.cpp similarity index 100% rename from libnd4j/include/graph/profiling/impl/NodeProfile.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/profiling/impl/NodeProfile.cpp diff --git a/libnd4j/include/graph/scheme/array.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/array.fbs similarity index 100% rename from libnd4j/include/graph/scheme/array.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/array.fbs diff --git a/libnd4j/include/graph/scheme/config.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/config.fbs similarity index 100% rename from libnd4j/include/graph/scheme/config.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/config.fbs diff --git a/libnd4j/include/graph/scheme/graph.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/graph.fbs similarity index 100% rename from libnd4j/include/graph/scheme/graph.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/graph.fbs diff --git a/libnd4j/include/graph/scheme/node.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/node.fbs similarity index 100% rename from libnd4j/include/graph/scheme/node.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/node.fbs diff --git a/libnd4j/include/graph/scheme/properties.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/properties.fbs similarity index 100% rename from libnd4j/include/graph/scheme/properties.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/properties.fbs diff --git a/libnd4j/include/graph/scheme/request.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/request.fbs similarity index 100% rename from libnd4j/include/graph/scheme/request.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/request.fbs diff --git a/libnd4j/include/graph/scheme/result.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/result.fbs similarity index 100% rename from libnd4j/include/graph/scheme/result.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/result.fbs diff --git a/libnd4j/include/graph/scheme/uigraphevents.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/uigraphevents.fbs similarity index 100% rename from libnd4j/include/graph/scheme/uigraphevents.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/uigraphevents.fbs diff --git a/libnd4j/include/graph/scheme/uigraphstatic.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/uigraphstatic.fbs similarity index 100% rename from libnd4j/include/graph/scheme/uigraphstatic.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/uigraphstatic.fbs diff --git a/libnd4j/include/graph/scheme/utils.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/utils.fbs similarity index 100% rename from libnd4j/include/graph/scheme/utils.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/utils.fbs diff --git a/libnd4j/include/graph/scheme/variable.fbs b/cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/variable.fbs similarity index 100% rename from libnd4j/include/graph/scheme/variable.fbs rename to cavis-native/cavis-native-lib/src/main/cpp/blas/graph/scheme/variable.fbs diff --git a/libnd4j/include/helpers/ArrayUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ArrayUtils.h similarity index 100% rename from libnd4j/include/helpers/ArrayUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ArrayUtils.h diff --git a/libnd4j/include/helpers/AttentionHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/AttentionHelper.h similarity index 100% rename from libnd4j/include/helpers/AttentionHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/AttentionHelper.h diff --git a/libnd4j/include/helpers/BenchmarkHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/BenchmarkHelper.h similarity index 100% rename from libnd4j/include/helpers/BenchmarkHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/BenchmarkHelper.h diff --git a/libnd4j/include/helpers/BitwiseUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/BitwiseUtils.h similarity index 100% rename from libnd4j/include/helpers/BitwiseUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/BitwiseUtils.h diff --git a/libnd4j/include/helpers/BlasHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/BlasHelper.h similarity index 100% rename from libnd4j/include/helpers/BlasHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/BlasHelper.h diff --git a/libnd4j/include/helpers/ConstantHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ConstantHelper.h similarity index 100% rename from libnd4j/include/helpers/ConstantHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ConstantHelper.h diff --git a/libnd4j/include/helpers/ConstantShapeHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ConstantShapeHelper.h similarity index 100% rename from libnd4j/include/helpers/ConstantShapeHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ConstantShapeHelper.h diff --git a/libnd4j/include/helpers/ConstantTadHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ConstantTadHelper.h similarity index 100% rename from libnd4j/include/helpers/ConstantTadHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ConstantTadHelper.h diff --git a/libnd4j/include/helpers/CudaLaunchHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/CudaLaunchHelper.h similarity index 100% rename from libnd4j/include/helpers/CudaLaunchHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/CudaLaunchHelper.h diff --git a/libnd4j/include/helpers/DebugHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/DebugHelper.h similarity index 100% rename from libnd4j/include/helpers/DebugHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/DebugHelper.h diff --git a/libnd4j/include/helpers/DebugInfo.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/DebugInfo.h similarity index 100% rename from libnd4j/include/helpers/DebugInfo.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/DebugInfo.h diff --git a/libnd4j/include/helpers/EigenValsAndVecs.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/EigenValsAndVecs.h similarity index 100% rename from libnd4j/include/helpers/EigenValsAndVecs.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/EigenValsAndVecs.h diff --git a/libnd4j/include/helpers/EnumUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/EnumUtils.h similarity index 100% rename from libnd4j/include/helpers/EnumUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/EnumUtils.h diff --git a/libnd4j/include/helpers/FullPivLU.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/FullPivLU.h similarity index 100% rename from libnd4j/include/helpers/FullPivLU.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/FullPivLU.h diff --git a/libnd4j/include/helpers/GradCheck.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/GradCheck.h similarity index 100% rename from libnd4j/include/helpers/GradCheck.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/GradCheck.h diff --git a/libnd4j/include/helpers/HessenbergAndSchur.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/HessenbergAndSchur.h similarity index 100% rename from libnd4j/include/helpers/HessenbergAndSchur.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/HessenbergAndSchur.h diff --git a/libnd4j/include/helpers/LoopKind.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/LoopKind.h similarity index 100% rename from libnd4j/include/helpers/LoopKind.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/LoopKind.h diff --git a/libnd4j/include/helpers/Loops.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/Loops.h similarity index 100% rename from libnd4j/include/helpers/Loops.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/Loops.h diff --git a/libnd4j/include/helpers/Loops.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/Loops.hpp similarity index 100% rename from libnd4j/include/helpers/Loops.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/Loops.hpp diff --git a/libnd4j/include/helpers/LoopsCoordsHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/LoopsCoordsHelper.h similarity index 100% rename from libnd4j/include/helpers/LoopsCoordsHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/LoopsCoordsHelper.h diff --git a/libnd4j/include/helpers/MKLDNNStream.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/MKLDNNStream.h similarity index 100% rename from libnd4j/include/helpers/MKLDNNStream.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/MKLDNNStream.h diff --git a/libnd4j/include/helpers/MmulHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/MmulHelper.h similarity index 100% rename from libnd4j/include/helpers/MmulHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/MmulHelper.h diff --git a/libnd4j/include/helpers/OmpLaunchHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/OmpLaunchHelper.h similarity index 100% rename from libnd4j/include/helpers/OmpLaunchHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/OmpLaunchHelper.h diff --git a/libnd4j/include/helpers/OpArgsHolder.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/OpArgsHolder.h similarity index 100% rename from libnd4j/include/helpers/OpArgsHolder.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/OpArgsHolder.h diff --git a/libnd4j/include/helpers/OpBenchmark.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/OpBenchmark.h similarity index 100% rename from libnd4j/include/helpers/OpBenchmark.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/OpBenchmark.h diff --git a/libnd4j/include/helpers/OpTracker.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/OpTracker.h similarity index 100% rename from libnd4j/include/helpers/OpTracker.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/OpTracker.h diff --git a/libnd4j/include/helpers/PointersManager.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/PointersManager.h similarity index 100% rename from libnd4j/include/helpers/PointersManager.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/PointersManager.h diff --git a/libnd4j/include/helpers/RandomLauncher.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/RandomLauncher.h similarity index 100% rename from libnd4j/include/helpers/RandomLauncher.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/RandomLauncher.h diff --git a/libnd4j/include/helpers/ShapeBuilders.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ShapeBuilders.h similarity index 100% rename from libnd4j/include/helpers/ShapeBuilders.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ShapeBuilders.h diff --git a/libnd4j/include/helpers/ShapeUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ShapeUtils.h similarity index 100% rename from libnd4j/include/helpers/ShapeUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/ShapeUtils.h diff --git a/libnd4j/include/helpers/SimpleReadWriteLock.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/SimpleReadWriteLock.h similarity index 100% rename from libnd4j/include/helpers/SimpleReadWriteLock.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/SimpleReadWriteLock.h diff --git a/libnd4j/include/helpers/Sqrtm.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/Sqrtm.h similarity index 100% rename from libnd4j/include/helpers/Sqrtm.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/Sqrtm.h diff --git a/libnd4j/include/helpers/StringUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/StringUtils.h similarity index 100% rename from libnd4j/include/helpers/StringUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/StringUtils.h diff --git a/libnd4j/include/helpers/TAD.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/TAD.h similarity index 100% rename from libnd4j/include/helpers/TAD.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/TAD.h diff --git a/libnd4j/include/helpers/benchmark/BasicSuit.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/BasicSuit.h similarity index 100% rename from libnd4j/include/helpers/benchmark/BasicSuit.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/BasicSuit.h diff --git a/libnd4j/include/helpers/benchmark/BoolParameters.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/BoolParameters.h similarity index 100% rename from libnd4j/include/helpers/benchmark/BoolParameters.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/BoolParameters.h diff --git a/libnd4j/include/helpers/benchmark/BroadcastBenchmark.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/BroadcastBenchmark.h similarity index 100% rename from libnd4j/include/helpers/benchmark/BroadcastBenchmark.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/BroadcastBenchmark.h diff --git a/libnd4j/include/helpers/benchmark/DeclarableBenchmark.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/DeclarableBenchmark.h similarity index 100% rename from libnd4j/include/helpers/benchmark/DeclarableBenchmark.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/DeclarableBenchmark.h diff --git a/libnd4j/include/helpers/benchmark/IntParameters.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/IntParameters.h similarity index 100% rename from libnd4j/include/helpers/benchmark/IntParameters.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/IntParameters.h diff --git a/libnd4j/include/helpers/benchmark/IntPowerParameters.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/IntPowerParameters.h similarity index 100% rename from libnd4j/include/helpers/benchmark/IntPowerParameters.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/IntPowerParameters.h diff --git a/libnd4j/include/helpers/benchmark/MatrixBenchmark.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/MatrixBenchmark.h similarity index 100% rename from libnd4j/include/helpers/benchmark/MatrixBenchmark.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/MatrixBenchmark.h diff --git a/libnd4j/include/helpers/benchmark/PairwiseBenchmark.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/PairwiseBenchmark.h similarity index 100% rename from libnd4j/include/helpers/benchmark/PairwiseBenchmark.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/PairwiseBenchmark.h diff --git a/libnd4j/include/helpers/benchmark/Parameters.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/Parameters.h similarity index 100% rename from libnd4j/include/helpers/benchmark/Parameters.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/Parameters.h diff --git a/libnd4j/include/helpers/benchmark/ParametersBatch.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/ParametersBatch.h similarity index 100% rename from libnd4j/include/helpers/benchmark/ParametersBatch.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/ParametersBatch.h diff --git a/libnd4j/include/helpers/benchmark/ParametersSpace.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/ParametersSpace.h similarity index 100% rename from libnd4j/include/helpers/benchmark/ParametersSpace.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/ParametersSpace.h diff --git a/libnd4j/include/helpers/benchmark/PredefinedParameters.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/PredefinedParameters.h similarity index 100% rename from libnd4j/include/helpers/benchmark/PredefinedParameters.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/PredefinedParameters.h diff --git a/libnd4j/include/helpers/benchmark/ReductionBenchmark.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/ReductionBenchmark.h similarity index 100% rename from libnd4j/include/helpers/benchmark/ReductionBenchmark.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/ReductionBenchmark.h diff --git a/libnd4j/include/helpers/benchmark/ScalarBenchmark.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/ScalarBenchmark.h similarity index 100% rename from libnd4j/include/helpers/benchmark/ScalarBenchmark.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/ScalarBenchmark.h diff --git a/libnd4j/include/helpers/benchmark/TransformBenchmark.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/TransformBenchmark.h similarity index 100% rename from libnd4j/include/helpers/benchmark/TransformBenchmark.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/benchmark/TransformBenchmark.h diff --git a/libnd4j/include/helpers/biDiagonalUp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/biDiagonalUp.h similarity index 100% rename from libnd4j/include/helpers/biDiagonalUp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/biDiagonalUp.h diff --git a/libnd4j/include/helpers/cpu/ConstantHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/ConstantHelper.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/ConstantHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/ConstantHelper.cpp diff --git a/libnd4j/include/helpers/cpu/ConstantShapeHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/ConstantShapeHelper.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/ConstantShapeHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/ConstantShapeHelper.cpp diff --git a/libnd4j/include/helpers/cpu/ConstantTadHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/ConstantTadHelper.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/ConstantTadHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/ConstantTadHelper.cpp diff --git a/libnd4j/include/helpers/cpu/MmulHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/MmulHelper.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/MmulHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/MmulHelper.cpp diff --git a/libnd4j/include/helpers/cpu/PointersManager.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/PointersManager.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/PointersManager.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/PointersManager.cpp diff --git a/libnd4j/include/helpers/cpu/cublasHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/cublasHelper.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/cublasHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/cublasHelper.cpp diff --git a/libnd4j/include/helpers/cpu/loops/IndexReductionLoops.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/IndexReductionLoops.hpp similarity index 100% rename from libnd4j/include/helpers/cpu/loops/IndexReductionLoops.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/IndexReductionLoops.hpp diff --git a/libnd4j/include/helpers/cpu/loops/IndexReductionLoops_int32.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/IndexReductionLoops_int32.cpp.in similarity index 100% rename from libnd4j/include/helpers/cpu/loops/IndexReductionLoops_int32.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/IndexReductionLoops_int32.cpp.in diff --git a/libnd4j/include/helpers/cpu/loops/IndexReductionLoops_int64.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/IndexReductionLoops_int64.cpp.in similarity index 100% rename from libnd4j/include/helpers/cpu/loops/IndexReductionLoops_int64.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/IndexReductionLoops_int64.cpp.in diff --git a/libnd4j/include/helpers/cpu/loops/Reduction3Loops.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/Reduction3Loops.cpp.in similarity index 100% rename from libnd4j/include/helpers/cpu/loops/Reduction3Loops.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/Reduction3Loops.cpp.in diff --git a/libnd4j/include/helpers/cpu/loops/Reduction3Loops.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/Reduction3Loops.hpp similarity index 100% rename from libnd4j/include/helpers/cpu/loops/Reduction3Loops.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/Reduction3Loops.hpp diff --git a/libnd4j/include/helpers/cpu/loops/ReductionLoops.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops.hpp similarity index 100% rename from libnd4j/include/helpers/cpu/loops/ReductionLoops.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops.hpp diff --git a/libnd4j/include/helpers/cpu/loops/ReductionLoops_bool.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_bool.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/loops/ReductionLoops_bool.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_bool.cpp diff --git a/libnd4j/include/helpers/cpu/loops/ReductionLoops_float.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_float.cpp.in similarity index 100% rename from libnd4j/include/helpers/cpu/loops/ReductionLoops_float.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_float.cpp.in diff --git a/libnd4j/include/helpers/cpu/loops/ReductionLoops_float.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_float.hpp similarity index 100% rename from libnd4j/include/helpers/cpu/loops/ReductionLoops_float.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_float.hpp diff --git a/libnd4j/include/helpers/cpu/loops/ReductionLoops_long.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_long.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/loops/ReductionLoops_long.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_long.cpp diff --git a/libnd4j/include/helpers/cpu/loops/ReductionLoops_same.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_same.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/loops/ReductionLoops_same.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/loops/ReductionLoops_same.cpp diff --git a/libnd4j/include/helpers/cpu/svd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/svd.cpp similarity index 100% rename from libnd4j/include/helpers/cpu/svd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cpu/svd.cpp diff --git a/libnd4j/include/helpers/cublasHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cublasHelper.h similarity index 100% rename from libnd4j/include/helpers/cublasHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cublasHelper.h diff --git a/libnd4j/include/helpers/cuda/ConstantHelper.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda/ConstantHelper.cu similarity index 100% rename from libnd4j/include/helpers/cuda/ConstantHelper.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda/ConstantHelper.cu diff --git a/libnd4j/include/helpers/cuda/ConstantShapeHelper.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda/ConstantShapeHelper.cu similarity index 100% rename from libnd4j/include/helpers/cuda/ConstantShapeHelper.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda/ConstantShapeHelper.cu diff --git a/libnd4j/include/helpers/cuda/ConstantTadHelper.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda/ConstantTadHelper.cu similarity index 100% rename from libnd4j/include/helpers/cuda/ConstantTadHelper.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda/ConstantTadHelper.cu diff --git a/libnd4j/include/helpers/cuda/PointersManager.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda/PointersManager.cu similarity index 100% rename from libnd4j/include/helpers/cuda/PointersManager.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda/PointersManager.cu diff --git a/libnd4j/include/helpers/cuda_off/MmulHelper.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda_off/MmulHelper.cu similarity index 100% rename from libnd4j/include/helpers/cuda_off/MmulHelper.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda_off/MmulHelper.cu diff --git a/libnd4j/include/helpers/cuda_off/cublasHelper.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda_off/cublasHelper.cu similarity index 100% rename from libnd4j/include/helpers/cuda_off/cublasHelper.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/cuda_off/cublasHelper.cu diff --git a/libnd4j/include/helpers/data_gen.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/data_gen.h similarity index 100% rename from libnd4j/include/helpers/data_gen.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/data_gen.h diff --git a/libnd4j/include/helpers/files.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/files.h similarity index 100% rename from libnd4j/include/helpers/files.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/files.h diff --git a/libnd4j/include/helpers/helper_generator.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/helper_generator.h similarity index 100% rename from libnd4j/include/helpers/helper_generator.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/helper_generator.h diff --git a/libnd4j/include/helpers/helper_hash.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/helper_hash.h similarity index 100% rename from libnd4j/include/helpers/helper_hash.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/helper_hash.h diff --git a/libnd4j/include/helpers/helper_ptrmap.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/helper_ptrmap.h similarity index 100% rename from libnd4j/include/helpers/helper_ptrmap.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/helper_ptrmap.h diff --git a/libnd4j/include/helpers/helper_random.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/helper_random.h similarity index 100% rename from libnd4j/include/helpers/helper_random.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/helper_random.h diff --git a/libnd4j/include/helpers/hhColPivQR.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/hhColPivQR.h similarity index 100% rename from libnd4j/include/helpers/hhColPivQR.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/hhColPivQR.h diff --git a/libnd4j/include/helpers/hhSequence.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/hhSequence.h similarity index 100% rename from libnd4j/include/helpers/hhSequence.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/hhSequence.h diff --git a/libnd4j/include/helpers/householder.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/householder.h similarity index 100% rename from libnd4j/include/helpers/householder.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/householder.h diff --git a/libnd4j/include/helpers/impl/ArrayUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/ArrayUtils.cpp similarity index 100% rename from libnd4j/include/helpers/impl/ArrayUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/ArrayUtils.cpp diff --git a/libnd4j/include/helpers/impl/AttentionHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/AttentionHelper.cpp similarity index 100% rename from libnd4j/include/helpers/impl/AttentionHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/AttentionHelper.cpp diff --git a/libnd4j/include/helpers/impl/BenchmarkHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/BenchmarkHelper.cpp similarity index 100% rename from libnd4j/include/helpers/impl/BenchmarkHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/BenchmarkHelper.cpp diff --git a/libnd4j/include/helpers/impl/BitwiseUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/BitwiseUtils.cpp similarity index 100% rename from libnd4j/include/helpers/impl/BitwiseUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/BitwiseUtils.cpp diff --git a/libnd4j/include/helpers/impl/BlasHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/BlasHelper.cpp similarity index 100% rename from libnd4j/include/helpers/impl/BlasHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/BlasHelper.cpp diff --git a/libnd4j/include/helpers/impl/CudaLaunchHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/CudaLaunchHelper.cpp similarity index 100% rename from libnd4j/include/helpers/impl/CudaLaunchHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/CudaLaunchHelper.cpp diff --git a/libnd4j/include/helpers/impl/DebugHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/DebugHelper.cpp similarity index 100% rename from libnd4j/include/helpers/impl/DebugHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/DebugHelper.cpp diff --git a/libnd4j/include/helpers/impl/EigenValsAndVecs.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/EigenValsAndVecs.cpp similarity index 100% rename from libnd4j/include/helpers/impl/EigenValsAndVecs.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/EigenValsAndVecs.cpp diff --git a/libnd4j/include/helpers/impl/EnumUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/EnumUtils.cpp similarity index 100% rename from libnd4j/include/helpers/impl/EnumUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/EnumUtils.cpp diff --git a/libnd4j/include/helpers/impl/FullPivLU.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/FullPivLU.cpp similarity index 100% rename from libnd4j/include/helpers/impl/FullPivLU.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/FullPivLU.cpp diff --git a/libnd4j/include/helpers/impl/GradCheck.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/GradCheck.cpp similarity index 100% rename from libnd4j/include/helpers/impl/GradCheck.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/GradCheck.cpp diff --git a/libnd4j/include/helpers/impl/HessenbergAndSchur.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/HessenbergAndSchur.cpp similarity index 100% rename from libnd4j/include/helpers/impl/HessenbergAndSchur.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/HessenbergAndSchur.cpp diff --git a/libnd4j/include/helpers/impl/MmulHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/MmulHelper.cpp similarity index 100% rename from libnd4j/include/helpers/impl/MmulHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/MmulHelper.cpp diff --git a/libnd4j/include/helpers/impl/OmpLaunchHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/OmpLaunchHelper.cpp similarity index 100% rename from libnd4j/include/helpers/impl/OmpLaunchHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/OmpLaunchHelper.cpp diff --git a/libnd4j/include/helpers/impl/OpArgsHolder.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/OpArgsHolder.cpp similarity index 100% rename from libnd4j/include/helpers/impl/OpArgsHolder.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/OpArgsHolder.cpp diff --git a/libnd4j/include/helpers/impl/OpBenchmark.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/OpBenchmark.cpp similarity index 100% rename from libnd4j/include/helpers/impl/OpBenchmark.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/OpBenchmark.cpp diff --git a/libnd4j/include/helpers/impl/OpTracker.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/OpTracker.cpp similarity index 100% rename from libnd4j/include/helpers/impl/OpTracker.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/OpTracker.cpp diff --git a/libnd4j/include/helpers/impl/Parameters.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/Parameters.cpp similarity index 100% rename from libnd4j/include/helpers/impl/Parameters.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/Parameters.cpp diff --git a/libnd4j/include/helpers/impl/RandomLauncher.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/RandomLauncher.cpp similarity index 100% rename from libnd4j/include/helpers/impl/RandomLauncher.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/RandomLauncher.cpp diff --git a/libnd4j/include/helpers/impl/ShapeBuilders.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/ShapeBuilders.cpp similarity index 100% rename from libnd4j/include/helpers/impl/ShapeBuilders.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/ShapeBuilders.cpp diff --git a/libnd4j/include/helpers/impl/ShapeUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/ShapeUtils.cpp similarity index 100% rename from libnd4j/include/helpers/impl/ShapeUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/ShapeUtils.cpp diff --git a/libnd4j/include/helpers/impl/SimpleReadWriteLock.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/SimpleReadWriteLock.cpp similarity index 100% rename from libnd4j/include/helpers/impl/SimpleReadWriteLock.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/SimpleReadWriteLock.cpp diff --git a/libnd4j/include/helpers/impl/Sqrtm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/Sqrtm.cpp similarity index 100% rename from libnd4j/include/helpers/impl/Sqrtm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/Sqrtm.cpp diff --git a/libnd4j/include/helpers/impl/StringUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/StringUtils.cpp similarity index 100% rename from libnd4j/include/helpers/impl/StringUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/StringUtils.cpp diff --git a/libnd4j/include/helpers/impl/TAD.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/TAD.cpp similarity index 100% rename from libnd4j/include/helpers/impl/TAD.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/TAD.cpp diff --git a/libnd4j/include/helpers/impl/biDiagonalUp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/biDiagonalUp.cpp similarity index 100% rename from libnd4j/include/helpers/impl/biDiagonalUp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/biDiagonalUp.cpp diff --git a/libnd4j/include/helpers/impl/helper_hash.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/helper_hash.cpp similarity index 100% rename from libnd4j/include/helpers/impl/helper_hash.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/helper_hash.cpp diff --git a/libnd4j/include/helpers/impl/hhColPivQR.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/hhColPivQR.cpp similarity index 100% rename from libnd4j/include/helpers/impl/hhColPivQR.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/hhColPivQR.cpp diff --git a/libnd4j/include/helpers/impl/hhSequence.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/hhSequence.cpp similarity index 100% rename from libnd4j/include/helpers/impl/hhSequence.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/hhSequence.cpp diff --git a/libnd4j/include/helpers/impl/householder.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/householder.cpp similarity index 100% rename from libnd4j/include/helpers/impl/householder.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/householder.cpp diff --git a/libnd4j/include/helpers/impl/jacobiSVD.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/jacobiSVD.cpp similarity index 100% rename from libnd4j/include/helpers/impl/jacobiSVD.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/jacobiSVD.cpp diff --git a/libnd4j/include/helpers/impl/logger.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/logger.cpp similarity index 100% rename from libnd4j/include/helpers/impl/logger.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/logger.cpp diff --git a/libnd4j/include/helpers/impl/shape.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/shape.cpp similarity index 100% rename from libnd4j/include/helpers/impl/shape.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/shape.cpp diff --git a/libnd4j/include/helpers/impl/unicode.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/unicode.cpp similarity index 100% rename from libnd4j/include/helpers/impl/unicode.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/impl/unicode.cpp diff --git a/libnd4j/include/helpers/jacobiSVD.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/jacobiSVD.h similarity index 100% rename from libnd4j/include/helpers/jacobiSVD.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/jacobiSVD.h diff --git a/libnd4j/include/helpers/logger.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/logger.h similarity index 100% rename from libnd4j/include/helpers/logger.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/logger.h diff --git a/libnd4j/include/helpers/mman.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/mman.h similarity index 100% rename from libnd4j/include/helpers/mman.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/mman.h diff --git a/libnd4j/include/helpers/shape.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/shape.h similarity index 99% rename from libnd4j/include/helpers/shape.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/shape.h index 63e532f32..f8e71f45c 100644 --- a/libnd4j/include/helpers/shape.h +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/shape.h @@ -4519,7 +4519,7 @@ INLINEDEF _CUDA_HD void maxIndToMinInd(int* maxIdxs, int* minIdxs, const Nd4jLon maxI = rankMax-1; N = 0; - int step; + int step = 0; maxOffsets[N++] = shape::getOffset(maxShapeInfo, indices); // nested loops - producing of absolute indices for max array @@ -4593,7 +4593,7 @@ INLINEDEF _CUDA_HD void maxIndToMinInd(int* maxIdxs, int* minIdxs, const Nd4jLon maxI = rankMax-1; N = 0; - int step; + int step=0; maxIdxs[N++] = shape::coords2index(maxShapeInfo, indices); // nested loops - producing of absolute indices for max array diff --git a/libnd4j/include/helpers/svd.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/svd.h similarity index 100% rename from libnd4j/include/helpers/svd.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/svd.h diff --git a/libnd4j/include/helpers/threshold.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/threshold.h similarity index 100% rename from libnd4j/include/helpers/threshold.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/threshold.h diff --git a/libnd4j/include/helpers/unicode.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/unicode.h similarity index 100% rename from libnd4j/include/helpers/unicode.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/helpers/unicode.h diff --git a/libnd4j/include/indexing/IndicesList.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/indexing/IndicesList.h similarity index 100% rename from libnd4j/include/indexing/IndicesList.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/indexing/IndicesList.h diff --git a/libnd4j/include/indexing/NDIndex.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/indexing/NDIndex.h similarity index 100% rename from libnd4j/include/indexing/NDIndex.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/indexing/NDIndex.h diff --git a/libnd4j/include/indexing/impl/IndicesList.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/indexing/impl/IndicesList.cpp similarity index 100% rename from libnd4j/include/indexing/impl/IndicesList.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/indexing/impl/IndicesList.cpp diff --git a/libnd4j/include/indexing/impl/NDIndex.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/indexing/impl/NDIndex.cpp similarity index 100% rename from libnd4j/include/indexing/impl/NDIndex.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/indexing/impl/NDIndex.cpp diff --git a/libnd4j/include/legacy/NativeOpExecutioner.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/NativeOpExecutioner.h similarity index 100% rename from libnd4j/include/legacy/NativeOpExecutioner.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/NativeOpExecutioner.h diff --git a/libnd4j/include/legacy/NativeOps.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/NativeOps.h old mode 100755 new mode 100644 similarity index 99% rename from libnd4j/include/legacy/NativeOps.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/NativeOps.h index 74e371a15..6bc1f4fe1 --- a/libnd4j/include/legacy/NativeOps.h +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/NativeOps.h @@ -91,7 +91,7 @@ extern "C" { ND4J_EXPORT int lastErrorCode(); /** - * This function returns last error message, if last error code > 0 + * This function returns last error message, if last error code > 0 * @return */ ND4J_EXPORT const char* lastErrorMessage(); @@ -1109,7 +1109,7 @@ static Nd4jPointer _numpyHeaderForNd4j(Nd4jPointer data,const Nd4jPointer shapeB npShape[i] = shape[i]; } - Nd4jLong length = shape::prodLong(shape,rank); + //Nd4jLong length = shape::prodLong(shape,rank); auto npHeader = cnpy::createNpyHeader(data,npShape,rank,wordSize); char *ret = new char[npHeader.size() + 1]; int count = 0; diff --git a/libnd4j/include/legacy/cpu/NativeOpExecutioner.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cpu/NativeOpExecutioner.cpp similarity index 100% rename from libnd4j/include/legacy/cpu/NativeOpExecutioner.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cpu/NativeOpExecutioner.cpp diff --git a/libnd4j/include/legacy/cpu/NativeOps.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cpu/NativeOps.cpp similarity index 100% rename from libnd4j/include/legacy/cpu/NativeOps.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cpu/NativeOps.cpp diff --git a/libnd4j/include/legacy/cuda/BlasVersionHelper.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cuda/BlasVersionHelper.cu similarity index 100% rename from libnd4j/include/legacy/cuda/BlasVersionHelper.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cuda/BlasVersionHelper.cu diff --git a/libnd4j/include/legacy/cuda/NativeOpExecutioner.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cuda/NativeOpExecutioner.cu similarity index 100% rename from libnd4j/include/legacy/cuda/NativeOpExecutioner.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cuda/NativeOpExecutioner.cu diff --git a/libnd4j/include/legacy/cuda/NativeOps.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cuda/NativeOps.cu old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/legacy/cuda/NativeOps.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/cuda/NativeOps.cu diff --git a/libnd4j/include/legacy/impl/Environment.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/impl/Environment.cpp similarity index 100% rename from libnd4j/include/legacy/impl/Environment.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/impl/Environment.cpp diff --git a/libnd4j/include/legacy/impl/cnpy.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/impl/cnpy.cpp similarity index 100% rename from libnd4j/include/legacy/impl/cnpy.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/legacy/impl/cnpy.cpp diff --git a/libnd4j/include/loops/BroadcastPairwiseConverter.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/BroadcastPairwiseConverter.h similarity index 100% rename from libnd4j/include/loops/BroadcastPairwiseConverter.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/BroadcastPairwiseConverter.h diff --git a/libnd4j/include/loops/BroadcastScalarConverter.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/BroadcastScalarConverter.h similarity index 100% rename from libnd4j/include/loops/BroadcastScalarConverter.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/BroadcastScalarConverter.h diff --git a/libnd4j/include/loops/ReduceType.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/ReduceType.h similarity index 100% rename from libnd4j/include/loops/ReduceType.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/ReduceType.h diff --git a/libnd4j/include/loops/broadcasting.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/broadcasting.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/loops/broadcasting.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/broadcasting.h diff --git a/libnd4j/include/loops/broadcasting_bool.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/broadcasting_bool.h similarity index 100% rename from libnd4j/include/loops/broadcasting_bool.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/broadcasting_bool.h diff --git a/libnd4j/include/loops/broadcasting_int.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/broadcasting_int.h similarity index 100% rename from libnd4j/include/loops/broadcasting_int.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/broadcasting_int.h diff --git a/libnd4j/include/loops/cpu/broadcasting.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/broadcasting.hpp similarity index 100% rename from libnd4j/include/loops/cpu/broadcasting.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/broadcasting.hpp diff --git a/libnd4j/include/loops/cpu/broadcasting_bool.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/broadcasting_bool.hpp similarity index 100% rename from libnd4j/include/loops/cpu/broadcasting_bool.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/broadcasting_bool.hpp diff --git a/libnd4j/include/loops/cpu/broadcasting_int.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/broadcasting_int.hpp similarity index 100% rename from libnd4j/include/loops/cpu/broadcasting_int.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/broadcasting_int.hpp diff --git a/libnd4j/include/loops/cpu/compilation_units/broadcast_bool_p.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/broadcast_bool_p.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/broadcast_bool_p.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/broadcast_bool_p.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/broadcast_int_p.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/broadcast_int_p.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/broadcast_int_p.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/broadcast_int_p.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/broadcast_p.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/broadcast_p.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/broadcast_p.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/broadcast_p.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/indexreduce_int32.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/indexreduce_int32.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/indexreduce_int32.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/indexreduce_int32.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/indexreduce_int64.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/indexreduce_int64.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/indexreduce_int64.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/indexreduce_int64.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/pairwise_p.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/pairwise_p.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/pairwise_p.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/pairwise_p.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/random.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/random.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/random.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/random.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/reduce3_bfloat16.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce3_bfloat16.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/reduce3_bfloat16.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce3_bfloat16.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/reduce3_double.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce3_double.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/reduce3_double.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce3_double.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/reduce3_float.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce3_float.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/reduce3_float.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce3_float.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/reduce3_float16.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce3_float16.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/reduce3_float16.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce3_float16.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/reduce_float.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce_float.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/reduce_float.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/reduce_float.cpp.in diff --git a/libnd4j/include/loops/cpu/compilation_units/scalar_p.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/scalar_p.cpp.in similarity index 100% rename from libnd4j/include/loops/cpu/compilation_units/scalar_p.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/compilation_units/scalar_p.cpp.in diff --git a/libnd4j/include/loops/cpu/indexreduce.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/indexreduce.hpp similarity index 100% rename from libnd4j/include/loops/cpu/indexreduce.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/indexreduce.hpp diff --git a/libnd4j/include/loops/cpu/pairwise.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/pairwise.hpp similarity index 100% rename from libnd4j/include/loops/cpu/pairwise.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/pairwise.hpp diff --git a/libnd4j/include/loops/cpu/pairwise_bool.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/pairwise_bool.cpp similarity index 100% rename from libnd4j/include/loops/cpu/pairwise_bool.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/pairwise_bool.cpp diff --git a/libnd4j/include/loops/cpu/pairwise_int.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/pairwise_int.cpp similarity index 100% rename from libnd4j/include/loops/cpu/pairwise_int.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/pairwise_int.cpp diff --git a/libnd4j/include/loops/cpu/random.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/random.hpp similarity index 100% rename from libnd4j/include/loops/cpu/random.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/random.hpp diff --git a/libnd4j/include/loops/cpu/reduce/reduce_bool.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce/reduce_bool.cpp similarity index 100% rename from libnd4j/include/loops/cpu/reduce/reduce_bool.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce/reduce_bool.cpp diff --git a/libnd4j/include/loops/cpu/reduce/reduce_float.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce/reduce_float.hpp similarity index 100% rename from libnd4j/include/loops/cpu/reduce/reduce_float.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce/reduce_float.hpp diff --git a/libnd4j/include/loops/cpu/reduce/reduce_long.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce/reduce_long.cpp similarity index 100% rename from libnd4j/include/loops/cpu/reduce/reduce_long.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce/reduce_long.cpp diff --git a/libnd4j/include/loops/cpu/reduce/reduce_same.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce/reduce_same.cpp similarity index 100% rename from libnd4j/include/loops/cpu/reduce/reduce_same.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce/reduce_same.cpp diff --git a/libnd4j/include/loops/cpu/reduce3.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce3.hpp similarity index 100% rename from libnd4j/include/loops/cpu/reduce3.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/reduce3.hpp diff --git a/libnd4j/include/loops/cpu/scalar.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/scalar.hpp similarity index 100% rename from libnd4j/include/loops/cpu/scalar.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/scalar.hpp diff --git a/libnd4j/include/loops/cpu/scalar_bool.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/scalar_bool.cpp similarity index 100% rename from libnd4j/include/loops/cpu/scalar_bool.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/scalar_bool.cpp diff --git a/libnd4j/include/loops/cpu/scalar_int.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/scalar_int.cpp similarity index 100% rename from libnd4j/include/loops/cpu/scalar_int.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/scalar_int.cpp diff --git a/libnd4j/include/loops/cpu/summarystatsreduce.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/summarystatsreduce.cpp similarity index 100% rename from libnd4j/include/loops/cpu/summarystatsreduce.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/summarystatsreduce.cpp diff --git a/libnd4j/include/loops/cpu/transform/transform_any.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_any.cpp similarity index 100% rename from libnd4j/include/loops/cpu/transform/transform_any.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_any.cpp diff --git a/libnd4j/include/loops/cpu/transform/transform_bool.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_bool.cpp similarity index 100% rename from libnd4j/include/loops/cpu/transform/transform_bool.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_bool.cpp diff --git a/libnd4j/include/loops/cpu/transform/transform_float.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_float.cpp similarity index 100% rename from libnd4j/include/loops/cpu/transform/transform_float.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_float.cpp diff --git a/libnd4j/include/loops/cpu/transform/transform_same.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_same.cpp similarity index 100% rename from libnd4j/include/loops/cpu/transform/transform_same.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_same.cpp diff --git a/libnd4j/include/loops/cpu/transform/transform_strict.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_strict.cpp similarity index 100% rename from libnd4j/include/loops/cpu/transform/transform_strict.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cpu/transform/transform_strict.cpp diff --git a/libnd4j/include/loops/cuda/broadcasting.chpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/broadcasting.chpp similarity index 100% rename from libnd4j/include/loops/cuda/broadcasting.chpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/broadcasting.chpp diff --git a/libnd4j/include/loops/cuda/broadcasting.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/broadcasting.cu similarity index 100% rename from libnd4j/include/loops/cuda/broadcasting.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/broadcasting.cu diff --git a/libnd4j/include/loops/cuda/broadcasting_bool.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/broadcasting_bool.cu similarity index 100% rename from libnd4j/include/loops/cuda/broadcasting_bool.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/broadcasting_bool.cu diff --git a/libnd4j/include/loops/cuda/broadcasting_int.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/broadcasting_int.cu similarity index 100% rename from libnd4j/include/loops/cuda/broadcasting_int.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/broadcasting_int.cu diff --git a/libnd4j/include/loops/cuda/compilation_units/broadcasting.cu.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/broadcasting.cu.in similarity index 100% rename from libnd4j/include/loops/cuda/compilation_units/broadcasting.cu.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/broadcasting.cu.in diff --git a/libnd4j/include/loops/cuda/compilation_units/pairwise.cu.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/pairwise.cu.in similarity index 100% rename from libnd4j/include/loops/cuda/compilation_units/pairwise.cu.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/pairwise.cu.in diff --git a/libnd4j/include/loops/cuda/compilation_units/reduce3.cu.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/reduce3.cu.in similarity index 100% rename from libnd4j/include/loops/cuda/compilation_units/reduce3.cu.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/reduce3.cu.in diff --git a/libnd4j/include/loops/cuda/compilation_units/reduce_float.cu.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/reduce_float.cu.in similarity index 100% rename from libnd4j/include/loops/cuda/compilation_units/reduce_float.cu.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/reduce_float.cu.in diff --git a/libnd4j/include/loops/cuda/compilation_units/scalar.cu.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/scalar.cu.in similarity index 100% rename from libnd4j/include/loops/cuda/compilation_units/scalar.cu.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/compilation_units/scalar.cu.in diff --git a/libnd4j/include/loops/cuda/indexreduce.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/indexreduce.cu similarity index 100% rename from libnd4j/include/loops/cuda/indexreduce.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/indexreduce.cu diff --git a/libnd4j/include/loops/cuda/inplace_loops/README.md b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/inplace_loops/README.md similarity index 100% rename from libnd4j/include/loops/cuda/inplace_loops/README.md rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/inplace_loops/README.md diff --git a/libnd4j/include/loops/cuda/inplace_loops/reduce_same_inplace.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/inplace_loops/reduce_same_inplace.h similarity index 100% rename from libnd4j/include/loops/cuda/inplace_loops/reduce_same_inplace.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/inplace_loops/reduce_same_inplace.h diff --git a/libnd4j/include/loops/cuda/inplace_loops/scalar_inplace.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/inplace_loops/scalar_inplace.h similarity index 100% rename from libnd4j/include/loops/cuda/inplace_loops/scalar_inplace.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/inplace_loops/scalar_inplace.h diff --git a/libnd4j/include/loops/cuda/inplace_loops/transform_strict_inplace.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/inplace_loops/transform_strict_inplace.h similarity index 100% rename from libnd4j/include/loops/cuda/inplace_loops/transform_strict_inplace.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/inplace_loops/transform_strict_inplace.h diff --git a/libnd4j/include/loops/cuda/legacy/grid_shaped.legacy b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/grid_shaped.legacy similarity index 100% rename from libnd4j/include/loops/cuda/legacy/grid_shaped.legacy rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/grid_shaped.legacy diff --git a/libnd4j/include/loops/cuda/legacy/grid_strided.legacy b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/grid_strided.legacy similarity index 100% rename from libnd4j/include/loops/cuda/legacy/grid_strided.legacy rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/grid_strided.legacy diff --git a/libnd4j/include/loops/cuda/legacy/reduce.legacy b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/reduce.legacy similarity index 100% rename from libnd4j/include/loops/cuda/legacy/reduce.legacy rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/reduce.legacy diff --git a/libnd4j/include/loops/cuda/legacy/scalar_temp.legacy b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/scalar_temp.legacy similarity index 100% rename from libnd4j/include/loops/cuda/legacy/scalar_temp.legacy rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/scalar_temp.legacy diff --git a/libnd4j/include/loops/cuda/legacy/transform.legacy b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/transform.legacy similarity index 100% rename from libnd4j/include/loops/cuda/legacy/transform.legacy rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/legacy/transform.legacy diff --git a/libnd4j/include/loops/cuda/pairwise.chpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/pairwise.chpp similarity index 100% rename from libnd4j/include/loops/cuda/pairwise.chpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/pairwise.chpp diff --git a/libnd4j/include/loops/cuda/pairwise.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/pairwise.cu similarity index 100% rename from libnd4j/include/loops/cuda/pairwise.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/pairwise.cu diff --git a/libnd4j/include/loops/cuda/pairwise_bool.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/pairwise_bool.cu similarity index 100% rename from libnd4j/include/loops/cuda/pairwise_bool.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/pairwise_bool.cu diff --git a/libnd4j/include/loops/cuda/pairwise_int.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/pairwise_int.cu similarity index 100% rename from libnd4j/include/loops/cuda/pairwise_int.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/pairwise_int.cu diff --git a/libnd4j/include/loops/cuda/random.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/random.cu similarity index 100% rename from libnd4j/include/loops/cuda/random.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/random.cu diff --git a/libnd4j/include/loops/cuda/reduce/reduce_bool.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce/reduce_bool.cu similarity index 100% rename from libnd4j/include/loops/cuda/reduce/reduce_bool.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce/reduce_bool.cu diff --git a/libnd4j/include/loops/cuda/reduce/reduce_float.chpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce/reduce_float.chpp similarity index 100% rename from libnd4j/include/loops/cuda/reduce/reduce_float.chpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce/reduce_float.chpp diff --git a/libnd4j/include/loops/cuda/reduce/reduce_long.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce/reduce_long.cu similarity index 100% rename from libnd4j/include/loops/cuda/reduce/reduce_long.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce/reduce_long.cu diff --git a/libnd4j/include/loops/cuda/reduce/reduce_same.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce/reduce_same.cu similarity index 100% rename from libnd4j/include/loops/cuda/reduce/reduce_same.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce/reduce_same.cu diff --git a/libnd4j/include/loops/cuda/reduce3.chpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce3.chpp similarity index 100% rename from libnd4j/include/loops/cuda/reduce3.chpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/reduce3.chpp diff --git a/libnd4j/include/loops/cuda/scalar.chpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/scalar.chpp similarity index 100% rename from libnd4j/include/loops/cuda/scalar.chpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/scalar.chpp diff --git a/libnd4j/include/loops/cuda/scalar.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/scalar.cu similarity index 100% rename from libnd4j/include/loops/cuda/scalar.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/scalar.cu diff --git a/libnd4j/include/loops/cuda/scalar_bool.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/scalar_bool.cu similarity index 100% rename from libnd4j/include/loops/cuda/scalar_bool.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/scalar_bool.cu diff --git a/libnd4j/include/loops/cuda/scalar_int.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/scalar_int.cu similarity index 100% rename from libnd4j/include/loops/cuda/scalar_int.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/scalar_int.cu diff --git a/libnd4j/include/loops/cuda/specials/accumulateKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/accumulateKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/accumulateKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/accumulateKernel.cu diff --git a/libnd4j/include/loops/cuda/specials/averagingKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/averagingKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/averagingKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/averagingKernel.cu diff --git a/libnd4j/include/loops/cuda/specials/bitonicArbitraryStep.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/bitonicArbitraryStep.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/bitonicArbitraryStep.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/bitonicArbitraryStep.cu diff --git a/libnd4j/include/loops/cuda/specials/bitonicSortStep.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/bitonicSortStep.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/bitonicSortStep.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/bitonicSortStep.cu diff --git a/libnd4j/include/loops/cuda/specials/concatKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/concatKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/concatKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/concatKernel.cu diff --git a/libnd4j/include/loops/cuda/specials/concatKernelHStack.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/concatKernelHStack.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/concatKernelHStack.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/concatKernelHStack.cu diff --git a/libnd4j/include/loops/cuda/specials/concatKernelScalar.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/concatKernelScalar.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/concatKernelScalar.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/concatKernelScalar.cu diff --git a/libnd4j/include/loops/cuda/specials/concatKernelVStack.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/concatKernelVStack.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/concatKernelVStack.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/concatKernelVStack.cu diff --git a/libnd4j/include/loops/cuda/specials/convertHalfs.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/convertHalfs.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/convertHalfs.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/convertHalfs.cu diff --git a/libnd4j/include/loops/cuda/specials/convertToHalf.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/convertToHalf.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/convertToHalf.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/convertToHalf.cu diff --git a/libnd4j/include/loops/cuda/specials/fillDimensionalIsMax.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/fillDimensionalIsMax.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/fillDimensionalIsMax.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/fillDimensionalIsMax.cu diff --git a/libnd4j/include/loops/cuda/specials/fillIsMax.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/fillIsMax.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/fillIsMax.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/fillIsMax.cu diff --git a/libnd4j/include/loops/cuda/specials/flatten.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/flatten.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/flatten.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/flatten.cu diff --git a/libnd4j/include/loops/cuda/specials/oesTad.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/oesTad.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/oesTad.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/oesTad.cu diff --git a/libnd4j/include/loops/cuda/specials/pullRowsKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/pullRowsKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/pullRowsKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/pullRowsKernel.cu diff --git a/libnd4j/include/loops/cuda/specials/setDiagonalKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/setDiagonalKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/setDiagonalKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/setDiagonalKernel.cu diff --git a/libnd4j/include/loops/cuda/specials/shuffleKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/shuffleKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/shuffleKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/shuffleKernel.cu diff --git a/libnd4j/include/loops/cuda/specials/swapUnsafeKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/swapUnsafeKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/swapUnsafeKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/swapUnsafeKernel.cu diff --git a/libnd4j/include/loops/cuda/specials/tearKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/tearKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/tearKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/tearKernel.cu diff --git a/libnd4j/include/loops/cuda/specials/tileKernel.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/tileKernel.cu similarity index 100% rename from libnd4j/include/loops/cuda/specials/tileKernel.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/specials/tileKernel.cu diff --git a/libnd4j/include/loops/cuda/summarystatsreduce.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/summarystatsreduce.cu similarity index 100% rename from libnd4j/include/loops/cuda/summarystatsreduce.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/summarystatsreduce.cu diff --git a/libnd4j/include/loops/cuda/transform/transform_any.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_any.cu similarity index 100% rename from libnd4j/include/loops/cuda/transform/transform_any.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_any.cu diff --git a/libnd4j/include/loops/cuda/transform/transform_bool.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_bool.cu similarity index 100% rename from libnd4j/include/loops/cuda/transform/transform_bool.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_bool.cu diff --git a/libnd4j/include/loops/cuda/transform/transform_float.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_float.cu similarity index 100% rename from libnd4j/include/loops/cuda/transform/transform_float.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_float.cu diff --git a/libnd4j/include/loops/cuda/transform/transform_same.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_same.cu similarity index 100% rename from libnd4j/include/loops/cuda/transform/transform_same.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_same.cu diff --git a/libnd4j/include/loops/cuda/transform/transform_strict.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_strict.cu similarity index 100% rename from libnd4j/include/loops/cuda/transform/transform_strict.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/transform/transform_strict.cu diff --git a/libnd4j/include/loops/cuda/type_conversions.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/type_conversions.cu similarity index 100% rename from libnd4j/include/loops/cuda/type_conversions.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/cuda/type_conversions.cu diff --git a/libnd4j/include/loops/grid_shaped.legacy b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/grid_shaped.legacy similarity index 100% rename from libnd4j/include/loops/grid_shaped.legacy rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/grid_shaped.legacy diff --git a/libnd4j/include/loops/grid_strided.legacy b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/grid_strided.legacy similarity index 100% rename from libnd4j/include/loops/grid_strided.legacy rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/grid_strided.legacy diff --git a/libnd4j/include/loops/impl/type_conversions.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/impl/type_conversions.cpp similarity index 100% rename from libnd4j/include/loops/impl/type_conversions.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/impl/type_conversions.cpp diff --git a/libnd4j/include/loops/indexreduce.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/indexreduce.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/loops/indexreduce.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/indexreduce.h diff --git a/libnd4j/include/loops/legacy_ops.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/legacy_ops.h similarity index 100% rename from libnd4j/include/loops/legacy_ops.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/legacy_ops.h diff --git a/libnd4j/include/loops/pairwise_bool.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/pairwise_bool.h similarity index 100% rename from libnd4j/include/loops/pairwise_bool.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/pairwise_bool.h diff --git a/libnd4j/include/loops/pairwise_int.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/pairwise_int.h similarity index 100% rename from libnd4j/include/loops/pairwise_int.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/pairwise_int.h diff --git a/libnd4j/include/loops/pairwise_transform.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/pairwise_transform.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/loops/pairwise_transform.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/pairwise_transform.h diff --git a/libnd4j/include/loops/random.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/random.h similarity index 100% rename from libnd4j/include/loops/random.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/random.h diff --git a/libnd4j/include/loops/reduce3.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce3.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/loops/reduce3.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce3.h diff --git a/libnd4j/include/loops/reduce_bool.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce_bool.h similarity index 100% rename from libnd4j/include/loops/reduce_bool.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce_bool.h diff --git a/libnd4j/include/loops/reduce_float.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce_float.h similarity index 100% rename from libnd4j/include/loops/reduce_float.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce_float.h diff --git a/libnd4j/include/loops/reduce_long.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce_long.h similarity index 100% rename from libnd4j/include/loops/reduce_long.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce_long.h diff --git a/libnd4j/include/loops/reduce_same.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce_same.h similarity index 100% rename from libnd4j/include/loops/reduce_same.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/reduce_same.h diff --git a/libnd4j/include/loops/scalar.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/scalar.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/loops/scalar.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/scalar.h diff --git a/libnd4j/include/loops/scalar_bool.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/scalar_bool.h similarity index 100% rename from libnd4j/include/loops/scalar_bool.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/scalar_bool.h diff --git a/libnd4j/include/loops/scalar_int.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/scalar_int.h similarity index 100% rename from libnd4j/include/loops/scalar_int.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/scalar_int.h diff --git a/libnd4j/include/loops/special_kernels.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/special_kernels.h similarity index 100% rename from libnd4j/include/loops/special_kernels.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/special_kernels.h diff --git a/libnd4j/include/loops/summarystatsreduce.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/summarystatsreduce.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/loops/summarystatsreduce.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/summarystatsreduce.h diff --git a/libnd4j/include/loops/transform_any.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_any.h similarity index 100% rename from libnd4j/include/loops/transform_any.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_any.h diff --git a/libnd4j/include/loops/transform_bool.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_bool.h similarity index 100% rename from libnd4j/include/loops/transform_bool.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_bool.h diff --git a/libnd4j/include/loops/transform_float.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_float.h similarity index 100% rename from libnd4j/include/loops/transform_float.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_float.h diff --git a/libnd4j/include/loops/transform_same.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_same.h similarity index 100% rename from libnd4j/include/loops/transform_same.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_same.h diff --git a/libnd4j/include/loops/transform_strict.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_strict.h similarity index 100% rename from libnd4j/include/loops/transform_strict.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/transform_strict.h diff --git a/libnd4j/include/loops/type_conversions.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/loops/type_conversions.h similarity index 100% rename from libnd4j/include/loops/type_conversions.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/loops/type_conversions.h diff --git a/libnd4j/include/math/platformmath.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/math/platformmath.h similarity index 100% rename from libnd4j/include/math/platformmath.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/math/platformmath.h diff --git a/libnd4j/include/math/templatemath.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/math/templatemath.h similarity index 100% rename from libnd4j/include/math/templatemath.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/math/templatemath.h diff --git a/libnd4j/include/memory/AllocationEntry.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/AllocationEntry.h similarity index 100% rename from libnd4j/include/memory/AllocationEntry.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/AllocationEntry.h diff --git a/libnd4j/include/memory/ExternalWorkspace.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/ExternalWorkspace.h similarity index 100% rename from libnd4j/include/memory/ExternalWorkspace.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/ExternalWorkspace.h diff --git a/libnd4j/include/memory/MemoryCounter.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryCounter.h similarity index 100% rename from libnd4j/include/memory/MemoryCounter.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryCounter.h diff --git a/libnd4j/include/memory/MemoryRegistrator.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryRegistrator.h similarity index 100% rename from libnd4j/include/memory/MemoryRegistrator.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryRegistrator.h diff --git a/libnd4j/include/memory/MemoryReport.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryReport.h similarity index 100% rename from libnd4j/include/memory/MemoryReport.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryReport.h diff --git a/libnd4j/include/memory/MemoryTracker.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryTracker.h similarity index 100% rename from libnd4j/include/memory/MemoryTracker.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryTracker.h diff --git a/libnd4j/include/memory/MemoryType.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryType.h similarity index 100% rename from libnd4j/include/memory/MemoryType.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryType.h diff --git a/libnd4j/include/memory/MemoryUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryUtils.h similarity index 100% rename from libnd4j/include/memory/MemoryUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/MemoryUtils.h diff --git a/libnd4j/include/memory/Workspace.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/Workspace.h similarity index 100% rename from libnd4j/include/memory/Workspace.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/Workspace.h diff --git a/libnd4j/include/memory/cpu/Workspace.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/cpu/Workspace.cpp similarity index 100% rename from libnd4j/include/memory/cpu/Workspace.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/cpu/Workspace.cpp diff --git a/libnd4j/include/memory/cuda/Workspace.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/cuda/Workspace.cu similarity index 100% rename from libnd4j/include/memory/cuda/Workspace.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/cuda/Workspace.cu diff --git a/libnd4j/include/memory/impl/AllocationEntry.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/AllocationEntry.cpp similarity index 100% rename from libnd4j/include/memory/impl/AllocationEntry.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/AllocationEntry.cpp diff --git a/libnd4j/include/memory/impl/ExternalWorkspace.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/ExternalWorkspace.cpp similarity index 100% rename from libnd4j/include/memory/impl/ExternalWorkspace.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/ExternalWorkspace.cpp diff --git a/libnd4j/include/memory/impl/MemoryCounter.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryCounter.cpp similarity index 100% rename from libnd4j/include/memory/impl/MemoryCounter.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryCounter.cpp diff --git a/libnd4j/include/memory/impl/MemoryRegistrator.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryRegistrator.cpp similarity index 100% rename from libnd4j/include/memory/impl/MemoryRegistrator.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryRegistrator.cpp diff --git a/libnd4j/include/memory/impl/MemoryReport.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryReport.cpp similarity index 100% rename from libnd4j/include/memory/impl/MemoryReport.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryReport.cpp diff --git a/libnd4j/include/memory/impl/MemoryTracker.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryTracker.cpp similarity index 100% rename from libnd4j/include/memory/impl/MemoryTracker.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryTracker.cpp diff --git a/libnd4j/include/memory/impl/MemoryUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryUtils.cpp similarity index 100% rename from libnd4j/include/memory/impl/MemoryUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/memory/impl/MemoryUtils.cpp diff --git a/libnd4j/include/ops/BroadcastBoolOpsTuple.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/BroadcastBoolOpsTuple.h similarity index 100% rename from libnd4j/include/ops/BroadcastBoolOpsTuple.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/BroadcastBoolOpsTuple.h diff --git a/libnd4j/include/ops/BroadcastIntOpsTuple.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/BroadcastIntOpsTuple.h similarity index 100% rename from libnd4j/include/ops/BroadcastIntOpsTuple.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/BroadcastIntOpsTuple.h diff --git a/libnd4j/include/ops/BroadcastOpsTuple.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/BroadcastOpsTuple.h similarity index 100% rename from libnd4j/include/ops/BroadcastOpsTuple.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/BroadcastOpsTuple.h diff --git a/libnd4j/include/ops/InputType.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/InputType.h similarity index 100% rename from libnd4j/include/ops/InputType.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/InputType.h diff --git a/libnd4j/include/ops/declarable/BooleanOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/BooleanOp.h similarity index 100% rename from libnd4j/include/ops/declarable/BooleanOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/BooleanOp.h diff --git a/libnd4j/include/ops/declarable/BroadcastableBoolOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/BroadcastableBoolOp.h similarity index 100% rename from libnd4j/include/ops/declarable/BroadcastableBoolOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/BroadcastableBoolOp.h diff --git a/libnd4j/include/ops/declarable/BroadcastableOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/BroadcastableOp.h similarity index 100% rename from libnd4j/include/ops/declarable/BroadcastableOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/BroadcastableOp.h diff --git a/libnd4j/include/ops/declarable/CustomOperations.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/CustomOperations.h similarity index 100% rename from libnd4j/include/ops/declarable/CustomOperations.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/CustomOperations.h diff --git a/libnd4j/include/ops/declarable/DeclarableCustomOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/DeclarableCustomOp.h similarity index 100% rename from libnd4j/include/ops/declarable/DeclarableCustomOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/DeclarableCustomOp.h diff --git a/libnd4j/include/ops/declarable/DeclarableListOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/DeclarableListOp.h similarity index 100% rename from libnd4j/include/ops/declarable/DeclarableListOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/DeclarableListOp.h diff --git a/libnd4j/include/ops/declarable/DeclarableOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/DeclarableOp.h similarity index 100% rename from libnd4j/include/ops/declarable/DeclarableOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/DeclarableOp.h diff --git a/libnd4j/include/ops/declarable/DeclarableReductionOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/DeclarableReductionOp.h similarity index 100% rename from libnd4j/include/ops/declarable/DeclarableReductionOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/DeclarableReductionOp.h diff --git a/libnd4j/include/ops/declarable/EmptyHandling.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/EmptyHandling.h similarity index 100% rename from libnd4j/include/ops/declarable/EmptyHandling.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/EmptyHandling.h diff --git a/libnd4j/include/ops/declarable/LegacyBroadcastBoolOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyBroadcastBoolOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyBroadcastBoolOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyBroadcastBoolOp.h diff --git a/libnd4j/include/ops/declarable/LegacyBroadcastOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyBroadcastOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyBroadcastOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyBroadcastOp.h diff --git a/libnd4j/include/ops/declarable/LegacyIndexReduceOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyIndexReduceOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyIndexReduceOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyIndexReduceOp.h diff --git a/libnd4j/include/ops/declarable/LegacyOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyOp.h diff --git a/libnd4j/include/ops/declarable/LegacyPairwiseTransformBoolOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyPairwiseTransformBoolOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyPairwiseTransformBoolOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyPairwiseTransformBoolOp.h diff --git a/libnd4j/include/ops/declarable/LegacyPairwiseTransformOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyPairwiseTransformOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyPairwiseTransformOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyPairwiseTransformOp.h diff --git a/libnd4j/include/ops/declarable/LegacyRandomOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyRandomOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyRandomOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyRandomOp.h diff --git a/libnd4j/include/ops/declarable/LegacyReduce3Op.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduce3Op.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyReduce3Op.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduce3Op.h diff --git a/libnd4j/include/ops/declarable/LegacyReduceBoolOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceBoolOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyReduceBoolOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceBoolOp.h diff --git a/libnd4j/include/ops/declarable/LegacyReduceFloatOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceFloatOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyReduceFloatOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceFloatOp.h diff --git a/libnd4j/include/ops/declarable/LegacyReduceLongOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceLongOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyReduceLongOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceLongOp.h diff --git a/libnd4j/include/ops/declarable/LegacyReduceOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyReduceOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceOp.h diff --git a/libnd4j/include/ops/declarable/LegacyReduceSameOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceSameOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyReduceSameOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyReduceSameOp.h diff --git a/libnd4j/include/ops/declarable/LegacyScalarBoolOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyScalarBoolOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyScalarBoolOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyScalarBoolOp.h diff --git a/libnd4j/include/ops/declarable/LegacyScalarOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyScalarOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyScalarOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyScalarOp.h diff --git a/libnd4j/include/ops/declarable/LegacyStatsOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyStatsOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyStatsOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyStatsOp.h diff --git a/libnd4j/include/ops/declarable/LegacyTransformAnyOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformAnyOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyTransformAnyOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformAnyOp.h diff --git a/libnd4j/include/ops/declarable/LegacyTransformBoolOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformBoolOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyTransformBoolOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformBoolOp.h diff --git a/libnd4j/include/ops/declarable/LegacyTransformFloatOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformFloatOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyTransformFloatOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformFloatOp.h diff --git a/libnd4j/include/ops/declarable/LegacyTransformOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyTransformOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformOp.h diff --git a/libnd4j/include/ops/declarable/LegacyTransformSameOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformSameOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyTransformSameOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformSameOp.h diff --git a/libnd4j/include/ops/declarable/LegacyTransformStrictOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformStrictOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LegacyTransformStrictOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LegacyTransformStrictOp.h diff --git a/libnd4j/include/ops/declarable/LogicOp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LogicOp.h similarity index 100% rename from libnd4j/include/ops/declarable/LogicOp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/LogicOp.h diff --git a/libnd4j/include/ops/declarable/OpDescriptor.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/OpDescriptor.h similarity index 100% rename from libnd4j/include/ops/declarable/OpDescriptor.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/OpDescriptor.h diff --git a/libnd4j/include/ops/declarable/OpRegistrator.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/OpRegistrator.h similarity index 100% rename from libnd4j/include/ops/declarable/OpRegistrator.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/OpRegistrator.h diff --git a/libnd4j/include/ops/declarable/OpTuple.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/OpTuple.h similarity index 100% rename from libnd4j/include/ops/declarable/OpTuple.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/OpTuple.h diff --git a/libnd4j/include/ops/declarable/PlatformHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/PlatformHelper.h similarity index 100% rename from libnd4j/include/ops/declarable/PlatformHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/PlatformHelper.h diff --git a/libnd4j/include/ops/declarable/generic/CustomOperations.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/CustomOperations.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/CustomOperations.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/CustomOperations.cpp diff --git a/libnd4j/include/ops/declarable/generic/README.md b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/README.md similarity index 100% rename from libnd4j/include/ops/declarable/generic/README.md rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/README.md diff --git a/libnd4j/include/ops/declarable/generic/bitwise/bits_hamming_distance.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/bits_hamming_distance.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/bits_hamming_distance.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/bits_hamming_distance.cpp diff --git a/libnd4j/include/ops/declarable/generic/bitwise/bitwise_and.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/bitwise_and.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/bitwise_and.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/bitwise_and.cpp diff --git a/libnd4j/include/ops/declarable/generic/bitwise/bitwise_or.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/bitwise_or.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/bitwise_or.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/bitwise_or.cpp diff --git a/libnd4j/include/ops/declarable/generic/bitwise/bitwise_xor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/bitwise_xor.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/bitwise_xor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/bitwise_xor.cpp diff --git a/libnd4j/include/ops/declarable/generic/bitwise/cyclic_rshift.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/cyclic_rshift.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/cyclic_rshift.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/cyclic_rshift.cpp diff --git a/libnd4j/include/ops/declarable/generic/bitwise/cyclic_shift.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/cyclic_shift.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/cyclic_shift.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/cyclic_shift.cpp diff --git a/libnd4j/include/ops/declarable/generic/bitwise/rshift.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/rshift.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/rshift.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/rshift.cpp diff --git a/libnd4j/include/ops/declarable/generic/bitwise/shift.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/shift.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/shift.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/shift.cpp diff --git a/libnd4j/include/ops/declarable/generic/bitwise/toggle_bits.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/toggle_bits.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/bitwise/toggle_bits.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/bitwise/toggle_bits.cpp diff --git a/libnd4j/include/ops/declarable/generic/blas/axpy.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/axpy.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/blas/axpy.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/axpy.cpp diff --git a/libnd4j/include/ops/declarable/generic/blas/batched_gemm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/batched_gemm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/blas/batched_gemm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/batched_gemm.cpp diff --git a/libnd4j/include/ops/declarable/generic/blas/matmul.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/matmul.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/blas/matmul.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/matmul.cpp diff --git a/libnd4j/include/ops/declarable/generic/blas/tensormmul.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/tensormmul.cpp similarity index 98% rename from libnd4j/include/ops/declarable/generic/blas/tensormmul.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/tensormmul.cpp index 090c0942e..159918d3c 100644 --- a/libnd4j/include/ops/declarable/generic/blas/tensormmul.cpp +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/blas/tensormmul.cpp @@ -141,10 +141,10 @@ CUSTOM_OP_IMPL(tensormmul_bp, 3, 2, false, 0, -1) { std::vector axesA = ShapeUtils::evalDimsToExclude(Arank, axes0); std::vector axesB = ShapeUtils::evalDimsToExclude(Brank, axes1); - + // rank always have to be divided by 2 std::vector axesAdLdC, axesBdLdC; if (dLdCrank > 1) { - axesAdLdC.resize(axesA.size()); + axesAdLdC.resize(dLdCrank / 2); std::iota(axesAdLdC.begin(), axesAdLdC.end(), 0); axesBdLdC = ShapeUtils::evalDimsToExclude(dLdCrank, axesAdLdC); } diff --git a/libnd4j/include/ops/declarable/generic/boolean/boolean_not.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/boolean_not.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/boolean_not.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/boolean_not.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/choose.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/choose.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/choose.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/choose.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/eq_scalar.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/eq_scalar.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/eq_scalar.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/eq_scalar.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/gt_scalar.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/gt_scalar.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/gt_scalar.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/gt_scalar.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/gte_scalar.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/gte_scalar.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/gte_scalar.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/gte_scalar.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/is_non_decreasing.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/is_non_decreasing.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/is_non_decreasing.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/is_non_decreasing.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/is_numeric_tensor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/is_numeric_tensor.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/is_numeric_tensor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/is_numeric_tensor.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/is_strictly_increasing.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/is_strictly_increasing.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/is_strictly_increasing.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/is_strictly_increasing.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/lt_scalar.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/lt_scalar.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/lt_scalar.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/lt_scalar.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/lte_scalar.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/lte_scalar.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/lte_scalar.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/lte_scalar.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/neq_scalar.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/neq_scalar.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/neq_scalar.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/neq_scalar.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/select.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/select.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/select.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/select.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/where.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/where.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/where.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/where.cpp diff --git a/libnd4j/include/ops/declarable/generic/boolean/where_np.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/where_np.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/boolean/where_np.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/boolean/where_np.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/add.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/add.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/add.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/add.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/assign.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/assign.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/assign.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/assign.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/atan2.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/atan2.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/atan2.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/atan2.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/boolean_and.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/boolean_and.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/boolean_and.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/boolean_and.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/boolean_or.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/boolean_or.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/boolean_or.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/boolean_or.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/boolean_xor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/boolean_xor.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/boolean_xor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/boolean_xor.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/divide.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/divide.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/divide.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/divide.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/divide_no_nan.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/divide_no_nan.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/divide_no_nan.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/divide_no_nan.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/equals.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/equals.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/equals.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/equals.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/floordiv.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/floordiv.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/floordiv.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/floordiv.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/floormod.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/floormod.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/floormod.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/floormod.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/greater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/greater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/greater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/greater.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/greater_equal.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/greater_equal.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/greater_equal.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/greater_equal.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/igamma.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/igamma.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/igamma.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/igamma.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/igammac.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/igammac.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/igammac.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/igammac.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/less.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/less.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/less.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/less.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/less_equal.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/less_equal.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/less_equal.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/less_equal.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/maximum.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/maximum.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/maximum.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/maximum.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/meshgrid.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/meshgrid.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/meshgrid.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/meshgrid.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/minimum.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/minimum.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/minimum.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/minimum.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/mod.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/mod.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/mod.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/mod.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/multiply.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/multiply.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/multiply.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/multiply.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/not_equals.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/not_equals.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/not_equals.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/not_equals.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/percentile.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/percentile.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/percentile.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/percentile.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/pow.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/pow.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/realdiv.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/realdiv.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/realdiv.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/realdiv.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/reverse_divide.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/reverse_divide.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/reverse_divide.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/reverse_divide.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/reverse_mod.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/reverse_mod.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/reverse_mod.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/reverse_mod.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/reverse_subtract.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/reverse_subtract.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/reverse_subtract.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/reverse_subtract.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/squared_subtract.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/squared_subtract.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/squared_subtract.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/squared_subtract.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/subtract.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/subtract.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/subtract.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/subtract.cpp diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/template.tpl b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/template.tpl similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/template.tpl rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/template.tpl diff --git a/libnd4j/include/ops/declarable/generic/broadcastable/truncatediv.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/truncatediv.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/broadcastable/truncatediv.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/broadcastable/truncatediv.cpp diff --git a/libnd4j/include/ops/declarable/generic/compat/README.md b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compat/README.md similarity index 100% rename from libnd4j/include/ops/declarable/generic/compat/README.md rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compat/README.md diff --git a/libnd4j/include/ops/declarable/generic/compat/compat_sparse_to_dense.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compat/compat_sparse_to_dense.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/compat/compat_sparse_to_dense.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compat/compat_sparse_to_dense.cpp diff --git a/libnd4j/include/ops/declarable/generic/compat/compat_string_split.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compat/compat_string_split.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/compat/compat_string_split.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compat/compat_string_split.cpp diff --git a/libnd4j/include/ops/declarable/generic/compression/bitmap.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compression/bitmap.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/compression/bitmap.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compression/bitmap.cpp diff --git a/libnd4j/include/ops/declarable/generic/compression/threshold.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compression/threshold.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/compression/threshold.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/compression/threshold.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/bitcast.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/bitcast.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/bitcast.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/bitcast.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/cast.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/cast.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/cast.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/cast.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/to_double.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_double.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/to_double.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_double.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/to_float16.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_float16.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/to_float16.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_float16.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/to_float32.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_float32.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/to_float32.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_float32.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/to_int32.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_int32.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/to_int32.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_int32.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/to_int64.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_int64.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/to_int64.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_int64.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/to_uint32.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_uint32.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/to_uint32.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_uint32.cpp diff --git a/libnd4j/include/ops/declarable/generic/datatypes/to_uint64.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_uint64.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/datatypes/to_uint64.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/datatypes/to_uint64.cpp diff --git a/libnd4j/include/ops/declarable/generic/flow/flow_control_ops.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/flow/flow_control_ops.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/flow/flow_control_ops.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/flow/flow_control_ops.cpp diff --git a/libnd4j/include/ops/declarable/generic/grad/broadcast_gradient_args.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/grad/broadcast_gradient_args.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/grad/broadcast_gradient_args.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/grad/broadcast_gradient_args.cpp diff --git a/libnd4j/include/ops/declarable/generic/helpers/BroadcastHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/helpers/BroadcastHelper.h similarity index 100% rename from libnd4j/include/ops/declarable/generic/helpers/BroadcastHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/helpers/BroadcastHelper.h diff --git a/libnd4j/include/ops/declarable/generic/helpers/ScatterHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/helpers/ScatterHelper.h similarity index 100% rename from libnd4j/include/ops/declarable/generic/helpers/ScatterHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/helpers/ScatterHelper.h diff --git a/libnd4j/include/ops/declarable/generic/images/adjust_contrast.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/adjust_contrast.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/adjust_contrast.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/adjust_contrast.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/adjust_hue.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/adjust_hue.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/adjust_hue.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/adjust_hue.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/adjust_saturation.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/adjust_saturation.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/adjust_saturation.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/adjust_saturation.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/crop_and_resize.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/crop_and_resize.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/crop_and_resize.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/crop_and_resize.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/draw_bounding_boxes.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/draw_bounding_boxes.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/draw_bounding_boxes.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/draw_bounding_boxes.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/extract_image_patches.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/extract_image_patches.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/extract_image_patches.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/extract_image_patches.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/hsvToRgb.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/hsvToRgb.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/hsvToRgb.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/hsvToRgb.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/image_resize.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/image_resize.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/image_resize.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/image_resize.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/resize_area.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_area.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/resize_area.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_area.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/resize_bicubic.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_bicubic.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/resize_bicubic.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_bicubic.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/resize_images.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_images.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/resize_images.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_images.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/resize_linear.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_linear.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/resize_linear.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_linear.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/resize_neighbor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_neighbor.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/resize_neighbor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/resize_neighbor.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/rgbToGrs.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/rgbToGrs.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/rgbToGrs.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/rgbToGrs.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/rgbToHsv.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/rgbToHsv.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/rgbToHsv.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/rgbToHsv.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/rgbToYiq.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/rgbToYiq.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/rgbToYiq.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/rgbToYiq.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/rgbToYuv.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/rgbToYuv.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/rgbToYuv.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/rgbToYuv.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/yiqToRgb.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/yiqToRgb.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/yiqToRgb.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/yiqToRgb.cpp diff --git a/libnd4j/include/ops/declarable/generic/images/yuvToRgb.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/yuvToRgb.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/images/yuvToRgb.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/images/yuvToRgb.cpp diff --git a/libnd4j/include/ops/declarable/generic/kernels/knn_mindistance.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/kernels/knn_mindistance.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/kernels/knn_mindistance.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/kernels/knn_mindistance.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/betaInc.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/betaInc.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/betaInc.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/betaInc.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/cholesky.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/cholesky.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/cholesky.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/cholesky.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/cross.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/cross.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/cross.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/cross.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/diag.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/diag.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/diag.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/diag.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/diagPart.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/diagPart.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/diagPart.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/diagPart.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/digamma.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/digamma.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/digamma.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/digamma.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/eye.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/eye.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/eye.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/eye.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/lgamma.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/lgamma.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/lgamma.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/lgamma.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/log1p.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/log1p.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/log1p.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/log1p.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/lstsq.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/lstsq.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/lstsq.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/lstsq.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/lup.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/lup.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/lup.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/lup.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/matrixDiagPart.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrixDiagPart.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/matrixDiagPart.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrixDiagPart.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/matrixSetDiag.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrixSetDiag.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/matrixSetDiag.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrixSetDiag.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/matrix_band_part.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrix_band_part.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/matrix_band_part.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrix_band_part.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/matrix_determinant.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrix_determinant.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/matrix_determinant.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrix_determinant.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/matrix_diag.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrix_diag.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/matrix_diag.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrix_diag.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/matrix_inverse.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrix_inverse.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/matrix_inverse.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/matrix_inverse.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/moments.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/moments.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/moments.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/moments.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/polygamma.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/polygamma.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/polygamma.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/polygamma.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/qr.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/qr.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/qr.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/qr.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/solve.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/solve.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/solve.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/solve.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/sqrtm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/sqrtm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/sqrtm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/sqrtm.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/sufficient_statistics.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/sufficient_statistics.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/sufficient_statistics.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/sufficient_statistics.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/svd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/svd.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/svd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/svd.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/trace.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/trace.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/trace.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/trace.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/tri.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/tri.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/tri.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/tri.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/triangular_solve.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/triangular_solve.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/triangular_solve.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/triangular_solve.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/triu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/triu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/triu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/triu.cpp diff --git a/libnd4j/include/ops/declarable/generic/linalg/zeta.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/zeta.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/linalg/zeta.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/linalg/zeta.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/clone_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/clone_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/clone_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/clone_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/create_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/create_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/create_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/create_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/gather_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/gather_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/gather_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/gather_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/pick_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/pick_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/pick_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/pick_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/read_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/read_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/read_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/read_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/scatter_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/scatter_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/scatter_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/scatter_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/size_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/size_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/size_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/size_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/split_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/split_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/split_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/split_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/stack_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/stack_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/stack_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/stack_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/unstack_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/unstack_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/unstack_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/unstack_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/list/write_list.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/write_list.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/list/write_list.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/list/write_list.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/absoluteDifference.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/absoluteDifference.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/absoluteDifference.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/absoluteDifference.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/cosineDistance.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/cosineDistance.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/cosineDistance.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/cosineDistance.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/ctcLoss.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/ctcLoss.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/ctcLoss.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/ctcLoss.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/hingeLoss.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/hingeLoss.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/hingeLoss.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/hingeLoss.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/huberLoss.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/huberLoss.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/huberLoss.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/huberLoss.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/l2_loss.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/l2_loss.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/l2_loss.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/l2_loss.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/logLoss.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/logLoss.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/logLoss.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/logLoss.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/log_poisson_loss.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/log_poisson_loss.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/log_poisson_loss.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/log_poisson_loss.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/meanPairWsSqErr.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/meanPairWsSqErr.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/meanPairWsSqErr.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/meanPairWsSqErr.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/meanSqErr.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/meanSqErr.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/meanSqErr.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/meanSqErr.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/sigmCrossEntropy.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/sigmCrossEntropy.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/sigmCrossEntropy.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/sigmCrossEntropy.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/softmaxCrossEntropy.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/softmaxCrossEntropy.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/softmaxCrossEntropy.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/softmaxCrossEntropy.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/softmaxCrossEntropyWithLogits.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/softmaxCrossEntropyWithLogits.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/softmaxCrossEntropyWithLogits.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/softmaxCrossEntropyWithLogits.cpp diff --git a/libnd4j/include/ops/declarable/generic/loss/sparseSoftmaxCrossEntropyWithLogits.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/sparseSoftmaxCrossEntropyWithLogits.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/loss/sparseSoftmaxCrossEntropyWithLogits.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/loss/sparseSoftmaxCrossEntropyWithLogits.cpp diff --git a/libnd4j/include/ops/declarable/generic/nlp/cbow.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nlp/cbow.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nlp/cbow.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nlp/cbow.cpp diff --git a/libnd4j/include/ops/declarable/generic/nlp/skipgram.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nlp/skipgram.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nlp/skipgram.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nlp/skipgram.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/crelu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/crelu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/crelu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/crelu.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/cube.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/cube.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/cube.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/cube.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/elu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/elu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/elu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/elu.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/hardsigmoid.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/hardsigmoid.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/hardsigmoid.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/hardsigmoid.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/hardtanh.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/hardtanh.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/hardtanh.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/hardtanh.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/identity.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/identity.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/identity.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/identity.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/identity_n.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/identity_n.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/identity_n.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/identity_n.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/lrelu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/lrelu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/lrelu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/lrelu.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/prelu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/prelu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/prelu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/prelu.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/rationaltanh.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/rationaltanh.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/rationaltanh.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/rationaltanh.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/rectifiedtanh.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/rectifiedtanh.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/rectifiedtanh.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/rectifiedtanh.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/relu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/relu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/relu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/relu.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/relu6.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/relu6.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/relu6.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/relu6.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/selu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/selu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/selu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/selu.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/sigmoid.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/sigmoid.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/sigmoid.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/sigmoid.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/softplus.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/softplus.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/softplus.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/softplus.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/softsign.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/softsign.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/softsign.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/softsign.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/tanh.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/tanh.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/tanh.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/tanh.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/activations/thresholdedrelu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/thresholdedrelu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/activations/thresholdedrelu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/activations/thresholdedrelu.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/apply_sgd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/apply_sgd.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/apply_sgd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/apply_sgd.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/batchnorm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/batchnorm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/batchnorm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/batchnorm.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/bias_add.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/bias_add.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/bias_add.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/bias_add.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/col2im.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/col2im.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/col2im.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/col2im.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/conv1d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/conv1d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/conv1d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/conv1d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/conv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/conv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/conv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/conv2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/conv3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/conv3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/conv3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/conv3d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/deconv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/deconv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/deconv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/deconv2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/deconv2d_tf.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/deconv2d_tf.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/deconv2d_tf.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/deconv2d_tf.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/deconv3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/deconv3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/deconv3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/deconv3d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/depthwiseConv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/depthwiseConv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/depthwiseConv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/depthwiseConv2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/dilation2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/dilation2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/dilation2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/dilation2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/im2col.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/im2col.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/im2col.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/im2col.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/ismax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/ismax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/ismax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/ismax.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/pointwiseConv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/pointwiseConv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/pointwiseConv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/pointwiseConv2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/sconv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/sconv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/sconv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/sconv2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/upsampling2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/upsampling2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/upsampling2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/upsampling2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/convo/upsampling3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/upsampling3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/convo/upsampling3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/convo/upsampling3d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/dot_product_attention.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/dot_product_attention.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/dot_product_attention.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/dot_product_attention.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/embedding_lookup.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/embedding_lookup.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/embedding_lookup.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/embedding_lookup.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/fusedBatchNorm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/fusedBatchNorm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/fusedBatchNorm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/fusedBatchNorm.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/layer_norm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/layer_norm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/layer_norm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/layer_norm.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/logSoftmax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/logSoftmax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/logSoftmax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/logSoftmax.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/lrn.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/lrn.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/lrn.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/lrn.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/multi_head_dot_product_attention.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/multi_head_dot_product_attention.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/multi_head_dot_product_attention.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/multi_head_dot_product_attention.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/pooling/avgpool2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/avgpool2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/pooling/avgpool2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/avgpool2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/pooling/avgpool3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/avgpool3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/pooling/avgpool3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/avgpool3d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/pooling/maxpool2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/maxpool2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/pooling/maxpool2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/maxpool2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/pooling/maxpool3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/maxpool3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/pooling/maxpool3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/maxpool3d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/pooling/maxpool_with_argmax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/maxpool_with_argmax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/pooling/maxpool_with_argmax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/maxpool_with_argmax.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/pooling/pnormpool2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/pnormpool2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/pooling/pnormpool2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/pooling/pnormpool2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/dynamicBidirectionalRNN.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/dynamicBidirectionalRNN.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/dynamicBidirectionalRNN.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/dynamicBidirectionalRNN.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/dynamicRNN.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/dynamicRNN.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/dynamicRNN.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/dynamicRNN.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/gru.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/gru.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/gru.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/gru.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/gruCell.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/gruCell.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/gruCell.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/gruCell.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/lstm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/lstm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstm.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/lstmBlock.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmBlock.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/lstmBlock.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmBlock.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/lstmBlockCell.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmBlockCell.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/lstmBlockCell.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmBlockCell.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/lstmCell.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmCell.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/lstmCell.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmCell.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/lstmLayer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmLayer.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/lstmLayer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmLayer.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/lstmLayerCell.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmLayerCell.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/lstmLayerCell.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/lstmLayerCell.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/sru.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/sru.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/sru.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/sru.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/sruCell.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/sruCell.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/sruCell.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/sruCell.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/staticBidirectionalRNN.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/staticBidirectionalRNN.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/staticBidirectionalRNN.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/staticBidirectionalRNN.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/recurrent/staticRNN.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/staticRNN.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/recurrent/staticRNN.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/recurrent/staticRNN.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/relu_layer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/relu_layer.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/relu_layer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/relu_layer.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/softmax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/softmax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/softmax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/softmax.cpp diff --git a/libnd4j/include/ops/declarable/generic/nn/xw_plus_b.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/xw_plus_b.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/nn/xw_plus_b.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/nn/xw_plus_b.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/assert.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/assert.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/assert.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/assert.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/bincount.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/bincount.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/bincount.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/bincount.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/broadcast_dynamic_shape.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/broadcast_dynamic_shape.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/broadcast_dynamic_shape.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/broadcast_dynamic_shape.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/check_numerics.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/check_numerics.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/check_numerics.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/check_numerics.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/compare_and_bitpack.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/compare_and_bitpack.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/compare_and_bitpack.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/compare_and_bitpack.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/confusion_matrix.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/confusion_matrix.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/confusion_matrix.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/confusion_matrix.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/expose.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/expose.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/expose.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/expose.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars_per_channel.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars_per_channel.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars_per_channel.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars_per_channel.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/in_top_k.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/in_top_k.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/in_top_k.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/in_top_k.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/listdiff.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/listdiff.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/listdiff.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/listdiff.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/non_max_suppression.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/non_max_suppression.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/non_max_suppression.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/non_max_suppression.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/non_max_suppression_overlaps.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/non_max_suppression_overlaps.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/non_max_suppression_overlaps.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/non_max_suppression_overlaps.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/normalize_moments.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/normalize_moments.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/normalize_moments.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/normalize_moments.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/nth_element.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/nth_element.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/nth_element.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/nth_element.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/onehot.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/onehot.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/onehot.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/onehot.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/rint.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/rint.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/rint.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/rint.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/roll.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/roll.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/roll.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/roll.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/segment_max.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_max.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/segment_max.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_max.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/segment_mean.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_mean.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/segment_mean.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_mean.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/segment_min.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_min.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/segment_min.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_min.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/segment_prod.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_prod.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/segment_prod.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_prod.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/segment_sum.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_sum.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/segment_sum.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/segment_sum.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/sequence_mask.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/sequence_mask.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/sequence_mask.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/sequence_mask.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/square.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/square.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/square.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/square.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/stop_gradient.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/stop_gradient.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/stop_gradient.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/stop_gradient.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/top_k.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/top_k.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/top_k.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/top_k.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/unique.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unique.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/unique.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unique.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_max.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_max.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_max.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_max.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_mean.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_mean.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_mean.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_mean.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_min.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_min.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_min.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_min.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_prod.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_prod.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_prod.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_prod.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_sqrt_n.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_sqrt_n.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_sqrt_n.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_sqrt_n.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_sum.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_sum.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_sum.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/unsorted_segment_sum.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/weighted_cross_entropy_with_logits.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/weighted_cross_entropy_with_logits.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/weighted_cross_entropy_with_logits.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/weighted_cross_entropy_with_logits.cpp diff --git a/libnd4j/include/ops/declarable/generic/parity_ops/zero_fraction.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/zero_fraction.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/parity_ops/zero_fraction.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/parity_ops/zero_fraction.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/bernoulli.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/bernoulli.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/bernoulli.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/bernoulli.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/dropout.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/dropout.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/dropout.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/dropout.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/exponential.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/exponential.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/exponential.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/exponential.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/gamma.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/gamma.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/gamma.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/gamma.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/get_seed.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/get_seed.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/get_seed.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/get_seed.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/multinomial.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/multinomial.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/multinomial.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/multinomial.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/normal.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/normal.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/normal.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/normal.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/poisson.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/poisson.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/poisson.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/poisson.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/random_crop.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/random_crop.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/random_crop.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/random_crop.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/random_shuffle.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/random_shuffle.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/random_shuffle.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/random_shuffle.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/set_seed.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/set_seed.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/set_seed.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/set_seed.cpp diff --git a/libnd4j/include/ops/declarable/generic/random/uniform.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/uniform.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/random/uniform.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/random/uniform.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/argamax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/argamax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/argamax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/argamax.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/argamin.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/argamin.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/argamin.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/argamin.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/argmax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/argmax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/argmax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/argmax.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/argmin.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/argmin.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/argmin.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/argmin.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/norm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/norm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/norm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/norm.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduceMean.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduceMean.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduceMean.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduceMean.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduceStDev.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduceStDev.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduceStDev.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduceStDev.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduceVariance.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduceVariance.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduceVariance.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduceVariance.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_dot.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_dot.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_dot.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_dot.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_logsumexp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_logsumexp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_logsumexp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_logsumexp.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_max.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_max.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_max.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_max.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_min.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_min.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_min.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_min.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_norm1.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_norm1.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_norm1.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_norm1.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_norm2.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_norm2.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_norm2.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_norm2.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_norm_max.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_norm_max.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_norm_max.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_norm_max.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_prod.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_prod.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_prod.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_prod.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_sqnorm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_sqnorm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_sqnorm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_sqnorm.cpp diff --git a/libnd4j/include/ops/declarable/generic/reduce/reduce_sum.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_sum.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/reduce/reduce_sum.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/reduce/reduce_sum.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/broadcast_to.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/broadcast_to.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/broadcast_to.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/broadcast_to.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/evaluate_reduction_shape.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/evaluate_reduction_shape.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/evaluate_reduction_shape.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/evaluate_reduction_shape.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/expand_dims.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/expand_dims.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/expand_dims.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/expand_dims.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/flatten.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/flatten.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/flatten.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/flatten.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/flatten_2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/flatten_2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/flatten_2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/flatten_2d.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/order.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/order.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/order.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/order.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/permute.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/permute.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/permute.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/permute.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/rank.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/rank.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/rank.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/rank.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/reshape.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/reshape.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/reshape.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/reshape.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/reshape_as.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/reshape_as.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/reshape_as.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/reshape_as.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/shape.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/shape.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/shape.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/shape.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/shapes.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/shapes.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/shapes.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/shapes.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/size.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/size.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/size.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/size.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/size_at.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/size_at.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/size_at.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/size_at.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/squeeze.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/squeeze.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/squeeze.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/squeeze.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/tile_to_shape.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/tile_to_shape.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/tile_to_shape.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/tile_to_shape.cpp diff --git a/libnd4j/include/ops/declarable/generic/shape/transpose.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/transpose.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/shape/transpose.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/shape/transpose.cpp diff --git a/libnd4j/include/ops/declarable/generic/strings/split_string.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/strings/split_string.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/strings/split_string.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/strings/split_string.cpp diff --git a/libnd4j/include/ops/declarable/generic/tensor/create.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/create.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tensor/create.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/create.cpp diff --git a/libnd4j/include/ops/declarable/generic/tensor/fill.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/fill.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tensor/fill.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/fill.cpp diff --git a/libnd4j/include/ops/declarable/generic/tensor/fill_as.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/fill_as.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tensor/fill_as.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/fill_as.cpp diff --git a/libnd4j/include/ops/declarable/generic/tensor/lin_space.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/lin_space.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tensor/lin_space.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/lin_space.cpp diff --git a/libnd4j/include/ops/declarable/generic/tensor/ones_as.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/ones_as.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tensor/ones_as.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/ones_as.cpp diff --git a/libnd4j/include/ops/declarable/generic/tensor/range.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/range.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tensor/range.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/range.cpp diff --git a/libnd4j/include/ops/declarable/generic/tensor/strided_slice.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/strided_slice.cpp similarity index 99% rename from libnd4j/include/ops/declarable/generic/tensor/strided_slice.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/strided_slice.cpp index 2faf04099..e98d67622 100644 --- a/libnd4j/include/ops/declarable/generic/tensor/strided_slice.cpp +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/strided_slice.cpp @@ -523,8 +523,7 @@ namespace sd { std::vector indices; bool result = _preprocess_strided_slice(&indices, &shape, input_shape, begin, end, strides, begin_mask, ellipsis_mask, end_mask, new_axis_mask, shrink_axis_mask, &is_identity, &is_simple_slice, &is_dim0); if (indices.size()) { - auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), 'c', - shape); + auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), 'c', shape); // if (inputLen > 1) { // newShape = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), 'c', // shape); @@ -533,7 +532,6 @@ namespace sd { // } return SHAPELIST(newShape); } - return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfo(ArrayOptions::dataType(inShape))); } diff --git a/libnd4j/include/ops/declarable/generic/tensor/zeros_as.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/zeros_as.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tensor/zeros_as.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tensor/zeros_as.cpp diff --git a/libnd4j/include/ops/declarable/generic/tests/noop.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/noop.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tests/noop.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/noop.cpp diff --git a/libnd4j/include/ops/declarable/generic/tests/test_output_reshape.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/test_output_reshape.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tests/test_output_reshape.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/test_output_reshape.cpp diff --git a/libnd4j/include/ops/declarable/generic/tests/test_scalar.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/test_scalar.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tests/test_scalar.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/test_scalar.cpp diff --git a/libnd4j/include/ops/declarable/generic/tests/testcustom.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/testcustom.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tests/testcustom.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/testcustom.cpp diff --git a/libnd4j/include/ops/declarable/generic/tests/testop2i2o.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/testop2i2o.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tests/testop2i2o.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/testop2i2o.cpp diff --git a/libnd4j/include/ops/declarable/generic/tests/testreduction.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/testreduction.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tests/testreduction.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tests/testreduction.cpp diff --git a/libnd4j/include/ops/declarable/generic/thrid_party/firas_sparse.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/thrid_party/firas_sparse.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/thrid_party/firas_sparse.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/thrid_party/firas_sparse.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/batch_to_space.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/batch_to_space.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/batch_to_space.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/batch_to_space.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/batch_to_space_nd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/batch_to_space_nd.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/batch_to_space_nd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/batch_to_space_nd.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/clip_by_averaged_norm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/clip_by_averaged_norm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/clip_by_averaged_norm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/clip_by_averaged_norm.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/clip_by_global_norm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/clip_by_global_norm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/clip_by_global_norm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/clip_by_global_norm.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/clip_by_norm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/clip_by_norm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/clip_by_norm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/clip_by_norm.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/clip_by_value.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/clip_by_value.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/clip_by_value.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/clip_by_value.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/concat.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/concat.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/concat.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/concat.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/cumprod.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/cumprod.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/cumprod.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/cumprod.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/cumsum.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/cumsum.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/cumsum.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/cumsum.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/depth_to_space.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/depth_to_space.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/depth_to_space.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/depth_to_space.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/dynamic_parititon.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/dynamic_parititon.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/dynamic_parititon.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/dynamic_parititon.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/dynamic_stitch.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/dynamic_stitch.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/dynamic_stitch.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/dynamic_stitch.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/floor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/floor.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/floor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/floor.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/gather.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/gather.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/gather.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/gather.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/gatherNd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/gatherNd.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/gatherNd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/gatherNd.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/hashcode.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/hashcode.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/hashcode.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/hashcode.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/histogram.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/histogram.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/histogram.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/histogram.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/histogram_fixed_width.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/histogram_fixed_width.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/histogram_fixed_width.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/histogram_fixed_width.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/invertPermutation.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/invertPermutation.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/invertPermutation.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/invertPermutation.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/merge_add.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/merge_add.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/merge_add.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/merge_add.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/merge_avg.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/merge_avg.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/merge_avg.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/merge_avg.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/merge_max.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/merge_max.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/merge_max.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/merge_max.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/merge_max_idx.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/merge_max_idx.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/merge_max_idx.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/merge_max_idx.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/mirrorPad.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/mirrorPad.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/mirrorPad.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/mirrorPad.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/pad.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/pad.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/pad.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/pad.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/parallelStack.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/parallelStack.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/parallelStack.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/parallelStack.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/repeat.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/repeat.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/repeat.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/repeat.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/reverse.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/reverse.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/reverse.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/reverse.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/reverseSequence.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/reverseSequence.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/reverseSequence.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/reverseSequence.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_add.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_add.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_add.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_add.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_div.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_div.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_div.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_div.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_max.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_max.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_max.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_max.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_min.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_min.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_min.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_min.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_mul.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_mul.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_mul.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_mul.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_nd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_nd.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_nd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_nd.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_nd_add.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_nd_add.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_nd_add.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_nd_add.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_nd_sub.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_nd_sub.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_nd_sub.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_nd_sub.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_nd_update.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_nd_update.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_nd_update.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_nd_update.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_sub.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_sub.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_sub.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_sub.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_upd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_upd.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_upd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_upd.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/scatter_update.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_update.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/scatter_update.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/scatter_update.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/slice.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/slice.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/slice.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/slice.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/space_to_batch.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/space_to_batch.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/space_to_batch.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/space_to_batch.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/space_to_batch_nd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/space_to_batch_nd.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/space_to_batch_nd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/space_to_batch_nd.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/space_to_depth.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/space_to_depth.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/space_to_depth.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/space_to_depth.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/split.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/split.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/split.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/split.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/split_v.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/split_v.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/split_v.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/split_v.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/stack.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/stack.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/stack.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/stack.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/standardize.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/standardize.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/standardize.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/standardize.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/tear.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/tear.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/tear.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/tear.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/tile.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/tile.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/tile.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/tile.cpp diff --git a/libnd4j/include/ops/declarable/generic/transforms/unstack.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/unstack.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/transforms/unstack.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/transforms/unstack.cpp diff --git a/libnd4j/include/ops/declarable/generic/tsne/cell_contains.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tsne/cell_contains.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tsne/cell_contains.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tsne/cell_contains.cpp diff --git a/libnd4j/include/ops/declarable/generic/tsne/edge_force.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tsne/edge_force.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tsne/edge_force.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tsne/edge_force.cpp diff --git a/libnd4j/include/ops/declarable/generic/tsne/gains.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tsne/gains.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tsne/gains.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tsne/gains.cpp diff --git a/libnd4j/include/ops/declarable/generic/tsne/symmetrized.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tsne/symmetrized.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/tsne/symmetrized.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/tsne/symmetrized.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/adaBeliefUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adaBeliefUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/adaBeliefUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adaBeliefUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/adaDeltaUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adaDeltaUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/adaDeltaUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adaDeltaUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/adaGradUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adaGradUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/adaGradUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adaGradUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/adaMaxUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adaMaxUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/adaMaxUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adaMaxUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/adamUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adamUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/adamUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/adamUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/amsGradUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/amsGradUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/amsGradUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/amsGradUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/nadamUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/nadamUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/nadamUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/nadamUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/nesterovsUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/nesterovsUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/nesterovsUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/nesterovsUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/rmsPropUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/rmsPropUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/rmsPropUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/rmsPropUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/updaters/sgdUpdater.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/sgdUpdater.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/updaters/sgdUpdater.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/updaters/sgdUpdater.cpp diff --git a/libnd4j/include/ops/declarable/generic/util/print_affinity.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/util/print_affinity.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/util/print_affinity.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/util/print_affinity.cpp diff --git a/libnd4j/include/ops/declarable/generic/util/print_variable.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/util/print_variable.cpp similarity index 100% rename from libnd4j/include/ops/declarable/generic/util/print_variable.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/generic/util/print_variable.cpp diff --git a/libnd4j/include/ops/declarable/headers/BarnesHutTsne.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/BarnesHutTsne.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/BarnesHutTsne.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/BarnesHutTsne.h diff --git a/libnd4j/include/ops/declarable/headers/activations.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/activations.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/activations.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/activations.h diff --git a/libnd4j/include/ops/declarable/headers/bitwise.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/bitwise.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/bitwise.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/bitwise.h diff --git a/libnd4j/include/ops/declarable/headers/blas.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/blas.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/blas.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/blas.h diff --git a/libnd4j/include/ops/declarable/headers/boolean.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/boolean.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/boolean.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/boolean.h diff --git a/libnd4j/include/ops/declarable/headers/broadcastable.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/broadcastable.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/broadcastable.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/broadcastable.h diff --git a/libnd4j/include/ops/declarable/headers/common.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/common.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/common.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/common.h diff --git a/libnd4j/include/ops/declarable/headers/compat.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/compat.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/compat.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/compat.h diff --git a/libnd4j/include/ops/declarable/headers/compression.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/compression.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/compression.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/compression.h diff --git a/libnd4j/include/ops/declarable/headers/convo.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/convo.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/convo.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/convo.h diff --git a/libnd4j/include/ops/declarable/headers/datatypes.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/datatypes.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/datatypes.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/datatypes.h diff --git a/libnd4j/include/ops/declarable/headers/images.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/images.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/images.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/images.h diff --git a/libnd4j/include/ops/declarable/headers/kernels.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/kernels.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/kernels.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/kernels.h diff --git a/libnd4j/include/ops/declarable/headers/list.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/list.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/list.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/list.h diff --git a/libnd4j/include/ops/declarable/headers/loss.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/loss.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/loss.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/loss.h diff --git a/libnd4j/include/ops/declarable/headers/nlp.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/nlp.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/nlp.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/nlp.h diff --git a/libnd4j/include/ops/declarable/headers/nn.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/nn.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/nn.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/nn.h diff --git a/libnd4j/include/ops/declarable/headers/parity_ops.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/parity_ops.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/parity_ops.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/parity_ops.h diff --git a/libnd4j/include/ops/declarable/headers/random.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/random.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/random.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/random.h diff --git a/libnd4j/include/ops/declarable/headers/recurrent.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/recurrent.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/recurrent.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/recurrent.h diff --git a/libnd4j/include/ops/declarable/headers/shape.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/shape.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/shape.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/shape.h diff --git a/libnd4j/include/ops/declarable/headers/strings.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/strings.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/strings.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/strings.h diff --git a/libnd4j/include/ops/declarable/headers/tests.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/tests.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/tests.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/tests.h diff --git a/libnd4j/include/ops/declarable/headers/third_party.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/third_party.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/third_party.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/third_party.h diff --git a/libnd4j/include/ops/declarable/headers/transforms.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/transforms.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/transforms.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/transforms.h diff --git a/libnd4j/include/ops/declarable/headers/updaters.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/updaters.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/updaters.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/updaters.h diff --git a/libnd4j/include/ops/declarable/headers/util.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/util.h similarity index 100% rename from libnd4j/include/ops/declarable/headers/util.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/headers/util.h diff --git a/libnd4j/include/ops/declarable/helpers/BarnesHutTsne.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/BarnesHutTsne.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/BarnesHutTsne.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/BarnesHutTsne.h diff --git a/libnd4j/include/ops/declarable/helpers/activations.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/activations.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/activations.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/activations.h diff --git a/libnd4j/include/ops/declarable/helpers/addBias.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/addBias.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/addBias.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/addBias.h diff --git a/libnd4j/include/ops/declarable/helpers/adjust_hue.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/adjust_hue.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/adjust_hue.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/adjust_hue.h diff --git a/libnd4j/include/ops/declarable/helpers/adjust_saturation.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/adjust_saturation.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/adjust_saturation.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/adjust_saturation.h diff --git a/libnd4j/include/ops/declarable/helpers/axis.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/axis.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/axis.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/axis.h diff --git a/libnd4j/include/ops/declarable/helpers/batched_gemm.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/batched_gemm.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/batched_gemm.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/batched_gemm.h diff --git a/libnd4j/include/ops/declarable/helpers/batchnorm.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/batchnorm.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/batchnorm.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/batchnorm.h diff --git a/libnd4j/include/ops/declarable/helpers/betaInc.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/betaInc.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/betaInc.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/betaInc.h diff --git a/libnd4j/include/ops/declarable/helpers/choose.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/choose.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/choose.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/choose.h diff --git a/libnd4j/include/ops/declarable/helpers/col2im.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/col2im.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/col2im.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/col2im.h diff --git a/libnd4j/include/ops/declarable/helpers/compare_elem.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/compare_elem.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/compare_elem.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/compare_elem.h diff --git a/libnd4j/include/ops/declarable/helpers/compression.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/compression.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/compression.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/compression.h diff --git a/libnd4j/include/ops/declarable/helpers/confusion.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/confusion.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/confusion.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/confusion.h diff --git a/libnd4j/include/ops/declarable/helpers/convolutions.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/convolutions.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/convolutions.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/convolutions.h diff --git a/libnd4j/include/ops/declarable/helpers/cpu/BarnesHutTsne.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/BarnesHutTsne.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/BarnesHutTsne.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/BarnesHutTsne.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/README.md b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/README.md similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/README.md rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/README.md diff --git a/libnd4j/include/ops/declarable/helpers/cpu/activations.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/activations.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/activations.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/activations.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/addBias.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/addBias.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/addBias.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/addBias.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/adjust_hue.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/adjust_hue.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/adjust_hue.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/adjust_hue.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/adjust_saturation.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/adjust_saturation.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/adjust_saturation.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/adjust_saturation.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/axis.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/axis.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/axis.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/axis.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/batched_gemm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/batched_gemm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/batched_gemm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/batched_gemm.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/batchnorm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/batchnorm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/batchnorm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/batchnorm.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/betaInc.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/betaInc.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/betaInc.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/betaInc.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/clip.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/clip.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/clip.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/clip.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/col2im.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/col2im.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/col2im.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/col2im.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compare_and_bitpack.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compare_and_bitpack.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compare_and_bitpack.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compare_and_bitpack.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compare_elem.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compare_elem.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compare_elem.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compare_elem.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compilation_units/argamax.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/argamax.cpp.in similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compilation_units/argamax.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/argamax.cpp.in diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compilation_units/argamin.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/argamin.cpp.in similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compilation_units/argamin.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/argamin.cpp.in diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compilation_units/argmax.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/argmax.cpp.in similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compilation_units/argmax.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/argmax.cpp.in diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compilation_units/argmin.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/argmin.cpp.in similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compilation_units/argmin.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/argmin.cpp.in diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compilation_units/crop_and_resize.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/crop_and_resize.cpp.in similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compilation_units/crop_and_resize.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compilation_units/crop_and_resize.cpp.in diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compression/compression.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compression/compression.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compression/compression.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compression/compression.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/compression/threshold.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compression/threshold.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/compression/threshold.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/compression/threshold.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/concat.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/concat.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/concat.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/concat.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/confusion.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/confusion.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/confusion.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/confusion.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_col2vol.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_col2vol.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_col2vol.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_col2vol.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_conv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_conv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_conv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_conv2d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_conv2dBP.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_conv2dBP.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_conv2dBP.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_conv2dBP.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_depthwiseConv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_depthwiseConv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_depthwiseConv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_depthwiseConv2d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_depthwiseConv2dBP.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_depthwiseConv2dBP.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_depthwiseConv2dBP.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_depthwiseConv2dBP.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_pooling2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_pooling2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_pooling2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_pooling2d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_pooling2dBP.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_pooling2dBP.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_pooling2dBP.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_pooling2dBP.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_pooling3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_pooling3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_pooling3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_pooling3d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_pooling3dBP.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_pooling3dBP.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_pooling3dBP.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_pooling3dBP.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_sconv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_sconv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_sconv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_sconv2d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_upsampling2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_upsampling2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_upsampling2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_upsampling2d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_upsampling2dBP.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_upsampling2dBP.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_upsampling2dBP.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_upsampling2dBP.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_upsampling3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_upsampling3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_upsampling3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_upsampling3d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_upsampling3dBP.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_upsampling3dBP.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_upsampling3dBP.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_upsampling3dBP.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/convolutions_vol2col.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_vol2col.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/convolutions_vol2col.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/convolutions_vol2col.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/crop_and_resize.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/crop_and_resize.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/crop_and_resize.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/crop_and_resize.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/crop_and_resize.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/crop_and_resize.hpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/crop_and_resize.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/crop_and_resize.hpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/cross.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/cross.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/cross.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/cross.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/ctcLoss.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/ctcLoss.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/ctcLoss.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/ctcLoss.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/d_t_s.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/d_t_s.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/d_t_s.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/d_t_s.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/diGamma.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/diGamma.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/diGamma.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/diGamma.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/diag.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/diag.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/diag.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/diag.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/dilation2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/dilation2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/dilation2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/dilation2d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/dropout.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/dropout.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/dropout.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/dropout.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/dynamic.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/dynamic.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/dynamic.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/dynamic.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/extract_patches.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/extract_patches.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/extract_patches.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/extract_patches.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/eye.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/eye.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/eye.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/eye.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/fake_quantization.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/fake_quantization.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/fake_quantization.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/fake_quantization.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/flatten.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/flatten.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/flatten.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/flatten.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/gather.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/gather.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/gather.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/gather.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/gatherTransforms.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/gatherTransforms.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/gatherTransforms.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/gatherTransforms.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/gradient.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/gradient.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/gradient.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/gradient.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/hamming.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/hamming.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/hamming.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/hamming.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/hashcode.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/hashcode.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/hashcode.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/hashcode.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/histogram.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/histogram.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/histogram.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/histogram.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/histogramFixedWidth.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/histogramFixedWidth.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/histogramFixedWidth.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/histogramFixedWidth.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/im2col.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/im2col.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/im2col.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/im2col.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/image_draw_bounding_boxes.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/image_draw_bounding_boxes.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/image_draw_bounding_boxes.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/image_draw_bounding_boxes.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/image_resize.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/image_resize.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/image_resize.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/image_resize.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/image_suppression.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/image_suppression.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/image_suppression.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/image_suppression.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/imagesHelpers.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/imagesHelpers.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/imagesHelpers.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/imagesHelpers.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/indexReductions.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/indexReductions.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/indexReductions.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/indexReductions.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/indexReductions.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/indexReductions.hpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/indexReductions.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/indexReductions.hpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/invertPermutation.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/invertPermutation.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/invertPermutation.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/invertPermutation.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/ismax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/ismax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/ismax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/ismax.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/legacy_helper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/legacy_helper.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/legacy_helper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/legacy_helper.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/lgamma.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lgamma.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/lgamma.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lgamma.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/lrn.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lrn.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/lrn.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lrn.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/lstm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lstm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/lstm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lstm.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/lstsq.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lstsq.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/lstsq.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lstsq.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/lup.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lup.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/lup.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/lup.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/matrixSetDiag.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/matrixSetDiag.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/matrixSetDiag.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/matrixSetDiag.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/matrix_band.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/matrix_band.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/matrix_band.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/matrix_band.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/matrix_diag_part.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/matrix_diag_part.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/matrix_diag_part.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/matrix_diag_part.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/max_pooling.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/max_pooling.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/max_pooling.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/max_pooling.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/merge.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/merge.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/merge.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/merge.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/meshgrid.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/meshgrid.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/meshgrid.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/meshgrid.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/minimax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/minimax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/minimax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/minimax.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/nth_element.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/nth_element.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/nth_element.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/nth_element.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/one_hot.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/one_hot.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/one_hot.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/one_hot.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/pad.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/pad.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/pad.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/pad.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/percentile.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/percentile.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/percentile.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/percentile.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/polyGamma.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/polyGamma.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/polyGamma.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/polyGamma.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/prefix.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/prefix.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/prefix.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/prefix.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/print_variable.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/print_variable.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/print_variable.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/print_variable.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/qr.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/qr.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/qr.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/qr.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/random.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/random.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/random.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/random.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/randomShuffle.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/randomShuffle.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/randomShuffle.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/randomShuffle.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/random_crop.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/random_crop.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/random_crop.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/random_crop.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/range.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/range.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/range.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/range.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/reverse.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/reverse.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/reverse.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/reverse.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/roll.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/roll.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/roll.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/roll.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/s_t_b.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/s_t_b.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/s_t_b.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/s_t_b.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/s_t_d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/s_t_d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/s_t_d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/s_t_d.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/scatter.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/scatter.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/scatter.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/scatter.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/scatterUpdateAndSimple.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/scatterUpdateAndSimple.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/scatterUpdateAndSimple.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/scatterUpdateAndSimple.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/segment.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/segment.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/segment.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/segment.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/sequence_mask.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/sequence_mask.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/sequence_mask.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/sequence_mask.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/sg_cb.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/sg_cb.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/sg_cb.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/sg_cb.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/shift.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/shift.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/shift.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/shift.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/softmax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/softmax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/softmax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/softmax.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/solve.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/solve.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/solve.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/solve.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/split.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/split.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/split.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/split.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/sru.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/sru.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/sru.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/sru.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/stack.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/stack.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/stack.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/stack.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/svd.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/svd.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/svd.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/svd.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/tile.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/tile.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/tile.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/tile.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/toggle_bits.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/toggle_bits.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/toggle_bits.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/toggle_bits.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/top_k.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/top_k.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/top_k.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/top_k.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/trace.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/trace.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/trace.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/trace.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/triangular_solve.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/triangular_solve.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/triangular_solve.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/triangular_solve.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/triu.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/triu.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/triu.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/triu.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterAdaBelief.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdaBelief.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterAdaBelief.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdaBelief.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterAdaDelta.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdaDelta.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterAdaDelta.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdaDelta.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterAdaGrad.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdaGrad.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterAdaGrad.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdaGrad.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterAdaMax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdaMax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterAdaMax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdaMax.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterAdam.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdam.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterAdam.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAdam.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterAmsGrad.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAmsGrad.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterAmsGrad.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterAmsGrad.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterNadam.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterNadam.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterNadam.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterNadam.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterNesterovs.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterNesterovs.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterNesterovs.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterNesterovs.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/updaterRmsProp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterRmsProp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/updaterRmsProp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/updaterRmsProp.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/weights.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/weights.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/weights.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/weights.cpp diff --git a/libnd4j/include/ops/declarable/helpers/cpu/zeta.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/zeta.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cpu/zeta.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cpu/zeta.cpp diff --git a/libnd4j/include/ops/declarable/helpers/crop_and_resize.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/crop_and_resize.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/crop_and_resize.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/crop_and_resize.h diff --git a/libnd4j/include/ops/declarable/helpers/cross.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cross.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cross.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cross.h diff --git a/libnd4j/include/ops/declarable/helpers/ctcLoss.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/ctcLoss.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/ctcLoss.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/ctcLoss.h diff --git a/libnd4j/include/ops/declarable/helpers/cuda/BarnesHutTsne.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/BarnesHutTsne.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/BarnesHutTsne.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/BarnesHutTsne.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/README.md b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/README.md similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/README.md rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/README.md diff --git a/libnd4j/include/ops/declarable/helpers/cuda/activations.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/activations.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/activations.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/activations.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/addBias.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/addBias.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/addBias.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/addBias.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/adjust_hue.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/adjust_hue.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/adjust_hue.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/adjust_hue.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/adjust_saturation.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/adjust_saturation.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/adjust_saturation.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/adjust_saturation.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/axis.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/axis.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/axis.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/axis.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/batched_gemm.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/batched_gemm.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/batched_gemm.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/batched_gemm.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/batchnorm.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/batchnorm.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/batchnorm.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/batchnorm.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/betaInc.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/betaInc.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/betaInc.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/betaInc.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/clip.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/clip.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/clip.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/clip.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/col2im.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/col2im.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/col2im.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/col2im.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/compare_and_bitpack.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/compare_and_bitpack.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/compare_and_bitpack.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/compare_and_bitpack.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/compare_elem.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/compare_elem.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/compare_elem.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/compare_elem.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/compression/compression.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/compression/compression.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/compression/compression.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/compression/compression.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/compression/threshold.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/compression/threshold.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/compression/threshold.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/compression/threshold.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/concat.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/concat.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/concat.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/concat.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/confusion.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/confusion.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/confusion.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/confusion.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_col2vol.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_col2vol.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_col2vol.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_col2vol.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_conv2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_conv2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_conv2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_conv2d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_conv2dBP.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_conv2dBP.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_conv2dBP.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_conv2dBP.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_depthwiseConv2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_depthwiseConv2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_depthwiseConv2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_depthwiseConv2d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_depthwiseConv2dBP.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_depthwiseConv2dBP.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_depthwiseConv2dBP.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_depthwiseConv2dBP.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_pooling2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_pooling2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_pooling2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_pooling2d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_pooling2dBP.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_pooling2dBP.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_pooling2dBP.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_pooling2dBP.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_pooling3d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_pooling3d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_pooling3d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_pooling3d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_pooling3dBP.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_pooling3dBP.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_pooling3dBP.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_pooling3dBP.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_sconv2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_sconv2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_sconv2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_sconv2d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_upsampling2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_upsampling2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_upsampling2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_upsampling2d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_upsampling2dBP.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_upsampling2dBP.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_upsampling2dBP.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_upsampling2dBP.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_upsampling3d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_upsampling3d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_upsampling3d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_upsampling3d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_upsampling3dBP.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_upsampling3dBP.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_upsampling3dBP.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_upsampling3dBP.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/convolutions_vol2col.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_vol2col.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/convolutions_vol2col.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/convolutions_vol2col.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/cross.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/cross.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/cross.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/cross.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/ctcLoss.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/ctcLoss.cu similarity index 98% rename from libnd4j/include/ops/declarable/helpers/cuda/ctcLoss.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/ctcLoss.cu index 1c5678f28..953dd6984 100644 --- a/libnd4j/include/ops/declarable/helpers/cuda/ctcLoss.cu +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/ctcLoss.cu @@ -13,9 +13,11 @@ * * SPDX-License-Identifier: Apache-2.0 *******************************************************************************/ + // // @author AbdelRauf // + #include #include #include diff --git a/libnd4j/include/ops/declarable/helpers/cuda/d_t_s.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/d_t_s.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/d_t_s.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/d_t_s.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/diGamma.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/diGamma.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/diGamma.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/diGamma.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/diag.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/diag.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/diag.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/diag.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/dilation2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/dilation2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/dilation2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/dilation2d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/dropout.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/dropout.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/dropout.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/dropout.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/dynamic.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/dynamic.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/dynamic.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/dynamic.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/extract_patches.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/extract_patches.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/extract_patches.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/extract_patches.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/fake_quantization.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/fake_quantization.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/fake_quantization.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/fake_quantization.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/flatten.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/flatten.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/flatten.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/flatten.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/gather.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/gather.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/gather.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/gather.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/gather_nd.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/gather_nd.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/gather_nd.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/gather_nd.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/gradient.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/gradient.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/gradient.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/gradient.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/hamming.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/hamming.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/hamming.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/hamming.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/hashcode.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/hashcode.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/hashcode.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/hashcode.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/histogram.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/histogram.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/histogram.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/histogram.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/histogramFixedWidth.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/histogramFixedWidth.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/histogramFixedWidth.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/histogramFixedWidth.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/im2col.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/im2col.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/im2col.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/im2col.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/image_draw_bounding_boxes.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/image_draw_bounding_boxes.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/image_draw_bounding_boxes.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/image_draw_bounding_boxes.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/image_resize.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/image_resize.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/image_resize.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/image_resize.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/image_resize_v2.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/image_resize_v2.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/image_resize_v2.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/image_resize_v2.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/image_suppression.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/image_suppression.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/image_suppression.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/image_suppression.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/imagesHelpers.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/imagesHelpers.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/imagesHelpers.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/imagesHelpers.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/indexReductions.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/indexReductions.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/indexReductions.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/indexReductions.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/ismax.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/ismax.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/ismax.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/ismax.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/legacy/relu.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/legacy/relu.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/legacy/relu.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/legacy/relu.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/legacy/tanh.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/legacy/tanh.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/legacy/tanh.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/legacy/tanh.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/legacy_helper.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/legacy_helper.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/legacy_helper.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/legacy_helper.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/lgamma.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lgamma.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/lgamma.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lgamma.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/lrn.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lrn.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/lrn.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lrn.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/lstm.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lstm.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/lstm.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lstm.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/lstsq.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lstsq.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/lstsq.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lstsq.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/lup.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lup.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/lup.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/lup.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/matrixSetDiag.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/matrixSetDiag.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/matrixSetDiag.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/matrixSetDiag.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/matrix_band.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/matrix_band.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/matrix_band.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/matrix_band.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/matrix_diag_part.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/matrix_diag_part.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/matrix_diag_part.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/matrix_diag_part.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/max_pooling.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/max_pooling.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/max_pooling.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/max_pooling.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/maximum.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/maximum.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/maximum.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/maximum.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/merge.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/merge.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/merge.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/merge.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/meshgrid.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/meshgrid.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/meshgrid.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/meshgrid.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/minimum.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/minimum.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/minimum.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/minimum.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/nth_element.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/nth_element.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/nth_element.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/nth_element.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/one_hot.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/one_hot.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/one_hot.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/one_hot.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/pad.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/pad.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/pad.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/pad.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/percentile.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/percentile.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/percentile.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/percentile.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/polyGamma.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/polyGamma.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/polyGamma.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/polyGamma.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/prefix.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/prefix.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/prefix.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/prefix.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/print_variable.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/print_variable.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/print_variable.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/print_variable.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/qr.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/qr.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/qr.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/qr.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/random.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/random.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/random.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/random.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/randomShuffle.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/randomShuffle.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/randomShuffle.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/randomShuffle.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/random_crop.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/random_crop.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/random_crop.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/random_crop.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/range.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/range.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/range.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/range.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/reverse.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/reverse.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/reverse.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/reverse.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/roll.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/roll.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/roll.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/roll.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/s_t_b.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/s_t_b.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/s_t_b.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/s_t_b.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/s_t_d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/s_t_d.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/s_t_d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/s_t_d.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/scatter.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/scatter.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/scatter.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/scatter.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/scatter_simple.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/scatter_simple.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/scatter_simple.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/scatter_simple.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/scatter_update.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/scatter_update.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/scatter_update.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/scatter_update.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/segment.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/segment.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/segment_max.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_max.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/segment_max.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_max.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/segment_mean.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_mean.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/segment_mean.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_mean.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/segment_min.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_min.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/segment_min.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_min.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/segment_prod.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_prod.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/segment_prod.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_prod.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/segment_sqrtn.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_sqrtn.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/segment_sqrtn.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_sqrtn.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/segment_sum.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_sum.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/segment_sum.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/segment_sum.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/sequence_mask.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/sequence_mask.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/sequence_mask.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/sequence_mask.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/sg_cb.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/sg_cb.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/sg_cb.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/sg_cb.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/shift.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/shift.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/shift.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/shift.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/solve.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/solve.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/solve.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/solve.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/split.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/split.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/split.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/split.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/sru.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/sru.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/sru.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/sru.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/stack.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/stack.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/stack.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/stack.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/svd.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/svd.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/svd.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/svd.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/toggle_bits.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/toggle_bits.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/toggle_bits.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/toggle_bits.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/top_k.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/top_k.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/top_k.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/top_k.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/transforms.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/transforms.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/transforms.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/transforms.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/triangular_solve.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/triangular_solve.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/triangular_solve.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/triangular_solve.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterAdaBelief.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdaBelief.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterAdaBelief.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdaBelief.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterAdaDelta.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdaDelta.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterAdaDelta.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdaDelta.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterAdaGrad.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdaGrad.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterAdaGrad.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdaGrad.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterAdaMax.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdaMax.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterAdaMax.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdaMax.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterAdam.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdam.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterAdam.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAdam.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterAmsGrad.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAmsGrad.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterAmsGrad.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterAmsGrad.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterNadam.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterNadam.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterNadam.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterNadam.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterNesterovs.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterNesterovs.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterNesterovs.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterNesterovs.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/updaterRmsProp.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterRmsProp.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/updaterRmsProp.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/updaterRmsProp.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/weights.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/weights.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/weights.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/weights.cu diff --git a/libnd4j/include/ops/declarable/helpers/cuda/zeta.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/zeta.cu similarity index 100% rename from libnd4j/include/ops/declarable/helpers/cuda/zeta.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/cuda/zeta.cu diff --git a/libnd4j/include/ops/declarable/helpers/d_t_s.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/d_t_s.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/d_t_s.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/d_t_s.h diff --git a/libnd4j/include/ops/declarable/helpers/diag.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/diag.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/diag.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/diag.h diff --git a/libnd4j/include/ops/declarable/helpers/dilation2d.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/dilation2d.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/dilation2d.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/dilation2d.h diff --git a/libnd4j/include/ops/declarable/helpers/dropout.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/dropout.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/dropout.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/dropout.h diff --git a/libnd4j/include/ops/declarable/helpers/dynamic.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/dynamic.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/dynamic.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/dynamic.h diff --git a/libnd4j/include/ops/declarable/helpers/extract_patches.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/extract_patches.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/extract_patches.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/extract_patches.h diff --git a/libnd4j/include/ops/declarable/helpers/fake_quantization.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/fake_quantization.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/fake_quantization.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/fake_quantization.h diff --git a/libnd4j/include/ops/declarable/helpers/flatten.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/flatten.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/flatten.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/flatten.h diff --git a/libnd4j/include/ops/declarable/helpers/gammaMathFunc.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/gammaMathFunc.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/gammaMathFunc.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/gammaMathFunc.h diff --git a/libnd4j/include/ops/declarable/helpers/gather.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/gather.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/gather.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/gather.h diff --git a/libnd4j/include/ops/declarable/helpers/gradient.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/gradient.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/gradient.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/gradient.h diff --git a/libnd4j/include/ops/declarable/helpers/gru.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/gru.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/gru.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/gru.h diff --git a/libnd4j/include/ops/declarable/helpers/hamming.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/hamming.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/hamming.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/hamming.h diff --git a/libnd4j/include/ops/declarable/helpers/hashcode.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/hashcode.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/hashcode.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/hashcode.h diff --git a/libnd4j/include/ops/declarable/helpers/helpers.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/helpers.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/helpers.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/helpers.h diff --git a/libnd4j/include/ops/declarable/helpers/histogram.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/histogram.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/histogram.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/histogram.h diff --git a/libnd4j/include/ops/declarable/helpers/histogramFixedWidth.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/histogramFixedWidth.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/histogramFixedWidth.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/histogramFixedWidth.h diff --git a/libnd4j/include/ops/declarable/helpers/im2col.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/im2col.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/im2col.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/im2col.h diff --git a/libnd4j/include/ops/declarable/helpers/image_draw_bounding_boxes.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/image_draw_bounding_boxes.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/image_draw_bounding_boxes.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/image_draw_bounding_boxes.h diff --git a/libnd4j/include/ops/declarable/helpers/image_resize.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/image_resize.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/image_resize.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/image_resize.h diff --git a/libnd4j/include/ops/declarable/helpers/image_suppression.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/image_suppression.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/image_suppression.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/image_suppression.h diff --git a/libnd4j/include/ops/declarable/helpers/imagesHelpers.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/imagesHelpers.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/imagesHelpers.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/imagesHelpers.h diff --git a/libnd4j/include/ops/declarable/helpers/impl/README.md b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/README.md similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/README.md rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/README.md diff --git a/libnd4j/include/ops/declarable/helpers/impl/choose.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/choose.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/choose.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/choose.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/gru.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/gru.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/gru.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/gru.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/knn_mindistance.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/knn_mindistance.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/knn_mindistance.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/knn_mindistance.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/listdiff.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/listdiff.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/listdiff.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/listdiff.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/lstm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/lstm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/lstm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/lstm.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/lstmLayer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/lstmLayer.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/lstmLayer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/lstmLayer.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/multiUnique.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/multiUnique.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/multiUnique.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/multiUnique.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/rnn.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/rnn.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/rnn.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/rnn.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/sparse_to_dense.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/sparse_to_dense.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/sparse_to_dense.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/sparse_to_dense.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/sqrtm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/sqrtm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/sqrtm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/sqrtm.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/unique.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/unique.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/unique.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/unique.cpp diff --git a/libnd4j/include/ops/declarable/helpers/impl/where.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/where.cpp similarity index 100% rename from libnd4j/include/ops/declarable/helpers/impl/where.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/impl/where.cpp diff --git a/libnd4j/include/ops/declarable/helpers/ismax.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/ismax.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/ismax.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/ismax.h diff --git a/libnd4j/include/ops/declarable/helpers/knn.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/knn.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/knn.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/knn.h diff --git a/libnd4j/include/ops/declarable/helpers/legacy_helpers.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/legacy_helpers.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/legacy_helpers.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/legacy_helpers.h diff --git a/libnd4j/include/ops/declarable/helpers/lgamma.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lgamma.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/lgamma.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lgamma.h diff --git a/libnd4j/include/ops/declarable/helpers/listdiff.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/listdiff.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/listdiff.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/listdiff.h diff --git a/libnd4j/include/ops/declarable/helpers/lrn.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lrn.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/lrn.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lrn.h diff --git a/libnd4j/include/ops/declarable/helpers/lstm.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lstm.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/lstm.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lstm.h diff --git a/libnd4j/include/ops/declarable/helpers/lstmBlock.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lstmBlock.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/lstmBlock.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lstmBlock.h diff --git a/libnd4j/include/ops/declarable/helpers/lstmLayer.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lstmLayer.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/lstmLayer.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lstmLayer.h diff --git a/libnd4j/include/ops/declarable/helpers/lstsq.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lstsq.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/lstsq.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lstsq.h diff --git a/libnd4j/include/ops/declarable/helpers/lup.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lup.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/lup.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/lup.h diff --git a/libnd4j/include/ops/declarable/helpers/matmul.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/matmul.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/matmul.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/matmul.h diff --git a/libnd4j/include/ops/declarable/helpers/matrixSetDiag.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/matrixSetDiag.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/matrixSetDiag.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/matrixSetDiag.h diff --git a/libnd4j/include/ops/declarable/helpers/matrix_band.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/matrix_band.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/matrix_band.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/matrix_band.h diff --git a/libnd4j/include/ops/declarable/helpers/matrix_diag_part.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/matrix_diag_part.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/matrix_diag_part.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/matrix_diag_part.h diff --git a/libnd4j/include/ops/declarable/helpers/max_pooling.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/max_pooling.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/max_pooling.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/max_pooling.h diff --git a/libnd4j/include/ops/declarable/helpers/meshgrid.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/meshgrid.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/meshgrid.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/meshgrid.h diff --git a/libnd4j/include/ops/declarable/helpers/minimax.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/minimax.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/minimax.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/minimax.h diff --git a/libnd4j/include/ops/declarable/helpers/multiUnique.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/multiUnique.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/multiUnique.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/multiUnique.h diff --git a/libnd4j/include/ops/declarable/helpers/nth_element.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/nth_element.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/nth_element.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/nth_element.h diff --git a/libnd4j/include/ops/declarable/helpers/one_hot.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/one_hot.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/one_hot.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/one_hot.h diff --git a/libnd4j/include/ops/declarable/helpers/percentile.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/percentile.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/percentile.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/percentile.h diff --git a/libnd4j/include/ops/declarable/helpers/prefix.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/prefix.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/prefix.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/prefix.h diff --git a/libnd4j/include/ops/declarable/helpers/print_variable.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/print_variable.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/print_variable.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/print_variable.h diff --git a/libnd4j/include/ops/declarable/helpers/qr.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/qr.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/qr.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/qr.h diff --git a/libnd4j/include/ops/declarable/helpers/random.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/random.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/random.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/random.h diff --git a/libnd4j/include/ops/declarable/helpers/random_crop.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/random_crop.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/random_crop.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/random_crop.h diff --git a/libnd4j/include/ops/declarable/helpers/range.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/range.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/range.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/range.h diff --git a/libnd4j/include/ops/declarable/helpers/reductions.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/reductions.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/reductions.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/reductions.h diff --git a/libnd4j/include/ops/declarable/helpers/reverse.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/reverse.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/reverse.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/reverse.h diff --git a/libnd4j/include/ops/declarable/helpers/rnn.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/rnn.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/rnn.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/rnn.h diff --git a/libnd4j/include/ops/declarable/helpers/roll.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/roll.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/roll.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/roll.h diff --git a/libnd4j/include/ops/declarable/helpers/s_t_b.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/s_t_b.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/s_t_b.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/s_t_b.h diff --git a/libnd4j/include/ops/declarable/helpers/s_t_d.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/s_t_d.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/s_t_d.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/s_t_d.h diff --git a/libnd4j/include/ops/declarable/helpers/scatter.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/scatter.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/scatter.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/scatter.h diff --git a/libnd4j/include/ops/declarable/helpers/segment.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/segment.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/segment.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/segment.h diff --git a/libnd4j/include/ops/declarable/helpers/segment_common.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/segment_common.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/segment_common.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/segment_common.h diff --git a/libnd4j/include/ops/declarable/helpers/sequence_mask.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sequence_mask.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/sequence_mask.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sequence_mask.h diff --git a/libnd4j/include/ops/declarable/helpers/sg_cb.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sg_cb.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/sg_cb.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sg_cb.h diff --git a/libnd4j/include/ops/declarable/helpers/shift.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/shift.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/shift.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/shift.h diff --git a/libnd4j/include/ops/declarable/helpers/solve.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/solve.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/solve.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/solve.h diff --git a/libnd4j/include/ops/declarable/helpers/sparse_to_dense.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sparse_to_dense.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/sparse_to_dense.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sparse_to_dense.h diff --git a/libnd4j/include/ops/declarable/helpers/sqrtm.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sqrtm.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/sqrtm.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sqrtm.h diff --git a/libnd4j/include/ops/declarable/helpers/sru.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sru.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/sru.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/sru.h diff --git a/libnd4j/include/ops/declarable/helpers/stack.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/stack.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/stack.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/stack.h diff --git a/libnd4j/include/ops/declarable/helpers/svd.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/svd.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/svd.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/svd.h diff --git a/libnd4j/include/ops/declarable/helpers/threshold.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/threshold.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/threshold.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/threshold.h diff --git a/libnd4j/include/ops/declarable/helpers/toggle_bits.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/toggle_bits.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/toggle_bits.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/toggle_bits.h diff --git a/libnd4j/include/ops/declarable/helpers/top_k.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/top_k.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/top_k.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/top_k.h diff --git a/libnd4j/include/ops/declarable/helpers/transforms.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/transforms.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/transforms.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/transforms.h diff --git a/libnd4j/include/ops/declarable/helpers/triangular_solve.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/triangular_solve.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/triangular_solve.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/triangular_solve.h diff --git a/libnd4j/include/ops/declarable/helpers/unique.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/unique.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/unique.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/unique.h diff --git a/libnd4j/include/ops/declarable/helpers/updatersHelpers.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/updatersHelpers.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/updatersHelpers.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/updatersHelpers.h diff --git a/libnd4j/include/ops/declarable/helpers/weights.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/weights.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/weights.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/weights.h diff --git a/libnd4j/include/ops/declarable/helpers/where.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/where.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/where.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/where.h diff --git a/libnd4j/include/ops/declarable/helpers/zeta.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/zeta.h similarity index 100% rename from libnd4j/include/ops/declarable/helpers/zeta.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/helpers/zeta.h diff --git a/libnd4j/include/ops/declarable/impl/BooleanOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/BooleanOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/BooleanOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/BooleanOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/BroadcastableBoolOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/BroadcastableBoolOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/BroadcastableBoolOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/BroadcastableBoolOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/BroadcastableOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/BroadcastableOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/BroadcastableOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/BroadcastableOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/DeclarableCustomOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/DeclarableCustomOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/DeclarableCustomOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/DeclarableCustomOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/DeclarableListOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/DeclarableListOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/DeclarableListOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/DeclarableListOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/DeclarableOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/DeclarableOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/DeclarableOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/DeclarableOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/DeclarableReductionOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/DeclarableReductionOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/DeclarableReductionOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/DeclarableReductionOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyBroadcastBoolOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyBroadcastBoolOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyBroadcastBoolOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyBroadcastBoolOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyBroadcastOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyBroadcastOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyBroadcastOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyBroadcastOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyIndexReduceOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyIndexReduceOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyIndexReduceOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyIndexReduceOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyPairwiseTransformBoolOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyPairwiseTransformBoolOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyPairwiseTransformBoolOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyPairwiseTransformBoolOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyPairwiseTransformOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyPairwiseTransformOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyPairwiseTransformOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyPairwiseTransformOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyRandomOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyRandomOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyRandomOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyRandomOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyReduce3Op.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduce3Op.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyReduce3Op.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduce3Op.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyReduceBoolOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceBoolOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyReduceBoolOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceBoolOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyReduceFloatOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceFloatOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyReduceFloatOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceFloatOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyReduceLongOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceLongOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyReduceLongOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceLongOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyReduceOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyReduceOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyReduceSameOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceSameOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyReduceSameOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyReduceSameOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyScalarBoolOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyScalarBoolOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyScalarBoolOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyScalarBoolOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyScalarOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyScalarOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyScalarOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyScalarOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyStatsOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyStatsOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyStatsOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyStatsOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyTransformAnyOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformAnyOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyTransformAnyOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformAnyOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyTransformBoolOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformBoolOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyTransformBoolOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformBoolOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyTransformFloatOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformFloatOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyTransformFloatOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformFloatOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyTransformOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyTransformOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyTransformSameOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformSameOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyTransformSameOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformSameOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LegacyTransformStrictOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformStrictOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LegacyTransformStrictOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LegacyTransformStrictOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/LogicOp.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LogicOp.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/LogicOp.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/LogicOp.cpp diff --git a/libnd4j/include/ops/declarable/impl/OpDescriptor.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/OpDescriptor.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/OpDescriptor.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/OpDescriptor.cpp diff --git a/libnd4j/include/ops/declarable/impl/OpRegistrator.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/OpRegistrator.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/OpRegistrator.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/OpRegistrator.cpp diff --git a/libnd4j/include/ops/declarable/impl/OpTuple.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/OpTuple.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/OpTuple.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/OpTuple.cpp diff --git a/libnd4j/include/ops/declarable/impl/PlatformHelper.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/PlatformHelper.cpp similarity index 100% rename from libnd4j/include/ops/declarable/impl/PlatformHelper.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/impl/PlatformHelper.cpp diff --git a/libnd4j/include/ops/declarable/platform/README.md b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/README.md similarity index 100% rename from libnd4j/include/ops/declarable/platform/README.md rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/README.md diff --git a/libnd4j/include/ops/declarable/platform/armcompute/armcomputeUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/armcomputeUtils.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/armcompute/armcomputeUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/armcomputeUtils.cpp diff --git a/libnd4j/include/ops/declarable/platform/armcompute/armcomputeUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/armcomputeUtils.h similarity index 100% rename from libnd4j/include/ops/declarable/platform/armcompute/armcomputeUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/armcomputeUtils.h diff --git a/libnd4j/include/ops/declarable/platform/armcompute/avgpooling2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/avgpooling2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/armcompute/avgpooling2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/avgpooling2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/armcompute/conv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/conv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/armcompute/conv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/conv2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/armcompute/deconv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/deconv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/armcompute/deconv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/deconv2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/armcompute/maxpooling2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/maxpooling2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/armcompute/maxpooling2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/armcompute/maxpooling2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/cudnn/avgpool2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/avgpool2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/avgpool2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/avgpool2d.cu diff --git a/libnd4j/include/ops/declarable/platform/cudnn/avgpool3d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/avgpool3d.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/avgpool3d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/avgpool3d.cu diff --git a/libnd4j/include/ops/declarable/platform/cudnn/batchnorm.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/batchnorm.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/batchnorm.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/batchnorm.cu diff --git a/libnd4j/include/ops/declarable/platform/cudnn/conv2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/conv2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/conv2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/conv2d.cu diff --git a/libnd4j/include/ops/declarable/platform/cudnn/conv3d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/conv3d.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/conv3d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/conv3d.cu diff --git a/libnd4j/include/ops/declarable/platform/cudnn/ctcloss.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/ctcloss.cu similarity index 99% rename from libnd4j/include/ops/declarable/platform/cudnn/ctcloss.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/ctcloss.cu index 7afeb4c4a..9e2fe727f 100644 --- a/libnd4j/include/ops/declarable/platform/cudnn/ctcloss.cu +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/ctcloss.cu @@ -20,7 +20,7 @@ #include "cudnnUtils.h" #include -#include +//#include namespace sd { diff --git a/libnd4j/include/ops/declarable/platform/cudnn/cudnnUtils.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/cudnnUtils.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/cudnnUtils.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/cudnnUtils.cu diff --git a/libnd4j/include/ops/declarable/platform/cudnn/cudnnUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/cudnnUtils.h similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/cudnnUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/cudnnUtils.h diff --git a/libnd4j/include/ops/declarable/platform/cudnn/depthwiseConv2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/depthwiseConv2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/depthwiseConv2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/depthwiseConv2d.cu diff --git a/libnd4j/include/ops/declarable/platform/cudnn/maxpool2d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/maxpool2d.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/maxpool2d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/maxpool2d.cu diff --git a/libnd4j/include/ops/declarable/platform/cudnn/maxpool3d.cu b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/maxpool3d.cu similarity index 100% rename from libnd4j/include/ops/declarable/platform/cudnn/maxpool3d.cu rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/cudnn/maxpool3d.cu diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/avgpooling2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/avgpooling2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/avgpooling2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/avgpooling2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/avgpooling3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/avgpooling3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/avgpooling3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/avgpooling3d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/batchnorm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/batchnorm.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/batchnorm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/batchnorm.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/concat.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/concat.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/concat.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/concat.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/conv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/conv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/conv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/conv2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/conv3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/conv3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/conv3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/conv3d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/deconv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/deconv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/deconv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/deconv2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/deconv2d_tf.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/deconv2d_tf.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/deconv2d_tf.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/deconv2d_tf.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/deconv3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/deconv3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/deconv3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/deconv3d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/depthwiseConv2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/depthwiseConv2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/depthwiseConv2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/depthwiseConv2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/lrn.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/lrn.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/lrn.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/lrn.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/lstmLayer.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/lstmLayer.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/lstmLayer.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/lstmLayer.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/matmul.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/matmul.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/matmul.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/matmul.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/maxpooling2d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/maxpooling2d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/maxpooling2d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/maxpooling2d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/maxpooling3d.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/maxpooling3d.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/maxpooling3d.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/maxpooling3d.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/mkldnnUtils.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/mkldnnUtils.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/mkldnnUtils.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/mkldnnUtils.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/mkldnnUtils.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/mkldnnUtils.h similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/mkldnnUtils.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/mkldnnUtils.h diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/softmax.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/softmax.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/softmax.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/softmax.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/tanh.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/tanh.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/tanh.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/tanh.cpp diff --git a/libnd4j/include/ops/declarable/platform/mkldnn/xw_plus_b.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/xw_plus_b.cpp similarity index 100% rename from libnd4j/include/ops/declarable/platform/mkldnn/xw_plus_b.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/declarable/platform/mkldnn/xw_plus_b.cpp diff --git a/libnd4j/include/ops/gemm.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/gemm.h similarity index 100% rename from libnd4j/include/ops/gemm.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/gemm.h diff --git a/libnd4j/include/ops/impl/BroadcastBoolOpsTuple.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/BroadcastBoolOpsTuple.cpp similarity index 100% rename from libnd4j/include/ops/impl/BroadcastBoolOpsTuple.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/BroadcastBoolOpsTuple.cpp diff --git a/libnd4j/include/ops/impl/BroadcastIntOpsTuple.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/BroadcastIntOpsTuple.cpp similarity index 100% rename from libnd4j/include/ops/impl/BroadcastIntOpsTuple.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/BroadcastIntOpsTuple.cpp diff --git a/libnd4j/include/ops/impl/BroadcastOpsTuple.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/BroadcastOpsTuple.cpp similarity index 100% rename from libnd4j/include/ops/impl/BroadcastOpsTuple.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/BroadcastOpsTuple.cpp diff --git a/libnd4j/include/ops/impl/compilation_units/specials_double.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/compilation_units/specials_double.cpp.in similarity index 100% rename from libnd4j/include/ops/impl/compilation_units/specials_double.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/compilation_units/specials_double.cpp.in diff --git a/libnd4j/include/ops/impl/compilation_units/specials_single.cpp.in b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/compilation_units/specials_single.cpp.in similarity index 100% rename from libnd4j/include/ops/impl/compilation_units/specials_single.cpp.in rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/compilation_units/specials_single.cpp.in diff --git a/libnd4j/include/ops/impl/gemm.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/gemm.cpp similarity index 100% rename from libnd4j/include/ops/impl/gemm.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/gemm.cpp diff --git a/libnd4j/include/ops/impl/specials_double.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/specials_double.hpp similarity index 100% rename from libnd4j/include/ops/impl/specials_double.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/specials_double.hpp diff --git a/libnd4j/include/ops/impl/specials_single.hpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/specials_single.hpp similarity index 100% rename from libnd4j/include/ops/impl/specials_single.hpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/specials_single.hpp diff --git a/libnd4j/include/ops/impl/specials_sparse.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/specials_sparse.cpp similarity index 100% rename from libnd4j/include/ops/impl/specials_sparse.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/impl/specials_sparse.cpp diff --git a/libnd4j/include/ops/meta_ops.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/meta_ops.h similarity index 100% rename from libnd4j/include/ops/meta_ops.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/meta_ops.h diff --git a/libnd4j/include/ops/ops.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/ops.h similarity index 100% rename from libnd4j/include/ops/ops.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/ops.h diff --git a/libnd4j/include/ops/random_ops.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/random_ops.h similarity index 100% rename from libnd4j/include/ops/random_ops.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/random_ops.h diff --git a/libnd4j/include/ops/special_random_ops.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/special_random_ops.h similarity index 100% rename from libnd4j/include/ops/special_random_ops.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/special_random_ops.h diff --git a/libnd4j/include/ops/specials.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/specials.h similarity index 100% rename from libnd4j/include/ops/specials.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/specials.h diff --git a/libnd4j/include/ops/specials_cuda.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/specials_cuda.h similarity index 100% rename from libnd4j/include/ops/specials_cuda.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/specials_cuda.h diff --git a/libnd4j/include/ops/specials_sparse.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/ops/specials_sparse.h similarity index 100% rename from libnd4j/include/ops/specials_sparse.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/ops/specials_sparse.h diff --git a/libnd4j/include/performance/benchmarking/BenchmarkSuit.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/BenchmarkSuit.h similarity index 100% rename from libnd4j/include/performance/benchmarking/BenchmarkSuit.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/BenchmarkSuit.h diff --git a/libnd4j/include/performance/benchmarking/FullBenchmarkSuit.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/FullBenchmarkSuit.h similarity index 100% rename from libnd4j/include/performance/benchmarking/FullBenchmarkSuit.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/FullBenchmarkSuit.h diff --git a/libnd4j/include/performance/benchmarking/LightBenchmarkSuit.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/LightBenchmarkSuit.h similarity index 100% rename from libnd4j/include/performance/benchmarking/LightBenchmarkSuit.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/LightBenchmarkSuit.h diff --git a/libnd4j/include/performance/benchmarking/impl/BenchmarkSuit.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/impl/BenchmarkSuit.cpp similarity index 100% rename from libnd4j/include/performance/benchmarking/impl/BenchmarkSuit.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/impl/BenchmarkSuit.cpp diff --git a/libnd4j/include/performance/benchmarking/impl/FullBenchmarkSuit.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/impl/FullBenchmarkSuit.cpp similarity index 100% rename from libnd4j/include/performance/benchmarking/impl/FullBenchmarkSuit.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/impl/FullBenchmarkSuit.cpp diff --git a/libnd4j/include/performance/benchmarking/impl/LightBenchmarkSuit.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/impl/LightBenchmarkSuit.cpp similarity index 100% rename from libnd4j/include/performance/benchmarking/impl/LightBenchmarkSuit.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/performance/benchmarking/impl/LightBenchmarkSuit.cpp diff --git a/libnd4j/include/samediff.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/samediff.h similarity index 100% rename from libnd4j/include/samediff.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/samediff.h diff --git a/libnd4j/include/system/BlasVersionHelper.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/BlasVersionHelper.h similarity index 100% rename from libnd4j/include/system/BlasVersionHelper.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/BlasVersionHelper.h diff --git a/libnd4j/include/system/Environment.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/Environment.h similarity index 100% rename from libnd4j/include/system/Environment.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/Environment.h diff --git a/libnd4j/include/system/buffer.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/buffer.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/system/buffer.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/buffer.h diff --git a/libnd4j/include/system/dll.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/dll.h similarity index 100% rename from libnd4j/include/system/dll.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/dll.h diff --git a/libnd4j/include/system/enum_boilerplate.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/enum_boilerplate.h similarity index 100% rename from libnd4j/include/system/enum_boilerplate.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/enum_boilerplate.h diff --git a/libnd4j/include/system/msvc.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/msvc.h similarity index 100% rename from libnd4j/include/system/msvc.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/msvc.h diff --git a/libnd4j/include/system/nd4jmalloc.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/nd4jmalloc.h similarity index 100% rename from libnd4j/include/system/nd4jmalloc.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/nd4jmalloc.h diff --git a/libnd4j/include/system/nd4jmemset.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/nd4jmemset.h similarity index 100% rename from libnd4j/include/system/nd4jmemset.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/nd4jmemset.h diff --git a/libnd4j/include/system/op_boilerplate.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/op_boilerplate.h similarity index 100% rename from libnd4j/include/system/op_boilerplate.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/op_boilerplate.h diff --git a/libnd4j/include/system/op_enums.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/op_enums.h similarity index 100% rename from libnd4j/include/system/op_enums.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/op_enums.h diff --git a/libnd4j/include/system/openmp_pragmas.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/openmp_pragmas.h similarity index 100% rename from libnd4j/include/system/openmp_pragmas.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/openmp_pragmas.h diff --git a/libnd4j/include/system/optype.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/optype.h similarity index 100% rename from libnd4j/include/system/optype.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/optype.h diff --git a/libnd4j/include/system/pairwise_util.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/pairwise_util.h old mode 100755 new mode 100644 similarity index 100% rename from libnd4j/include/system/pairwise_util.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/pairwise_util.h diff --git a/libnd4j/include/system/platform_boilerplate.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/platform_boilerplate.h similarity index 100% rename from libnd4j/include/system/platform_boilerplate.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/platform_boilerplate.h diff --git a/libnd4j/include/system/play.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/play.h similarity index 100% rename from libnd4j/include/system/play.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/play.h diff --git a/libnd4j/include/system/pointercast.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/pointercast.h similarity index 100% rename from libnd4j/include/system/pointercast.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/pointercast.h diff --git a/libnd4j/include/system/type_boilerplate.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/type_boilerplate.h similarity index 100% rename from libnd4j/include/system/type_boilerplate.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/type_boilerplate.h diff --git a/libnd4j/include/system/util.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/system/util.h similarity index 100% rename from libnd4j/include/system/util.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/system/util.h diff --git a/libnd4j/include/types/bfloat16.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/bfloat16.h similarity index 99% rename from libnd4j/include/types/bfloat16.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/bfloat16.h index 2817e82c7..f8aec43db 100644 --- a/libnd4j/include/types/bfloat16.h +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/bfloat16.h @@ -75,10 +75,13 @@ return this->_data == 0 ? false : true; } + /* template ::value>::type> local_def explicit operator T() const { return static_cast(static_cast(*this)); } +*/ + local_def bfloat16& operator=(const bool rhs) { *this = (float)rhs ? 1.f: 0.f; diff --git a/libnd4j/include/types/float16.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/float16.h similarity index 95% rename from libnd4j/include/types/float16.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/float16.h index ed0b1b3ae..69831609e 100644 --- a/libnd4j/include/types/float16.h +++ b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/float16.h @@ -23,7 +23,7 @@ #include #include #include -#if defined(__INTEL_COMPILER) || defined(SD_F16C) +#if defined(__INTEL_COMPILER) #include #endif @@ -32,9 +32,6 @@ struct bfloat16; #ifdef __CUDACC__ #include -#if CUDA_VERSION_MAJOR != 8 -// CUDA_9 and above - struct ihalf : public __half { public: __host__ __device__ ihalf() : half() { @@ -54,26 +51,6 @@ struct ihalf : public __half { } }; -#else -struct ihalf : public __half { - public: - __host__ __device__ ihalf() : half() { - // - } - - inline __host__ __device__ unsigned short * getXP() { - return &this->x; - } - - inline __host__ __device__ unsigned short getX() const { - return this->x; - } - - inline __host__ __device__ void assign(const half f) { - this->x = ((__half *) &f)->x; - } -}; -#endif // CUDA_8 #else struct __half { @@ -123,7 +100,7 @@ static local_def unsigned short hneg(unsigned short h) { } -#if defined(__INTEL_COMPILER) || defined(SD_F16C) +#if defined(__INTEL_COMPILER) //_Pragma("omp declare simd") inline local_def float cpu_ihalf2float(ihalf h) { return _cvtsh_ss(h.getX()); @@ -158,7 +135,7 @@ local_def float cpu_ihalf2float(ihalf h) { } #endif -#if defined(__INTEL_COMPILER) || defined(SD_F16C) +#if defined(__INTEL_COMPILER) //_Pragma("omp declare simd") inline local_def ihalf cpu_float2ihalf_rn(float f) { ihalf ret; @@ -363,11 +340,7 @@ struct float16 { local_def friend float16 operator*(const float16& a, const float16& b) { return __hmul(a.data, b.data); } local_def friend float16 operator/(const float16& a, const float16& b) { - #if CUDA_VERSION_MAJOR == 8 - return hdiv(a.data, b.data); - #else - return __hdiv(a.data, b.data); - #endif + return __hdiv(a.data, b.data); } #else local_def friend float16 operator+(const float16& a, const float16& b) { return float16((float)a + (float)b); } diff --git a/libnd4j/include/types/float8.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/float8.h similarity index 100% rename from libnd4j/include/types/float8.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/float8.h diff --git a/libnd4j/include/types/impl/float8.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/float8.cpp similarity index 100% rename from libnd4j/include/types/impl/float8.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/float8.cpp diff --git a/libnd4j/include/types/impl/int16.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/int16.cpp similarity index 100% rename from libnd4j/include/types/impl/int16.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/int16.cpp diff --git a/libnd4j/include/types/impl/int8.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/int8.cpp similarity index 100% rename from libnd4j/include/types/impl/int8.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/int8.cpp diff --git a/libnd4j/include/types/impl/pair.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/pair.cpp similarity index 100% rename from libnd4j/include/types/impl/pair.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/pair.cpp diff --git a/libnd4j/include/types/impl/triple.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/triple.cpp similarity index 100% rename from libnd4j/include/types/impl/triple.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/triple.cpp diff --git a/libnd4j/include/types/impl/uint16.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/uint16.cpp similarity index 100% rename from libnd4j/include/types/impl/uint16.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/uint16.cpp diff --git a/libnd4j/include/types/impl/uint8.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/uint8.cpp similarity index 100% rename from libnd4j/include/types/impl/uint8.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/uint8.cpp diff --git a/libnd4j/include/types/impl/utf8string.cpp b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/utf8string.cpp similarity index 100% rename from libnd4j/include/types/impl/utf8string.cpp rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/impl/utf8string.cpp diff --git a/libnd4j/include/types/int16.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/int16.h similarity index 100% rename from libnd4j/include/types/int16.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/int16.h diff --git a/libnd4j/include/types/int8.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/int8.h similarity index 100% rename from libnd4j/include/types/int8.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/int8.h diff --git a/libnd4j/include/types/pair.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/pair.h similarity index 100% rename from libnd4j/include/types/pair.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/pair.h diff --git a/libnd4j/include/types/triple.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/triple.h similarity index 100% rename from libnd4j/include/types/triple.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/triple.h diff --git a/libnd4j/include/types/types.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/types.h similarity index 100% rename from libnd4j/include/types/types.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/types.h diff --git a/libnd4j/include/types/u32.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/u32.h similarity index 100% rename from libnd4j/include/types/u32.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/u32.h diff --git a/libnd4j/include/types/u64.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/u64.h similarity index 100% rename from libnd4j/include/types/u64.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/u64.h diff --git a/libnd4j/include/types/uint16.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/uint16.h similarity index 100% rename from libnd4j/include/types/uint16.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/uint16.h diff --git a/libnd4j/include/types/uint8.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/uint8.h similarity index 100% rename from libnd4j/include/types/uint8.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/uint8.h diff --git a/libnd4j/include/types/utf8string.h b/cavis-native/cavis-native-lib/src/main/cpp/blas/types/utf8string.h similarity index 100% rename from libnd4j/include/types/utf8string.h rename to cavis-native/cavis-native-lib/src/main/cpp/blas/types/utf8string.h diff --git a/libnd4j/include/config.h.in b/cavis-native/cavis-native-lib/src/main/include/config.h.in similarity index 100% rename from libnd4j/include/config.h.in rename to cavis-native/cavis-native-lib/src/main/include/config.h.in diff --git a/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/Dummy.java b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/Dummy.java new file mode 100644 index 000000000..35a1c3778 --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/Dummy.java @@ -0,0 +1,25 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.nd4j.nativeblas; + +public class Dummy { +} diff --git a/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cpu/Nd4jCpuHelper.java b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cpu/Nd4jCpuHelper.java new file mode 100644 index 000000000..16d52d425 --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cpu/Nd4jCpuHelper.java @@ -0,0 +1,27 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.nd4j.nativeblas.cpu; + +import org.nd4j.nativeblas.NativeOps; + +public abstract class Nd4jCpuHelper extends Nd4jCpuPresets implements NativeOps { +} diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native-preset/src/main/java/org/nd4j/nativeblas/Nd4jCpuPresets.java b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cpu/Nd4jCpuPresets.java similarity index 94% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native-preset/src/main/java/org/nd4j/nativeblas/Nd4jCpuPresets.java rename to cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cpu/Nd4jCpuPresets.java index 2077d1431..cb8e71315 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native-preset/src/main/java/org/nd4j/nativeblas/Nd4jCpuPresets.java +++ b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cpu/Nd4jCpuPresets.java @@ -1,24 +1,25 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ -package org.nd4j.nativeblas; +package org.nd4j.nativeblas.cpu; import org.bytedeco.javacpp.annotation.Platform; import org.bytedeco.javacpp.annotation.Properties; @@ -36,7 +37,7 @@ import java.util.Scanner; * * @author saudet */ -@Properties(inherit = openblas.class, target = "org.nd4j.nativeblas.Nd4jCpu", helper = "org.nd4j.nativeblas.Nd4jCpuHelper", +@Properties(inherit = openblas.class, target = "org.nd4j.nativeblas.Nd4jCpu", helper = "org.nd4j.nativeblas.cpu.Nd4jCpuHelper", value = {@Platform(define = "LIBND4J_ALL_OPS", include = { "memory/MemoryType.h", "array/DataType.h", @@ -148,7 +149,8 @@ import java.util.Scanner; "lapacke_utils.h", "cnpy/cnpy.h" }, - compiler = {"cpp11", "nowarnings"}, + // compiler = {"cpp11", "nowarnings"}, + compiler = {"cpp11"}, library = "jnind4jcpu", link = "nd4jcpu", preload = "libnd4jcpu"), @Platform(value = "linux", preload = "gomp@.1", preloadpath = {"/lib64/", "/lib/", "/usr/lib64/", "/usr/lib/"}), @Platform(value = "linux-armhf", preloadpath = {"/usr/arm-linux-gnueabihf/lib/", "/usr/lib/arm-linux-gnueabihf/"}), diff --git a/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cuda/Nd4jCudaHelper.java b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cuda/Nd4jCudaHelper.java new file mode 100644 index 000000000..6d70238e2 --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cuda/Nd4jCudaHelper.java @@ -0,0 +1,27 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package org.nd4j.nativeblas.cuda; + +import org.nd4j.nativeblas.NativeOps; + +public abstract class Nd4jCudaHelper extends Nd4jCudaPresets implements NativeOps { +} diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/java/org/nd4j/nativeblas/Nd4jCudaPresets.java b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cuda/Nd4jCudaPresets.java similarity index 90% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/java/org/nd4j/nativeblas/Nd4jCudaPresets.java rename to cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cuda/Nd4jCudaPresets.java index ab2cc0550..eeecf849e 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/java/org/nd4j/nativeblas/Nd4jCudaPresets.java +++ b/cavis-native/cavis-native-lib/src/main/java/org/nd4j/nativeblas/cuda/Nd4jCudaPresets.java @@ -1,26 +1,26 @@ /* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * */ -package org.nd4j.nativeblas; +package org.nd4j.nativeblas.cuda; -import java.util.List; import org.bytedeco.javacpp.ClassProperties; import org.bytedeco.javacpp.LoadEnabled; import org.bytedeco.javacpp.Loader; @@ -30,11 +30,13 @@ import org.bytedeco.javacpp.tools.Info; import org.bytedeco.javacpp.tools.InfoMap; import org.bytedeco.javacpp.tools.InfoMapper; +import java.util.List; + /** * * @author saudet */ -@Properties(target = "org.nd4j.nativeblas.Nd4jCuda", helper = "org.nd4j.nativeblas.Nd4jCudaHelper", +@Properties(target = "org.nd4j.nativeblas.Nd4jCuda", helper = "org.nd4j.nativeblas.cuda.Nd4jCudaHelper", value = {@Platform(define = "LIBND4J_ALL_OPS", include = { "array/DataType.h", "array/DataBuffer.h", @@ -123,12 +125,12 @@ import org.bytedeco.javacpp.tools.InfoMapper; "cnpy/cnpy.h" }, compiler = {"cpp11", "nowarnings"}, - library = "jnind4jcuda", link = "nd4jcuda", preload = "libnd4jcuda"), + library = "jnind4jcuda", link = "nd4jcuda", preload = "nd4jcuda"), @Platform(value = "linux", preload = "gomp@.1", preloadpath = {"/lib64/", "/lib/", "/usr/lib64/", "/usr/lib/"}), @Platform(value = "linux-armhf", preloadpath = {"/usr/arm-linux-gnueabihf/lib/", "/usr/lib/arm-linux-gnueabihf/"}), @Platform(value = "linux-arm64", preloadpath = {"/usr/aarch64-linux-gnu/lib/", "/usr/lib/aarch64-linux-gnu/"}), @Platform(value = "linux-ppc64", preloadpath = {"/usr/powerpc64-linux-gnu/lib/", "/usr/powerpc64le-linux-gnu/lib/", "/usr/lib/powerpc64-linux-gnu/", "/usr/lib/powerpc64le-linux-gnu/"}), - @Platform(value = "windows", preload = {"libwinpthread-1", "libgcc_s_seh-1", "libgomp-1", "libstdc++-6", "libnd4jcpu"}) }) + @Platform(value = "windows", preload = {"libwinpthread-1", "libgcc_s_seh-1", "libgomp-1", "libstdc++-6", "nd4jcuda"}) }) public class Nd4jCudaPresets implements LoadEnabled, InfoMapper { @Override public void init(ClassProperties properties) { @@ -146,9 +148,9 @@ public class Nd4jCudaPresets implements LoadEnabled, InfoMapper { "cudnn_adv_train", "cudnn_cnn_infer", "cudnn_cnn_train"}; for (String lib : libs) { if (platform.startsWith("linux")) { - lib += lib.startsWith("cudnn") ? "@.8" : lib.equals("curand") || lib.equals("cusolver") ? "@.10" : lib.equals("cudart") ? "@.11.0" : "@.11"; + lib += lib.startsWith("cudnn") ? "@.8" : lib.equals("curand") ? "@.10" : lib.equals("cudart") ? "@.11.0" : "@.11"; } else if (platform.startsWith("windows")) { - lib += lib.startsWith("cudnn") ? "64_8" : lib.equals("curand") || lib.equals("cusolver") ? "64_10" : lib.equals("cudart") ? "64_110" : "64_11"; + lib += lib.startsWith("cudnn") ? "64_8" : lib.equals("curand") ? "64_10" : lib.equals("cudart") ? "64_110" : "64_11"; } else { continue; // no CUDA } diff --git a/cavis-native/cavis-native-lib/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor b/cavis-native/cavis-native-lib/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor new file mode 100644 index 000000000..cb9fb036e --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/resources/META-INF/services/org.nd4j.linalg.compression.NDArrayCompressor @@ -0,0 +1,23 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + + +org.nd4j.linalg.cpu.nativecpu.compression.CpuThreshold \ No newline at end of file diff --git a/cavis-native/cavis-native-lib/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend b/cavis-native/cavis-native-lib/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend new file mode 100644 index 000000000..89e271523 --- /dev/null +++ b/cavis-native/cavis-native-lib/src/main/resources/META-INF/services/org.nd4j.linalg.factory.Nd4jBackend @@ -0,0 +1,23 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# + +#org.nd4j.linalg.jcublas.JCublasBackend +org.nd4j.linalg.cpu.nativecpu.CpuBackend \ No newline at end of file diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/resources/function_threads.properties b/cavis-native/cavis-native-lib/src/main/resources/function_threads.properties similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/resources/function_threads.properties rename to cavis-native/cavis-native-lib/src/main/resources/function_threads.properties diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/resources/native.properties b/cavis-native/cavis-native-lib/src/main/resources/native.properties similarity index 100% rename from nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda-preset/src/main/resources/native.properties rename to cavis-native/cavis-native-lib/src/main/resources/native.properties diff --git a/cavis-nd4j/build.gradle b/cavis-nd4j/build.gradle new file mode 100644 index 000000000..91d22ef69 --- /dev/null +++ b/cavis-nd4j/build.gradle @@ -0,0 +1,7 @@ +subprojects { + group = group + ".cavis-nd4j" + + apply plugin: "java-library" + apply plugin: "maven-publish" + apply plugin: "signing" +} \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-aeron/build.gradle b/cavis-nd4j/cavis-nd4j-aeron/build.gradle new file mode 100644 index 000000000..b511bd100 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-aeron/build.gradle @@ -0,0 +1,31 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation "io.aeron:aeron-all:1.32.0" + implementation "org.slf4j:slf4j-api" + implementation projects.cavisDnn.cavisDnnApi + implementation "com.google.guava:guava" + testImplementation 'ch.qos.logback:logback-core' + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testImplementation "org.apache.commons:commons-lang3" +} \ No newline at end of file diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronConnectionInformation.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronConnectionInformation.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronConnectionInformation.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronConnectionInformation.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArrayPublisher.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySerde.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySerde.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySerde.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySerde.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java similarity index 97% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java index 8a57d698d..91a3dbea5 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronNDArraySubscriber.java @@ -25,6 +25,8 @@ import io.aeron.FragmentAssembler; import io.aeron.Subscription; import lombok.Builder; import lombok.Data; +import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import org.agrona.CloseHelper; import org.agrona.concurrent.SigInt; import org.slf4j.Logger; @@ -37,6 +39,7 @@ import java.util.concurrent.atomic.AtomicBoolean; @Data @Builder +@Slf4j public class AeronNDArraySubscriber implements AutoCloseable { // The channel (an endpoint identifier) to receive messages from private String channel; @@ -50,7 +53,7 @@ public class AeronNDArraySubscriber implements AutoCloseable { private Aeron.Context ctx; private AtomicBoolean running = new AtomicBoolean(true); private final AtomicBoolean init = new AtomicBoolean(false); - private static Logger log = LoggerFactory.getLogger(AeronNDArraySubscriber.class); + private NDArrayCallback ndArrayCallback; private Aeron aeron; private Subscription subscription; @@ -118,7 +121,8 @@ public class AeronNDArraySubscriber implements AutoCloseable { if (aeron == null) throw new IllegalStateException("No aeron instance defined"); boolean started = false; - while (!started) { + int tries=0; + while (!started && tries < 10) { try (final Subscription subscription = aeron.addSubscription(channel, streamId)) { this.subscription = subscription; log.info("Beginning subscribe on channel " + channel + " and stream " + streamId); @@ -128,6 +132,8 @@ public class AeronNDArraySubscriber implements AutoCloseable { } catch (Exception e) { log.warn("Unable to connect...trying again on channel " + channel, e); + } finally { + tries++; } } @@ -168,7 +174,7 @@ public class AeronNDArraySubscriber implements AutoCloseable { try { subscriber.launch(); } catch (Exception e) { - log.error("",e); + log.error(e.getMessage(),e); } }); diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronUtil.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronUtil.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronUtil.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronUtil.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/LowLatencyMediaDriver.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/LowLatencyMediaDriver.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/LowLatencyMediaDriver.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/LowLatencyMediaDriver.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayCallback.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayCallback.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayCallback.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayCallback.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayFragmentHandler.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayFragmentHandler.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayFragmentHandler.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayFragmentHandler.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayHolder.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayHolder.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayHolder.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayHolder.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/NDArrayMessage.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulator.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulator.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulator.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulator.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/InMemoryChunkAccumulator.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/InMemoryChunkAccumulator.java similarity index 98% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/InMemoryChunkAccumulator.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/InMemoryChunkAccumulator.java index 56d4a924f..b725d3c04 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/InMemoryChunkAccumulator.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/InMemoryChunkAccumulator.java @@ -20,7 +20,7 @@ package org.nd4j.aeron.ipc.chunk; -import org.nd4j.shade.guava.collect.Maps; +import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; import org.nd4j.aeron.ipc.NDArrayMessage; diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunk.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunk.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunk.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunk.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponder.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/HostPortPublisher.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/NDArrayResponseFragmentHandler.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/NDArrayResponseFragmentHandler.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/NDArrayResponseFragmentHandler.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/response/NDArrayResponseFragmentHandler.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ndarrayholder/InMemoryNDArrayHolder.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ndarrayholder/InMemoryNDArrayHolder.java similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ndarrayholder/InMemoryNDArrayHolder.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/ndarrayholder/InMemoryNDArrayHolder.java diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/util/BufferUtil.java b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/util/BufferUtil.java similarity index 96% rename from nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/util/BufferUtil.java rename to cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/util/BufferUtil.java index 8306f4604..0ea6770e9 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/util/BufferUtil.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/main/java/org/nd4j/aeron/util/BufferUtil.java @@ -21,7 +21,6 @@ package org.nd4j.aeron.util; -import java.nio.Buffer; import java.nio.ByteBuffer; /** @@ -66,8 +65,7 @@ public class BufferUtil { all.put(curr); } - Buffer buffer = (Buffer) all; - buffer.flip(); + all.flip(); return all; } diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/AeronNDArraySerdeTest.java b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/AeronNDArraySerdeTest.java similarity index 81% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/AeronNDArraySerdeTest.java rename to cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/AeronNDArraySerdeTest.java index ed1532a7d..15aa2cc73 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/AeronNDArraySerdeTest.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/AeronNDArraySerdeTest.java @@ -22,27 +22,18 @@ package org.nd4j.aeron.ipc; import org.agrona.concurrent.UnsafeBuffer; import org.apache.commons.lang3.time.StopWatch; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import javax.annotation.concurrent.NotThreadSafe; import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -@NotThreadSafe -@Disabled("Tests are too flaky") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(120) public class AeronNDArraySerdeTest extends BaseND4JTest { @Test @@ -50,35 +41,33 @@ public class AeronNDArraySerdeTest extends BaseND4JTest { INDArray arr = Nd4j.scalar(1.0); UnsafeBuffer buffer = AeronNDArraySerde.toBuffer(arr); INDArray back = AeronNDArraySerde.toArray(buffer); - assertEquals(arr, back); + Assertions.assertEquals(arr, back); } @Test public void testToAndFromCompressed() { INDArray arr = Nd4j.scalar(1.0); INDArray compress = Nd4j.getCompressor().compress(arr, "GZIP"); - assertTrue(compress.isCompressed()); + Assertions.assertTrue(compress.isCompressed()); UnsafeBuffer buffer = AeronNDArraySerde.toBuffer(compress); INDArray back = AeronNDArraySerde.toArray(buffer); INDArray decompressed = Nd4j.getCompressor().decompress(compress); - assertEquals(arr, decompressed); - assertEquals(arr, back); + Assertions.assertEquals(arr, decompressed); + Assertions.assertEquals(arr, back); } @Test - @Disabled // timeout, skip step ignored public void testToAndFromCompressedLarge() { - skipUnlessIntegrationTests(); INDArray arr = Nd4j.zeros((int) 1e7); INDArray compress = Nd4j.getCompressor().compress(arr, "GZIP"); - assertTrue(compress.isCompressed()); + Assertions.assertTrue(compress.isCompressed()); UnsafeBuffer buffer = AeronNDArraySerde.toBuffer(compress); INDArray back = AeronNDArraySerde.toArray(buffer); INDArray decompressed = Nd4j.getCompressor().decompress(compress); - assertEquals(arr, decompressed); - assertEquals(arr, back); + Assertions.assertEquals(arr, decompressed); + Assertions.assertEquals(arr, back); } @@ -113,7 +102,6 @@ public class AeronNDArraySerdeTest extends BaseND4JTest { } - @Override public long getTimeoutMilliseconds() { return Long.MAX_VALUE; } diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java similarity index 79% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java rename to cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java index 9fb249947..85ec9f01b 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/LargeNdArrayIpcTest.java @@ -24,24 +24,20 @@ import io.aeron.Aeron; import io.aeron.driver.MediaDriver; import lombok.extern.slf4j.Slf4j; import org.agrona.CloseHelper; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import javax.annotation.concurrent.NotThreadSafe; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.jupiter.api.Assertions.assertFalse; @Slf4j -@NotThreadSafe -@Disabled("Tests are too flaky") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(120) public class LargeNdArrayIpcTest extends BaseND4JTest { private MediaDriver mediaDriver; private Aeron.Context ctx; @@ -51,39 +47,34 @@ public class LargeNdArrayIpcTest extends BaseND4JTest { @Override public long getTimeoutMilliseconds() { - return 180000L; + return 12000L; } @BeforeEach public void before() { - if(isIntegrationTests()) { - //MediaDriver.loadPropertiesFile("aeron.properties"); - MediaDriver.Context ctx = AeronUtil.getMediaDriverContext(length); - mediaDriver = MediaDriver.launchEmbedded(ctx); - System.out.println("Using media driver directory " + mediaDriver.aeronDirectoryName()); - System.out.println("Launched media driver"); - } + } @AfterEach public void after() { - if(isIntegrationTests()) { CloseHelper.quietClose(mediaDriver); - } } @Test - @Disabled public void testMultiThreadedIpcBig() throws Exception { - skipUnlessIntegrationTests(); //Long-running test - don't run as part of unit tests by default + //MediaDriver.loadPropertiesFile("aeron.properties"); + MediaDriver.Context mctx = AeronUtil.getMediaDriverContext(length).useWindowsHighResTimer(true); + mediaDriver = MediaDriver.launchEmbedded(mctx); + System.out.println("Using media driver directory " + mediaDriver.aeronDirectoryName()); + System.out.println("Launched media driver"); int length = (int) 1e7; INDArray arr = Nd4j.ones(length); AeronNDArrayPublisher publisher; ctx = new Aeron.Context() - .driverTimeoutMs(1000000).availableImageHandler(AeronUtil::printAvailableImage) + .driverTimeoutMs(10000).availableImageHandler(AeronUtil::printAvailableImage) .unavailableImageHandler(AeronUtil::printUnavailableImage) - .aeronDirectoryName(mediaDriver.aeronDirectoryName()).keepAliveIntervalNs(1000000) + .aeronDirectoryName(mediaDriver.aeronDirectoryName()) .errorHandler(err -> err.printStackTrace()); final AtomicBoolean running = new AtomicBoolean(true); @@ -129,9 +120,9 @@ public class LargeNdArrayIpcTest extends BaseND4JTest { subscribers[i] = subscriber; } - Thread.sleep(10000); + Thread.sleep(1000); - publisher = AeronNDArrayPublisher.builder().publishRetryTimeOut(300000).streamId(streamId).channel(channel) + publisher = AeronNDArrayPublisher.builder().publishRetryTimeOut(3000).streamId(streamId).channel(channel) .aeron(aeron).build(); @@ -142,10 +133,6 @@ public class LargeNdArrayIpcTest extends BaseND4JTest { } - Thread.sleep(30000); - - - for (int i = 0; i < numSubscribers; i++) CloseHelper.close(subscribers[i]); CloseHelper.close(aeron); @@ -157,10 +144,10 @@ public class LargeNdArrayIpcTest extends BaseND4JTest { private Aeron.Context getContext() { if (ctx == null) - ctx = new Aeron.Context().driverTimeoutMs(1000000) + ctx = new Aeron.Context().driverTimeoutMs(10000) .availableImageHandler(AeronUtil::printAvailableImage) .unavailableImageHandler(AeronUtil::printUnavailableImage) - .aeronDirectoryName(mediaDriver.aeronDirectoryName()).keepAliveIntervalNs(100000) + .aeronDirectoryName(mediaDriver.aeronDirectoryName()) .errorHandler(err -> err.printStackTrace()); return ctx; } diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NDArrayMessageTest.java b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NDArrayMessageTest.java similarity index 79% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NDArrayMessageTest.java rename to cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NDArrayMessageTest.java index c452fd231..872ea3682 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NDArrayMessageTest.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NDArrayMessageTest.java @@ -21,24 +21,14 @@ package org.nd4j.aeron.ipc; import org.agrona.DirectBuffer; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import javax.annotation.concurrent.NotThreadSafe; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@NotThreadSafe -@Disabled("Tests are too flaky") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(120) public class NDArrayMessageTest extends BaseND4JTest { @Test @@ -47,13 +37,13 @@ public class NDArrayMessageTest extends BaseND4JTest { DirectBuffer bufferConvert = NDArrayMessage.toBuffer(message); bufferConvert.byteBuffer().rewind(); NDArrayMessage newMessage = NDArrayMessage.fromBuffer(bufferConvert, 0); - assertEquals(message, newMessage); + Assertions.assertEquals(message, newMessage); INDArray compressed = Nd4j.getCompressor().compress(Nd4j.scalar(1.0), "GZIP"); NDArrayMessage messageCompressed = NDArrayMessage.wholeArrayUpdate(compressed); DirectBuffer bufferConvertCompressed = NDArrayMessage.toBuffer(messageCompressed); NDArrayMessage newMessageTest = NDArrayMessage.fromBuffer(bufferConvertCompressed, 0); - assertEquals(messageCompressed, newMessageTest); + Assertions.assertEquals(messageCompressed, newMessageTest); } diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NdArrayIpcTest.java b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NdArrayIpcTest.java similarity index 91% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NdArrayIpcTest.java rename to cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NdArrayIpcTest.java index 42ed02c0f..999e11281 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NdArrayIpcTest.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/NdArrayIpcTest.java @@ -25,25 +25,16 @@ import io.aeron.driver.MediaDriver; import org.agrona.CloseHelper; import org.junit.jupiter.api.*; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.concurrent.NotThreadSafe; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; -import static org.junit.jupiter.api.Assertions.assertFalse; - -@NotThreadSafe -@Disabled("Tests are too flaky") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(120) public class NdArrayIpcTest extends BaseND4JTest { private MediaDriver mediaDriver; private static Logger log = LoggerFactory.getLogger(NdArrayIpcTest.class); @@ -59,24 +50,19 @@ public class NdArrayIpcTest extends BaseND4JTest { @BeforeEach public void before() { - if(isIntegrationTests()) { MediaDriver.Context ctx = AeronUtil.getMediaDriverContext(length); mediaDriver = MediaDriver.launchEmbedded(ctx); System.out.println("Using media driver directory " + mediaDriver.aeronDirectoryName()); System.out.println("Launched media driver"); - } } @AfterEach public void after() { - if(isIntegrationTests()) { CloseHelper.quietClose(mediaDriver); - } } @Test public void testMultiThreadedIpc() throws Exception { - skipUnlessIntegrationTests(); //Long-running test - don't run as part of unit tests by default ExecutorService executorService = Executors.newFixedThreadPool(4); INDArray arr = Nd4j.scalar(1.0); @@ -155,12 +141,11 @@ public class NdArrayIpcTest extends BaseND4JTest { CloseHelper.close(publisher); CloseHelper.close(aeron); Thread.sleep(10000); - assertFalse(running.get()); + Assertions.assertFalse(running.get()); } @Test public void testIpc() throws Exception { - skipUnlessIntegrationTests(); //Long-running test - don't run as part of unit tests by default INDArray arr = Nd4j.scalar(1.0); @@ -208,7 +193,7 @@ public class NdArrayIpcTest extends BaseND4JTest { while (!subscriber.launched()) Thread.sleep(1000); - Thread.sleep(10000); + Thread.sleep(1000); AeronNDArrayPublisher publisher = AeronNDArrayPublisher.builder().streamId(streamId).aeron(aeron).channel(channel).build(); @@ -220,7 +205,7 @@ public class NdArrayIpcTest extends BaseND4JTest { Thread.sleep(30000); - assertFalse(running.get()); + Assertions.assertFalse(running.get()); publisher.close(); diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulatorTests.java b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulatorTests.java similarity index 79% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulatorTests.java rename to cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulatorTests.java index fe1e5d47e..0665b4a03 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulatorTests.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/ChunkAccumulatorTests.java @@ -20,24 +20,14 @@ package org.nd4j.aeron.ipc.chunk; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.aeron.ipc.NDArrayMessage; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; -import javax.annotation.concurrent.NotThreadSafe; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@NotThreadSafe -@Disabled("Tests are too flaky") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(120) public class ChunkAccumulatorTests extends BaseND4JTest { @Test @@ -51,7 +41,7 @@ public class ChunkAccumulatorTests extends BaseND4JTest { } NDArrayMessage message1 = chunkAccumulator.reassemble(chunks[0].getId()); - assertEquals(message, message1); + Assertions.assertEquals(message, message1); } } diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunkTests.java b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunkTests.java similarity index 86% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunkTests.java rename to cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunkTests.java index f756b8265..7aad3b0c9 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunkTests.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/chunk/NDArrayMessageChunkTests.java @@ -21,27 +21,18 @@ package org.nd4j.aeron.ipc.chunk; import org.agrona.DirectBuffer; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.aeron.ipc.NDArrayMessage; import org.nd4j.aeron.util.BufferUtil; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; -import javax.annotation.concurrent.NotThreadSafe; import java.nio.ByteBuffer; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -@NotThreadSafe -@Disabled("Tests are too flaky") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(120) public class NDArrayMessageChunkTests extends BaseND4JTest { @Test @@ -73,7 +64,7 @@ public class NDArrayMessageChunkTests extends BaseND4JTest { byte[] arrays2 = new byte[concatAll.capacity()]; concatAll.rewind(); concatAll.get(arrays2); - assertArrayEquals(arrays, arrays2); + Assertions.assertArrayEquals(arrays, arrays2); NDArrayMessage message1 = NDArrayMessage.fromChunks(chunks); assertEquals(message, message1); diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponseTest.java b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponseTest.java similarity index 92% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponseTest.java rename to cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponseTest.java index 140c35e95..cdecd76a0 100644 --- a/nd4j/nd4j-serde/nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponseTest.java +++ b/cavis-nd4j/cavis-nd4j-aeron/src/test/java/org/nd4j/aeron/ipc/response/AeronNDArrayResponseTest.java @@ -26,40 +26,30 @@ import io.aeron.driver.ThreadingMode; import lombok.extern.slf4j.Slf4j; import org.agrona.CloseHelper; import org.agrona.concurrent.BusySpinIdleStrategy; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.aeron.ipc.*; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import javax.annotation.concurrent.NotThreadSafe; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import static org.junit.jupiter.api.Assertions.assertEquals; - @Slf4j -@NotThreadSafe -@Disabled("Tests are too flaky") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(120) public class AeronNDArrayResponseTest extends BaseND4JTest { private MediaDriver mediaDriver; - @Override public long getTimeoutMilliseconds() { return 180000L; } @BeforeEach public void before() { - if(isIntegrationTests()) { + final MediaDriver.Context ctx = new MediaDriver.Context().threadingMode(ThreadingMode.SHARED).dirDeleteOnShutdown(true) .dirDeleteOnStart(true) @@ -69,13 +59,12 @@ public class AeronNDArrayResponseTest extends BaseND4JTest { mediaDriver = MediaDriver.launchEmbedded(ctx); System.out.println("Using media driver directory " + mediaDriver.aeronDirectoryName()); System.out.println("Launched media driver"); - } + } @Test public void testResponse() throws Exception { - skipUnlessIntegrationTests(); //Long-running test - don't run as part of unit tests by default int streamId = 10; int responderStreamId = 11; @@ -184,7 +173,7 @@ public class AeronNDArrayResponseTest extends BaseND4JTest { - assertEquals(expectedResponses, count.get()); + Assertions.assertEquals(expectedResponses, count.get()); System.out.println("After"); diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/resources/aeron.properties b/cavis-nd4j/cavis-nd4j-aeron/src/test/resources/aeron.properties similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/resources/aeron.properties rename to cavis-nd4j/cavis-nd4j-aeron/src/test/resources/aeron.properties diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/resources/log4j.properties b/cavis-nd4j/cavis-nd4j-aeron/src/test/resources/log4j.properties similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/resources/log4j.properties rename to cavis-nd4j/cavis-nd4j-aeron/src/test/resources/log4j.properties diff --git a/nd4j/nd4j-serde/nd4j-aeron/src/test/resources/logback.xml b/cavis-nd4j/cavis-nd4j-aeron/src/test/resources/logback.xml similarity index 100% rename from nd4j/nd4j-serde/nd4j-aeron/src/test/resources/logback.xml rename to cavis-nd4j/cavis-nd4j-aeron/src/test/resources/logback.xml diff --git a/cavis-nd4j/cavis-nd4j-common-tests/build.gradle b/cavis-nd4j/cavis-nd4j-common-tests/build.gradle new file mode 100644 index 000000000..1f3f41d50 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common-tests/build.gradle @@ -0,0 +1,22 @@ +plugins { + id 'java-library' + id 'maven-publish' +} + +dependencies { + + implementation 'org.junit.jupiter:junit-jupiter-api' + implementation 'org.junit.jupiter:junit-jupiter-engine' + + + implementation project(":cavis-dnn:cavis-dnn-api") + implementation project(":cavis-dnn:cavis-dnn-common") + + implementation ("org.reflections:reflections") { + exclude group: 'com.google.code.findbugs', module: '*' + } + + implementation 'org.springframework:spring-core' + implementation "org.bytedeco:javacpp" + implementation 'org.slf4j:slf4j-api' +} diff --git a/nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/AbstractAssertTestsClass.java b/cavis-nd4j/cavis-nd4j-common-tests/src/main/java/org/nd4j/common/tests/AbstractAssertTestsClass.java similarity index 98% rename from nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/AbstractAssertTestsClass.java rename to cavis-nd4j/cavis-nd4j-common-tests/src/main/java/org/nd4j/common/tests/AbstractAssertTestsClass.java index ff5251175..5c654b0b2 100644 --- a/nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/AbstractAssertTestsClass.java +++ b/cavis-nd4j/cavis-nd4j-common-tests/src/main/java/org/nd4j/common/tests/AbstractAssertTestsClass.java @@ -20,6 +20,7 @@ package org.nd4j.common.tests; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; import org.reflections.Reflections; import org.reflections.scanners.MethodAnnotationsScanner; import org.reflections.util.ClasspathHelper; @@ -27,8 +28,6 @@ import org.reflections.util.ConfigurationBuilder; import java.lang.reflect.Method; import java.util.*; -import org.junit.jupiter.api.Test; - @Slf4j public abstract class AbstractAssertTestsClass extends BaseND4JTest { @@ -45,7 +44,7 @@ public abstract class AbstractAssertTestsClass extends BaseND4JTest { } @Test - public void checkTestClasses() { + public void checkTestClasses(){ Reflections reflections = new Reflections(new ConfigurationBuilder() .setUrls(ClasspathHelper.forPackage(getPackageName())) .setScanners(new MethodAnnotationsScanner())); diff --git a/nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/BaseND4JTest.java b/cavis-nd4j/cavis-nd4j-common-tests/src/main/java/org/nd4j/common/tests/BaseND4JTest.java similarity index 81% rename from nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/BaseND4JTest.java rename to cavis-nd4j/cavis-nd4j-common-tests/src/main/java/org/nd4j/common/tests/BaseND4JTest.java index f6f620ead..937f46a9d 100644 --- a/nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/BaseND4JTest.java +++ b/cavis-nd4j/cavis-nd4j-common-tests/src/main/java/org/nd4j/common/tests/BaseND4JTest.java @@ -20,12 +20,12 @@ package org.nd4j.common.tests; -import ch.qos.logback.classic.LoggerContext; + +import lombok.extern.log4j.Log4j2; import lombok.extern.slf4j.Slf4j; import org.bytedeco.javacpp.Pointer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; import org.nd4j.common.base.Preconditions; import org.nd4j.common.config.ND4JSystemProperties; import org.nd4j.linalg.api.buffer.DataType; @@ -33,17 +33,12 @@ import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.api.ops.executioner.OpExecutioner; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.profiler.ProfilerConfig; -import org.slf4j.ILoggerFactory; -import org.slf4j.LoggerFactory; import java.lang.management.ManagementFactory; import java.util.List; import java.util.Map; import java.util.Properties; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - - @Slf4j public abstract class BaseND4JTest { @@ -55,7 +50,7 @@ public abstract class BaseND4JTest { * Override this method to set the default timeout for methods in the test class */ public long getTimeoutMilliseconds(){ - return 180_000; + return 90_000; } /** @@ -95,7 +90,7 @@ public abstract class BaseND4JTest { /** * @return True if integration tests maven profile is enabled, false otherwise. */ - public boolean isIntegrationTests() { + public boolean isIntegrationTests(){ if(integrationTest == null){ String prop = System.getenv("DL4J_INTEGRATION_TESTS"); integrationTest = Boolean.parseBoolean(prop); @@ -103,23 +98,14 @@ public abstract class BaseND4JTest { return integrationTest; } - /** - * Call this as the first line of a test in order to skip that test, only when the integration tests maven profile is not enabled. - * This can be used to dynamically skip integration tests when the integration test profile is not enabled. - * Note that the integration test profile is not enabled by default - "integration-tests" profile - */ - public void skipUnlessIntegrationTests() { - assumeTrue( isIntegrationTests(),"Skipping integration test - integration profile is not enabled"); - } - @BeforeEach - public void beforeTest(TestInfo testInfo) { - log.info("{}.{}", getClass().getSimpleName(), testInfo.getTestMethod().get().getName()); + public void beforeTest(){ //Suppress ND4J initialization - don't need this logged for every test... System.setProperty(ND4JSystemProperties.LOG_INITIALIZATION, "false"); System.setProperty(ND4JSystemProperties.ND4J_IGNORE_AVX, "true"); - Nd4j.getExecutioner().setProfilingMode(getProfilingMode()); - Nd4j.getExecutioner().setProfilingConfig(ProfilerConfig.builder().build()); + + //Nd4j.getExecutioner().setProfilingMode(getProfilingMode()); + Nd4j.setDefaultDataTypes(getDataType(), getDefaultFPDataType()); Nd4j.getExecutioner().setProfilingConfig(ProfilerConfig.builder().build()); Nd4j.getExecutioner().enableDebugMode(false); @@ -134,7 +120,7 @@ public abstract class BaseND4JTest { } @AfterEach - public void afterTest(TestInfo testInfo) { + public void afterTest(){ //Attempt to keep workspaces isolated between tests Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); MemoryWorkspace currWS = Nd4j.getMemoryManager().getCurrentWorkspace(); @@ -145,12 +131,7 @@ public abstract class BaseND4JTest { log.error("Open workspace leaked from test! Exiting - {}, isOpen = {} - {}", currWS.getId(), currWS.isScopeActive(), currWS); System.out.println("Open workspace leaked from test! Exiting - " + currWS.getId() + ", isOpen = " + currWS.isScopeActive() + " - " + currWS); System.out.flush(); - //Try to flush logs also: - try{ Thread.sleep(1000); } catch (InterruptedException e){ } - ILoggerFactory lf = LoggerFactory.getILoggerFactory(); - if( lf instanceof LoggerContext){ - ((LoggerContext)lf).stop(); - } + try{ Thread.sleep(1000); } catch (InterruptedException e){ } System.exit(1); } @@ -167,8 +148,7 @@ public abstract class BaseND4JTest { int threadsAfter = ManagementFactory.getThreadMXBean().getThreadCount(); long duration = System.currentTimeMillis() - startTime; - sb.append(getClass().getSimpleName()).append(".").append( testInfo.getTestMethod().get().getName()) - .append(": ").append(duration).append(" ms") + sb .append(": ").append(duration).append(" ms") .append(", threadCount: (").append(threadCountBefore).append("->").append(threadsAfter).append(")") .append(", jvmTotal=").append(jvmTotal) .append(", jvmMax=").append(jvmMax) diff --git a/nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/ResourceUtils.java b/cavis-nd4j/cavis-nd4j-common-tests/src/main/java/org/nd4j/common/tests/ResourceUtils.java similarity index 100% rename from nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/ResourceUtils.java rename to cavis-nd4j/cavis-nd4j-common-tests/src/main/java/org/nd4j/common/tests/ResourceUtils.java diff --git a/cavis-nd4j/cavis-nd4j-common/build.gradle b/cavis-nd4j/cavis-nd4j-common/build.gradle new file mode 100644 index 000000000..e801f3b5c --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/build.gradle @@ -0,0 +1,32 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +dependencies { + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "com.google.guava:guava" + implementation 'org.slf4j:slf4j-api' + implementation "commons-io:commons-io" + implementation "org.apache.commons:commons-math3" + implementation "org.apache.commons:commons-lang3" + implementation "org.apache.commons:commons-compress" + implementation "commons-codec:commons-codec" + testImplementation projects.cavisNd4j.cavisNd4jCommonTests +} \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/base/Preconditions.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/base/Preconditions.java new file mode 100644 index 000000000..c8bc3966d --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/base/Preconditions.java @@ -0,0 +1,750 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.base; + +import org.nd4j.common.config.ND4JClassLoading; + +import java.util.*; + +public final class Preconditions { + private static final Map FORMATTERS = new HashMap<>(); + static { + ServiceLoader sl = ND4JClassLoading.loadService(PreconditionsFormat.class); + for (PreconditionsFormat pf : sl) { + List formatTags = pf.formatTags(); + for(String s : formatTags){ + FORMATTERS.put(s, pf); + } + } + } + + private Preconditions() { + } + + /** + * Check the specified boolean argument. Throws an IllegalArgumentException if {@code b} is false + * + * @param b Argument to check + */ + public static void checkArgument(boolean b) { + if (!b) { + throw new IllegalArgumentException(); + } + } + + /** + * Check the specified boolean argument. Throws an IllegalArgumentException with the specified message if {@code b} is false + * + * @param b Argument to check + * @param message Message for exception. May be null + */ + public static void checkArgument(boolean b, String message) { + if (!b) { + throwEx(message); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, int arg1) { + if (!b) { + throwEx(msg, arg1); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, long arg1) { + if (!b) { + throwEx(msg, arg1); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, double arg1) { + if (!b) { + throwEx(msg, arg1); + } + } + + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, Object arg1) { + if (!b) { + throwEx(msg, arg1); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, int arg1, int arg2) { + if (!b) { + throwEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, long arg1, long arg2) { + if (!b) { + throwEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, double arg1, double arg2) { + if (!b) { + throwEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, Object arg1, Object arg2) { + if (!b) { + throwEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, int arg1, int arg2, int arg3) { + if (!b) { + throwEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, long arg1, long arg2, long arg3) { + if (!b) { + throwEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, double arg1, double arg2, double arg3) { + if (!b) { + throwEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, Object arg1, Object arg2, Object arg3) { + if (!b) { + throwEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, int arg1, int arg2, int arg3, int arg4) { + if (!b) { + throwEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, long arg1, long arg2, long arg3, long arg4) { + if (!b) { + throwEx(msg, arg1, arg2, arg3, arg4); + } + } + + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, double arg1, double arg2, double arg3, double arg4) { + if (!b) { + throwEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, Object arg1, Object arg2, Object arg3, Object arg4) { + if (!b) { + throwEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, Object arg1, Object arg2, Object arg3, Object arg4, Object arg5) { + if (!b) { + throwEx(msg, arg1, arg2, arg3, arg4, arg5); + } + } + + /** + * See {@link #checkArgument(boolean, String, Object...)} + */ + public static void checkArgument(boolean b, String msg, Object arg1, Object arg2, Object arg3, Object arg4, Object arg5, Object arg6) { + if (!b) { + throwEx(msg, arg1, arg2, arg3, arg4, arg5, arg6); + } + } + + /** + * Check the specified boolean argument. Throws an IllegalArgumentException with the specified message if {@code b} is false. + * Note that the message may specify argument locations using "%s" - for example, + * {@code checkArgument(false, "Got %s values, expected %s", 3, "more"} would throw an IllegalArgumentException + * with the message "Got 3 values, expected more" + * + * @param b Argument to check + * @param message Message for exception. May be null. + * @param args Arguments to place in message + */ + public static void checkArgument(boolean b, String message, Object... args) { + if (!b) { + throwEx(message, args); + } + } + + + /** + * Check the specified boolean argument. Throws an IllegalStateException if {@code b} is false + * + * @param b State to check + */ + public static void checkState(boolean b) { + if (!b) { + throw new IllegalStateException(); + } + } + + /** + * Check the specified boolean argument. Throws an IllegalStateException with the specified message if {@code b} is false + * + * @param b State to check + * @param message Message for exception. May be null + */ + public static void checkState(boolean b, String message) { + if (!b) { + throwStateEx(message); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, int arg1) { + if (!b) { + throwStateEx(msg, arg1); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, long arg1) { + if (!b) { + throwStateEx(msg, arg1); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, double arg1) { + if (!b) { + throwStateEx(msg, arg1); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, Object arg1) { + if (!b) { + throwStateEx(msg, arg1); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, int arg1, int arg2) { + if (!b) { + throwStateEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, long arg1, long arg2) { + if (!b) { + throwStateEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, double arg1, double arg2) { + if (!b) { + throwStateEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, Object arg1, Object arg2) { + if (!b) { + throwStateEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, int arg1, int arg2, int arg3) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, long arg1, long arg2, long arg3) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, double arg1, double arg2, double arg3) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, Object arg1, Object arg2, Object arg3) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, int arg1, int arg2, int arg3, int arg4) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, long arg1, long arg2, long arg3, long arg4) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, double arg1, double arg2, double arg3, double arg4) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, Object arg1, Object arg2, Object arg3, Object arg4) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, Object arg1, Object arg2, Object arg3, Object arg4, Object arg5) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3, arg4, arg5); + } + } + + /** + * See {@link #checkState(boolean, String, Object...)} + */ + public static void checkState(boolean b, String msg, Object arg1, Object arg2, Object arg3, Object arg4, Object arg5, Object arg6) { + if (!b) { + throwStateEx(msg, arg1, arg2, arg3, arg4, arg5, arg6); + } + } + + /** + * Check the specified boolean argument. Throws an IllegalStateException with the specified message if {@code b} is false. + * Note that the message may specify argument locations using "%s" - for example, + * {@code checkArgument(false, "Got %s values, expected %s", 3, "more"} would throw an IllegalStateException + * with the message "Got 3 values, expected more" + * + * @param b Argument to check + * @param message Message for exception. May be null. + * @param args Arguments to place in message + */ + public static void checkState(boolean b, String message, Object... args) { + if (!b) { + throwStateEx(message, args); + } + } + + + /** + * Check the specified boolean argument. Throws an NullPointerException if {@code o} is false + * + * @param o Object to check + */ + public static void checkNotNull(Object o) { + if (o == null) { + throw new NullPointerException(); + } + } + + /** + * Check the specified boolean argument. Throws an NullPointerException with the specified message if {@code o} is false + * + * @param o Object to check + * @param message Message for exception. May be null + */ + public static void checkNotNull(Object o, String message) { + if (o == null) { + throwNullPointerEx(message); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, int arg1) { + if (o == null) { + throwNullPointerEx(msg, arg1); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, long arg1) { + if (o == null) { + throwNullPointerEx(msg, arg1); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, double arg1) { + if (o == null) { + throwNullPointerEx(msg, arg1); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, Object arg1) { + if (o == null) { + throwNullPointerEx(msg, arg1); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, int arg1, int arg2) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, long arg1, long arg2) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, double arg1, double arg2) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, Object arg1, Object arg2) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, int arg1, int arg2, int arg3) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, long arg1, long arg2, long arg3) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, double arg1, double arg2, double arg3) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, Object arg1, Object arg2, Object arg3) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2, arg3); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, int arg1, int arg2, int arg3, int arg4) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, long arg1, long arg2, long arg3, long arg4) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, double arg1, double arg2, double arg3, double arg4) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * See {@link #checkNotNull(Object, String, Object...)} + */ + public static void checkNotNull(Object o, String msg, Object arg1, Object arg2, Object arg3, Object arg4) { + if (o == null) { + throwNullPointerEx(msg, arg1, arg2, arg3, arg4); + } + } + + /** + * Check the specified boolean argument. Throws an IllegalStateException with the specified message if {@code o} is false. + * Note that the message may specify argument locations using "%s" - for example, + * {@code checkArgument(false, "Got %s values, expected %s", 3, "more"} would throw an IllegalStateException + * with the message "Got 3 values, expected more" + * + * @param o Object to check + * @param message Message for exception. May be null. + * @param args Arguments to place in message + */ + public static void checkNotNull(Object o, String message, Object... args) { + if (o == null) { + throwStateEx(message, args); + } + } + + public static void throwEx(String message, Object... args) { + String f = format(message, args); + throw new IllegalArgumentException(f); + } + + public static void throwStateEx(String message, Object... args) { + String f = format(message, args); + throw new IllegalStateException(f); + } + + public static void throwNullPointerEx(String message, Object... args) { + String f = format(message, args); + throw new NullPointerException(f); + } + + private static String format(String message, Object... args) { + if (message == null) { + message = ""; + } + if (args == null) { + args = new Object[]{"null"}; + } + + StringBuilder sb = new StringBuilder(); + + int indexOfStart = 0; + boolean consumedMessageFully = false; + for (int i = 0; i < args.length; i++) { + //First: scan for next tag. This could be a %s, or it could be a custom loader for Preconditions class (PreconditionsFormat) + int nextIdx = message.indexOf("%s", indexOfStart); + + int nextCustom = -1; + String nextCustomTag = null; + for(String s : FORMATTERS.keySet()){ + int idxThis = message.indexOf(s, indexOfStart); + if(idxThis > 0 && (nextCustom < 0 || idxThis < nextCustom)){ + nextCustom = idxThis; + nextCustomTag = s; + } + } + + if (nextIdx < 0 && nextCustom < 0) { + //Malformed message: No more "%s" (or custom tags) to replace, but more message args + if (!consumedMessageFully) { + sb.append(message.substring(indexOfStart)); + consumedMessageFully = true; + sb.append(" ["); + while (i < args.length) { + sb.append(formatArg(args[i])); + if (i < args.length - 1) { + sb.append(","); + } + i++; + } + sb.append("]"); + } + } else { + if(nextCustom < 0 || (nextIdx > 0 && nextIdx < nextCustom)){ + //%s tag + sb.append(message.substring(indexOfStart, nextIdx)) + .append(formatArg(args[i])); + indexOfStart = nextIdx + 2; + } else { + //Custom tag + sb.append(message.substring(indexOfStart, nextCustom)); + String s = FORMATTERS.get(nextCustomTag).format(nextCustomTag, args[i]); + sb.append(s); + indexOfStart = nextCustom + nextCustomTag.length(); + } + } + } + if (!consumedMessageFully) { + sb.append(message.substring(indexOfStart)); + } + + return sb.toString(); + } + + private static String formatArg(Object o){ + if(o == null){ + return "null"; + } + if(o.getClass().isArray()){ + return formatArray(o); + } + return o.toString(); + } + + public static String formatArray(Object o){ + if(o == null) + return "null"; + + if(o.getClass().getComponentType().isPrimitive()){ + if(o instanceof byte[]) { + return Arrays.toString((byte[])o); + } else if(o instanceof int[]){ + return Arrays.toString((int[])o); + } else if(o instanceof long[]){ + return Arrays.toString((long[])o); + } else if(o instanceof float[]){ + return Arrays.toString((float[])o); + } else if(o instanceof double[]){ + return Arrays.toString((double[])o); + } else if(o instanceof char[]){ + return Arrays.toString((char[])o); + } else if(o instanceof boolean[]) { + return Arrays.toString((boolean[])o); + } else if(o instanceof short[]){ + return Arrays.toString((short[])o); + } else { + //Should never happen + return o.toString(); + } + } else { + Object[] arr = (Object[])o; + return Arrays.toString(arr); + } + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/base/PreconditionsFormat.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/base/PreconditionsFormat.java new file mode 100644 index 000000000..8d598d546 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/base/PreconditionsFormat.java @@ -0,0 +1,31 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.base; + +import java.util.List; + +public interface PreconditionsFormat { + + List formatTags(); + + String format(String tag, Object arg); + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/CompactHeapStringList.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/CompactHeapStringList.java new file mode 100644 index 000000000..b7a25f248 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/CompactHeapStringList.java @@ -0,0 +1,349 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.collection; + +import java.util.*; + +public class CompactHeapStringList implements List { + public static final int DEFAULT_REALLOCATION_BLOCK_SIZE_BYTES = 8 * 1024 * 1024; //8MB + public static final int DEFAULT_INTEGER_REALLOCATION_BLOCK_SIZE_BYTES = 1024 * 1024; //1MB - 262144 ints, 131k entries + + private final int reallocationBlockSizeBytes; + private final int reallocationIntegerBlockSizeBytes; + private int usedCount = 0; + private int nextDataOffset = 0; + private char[] data; + private int[] offsetAndLength; + + public CompactHeapStringList() { + this(DEFAULT_REALLOCATION_BLOCK_SIZE_BYTES, DEFAULT_INTEGER_REALLOCATION_BLOCK_SIZE_BYTES); + } + + /** + * + * @param reallocationBlockSizeBytes Number of bytes by which to increase the char[], when allocating a new storage array + * @param intReallocationBlockSizeBytes Number of bytes by which to increase the int[], when allocating a new storage array + */ + public CompactHeapStringList(int reallocationBlockSizeBytes, int intReallocationBlockSizeBytes) { + this.reallocationBlockSizeBytes = reallocationBlockSizeBytes; + this.reallocationIntegerBlockSizeBytes = intReallocationBlockSizeBytes; + + this.data = new char[this.reallocationBlockSizeBytes / 2]; + this.offsetAndLength = new int[this.reallocationIntegerBlockSizeBytes / 4]; + } + + @Override + public int size() { + return usedCount; + } + + @Override + public boolean isEmpty() { + return usedCount == 0; + } + + @Override + public boolean contains(Object o) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public Iterator iterator() { + return new CompactHeapStringListIterator(); + } + + @Override + public String[] toArray() { + String[] str = new String[usedCount]; + for (int i = 0; i < usedCount; i++) { + str[i] = get(i); + } + return str; + } + + @Override + public T[] toArray(T[] a) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public boolean add(String s) { + int length = s.length(); + //3 possibilities: + //(a) doesn't fit in char[] + //(b) doesn't fit in int[] + //(c) fits OK in both + + if (nextDataOffset + length > data.length) { + //Allocate new data array, if possible + if (nextDataOffset > Integer.MAX_VALUE - length) { + throw new UnsupportedOperationException( + "Cannot allocate new data char[]: required array size exceeds Integer.MAX_VALUE"); + } + int toAdd = Math.max(reallocationBlockSizeBytes / 2, length); + int newLength = data.length + Math.min(toAdd, Integer.MAX_VALUE - data.length); + data = Arrays.copyOf(data, newLength); + } + if (2 * (usedCount + 1) >= offsetAndLength.length) { + if (offsetAndLength.length >= Integer.MAX_VALUE - 2) { + //Should normally never happen + throw new UnsupportedOperationException( + "Cannot allocate new offset int[]: required array size exceeds Integer.MAX_VALUE"); + } + int newLength = offsetAndLength.length + Math.min(reallocationIntegerBlockSizeBytes / 4, + Integer.MAX_VALUE - offsetAndLength.length); + offsetAndLength = Arrays.copyOf(offsetAndLength, newLength); + } + + + s.getChars(0, length, data, nextDataOffset); + offsetAndLength[2 * usedCount] = nextDataOffset; + offsetAndLength[2 * usedCount + 1] = length; + nextDataOffset += length; + usedCount++; + + return true; + } + + @Override + public boolean remove(Object o) { + //In principle we *could* do this with array copies + throw new UnsupportedOperationException("Remove not supported"); + } + + @Override + public boolean containsAll(Collection c) { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public boolean addAll(Collection c) { + for (String s : c) { + add(s); + } + return c.size() > 0; + } + + @Override + public boolean addAll(int index, Collection c) { + //This is conceivably possible with array copies and adjusting the indices + throw new UnsupportedOperationException("Add all at specified index: Not supported"); + } + + @Override + public boolean removeAll(Collection c) { + throw new UnsupportedOperationException("Remove all: Not supported"); + } + + @Override + public boolean retainAll(Collection c) { + throw new UnsupportedOperationException("Retain all: Not supported"); + } + + @Override + public void clear() { + usedCount = 0; + nextDataOffset = 0; + data = new char[reallocationBlockSizeBytes / 2]; + offsetAndLength = new int[reallocationIntegerBlockSizeBytes / 4]; + } + + @Override + public String get(int index) { + if (index >= usedCount) { + throw new IllegalArgumentException("Invalid index: " + index + " >= size(). Size = " + usedCount); + } + int offset = offsetAndLength[2 * index]; + int length = offsetAndLength[2 * index + 1]; + return new String(data, offset, length); + } + + @Override + public String set(int index, String element) { + //This *could* be done with array copy ops... + throw new UnsupportedOperationException( + "Set specified index: not supported due to serialized storage structure"); + } + + @Override + public void add(int index, String element) { + //This *could* be done with array copy ops... + throw new UnsupportedOperationException( + "Set specified index: not supported due to serialized storage structure"); + } + + @Override + public String remove(int index) { + throw new UnsupportedOperationException("Remove: not supported"); + } + + @Override + public int indexOf(Object o) { + if (!(o instanceof String)) { + return -1; + } + + String str = (String) o; + char[] ch = str.toCharArray(); + + + for (int i = 0; i < usedCount; i++) { + if (offsetAndLength[2 * i + 1] != ch.length) { + //Can't be this one: lengths differ + continue; + } + int offset = offsetAndLength[2 * i]; + + boolean matches = true; + for (int j = 0; j < ch.length; j++) { + if (data[offset + j] != ch[j]) { + matches = false; + break; + } + } + if (matches) { + return i; + } + } + + return -1; + } + + @Override + public int lastIndexOf(Object o) { + if (!(o instanceof String)) { + return -1; + } + + String str = (String) o; + char[] ch = str.toCharArray(); + + + for (int i = usedCount - 1; i >= 0; i--) { + if (offsetAndLength[2 * i + 1] != ch.length) { + //Can't be this one: lengths differ + continue; + } + int offset = offsetAndLength[2 * i]; + + boolean matches = true; + for (int j = 0; j < ch.length; j++) { + if (data[offset + j] != ch[j]) { + matches = false; + break; + } + } + if (matches) { + return i; + } + } + + return -1; + } + + @Override + public ListIterator listIterator() { + return new CompactHeapStringListIterator(); + } + + @Override + public ListIterator listIterator(int index) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public List subList(int fromIndex, int toIndex) { + throw new UnsupportedOperationException("Not supported"); + } + + @Override + public boolean equals(Object o) { + if (o == this) + return true; + if (!(o instanceof List)) + return false; + + ListIterator e1 = listIterator(); + ListIterator e2 = ((List) o).listIterator(); + while (e1.hasNext() && e2.hasNext()) { + String o1 = e1.next(); + Object o2 = e2.next(); + if (!(o1 == null ? o2 == null : o1.equals(o2))) + return false; + } + return !(e1.hasNext() || e2.hasNext()); + } + + private class CompactHeapStringListIterator implements Iterator, ListIterator { + private int currIdx = 0; + + @Override + public boolean hasNext() { + return currIdx < usedCount; + } + + @Override + public String next() { + if (!hasNext()) { + throw new NoSuchElementException("No next element"); + } + return get(currIdx++); + } + + @Override + public boolean hasPrevious() { + return currIdx > 0; + } + + @Override + public String previous() { + if (!hasPrevious()) { + throw new NoSuchElementException(); + } + return get(currIdx--); + } + + @Override + public int nextIndex() { + return currIdx; + } + + @Override + public int previousIndex() { + return currIdx; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void set(String s) { + throw new UnsupportedOperationException(); + } + + @Override + public void add(String s) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/IntArrayKeyMap.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/IntArrayKeyMap.java new file mode 100644 index 000000000..2ed1b154d --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/IntArrayKeyMap.java @@ -0,0 +1,159 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.collection; + +import lombok.Getter; +import org.nd4j.common.base.Preconditions; +import com.google.common.primitives.Ints; + +import java.util.*; + +public class IntArrayKeyMap implements Map { + + private Map map = new LinkedHashMap<>(); + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean containsKey(Object o) { + return map.containsKey(new IntArray((int[]) o)); + } + + @Override + public boolean containsValue(Object o) { + return map.containsValue(new IntArray((int[]) o)); + } + + @Override + public V get(Object o) { + return map.get(new IntArray((int[]) o)); + } + + @Override + public V put(int[] ints, V v) { + return map.put(new IntArray(ints),v); + } + + @Override + public V remove(Object o) { + return map.remove(new IntArray((int[]) o)); + } + + @Override + public void putAll(Map map) { + for(Entry entry : map.entrySet()) { + this.map.put(new IntArray(entry.getKey()),entry.getValue()); + } + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public Set keySet() { + Set intArrays = map.keySet(); + Set ret = new LinkedHashSet<>(); + for(IntArray intArray : intArrays) + ret.add(intArray.backingArray); + return ret; + } + + @Override + public Collection values() { + return map.values(); + } + + @Override + public Set> entrySet() { + Set> intArrays = map.entrySet(); + Set> ret = new LinkedHashSet<>(); + for(Map.Entry intArray : intArrays) { + final Map.Entry intArray2 = intArray; + ret.add(new Map.Entry() { + @Override + public int[] getKey() { + return intArray2.getKey().backingArray; + } + + @Override + public V getValue() { + return intArray2.getValue(); + } + + @Override + public V setValue(V v) { + return intArray2.setValue(v); + } + }); + } + return ret; + } + + + public static class IntArray implements Comparable { + @Getter + private int[] backingArray; + + public IntArray(int[] backingArray) { + Preconditions.checkNotNull(backingArray,"Backing array must not be null!"); + this.backingArray = Ints.toArray(new LinkedHashSet<>(Ints.asList(backingArray))); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IntArray intArray = (IntArray) o; + + return Arrays.equals(intArray.backingArray,backingArray); + } + + @Override + public int hashCode() { + return Arrays.hashCode(backingArray); + } + + @Override + public int compareTo(IntArray intArray) { + if(this.backingArray.length == 0 || intArray.backingArray.length == 0) { + return 1; + } + + else if(Arrays.equals(backingArray,intArray.backingArray)) + return 1; + + return Ints.compare(Ints.max(backingArray),Ints.max(intArray.backingArray)); + } + } + + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/IntArrayKeySet.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/IntArrayKeySet.java new file mode 100644 index 000000000..1a8893cda --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/IntArrayKeySet.java @@ -0,0 +1,113 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.collection; + +import java.util.*; + +public class IntArrayKeySet implements Set { + private Set set = new LinkedHashSet<>(); + @Override + public int size() { + return set.size(); + } + + @Override + public boolean isEmpty() { + return set.isEmpty(); + } + + @Override + public boolean contains(Object o) { + return set.contains(new IntArrayKeyMap.IntArray((int[]) o)); + } + + @Override + public Iterator iterator() { + List ret = new ArrayList<>(); + for(IntArrayKeyMap.IntArray arr : set) { + ret.add(arr.getBackingArray()); + } + + return ret.iterator(); + } + + @Override + public Object[] toArray() { + Object[] ret = new Object[size()]; + int count = 0; + for(IntArrayKeyMap.IntArray intArray : set) { + ret[count++] = intArray.getBackingArray(); + } + + return ret; + } + + @Override + public T[] toArray(T[] ts) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean add(int[] ints) { + return set.add(new IntArrayKeyMap.IntArray(ints)); + } + + @Override + public boolean remove(Object o) { + return set.remove(new IntArrayKeyMap.IntArray((int[]) o)); + } + + @Override + public boolean containsAll(Collection collection) { + return set.containsAll(getCollection(collection)); + + } + + @Override + public boolean addAll(Collection collection) { + return set.addAll(getCollection(collection)); + } + + @Override + public boolean retainAll(Collection collection) { + return set.retainAll(getCollection(collection)); + } + + @Override + public boolean removeAll(Collection collection) { + return set.removeAll(getCollection(collection)); + } + + @Override + public void clear() { + set.clear(); + } + + private Collection getCollection(Collection coll) { + List ret = new ArrayList<>(); + Collection casted = (Collection) coll; + for(int[] arr : casted) { + ret.add(new IntArrayKeyMap.IntArray(arr)); + } + return ret; + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalMap.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalMap.java new file mode 100644 index 000000000..a88871152 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalMap.java @@ -0,0 +1,460 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.collection; + +import org.nd4j.common.primitives.Pair; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Multiple key map + */ +public class MultiDimensionalMap implements Serializable { + + private Map, V> backedMap; + + /** + * Thread safe sorted map implementation + * @param + * @param + * @param + * @return + */ + public static MultiDimensionalMap newThreadSafeTreeBackedMap() { + return new MultiDimensionalMap<>(new ConcurrentSkipListMap, V>()); + } + + /** + * Thread safe hash map implementation + * @param + * @param + * @param + * @return + */ + public static MultiDimensionalMap newThreadSafeHashBackedMap() { + return new MultiDimensionalMap<>(new ConcurrentHashMap, V>()); + } + + /** + * Thread safe hash map impl + * @param + * @param + * @param + * @return + */ + public static MultiDimensionalMap newHashBackedMap() { + return new MultiDimensionalMap<>(new HashMap, V>()); + } + + /** + * Tree map implementation + * @param + * @param + * @param + * @return + */ + public static MultiDimensionalMap newTreeBackedMap() { + return new MultiDimensionalMap<>(new TreeMap, V>()); + } + + public MultiDimensionalMap(Map, V> backedMap) { + this.backedMap = backedMap; + } + + protected MultiDimensionalMap(){ } + + /** + * Returns the number of key-value mappings in this map. If the + * map contains more than Integer.MAX_VALUE elements, returns + * Integer.MAX_VALUE. + * + * @return the number of key-value mappings in this map + */ + public int size() { + return backedMap.size(); + } + + /** + * Returns true if this map contains no key-value mappings. + * + * @return true if this map contains no key-value mappings + */ + public boolean isEmpty() { + return backedMap.isEmpty(); + } + + /** + * Returns true if this map contains a mapping for the specified + * key. More formally, returns true if and only if + * this map contains a mapping for a key k such that + * (key==null ? k==null : key.equals(k)). (There can be + * at most one such mapping.) + * + * @param key key whose presence in this map is to be tested + * @return true if this map contains a mapping for the specified + * key + * @throws ClassCastException if the key is of an inappropriate type for + * this map + * (optional) + * @throws NullPointerException if the specified key is null and this map + * does not permit null keys + * (optional) + */ + + public boolean containsKey(Object key) { + return backedMap.containsKey(key); + } + + /** + * Returns true if this map maps one or more keys to the + * specified value. More formally, returns true if and only if + * this map contains at least one mapping to a value v such that + * (value==null ? v==null : value.equals(v)). This operation + * will probably require time linear in the map size for most + * implementations of the Map interface. + * + * @param value value whose presence in this map is to be tested + * @return true if this map maps one or more keys to the + * specified value + * @throws ClassCastException if the value is of an inappropriate type for + * this map + * (optional) + * @throws NullPointerException if the specified value is null and this + * map does not permit null values + * (optional) + */ + + public boolean containsValue(Object value) { + return backedMap.containsValue(value); + } + + /** + * Returns the value to which the specified key is mapped, + * or {@code null} if this map contains no mapping for the key. + *

+ *

More formally, if this map contains a mapping from a key + * {@code k} to a value {@code v} such that {@code (key==null ? k==null : + * key.equals(k))}, then this method returns {@code v}; otherwise + * it returns {@code null}. (There can be at most one such mapping.) + *

+ *

If this map permits null values, then a return value of + * {@code null} does not necessarily indicate that the map + * contains no mapping for the key; it's also possible that the map + * explicitly maps the key to {@code null}. The {@link #containsKey + * containsKey} operation may be used to distinguish these two cases. + * + * @param key the key whose associated value is to be returned + * @return the value to which the specified key is mapped, or + * {@code null} if this map contains no mapping for the key + * @throws ClassCastException if the key is of an inappropriate type for + * this map + * (optional) + * @throws NullPointerException if the specified key is null and this map + * does not permit null keys + * (optional) + */ + + public V get(Object key) { + return backedMap.get(key); + } + + /** + * Associates the specified value with the specified key in this map + * (optional operation). If the map previously contained a mapping for + * the key, the old value is replaced by the specified value. (A map + * m is said to contain a mapping for a key k if and only + * if {@link #containsKey(Object) m.containsKey(k)} would return + * true.) + * + * @param key key with which the specified value is to be associated + * @param value value to be associated with the specified key + * @return the previous value associated with key, or + * null if there was no mapping for key. + * (A null return can also indicate that the map + * previously associated null with key, + * if the implementation supports null values.) + * @throws UnsupportedOperationException if the put operation + * is not supported by this map + * @throws ClassCastException if the class of the specified key or value + * prevents it from being stored in this map + * @throws NullPointerException if the specified key or value is null + * and this map does not permit null keys or values + * @throws IllegalArgumentException if some property of the specified key + * or value prevents it from being stored in this map + */ + + public V put(Pair key, V value) { + return backedMap.put(key, value); + } + + /** + * Removes the mapping for a key from this map if it is present + * (optional operation). More formally, if this map contains a mapping + * from key k to value v such that + * (key==null ? k==null : key.equals(k)), that mapping + * is removed. (The map can contain at most one such mapping.) + *

+ *

Returns the value to which this map previously associated the key, + * or null if the map contained no mapping for the key. + *

+ *

If this map permits null values, then a return value of + * null does not necessarily indicate that the map + * contained no mapping for the key; it's also possible that the map + * explicitly mapped the key to null. + *

+ *

The map will not contain a mapping for the specified key once the + * call returns. + * + * @param key key whose mapping is to be removed from the map + * @return the previous value associated with key, or + * null if there was no mapping for key. + * @throws UnsupportedOperationException if the remove operation + * is not supported by this map + * @throws ClassCastException if the key is of an inappropriate type for + * this map + * (optional) + * @throws NullPointerException if the specified key is null and this + * map does not permit null keys + * (optional) + */ + + public V remove(Object key) { + return backedMap.remove(key); + } + + /** + * Copies all of the mappings from the specified map to this map + * (optional operation). The effect of this call is equivalent to that + * of calling {@link Map<>#put(k, v)} on this map once + * for each mapping from key k to value v in the + * specified map. The behavior of this operation is undefined if the + * specified map is modified while the operation is in progress. + * + * @param m mappings to be stored in this map + * @throws UnsupportedOperationException if the putAll operation + * is not supported by this map + * @throws ClassCastException if the class of a key or value in the + * specified map prevents it from being stored in this map + * @throws NullPointerException if the specified map is null, or if + * this map does not permit null keys or values, and the + * specified map contains null keys or values + * @throws IllegalArgumentException if some property of a key or value in + * the specified map prevents it from being stored in this map + */ + + public void putAll(Map, ? extends V> m) { + backedMap.putAll(m); + } + + /** + * Removes all of the mappings from this map (optional operation). + * The map will be empty after this call returns. + * + * @throws UnsupportedOperationException if the clear operation + * is not supported by this map + */ + + public void clear() { + backedMap.clear(); + } + + /** + * Returns a {@link Set} view of the keys contained in this map. + * The applyTransformToDestination is backed by the map, so changes to the map are + * reflected in the applyTransformToDestination, and vice-versa. If the map is modified + * while an iteration over the applyTransformToDestination is in progress (except through + * the iterator's own remove operation), the results of + * the iteration are undefined. The applyTransformToDestination supports element removal, + * which removes the corresponding mapping from the map, via the + * Iterator.remove, Set.remove, + * removeAll, retainAll, and clear + * operations. It does not support the add or addAll + * operations. + * + * @return a applyTransformToDestination view of the keys contained in this map + */ + + public Set> keySet() { + return backedMap.keySet(); + } + + /** + * Returns a {@link Collection} view of the values contained in this map. + * The collection is backed by the map, so changes to the map are + * reflected in the collection, and vice-versa. If the map is + * modified while an iteration over the collection is in progress + * (except through the iterator's own remove operation), + * the results of the iteration are undefined. The collection + * supports element removal, which removes the corresponding + * mapping from the map, via the Iterator.remove, + * Collection.remove, removeAll, + * retainAll and clear operations. It does not + * support the add or addAll operations. + * + * @return a collection view of the values contained in this map + */ + + public Collection values() { + return backedMap.values(); + } + + /** + * Returns a {@link Set} view of the mappings contained in this map. + * The applyTransformToDestination is backed by the map, so changes to the map are + * reflected in the applyTransformToDestination, and vice-versa. If the map is modified + * while an iteration over the applyTransformToDestination is in progress (except through + * the iterator's own remove operation, or through the + * setValue operation on a map entry returned by the + * iterator) the results of the iteration are undefined. The applyTransformToDestination + * supports element removal, which removes the corresponding + * mapping from the map, via the Iterator.remove, + * Set.remove, removeAll, retainAll and + * clear operations. It does not support the + * add or addAll operations. + * + * @return a applyTransformToDestination view of the mappings contained in this map + */ + + public Set> entrySet() { + Set> ret = new HashSet<>(); + for (Pair pair : backedMap.keySet()) { + ret.add(new Entry<>(pair.getFirst(), pair.getSecond(), backedMap.get(pair))); + } + return ret; + } + + public V get(K k, T t) { + return get(new Pair<>(k, t)); + } + + public void put(K k, T t, V v) { + put(new Pair<>(k, t), v); + } + + + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof MultiDimensionalMap)) + return false; + + MultiDimensionalMap that = (MultiDimensionalMap) o; + + return !(backedMap != null ? !backedMap.equals(that.backedMap) : that.backedMap != null); + + } + + + public int hashCode() { + return backedMap != null ? backedMap.hashCode() : 0; + } + + + public String toString() { + return "MultiDimensionalMap{" + "backedMap=" + backedMap + '}'; + } + + + public boolean contains(K k, T t) { + return containsKey(new Pair<>(k, t)); + } + + + public static class Entry implements Map.Entry, V> { + + private K firstKey; + private T secondKey; + private V value; + + public Entry(K firstKey, T secondKey, V value) { + this.firstKey = firstKey; + this.secondKey = secondKey; + this.value = value; + } + + public K getFirstKey() { + return firstKey; + } + + public void setFirstKey(K firstKey) { + this.firstKey = firstKey; + } + + public T getSecondKey() { + return secondKey; + } + + public void setSecondKey(T secondKey) { + this.secondKey = secondKey; + } + + public V getValue() { + return value; + } + + /** + * Replaces the value corresponding to this entry with the specified + * value (optional operation). (Writes through to the map.) The + * behavior of this call is undefined if the mapping has already been + * removed from the map (by the iterator's remove operation). + * + * @param value new value to be stored in this entry + * @return old value corresponding to the entry + * @throws UnsupportedOperationException if the put operation + * is not supported by the backing map + * @throws ClassCastException if the class of the specified value + * prevents it from being stored in the backing map + * @throws NullPointerException if the backing map does not permit + * null values, and the specified value is null + * @throws IllegalArgumentException if some property of this value + * prevents it from being stored in the backing map + * @throws IllegalStateException implementations may, but are not + * required to, throw this exception if the entry has been + * removed from the backing map. + */ + + public V setValue(V value) { + V old = this.value; + this.value = value; + return old; + } + + + /** + * Returns the key corresponding to this entry. + * + * @return the key corresponding to this entry + * @throws IllegalStateException implementations may, but are not + * required to, throw this exception if the entry has been + * removed from the backing map. + */ + + public Pair getKey() { + return new Pair<>(firstKey, secondKey); + } + } + + + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalSet.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalSet.java new file mode 100644 index 000000000..c5712d3eb --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collection/MultiDimensionalSet.java @@ -0,0 +1,359 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.collection; + +import org.nd4j.common.primitives.Pair; + +import java.util.*; +import java.util.concurrent.ConcurrentSkipListSet; + +public class MultiDimensionalSet implements Set> { + + + private Set> backedSet; + + public MultiDimensionalSet(Set> backedSet) { + this.backedSet = backedSet; + } + + public static MultiDimensionalSet hashSet() { + return new MultiDimensionalSet<>(new HashSet>()); + } + + + public static MultiDimensionalSet treeSet() { + return new MultiDimensionalSet<>(new TreeSet>()); + } + + + + public static MultiDimensionalSet concurrentSkipListSet() { + return new MultiDimensionalSet<>(new ConcurrentSkipListSet>()); + } + + /** + * Returns the number of elements in this applyTransformToDestination (its cardinality). If this + * applyTransformToDestination contains more than Integer.MAX_VALUE elements, returns + * Integer.MAX_VALUE. + * + * @return the number of elements in this applyTransformToDestination (its cardinality) + */ + @Override + public int size() { + return backedSet.size(); + } + + /** + * Returns true if this applyTransformToDestination contains no elements. + * + * @return true if this applyTransformToDestination contains no elements + */ + @Override + public boolean isEmpty() { + return backedSet.isEmpty(); + } + + /** + * Returns true if this applyTransformToDestination contains the specified element. + * More formally, returns true if and only if this applyTransformToDestination + * contains an element e such that + * (o==null ? e==null : o.equals(e)). + * + * @param o element whose presence in this applyTransformToDestination is to be tested + * @return true if this applyTransformToDestination contains the specified element + * @throws ClassCastException if the type of the specified element + * is incompatible with this applyTransformToDestination + * (optional) + * @throws NullPointerException if the specified element is null and this + * applyTransformToDestination does not permit null elements + * (optional) + */ + @Override + public boolean contains(Object o) { + return backedSet.contains(o); + } + + /** + * Returns an iterator over the elements in this applyTransformToDestination. The elements are + * returned in no particular order (unless this applyTransformToDestination is an instance of some + * class that provides a guarantee). + * + * @return an iterator over the elements in this applyTransformToDestination + */ + @Override + public Iterator> iterator() { + return backedSet.iterator(); + } + + /** + * Returns an array containing all of the elements in this applyTransformToDestination. + * If this applyTransformToDestination makes any guarantees as to what order its elements + * are returned by its iterator, this method must return the + * elements in the same order. + *

+ *

The returned array will be "safe" in that no references to it + * are maintained by this applyTransformToDestination. (In other words, this method must + * allocate a new array even if this applyTransformToDestination is backed by an array). + * The caller is thus free to modify the returned array. + *

+ *

This method acts as bridge between array-based and collection-based + * APIs. + * + * @return an array containing all the elements in this applyTransformToDestination + */ + @Override + public Object[] toArray() { + return backedSet.toArray(); + } + + /** + * Returns an array containing all of the elements in this applyTransformToDestination; the + * runtime type of the returned array is that of the specified array. + * If the applyTransformToDestination fits in the specified array, it is returned therein. + * Otherwise, a new array is allocated with the runtime type of the + * specified array and the size of this applyTransformToDestination. + *

+ *

If this applyTransformToDestination fits in the specified array with room to spare + * (i.e., the array has more elements than this applyTransformToDestination), the element in + * the array immediately following the end of the applyTransformToDestination is applyTransformToDestination to + * null. (This is useful in determining the length of this + * applyTransformToDestination only if the caller knows that this applyTransformToDestination does not contain + * any null elements.) + *

+ *

If this applyTransformToDestination makes any guarantees as to what order its elements + * are returned by its iterator, this method must return the elements + * in the same order. + *

+ *

Like the {@link #toArray()} method, this method acts as bridge between + * array-based and collection-based APIs. Further, this method allows + * precise control over the runtime type of the output array, and may, + * under certain circumstances, be used to save allocation costs. + *

+ *

Suppose x is a applyTransformToDestination known to contain only strings. + * The following code can be used to dump the applyTransformToDestination into a newly allocated + * array of String: + *

+ * 

+     *     String[] y = x.toArray(new String[0]);
+ * + * Note that toArray(new Object[0]) is identical in function to + * toArray(). + * + * @param a the array into which the elements of this applyTransformToDestination are to be + * stored, if it is big enough; otherwise, a new array of the same + * runtime type is allocated for this purpose. + * @return an array containing all the elements in this applyTransformToDestination + * @throws ArrayStoreException if the runtime type of the specified array + * is not a supertype of the runtime type of every element in this + * applyTransformToDestination + * @throws NullPointerException if the specified array is null + */ + @Override + public T[] toArray(T[] a) { + return backedSet.toArray(a); + } + + /** + * Adds the specified element to this applyTransformToDestination if it is not already present + * (optional operation). More formally, adds the specified element + * e to this applyTransformToDestination if the applyTransformToDestination contains no element e2 + * such that + * (e==null ? e2==null : e.equals(e2)). + * If this applyTransformToDestination already contains the element, the call leaves the applyTransformToDestination + * unchanged and returns false. In combination with the + * restriction on constructors, this ensures that sets never contain + * duplicate elements. + *

+ *

The stipulation above does not imply that sets must accept all + * elements; sets may refuse to add any particular element, including + * null, and throw an exception, as described in the + * specification for {@link Collection#add Collection.add}. + * Individual applyTransformToDestination implementations should clearly document any + * restrictions on the elements that they may contain. + * + * @param kvPair element to be added to this applyTransformToDestination + * @return true if this applyTransformToDestination did not already contain the specified + * element + * @throws UnsupportedOperationException if the add operation + * is not supported by this applyTransformToDestination + * @throws ClassCastException if the class of the specified element + * prevents it from being added to this applyTransformToDestination + * @throws NullPointerException if the specified element is null and this + * applyTransformToDestination does not permit null elements + * @throws IllegalArgumentException if some property of the specified element + * prevents it from being added to this applyTransformToDestination + */ + @Override + public boolean add(Pair kvPair) { + return backedSet.add(kvPair); + } + + /** + * Removes the specified element from this applyTransformToDestination if it is present + * (optional operation). More formally, removes an element e + * such that + * (o==null ? e==null : o.equals(e)), if + * this applyTransformToDestination contains such an element. Returns true if this applyTransformToDestination + * contained the element (or equivalently, if this applyTransformToDestination changed as a + * result of the call). (This applyTransformToDestination will not contain the element once the + * call returns.) + * + * @param o object to be removed from this applyTransformToDestination, if present + * @return true if this applyTransformToDestination contained the specified element + * @throws ClassCastException if the type of the specified element + * is incompatible with this applyTransformToDestination + * (optional) + * @throws NullPointerException if the specified element is null and this + * applyTransformToDestination does not permit null elements + * (optional) + * @throws UnsupportedOperationException if the remove operation + * is not supported by this applyTransformToDestination + */ + @Override + public boolean remove(Object o) { + return backedSet.remove(o); + } + + /** + * Returns true if this applyTransformToDestination contains all of the elements of the + * specified collection. If the specified collection is also a applyTransformToDestination, this + * method returns true if it is a subset of this applyTransformToDestination. + * + * @param c collection to be checked for containment in this applyTransformToDestination + * @return true if this applyTransformToDestination contains all of the elements of the + * specified collection + * @throws ClassCastException if the types of one or more elements + * in the specified collection are incompatible with this + * applyTransformToDestination + * (optional) + * @throws NullPointerException if the specified collection contains one + * or more null elements and this applyTransformToDestination does not permit null + * elements + * (optional), + * or if the specified collection is null + * @see #contains(Object) + */ + @Override + public boolean containsAll(Collection c) { + return backedSet.containsAll(c); + } + + /** + * Adds all of the elements in the specified collection to this applyTransformToDestination if + * they're not already present (optional operation). If the specified + * collection is also a applyTransformToDestination, the addAll operation effectively + * modifies this applyTransformToDestination so that its value is the union of the two + * sets. The behavior of this operation is undefined if the specified + * collection is modified while the operation is in progress. + * + * @param c collection containing elements to be added to this applyTransformToDestination + * @return true if this applyTransformToDestination changed as a result of the call + * @throws UnsupportedOperationException if the addAll operation + * is not supported by this applyTransformToDestination + * @throws ClassCastException if the class of an element of the + * specified collection prevents it from being added to this applyTransformToDestination + * @throws NullPointerException if the specified collection contains one + * or more null elements and this applyTransformToDestination does not permit null + * elements, or if the specified collection is null + * @throws IllegalArgumentException if some property of an element of the + * specified collection prevents it from being added to this applyTransformToDestination + * @see #add(Object) + */ + @Override + public boolean addAll(Collection> c) { + return backedSet.addAll(c); + } + + /** + * Retains only the elements in this applyTransformToDestination that are contained in the + * specified collection (optional operation). In other words, removes + * from this applyTransformToDestination all of its elements that are not contained in the + * specified collection. If the specified collection is also a applyTransformToDestination, this + * operation effectively modifies this applyTransformToDestination so that its value is the + * intersection of the two sets. + * + * @param c collection containing elements to be retained in this applyTransformToDestination + * @return true if this applyTransformToDestination changed as a result of the call + * @throws UnsupportedOperationException if the retainAll operation + * is not supported by this applyTransformToDestination + * @throws ClassCastException if the class of an element of this applyTransformToDestination + * is incompatible with the specified collection + * (optional) + * @throws NullPointerException if this applyTransformToDestination contains a null element and the + * specified collection does not permit null elements + * (optional), + * or if the specified collection is null + * @see #remove(Object) + */ + @Override + public boolean retainAll(Collection c) { + return backedSet.retainAll(c); + } + + /** + * Removes from this applyTransformToDestination all of its elements that are contained in the + * specified collection (optional operation). If the specified + * collection is also a applyTransformToDestination, this operation effectively modifies this + * applyTransformToDestination so that its value is the asymmetric applyTransformToDestination difference of + * the two sets. + * + * @param c collection containing elements to be removed from this applyTransformToDestination + * @return true if this applyTransformToDestination changed as a result of the call + * @throws UnsupportedOperationException if the removeAll operation + * is not supported by this applyTransformToDestination + * @throws ClassCastException if the class of an element of this applyTransformToDestination + * is incompatible with the specified collection + * (optional) + * @throws NullPointerException if this applyTransformToDestination contains a null element and the + * specified collection does not permit null elements + * (optional), + * or if the specified collection is null + * @see #remove(Object) + * @see #contains(Object) + */ + @Override + public boolean removeAll(Collection c) { + return backedSet.removeAll(c); + } + + /** + * Removes all of the elements from this applyTransformToDestination (optional operation). + * The applyTransformToDestination will be empty after this call returns. + * + * @throws UnsupportedOperationException if the clear method + * is not supported by this applyTransformToDestination + */ + @Override + public void clear() { + backedSet.clear(); + } + + + + public boolean contains(K k, V v) { + return contains(new Pair<>(k, v)); + } + + public void add(K k, V v) { + add(new Pair<>(k, v)); + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collections/WeakIdentityHashMap.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collections/WeakIdentityHashMap.java new file mode 100644 index 000000000..cdaabaf5a --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/collections/WeakIdentityHashMap.java @@ -0,0 +1,173 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.collections; + +import lombok.*; + +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; +import java.util.*; + +public class WeakIdentityHashMap implements Map { + + protected final Map, V> map; + protected final ReferenceQueue refQueue; + + public WeakIdentityHashMap(){ + map = new HashMap<>(); + refQueue = new ReferenceQueue<>(); + } + + //Clear references to any map keys that have been GC'd + protected void clearReferences(){ + Reference r; + while((r = refQueue.poll()) != null){ + map.remove(r); + } + } + + @Override + public int size() { + clearReferences(); + return map.size(); + } + + @Override + public boolean isEmpty() { + clearReferences(); + return map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + clearReferences(); + return map.containsKey(new KeyRef<>(key)); + } + + @Override + public boolean containsValue(Object value) { + clearReferences(); + return map.containsValue(value); + } + + @Override + public V get(Object key) { + clearReferences(); + return map.get(new KeyRef<>(key)); + } + + @Override + public V put(K key, V value) { + clearReferences(); + map.put(new KeyRef<>(key), value); + return value; + } + + @Override + public V remove(Object key) { + clearReferences(); + return map.remove(new KeyRef<>(key)); + } + + @Override + public void putAll(Map m) { + clearReferences(); + for(Map.Entry e : m.entrySet()){ + map.put(new KeyRef<>(e.getKey()), e.getValue()); + } + } + + @Override + public void clear() { + map.clear(); + clearReferences(); + } + + @Override + public Set keySet() { + clearReferences(); + Set ret = new HashSet<>(); + for(KeyRef k : map.keySet() ){ + K key = k.get(); + if(key != null) + ret.add(key); + } + return ret; + } + + @Override + public Collection values() { + clearReferences(); + return map.values(); + } + + @Override + public Set> entrySet() { + clearReferences(); + Set> ret = new HashSet<>(); + for(Map.Entry, V> e : map.entrySet()){ + K k = e.getKey().get(); + if(k != null){ + ret.add(new Entry(k, e.getValue())); + } + } + return ret; + } + + + protected static class KeyRef extends WeakReference { + private final int hash; + public KeyRef(@NonNull K referent) { + super(referent); + this.hash = System.identityHashCode(referent); + } + + @Override + public int hashCode(){ + return hash; + } + + @Override + public boolean equals(Object o){ + if(this == o){ + return true; + } + if(o instanceof WeakReference){ + return this.get() == ((WeakReference) o).get(); + } + return false; + } + } + + @Data + @AllArgsConstructor + protected static class Entry implements Map.Entry { + protected K key; + protected V value; + + @Override + public V setValue(V value){ + this.value = value; + return value; + } + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/config/ND4JClassLoading.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/config/ND4JClassLoading.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/config/ND4JClassLoading.java rename to cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/config/ND4JClassLoading.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/config/ND4JEnvironmentVars.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/config/ND4JEnvironmentVars.java new file mode 100644 index 000000000..ecd1a80a1 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/config/ND4JEnvironmentVars.java @@ -0,0 +1,179 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.config; + +public class ND4JEnvironmentVars { + + /** + * Applicability: nd4j-native, when multiple backends are on classpath
+ * Description: Defines the priority that the CPU/Native backend should be loaded (or attempt to be loaded). If this + * is set to a higher value than {@link #BACKEND_PRIORITY_GPU} (which has default value 100) the native backend + * will be loaded in preference to the CUDA backend, when both are on the classpath. Default value: 0 + */ + public static final String BACKEND_PRIORITY_CPU = "BACKEND_PRIORITY_CPU"; + /** + * Applicability: nd4j-cuda-xx, when multiple backends are on classpath
+ * Description: Defines the priority that the CUDA (GPU) backend should be loaded (or attempt to be loaded). If this + * is set to a higher value than {@link #BACKEND_PRIORITY_CPU} (which has default value 0) the GPU backend + * will be loaded in preference to the CUDA backend, when both are on the classpath. Default value: 100 - hence + * by default, the CUDA backend will be loaded when both it and the CPU/native backend are on the classpath + */ + public static final String BACKEND_PRIORITY_GPU = "BACKEND_PRIORITY_GPU"; + /** + * Applicability: always - but only if an ND4J backend cannot be found/loaded via standard ServiceLoader mechanisms
+ * Description: Set this environment variable to a set fully qualified JAR files to attempt to load before failing on + * not loading a backend. JAR files should be semi-colon delimited; i.e., "/some/file.jar;/other/path.jar". + * This should rarely be required in practice - for example, only in dynamic class loading/dynamic classpath scenarios
+ * For equivalent system property, see {@link ND4JSystemProperties#DYNAMIC_LOAD_CLASSPATH_PROPERTY} for the equivalent + * system property (that will take precidence if both are set) + */ + public static final String BACKEND_DYNAMIC_LOAD_CLASSPATH = "ND4J_DYNAMIC_LOAD_CLASSPATH"; + /** + * Applicability: nd4j-native backend
+ * Description: Sets the number of OpenMP parallel threads for ND4J native operations (and also native BLAS libraries + * such as Intel MKL and OpenBLAS). + * By default, this will be set to the number of physical cores (i.e., excluding hyperthreading cores), which usually + * provides optimal performance. Setting this to a larger value than the number of physical cores (for example, equal + * to number of logical cores - i.e., setting to 16 on an 8-core + hypethreading processor) - can result in reduced + * performance
+ * Note that if you have a significant number of parallel Java threads (for example, Spark or ParallelWrapper), or + * you want to keep some cores free for other programs - you may want to reduce this value. + * + * @see #ND4J_SKIP_BLAS_THREADS + */ + public static final String OMP_NUM_THREADS = "OMP_NUM_THREADS"; + /** + * Applicability: nd4j-native backend
+ * Description: Skips the setting of the {@link #OMP_NUM_THREADS} property for ND4J ops. Note that this property + * will usually still take effect for native BLAS libraries (MKL, OpenBLAS) even if this property is set + */ + public static final String ND4J_SKIP_BLAS_THREADS = "ND4J_SKIP_BLAS_THREADS"; + /** + * Applicability: nd4j-native backend
+ * Description: Whether build-in BLAS matrix multiplication (GEMM) should be used instead of the native BLAS + * library such as MKL or OpenBLAS. This can have a noticable performance impact for these ops. + * Note that this is typically only useful as a workaround (or test) for bugs in these underlying native libraries, + * which are rare (but do occasionally occur on some platforms) + */ + public static final String ND4J_FALLBACK = "ND4J_FALLBACK"; + /** + * Applicability: nd4j-parameter-server
+ * Usage: A fallback for determining the local IP the parameter server, if other approaches fail to determine the + * local IP + */ + public static final String DL4J_VOID_IP = "DL4J_VOID_IP"; + /** + * Applicability: nd4j-cuda-xx
+ * Description: + */ + public static final String ND4J_CUDA_MAX_BLOCK_SIZE = "ND4J_CUDA_MAX_BLOCK_SIZE"; + /** + * Applicability: nd4j-cuda-xx
+ * Description: + */ + public static final String ND4J_CUDA_MIN_BLOCK_SIZE = "ND4J_CUDA_MIN_BLOCK_SIZE"; + /** + * Applicability: nd4j-cuda-xx
+ * Description: + */ + public static final String ND4J_CUDA_MAX_GRID_SIZE = "ND4J_CUDA_MAX_GRID_SIZE"; + + /** + * Applicability: nd4j-cuda-xx
+ * Description: This variable defines how many concurrent threads will be able to use same device. Keep in mind, this doesn't affect natural CUDA limitations + */ + public static final String ND4J_CUDA_MAX_CONTEXTS = "ND4J_CUDA_MAX_CONTEXTS"; + + /** + * Applicability: nd4j-cuda-xx used on multi-GPU systems
+ * Description: If set, only a single GPU will be used by ND4J, even if multiple GPUs are available in the system + */ + public static final String ND4J_CUDA_FORCE_SINGLE_GPU = "ND4J_CUDA_FORCE_SINGLE_GPU"; + /** + * Applicability: nd4j-cuda-xx
+ * Description: + */ + public static final String ND4J_CUDA_USE_PREALLOCATION = "ND4J_CUDA_USE_PREALLOCATION"; + /** + * Applicability: nd4j-cuda-xx
+ * Description: + */ + public static final String ND4J_CUDA_MAX_DEVICE_CACHE = "ND4J_CUDA_MAX_DEVICE_CACHE"; + /** + * Applicability: nd4j-cuda-xx
+ * Description: + */ + public static final String ND4J_CUDA_MAX_HOST_CACHE = "ND4J_CUDA_MAX_HOST_CACHE"; + /** + * Applicability: nd4j-cuda-xx
+ * Description: + */ + public static final String ND4J_CUDA_MAX_DEVICE_ALLOCATION = "ND4J_CUDA_MAX_DEVICE_ALLOCATION"; + + /** + * Applicability: nd4j-native + */ + public static final String ND4J_MKL_FALLBACK = "ND4J_MKL_FALLBACK"; + + public static final String ND4J_RESOURCES_CACHE_DIR = "ND4J_RESOURCES_CACHE_DIR"; + + /** + * Applicability: nd4j-native
+ * Description: Set to true to avoid logging AVX warnings (i.e., running generic x86 binaries on an AVX2 system) + */ + public static final String ND4J_IGNORE_AVX = "ND4J_IGNORE_AVX"; + + /** + * This variable defines how many threads will be used in ThreadPool for parallel execution of linear algebra. + * Default value: number of threads supported by this system. + */ + public static final String SD_MAX_THREADS = "SD_MAX_THREADS"; + + /** + * This variable defines how many threads will be used for any 1 linear algebra operation. + * Default value: number of threads supported by this system. + */ + public static final String SD_MASTER_THREADS = "SD_MASTER_THREADS"; + + /** + * If set, this variable disables use of optimized platform helpers (i.e. mkldnn or cuDNN) + */ + public static final String SD_FORBID_HELPERS = "SD_FORBID_HELPERS"; + + /** + * If set, this variables defines how much memory application is allowed to use off-heap. + * PLEASE NOTE: this option is separate from JVM XMS/XMX options + */ + public static final String SD_MAX_PRIMARY_BYTES = "SD_MAX_PRIMARY_BYTES"; + + /** + * If set, this variable defines how much memory application is allowed to use ON ALL computational devices COMBINED. + */ + public static final String SD_MAX_SPECIAL_BYTES = "SD_MAX_SPECIAL_BYTES"; + + /** + * If set, this variable defines how much memory application is allowed to use on any one computational device + */ + public static final String SD_MAX_DEVICE_BYTES = "SD_MAX_DEVICE_BYTES"; + + private ND4JEnvironmentVars() { + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/config/ND4JSystemProperties.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/config/ND4JSystemProperties.java new file mode 100644 index 000000000..ba1d011cd --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/config/ND4JSystemProperties.java @@ -0,0 +1,164 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.config; + +import org.nd4j.common.resources.Resources; +import org.nd4j.common.resources.strumpf.ResourceFile; +import org.nd4j.common.resources.strumpf.StrumpfResolver; + +import java.io.File; +import java.net.URL; + +public class ND4JSystemProperties { + + /** + * Applicability: Always
+ * Description: Sets the default datatype for ND4J - should be one of "float", "double", "half". + * ND4J is set to float (32-bit floating point values) by default. + */ + public static final String DTYPE = "dtype"; + /** + * Applicability: Always
+ * Description: By default, ND4J will log some information when the library has completed initialization, such as the + * backend (CPU or CUDA), CPU/Devices, memory etc. This system property can be used to disable the logging of this + * initialization information + */ + public static final String LOG_INITIALIZATION = "org.nd4j.log.initialization"; + + /** + * Applicability: nd4j-native when running non-AVX binary on an AVX compatible CPU
+ * Description: Set to true to avoid logging AVX warnings (i.e., running generic x86 binaries on an AVX2 system) + */ + public static final String ND4J_IGNORE_AVX = "org.nd4j.avx.ignore"; + + /** + * Applicability: Always
+ * Description: This system property defines the maximum amount of off-heap memory that can be used. + * ND4J uses off-heap memory for storage of all INDArray data. This off-heap memory is a different + * pool of memory to the on-heap JVM memory (configured using standard Java Xms/Xmx options). + * Default: 2x Java XMX setting + * + * @see #JAVACPP_MEMORY_MAX_PHYSICAL_BYTES + */ + public static final String JAVACPP_MEMORY_MAX_BYTES = "org.bytedeco.javacpp.maxbytes"; + /** + * Applicability: Always
+ * Description: This system property defines the maximum total amount of memory that the process can use - it is + * the sum of both off-heap and on-heap memory. This can be used to provide an upper bound on the maximum amount + * of memory (of all types) that ND4J will use + * + * @see #JAVACPP_MEMORY_MAX_BYTES + */ + public static final String JAVACPP_MEMORY_MAX_PHYSICAL_BYTES = "org.bytedeco.javacpp.maxphysicalbytes"; + + /** + * Applicability: ND4J Temporary file creation/extraction for ClassPathResource, memory mapped workspaces, and
+ * Description: Specify the local directory where temporary files will be written. If not specified, the default + * Java temporary directory (java.io.tmpdir system property) will generally be used. + */ + public static final String ND4J_TEMP_DIR_PROPERTY = "org.nd4j.tempdir"; + + /** + * Applicability: always - but only if an ND4J backend cannot be found/loaded via standard ServiceLoader mechanisms
+ * Description: Set this property to a set fully qualified JAR files to attempt to load before failing on + * not loading a backend. JAR files should be semi-colon delimited; i.e., "/some/file.jar;/other/path.jar". + * This should rarely be required in practice - for example, only in dynamic class loading/dynamic classpath scenarios
+ * For equivalent system property, see {@link ND4JEnvironmentVars#BACKEND_DYNAMIC_LOAD_CLASSPATH} for the equivalent + * system property (the system property will take precidence if both are set) + */ + public static final String DYNAMIC_LOAD_CLASSPATH_PROPERTY = "org.nd4j.backend.dynamicbackend"; + /** + * Applicability: Always
+ * Description Setting the system property to false will stop ND4J from performing the version check, and logging any + * warnings/errors. By default, the version check is enabled.
+ * Note: the version check is there for a reason! Using incompatible versions of ND4J/DL4J etc is likely to cause + * issues, and should be avoided. + */ + public static final String VERSION_CHECK_PROPERTY = "org.nd4j.versioncheck"; + /** + * Applicability: always
+ * Description: Used to specify the maximum number of elements (numbers) to print when using DataBuffer.toString(). + * Use -1 to print all elements (i.e., no limit). This is usually to avoid expensive toString() calls on buffers + * which may have millions of elements - for example, in a debugger
+ * Default: 1000 + */ + public static final String DATABUFFER_TO_STRING_MAX_ELEMENTS = "org.nd4j.databuffer.tostring.maxelements"; + /** + * Applicability: nd4j-native backend, when multiple BLAS libraries are available
+ * Description: This system property can be used to control which BLAS library is loaded and used by ND4J. + * For example, {@code org.bytedeco.javacpp.openblas.load=mkl_rt} can be used to load a default installation of MKL. + * However, MKL is liked with by default (when available) so setting this option explicitly is not usually required. + * For more details, see https://github.com/bytedeco/javacpp-presets/tree/master/openblas#documentation + */ + public static final String ND4J_CPU_LOAD_OPENBLAS = "org.bytedeco.openblas.load"; + /** + * Applicability: nd4j-native backend, when multiple BLAS libraries are available
+ * Description: This system property can be used to control which BLAS library is loaded and used by ND4J. + * Similar to {@link #ND4J_CPU_LOAD_OPENBLAS} but when this is set, LAPACK will not be loaded + */ + public static final String ND4J_CPU_LOAD_OPENBLAS_NOLAPACK = "org.bytedeco.openblas_nolapack.load"; + /** + * Applicability: nd4j-parameter-server, dl4j-spark (gradient sharing training master)
+ * Description: Aeros in a high-performance communication library used in distributed computing contexts in some + * places in ND4J and DL4J. This term buffer length determines the maximum message length that can be sent via Aeron + * in a single message. It can be increased to avoid exceptions such as {@code Encoded message exceeds maxMessageLength of 2097152}, + * at the expense of increased memory consumption (memory consumption is a multiple of this). It is specified in bytes + * with no unit suffix. Default value: 33554432 (32MB). + * IMPORTANT: This value must be an exact power of 2.
+ * Note also the maximum effective size is 128MB (134217728) (due to Aeron internal limits - beyond which increasing + * the buffer size will have no effect) + */ + public static final String AERON_TERM_BUFFER_PROP = "aeron.term.buffer.length"; + + /** + * Applicability: nd4j-common {@link Resources} class (and hence {@link StrumpfResolver})
+ * Description: When resolving resources from a Strumpf resource file (Example: {@code Resources.asFile("myFile.txt")} + * where should the remote files be downloaded to?
+ * This is generally used for resolving test resources, but can be used for Strumpf resource files generally. + */ + public static final String RESOURCES_CACHE_DIR = "org.nd4j.test.resources.cache.dir"; + + /** + * Applicability: nd4j-common {@link Resources} class (and hence {@link StrumpfResolver})
+ * Description: When resolving resources from a Strumpf resource file (Example: {@code Resources.asFile("myFile.txt")} + * what should be the connection timeout, as used by {@link org.apache.commons.io.FileUtils#copyURLToFile(URL, File, int, int)}
+ * Default: {@link ResourceFile#DEFAULT_CONNECTION_TIMEOUT} + */ + public static final String RESOURCES_CONNECTION_TIMEOUT = "org.nd4j.resources.download.connectiontimeout"; + + /** + * Applicability: nd4j-common {@link Resources} class (and hence {@link StrumpfResolver})
+ * Description: When resolving resources from a Strumpf resource file (Example: {@code Resources.asFile("myFile.txt")} + * what should be the connection timeout, as used by {@link org.apache.commons.io.FileUtils#copyURLToFile(URL, File, int, int)}
+ * Default: {@link ResourceFile#DEFAULT_READ_TIMEOUT} + */ + public static final String RESOURCES_READ_TIMEOUT = "org.nd4j.resources.download.readtimeout"; + + /** + * Applicability: nd4j-common {@link Resources} class (and hence {@link StrumpfResolver})
+ * Description: When resolving resources, what local directories should be checked (in addition to the classpath) for files? + * This is optional. Multiple directories may be specified, using comma-separated paths + */ + public static final String RESOURCES_LOCAL_DIRS = "org.nd4j.strumpf.resource.dirs"; + + private ND4JSystemProperties() { + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiConsumer.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiConsumer.java new file mode 100644 index 000000000..439ee14bd --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiConsumer.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +public interface BiConsumer { + + /** + * Perform the operation on the given arguments + * + * @param t First input + * @param u Second input + */ + void accept(T t, U u); + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiFunction.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiFunction.java new file mode 100644 index 000000000..debaa89f1 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiFunction.java @@ -0,0 +1,34 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +public interface BiFunction { + + /** + * Apply the function and return the result + * + * @param t First argument + * @param u Second argument + * @return Result + */ + R apply(T t, U u); + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiPredicate.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiPredicate.java new file mode 100644 index 000000000..52450dde5 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/BiPredicate.java @@ -0,0 +1,34 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +public interface BiPredicate { + + /** + * Evaluate the predicate + * + * @param t First argument + * @param u Second argument + * @return Result + */ + boolean test(T t, U u); + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Consumer.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Consumer.java new file mode 100644 index 000000000..da6add192 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Consumer.java @@ -0,0 +1,30 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +public interface Consumer { + + /** + * Perform the operation on the input + * @param t Input + */ + void accept(T t); +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Function.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Function.java new file mode 100644 index 000000000..fa8ec40c6 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Function.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +public interface Function { + + /** + * Apply the function to the argument, and return the result + * + * @param t Input + * @return Result + */ + R apply (T t); + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/FunctionalUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/FunctionalUtils.java new file mode 100644 index 000000000..c62ae71a5 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/FunctionalUtils.java @@ -0,0 +1,133 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +import org.nd4j.common.primitives.Pair; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class FunctionalUtils { + + + /** + * For each key in left and right, cogroup returns the list of values + * as a pair for each value present in left as well as right. + * @param left the left list of pairs to join + * @param right the right list of pairs to join + * @param the key type + * @param the value type + * @return a map of the list of values by key for each value in the left as well as the right + * with each element in the pair representing the values in left and right respectively. + */ + public static Map,List>> cogroup(List> left,List> right) { + Map,List>> ret = new HashMap<>(); + + //group by key first to consolidate values + Map> leftMap = groupByKey(left); + Map> rightMap = groupByKey(right); + + /** + * Iterate over each key in the list + * adding values to the left items + * as values are found in the list. + */ + for(Map.Entry> entry : leftMap.entrySet()) { + K key = entry.getKey(); + if(!ret.containsKey(key)) { + List leftListPair = new ArrayList<>(); + List rightListPair = new ArrayList<>(); + Pair,List> p = Pair.of(leftListPair,rightListPair); + ret.put(key,p); + } + + Pair,List> p = ret.get(key); + p.getFirst().addAll(entry.getValue()); + + + } + + /** + * Iterate over each key in the list + * adding values to the right items + * as values are found in the list. + */ + for(Map.Entry> entry : rightMap.entrySet()) { + K key = entry.getKey(); + if(!ret.containsKey(key)) { + List leftListPair = new ArrayList<>(); + List rightListPair = new ArrayList<>(); + Pair,List> p = Pair.of(leftListPair,rightListPair); + ret.put(key,p); + } + + Pair,List> p = ret.get(key); + p.getSecond().addAll(entry.getValue()); + + } + + return ret; + } + + /** + * Group the input pairs by the key of each pair. + * @param listInput the list of pairs to group + * @param the key type + * @param the value type + * @return a map representing a grouping of the + * keys by the given input key type and list of values + * in the grouping. + */ + public static Map> groupByKey(List> listInput) { + Map> ret = new HashMap<>(); + for(Pair pair : listInput) { + List currList = ret.get(pair.getFirst()); + if(currList == null) { + currList = new ArrayList<>(); + ret.put(pair.getFirst(),currList); + } + + currList.add(pair.getSecond()); + } + + return ret; + } + + /** + * Convert a map with a set of entries of type K for key + * and V for value in to a list of {@link Pair} + * @param map the map to collapse + * @param the key type + * @param the value type + * @return the collapsed map as a {@link List} + */ + public static List> mapToPair(Map map) { + List> ret = new ArrayList<>(map.size()); + for(Map.Entry entry : map.entrySet()) { + ret.add(Pair.of(entry.getKey(),entry.getValue())); + } + + return ret; + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Predicate.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Predicate.java new file mode 100644 index 000000000..a0d2cfc93 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Predicate.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +public interface Predicate { + + /** + * Returns the result of the predicate on the given input + * + * @param t Input + * @return Result + */ + boolean test(T t); + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Supplier.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Supplier.java new file mode 100644 index 000000000..d29ca3ee8 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/Supplier.java @@ -0,0 +1,35 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +/** + * A supplier of results with no input arguments + * + * @param Type of result + */ +public interface Supplier { + + /** + * @return Result + */ + T get(); + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/UnaryOperator.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/UnaryOperator.java new file mode 100644 index 000000000..894e0263b --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/function/UnaryOperator.java @@ -0,0 +1,24 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.function; + +public interface UnaryOperator extends Function { +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/holder/ObjectMapperHolder.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/holder/ObjectMapperHolder.java new file mode 100644 index 000000000..0cd5166a1 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/holder/ObjectMapperHolder.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.holder; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class ObjectMapperHolder { + + private static ObjectMapper objectMapper = getMapper(); + + private ObjectMapperHolder() {} + + + /** + * Get a single object mapper for use + * with reading and writing json + * @return + */ + public static ObjectMapper getJsonMapper() { + return objectMapper; + } + + private static ObjectMapper getMapper() { + ObjectMapper om = new ObjectMapper(); + //Serialize fields only, not using getters + //Not all getters are supported - for example, UserEntity + om.setVisibilityChecker(om.getSerializationConfig() + .getDefaultVisibilityChecker() + .withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withSetterVisibility(JsonAutoDetect.Visibility.NONE) + .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)); + om.setSerializationInclusion(JsonInclude.Include.NON_NULL); + return om; + } + + + +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/AbstractFileResolvingResource.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/AbstractFileResolvingResource.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/AbstractFileResolvingResource.java rename to cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/AbstractFileResolvingResource.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/AbstractResource.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/AbstractResource.java new file mode 100644 index 000000000..a6595a0e3 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/AbstractResource.java @@ -0,0 +1,132 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +public abstract class AbstractResource implements Resource { + public AbstractResource() {} + + public boolean exists() { + try { + return this.getFile().exists(); + } catch (IOException var4) { + try { + InputStream isEx = this.getInputStream(); + isEx.close(); + return true; + } catch (Throwable var3) { + return false; + } + } + } + + public boolean isReadable() { + return true; + } + + public boolean isOpen() { + return false; + } + + public URL getURL() throws IOException { + throw new FileNotFoundException(this.getDescription() + " cannot be resolved to URL"); + } + + public URI getURI() throws IOException { + URL url = this.getURL(); + + try { + return ResourceUtils.toURI(url); + } catch (URISyntaxException var3) { + throw new IOException("Invalid URI [" + url + "]", var3); + } + } + + public File getFile() throws IOException { + throw new FileNotFoundException(this.getDescription() + " cannot be resolved to absolute file path"); + } + + public long contentLength() throws IOException { + InputStream is = this.getInputStream(); + Assert.state(is != null, "resource input stream must not be null"); + + try { + long size = 0L; + + int read; + for (byte[] buf = new byte[255]; (read = is.read(buf)) != -1; size += (long) read) { + ; + } + + long var6 = size; + return var6; + } finally { + try { + is.close(); + } catch (IOException var14) { + ; + } + + } + } + + public long lastModified() throws IOException { + long lastModified = this.getFileForLastModifiedCheck().lastModified(); + if (lastModified == 0L) { + throw new FileNotFoundException(this.getDescription() + + " cannot be resolved in the file system for resolving its last-modified timestamp"); + } else { + return lastModified; + } + } + + protected File getFileForLastModifiedCheck() throws IOException { + return this.getFile(); + } + + public Resource createRelative(String relativePath) throws IOException { + throw new FileNotFoundException("Cannot create a relative resource for " + this.getDescription()); + } + + public String getFilename() { + return null; + } + + public String toString() { + return this.getDescription(); + } + + public boolean equals(Object obj) { + return obj == this + || obj instanceof Resource && ((Resource) obj).getDescription().equals(this.getDescription()); + } + + public int hashCode() { + return this.getDescription().hashCode(); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/Assert.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/Assert.java new file mode 100644 index 000000000..5da760b45 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/Assert.java @@ -0,0 +1,175 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + + +import java.util.Collection; +import java.util.Map; + +public abstract class Assert { + public Assert() {} + + public static void isTrue(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + public static void isTrue(boolean expression) { + isTrue(expression, "[Assertion failed] - this expression must be true"); + } + + public static void isNull(Object object, String message) { + if (object != null) { + throw new IllegalArgumentException(message); + } + } + + public static void isNull(Object object) { + isNull(object, "[Assertion failed] - the object argument must be null"); + } + + public static void notNull(Object object, String message) { + if (object == null) { + throw new IllegalArgumentException(message); + } + } + + public static void notNull(Object object) { + notNull(object, "[Assertion failed] - this argument is required; it must not be null"); + } + + public static void hasLength(String text, String message) { + if (!StringUtils.hasLength(text)) { + throw new IllegalArgumentException(message); + } + } + + public static void hasLength(String text) { + hasLength(text, "[Assertion failed] - this String argument must have length; it must not be null or empty"); + } + + public static void hasText(String text, String message) { + if (!StringUtils.hasText(text)) { + throw new IllegalArgumentException(message); + } + } + + public static void hasText(String text) { + hasText(text, "[Assertion failed] - this String argument must have text; it must not be null, empty, or blank"); + } + + public static void doesNotContain(String textToSearch, String substring, String message) { + if (StringUtils.hasLength(textToSearch) && StringUtils.hasLength(substring) + && textToSearch.contains(substring)) { + throw new IllegalArgumentException(message); + } + } + + public static void doesNotContain(String textToSearch, String substring) { + doesNotContain(textToSearch, substring, + "[Assertion failed] - this String argument must not contain the substring [" + substring + "]"); + } + + public static void notEmpty(Object[] array, String message) { + if (ObjectUtils.isEmpty(array)) { + throw new IllegalArgumentException(message); + } + } + + public static void notEmpty(Object[] array) { + notEmpty(array, "[Assertion failed] - this array must not be empty: it must contain at least 1 element"); + } + + public static void noNullElements(Object[] array, String message) { + if (array != null) { + Object[] arr$ = array; + int len$ = array.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Object element = arr$[i$]; + if (element == null) { + throw new IllegalArgumentException(message); + } + } + } + + } + + public static void noNullElements(Object[] array) { + noNullElements(array, "[Assertion failed] - this array must not contain any null elements"); + } + + public static void notEmpty(Collection collection, String message) { + if (CollectionUtils.isEmpty(collection)) { + throw new IllegalArgumentException(message); + } + } + + public static void notEmpty(Collection collection) { + notEmpty(collection, + "[Assertion failed] - this collection must not be empty: it must contain at least 1 element"); + } + + public static void notEmpty(Map map, String message) { + if (CollectionUtils.isEmpty(map)) { + throw new IllegalArgumentException(message); + } + } + + public static void notEmpty(Map map) { + notEmpty(map, "[Assertion failed] - this map must not be empty; it must contain at least one entry"); + } + + public static void isInstanceOf(Class clazz, Object obj) { + isInstanceOf(clazz, obj, ""); + } + + public static void isInstanceOf(Class type, Object obj, String message) { + notNull(type, "Type to check against must not be null"); + if (!type.isInstance(obj)) { + throw new IllegalArgumentException((StringUtils.hasLength(message) ? message + " " : "") + + "Object of class [" + (obj != null ? obj.getClass().getName() : "null") + + "] must be an instance of " + type); + } + } + + public static void isAssignable(Class superType, Class subType) { + isAssignable(superType, subType, ""); + } + + public static void isAssignable(Class superType, Class subType, String message) { + notNull(superType, "Type to check against must not be null"); + if (subType == null || !superType.isAssignableFrom(subType)) { + throw new IllegalArgumentException(message + subType + " is not assignable to " + superType); + } + } + + public static void state(boolean expression, String message) { + if (!expression) { + throw new IllegalStateException(message); + } + } + + public static void state(boolean expression) { + state(expression, "[Assertion failed] - this state invariant must be true"); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ClassPathResource.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ClassPathResource.java new file mode 100644 index 000000000..cf3d45944 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ClassPathResource.java @@ -0,0 +1,439 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.nd4j.common.base.Preconditions; +import org.nd4j.common.config.ND4JClassLoading; + +import java.io.*; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.attribute.FileAttribute; +import java.util.Enumeration; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class ClassPathResource extends AbstractFileResolvingResource { + + private final String path; + private ClassLoader classLoader; + private Class clazz; + + public ClassPathResource(String path) { + this(path, (ClassLoader) null); + } + + public ClassPathResource(String path, ClassLoader classLoader) { + Assert.notNull(path, "Path must not be null"); + String pathToUse = StringUtils.cleanPath(path); + if (pathToUse.startsWith("/")) { + pathToUse = pathToUse.substring(1); + } + + this.path = pathToUse; + this.classLoader = classLoader != null ? classLoader : ND4JClassLoading.getNd4jClassloader(); + } + + public ClassPathResource(String path, Class clazz) { + Assert.notNull(path, "Path must not be null"); + this.path = StringUtils.cleanPath(path); + this.clazz = clazz; + } + + protected ClassPathResource(String path, ClassLoader classLoader, Class clazz) { + this.path = StringUtils.cleanPath(path); + this.classLoader = classLoader; + this.clazz = clazz; + } + + public final String getPath() { + return this.path; + } + + public final ClassLoader getClassLoader() { + return this.classLoader != null ? this.classLoader : this.clazz.getClassLoader(); + } + + /** + * Get the File. + * If the file cannot be accessed directly (for example, it is in a JAR file), we will attempt to extract it from + * the JAR and copy it to the temporary directory, using {@link #getTempFileFromArchive()} + * + * @return The File, or a temporary copy if it can not be accessed directly + * @throws IOException + */ + @Override + public File getFile() throws IOException { + try{ + return super.getFile(); + } catch (FileNotFoundException e){ + //java.io.FileNotFoundException: class path resource [iris.txt] cannot be resolved to absolute file path because + // it does not reside in the file system: jar:file:/.../dl4j-test-resources-0.9.2-SNAPSHOT.jar!/iris.txt + return getTempFileFromArchive(); + } + } + + + /** + * Get a temp file from the classpath.
+ * This is for resources where a file is needed and the classpath resource is in a jar file. The file is copied + * to the default temporary directory, using {@link Files#createTempFile(String, String, FileAttribute[])}. + * Consequently, the extracted file will have a different filename to the extracted one. + * + * @return the temp file + * @throws IOException If an error occurs when files are being copied + * @see #getTempFileFromArchive(File) + */ + public File getTempFileFromArchive() throws IOException { + return getTempFileFromArchive(null); + } + + /** + * Get a temp file from the classpath, and (optionally) place it in the specified directory
+ * Note that:
+ * - If the directory is not specified, the file is copied to the default temporary directory, using + * {@link Files#createTempFile(String, String, FileAttribute[])}. Consequently, the extracted file will have a + * different filename to the extracted one.
+ * - If the directory *is* specified, the file is copied directly - and the original filename is maintained + * + * @param rootDirectory May be null. If non-null, copy to the specified directory + * @return the temp file + * @throws IOException If an error occurs when files are being copied + * @see #getTempFileFromArchive(File) + */ + public File getTempFileFromArchive(File rootDirectory) throws IOException { + InputStream is = getInputStream(); + File tmpFile; + if(rootDirectory != null){ + //Maintain original file names, as it's going in a directory... + tmpFile = new File(rootDirectory, FilenameUtils.getName(path)); + } else { + tmpFile = Files.createTempFile(FilenameUtils.getName(path), "tmp").toFile(); + } + + tmpFile.deleteOnExit(); + + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tmpFile)); + + IOUtils.copy(is, bos); + bos.flush(); + bos.close(); + return tmpFile; + } + + /** + * Extract the directory recursively to the specified location. Current ClassPathResource must point to + * a directory.
+ * For example, if classpathresource points to "some/dir/", then the contents - not including the parent directory "dir" - + * will be extracted or copied to the specified destination.
+ * @param destination Destination directory. Must exist + */ + public void copyDirectory(File destination) throws IOException { + Preconditions.checkState(destination.exists() && destination.isDirectory(), "Destination directory must exist and be a directory: %s", destination); + + + URL url = this.getUrl(); + + if (isJarURL(url)) { + /* + This is actually request for file, that's packed into jar. Probably the current one, but that doesn't matters. + */ + InputStream stream = null; + ZipFile zipFile = null; + try { + GetStreamFromZip getStreamFromZip = new GetStreamFromZip(url, path).invoke(); + ZipEntry entry = getStreamFromZip.getEntry(); + stream = getStreamFromZip.getStream(); + zipFile = getStreamFromZip.getZipFile(); + + Preconditions.checkState(entry.isDirectory(), "Source must be a directory: %s", entry.getName()); + + String pathNoSlash = this.path; + if(pathNoSlash.endsWith("/") || pathNoSlash.endsWith("\\")){ + pathNoSlash = pathNoSlash.substring(0, pathNoSlash.length()-1); + } + + Enumeration entries = zipFile.entries(); + while(entries.hasMoreElements()){ + ZipEntry e = entries.nextElement(); + String name = e.getName(); + if(name.startsWith(pathNoSlash) && name.length() > pathNoSlash.length() && (name.charAt(pathNoSlash.length()) == '/' || name.charAt(pathNoSlash.length()) == '\\')){ //second condition: to avoid "/dir/a/" and "/dir/abc/" both matching startsWith + + String relativePath = name.substring(this.path.length()); + + File extractTo = new File(destination, relativePath); + if(e.isDirectory()){ + extractTo.mkdirs(); + } else { + try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(extractTo))){ + InputStream is = getInputStream(name, clazz, classLoader); + IOUtils.copy(is, bos); + } + } + } + } + + stream.close(); + zipFile.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + if(stream != null) + IOUtils.closeQuietly(stream); + if(zipFile != null) + IOUtils.closeQuietly(zipFile); + } + + } else { + File source; + try{ + source = new File(url.toURI()); + } catch (URISyntaxException e) { + throw new IOException("Error converting URL to a URI - path may be invalid? Path=" + url); + } + Preconditions.checkState(source.isDirectory(), "Source must be a directory: %s", source); + Preconditions.checkState(destination.exists() && destination.isDirectory(), "Destination must be a directory and must exist: %s", destination); + FileUtils.copyDirectory(source, destination); + } + } + + public boolean exists() { + URL url; + if (this.clazz != null) { + url = this.clazz.getResource(this.path); + } else { + url = this.classLoader.getResource(this.path); + } + + return url != null; + } + + public InputStream getInputStream() throws IOException { + return getInputStream(path, clazz, classLoader); + } + + + private static InputStream getInputStream(String path, Class clazz, ClassLoader classLoader) throws IOException { + InputStream is; + if (clazz != null) { + is = clazz.getResourceAsStream(path); + } else { + is = classLoader.getResourceAsStream(path); + } + + if (is == null) { + throw new FileNotFoundException(path + " cannot be opened because it does not exist"); + } else { + if(is instanceof BufferedInputStream) + return is; + return new BufferedInputStream(is); + } + } + + public URL getURL() throws IOException { + URL url; + if (this.clazz != null) { + url = this.clazz.getResource(this.path); + } else { + url = this.classLoader.getResource(this.path); + } + + if (url == null) { + throw new FileNotFoundException( + this.getDescription() + " cannot be resolved to URL because it does not exist"); + } else { + return url; + } + } + + public Resource createRelative(String relativePath) { + String pathToUse = StringUtils.applyRelativePath(this.path, relativePath); + return new ClassPathResource(pathToUse, this.classLoader, this.clazz); + } + + public String getFilename() { + return StringUtils.getFilename(this.path); + } + + public String getDescription() { + StringBuilder builder = new StringBuilder("class path resource ["); + String pathToUse = this.path; + if (this.clazz != null && !pathToUse.startsWith("/")) { + builder.append(ResourceUtils.classPackageAsResourcePath(this.clazz)); + builder.append('/'); + } + + if (pathToUse.startsWith("/")) { + pathToUse = pathToUse.substring(1); + } + + builder.append(pathToUse); + builder.append(']'); + return builder.toString(); + } + + public boolean equals(Object obj) { + if (obj == this) { + return true; + } else if (!(obj instanceof ClassPathResource)) { + return false; + } else { + ClassPathResource otherRes = (ClassPathResource) obj; + return this.path.equals(otherRes.path) && ObjectUtils.nullSafeEquals(this.classLoader, otherRes.classLoader) + && ObjectUtils.nullSafeEquals(this.clazz, otherRes.clazz); + } + } + + public int hashCode() { + return this.path.hashCode(); + } + + /** + * Returns URL of the requested resource + * + * @return URL of the resource, if it's available in current Jar + */ + private URL getUrl() { + ClassLoader loader = null; + try { + loader = ND4JClassLoading.getNd4jClassloader(); + } catch (Exception e) { + // do nothing + } + + if (loader == null) { + loader = ClassPathResource.class.getClassLoader(); + } + + URL url = loader.getResource(this.path); + if (url == null) { + // try to check for mis-used starting slash + // TODO: see TODO below + if (this.path.startsWith("/")) { + url = loader.getResource(this.path.replaceFirst("[\\\\/]", "")); + if (url != null) + return url; + } else { + // try to add slash, to make clear it's not an issue + // TODO: change this mechanic to actual path purifier + url = loader.getResource("/" + this.path); + if (url != null) + return url; + } + throw new IllegalStateException("Resource '" + this.path + "' cannot be found."); + } + return url; + } + + /** + * Checks, if proposed URL is packed into archive. + * + * @param url URL to be checked + * @return True, if URL is archive entry, False otherwise + */ + private static boolean isJarURL(URL url) { + String protocol = url.getProtocol(); + return "jar".equals(protocol) || "zip".equals(protocol) || "wsjar".equals(protocol) + || "code-source".equals(protocol) && url.getPath().contains("!/"); + } + + private class GetStreamFromZip { + private URL url; + private ZipFile zipFile; + private ZipEntry entry; + private InputStream stream; + private String resourceName; + + public GetStreamFromZip(URL url, String resourceName) { + this.url = url; + this.resourceName = resourceName; + } + + public URL getUrl() { + return url; + } + + public ZipFile getZipFile() { + return zipFile; + } + + public ZipEntry getEntry() { + return entry; + } + + public InputStream getStream() { + return stream; + } + + public GetStreamFromZip invoke() throws IOException { + url = extractActualUrl(url); + + zipFile = new ZipFile(url.getFile()); + entry = zipFile.getEntry(this.resourceName); + if (entry == null) { + if (this.resourceName.startsWith("/")) { + entry = zipFile.getEntry(this.resourceName.replaceFirst("/", "")); + if (entry == null) { + throw new FileNotFoundException("Resource " + this.resourceName + " not found"); + } + } else + throw new FileNotFoundException("Resource " + this.resourceName + " not found"); + } + + stream = zipFile.getInputStream(entry); + return this; + } + } + + /** + * Extracts parent Jar URL from original ClassPath entry URL. + * + * @param jarUrl Original URL of the resource + * @return URL of the Jar file, containing requested resource + * @throws MalformedURLException + */ + private URL extractActualUrl(URL jarUrl) throws MalformedURLException { + String urlFile = jarUrl.getFile(); + int separatorIndex = urlFile.indexOf("!/"); + if (separatorIndex != -1) { + String jarFile = urlFile.substring(0, separatorIndex); + + try { + return new URL(jarFile); + } catch (MalformedURLException var5) { + if (!jarFile.startsWith("/")) { + jarFile = "/" + jarFile; + } + + return new URL("file:" + jarFile); + } + } else { + return jarUrl; + } + } + + +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/CollectionUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/CollectionUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/CollectionUtils.java rename to cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/CollectionUtils.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/InputStreamSource.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/InputStreamSource.java new file mode 100644 index 000000000..da66184be --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/InputStreamSource.java @@ -0,0 +1,29 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + + +import java.io.IOException; +import java.io.InputStream; + +public interface InputStreamSource { + InputStream getInputStream() throws IOException; +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ObjectUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ObjectUtils.java new file mode 100644 index 000000000..e1dcf32e9 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ObjectUtils.java @@ -0,0 +1,698 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import java.lang.reflect.Array; +import java.util.Arrays; + +public abstract class ObjectUtils { + private static final int INITIAL_HASH = 7; + private static final int MULTIPLIER = 31; + private static final String EMPTY_STRING = ""; + private static final String NULL_STRING = "null"; + private static final String ARRAY_START = "{"; + private static final String ARRAY_END = "}"; + private static final String EMPTY_ARRAY = "{}"; + private static final String ARRAY_ELEMENT_SEPARATOR = ", "; + + public ObjectUtils() {} + + public static boolean isCheckedException(Throwable ex) { + return !(ex instanceof RuntimeException) && !(ex instanceof Error); + } + + public static boolean isCompatibleWithThrowsClause(Throwable ex, Class[] declaredExceptions) { + if (!isCheckedException(ex)) { + return true; + } else { + if (declaredExceptions != null) { + for (int i = 0; i < declaredExceptions.length; ++i) { + if (declaredExceptions[i].isAssignableFrom(ex.getClass())) { + return true; + } + } + } + + return false; + } + } + + public static boolean isArray(Object obj) { + return obj != null && obj.getClass().isArray(); + } + + public static boolean isEmpty(Object[] array) { + return array == null || array.length == 0; + } + + public static boolean containsElement(Object[] array, Object element) { + if (array == null) { + return false; + } else { + Object[] arr$ = array; + int len$ = array.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Object arrayEle = arr$[i$]; + if (nullSafeEquals(arrayEle, element)) { + return true; + } + } + + return false; + } + } + + public static boolean containsConstant(Enum[] enumValues, String constant) { + return containsConstant(enumValues, constant, false); + } + + public static boolean containsConstant(Enum[] enumValues, String constant, boolean caseSensitive) { + Enum[] arr$ = enumValues; + int len$ = enumValues.length; + int i$ = 0; + + while (true) { + if (i$ >= len$) { + return false; + } + + Enum candidate = arr$[i$]; + if (caseSensitive) { + if (candidate.toString().equals(constant)) { + break; + } + } else if (candidate.toString().equalsIgnoreCase(constant)) { + break; + } + + ++i$; + } + + return true; + } + + public static > E caseInsensitiveValueOf(E[] enumValues, String constant) { + Enum[] arr$ = enumValues; + int len$ = enumValues.length; + + for (int i$ = 0; i$ < len$; ++i$) { + Enum candidate = arr$[i$]; + if (candidate.toString().equalsIgnoreCase(constant)) { + return (E) candidate; + } + } + + throw new IllegalArgumentException(String.format("constant [%s] does not exist in enum opType %s", + new Object[] {constant, enumValues.getClass().getComponentType().getName()})); + } + + public static A[] addObjectToArray(A[] array, O obj) { + Class compType = Object.class; + if (array != null) { + compType = array.getClass().getComponentType(); + } else if (obj != null) { + compType = obj.getClass(); + } + + int newArrLength = array != null ? array.length + 1 : 1; + Object[] newArr = (Object[]) Array.newInstance(compType, newArrLength); + if (array != null) { + System.arraycopy(array, 0, newArr, 0, array.length); + } + + newArr[newArr.length - 1] = obj; + return (A[]) newArr; + } + + public static Object[] toObjectArray(Object source) { + if (source instanceof Object[]) { + return (Object[]) source; + } else if (source == null) { + return new Object[0]; + } else if (!source.getClass().isArray()) { + throw new IllegalArgumentException("Source is not an array: " + source); + } else { + int length = Array.getLength(source); + if (length == 0) { + return new Object[0]; + } else { + Class wrapperType = Array.get(source, 0).getClass(); + Object[] newArray = (Object[]) Array.newInstance(wrapperType, length); + + for (int i = 0; i < length; ++i) { + newArray[i] = Array.get(source, i); + } + + return newArray; + } + } + } + + public static boolean nullSafeEquals(Object o1, Object o2) { + if (o1 == o2) { + return true; + } else if (o1 != null && o2 != null) { + if (o1.equals(o2)) { + return true; + } else { + if (o1.getClass().isArray() && o2.getClass().isArray()) { + if (o1 instanceof Object[] && o2 instanceof Object[]) { + return Arrays.equals((Object[]) o1, (Object[]) o2); + } + + if (o1 instanceof boolean[] && o2 instanceof boolean[]) { + return Arrays.equals((boolean[]) o1, (boolean[]) o2); + } + + if (o1 instanceof byte[] && o2 instanceof byte[]) { + return Arrays.equals((byte[]) o1, (byte[]) o2); + } + + if (o1 instanceof char[] && o2 instanceof char[]) { + return Arrays.equals((char[]) o1, (char[]) o2); + } + + if (o1 instanceof double[] && o2 instanceof double[]) { + return Arrays.equals((double[]) o1, (double[]) o2); + } + + if (o1 instanceof float[] && o2 instanceof float[]) { + return Arrays.equals((float[]) o1, (float[]) o2); + } + + if (o1 instanceof int[] && o2 instanceof int[]) { + return Arrays.equals((int[]) o1, (int[]) o2); + } + + if (o1 instanceof long[] && o2 instanceof long[]) { + return Arrays.equals((long[]) o1, (long[]) o2); + } + + if (o1 instanceof short[] && o2 instanceof short[]) { + return Arrays.equals((short[]) o1, (short[]) o2); + } + } + + return false; + } + } else { + return false; + } + } + + public static int nullSafeHashCode(Object obj) { + if (obj == null) { + return 0; + } else { + if (obj.getClass().isArray()) { + if (obj instanceof Object[]) { + return nullSafeHashCode((Object[]) obj); + } + + if (obj instanceof boolean[]) { + return nullSafeHashCode((boolean[]) obj); + } + + if (obj instanceof byte[]) { + return nullSafeHashCode((byte[]) obj); + } + + if (obj instanceof char[]) { + return nullSafeHashCode((char[]) obj); + } + + if (obj instanceof double[]) { + return nullSafeHashCode((double[]) obj); + } + + if (obj instanceof float[]) { + return nullSafeHashCode((float[]) obj); + } + + if (obj instanceof int[]) { + return nullSafeHashCode((int[]) obj); + } + + if (obj instanceof long[]) { + return nullSafeHashCode((long[]) obj); + } + + if (obj instanceof short[]) { + return nullSafeHashCode((short[]) obj); + } + } + + return obj.hashCode(); + } + } + + public static int nullSafeHashCode(Object[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + nullSafeHashCode(array[i]); + } + + return hash; + } + } + + public static int nullSafeHashCode(boolean[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + hashCode(array[i]); + } + + return hash; + } + } + + public static int nullSafeHashCode(byte[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + array[i]; + } + + return hash; + } + } + + public static int nullSafeHashCode(char[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + array[i]; + } + + return hash; + } + } + + public static int nullSafeHashCode(double[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + hashCode(array[i]); + } + + return hash; + } + } + + public static int nullSafeHashCode(float[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + hashCode(array[i]); + } + + return hash; + } + } + + public static int nullSafeHashCode(int[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + array[i]; + } + + return hash; + } + } + + public static int nullSafeHashCode(long[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + hashCode(array[i]); + } + + return hash; + } + } + + public static int nullSafeHashCode(short[] array) { + if (array == null) { + return 0; + } else { + int hash = 7; + int arraySize = array.length; + + for (int i = 0; i < arraySize; ++i) { + hash = 31 * hash + array[i]; + } + + return hash; + } + } + + public static int hashCode(boolean bool) { + return bool ? 1231 : 1237; + } + + public static int hashCode(double dbl) { + long bits = Double.doubleToLongBits(dbl); + return hashCode(bits); + } + + public static int hashCode(float flt) { + return Float.floatToIntBits(flt); + } + + public static int hashCode(long lng) { + return (int) (lng ^ lng >>> 32); + } + + public static String identityToString(Object obj) { + return obj == null ? "" : obj.getClass().getName() + "@" + getIdentityHexString(obj); + } + + public static String getIdentityHexString(Object obj) { + return Integer.toHexString(System.identityHashCode(obj)); + } + + public static String getDisplayString(Object obj) { + return obj == null ? "" : nullSafeToString(obj); + } + + public static String nullSafeClassName(Object obj) { + return obj != null ? obj.getClass().getName() : "null"; + } + + public static String nullSafeToString(Object obj) { + if (obj == null) { + return "null"; + } else if (obj instanceof String) { + return (String) obj; + } else if (obj instanceof Object[]) { + return nullSafeToString((Object[]) obj); + } else if (obj instanceof boolean[]) { + return nullSafeToString((boolean[]) obj); + } else if (obj instanceof byte[]) { + return nullSafeToString((byte[]) obj); + } else if (obj instanceof char[]) { + return nullSafeToString((char[]) obj); + } else if (obj instanceof double[]) { + return nullSafeToString((double[]) obj); + } else if (obj instanceof float[]) { + return nullSafeToString((float[]) obj); + } else if (obj instanceof int[]) { + return nullSafeToString((int[]) obj); + } else if (obj instanceof long[]) { + return nullSafeToString((long[]) obj); + } else if (obj instanceof short[]) { + return nullSafeToString((short[]) obj); + } else { + String str = obj.toString(); + return str != null ? str : ""; + } + } + + public static String nullSafeToString(Object[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append(String.valueOf(array[i])); + } + + sb.append("}"); + return sb.toString(); + } + } + } + + public static String nullSafeToString(boolean[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append(array[i]); + } + + sb.append("}"); + return sb.toString(); + } + } + } + + public static String nullSafeToString(byte[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append(array[i]); + } + + sb.append("}"); + return sb.toString(); + } + } + } + + public static String nullSafeToString(char[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append("\'").append(array[i]).append("\'"); + } + + sb.append("}"); + return sb.toString(); + } + } + } + + public static String nullSafeToString(double[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append(array[i]); + } + + sb.append("}"); + return sb.toString(); + } + } + } + + public static String nullSafeToString(float[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append(array[i]); + } + + sb.append("}"); + return sb.toString(); + } + } + } + + public static String nullSafeToString(int[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append(array[i]); + } + + sb.append("}"); + return sb.toString(); + } + } + } + + public static String nullSafeToString(long[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append(array[i]); + } + + sb.append("}"); + return sb.toString(); + } + } + } + + public static String nullSafeToString(short[] array) { + if (array == null) { + return "null"; + } else { + int length = array.length; + if (length == 0) { + return "{}"; + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < length; ++i) { + if (i == 0) { + sb.append("{"); + } else { + sb.append(", "); + } + + sb.append(array[i]); + } + + sb.append("}"); + return sb.toString(); + } + } + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/ReflectionUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ReflectionUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/ReflectionUtils.java rename to cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ReflectionUtils.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/Resource.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/Resource.java new file mode 100644 index 000000000..1a7654848 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/Resource.java @@ -0,0 +1,102 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URL; + +public interface Resource extends InputStreamSource { + /** + * Whether the resource exists on the classpath + * @return + */ + boolean exists(); + + /** + * + * @return + */ + boolean isReadable(); + + /** + * + * @return + */ + boolean isOpen(); + + /** + * + * @return + * @throws IOException + */ + URL getURL() throws IOException; + + /** + * + * @return + * @throws IOException + */ + URI getURI() throws IOException; + + /** + * + * @return + * @throws IOException + */ + File getFile() throws IOException; + + /** + * + * @return + * @throws IOException + */ + long contentLength() throws IOException; + + /** + * + * @return + * @throws IOException + */ + long lastModified() throws IOException; + + /** + * + * @param var1 + * @return + * @throws IOException + */ + Resource createRelative(String var1) throws IOException; + + /** + * + * @return + */ + String getFilename(); + + /** + * + * @return + */ + String getDescription(); +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ResourceUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ResourceUtils.java new file mode 100644 index 000000000..32dc203e0 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/ResourceUtils.java @@ -0,0 +1,193 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import org.nd4j.common.config.ND4JClassLoading; + +import java.io.File; +import java.io.FileNotFoundException; +import java.net.*; +import java.util.Objects; + + +public abstract class ResourceUtils { + public static final String CLASSPATH_URL_PREFIX = "classpath:"; + public static final String FILE_URL_PREFIX = "file:"; + public static final String URL_PROTOCOL_FILE = "file"; + public static final String URL_PROTOCOL_JAR = "jar"; + public static final String URL_PROTOCOL_ZIP = "zip"; + public static final String URL_PROTOCOL_VFSZIP = "vfszip"; + public static final String URL_PROTOCOL_VFS = "vfs"; + public static final String URL_PROTOCOL_WSJAR = "wsjar"; + public static final String URL_PROTOCOL_CODE_SOURCE = "code-source"; + public static final String JAR_URL_SEPARATOR = "!/"; + + public ResourceUtils() {} + + public static boolean isUrl(String resourceLocation) { + if (resourceLocation == null) { + return false; + } else if (resourceLocation.startsWith("classpath:")) { + return true; + } else { + try { + new URL(resourceLocation); + return true; + } catch (MalformedURLException var2) { + return false; + } + } + } + + public static URL getURL(String resourceLocation) throws FileNotFoundException { + Assert.notNull(resourceLocation, "Resource location must not be null"); + if (resourceLocation.startsWith("classpath:")) { + String ex = resourceLocation.substring("classpath:".length()); + URL ex2 = ND4JClassLoading.getNd4jClassloader().getResource(ex); + if (ex2 == null) { + String description = "class path resource [" + ex + "]"; + throw new FileNotFoundException(description + " cannot be resolved to URL because it does not exist"); + } else { + return ex2; + } + } else { + try { + return new URL(resourceLocation); + } catch (MalformedURLException var5) { + try { + return (new File(resourceLocation)).toURI().toURL(); + } catch (MalformedURLException var4) { + throw new FileNotFoundException("Resource location [" + resourceLocation + + "] is neither a URL not a well-formed file path"); + } + } + } + } + + public static File getFile(String resourceLocation) throws FileNotFoundException { + Assert.notNull(resourceLocation, "Resource location must not be null"); + if (resourceLocation.startsWith("classpath:")) { + String ex = resourceLocation.substring("classpath:".length()); + String description = "class path resource [" + ex + "]"; + URL url = ND4JClassLoading.getNd4jClassloader().getResource(ex); + if (url == null) { + throw new FileNotFoundException(description + " cannot be resolved to absolute file path " + + "because it does not reside in the file system"); + } else { + return getFile(url, description); + } + } else { + try { + return getFile(new URL(resourceLocation)); + } catch (MalformedURLException var4) { + return new File(resourceLocation); + } + } + } + + public static File getFile(URL resourceUrl) throws FileNotFoundException { + return getFile(resourceUrl, "URL"); + } + + public static File getFile(URL resourceUrl, String description) throws FileNotFoundException { + Assert.notNull(resourceUrl, "Resource URL must not be null"); + if (!"file".equals(resourceUrl.getProtocol())) { + throw new FileNotFoundException(description + " cannot be resolved to absolute file path " + + "because it does not reside in the file system: " + resourceUrl); + } else { + try { + return new File(toURI(resourceUrl).getSchemeSpecificPart()); + } catch (URISyntaxException var3) { + return new File(resourceUrl.getFile()); + } + } + } + + public static File getFile(URI resourceUri) throws FileNotFoundException { + return getFile(resourceUri, "URI"); + } + + public static File getFile(URI resourceUri, String description) throws FileNotFoundException { + Assert.notNull(resourceUri, "Resource URI must not be null"); + if (!"file".equals(resourceUri.getScheme())) { + throw new FileNotFoundException(description + " cannot be resolved to absolute file path " + + "because it does not reside in the file system: " + resourceUri); + } else { + return new File(resourceUri.getSchemeSpecificPart()); + } + } + + public static boolean isFileURL(URL url) { + String protocol = url.getProtocol(); + return "file".equals(protocol) || protocol.startsWith("vfs"); + } + + public static boolean isJarURL(URL url) { + String protocol = url.getProtocol(); + return "jar".equals(protocol) || "zip".equals(protocol) || "wsjar".equals(protocol) + || "code-source".equals(protocol) && url.getPath().contains("!/"); + } + + public static URL extractJarFileURL(URL jarUrl) throws MalformedURLException { + String urlFile = jarUrl.getFile(); + int separatorIndex = urlFile.indexOf("!/"); + if (separatorIndex != -1) { + String jarFile = urlFile.substring(0, separatorIndex); + + try { + return new URL(jarFile); + } catch (MalformedURLException var5) { + if (!jarFile.startsWith("/")) { + jarFile = "/" + jarFile; + } + + return new URL("file:" + jarFile); + } + } else { + return jarUrl; + } + } + + public static URI toURI(URL url) throws URISyntaxException { + return toURI(url.toString()); + } + + public static URI toURI(String location) throws URISyntaxException { + return new URI(StringUtils.replace(location, " ", "%20")); + } + + public static void useCachesIfNecessary(URLConnection con) { + con.setUseCaches(con.getClass().getSimpleName().startsWith("JNLP")); + } + + public static String classPackageAsResourcePath(Class clazz) { + Objects.requireNonNull(clazz); + + String className = clazz.getName(); + int packageEndIndex = className.lastIndexOf(46); + if (packageEndIndex == -1) { + return ""; + } else { + String packageName = className.substring(0, packageEndIndex); + return packageName.replace('.', '/'); + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/StringUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/StringUtils.java new file mode 100644 index 000000000..9f4fecbec --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/StringUtils.java @@ -0,0 +1,717 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import java.util.*; + + +public abstract class StringUtils { + private static final String FOLDER_SEPARATOR = "/"; + private static final String WINDOWS_FOLDER_SEPARATOR = "\\"; + private static final String TOP_PATH = ".."; + private static final String CURRENT_PATH = "."; + private static final char EXTENSION_SEPARATOR = '.'; + + public StringUtils() {} + + public static boolean isEmpty(Object str) { + return str == null || "".equals(str); + } + + public static boolean hasLength(CharSequence str) { + return str != null && str.length() > 0; + } + + public static boolean hasLength(String str) { + return hasLength((CharSequence) str); + } + + public static boolean hasText(CharSequence str) { + if (!hasLength(str)) { + return false; + } else { + int strLen = str.length(); + + for (int i = 0; i < strLen; ++i) { + if (!Character.isWhitespace(str.charAt(i))) { + return true; + } + } + + return false; + } + } + + public static boolean hasText(String str) { + return hasText((CharSequence) str); + } + + public static boolean containsWhitespace(CharSequence str) { + if (!hasLength(str)) { + return false; + } else { + int strLen = str.length(); + + for (int i = 0; i < strLen; ++i) { + if (Character.isWhitespace(str.charAt(i))) { + return true; + } + } + + return false; + } + } + + public static boolean containsWhitespace(String str) { + return containsWhitespace((CharSequence) str); + } + + public static String trimWhitespace(String str) { + if (!hasLength(str)) { + return str; + } else { + StringBuilder sb = new StringBuilder(str); + + while (sb.length() > 0 && Character.isWhitespace(sb.charAt(0))) { + sb.deleteCharAt(0); + } + + while (sb.length() > 0 && Character.isWhitespace(sb.charAt(sb.length() - 1))) { + sb.deleteCharAt(sb.length() - 1); + } + + return sb.toString(); + } + } + + public static String trimAllWhitespace(String str) { + if (!hasLength(str)) { + return str; + } else { + StringBuilder sb = new StringBuilder(str); + int index = 0; + + while (sb.length() > index) { + if (Character.isWhitespace(sb.charAt(index))) { + sb.deleteCharAt(index); + } else { + ++index; + } + } + + return sb.toString(); + } + } + + public static String trimLeadingWhitespace(String str) { + if (!hasLength(str)) { + return str; + } else { + StringBuilder sb = new StringBuilder(str); + + while (sb.length() > 0 && Character.isWhitespace(sb.charAt(0))) { + sb.deleteCharAt(0); + } + + return sb.toString(); + } + } + + public static String trimTrailingWhitespace(String str) { + if (!hasLength(str)) { + return str; + } else { + StringBuilder sb = new StringBuilder(str); + + while (sb.length() > 0 && Character.isWhitespace(sb.charAt(sb.length() - 1))) { + sb.deleteCharAt(sb.length() - 1); + } + + return sb.toString(); + } + } + + public static String trimLeadingCharacter(String str, char leadingCharacter) { + if (!hasLength(str)) { + return str; + } else { + StringBuilder sb = new StringBuilder(str); + + while (sb.length() > 0 && sb.charAt(0) == leadingCharacter) { + sb.deleteCharAt(0); + } + + return sb.toString(); + } + } + + public static String trimTrailingCharacter(String str, char trailingCharacter) { + if (!hasLength(str)) { + return str; + } else { + StringBuilder sb = new StringBuilder(str); + + while (sb.length() > 0 && sb.charAt(sb.length() - 1) == trailingCharacter) { + sb.deleteCharAt(sb.length() - 1); + } + + return sb.toString(); + } + } + + public static boolean startsWithIgnoreCase(String str, String prefix) { + if (str != null && prefix != null) { + if (str.startsWith(prefix)) { + return true; + } else if (str.length() < prefix.length()) { + return false; + } else { + String lcStr = str.substring(0, prefix.length()).toLowerCase(); + String lcPrefix = prefix.toLowerCase(); + return lcStr.equals(lcPrefix); + } + } else { + return false; + } + } + + public static boolean endsWithIgnoreCase(String str, String suffix) { + if (str != null && suffix != null) { + if (str.endsWith(suffix)) { + return true; + } else if (str.length() < suffix.length()) { + return false; + } else { + String lcStr = str.substring(str.length() - suffix.length()).toLowerCase(); + String lcSuffix = suffix.toLowerCase(); + return lcStr.equals(lcSuffix); + } + } else { + return false; + } + } + + public static boolean substringMatch(CharSequence str, int index, CharSequence substring) { + for (int j = 0; j < substring.length(); ++j) { + int i = index + j; + if (i >= str.length() || str.charAt(i) != substring.charAt(j)) { + return false; + } + } + + return true; + } + + public static int countOccurrencesOf(String str, String sub) { + if (str != null && sub != null && str.length() != 0 && sub.length() != 0) { + int count = 0; + + int idx; + for (int pos = 0; (idx = str.indexOf(sub, pos)) != -1; pos = idx + sub.length()) { + ++count; + } + + return count; + } else { + return 0; + } + } + + public static String replace(String inString, String oldPattern, String newPattern) { + if (hasLength(inString) && hasLength(oldPattern) && newPattern != null) { + StringBuilder sb = new StringBuilder(); + int pos = 0; + int index = inString.indexOf(oldPattern); + + for (int patLen = oldPattern.length(); index >= 0; index = inString.indexOf(oldPattern, pos)) { + sb.append(inString.substring(pos, index)); + sb.append(newPattern); + pos = index + patLen; + } + + sb.append(inString.substring(pos)); + return sb.toString(); + } else { + return inString; + } + } + + public static String delete(String inString, String pattern) { + return replace(inString, pattern, ""); + } + + public static String deleteAny(String inString, String charsToDelete) { + if (hasLength(inString) && hasLength(charsToDelete)) { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < inString.length(); ++i) { + char c = inString.charAt(i); + if (charsToDelete.indexOf(c) == -1) { + sb.append(c); + } + } + + return sb.toString(); + } else { + return inString; + } + } + + public static String quote(String str) { + return str != null ? "\'" + str + "\'" : null; + } + + public static Object quoteIfString(Object obj) { + return obj instanceof String ? quote((String) obj) : obj; + } + + public static String unqualify(String qualifiedName) { + return unqualify(qualifiedName, '.'); + } + + public static String unqualify(String qualifiedName, char separator) { + return qualifiedName.substring(qualifiedName.lastIndexOf(separator) + 1); + } + + public static String capitalize(String str) { + return changeFirstCharacterCase(str, true); + } + + public static String uncapitalize(String str) { + return changeFirstCharacterCase(str, false); + } + + private static String changeFirstCharacterCase(String str, boolean capitalize) { + if (str != null && str.length() != 0) { + StringBuilder sb = new StringBuilder(str.length()); + if (capitalize) { + sb.append(Character.toUpperCase(str.charAt(0))); + } else { + sb.append(Character.toLowerCase(str.charAt(0))); + } + + sb.append(str.substring(1)); + return sb.toString(); + } else { + return str; + } + } + + public static String getFilename(String path) { + if (path == null) { + return null; + } else { + int separatorIndex = path.lastIndexOf("/"); + return separatorIndex != -1 ? path.substring(separatorIndex + 1) : path; + } + } + + public static String getFilenameExtension(String path) { + if (path == null) { + return null; + } else { + int extIndex = path.lastIndexOf(46); + if (extIndex == -1) { + return null; + } else { + int folderIndex = path.lastIndexOf("/"); + return folderIndex > extIndex ? null : path.substring(extIndex + 1); + } + } + } + + public static String stripFilenameExtension(String path) { + if (path == null) { + return null; + } else { + int extIndex = path.lastIndexOf(46); + if (extIndex == -1) { + return path; + } else { + int folderIndex = path.lastIndexOf("/"); + return folderIndex > extIndex ? path : path.substring(0, extIndex); + } + } + } + + public static String applyRelativePath(String path, String relativePath) { + int separatorIndex = path.lastIndexOf("/"); + if (separatorIndex != -1) { + String newPath = path.substring(0, separatorIndex); + if (!relativePath.startsWith("/")) { + newPath = newPath + "/"; + } + + return newPath + relativePath; + } else { + return relativePath; + } + } + + public static String cleanPath(String path) { + if (path == null) { + return null; + } else { + String pathToUse = replace(path, "\\", "/"); + int prefixIndex = pathToUse.indexOf(":"); + String prefix = ""; + if (prefixIndex != -1) { + prefix = pathToUse.substring(0, prefixIndex + 1); + pathToUse = pathToUse.substring(prefixIndex + 1); + } + + if (pathToUse.startsWith("/")) { + prefix = prefix + "/"; + pathToUse = pathToUse.substring(1); + } + + String[] pathArray = delimitedListToStringArray(pathToUse, "/"); + LinkedList pathElements = new LinkedList(); + int tops = 0; + + int i; + for (i = pathArray.length - 1; i >= 0; --i) { + String element = pathArray[i]; + if (!".".equals(element)) { + if ("..".equals(element)) { + ++tops; + } else if (tops > 0) { + --tops; + } else { + pathElements.add(0, element); + } + } + } + + for (i = 0; i < tops; ++i) { + pathElements.add(0, ".."); + } + + return prefix + collectionToDelimitedString(pathElements, "/"); + } + } + + public static boolean pathEquals(String path1, String path2) { + return cleanPath(path1).equals(cleanPath(path2)); + } + + public static Locale parseLocaleString(String localeString) { + String[] parts = tokenizeToStringArray(localeString, "_ ", false, false); + String language = parts.length > 0 ? parts[0] : ""; + String country = parts.length > 1 ? parts[1] : ""; + validateLocalePart(language); + validateLocalePart(country); + String variant = ""; + if (parts.length >= 2) { + int endIndexOfCountryCode = localeString.lastIndexOf(country) + country.length(); + variant = trimLeadingWhitespace(localeString.substring(endIndexOfCountryCode)); + if (variant.startsWith("_")) { + variant = trimLeadingCharacter(variant, '_'); + } + } + + return language.length() > 0 ? new Locale(language, country, variant) : null; + } + + private static void validateLocalePart(String localePart) { + for (int i = 0; i < localePart.length(); ++i) { + char ch = localePart.charAt(i); + if (ch != 95 && ch != 32 && !Character.isLetterOrDigit(ch)) { + throw new IllegalArgumentException("Locale part \"" + localePart + "\" contains invalid characters"); + } + } + + } + + public static String toLanguageTag(Locale locale) { + return locale.getLanguage() + (hasText(locale.getCountry()) ? "-" + locale.getCountry() : ""); + } + + public static String[] addStringToArray(String[] array, String str) { + if (ObjectUtils.isEmpty(array)) { + return new String[] {str}; + } else { + String[] newArr = new String[array.length + 1]; + System.arraycopy(array, 0, newArr, 0, array.length); + newArr[array.length] = str; + return newArr; + } + } + + public static String[] concatenateStringArrays(String[] array1, String[] array2) { + if (ObjectUtils.isEmpty(array1)) { + return array2; + } else if (ObjectUtils.isEmpty(array2)) { + return array1; + } else { + String[] newArr = new String[array1.length + array2.length]; + System.arraycopy(array1, 0, newArr, 0, array1.length); + System.arraycopy(array2, 0, newArr, array1.length, array2.length); + return newArr; + } + } + + public static String[] mergeStringArrays(String[] array1, String[] array2) { + if (ObjectUtils.isEmpty(array1)) { + return array2; + } else if (ObjectUtils.isEmpty(array2)) { + return array1; + } else { + ArrayList result = new ArrayList(); + result.addAll(Arrays.asList(array1)); + String[] arr$ = array2; + int len$ = array2.length; + + for (int i$ = 0; i$ < len$; ++i$) { + String str = arr$[i$]; + if (!result.contains(str)) { + result.add(str); + } + } + + return toStringArray(result); + } + } + + public static String[] sortStringArray(String[] array) { + if (ObjectUtils.isEmpty(array)) { + return new String[0]; + } else { + Arrays.sort(array); + return array; + } + } + + public static String[] toStringArray(Collection collection) { + return collection == null ? null : collection.toArray(new String[collection.size()]); + } + + public static String[] toStringArray(Enumeration enumeration) { + if (enumeration == null) { + return null; + } else { + ArrayList list = Collections.list(enumeration); + return (String[]) list.toArray(new String[list.size()]); + } + } + + public static String[] trimArrayElements(String[] array) { + if (ObjectUtils.isEmpty(array)) { + return new String[0]; + } else { + String[] result = new String[array.length]; + + for (int i = 0; i < array.length; ++i) { + String element = array[i]; + result[i] = element != null ? element.trim() : null; + } + + return result; + } + } + + public static String[] removeDuplicateStrings(String[] array) { + if (ObjectUtils.isEmpty(array)) { + return array; + } else { + TreeSet set = new TreeSet(); + String[] arr$ = array; + int len$ = array.length; + + for (int i$ = 0; i$ < len$; ++i$) { + String element = arr$[i$]; + set.add(element); + } + + return toStringArray(set); + } + } + + public static String[] split(String toSplit, String delimiter) { + if (hasLength(toSplit) && hasLength(delimiter)) { + int offset = toSplit.indexOf(delimiter); + if (offset < 0) { + return null; + } else { + String beforeDelimiter = toSplit.substring(0, offset); + String afterDelimiter = toSplit.substring(offset + delimiter.length()); + return new String[] {beforeDelimiter, afterDelimiter}; + } + } else { + return null; + } + } + + public static Properties splitArrayElementsIntoProperties(String[] array, String delimiter) { + return splitArrayElementsIntoProperties(array, delimiter, null); + } + + public static Properties splitArrayElementsIntoProperties(String[] array, String delimiter, String charsToDelete) { + if (ObjectUtils.isEmpty(array)) { + return null; + } else { + Properties result = new Properties(); + String[] arr$ = array; + int len$ = array.length; + + for (int i$ = 0; i$ < len$; ++i$) { + String element = arr$[i$]; + if (charsToDelete != null) { + element = deleteAny(element, charsToDelete); + } + + String[] splittedElement = split(element, delimiter); + if (splittedElement != null) { + result.setProperty(splittedElement[0].trim(), splittedElement[1].trim()); + } + } + + return result; + } + } + + public static String[] tokenizeToStringArray(String str, String delimiters) { + return tokenizeToStringArray(str, delimiters, true, true); + } + + public static String[] tokenizeToStringArray(String str, String delimiters, boolean trimTokens, + boolean ignoreEmptyTokens) { + if (str == null) { + return null; + } else { + StringTokenizer st = new StringTokenizer(str, delimiters); + ArrayList tokens = new ArrayList(); + + while (st.hasMoreTokens()) { + String token = st.nextToken(); + if (trimTokens) { + token = token.trim(); + } + + if (!ignoreEmptyTokens || token.length() > 0) { + tokens.add(token); + } + } + + return toStringArray(tokens); + } + } + + public static String[] delimitedListToStringArray(String str, String delimiter) { + return delimitedListToStringArray(str, delimiter, null); + } + + public static String[] delimitedListToStringArray(String str, String delimiter, String charsToDelete) { + if (str == null) { + return new String[0]; + } else if (delimiter == null) { + return new String[] {str}; + } else { + ArrayList result = new ArrayList(); + int pos; + if ("".equals(delimiter)) { + for (pos = 0; pos < str.length(); ++pos) { + result.add(deleteAny(str.substring(pos, pos + 1), charsToDelete)); + } + } else { + int delPos; + for (pos = 0; (delPos = str.indexOf(delimiter, pos)) != -1; pos = delPos + delimiter.length()) { + result.add(deleteAny(str.substring(pos, delPos), charsToDelete)); + } + + if (str.length() > 0 && pos <= str.length()) { + result.add(deleteAny(str.substring(pos), charsToDelete)); + } + } + + return toStringArray(result); + } + } + + public static String[] commaDelimitedListToStringArray(String str) { + return delimitedListToStringArray(str, ","); + } + + public static Set commaDelimitedListToSet(String str) { + TreeSet set = new TreeSet(); + String[] tokens = commaDelimitedListToStringArray(str); + String[] arr$ = tokens; + int len$ = tokens.length; + + for (int i$ = 0; i$ < len$; ++i$) { + String token = arr$[i$]; + set.add(token); + } + + return set; + } + + public static String collectionToDelimitedString(Collection coll, String delim, String prefix, String suffix) { + if (CollectionUtils.isEmpty(coll)) { + return ""; + } else { + StringBuilder sb = new StringBuilder(); + Iterator it = coll.iterator(); + + while (it.hasNext()) { + sb.append(prefix).append(it.next()).append(suffix); + if (it.hasNext()) { + sb.append(delim); + } + } + + return sb.toString(); + } + } + + public static String collectionToDelimitedString(Collection coll, String delim) { + return collectionToDelimitedString(coll, delim, "", ""); + } + + public static String collectionToCommaDelimitedString(Collection coll) { + return collectionToDelimitedString(coll, ","); + } + + public static String arrayToDelimitedString(Object[] arr, String delim) { + if (ObjectUtils.isEmpty(arr)) { + return ""; + } else if (arr.length == 1) { + return ObjectUtils.nullSafeToString(arr[0]); + } else { + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < arr.length; ++i) { + if (i > 0) { + sb.append(delim); + } + + sb.append(arr[i]); + } + + return sb.toString(); + } + } + + public static String arrayToCommaDelimitedString(Object[] arr) { + return arrayToDelimitedString(arr, ","); + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/VfsResource.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/VfsResource.java new file mode 100644 index 000000000..1174d7131 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/VfsResource.java @@ -0,0 +1,104 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; + +public class VfsResource extends AbstractResource { + private final Object resource; + + public VfsResource(Object resources) { + Assert.notNull(resources, "VirtualFile must not be null"); + this.resource = resources; + } + + public InputStream getInputStream() throws IOException { + return VfsUtils.getInputStream(this.resource); + } + + public boolean exists() { + return VfsUtils.exists(this.resource); + } + + public boolean isReadable() { + return VfsUtils.isReadable(this.resource); + } + + public URL getURL() throws IOException { + try { + return VfsUtils.getURL(this.resource); + } catch (Exception var2) { + throw new IOException("Failed to obtain URL for file " + this.resource, var2); + } + } + + public URI getURI() throws IOException { + try { + return VfsUtils.getURI(this.resource); + } catch (Exception var2) { + throw new IOException("Failed to obtain URI for " + this.resource, var2); + } + } + + public File getFile() throws IOException { + return VfsUtils.getFile(this.resource); + } + + public long contentLength() throws IOException { + return VfsUtils.getSize(this.resource); + } + + public long lastModified() throws IOException { + return VfsUtils.getLastModified(this.resource); + } + + public Resource createRelative(String relativePath) throws IOException { + if (!relativePath.startsWith(".") && relativePath.contains("/")) { + try { + return new VfsResource(VfsUtils.getChild(this.resource, relativePath)); + } catch (IOException var3) { + + } + } + + return new VfsResource(VfsUtils.getRelative(new URL(this.getURL(), relativePath))); + } + + public String getFilename() { + return VfsUtils.getName(this.resource); + } + + public String getDescription() { + return this.resource.toString(); + } + + public boolean equals(Object obj) { + return obj == this || obj instanceof VfsResource && this.resource.equals(((VfsResource) obj).resource); + } + + public int hashCode() { + return this.resource.hashCode(); + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/common/io/VfsUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/VfsUtils.java similarity index 100% rename from nd4j/nd4j-common/src/main/java/org/nd4j/common/io/VfsUtils.java rename to cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/io/VfsUtils.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/FileBatch.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/FileBatch.java new file mode 100644 index 000000000..a7147d798 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/FileBatch.java @@ -0,0 +1,160 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.loader; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +@AllArgsConstructor +@Data +public class FileBatch implements Serializable { + /** + * Name of the file in the zip file that contains the original paths/filenames + */ + public static final String ORIGINAL_PATHS_FILENAME = "originalUris.txt"; + + private final List fileBytes; + private final List originalUris; + + /** + * Read a FileBatch from the specified file. This method assumes the FileBatch was previously saved to + * zip format using {@link #writeAsZip(File)} or {@link #writeAsZip(OutputStream)} + * + * @param file File to read from + * @return The loaded FileBatch + * @throws IOException If an error occurs during reading + */ + public static FileBatch readFromZip(File file) throws IOException { + try (FileInputStream fis = new FileInputStream(file)) { + return readFromZip(fis); + } + } + + /** + * Read a FileBatch from the specified input stream. This method assumes the FileBatch was previously saved to + * zip format using {@link #writeAsZip(File)} or {@link #writeAsZip(OutputStream)} + * + * @param is Input stream to read from + * @return The loaded FileBatch + * @throws IOException If an error occurs during reading + */ + public static FileBatch readFromZip(InputStream is) throws IOException { + String originalUris = null; + Map bytesMap = new HashMap<>(); + try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(is))) { + ZipEntry ze; + while ((ze = zis.getNextEntry()) != null) { + String name = ze.getName(); + byte[] bytes = IOUtils.toByteArray(zis); + if (name.equals(ORIGINAL_PATHS_FILENAME)) { + originalUris = new String(bytes, 0, bytes.length, StandardCharsets.UTF_8); + } else { + int idxSplit = name.indexOf("_"); + int idxSplit2 = name.indexOf("."); + int fileIdx = Integer.parseInt(name.substring(idxSplit + 1, idxSplit2)); + bytesMap.put(fileIdx, bytes); + } + } + } + + List list = new ArrayList<>(bytesMap.size()); + for (int i = 0; i < bytesMap.size(); i++) { + list.add(bytesMap.get(i)); + } + + List origPaths = Arrays.asList(originalUris.split("\n")); + return new FileBatch(list, origPaths); + } + + /** + * Create a FileBatch from the specified files + * + * @param files Files to create the FileBatch from + * @return The created FileBatch + * @throws IOException If an error occurs during reading of the file content + */ + public static FileBatch forFiles(File... files) throws IOException { + return forFiles(Arrays.asList(files)); + } + + /** + * Create a FileBatch from the specified files + * + * @param files Files to create the FileBatch from + * @return The created FileBatch + * @throws IOException If an error occurs during reading of the file content + */ + public static FileBatch forFiles(List files) throws IOException { + List origPaths = new ArrayList<>(files.size()); + List bytes = new ArrayList<>(files.size()); + for (File f : files) { + bytes.add(FileUtils.readFileToByteArray(f)); + origPaths.add(f.toURI().toString()); + } + return new FileBatch(bytes, origPaths); + } + + /** + * Write the FileBatch to the specified File, in zip file format + * + * @param f File to write to + * @throws IOException If an error occurs during writing + */ + public void writeAsZip(File f) throws IOException { + writeAsZip(new FileOutputStream(f)); + } + + /** + * @param os Write the FileBatch to the specified output stream, in zip file format + * @throws IOException If an error occurs during writing + */ + public void writeAsZip(OutputStream os) throws IOException { + try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(os))) { + + //Write original paths as a text file: + ZipEntry ze = new ZipEntry(ORIGINAL_PATHS_FILENAME); + String originalUrisJoined = StringUtils.join(originalUris, "\n"); //Java String.join is Java 8 + zos.putNextEntry(ze); + zos.write(originalUrisJoined.getBytes(StandardCharsets.UTF_8)); + + for (int i = 0; i < fileBytes.size(); i++) { + String ext = FilenameUtils.getExtension(originalUris.get(i)); + if (ext == null || ext.isEmpty()) + ext = "bin"; + String name = "file_" + i + "." + ext; + ze = new ZipEntry(name); + zos.putNextEntry(ze); + zos.write(fileBytes.get(i)); + } + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/Loader.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/Loader.java new file mode 100644 index 000000000..e0e0a6823 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/Loader.java @@ -0,0 +1,29 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.loader; + +import java.io.IOException; +import java.io.Serializable; + +public interface Loader extends Serializable { + + T load(Source source) throws IOException; +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSource.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSource.java new file mode 100644 index 000000000..2955fb51a --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSource.java @@ -0,0 +1,37 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.loader; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.io.*; + +@AllArgsConstructor +public class LocalFileSource implements Source { + @Getter + private String path; + + @Override + public InputStream getInputStream() throws IOException { + return new BufferedInputStream(new FileInputStream(new File(path))); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSourceFactory.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSourceFactory.java new file mode 100644 index 000000000..a1c25af66 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/LocalFileSourceFactory.java @@ -0,0 +1,28 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.loader; + +public class LocalFileSourceFactory implements SourceFactory { + @Override + public Source getSource(String path) { + return new LocalFileSource(path); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/Source.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/Source.java new file mode 100644 index 000000000..7657391f7 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/Source.java @@ -0,0 +1,30 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.loader; + +import java.io.IOException; +import java.io.InputStream; + +public interface Source { + InputStream getInputStream() throws IOException; + + String getPath(); +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/SourceFactory.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/SourceFactory.java new file mode 100644 index 000000000..4d94be995 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/loader/SourceFactory.java @@ -0,0 +1,27 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.loader; + +import java.io.Serializable; + +public interface SourceFactory extends Serializable { + Source getSource(String path); +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Atomic.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Atomic.java new file mode 100644 index 000000000..fed1b1405 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Atomic.java @@ -0,0 +1,127 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.NoArgsConstructor; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Objects; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * + * @param + */ +@NoArgsConstructor +public class Atomic implements Serializable { + private volatile T value; + private transient ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + + public Atomic(T initialValue) { + this.value = initialValue; + } + + /** + * This method assigns new value + * @param value + */ + public void set(T value) { + try { + lock.writeLock().lock(); + + this.value = value; + } finally { + lock.writeLock().unlock(); + } + } + + /** + * This method returns current value + * @return + */ + public T get() { + try { + lock.readLock().lock(); + + return this.value; + } finally { + lock.readLock().unlock(); + } + } + + + + /** + * This method implements compare-and-swap + * + * @param expected + * @param newValue + * @return true if value was swapped, false otherwise + */ + public boolean cas(T expected, T newValue) { + try { + lock.writeLock().lock(); + + if (Objects.equals(value, expected)) { + this.value = newValue; + return true; + } else + return false; + } finally { + lock.writeLock().unlock(); + } + } + + private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + lock = new ReentrantReadWriteLock(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Atomic atomic = (Atomic) o; + try { + this.lock.readLock().lock(); + atomic.lock.readLock().lock(); + + return Objects.equals(this.value, atomic.value); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + atomic.lock.readLock().unlock(); + this.lock.readLock().unlock(); + } + } + + @Override + public int hashCode() { + try { + this.lock.readLock().lock(); + + return Objects.hash(value); + } finally { + this.lock.readLock().unlock(); + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/AtomicBoolean.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/AtomicBoolean.java new file mode 100644 index 000000000..62a61c185 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/AtomicBoolean.java @@ -0,0 +1,48 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +public class AtomicBoolean extends java.util.concurrent.atomic.AtomicBoolean { + + public AtomicBoolean(boolean initialValue){ + super(initialValue); + } + + public AtomicBoolean(){ + this(false); + } + + @Override + public boolean equals(Object o){ + if(o instanceof AtomicBoolean){ + return get() == ((AtomicBoolean)o).get(); + } else if(o instanceof Boolean){ + return get() == ((Boolean)o); + } + return false; + } + + @Override + public int hashCode(){ + return get() ? 1 : 0; + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/AtomicDouble.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/AtomicDouble.java new file mode 100644 index 000000000..fb9fb79a0 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/AtomicDouble.java @@ -0,0 +1,59 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import org.nd4j.common.primitives.serde.JsonDeserializerAtomicDouble; +import org.nd4j.common.primitives.serde.JsonSerializerAtomicDouble; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize(using = JsonSerializerAtomicDouble.class) +@JsonDeserialize(using = JsonDeserializerAtomicDouble.class) +public class AtomicDouble extends com.google.common.util.concurrent.AtomicDouble { + + public AtomicDouble(){ + this(0.0); + } + + public AtomicDouble(@JsonProperty("value") double value){ + super(value); + } + + public AtomicDouble(float value){ + this((double)value); + } + + @Override + public boolean equals(Object o){ + //NOTE: com.google.common.util.concurrent.AtomicDouble extends Number, hence this class extends number + if(o instanceof Number){ + return get() == ((Number)o).doubleValue(); + } + return false; + } + + @Override + public int hashCode(){ + //return Double.hashCode(get()); //Java 8+ + return Double.valueOf(get()).hashCode(); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Counter.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Counter.java new file mode 100644 index 000000000..746bd0105 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Counter.java @@ -0,0 +1,327 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +public class Counter implements Serializable { + private static final long serialVersionUID = 119L; + + protected ConcurrentHashMap map = new ConcurrentHashMap<>(); + protected AtomicDouble totalCount = new AtomicDouble(0); + protected AtomicBoolean dirty = new AtomicBoolean(false); + + public Counter() { + + } + + public double getCount(T element) { + AtomicDouble t = map.get(element); + if (t == null) + return 0.0; + + return t.get(); + } + + public void incrementCount(T element, double inc) { + AtomicDouble t = map.get(element); + if (t != null) + t.addAndGet(inc); + else { + map.put(element, new AtomicDouble(inc)); + } + + totalCount.addAndGet(inc); + } + + /** + * This method will increment all elements in collection + * + * @param elements + * @param inc + */ + public void incrementAll(Collection elements, double inc) { + for (T element: elements) { + incrementCount(element, inc); + } + } + + /** + * This method will increment counts of this counter by counts from other counter + * @param other + */ + public void incrementAll(Counter other) { + for (T2 element: other.keySet()) { + double cnt = other.getCount(element); + incrementCount(element, cnt); + } + } + + /** + * This method returns probability of given element + * + * @param element + * @return + */ + public double getProbability(T element) { + if (totalCount() <= 0.0) + throw new IllegalStateException("Can't calculate probability with empty counter"); + + return getCount(element) / totalCount(); + } + + /** + * This method sets new counter value for given element + * + * @param element element to be updated + * @param count new counter value + * @return previous value + */ + public double setCount(T element, double count) { + AtomicDouble t = map.get(element); + if (t != null) { + double val = t.getAndSet(count); + dirty.set(true); + return val; + } else { + map.put(element, new AtomicDouble(count)); + totalCount.addAndGet(count); + return 0; + } + + } + + /** + * This method returns Set of elements used in this counter + * + * @return + */ + public Set keySet() { + return map.keySet(); + } + + /** + * This method returns TRUE if counter has no elements, FALSE otherwise + * + * @return + */ + public boolean isEmpty() { + return map.size() == 0; + } + + /** + * This method returns Set of this counter + * @return + */ + public Set> entrySet() { + return map.entrySet(); + } + + /** + * This method returns List of elements, sorted by their counts + * @return + */ + public List keySetSorted() { + List result = new ArrayList<>(); + + PriorityQueue> pq = asPriorityQueue(); + while (!pq.isEmpty()) { + result.add(pq.poll().getFirst()); + } + + return result; + } + + /** + * This method will apply normalization to counter values and totals. + */ + public void normalize() { + for (T key : keySet()) { + setCount(key, getCount(key) / totalCount.get()); + } + + rebuildTotals(); + } + + protected void rebuildTotals() { + totalCount.set(0); + for (T key : keySet()) { + totalCount.addAndGet(getCount(key)); + } + + dirty.set(false); + } + + /** + * This method returns total sum of counter values + * @return + */ + public double totalCount() { + if (dirty.get()) + rebuildTotals(); + + return totalCount.get(); + } + + /** + * This method removes given key from counter + * + * @param element + * @return counter value + */ + public double removeKey(T element) { + AtomicDouble v = map.remove(element); + dirty.set(true); + + if (v != null) + return v.get(); + else + return 0.0; + } + + /** + * This method returns element with highest counter value + * + * @return + */ + public T argMax() { + double maxCount = -Double.MAX_VALUE; + T maxKey = null; + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue().get() > maxCount || maxKey == null) { + maxKey = entry.getKey(); + maxCount = entry.getValue().get(); + } + } + return maxKey; + } + + /** + * This method will remove all elements with counts below given threshold from counter + * @param threshold + */ + public void dropElementsBelowThreshold(double threshold) { + Iterator iterator = keySet().iterator(); + while (iterator.hasNext()) { + T element = iterator.next(); + double val = map.get(element).get(); + if (val < threshold) { + iterator.remove(); + dirty.set(true); + } + } + + } + + /** + * This method checks, if element exist in this counter + * + * @param element + * @return + */ + public boolean containsElement(T element) { + return map.containsKey(element); + } + + /** + * This method effectively resets counter to empty state + */ + public void clear() { + map.clear(); + totalCount.set(0.0); + dirty.set(false); + } + + @Override + public boolean equals(Object o){ + if(!(o instanceof Counter)) + return false; + Counter c2 = (Counter)o; + return map.equals(c2.map); + } + + @Override + public int hashCode(){ + return map.hashCode(); + } + + /** + * Returns total number of tracked elements + * + * @return + */ + public int size() { + return map.size(); + } + + /** + * This method removes all elements except of top N by counter values + * @param N + */ + public void keepTopNElements(int N){ + PriorityQueue> queue = asPriorityQueue(); + clear(); + for (int e = 0; e < N; e++) { + Pair pair = queue.poll(); + if (pair != null) + incrementCount(pair.getFirst(), pair.getSecond()); + } + } + + + public PriorityQueue> asPriorityQueue() { + PriorityQueue> pq = new PriorityQueue<>(Math.max(1,map.size()), new PairComparator()); + for (Map.Entry entry : map.entrySet()) { + pq.add(Pair.create(entry.getKey(), entry.getValue().get())); + } + + return pq; + } + + + public PriorityQueue> asReversedPriorityQueue() { + PriorityQueue> pq = new PriorityQueue<>(Math.max(1,map.size()), new ReversedPairComparator()); + for (Map.Entry entry : map.entrySet()) { + pq.add(Pair.create(entry.getKey(), entry.getValue().get())); + } + + return pq; + } + + public class PairComparator implements Comparator> { + + @Override + public int compare(Pair o1, Pair o2) { + return Double.compare(o2.value, o1.value); + } + } + + public class ReversedPairComparator implements Comparator> { + + @Override + public int compare(Pair o1, Pair o2) { + return Double.compare(o1.value, o2.value); + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/CounterMap.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/CounterMap.java new file mode 100644 index 000000000..1cc6758e6 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/CounterMap.java @@ -0,0 +1,252 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.EqualsAndHashCode; + +import java.io.Serializable; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@EqualsAndHashCode +public class CounterMap implements Serializable{ + private static final long serialVersionUID = 119L; + + protected Map> maps = new ConcurrentHashMap<>(); + + public CounterMap() { + + } + + /** + * This method checks if this CounterMap has any values stored + * + * @return + */ + public boolean isEmpty() { + return maps.isEmpty(); + } + + /** + * This method checks if this CounterMap has any values stored for a given first element + * + * @param element + * @return + */ + public boolean isEmpty(F element){ + if (isEmpty()) + return true; + + Counter m = maps.get(element); + if (m == null) + return true; + else + return m.isEmpty(); + } + + /** + * This method will increment values of this counter, by counts of other counter + * + * @param other + */ + public void incrementAll(CounterMap other) { + for (Map.Entry> entry : other.maps.entrySet()) { + F key = entry.getKey(); + Counter innerCounter = entry.getValue(); + for (Map.Entry innerEntry : innerCounter.entrySet()) { + S value = innerEntry.getKey(); + incrementCount(key, value, innerEntry.getValue().get()); + } + } + } + + /** + * This method will increment counts for a given first/second pair + * + * @param first + * @param second + * @param inc + */ + public void incrementCount(F first, S second, double inc) { + Counter counter = maps.get(first); + if (counter == null) { + counter = new Counter(); + maps.put(first, counter); + } + + counter.incrementCount(second, inc); + } + + /** + * This method returns counts for a given first/second pair + * + * @param first + * @param second + * @return + */ + public double getCount(F first, S second) { + Counter counter = maps.get(first); + if (counter == null) + return 0.0; + + return counter.getCount(second); + } + + /** + * This method allows you to set counter value for a given first/second pair + * + * @param first + * @param second + * @param value + * @return + */ + public double setCount(F first, S second, double value) { + Counter counter = maps.get(first); + if (counter == null) { + counter = new Counter(); + maps.put(first, counter); + } + + return counter.setCount(second, value); + } + + /** + * This method returns pair of elements with a max value + * + * @return + */ + public Pair argMax() { + Double maxCount = -Double.MAX_VALUE; + Pair maxKey = null; + for (Map.Entry> entry : maps.entrySet()) { + Counter counter = entry.getValue(); + S localMax = counter.argMax(); + if (counter.getCount(localMax) > maxCount || maxKey == null) { + maxKey = new Pair(entry.getKey(), localMax); + maxCount = counter.getCount(localMax); + } + } + return maxKey; + } + + /** + * This method purges all counters + */ + public void clear() { + maps.clear(); + } + + /** + * This method purges counter for a given first element + * @param element + */ + public void clear(F element) { + Counter s = maps.get(element); + if (s != null) + s.clear(); + } + + /** + * This method returns Set of all first elements + * @return + */ + public Set keySet() { + return maps.keySet(); + } + + /** + * This method returns counter for a given first element + * + * @param first + * @return + */ + public Counter getCounter(F first) { + return maps.get(first); + } + + /** + * This method returns Iterator of all first/second pairs stored in this counter + * + * @return + */ + public Iterator> getIterator() { + return new Iterator>() { + + Iterator outerIt; + Iterator innerIt; + F curKey; + + { + outerIt = keySet().iterator(); + } + + private boolean hasInside() { + if (innerIt == null || !innerIt.hasNext()) { + if (!outerIt.hasNext()) { + return false; + } + curKey = outerIt.next(); + innerIt = getCounter(curKey).keySet().iterator(); + } + return true; + } + + public boolean hasNext() { + return hasInside(); + } + + public Pair next() { + hasInside(); + if (curKey == null) + throw new RuntimeException("Outer element can't be null"); + + return Pair.makePair(curKey, innerIt.next()); + } + + public void remove() { + // + } + }; + } + + /** + * This method returns number of First elements in this CounterMap + * @return + */ + public int size() { + return maps.size(); + } + + /** + * This method returns total number of elements in this CounterMap + * @return + */ + public int totalSize() { + int size = 0; + for (F first: keySet()) { + size += getCounter(first).size(); + } + + return size; + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutablePair.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutablePair.java new file mode 100644 index 000000000..b077608eb --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutablePair.java @@ -0,0 +1,72 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.*; + +import java.io.Serializable; + +@AllArgsConstructor +@Data +@Builder +public class ImmutablePair implements Serializable { + private static final long serialVersionUID = 119L; + + protected ImmutablePair() { + // + } + + @Setter(AccessLevel.NONE) protected K key; + @Setter(AccessLevel.NONE) protected V value; + + public K getLeft() { + return key; + } + + public V getRight() { + return value; + } + + public K getFirst() { + return key; + } + + public V getSecond() { + return value; + } + + + public static ImmutablePair of(T key, E value) { + return new ImmutablePair(key, value); + } + + public static ImmutablePair makePair(T key, E value) { + return new ImmutablePair(key, value); + } + + public static ImmutablePair create(T key, E value) { + return new ImmutablePair(key, value); + } + + public static ImmutablePair pairOf(T key, E value) { + return new ImmutablePair(key, value); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableQuad.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableQuad.java new file mode 100644 index 000000000..fec871d8b --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableQuad.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.*; + +import java.io.Serializable; + +@Data +@AllArgsConstructor +@Builder +public class ImmutableQuad implements Serializable { + private static final long serialVersionUID = 119L; + + @Setter(AccessLevel.NONE) protected F first; + @Setter(AccessLevel.NONE) protected S second; + @Setter(AccessLevel.NONE) protected T third; + @Setter(AccessLevel.NONE) protected O fourth; + + public static ImmutableQuad quadOf(F first, S second, T third, O fourth) { + return new ImmutableQuad(first, second, third, fourth); + } + + public static ImmutableQuad of(F first, S second, T third, O fourth) { + return new ImmutableQuad(first, second, third, fourth); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableTriple.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableTriple.java new file mode 100644 index 000000000..c0f8af9d1 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/ImmutableTriple.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.*; + +import java.io.Serializable; + +@Data +@AllArgsConstructor +@Builder +public class ImmutableTriple implements Serializable { + private static final long serialVersionUID = 119L; + + protected ImmutableTriple() { + + } + + @Setter(AccessLevel.NONE) protected F first; + @Setter(AccessLevel.NONE) protected S second; + @Setter(AccessLevel.NONE) protected T third; + + + public F getLeft() { + return first; + } + + public S getMiddle() { + return second; + } + + public T getRight() { + return third; + } + + public static ImmutableTriple tripleOf(F first, S second, T third) { + return new ImmutableTriple(first, second, third); + } + + public static ImmutableTriple of(F first, S second, T third) { + return new ImmutableTriple(first, second, third); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Optional.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Optional.java new file mode 100644 index 000000000..ba947cacd --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Optional.java @@ -0,0 +1,114 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.EqualsAndHashCode; +import lombok.NonNull; + +import java.util.NoSuchElementException; + +@EqualsAndHashCode +public class Optional { + private static final Optional EMPTY = new Optional(); + + private final T value; + + private Optional(){ + this(null); + } + + private Optional(T value){ + this.value = value; + } + + /** + * Returns an empty Optional instance. No value is present for this Optional. + * + */ + public static Optional empty(){ + return (Optional)EMPTY; + } + + /** + * Returns an Optional with the specified present non-null value. + * + * @param value the value to be present, which must be non-null + * @return an Optional with the value present + */ + public static Optional of(@NonNull T value){ + return new Optional<>(value); + } + + /** + * Returns an Optional describing the specified value, if non-null, otherwise returns an empty Optional. + * + * @param value the possibly-null value to describe + * @return an Optional with a present value if the specified value is non-null, otherwise an empty Optional + */ + public static Optional ofNullable(T value){ + if(value == null){ + return empty(); + } + return new Optional<>(value); + } + + /** + * If a value is present in this Optional, returns the value, otherwise throws NoSuchElementException. + * + * @return the non-null value held by this Optional + * @throws NoSuchElementException - if there is no value present + */ + public T get(){ + if (!isPresent()) { + throw new NoSuchElementException("Optional is empty"); + } + return value; + } + + /** + * Return true if there is a value present, otherwise false. + * + * @return true if there is a value present, otherwise false + */ + public boolean isPresent(){ + return value != null; + } + + /** + * Return the value if present, otherwise return other. + * + * @param other the value to be returned if there is no value present, may be null + * @return + */ + public T orElse(T other){ + if(isPresent()){ + return get(); + } + return other; + } + + public String toString(){ + if(isPresent()){ + return "Optional(" + value.toString() + ")"; + } + return "Optional()"; + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Pair.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Pair.java new file mode 100644 index 000000000..223f37fa4 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Pair.java @@ -0,0 +1,95 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.Arrays; +import org.nd4j.common.base.Preconditions; + +@AllArgsConstructor +@Data +@NoArgsConstructor +@Builder +public class Pair implements Serializable { + private static final long serialVersionUID = 119L; + + protected K key; + protected V value; + + @Override + public String toString() { + return "Pair{" + + "key=" + (key instanceof int[] ? Arrays.toString((int[]) key) : key) + + ", value=" + (value instanceof int[] ? Arrays.toString((int[]) value) : value) + + '}'; + } + + public K getLeft() { + return key; + } + + public V getRight() { + return value; + } + + public K getFirst() { + return key; + } + + public V getSecond() { + return value; + } + + public void setFirst(K first) { + key = first; + } + + public void setSecond(V second) { + value = second; + } + + public static Pair of(T key, E value) { + return new Pair(key, value); + } + + public static Pair makePair(T key, E value) { + return new Pair(key, value); + } + + public static Pair create(T key, E value) { + return new Pair(key, value); + } + + public static Pair pairOf(T key, E value) { + return new Pair(key, value); + } + + public static Pair fromArray(T[] arr){ + Preconditions.checkArgument(arr.length == 2, + "Can only create a pair from an array with two values, got %s", arr.length); + return new Pair<>(arr[0], arr[1]); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Quad.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Quad.java new file mode 100644 index 000000000..50ff187b7 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Quad.java @@ -0,0 +1,49 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class Quad implements Serializable { + private static final long serialVersionUID = 119L; + + protected F first; + protected S second; + protected T third; + protected O fourth; + + public static Quad quadOf(F first, S second, T third, O fourth) { + return new Quad<>(first, second, third, fourth); + } + + public static Quad of(F first, S second, T third, O fourth) { + return new Quad<>(first, second, third, fourth); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/SynchronizedObject.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/SynchronizedObject.java new file mode 100644 index 000000000..3b76c1f41 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/SynchronizedObject.java @@ -0,0 +1,67 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import java.io.Serializable; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class SynchronizedObject implements Serializable { + protected T value; + protected transient ReentrantReadWriteLock lock; + + public SynchronizedObject() { + lock = new ReentrantReadWriteLock(); + } + + public SynchronizedObject(T value) { + this(); + + this.set(value); + } + + /** + * This method returns stored value via read lock + * @return + */ + public final T get() { + try { + lock.readLock().lock(); + + return value; + } finally { + lock.readLock().unlock(); + } + } + + /** + * This method updates stored value via write lock + * @param value + */ + public final void set(T value) { + try { + lock.writeLock().lock(); + + this.value = value; + } finally { + lock.writeLock().unlock(); + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Triple.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Triple.java new file mode 100644 index 000000000..60dde2b25 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/Triple.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class Triple implements Serializable { + private static final long serialVersionUID = 119L; + + protected F first; + protected S second; + protected T third; + + + public F getLeft() { + return first; + } + + public S getMiddle() { + return second; + } + + public T getRight() { + return third; + } + + public static Triple tripleOf(F first, S second, T third) { + return new Triple<>(first, second, third); + } + + public static Triple of(F first, S second, T third) { + return new Triple<>(first, second, third); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicBoolean.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicBoolean.java new file mode 100644 index 000000000..6c807feea --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicBoolean.java @@ -0,0 +1,39 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives.serde; + +import org.nd4j.common.primitives.AtomicBoolean; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; + +public class JsonDeserializerAtomicBoolean extends JsonDeserializer { + @Override + public AtomicBoolean deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + boolean value = node.asBoolean(); + return new AtomicBoolean(value); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicDouble.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicDouble.java new file mode 100644 index 000000000..d777b0072 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonDeserializerAtomicDouble.java @@ -0,0 +1,39 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives.serde; + +import org.nd4j.common.primitives.AtomicDouble; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; + +public class JsonDeserializerAtomicDouble extends JsonDeserializer { + @Override + public AtomicDouble deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + double value = node.asDouble(); + return new AtomicDouble(value); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicBoolean.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicBoolean.java new file mode 100644 index 000000000..c10f1bc95 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicBoolean.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives.serde; + +import org.nd4j.common.primitives.AtomicBoolean; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +public class JsonSerializerAtomicBoolean extends JsonSerializer { + @Override + public void serialize(AtomicBoolean atomicDouble, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException { + jsonGenerator.writeBoolean(atomicDouble.get()); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicDouble.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicDouble.java new file mode 100644 index 000000000..1f9041ccd --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/primitives/serde/JsonSerializerAtomicDouble.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives.serde; + +import org.nd4j.common.primitives.AtomicDouble; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +public class JsonSerializerAtomicDouble extends JsonSerializer { + @Override + public void serialize(AtomicDouble atomicDouble, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException { + jsonGenerator.writeNumber(atomicDouble.doubleValue()); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Downloader.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Downloader.java new file mode 100644 index 000000000..ae8e66967 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Downloader.java @@ -0,0 +1,155 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.nd4j.common.util.ArchiveUtils; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; + +@Slf4j +public class Downloader { + /** + * Default connection timeout in milliseconds when using {@link FileUtils#copyURLToFile(URL, File, int, int)} + */ + public static final int DEFAULT_CONNECTION_TIMEOUT = 60000; + /** + * Default read timeout in milliseconds when using {@link FileUtils#copyURLToFile(URL, File, int, int)} + */ + public static final int DEFAULT_READ_TIMEOUT = 60000; + + private Downloader(){ } + + /** + * As per {@link #download(String, URL, File, String, int, int, int)} with the connection and read timeouts + * set to their default values - {@link #DEFAULT_CONNECTION_TIMEOUT} and {@link #DEFAULT_READ_TIMEOUT} respectively + */ + public static void download(String name, URL url, File f, String targetMD5, int maxTries) throws IOException { + download(name, url, f, targetMD5, maxTries, DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT); + } + + /** + * Download the specified URL to the specified file, and verify that the target MD5 matches + * + * @param name Name (mainly for providing useful exceptions) + * @param url URL to download + * @param f Destination file + * @param targetMD5 Expected MD5 for file + * @param maxTries Maximum number of download attempts before failing and throwing an exception + * @param connectionTimeout connection timeout in milliseconds, as used by {@link org.apache.commons.io.FileUtils#copyURLToFile(URL, File, int, int)} + * @param readTimeout read timeout in milliseconds, as used by {@link org.apache.commons.io.FileUtils#copyURLToFile(URL, File, int, int)} + * @throws IOException If an error occurs during downloading + */ + public static void download(String name, URL url, File f, String targetMD5, int maxTries, int connectionTimeout, int readTimeout) throws IOException { + download(name, url, f, targetMD5, maxTries, 0, connectionTimeout, readTimeout); + } + + private static void download(String name, URL url, File f, String targetMD5, int maxTries, int attempt, int connectionTimeout, int readTimeout) throws IOException { + boolean isCorrectFile = f.exists() && f.isFile() && checkMD5OfFile(targetMD5, f); + if (attempt < maxTries) { + if(!isCorrectFile) { + FileUtils.copyURLToFile(url, f, connectionTimeout, readTimeout); + if (!checkMD5OfFile(targetMD5, f)) { + f.delete(); + download(name, url, f, targetMD5, maxTries, attempt + 1, connectionTimeout, readTimeout); + } + } + } else if (!isCorrectFile) { + //Too many attempts + throw new IOException("Could not download " + name + " from " + url + "\n properly despite trying " + maxTries + + " times, check your connection."); + } + } + + /** + * As per {@link #downloadAndExtract(String, URL, File, File, String, int, int, int)} with the connection and read timeouts + * * set to their default values - {@link #DEFAULT_CONNECTION_TIMEOUT} and {@link #DEFAULT_READ_TIMEOUT} respectively + */ + public static void downloadAndExtract(String name, URL url, File f, File extractToDir, String targetMD5, int maxTries) throws IOException { + downloadAndExtract(name, url, f, extractToDir, targetMD5, maxTries, DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT); + } + + /** + * Download the specified URL to the specified file, verify that the MD5 matches, and then extract it to the specified directory.
+ * Note that the file must be an archive, with the correct file extension: .zip, .jar, .tar.gz, .tgz or .gz + * + * @param name Name (mainly for providing useful exceptions) + * @param url URL to download + * @param f Destination file + * @param extractToDir Destination directory to extract all files + * @param targetMD5 Expected MD5 for file + * @param maxTries Maximum number of download attempts before failing and throwing an exception + * @param connectionTimeout connection timeout in milliseconds, as used by {@link org.apache.commons.io.FileUtils#copyURLToFile(URL, File, int, int)} + * @param readTimeout read timeout in milliseconds, as used by {@link org.apache.commons.io.FileUtils#copyURLToFile(URL, File, int, int)} + * @throws IOException If an error occurs during downloading + */ + public static void downloadAndExtract(String name, URL url, File f, File extractToDir, String targetMD5, int maxTries, + int connectionTimeout, int readTimeout) throws IOException { + downloadAndExtract(0, maxTries, name, url, f, extractToDir, targetMD5, connectionTimeout, readTimeout); + } + + private static void downloadAndExtract(int attempt, int maxTries, String name, URL url, File f, File extractToDir, + String targetMD5, int connectionTimeout, int readTimeout) throws IOException { + boolean isCorrectFile = f.exists() && f.isFile() && checkMD5OfFile(targetMD5, f); + if (attempt < maxTries) { + if(!isCorrectFile) { + FileUtils.copyURLToFile(url, f, connectionTimeout, readTimeout); + if (!checkMD5OfFile(targetMD5, f)) { + f.delete(); + downloadAndExtract(attempt + 1, maxTries, name, url, f, extractToDir, targetMD5, connectionTimeout, readTimeout); + } + } + // try extracting + try{ + ArchiveUtils.unzipFileTo(f.getAbsolutePath(), extractToDir.getAbsolutePath(), false); + } catch (Throwable t){ + log.warn("Error extracting {} files from file {} - retrying...", name, f.getAbsolutePath(), t); + f.delete(); + downloadAndExtract(attempt + 1, maxTries, name, url, f, extractToDir, targetMD5, connectionTimeout, readTimeout); + } + } else if (!isCorrectFile) { + //Too many attempts + throw new IOException("Could not download and extract " + name + " from " + url.getPath() + "\n properly despite trying " + maxTries + + " times, check your connection. File info:" + "\nTarget MD5: " + targetMD5 + + "\nHash matches: " + checkMD5OfFile(targetMD5, f) + "\nIs valid file: " + f.isFile()); + } + } + + /** + * Check the MD5 of the specified file + * @param targetMD5 Expected MD5 + * @param file File to check + * @return True if MD5 matches, false otherwise + */ + public static boolean checkMD5OfFile(String targetMD5, File file) throws IOException { + InputStream in = FileUtils.openInputStream(file); + String trueMd5 = DigestUtils.md5Hex(in); + IOUtils.closeQuietly(in); + return (targetMD5.equals(trueMd5)); + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Resolver.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Resolver.java new file mode 100644 index 000000000..8b8b05507 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Resolver.java @@ -0,0 +1,92 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources; + +import java.io.File; +import java.io.InputStream; + +public interface Resolver { + + /** + * Priority of this resolver. 0 is highest priority (check first), larger values are lower priority (check last) + */ + int priority(); + + /** + * Determine if the specified file resource can be resolved by {@link #asFile(String)} and {@link #asStream(String)} + * + * @param resourcePath Path of the resource to be resolved + * @return True if this resolver is able to resolve the resource file - i.e., whether it is a valid path and exists + */ + boolean exists(String resourcePath); + + /** + * Determine if the specified directory resource can be resolved by {@link #copyDirectory(String, File)} + * + * @param dirPath Path of the directory resource to be resolved + * @return True if this resolver is able to resolve the directory - i.e., whether it is a valid path and exists + */ + boolean directoryExists(String dirPath); + + /** + * Get the specified resources as a standard local file. + * Note that the resource must exist as determined by {@link #exists(String)} + * + * @param resourcePath Path of the resource. + * @return The local file version of the resource + */ + File asFile(String resourcePath); + + /** + * Get the specified resources as an input stream. + * Note that the resource must exist as determined by {@link #exists(String)} + * + * @param resourcePath Path of the resource. + * @return The resource as an input stream + */ + InputStream asStream(String resourcePath); + + /** + * Copy the directory resource (recursively) to the specified destination directory + * + * @param dirPath Path of the resource directory to resolve + * @param destinationDir Where the files should be copied to + */ + void copyDirectory(String dirPath, File destinationDir); + + /** + * @return True if the resolver has a local cache directory, as returned by {@link #localCacheRoot()} + */ + boolean hasLocalCache(); + + /** + * @return Root directory of the local cache, or null if {@link #hasLocalCache()} returns false + */ + File localCacheRoot(); + + /** + * Normalize the path that may be a resource reference. + * For example: "someDir/myFile.zip.resource_reference" --> "someDir/myFile.zip" + * Returns null if the file cannot be resolved. + * If the file is not a reference, the original path is returned + */ + String normalizePath(String path); +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Resources.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Resources.java new file mode 100644 index 000000000..f8fa974f4 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/Resources.java @@ -0,0 +1,157 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.config.ND4JClassLoading; +import org.nd4j.common.resources.strumpf.StrumpfResolver; + +import java.io.File; +import java.io.InputStream; +import java.util.*; + +@Slf4j +public class Resources { + private static Resources INSTANCE = new Resources(); + + protected final List resolvers; + + protected Resources() { + ServiceLoader loader = ND4JClassLoading.loadService(Resolver.class); + + resolvers = new ArrayList<>(); + resolvers.add(new StrumpfResolver()); + for (Resolver resolver : loader) { + resolvers.add(resolver); + } + + //Sort resolvers by priority: check resolvers with lower numbers first + Collections.sort(resolvers, new Comparator() { + @Override + public int compare(Resolver r1, Resolver r2) { + return Integer.compare(r1.priority(), r2.priority()); + } + }); + } + + /** + * Check if the specified resource exists (can be resolved by any method) hence can be loaded by {@link #asFile(String)} + * or {@link #asStream(String)} + * + * @param resourcePath Path of the resource to be resolved + * @return Whether the resource can be resolved or not + */ + public static boolean exists(@NonNull String resourcePath) { + return INSTANCE.resourceExists(resourcePath); + } + + /** + * Get the specified resource as a local file. + * If it cannot be found (i.e., {@link #exists(String)} returns false) this method will throw an exception. + * + * @param resourcePath Path of the resource to get + * @return Resource file + */ + public static File asFile(@NonNull String resourcePath) { + return INSTANCE.getAsFile(resourcePath); + } + + /** + * Get the specified resource as an input stream.
+ * If it cannot be found (i.e., {@link #exists(String)} returns false) this method will throw an exception. + * + * @param resourcePath Path of the resource to get + * @return Resource stream + */ + public static InputStream asStream(@NonNull String resourcePath) { + return INSTANCE.getAsStream(resourcePath); + } + + /** + * Copy the contents of the specified directory (path) to the specified destination directory, resolving any resources in the process + * + * @param directoryPath Directory to copy contents of + * @param destinationDir Destination + */ + public static void copyDirectory(@NonNull String directoryPath, @NonNull File destinationDir) { + INSTANCE.copyDir(directoryPath, destinationDir); + } + + /** + * Normalize the path that may be a resource reference. + * For example: "someDir/myFile.zip.resource_reference" --> "someDir/myFile.zip" + * Returns null if the file cannot be resolved. + * If the file is not a reference, the original path is returned + */ + public static String normalizePath(String path){ + return INSTANCE.normalize(path); + } + + protected boolean resourceExists(String resourcePath) { + for (Resolver r : resolvers) { + if (r.exists(resourcePath)) + return true; + } + + return false; + } + + protected File getAsFile(String resourcePath) { + for (Resolver r : resolvers) { + if (r.exists(resourcePath)) { + return r.asFile(resourcePath); + } + } + + throw new IllegalStateException("Cannot resolve resource (not found): none of " + resolvers.size() + + " resolvers can resolve resource \"" + resourcePath + "\" - available resolvers: " + resolvers.toString()); + } + + public InputStream getAsStream(String resourcePath) { + for (Resolver r : resolvers) { + if (r.exists(resourcePath)) { + log.debug("Resolved resource with resolver " + r.getClass().getName() + " for path " + resourcePath); + return r.asStream(resourcePath); + } + } + + throw new IllegalStateException("Cannot resolve resource (not found): none of " + resolvers.size() + + " resolvers can resolve resource \"" + resourcePath + "\" - available resolvers: " + resolvers.toString()); + } + + public void copyDir(String directoryPath, File destinationDir) { + for (Resolver r : resolvers) { + if (r.directoryExists(directoryPath)) { + r.copyDirectory(directoryPath, destinationDir); + return; + } + } + } + + public String normalize(String path){ + for(Resolver r : resolvers){ + path = r.normalizePath(path); + } + return path; + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/strumpf/ResourceFile.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/strumpf/ResourceFile.java new file mode 100644 index 000000000..0141be02f --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/strumpf/ResourceFile.java @@ -0,0 +1,259 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources.strumpf; + +import org.nd4j.common.config.ND4JSystemProperties; +import com.google.common.io.Files; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.nd4j.common.base.Preconditions; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +import java.io.*; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +@AllArgsConstructor +@NoArgsConstructor +@Data +@JsonIgnoreProperties("filePath") +@Slf4j +public class ResourceFile { + /** + * Default value for resource downloading connection timeout - see {@link ND4JSystemProperties#RESOURCES_CONNECTION_TIMEOUT} + */ + public static final int DEFAULT_CONNECTION_TIMEOUT = 60000; //Timeout for connections to be established + /** + * Default value for resource downloading read timeout - see {@link ND4JSystemProperties#RESOURCES_READ_TIMEOUT} + */ + public static final int DEFAULT_READ_TIMEOUT = 60000; //Timeout for amount of time between connection established and data is available + protected static final String PATH_KEY = "full_remote_path"; + protected static final String HASH = "_hash"; + protected static final String COMPRESSED_HASH = "_compressed_hash"; + + protected static final int MAX_DOWNLOAD_ATTEMPTS = 3; + + public static final ObjectMapper MAPPER = newMapper(); + + //Note: Field naming to match Strumpf JSON format + protected int current_version; + protected Map v1; + + //Not in JSON: + protected String filePath; + + public static ResourceFile fromFile(String path) { + return fromFile(new File(path)); + } + + public static ResourceFile fromFile(File file) { + String s; + try { + s = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + ResourceFile rf = MAPPER.readValue(s, ResourceFile.class); + rf.setFilePath(file.getPath()); + return rf; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String relativePath() { + String hashKey = null; + for (String key : v1.keySet()) { + if (key.endsWith(HASH) && !key.endsWith(COMPRESSED_HASH)) { + hashKey = key; + break; + } + } + if (hashKey == null) { + throw new IllegalStateException("Could not find _hash in resource reference file: " + filePath); + } + + String relativePath = hashKey.substring(0, hashKey.length() - 5); //-5 to remove "_hash" suffix + return relativePath.replaceAll("\\\\", "/"); + } + + public boolean localFileExistsAndValid(File cacheRootDir) { + + File file = getLocalFile(cacheRootDir); + if (!file.exists()) { + return false; + } + + //File exists... but is it valid? + String sha256Property = relativePath() + HASH; + String expSha256 = v1.get(sha256Property); + + Preconditions.checkState(expSha256 != null, "Expected JSON property %s was not found in resource reference file %s", sha256Property, filePath); + + String actualSha256 = sha256(file); + if (!expSha256.equals(actualSha256)) { + return false; + } + return true; + } + + /** + * Get the local file - or where it *would* be if it has been downloaded. If it does not exist, it will not be downloaded here + * + * @return + */ + protected File getLocalFile(File cacheRootDir) { + String relativePath = relativePath(); + + //For resolving local files with different versions, we want paths like: + // ".../dir/filename.txt__v1/filename.txt" + // ".../dir/filename.txt__v2/filename.txt" + //This is to support multiple versions of files simultaneously... for example, different projects needing different + // versions, or supporting old versions of resource files etc + + int lastSlash = Math.max(relativePath.lastIndexOf('/'), relativePath.lastIndexOf('\\')); + String filename; + if (lastSlash < 0) { + filename = relativePath; + } else { + filename = relativePath.substring(lastSlash + 1); + } + + File parentDir = new File(cacheRootDir, relativePath + "__v" + current_version); + File file = new File(parentDir, filename); + return file; + } + + /** + * Get the local file - downloading and caching if required + * + * @return + */ + public File localFile(File cacheRootDir) { + if (localFileExistsAndValid(cacheRootDir)) { + return getLocalFile(cacheRootDir); + } + + //Need to download and extract... + String remotePath = v1.get(PATH_KEY); + Preconditions.checkState(remotePath != null, "No remote path was found in resource reference file %s", filePath); + File f = getLocalFile(cacheRootDir); + + File tempDir = Files.createTempDir(); + File tempFile = new File(tempDir, FilenameUtils.getName(remotePath)); + + String sha256PropertyCompressed = relativePath() + COMPRESSED_HASH; + + String sha256Compressed = v1.get(sha256PropertyCompressed); + Preconditions.checkState(sha256Compressed != null, "Expected JSON property %s was not found in resource reference file %s", sha256PropertyCompressed, filePath); + + String sha256Property = relativePath() + HASH; + String sha256Uncompressed = v1.get(sha256Property); + + String connTimeoutStr = System.getProperty(ND4JSystemProperties.RESOURCES_CONNECTION_TIMEOUT); + String readTimeoutStr = System.getProperty(ND4JSystemProperties.RESOURCES_READ_TIMEOUT); + boolean validCTimeout = connTimeoutStr != null && connTimeoutStr.matches("\\d+"); + boolean validRTimeout = readTimeoutStr != null && readTimeoutStr.matches("\\d+"); + + int connectTimeout = validCTimeout ? Integer.parseInt(connTimeoutStr) : DEFAULT_CONNECTION_TIMEOUT; + int readTimeout = validRTimeout ? Integer.parseInt(readTimeoutStr) : DEFAULT_READ_TIMEOUT; + + try { + boolean correctHash = false; + for (int tryCount = 0; tryCount < MAX_DOWNLOAD_ATTEMPTS; tryCount++) { + try { + if (tempFile.exists()) + tempFile.delete(); + log.info("Downloading remote resource {} to {}", remotePath, tempFile); + FileUtils.copyURLToFile(new URL(remotePath), tempFile, connectTimeout, readTimeout); + //Now: check if downloaded archive hash is OK + String hash = sha256(tempFile); + correctHash = sha256Compressed.equals(hash); + if (!correctHash) { + log.warn("Download of file {} failed: expected hash {} vs. actual hash {}", remotePath, sha256Compressed, hash); + continue; + } + log.info("Downloaded {} to temporary file {}", remotePath, tempFile); + break; + } catch (Throwable t) { + if (tryCount == MAX_DOWNLOAD_ATTEMPTS - 1) { + throw new RuntimeException("Error downloading test resource: " + remotePath, t); + } + log.warn("Error downloading test resource, retrying... {}", remotePath, t); + } + } + + if (!correctHash) { + throw new RuntimeException("Could not successfully download with correct hash file after " + MAX_DOWNLOAD_ATTEMPTS + + " attempts: " + remotePath); + } + + //Now, extract: + f.getParentFile().mkdirs(); + try (OutputStream os = new BufferedOutputStream(new FileOutputStream(f)); + InputStream is = new BufferedInputStream(new GzipCompressorInputStream(new FileInputStream(tempFile)))) { + IOUtils.copy(is, os); + } catch (IOException e) { + throw new RuntimeException("Error extracting resource file", e); + } + log.info("Extracted {} to {}", tempFile, f); + + //Check extracted file hash: + String extractedHash = sha256(f); + if (!extractedHash.equals(sha256Uncompressed)) { + throw new RuntimeException("Extracted file hash does not match expected hash: " + remotePath + + " -> " + f.getAbsolutePath() + " - expected has " + sha256Uncompressed + ", actual hash " + extractedHash); + } + + } finally { + tempFile.delete(); + } + + return f; + } + + public static String sha256(File f) { + try (InputStream is = new BufferedInputStream(new FileInputStream(f))) { + return DigestUtils.sha256Hex(is); + } catch (IOException e) { + throw new RuntimeException("Error when hashing file: " + f.getPath(), e); + } + } + + + public static final ObjectMapper newMapper() { + ObjectMapper ret = new ObjectMapper(); + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); + ret.enable(SerializationFeature.INDENT_OUTPUT); + return ret; + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/strumpf/StrumpfResolver.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/strumpf/StrumpfResolver.java new file mode 100644 index 000000000..54ff89459 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/resources/strumpf/StrumpfResolver.java @@ -0,0 +1,281 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources.strumpf; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.nd4j.common.config.ND4JEnvironmentVars; +import org.nd4j.common.config.ND4JSystemProperties; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.common.resources.Resolver; + +import java.io.*; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@Slf4j +public class StrumpfResolver implements Resolver { + public static final String DEFAULT_CACHE_DIR = new File(System.getProperty("user.home"), ".cache/nd4j/test_resources").getAbsolutePath(); + public static final String REF = ".resource_reference"; + + protected final List localResourceDirs; + protected final File cacheDir; + + public StrumpfResolver() { + + String localDirs = System.getProperty(ND4JSystemProperties.RESOURCES_LOCAL_DIRS, null); + + if (localDirs != null && !localDirs.isEmpty()) { + String[] split = localDirs.split(","); + localResourceDirs = Arrays.asList(split); + } else { + localResourceDirs = null; + } + + String cd = System.getenv(ND4JEnvironmentVars.ND4J_RESOURCES_CACHE_DIR); + if(cd == null || cd.isEmpty()) { + cd = System.getProperty(ND4JSystemProperties.RESOURCES_CACHE_DIR, DEFAULT_CACHE_DIR); + } + cacheDir = new File(cd); + cacheDir.mkdirs(); + } + + public int priority() { + return 100; + } + + @Override + public boolean exists(@NonNull String resourcePath) { + //First: check local dirs (if any exist) + if (localResourceDirs != null && !localResourceDirs.isEmpty()) { + for (String s : localResourceDirs) { + //Check for standard file: + File f1 = new File(s, resourcePath); + if (f1.exists() && f1.isFile()) { + //OK - found actual file + return true; + } + + //Check for reference file: + File f2 = new File(s, resourcePath + REF); + if (f2.exists() && f2.isFile()) { + //OK - found resource reference + return false; + } + } + } + + //Second: Check classpath + ClassPathResource cpr = new ClassPathResource(resourcePath + REF); + if (cpr.exists()) { + return true; + } + + cpr = new ClassPathResource(resourcePath); + if (cpr.exists()) { + return true; + } + + return false; + } + + @Override + public boolean directoryExists(String dirPath) { + //First: check local dirs (if any) + if (localResourceDirs != null && !localResourceDirs.isEmpty()) { + for (String s : localResourceDirs) { + File f1 = new File(s, dirPath); + if (f1.exists() && f1.isDirectory()) { + //OK - found directory + return true; + } + } + } + + //Second: Check classpath + ClassPathResource cpr = new ClassPathResource(dirPath); + if (cpr.exists()) { + return true; + } + + return false; + } + + @Override + public File asFile(String resourcePath) { + assertExists(resourcePath); + + if (localResourceDirs != null && !localResourceDirs.isEmpty()) { + for (String s : localResourceDirs) { + File f1 = new File(s, resourcePath); + if (f1.exists() && f1.isFile()) { + //OK - found actual file + return f1; + } + + //Check for reference file: + File f2 = new File(s, resourcePath + REF); + if (f2.exists() && f2.isFile()) { + //OK - found resource reference. Need to download to local cache... and/or validate what we have in cache + ResourceFile rf = ResourceFile.fromFile(s); + return rf.localFile(cacheDir); + } + } + } + + + //Second: Check classpath for references (and actual file) + ClassPathResource cpr = new ClassPathResource(resourcePath + REF); + if (cpr.exists()) { + ResourceFile rf; + try { + rf = ResourceFile.fromFile(cpr.getFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return rf.localFile(cacheDir); + } + + cpr = new ClassPathResource(resourcePath); + if (cpr.exists()) { + try { + return cpr.getFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + throw new RuntimeException("Could not find resource file that should exist: " + resourcePath); + } + + @Override + public InputStream asStream(String resourcePath) { + File f = asFile(resourcePath); + log.debug("Resolved resource " + resourcePath + " as file at absolute path " + f.getAbsolutePath()); + try { + return new BufferedInputStream(new FileInputStream(f)); + } catch (FileNotFoundException e) { + throw new RuntimeException("Error reading file for resource: \"" + resourcePath + "\" resolved to \"" + f + "\""); + } + } + + @Override + public void copyDirectory(String dirPath, File destinationDir) { + //First: check local resource dir + boolean resolved = false; + if (localResourceDirs != null && !localResourceDirs.isEmpty()) { + for (String s : localResourceDirs) { + File f1 = new File(s, dirPath); + try { + FileUtils.copyDirectory(f1, destinationDir); + resolved = true; + break; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + //Second: Check classpath + if (!resolved) { + ClassPathResource cpr = new ClassPathResource(dirPath); + if (cpr.exists()) { + try { + cpr.copyDirectory(destinationDir); + resolved = true; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + if (!resolved) { + throw new RuntimeException("Unable to find resource directory for path: " + dirPath); + } + + //Finally, scan directory (recursively) and replace any resource files with actual files... + final List toResolve = new ArrayList<>(); + try { + Files.walkFileTree(destinationDir.toPath(), new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.toString().endsWith(REF)) { + toResolve.add(file); + } + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + + if (toResolve.size() > 0) { + for (Path p : toResolve) { + File localFile = ResourceFile.fromFile(p.toFile()).localFile(cacheDir); + String newPath = p.toFile().getAbsolutePath(); + newPath = newPath.substring(0, newPath.length() - REF.length()); + File destination = new File(newPath); + try { + FileUtils.copyFile(localFile, destination); + } catch (IOException e) { + throw new RuntimeException(e); + } + try { + FileUtils.forceDelete(p.toFile()); + } catch (IOException e) { + throw new RuntimeException("Error deleting temporary reference file", e); + } + } + } + } + + @Override + public boolean hasLocalCache() { + return true; + } + + @Override + public File localCacheRoot() { + return cacheDir; + } + + @Override + public String normalizePath(@NonNull String path) { + if(path.endsWith(REF)){ + return path.substring(0, path.length()-REF.length()); + } + return path; + } + + + protected void assertExists(String resourcePath) { + if (!exists(resourcePath)) { + throw new IllegalStateException("Could not find resource with path \"" + resourcePath + "\" in local directories (" + + localResourceDirs + ") or in classpath"); + } + } + + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/BTools.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/BTools.java new file mode 100644 index 000000000..7e4d06b49 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/BTools.java @@ -0,0 +1,386 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; + +//B = Base +public class BTools { + // + + /** + * getMtLvESS
+ * public static String getMtLvESS( int mtLv )
+ * Returns string. String length create indentation(shift) of other text.
+ * Indentation depends on method level - great method level, great indentation.
+ * Main method has method level 0.
+ * Other called method has method level 1, 2,...N.
+ * @param mtLv - method level + * @return method level external shift string + */ + public static String getMtLvESS( int mtLv ) { + // MtLvESS = Method Level External Shift String + // + if ( mtLv < 0 ) return "?"; + // + String Result = ""; + // + // String LvS = ". "; + String LvS = "."; + // + for ( int K = 1; K <= mtLv; K ++ ) { + // + Result = Result + LvS; + } + // + return Result; + } + + /** + * getMtLvISS
+ * public static String getMtLvISS()
+ * Returns string. String create indentation(shift)
+ * internal text to start text of method.
+ * + * @return method level internal shift string + */ + public static String getMtLvISS() { + // MtLvISS = Method Level Intern Shift String + // + // String Result = ".."; + // String Result = "~"; + String Result = " "; + // + return Result; + } + + /** + * getSpaces
+ * public static String getSpaces( int SpacesCount )
+ * Returns asked count of spaces.
+ * If count of spaces is < 0 returns '?'. + * @param SpacesCount = spaces count + * @return spaces + */ + public static String getSpaces( int SpacesCount ) { + // + if ( SpacesCount < 0 ) return "?"; + // + String Info = ""; + // + for ( int K = 1; K <= SpacesCount; K ++ ) { + Info += " "; + } + // + // + return Info; + } + + /** + * getSBln
+ * public static String getSBln( boolean... blnA )
+ * Returns boolean(s) converted to char (true = 'T'; false = 'F')
+ * If blnA.length is > 1 returns chars without separator.
+ * If blnA is '{ true, false, true }' returns 'TFT'.
+ * If blnA is null returns '?'.
+ * If blnA.length is 0 returns '?'.
+ * @param blnA + * @return boolean(s) as string + */ + public static String getSBln( boolean... blnA ) { + // + String Info = ""; + // + if ( blnA == null ) return "?"; + if ( blnA.length == 0 ) return "?"; + // + for ( int K = 0; K < blnA.length; K ++ ) { + // + Info += ( blnA[ K ] )? "T" : "F"; + } + // + return Info; + } + + /** + * getSDbl
+ * public static String getSDbl( double Value, int DecPrec )
+ * Returns double converted to string.
+ * If Value is Double.NaN returns "NaN".
+ * If DecPrec is < 0 is DecPrec set 0.
+ * + * @param Value - value + * @param DecPrec - decimal precision + * @return double as string + */ + public static String getSDbl( double Value, int DecPrec ) { + // + String Result = ""; + // + if ( Double.isNaN( Value ) ) return "NaN"; + // + if ( DecPrec < 0 ) DecPrec = 0; + // + String DFS = "###,###,##0"; + // + if ( DecPrec > 0 ) { + int idx = 0; + DFS += "."; + while ( idx < DecPrec ) { + DFS = DFS + "0"; + idx ++; + if ( idx > 100 ) break; + } + } + // +// Locale locale = new Locale("en", "UK"); + // + DecimalFormatSymbols DcmFrmSmb = new DecimalFormatSymbols( Locale.getDefault()); + DcmFrmSmb.setDecimalSeparator('.'); + DcmFrmSmb.setGroupingSeparator(' '); + // + DecimalFormat DcmFrm; + // + DcmFrm = new DecimalFormat( DFS, DcmFrmSmb ); + // + // DcmFrm.setGroupingSize( 3 ); + // + Result = DcmFrm.format( Value ); + // + return Result; + } + + /** + * getSDbl
+ * public static String getSDbl( double Value, int DecPrec, boolean ShowPlusSign )
+ * Returns double converted to string.
+ * If Value is Double.NaN returns "NaN".
+ * If DecPrec is < 0 is DecPrec set 0.
+ * If ShowPlusSign is true:
+ * - If Value is > 0 sign is '+'.
+ * - If Value is 0 sign is ' '.
+ * @param Value - value + * @param DecPrec - decimal precision + * @param ShowPlusSign - show plus sign + * @return double as string + */ + public static String getSDbl( double Value, int DecPrec, boolean ShowPlusSign ) { + // + String PlusSign = ""; + // + if ( ShowPlusSign && Value > 0 ) PlusSign = "+"; + if ( ShowPlusSign && Value == 0 ) PlusSign = " "; + // + return PlusSign + getSDbl( Value, DecPrec ); + } + + /** + * getSDbl
+ * public static String getSDbl( double Value, int DecPrec, boolean ShowPlusSign, int StringLength )
+ * Returns double converted to string.
+ * If Value is Double.NaN returns "NaN".
+ * If DecPrec is < 0 is DecPrec set 0.
+ * If ShowPlusSign is true:
+ * - If Value is > 0 sign is '+'.
+ * - If Value is 0 sign is ' '.
+ * If StringLength is > base double string length
+ * before base double string adds relevant spaces.
+ * If StringLength is <= base double string length
+ * returns base double string.
+ * @param Value - value + * @param DecPrec - decimal precision + * @param ShowPlusSign - show plus sign + * @param StringLength - string length + * @return double as string + */ + public static String getSDbl( double Value, int DecPrec, boolean ShowPlusSign, int StringLength ) { + // + String Info = ""; + // + String SDbl = getSDbl( Value, DecPrec, ShowPlusSign ); + // + if ( SDbl.length() >= StringLength ) return SDbl; + // +// String SpacesS = " "; + String SpacesS = getSpaces( StringLength ); + // + Info = SpacesS.substring( 0, StringLength - SDbl.length() ) + SDbl; + // + return Info; + } + + /** + * getSInt
+ * public static String getSInt( int Value, int CharsCount )
+ * Returns int converted to string.
+ * If CharsCount > base int string length
+ * before base int string adds relevant spaces.
+ * If CharsCount <= base int string length
+ * returns base int string.
+ * @param Value - value + * @param CharsCount - chars count + * @return int as string + */ + public static String getSInt( int Value, int CharsCount ) { + // + return getSInt( Value, CharsCount, ' ' ); + } + + /** + * getSInt
+ * public static String getSInt( int Value, int CharsCount, char LeadingChar )
+ * Returns int converted to string.
+ * If CharsCount > base int string length
+ * before base int string adds relevant leading chars.
+ * If CharsCount <= base int string length
+ * returns base int string.
+ * + * @param Value - value + * @param CharsCount - chars count + * @param LeadingChar - leading char + * @return int as string + */ + public static String getSInt( int Value, int CharsCount, char LeadingChar ) { + // + String Result = ""; + // + if ( CharsCount <= 0 ) { + return getSInt( Value ); + } + // + String FormatS = ""; + if ( LeadingChar == '0' ) { + FormatS = "%" + LeadingChar + Integer.toString( CharsCount ) + "d"; + } + else { + FormatS = "%" + Integer.toString( CharsCount ) + "d"; + } + // + Result = String.format( FormatS, Value ); + // + return Result; + } + + /** + * getSInt
+ * public static String getSInt( int Value )
+ * Returns int converted to string.
+ * @param Value + * @return int as string + */ + public static String getSInt( int Value ) { + // + String Result = ""; + // + Result = String.format( "%d", Value ); + // + return Result; + } + + /** + * getSIntA
+ * public static String getSIntA( int... intA )
+ * Returns intA converted to string.
+ * Strings are separated with ", ".
+ * If intA is null returns '?'.
+ * If intA.length is 0 returns '?'.
+ * @param intA - int value(s) (one or more) + * @return int... as string + */ +// public static String getSIntA( int[] intA ) { + public static String getSIntA( int... intA ) { + // + String Info = ""; + // + if ( intA == null ) return "?"; + if ( intA.length == 0 ) return "?"; + // + for ( int K = 0; K < intA.length; K ++ ) { + // + Info += ( Info.isEmpty() )? "" : ", "; + Info += BTools.getSInt( intA[ K ] ); + } + // + return Info; + } + + /** + * getIndexCharsCount
+ * public static int getIndexCharsCount( int MaxIndex )
+ * Returns chars count for max value of index.
+ * Example: Max value of index is 150 and chars count is 3.
+ * It is important for statement of indexed values.
+ * Index columns can have the same width for all rouws.
+ * @param MaxIndex - max value of index + * @return chars count for max value of index + */ + public static int getIndexCharsCount( int MaxIndex ) { + // + int CharsCount = 1; + // + if ( MaxIndex <= 0 ) return 1; + // + CharsCount = (int)Math.log10( MaxIndex ) + 1; + // + return CharsCount; + } + + /** + * getSLcDtTm
+ * public static String getSLcDtTm()
+ * Returns local datetime as string.
+ * Datetime format is "mm:ss.SSS".
+ * @return local datetime as string + */ + public static String getSLcDtTm() { + // + return getSLcDtTm( "mm:ss.SSS" ); + } + + /** + * getSLcDtTm
+ * public static String getSLcDtTm( String FormatS )
+ * Returns local datetime as string.
+ * Datetime format is param.
+ * @param FormatS datetime format + * @return local datetime as string + */ + public static String getSLcDtTm( String FormatS ) { + // + String Result = "?"; + // + LocalDateTime LDT = LocalDateTime.now(); + // + Result = "LDTm: " + LDT.format( DateTimeFormatter.ofPattern( FormatS ) ); + // + return Result; + } + + + + + + +} \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/InfoLine.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/InfoLine.java new file mode 100644 index 000000000..d52cc8e39 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/InfoLine.java @@ -0,0 +1,113 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import java.util.ArrayList; +import java.util.List; + + +public class InfoLine { + // + public InfoLine() { + // + } + // + public List< InfoValues > ivL = new ArrayList< InfoValues >(); + // + + /** + * Returns titles line as string appointed by title index (0..5).
+ * Columns are separated with char '|'.
+ * If title index is < 0 returns "?".
+ * If title index is > 5 returns "?".
+ * @param mtLv - method level + * @param title_I - title index + * @return titles line as string + */ + public String getTitleLine( int mtLv, int title_I ) { + // + String info = ""; + // + if ( title_I < 0 ) return "?"; + if ( title_I > 5 ) return "?"; + // + info = ""; + info += BTools.getMtLvESS( mtLv ); + info += BTools.getMtLvISS(); + info += "|"; + // + InfoValues i_IV; + // + String i_ValuesS = ""; + // + int i_VSLen = -1; + // + String i_TitleS = ""; + // + for ( int i = 0; i < ivL.size(); i ++ ) { + // + i_IV = ivL.get( i ); + // + i_ValuesS = i_IV.getValues(); + // + i_VSLen = i_ValuesS.length(); + // + i_TitleS = ( title_I < i_IV.titleA.length )? i_IV.titleA[ title_I ] : ""; + // + i_TitleS = i_TitleS + BTools.getSpaces( i_VSLen ); + // + info += i_TitleS.substring( 0, i_VSLen - 1 ); + // + info += "|"; + } + // + return info; + } + + /** + * Returns values line as string.
+ * Columns are separated with char '|'.
+ * @param mtLv - method level + * @return values line as string + */ + public String getValuesLine( int mtLv ) { + // + String info = ""; + // + info += BTools.getMtLvESS( mtLv ); + info += BTools.getMtLvISS(); + info += "|"; + // + InfoValues i_IV; + // + for ( int i = 0; i < ivL.size(); i ++ ) { + // + i_IV = ivL.get( i ); + // + info += i_IV.getValues(); + } + // + return info; + } + + + +} \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/InfoValues.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/InfoValues.java new file mode 100644 index 000000000..8edc67945 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/InfoValues.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import java.util.ArrayList; +import java.util.List; + +public class InfoValues { + // + public InfoValues( String... titleA ) { + // + for ( int i = 0; i < this.titleA.length; i++ ) this.titleA[ i ] = ""; + // + int Max_K = Math.min( this.titleA.length - 1, titleA.length - 1 ); + // + if (Max_K + 1 >= 0){ + System.arraycopy(titleA, 0, this.titleA, 0, Max_K + 1); + } + // + } + // + /** + * Title array.
+ */ + public String[] titleA = new String[ 6 ]; + // + // VS = Values String + /** + * Values string list.
+ */ + public List< String > vsL = new ArrayList< String >(); + // + + /** + * Returns values.
+ * This method use class InfoLine.
+ * This method is not intended for external use.
+ * @return + */ + public String getValues() { + // + String info = ""; + // + for ( int i = 0; i < vsL.size(); i ++ ) { + // + info += vsL.get( i ) + "|"; + } + // + return info; + } + +} \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/PropertyParser.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/PropertyParser.java new file mode 100644 index 000000000..52d805de4 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/PropertyParser.java @@ -0,0 +1,296 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import java.util.Properties; + +/** + * PropertyParser + * + * @author gagatust + */ +public class PropertyParser { + + private Properties properties; + + public PropertyParser(Properties properties) { + this.properties = properties; + } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } + + /** + * Parse property. + * + * @param name property name + * @return property + */ + public String parseString(String name) { + String property = getProperties().getProperty(name); + if (property == null) { + throw new NullPointerException(); + } + return property; + } + + /** + * Parse property. + * + * @param name property name + * @return property + */ + public int parseInt(String name) { + return Integer.parseInt(getProperties().getProperty(name)); + } + + /** + * Parse property. + * + * @param name property name + * @return property + */ + public boolean parseBoolean(String name) { + String property = getProperties().getProperty(name); + if (property == null) { + throw new IllegalArgumentException(); + } + return Boolean.parseBoolean(property); + } + + /** + * Parse property. + * + * @param name property name + * @return property + */ + public float parseFloat(String name) { + return Float.parseFloat(getProperties().getProperty(name)); + } + + /** + * Parse property. + * + * @param name property name + * @return property + */ + public double parseDouble(String name) { + return Double.parseDouble(getProperties().getProperty(name)); + } + + /** + * Parse property. + * + * @param name property name + * @return property + */ + public long parseLong(String name) { + return Long.parseLong(getProperties().getProperty(name)); + } + + /** + * Parse property. + * + * @param name property name + * @return property + */ + public char parseChar(String name) { + String property = getProperties().getProperty(name); + if (property.length() != 1) { + throw new IllegalArgumentException(name + " property is't char"); + } + return property.charAt(0); + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @return property + */ + public String toString(String name) { + return toString(name, ""); + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @return property + */ + public int toInt(String name) { + return toInt(name, 0); + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @return property + */ + public boolean toBoolean(String name) { + return toBoolean(name, false); + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @return property + */ + public float toFloat(String name) { + return toFloat(name, 0.0f); + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @return property + */ + public double toDouble(String name) { + return toDouble(name, 0.0); + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @return property + */ + public long toLong(String name) { + return toLong(name, 0); + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @return property + */ + public char toChar(String name) { + return toChar(name, '\u0000'); + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @param defaultValue default value + * @return property + */ + public String toString(String name, String defaultValue) { + String property = getProperties().getProperty(name); + return property != null ? property : defaultValue; + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @param defaultValue default value + * @return property + */ + public int toInt(String name, int defaultValue) { + try { + return parseInt(name); + } catch (Exception e) { + return defaultValue; + } + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @param defaultValue default value + * @return property + */ + public boolean toBoolean(String name, boolean defaultValue) { + String property = getProperties().getProperty(name); + return property != null ? Boolean.parseBoolean(property) : defaultValue; + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @param defaultValue default value + * @return property + */ + public float toFloat(String name, float defaultValue) { + try { + return parseFloat(name); + } catch (Exception e) { + return defaultValue; + } + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @param defaultValue default value + * @return property + */ + public double toDouble(String name, double defaultValue) { + try { + return parseDouble(name); + } catch (Exception e) { + return defaultValue; + } + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @param defaultValue default value + * @return property + */ + public long toLong(String name, long defaultValue) { + try { + return parseLong(name); + } catch (Exception e) { + return defaultValue; + } + } + + /** + * Get property. The method returns the default value if the property is not parsed. + * + * @param name property name + * @param defaultValue default value + * @return property + */ + public char toChar(String name, char defaultValue) { + try { + return parseChar(name); + } catch (Exception e) { + return defaultValue; + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/SIS.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/SIS.java new file mode 100644 index 000000000..b10296fcc --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/tools/SIS.java @@ -0,0 +1,450 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.PrintStream; +import java.io.Writer; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + + + +public class SIS { + // System Informations Saving + // + private String baseModuleCode = "SIS"; + private String moduleCode = "?"; + // + private PrintStream out; + @SuppressWarnings("unused") + private PrintStream err; + // + private String fullFileName = "?"; + // + private boolean wasOpenedFile = false; + private boolean wasClosedFile = false; + // + private File sis_File; + private Writer sis_Writer; + // + private int writerErrorInfoCount = 0; + private int closedFileInfoCount = 0; + // + private long charsCount = 0; + // + + /** + * initValues
+ * public void initValues( int mtLv, String superiorModuleCode,
+ * PrintStream out, PrintStream err )
+ * Initialize values for console - not file.
+ * @param mtLv - method level + * @param superiorModuleCode - superior module code + * @param out - console standard output + * @param err - console error output (not used) + */ + public void initValues( + int mtLv, + String superiorModuleCode, + PrintStream out, + PrintStream err + ) { + // + mtLv ++; + // + moduleCode = superiorModuleCode + "." + baseModuleCode; + // + this.out = out; + this.err = err; + // + } + + /** + * initValues
+ * public void initValues( int mtLv, String superiorModuleCode,
+ * PrintStream out, PrintStream err, String fileDrcS,
+ * String base_FileCode, String spc_FileCode,
+ * boolean ShowBriefInfo, boolean ShowFullInfo )
+ * Initialize values for console and file.
+ * fullFileName =
+ * "Z" +
+ * TimeS + "_" +
+ * base_FileCode + "_" +
+ * spc_FileCode +
+ * ".txt";
+ * TimeS (time string) format: "yyyyMMdd'_'HHmmss.SSS"
+ * @param mtLv - method level + * @param superiorModuleCode - superior module code + * @param out - console standard output + * @param err - console error output (not used) + * @param fileDrcS - file directory as string + * @param base_FileCode - base part of file code + * @param spc_FileCode - specifying part of file code + * @param ShowBriefInfo - show brief informations + * @param ShowFullInfo - show full informations + */ + public void initValues( + int mtLv, + String superiorModuleCode, + PrintStream out, + PrintStream err, + String fileDrcS, + String base_FileCode, + String spc_FileCode, + boolean ShowBriefInfo, + boolean ShowFullInfo + ) { + // + mtLv ++; + // + moduleCode = superiorModuleCode + "." + baseModuleCode; + // + String methodName = moduleCode + "." + "initValues"; + // + this.out = out; + this.err = err; + // + if ( ShowBriefInfo || ShowFullInfo ) { + out.format( "" ); + out.format( BTools.getMtLvESS( mtLv ) ); + out.format( methodName + ": " ); + out.format( "fileDrcS: " + fileDrcS + "; " ); + out.format( "base_FileCode: " + base_FileCode + "; " ); + out.format( "spc_FileCode: " + spc_FileCode + "; " ); +// out.format( "STm: %s; ", Tools.getSDatePM( System.currentTimeMillis(), "HH:mm:ss" ) + "; " ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + // + initFile( mtLv, fileDrcS, base_FileCode, spc_FileCode, ShowBriefInfo, ShowFullInfo ); + // + } + + private void initFile( + int mtLv, + String fileDrcS, + String base_FileCode, + String spc_FileCode, + boolean ShowBriefInfo, + boolean ShowFullInfo + ) { + // + mtLv ++; + // + String oinfo = ""; + // + String methodName = moduleCode + "." + "initFile"; + // + if ( ShowBriefInfo || ShowFullInfo ) { + out.format( "" ); + out.format( BTools.getMtLvESS( mtLv ) ); + out.format( methodName + ": " ); + out.format( "fileDrcS: " + fileDrcS + "; " ); + out.format( "base_FileCode: " + base_FileCode + "; " ); + out.format( "spc_FileCode: " + spc_FileCode + "; " ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + // + spc_FileCode = spc_FileCode.replace( ":", "" ); + spc_FileCode = spc_FileCode.replace( "/", "" ); + spc_FileCode = spc_FileCode.replace( ".", "" ); + // + File fileDrc = new File( fileDrcS ); + // + if ( !fileDrc.exists() ) { + fileDrc.mkdirs(); + // + out.format( "" ); + out.format( BTools.getMtLvESS( mtLv ) ); + out.format( methodName + ": " ); + out.format( "fileDrcS: %s; ", fileDrcS ); + out.format( "Directory was created; " ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + // + LocalDateTime LDT = LocalDateTime.now(); + // + String TimeS = LDT.format( DateTimeFormatter.ofPattern( "yyyyMMdd'_'HHmmss.SSS" ) ); + // + fullFileName = + "Z" + + TimeS + "_" + + base_FileCode + + "_" + + spc_FileCode + + ".txt"; + // + sis_File = new File( fileDrcS, fullFileName ); + // + sis_File.setReadable( true ); + // + if ( sis_File.exists() ) { + if ( ShowBriefInfo || ShowFullInfo ) { + out.format( "" ); + out.format( BTools.getMtLvESS( mtLv ) ); + out.format( BTools.getMtLvISS() ); + out.format( "delete File; " ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + sis_File.delete(); + } + // + try { + sis_File.createNewFile(); + } + catch ( Exception Exc ) { + // Exc.printStackTrace( Err_PS ); + out.format( "===" ); + out.format( methodName + ": " ); + out.format( "create New File error !!! " ); + out.format( "Exception: %s; ", Exc.getMessage() ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + out.format( "===" ); + out.format( BTools.getMtLvISS() ); + out.format( "fileDrcS: " + fileDrcS + "; " ); + out.format( "fullFileName: " + fullFileName + "; " ); + out.format( "%n" ); + // + return; + } + // + if ( ShowFullInfo ) { + out.format( "" ); + out.format( BTools.getMtLvESS( mtLv ) ); + out.format( BTools.getMtLvISS() ); + out.format( "fullFileName: " + fullFileName + "; " ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + // + try { + sis_Writer = new BufferedWriter( new FileWriter( sis_File ) ); + } + catch ( Exception Exc ) { + out.format( "===" ); + out.format( methodName + ": " ); + out.format( "create New Writer: " ); + out.format( "Exception: %s; ", Exc.getMessage() ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + // + return ; + } + // + wasOpenedFile = true; + // + if ( ShowFullInfo ) { + oinfo = ""; + oinfo += BTools.getMtLvESS( mtLv ); + oinfo += methodName + ": "; + oinfo += "fullFileName: " + fullFileName + "; "; + out.format( "%s", BTools.getSLcDtTm() ); + info( oinfo ); + } + // + } + + /** + * getfullFileName
+ * public String getfullFileName()
+ * Returns full file name
+ * @return full file name + */ + public String getfullFileName() { + // + return fullFileName; + } + + /** + * info
+ * public void info( String oinfo )
+ * This method is input for informations.
+ * Informations are showed in console and saved in file.
+ * @param oinfo - information + */ + public void info( String oinfo ) { + // + String methodName = moduleCode + "." + "info"; + // + out.format( "%s%n", oinfo ); + // + charsCount += oinfo.length(); + // + String FOInfo = getFullInfoString( oinfo ); + // + if ( !isFileOpen( methodName ) ) return; + // + outFile( FOInfo ); + // + flushFile(); + // + } + + /** + * getcharsCount
+ * public long getcharsCount()
+ * Returns chars count counted from SIS creating.
+ * @return chars count + */ + public long getcharsCount() { + // + return charsCount; + } + + private String getFullInfoString( String oinfo ) { + // + String Result = ""; + // + LocalDateTime LDT = LocalDateTime.now(); + // + String TimeS = LDT.format( DateTimeFormatter.ofPattern( "yyyy.MM.dd HH:mm:ss.SSS" ) ); + // + Result = + TimeS + + ": " + + oinfo + + "\r\n" + + ""; + // + return Result; + } + + private boolean isFileOpen( String SourceMethodName ) { + // + if ( !wasOpenedFile ) return false; + if ( !wasClosedFile ) return true; + // + String methodName = moduleCode + "." + "isFileOpen"; + // + closedFileInfoCount ++; + if ( closedFileInfoCount <= 3 ) { + out.format( "===" ); +// out.format( methodName + ": " ); + out.format( methodName + "(from " + SourceMethodName + "): " ); + out.format( "File is closed !!!; " ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + // + return false; + } + + private void outFile( String FOInfo ) { + // + String methodName = moduleCode + "." + "outFile"; + // + try { + sis_Writer.write( FOInfo ); + } + catch ( Exception Exc ) { + if ( writerErrorInfoCount < 2 ) { + writerErrorInfoCount ++; + out.format( "===" ); + out.format( methodName + ": " ); + out.format( "Writer.write error !!!; " ); + out.format( "Exception: %s; ", Exc.getMessage() ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + // + } + // + } + + private void flushFile() { + // + String methodName = moduleCode + "." + "flushFile"; + // + try { + sis_Writer.flush(); + } + catch ( Exception Exc ) { + out.format( "===" ); + out.format( methodName + ": " ); + out.format( "Writer.flush error !!!; " ); + out.format( "Exception: %s; ", Exc.getMessage() ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + // + } + + /** + * onStop
+ * public void onStop( int mtLv )
+ * This method should be called at the end of program.
+ * Close file.
+ * @param mtLv - method level + */ + public void onStop( int mtLv ) { + // + mtLv ++; + // + String oinfo = ""; + // + String methodName = moduleCode + "." + "onStop"; + // + oinfo = ""; + oinfo += BTools.getMtLvESS( mtLv ); + oinfo += methodName + ": "; + oinfo += BTools.getSLcDtTm(); + info( oinfo ); + // + closeFile(); + // + } + + + private void closeFile() { + // + String methodName = moduleCode + "." + "closeFile"; + // + flushFile(); + // + try { + sis_Writer.close(); + } + catch ( Exception Exc ) { + out.format( "===" ); + out.format( methodName + ": " ); + out.format( "Writer.close error !!!; " ); + out.format( "Exception: %s; ", Exc.getMessage() ); + out.format( "%s", BTools.getSLcDtTm() ); + out.format( "%n" ); + } + // + wasClosedFile = true; + // + } + + + + + + +} \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/AbstractNumber.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/AbstractNumber.java new file mode 100644 index 000000000..71f31e9ab --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/AbstractNumber.java @@ -0,0 +1,31 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +public interface AbstractNumber { + AbstractNumber add(AbstractNumber b); + + AbstractNumber sub(AbstractNumber b); + + AbstractNumber mult(AbstractNumber b); + + AbstractNumber div(AbstractNumber b); +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ArchiveUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ArchiveUtils.java new file mode 100644 index 000000000..317c5a23d --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ArchiveUtils.java @@ -0,0 +1,277 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.nd4j.common.base.Preconditions; + +import java.io.*; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; + +/** + * @author Adam Gibson + */ +@Slf4j +public class ArchiveUtils { + + protected ArchiveUtils() { + } + + /** + * Extracts all files from the archive to the specified destination.
+ * Note: Logs the path of all extracted files by default. Use {@link #unzipFileTo(String, String, boolean)} if + * logging is not desired.
+ * Can handle .zip, .jar, .tar.gz, .tgz, .tar, and .gz formats. + * Format is interpreted from the filename + * + * @param file the file to extract the files from + * @param dest the destination directory. Will be created if it does not exist + * @throws IOException If an error occurs accessing the files or extracting + */ + public static void unzipFileTo(String file, String dest) throws IOException { + unzipFileTo(file, dest, true); + } + + /** + * Extracts all files from the archive to the specified destination, optionally logging the extracted file path.
+ * Can handle .zip, .jar, .tar.gz, .tgz, .tar, and .gz formats. + * Format is interpreted from the filename + * + * @param file the file to extract the files from + * @param dest the destination directory. Will be created if it does not exist + * @param logFiles If true: log the path of every extracted file; if false do not log + * @throws IOException If an error occurs accessing the files or extracting + */ + public static void unzipFileTo(String file, String dest, boolean logFiles) throws IOException { + File target = new File(file); + if (!target.exists()) + throw new IllegalArgumentException("Archive doesnt exist"); + if (!new File(dest).exists()) + new File(dest).mkdirs(); + FileInputStream fin = new FileInputStream(target); + int BUFFER = 2048; + byte data[] = new byte[BUFFER]; + + if (file.endsWith(".zip") || file.endsWith(".jar")) { + try(ZipInputStream zis = new ZipInputStream(fin)) { + //get the zipped file list entry + ZipEntry ze = zis.getNextEntry(); + + while (ze != null) { + String fileName = ze.getName(); + + String canonicalDestinationDirPath = new File(dest).getCanonicalPath(); + File newFile = new File(dest + File.separator + fileName); + String canonicalDestinationFile = newFile.getCanonicalPath(); + + if (!canonicalDestinationFile.startsWith(canonicalDestinationDirPath + File.separator)) { + log.debug("Attempt to unzip entry is outside of the target dir"); + throw new IOException("Entry is outside of the target dir: "); + } + + if (ze.isDirectory()) { + newFile.mkdirs(); + zis.closeEntry(); + ze = zis.getNextEntry(); + continue; + } + + FileOutputStream fos = new FileOutputStream(newFile); + + int len; + while ((len = zis.read(data)) > 0) { + fos.write(data, 0, len); + } + + fos.close(); + ze = zis.getNextEntry(); + if(logFiles) { + log.info("File extracted: " + newFile.getAbsoluteFile()); + } + } + + zis.closeEntry(); + } + } else if (file.endsWith(".tar.gz") || file.endsWith(".tgz") || file.endsWith(".tar")) { + BufferedInputStream in = new BufferedInputStream(fin); + TarArchiveInputStream tarIn; + if(file.endsWith(".tar")){ + //Not compressed + tarIn = new TarArchiveInputStream(in); + } else { + GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in); + tarIn = new TarArchiveInputStream(gzIn); + } + + TarArchiveEntry entry; + /* Read the tar entries using the getNextEntry method **/ + while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { + if(logFiles) { + log.info("Extracting: " + entry.getName()); + } + /* If the entry is a directory, create the directory. */ + + if (entry.isDirectory()) { + File f = new File(dest + File.separator + entry.getName()); + f.mkdirs(); + } + /* + * If the entry is a file,write the decompressed file to the disk + * and close destination stream. + */ + else { + int count; + try(FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName()); + BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER);) { + while ((count = tarIn.read(data, 0, BUFFER)) != -1) { + destStream.write(data, 0, count); + } + + destStream.flush(); + IOUtils.closeQuietly(destStream); + } + } + } + + // Close the input stream + tarIn.close(); + } else if (file.endsWith(".gz")) { + File extracted = new File(target.getParent(), target.getName().replace(".gz", "")); + if (extracted.exists()) + extracted.delete(); + extracted.createNewFile(); + try (GZIPInputStream is2 = new GZIPInputStream(fin); OutputStream fos = FileUtils.openOutputStream(extracted)) { + IOUtils.copyLarge(is2, fos); + fos.flush(); + } + } else { + throw new IllegalStateException("Unable to infer file type (compression format) from source file name: " + + file); + } + target.delete(); + } + + /** + * List all of the files and directories in the specified tar.gz file + * + * @param tarFile A .tar file + * @return List of files and directories + */ + public static List tarListFiles(File tarFile) throws IOException { + Preconditions.checkState(!tarFile.getPath().endsWith(".tar.gz"), ".tar.gz files should not use this method - use tarGzListFiles instead"); + return tarGzListFiles(tarFile, false); + } + + /** + * List all of the files and directories in the specified tar.gz file + * + * @param tarGzFile A tar.gz file + * @return List of files and directories + */ + public static List tarGzListFiles(File tarGzFile) throws IOException { + return tarGzListFiles(tarGzFile, true); + } + + protected static List tarGzListFiles(File file, boolean isTarGz) throws IOException { + try(TarArchiveInputStream tin = + isTarGz ? new TarArchiveInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(file)))) : + new TarArchiveInputStream(new BufferedInputStream(new FileInputStream(file)))) { + ArchiveEntry entry; + List out = new ArrayList<>(); + while((entry = tin.getNextTarEntry()) != null){ + String name = entry.getName(); + out.add(name); + } + return out; + } + } + + /** + * List all of the files and directories in the specified .zip file + * + * @param zipFile Zip file + * @return List of files and directories + */ + public static List zipListFiles(File zipFile) throws IOException { + List out = new ArrayList<>(); + try (ZipFile zf = new ZipFile(zipFile)) { + Enumeration entries = zf.entries(); + while (entries.hasMoreElements()) { + ZipEntry ze = (ZipEntry) entries.nextElement(); + out.add(ze.getName()); + } + } + return out; + } + + /** + * Extract a single file from a .zip file. Does not support directories + * + * @param zipFile Zip file to extract from + * @param destination Destination file + * @param pathInZip Path in the zip to extract + * @throws IOException If exception occurs while reading/writing + */ + public static void zipExtractSingleFile(File zipFile, File destination, String pathInZip) throws IOException { + try (ZipFile zf = new ZipFile(zipFile); InputStream is = new BufferedInputStream(zf.getInputStream(zf.getEntry(pathInZip))); + OutputStream os = new BufferedOutputStream(new FileOutputStream(destination))) { + IOUtils.copy(is, os); + } + } + + /** + * Extract a single file from a tar.gz file. Does not support directories. + * NOTE: This should not be used for batch extraction of files, due to the need to iterate over the entries until the + * specified entry is found. Use {@link #unzipFileTo(String, String)} for batch extraction instead + * + * @param tarGz A tar.gz file + * @param destination The destination file to extract to + * @param pathInTarGz The path in the tar.gz file to extract + */ + public static void tarGzExtractSingleFile(File tarGz, File destination, String pathInTarGz) throws IOException { + try(TarArchiveInputStream tin = new TarArchiveInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(tarGz))))) { + ArchiveEntry entry; + boolean extracted = false; + while((entry = tin.getNextTarEntry()) != null){ + String name = entry.getName(); + if(pathInTarGz.equals(name)){ + try(OutputStream os = new BufferedOutputStream(new FileOutputStream(destination))){ + IOUtils.copy(tin, os); + } + extracted = true; + } + } + Preconditions.checkState(extracted, "No file was extracted. File not found? %s", pathInTarGz); + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ArrayUtil.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ArrayUtil.java new file mode 100644 index 000000000..8a30f0e48 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ArrayUtil.java @@ -0,0 +1,3623 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import lombok.val; +import org.apache.commons.lang3.RandomUtils; +import org.nd4j.common.base.Preconditions; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.Array; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * @author Adam Gibson + */ +public class ArrayUtil { + + + private ArrayUtil() {} + + + /** + * Returns true if any array elements are negative. + * If the array is null, it returns false + * @param arr the array to test + * @return + */ + public static boolean containsAnyNegative(int[] arr) { + if(arr == null) + return false; + + for(int i = 0; i < arr.length; i++) { + if(arr[i] < 0) + return true; + } + return false; + } + + public static boolean containsAnyNegative(long[] arr) { + if(arr == null) + return false; + + for(int i = 0; i < arr.length; i++) { + if(arr[i] < 0) + return true; + } + return false; + } + + public static boolean contains(int[] arr, int value){ + if(arr == null) + return false; + for( int i : arr ) { + if (i == value) + return true; + } + return false; + } + + public static boolean contains(long[] arr, int value){ + if(arr == null) + return false; + for( long i : arr ) { + if (i == value) + return true; + } + return false; + } + + /** + * + * @param arrs + * @param check + * @return + */ + public static boolean anyLargerThan(int[] arrs, int check) { + for(int i = 0; i < arrs.length; i++) { + if(arrs[i] > check) + return true; + } + + return false; + } + + + /** + * + * @param arrs + * @param check + * @return + */ + public static boolean anyLessThan(int[] arrs, int check) { + for(int i = 0; i < arrs.length; i++) { + if(arrs[i] < check) + return true; + } + + return false; + } + + + /** + * Convert a int array to a string array + * @param arr the array to convert + * @return the equivalent string array + */ + public static String[] convertToString(int[] arr) { + Preconditions.checkNotNull(arr); + String[] ret = new String[arr.length]; + for(int i = 0; i < arr.length; i++) { + ret[i] = String.valueOf(arr[i]); + } + + return ret; + } + + + /** + * Proper comparison contains for list of int + * arrays + * @param list the to search + * @param target the target int array + * @return whether the given target + * array is contained in the list + */ + public static boolean listOfIntsContains(List list,int[] target) { + for(int[] arr : list) + if(Arrays.equals(target,arr)) + return true; + return false; + } + + /** + * Repeat a value n times + * @param n the number of times to repeat + * @param toReplicate the value to repeat + * @return an array of length n filled with the + * given value + */ + public static int[] nTimes(int n, int toReplicate) { + int[] ret = new int[n]; + Arrays.fill(ret, toReplicate); + return ret; + } + + public static long[] nTimes(long n, long toReplicate) { + if (n > Integer.MAX_VALUE) + throw new RuntimeException("Index overflow in nTimes"); + val ret = new long[(int) n]; + Arrays.fill(ret, toReplicate); + return ret; + } + + public static T[] nTimes(int n, T toReplicate, Class tClass){ + Preconditions.checkState(n>=0, "Invalid number of times to replicate: must be >= 0, got %s", n); + T[] out = (T[])Array.newInstance(tClass, n); + for( int i=0; i set = new HashSet<>(); + for (int i : toTest) { + if (!set.contains(i)) + set.add(i); + else + return false; + } + + return true; + } + + /** + * Credit to mikio braun from jblas + *

+ * Create a random permutation of the numbers 0, ..., size - 1. + *

+ * see Algorithm P, D.E. Knuth: The Art of Computer Programming, Vol. 2, p. 145 + */ + public static int[] randomPermutation(int size) { + Random r = new Random(); + int[] result = new int[size]; + + for (int j = 0; j < size; j++) { + result[j] = j + 1; + } + + for (int j = size - 1; j > 0; j--) { + int k = r.nextInt(j); + int temp = result[j]; + result[j] = result[k]; + result[k] = temp; + } + + return result; + } + + + public static short toBFloat16(float data) { + return (short) (Float.floatToIntBits(data) << 16); + } + + public static short toBFloat16(double data) { + return toBFloat16((float) data); + } + + public static short toHalf(float data) { + return fromFloat(data); + } + + public static short toHalf(double data) { + return fromFloat((float) data); + } + + public static short[] toHalfs(float[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = fromFloat(data[i]); + } + return ret; + } + + public static short[] toHalfs(int[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = fromFloat((float) data[i]); + } + return ret; + } + + public static short[] toHalfs(long[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = fromFloat((float) data[i]); + } + return ret; + } + + public static short[] toBfloats(float[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = toBFloat16(data[i]); + } + return ret; + } + + public static short[] toBfloats(int[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = toBFloat16((float) data[i]); + } + return ret; + } + + public static short[] toBfloats(long[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = toBFloat16((float) data[i]); + } + return ret; + } + + public static long[] toLongs(byte[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static long[] toLongs(boolean[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = data[i] ? 1 : 0; + } + return ret; + } + + public static long[] toLongs(short[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static long[] toLongs(int[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static long[] toLongs(float[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static long[] toLongs(double[] data) { + val ret = new long[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = (long) data[i]; + } + return ret; + } + + public static short[] toHalfs(double[] data) { + short[] ret = new short[data.length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = fromFloat((float) data[i]); + } + return ret; + } + + public static short fromFloat(float v) { + if (Float.isNaN(v)) + return (short) 0x7fff; + if (v == Float.POSITIVE_INFINITY) + return (short) 0x7c00; + if (v == Float.NEGATIVE_INFINITY) + return (short) 0xfc00; + if (v == 0.0f) + return (short) 0x0000; + if (v == -0.0f) + return (short) 0x8000; + if (v > 65504.0f) + return 0x7bff; // max value supported by half float + if (v < -65504.0f) + return (short) (0x7bff | 0x8000); + if (v > 0.0f && v < 5.96046E-8f) + return 0x0001; + if (v < 0.0f && v > -5.96046E-8f) + return (short) 0x8001; + + final int f = Float.floatToIntBits(v); + + return (short) (((f >> 16) & 0x8000) | ((((f & 0x7f800000) - 0x38000000) >> 13) & 0x7c00) + | ((f >> 13) & 0x03ff)); + } + + public static int[] toInts(float[] data) { + int[] ret = new int[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = (int) data[i]; + return ret; + } + + public static int[] toInts(double[] data) { + int[] ret = new int[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = (int) data[i]; + return ret; + } + + public static byte[] toBytes(int[] array) { + val retVal = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + retVal[i] = (byte) array[i]; + } + return retVal; + } + + public static byte[] toBytes(float[] array) { + val retVal = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + retVal[i] = (byte) array[i]; + } + return retVal; + } + + public static byte[] toBytes(double[] array) { + val retVal = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + retVal[i] = (byte) array[i]; + } + return retVal; + } + + public static byte[] toBytes(long[] array) { + val retVal = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + retVal[i] = (byte) array[i]; + } + return retVal; + } + + public static int[] toInts(long[] array) { + int[] retVal = new int[array.length]; + + for (int i = 0; i < array.length; i++) { + retVal[i] = (int) array[i]; + } + + return retVal; + } + + + public static int[] mod(int[] input,int mod) { + int[] ret = new int[input.length]; + for(int i = 0; i < ret.length; i++) { + ret[i] = input[i] % mod; + } + + return ret; + } + + + /** + * Calculate the offset for a given stride array + * @param stride the stride to use + * @param i the offset to calculate for + * @return the offset for the given + * stride + */ + public static int offsetFor(int[] stride, int i) { + int ret = 0; + for (int j = 0; j < stride.length; j++) + ret += (i * stride[j]); + return ret; + + } + + /** + * Sum of an int array + * @param add the elements + * to calculate the sum for + * @return the sum of this array + */ + public static int sum(List add) { + if (add.isEmpty()) + return 0; + int ret = 0; + for (int i = 0; i < add.size(); i++) + ret += add.get(i); + return ret; + } + + /** + * Sum of an int array + * @param add the elements + * to calculate the sum for + * @return the sum of this array + */ + public static int sum(int[] add) { + if (add.length < 1) + return 0; + int ret = 0; + for (int i = 0; i < add.length; i++) + ret += add[i]; + return ret; + } + + public static long sumLong(long... add) { + if (add.length < 1) + return 0; + int ret = 0; + for (int i = 0; i < add.length; i++) + ret += add[i]; + return ret; + } + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static int prod(List mult) { + if (mult.isEmpty()) + return 0; + int ret = 1; + for (int i = 0; i < mult.size(); i++) + ret *= mult.get(i); + return ret; + } + + + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static int prod(long... mult) { + if (mult.length < 1) + return 0; + int ret = 1; + for (int i = 0; i < mult.length; i++) + ret *= mult[i]; + return ret; + } + + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static int prod(int... mult) { + if (mult.length < 1) + return 0; + int ret = 1; + for (int i = 0; i < mult.length; i++) + ret *= mult[i]; + return ret; + } + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static long prodLong(List mult) { + if (mult.isEmpty()) + return 0; + long ret = 1; + for (int i = 0; i < mult.size(); i++) + ret *= mult.get(i).longValue(); + return ret; + } + + + /** + * Product of an int array + * @param mult the elements + * to calculate the sum for + * @return the product of this array + */ + public static long prodLong(int... mult) { + if (mult.length < 1) + return 0; + long ret = 1; + for (int i = 0; i < mult.length; i++) + ret *= mult[i]; + return ret; + } + + public static long prodLong(long... mult) { + if (mult.length < 1) + return 0; + long ret = 1; + for (int i = 0; i < mult.length; i++) + ret *= mult[i]; + return ret; + } + + public static boolean equals(float[] data, double[] data2) { + if (data.length != data2.length) + return false; + for (int i = 0; i < data.length; i++) { + double equals = Math.abs(data2[i] - data[i]); + if (equals > 1e-6) + return false; + } + return true; + } + + + public static int[] consArray(int a, int[] as) { + int len = as.length; + int[] nas = new int[len + 1]; + nas[0] = a; + System.arraycopy(as, 0, nas, 1, len); + return nas; + } + + + /** + * Returns true if any of the elements are zero + * @param as + * @return + */ + public static boolean isZero(int[] as) { + for (int i = 0; i < as.length; i++) { + if (as[i] == 0) + return true; + } + return false; + } + + public static boolean isZero(long[] as) { + for (int i = 0; i < as.length; i++) { + if (as[i] == 0L) + return true; + } + return false; + } + + public static boolean anyMore(int[] target, int[] test) { + Preconditions.checkArgument(target.length == test.length, "Unable to compare: different sizes: length %s vs. %s", target.length, test.length); + for (int i = 0; i < target.length; i++) { + if (target[i] > test[i]) + return true; + } + return false; + } + + + public static boolean anyLess(int[] target, int[] test) { + Preconditions.checkArgument(target.length == test.length, "Unable to compare: different sizes: length %s vs. %s", target.length, test.length); + for (int i = 0; i < target.length; i++) { + if (target[i] < test[i]) + return true; + } + return false; + } + + public static boolean lessThan(int[] target, int[] test) { + Preconditions.checkArgument(target.length == test.length, "Unable to compare: different sizes: length %s vs. %s", target.length, test.length); + for (int i = 0; i < target.length; i++) { + if (target[i] < test[i]) + return true; + if (target[i] > test[i]) + return false; + } + return false; + } + + public static boolean greaterThan(int[] target, int[] test) { + Preconditions.checkArgument(target.length == test.length, "Unable to compare: different sizes: length %s vs. %s", target.length, test.length); + for (int i = 0; i < target.length; i++) { + if (target[i] > test[i]) + return true; + if (target[i] < test[i]) + return false; + } + return false; + } + + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static int calcOffset(List shape, List offsets, List strides) { + if (shape.size() != offsets.size() || shape.size() != strides.size()) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + int ret = 0; + for (int i = 0; i < offsets.size(); i++) { + //we should only do this in the general case, not on vectors + //the reason for this is we force everything including scalars + //to be 2d + if (shape.get(i) == 1 && offsets.size() > 2 && i > 0) + continue; + ret += offsets.get(i) * strides.get(i); + } + + return ret; + } + + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static int calcOffset(int[] shape, int[] offsets, int[] strides) { + if (shape.length != offsets.length || shape.length != strides.length) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + + int ret = 0; + for (int i = 0; i < offsets.length; i++) { + if (shape[i] == 1) + continue; + ret += offsets[i] * strides[i]; + } + + return ret; + } + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static long calcOffset(long[] shape, long[] offsets, long[] strides) { + if (shape.length != offsets.length || shape.length != strides.length) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + + long ret = 0; + for (int i = 0; i < offsets.length; i++) { + if (shape[i] == 1) + continue; + ret += offsets[i] * strides[i]; + } + + return ret; + } + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static long calcOffsetLong(List shape, List offsets, List strides) { + if (shape.size() != offsets.size() || shape.size() != strides.size()) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + long ret = 0; + for (int i = 0; i < offsets.size(); i++) { + //we should only do this in the general case, not on vectors + //the reason for this is we force everything including scalars + //to be 2d + if (shape.get(i) == 1 && offsets.size() > 2 && i > 0) + continue; + ret += (long) offsets.get(i) * strides.get(i); + } + + return ret; + } + + + public static long calcOffsetLong2(List shape, List offsets, List strides) { + if (shape.size() != offsets.size() || shape.size() != strides.size()) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + long ret = 0; + for (int i = 0; i < offsets.size(); i++) { + //we should only do this in the general case, not on vectors + //the reason for this is we force everything including scalars + //to be 2d + if (shape.get(i) == 1 && offsets.size() > 2 && i > 0) + continue; + ret += (long) offsets.get(i) * strides.get(i); + } + + return ret; + } + + + /** + * Compute the offset + * based on teh shape strides and offsets + * @param shape the shape to compute + * @param offsets the offsets to compute + * @param strides the strides to compute + * @return the offset for the given shape,offset,and strides + */ + public static long calcOffsetLong(int[] shape, int[] offsets, int[] strides) { + if (shape.length != offsets.length || shape.length != strides.length) + throw new IllegalArgumentException("Shapes,strides, and offsets must be the same size"); + + long ret = 0; + for (int i = 0; i < offsets.length; i++) { + if (shape[i] == 1) + continue; + ret += (long) offsets[i] * strides[i]; + } + + return ret; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static int dotProduct(List xs, List ys) { + int result = 0; + int n = xs.size(); + + if (ys.size() != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += xs.get(i) * ys.get(i); + } + return result; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static int dotProduct(int[] xs, int[] ys) { + int result = 0; + int n = xs.length; + + if (ys.length != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += xs[i] * ys[i]; + } + return result; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static long dotProductLong(List xs, List ys) { + long result = 0; + int n = xs.size(); + + if (ys.size() != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += (long) xs.get(i) * ys.get(i); + } + return result; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static long dotProductLong2(List xs, List ys) { + long result = 0; + int n = xs.size(); + + if (ys.size() != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += (long) xs.get(i) * ys.get(i); + } + return result; + } + + /** + * + * @param xs + * @param ys + * @return + */ + public static long dotProductLong(int[] xs, int[] ys) { + long result = 0; + int n = xs.length; + + if (ys.length != n) + throw new IllegalArgumentException("Different array sizes"); + + for (int i = 0; i < n; i++) { + result += (long) xs[i] * ys[i]; + } + return result; + } + + + public static int[] empty() { + return new int[0]; + } + + + public static int[] of(int... arr) { + return arr; + } + + public static int[] copy(int[] copy) { + int[] ret = new int[copy.length]; + System.arraycopy(copy, 0, ret, 0, ret.length); + return ret; + } + + public static long[] copy(long[] copy) { + long[] ret = new long[copy.length]; + System.arraycopy(copy, 0, ret, 0, ret.length); + return ret; + } + + + public static double[] doubleCopyOf(float[] data) { + double[] ret = new double[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = data[i]; + return ret; + } + + public static float[] floatCopyOf(double[] data) { + if (data.length == 0) + return new float[1]; + float[] ret = new float[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = (float) data[i]; + return ret; + } + + + /** + * Returns a subset of an array from 0 to "to" (exclusive) + * + * @param data the data to getFromOrigin a subset of + * @param to the end point of the data + * @return the subset of the data specified + */ + public static double[] range(double[] data, int to) { + return range(data, to, 1); + } + + + /** + * Returns a subset of an array from 0 to "to" (exclusive) using the specified stride + * + * @param data the data to getFromOrigin a subset of + * @param to the end point of the data + * @param stride the stride to go through the array + * @return the subset of the data specified + */ + public static double[] range(double[] data, int to, int stride) { + return range(data, to, stride, 1); + } + + + /** + * Returns a subset of an array from 0 to "to" + * using the specified stride + * + * @param data the data to getFromOrigin a subset of + * @param to the end point of the data + * @param stride the stride to go through the array + * @param numElementsEachStride the number of elements to collect at each stride + * @return the subset of the data specified + */ + public static double[] range(double[] data, int to, int stride, int numElementsEachStride) { + double[] ret = new double[to / stride]; + if (ret.length < 1) + ret = new double[1]; + int count = 0; + for (int i = 0; i < data.length; i += stride) { + for (int j = 0; j < numElementsEachStride; j++) { + if (i + j >= data.length || count >= ret.length) + break; + ret[count++] = data[i + j]; + } + } + return ret; + } + + public static List toList(int... ints){ + if(ints == null){ + return null; + } + List ret = new ArrayList<>(); + for (int anInt : ints) { + ret.add(anInt); + } + return ret; + } + + public static int[] toArray(List list) { + int[] ret = new int[list.size()]; + for (int i = 0; i < list.size(); i++) + ret[i] = list.get(i); + return ret; + } + + public static long[] toArrayLong(List list) { + long[] ret = new long[list.size()]; + for (int i = 0; i < list.size(); i++) + ret[i] = list.get(i); + return ret; + } + + + public static double[] toArrayDouble(List list) { + double[] ret = new double[list.size()]; + for (int i = 0; i < list.size(); i++) + ret[i] = list.get(i); + return ret; + + } + + + /** + * Generate an int array ranging from "from" to "to". + * The total number of elements is (from-to)/increment - i.e., range(0,2,1) returns [0,1] + * If from is > to this method will count backwards + * + * @param from the from + * @param to the end point of the data + * @param increment the amount to increment by + * @return the int array with a length equal to absoluteValue(from - to) + */ + public static int[] range(int from, int to, int increment) { + int diff = Math.abs(from - to); + int[] ret = new int[diff / increment]; + if (ret.length < 1) + ret = new int[1]; + + if (from < to) { + int count = 0; + for (int i = from; i < to; i += increment) { + if (count >= ret.length) + break; + ret[count++] = i; + } + } else if (from > to) { + int count = 0; + for (int i = from - 1; i >= to; i -= increment) { + if (count >= ret.length) + break; + ret[count++] = i; + } + } + + return ret; + } + + + public static long[] range(long from, long to, long increment) { + long diff = Math.abs(from - to); + long[] ret = new long[(int) (diff / increment)]; + if (ret.length < 1) + ret = new long[1]; + + if (from < to) { + int count = 0; + for (long i = from; i < to; i += increment) { + if (count >= ret.length) + break; + ret[count++] = i; + } + } else if (from > to) { + int count = 0; + for (int i = (int) from - 1; i >= to; i -= increment) { + if (count >= ret.length) + break; + ret[count++] = i; + } + } + + return ret; + } + + /** + * Generate an int array ranging from "from" to "to". + * The total number of elements is (from-to) - i.e., range(0,2) returns [0,1] + * If from is > to this method will count backwards + * + * @param from the from + * @param to the end point of the data + * @return the int array with a length equal to absoluteValue(from - to) + */ + public static int[] range(int from, int to) { + if (from == to) + return new int[0]; + return range(from, to, 1); + } + + public static long[] range(long from, long to) { + if (from == to) + return new long[0]; + return range(from, to, 1); + } + + public static double[] toDoubles(int[] ints) { + double[] ret = new double[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (double) ints[i]; + return ret; + } + + public static double[] toDoubles(long[] ints) { + double[] ret = new double[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (double) ints[i]; + return ret; + } + + public static double[] toDoubles(float[] ints) { + double[] ret = new double[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (double) ints[i]; + return ret; + } + + public static float[] toFloats(int[][] ints) { + return toFloats(Ints.concat(ints)); + } + + public static double[] toDoubles(int[][] ints) { + return toDoubles(Ints.concat(ints)); + } + + public static short[] toShorts(long[] ints) { + val ret = new short[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (short) ints[i]; + return ret; + } + + public static short[] toShorts(int[] ints) { + val ret = new short[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (short) ints[i]; + return ret; + } + + public static short[] toShorts(float[] ints) { + val ret = new short[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (short) ints[i]; + return ret; + } + + public static short[] toShorts(double[] ints) { + val ret = new short[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (short) ints[i]; + return ret; + } + + public static float[] toFloats(int[] ints) { + float[] ret = new float[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (float) ints[i]; + return ret; + } + + public static float[] toFloats(long[] ints) { + float[] ret = new float[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (float) ints[i]; + return ret; + } + + public static float[] toFloats(double[] ints) { + float[] ret = new float[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = (float) ints[i]; + return ret; + } + + public static int[] cutBelowZero(int[] data) { + val ret = new int[data.length]; + for (int i = 0; i < data.length; i++) + ret[i] = data[i] < 0 ? 0 : data[i]; + return ret; + } + + public static long[] cutBelowZero(long[] data) { + val ret = new long[data.length]; + for (int i = 0; i < data.length; i++) + ret[i] = data[i] < 0 ? 0 : data[i]; + return ret; + } + + public static short[] cutBelowZero(short[] data) { + val ret = new short[data.length]; + for (int i = 0; i < data.length; i++) + ret[i] = data[i] < 0 ? 0 : data[i]; + return ret; + } + + public static byte[] cutBelowZero(byte[] data) { + val ret = new byte[data.length]; + for (int i = 0; i < data.length; i++) + ret[i] = data[i] < 0 ? 0 : data[i]; + return ret; + } + + /** + * Return a copy of this array with the + * given index omitted + * + * @param data the data to copy + * @param index the index of the item to remove + * @param newValue the newValue to replace + * @return the new array with the omitted + * item + */ + public static int[] replace(int[] data, int index, int newValue) { + int[] copy = copy(data); + copy[index] = newValue; + return copy; + } + + /** + * Return a copy of this array with only the + * given index(es) remaining + * + * @param data the data to copy + * @param index the index of the item to remove + * @return the new array with the omitted + * item + */ + public static int[] keep(int[] data, int... index) { + if (index.length == data.length) + return data; + + int[] ret = new int[index.length]; + int count = 0; + for (int i = 0; i < data.length; i++) + if (Ints.contains(index, i)) + ret[count++] = data[i]; + + return ret; + } + + /** + * Return a copy of this array with only the + * given index(es) remaining + * + * @param data the data to copy + * @param index the index of the item to remove + * @return the new array with the omitted + * item + */ + public static long[] keep(long[] data, int... index) { + if (index.length == data.length) + return data; + + long[] ret = new long[index.length]; + int count = 0; + for (int i = 0; i < data.length; i++) + if (Ints.contains(index, i)) + ret[count++] = data[i]; + + return ret; + } + + + /** + * Return a copy of this array with the + * given index omitted + * + * PLEASE NOTE: index to be omitted must exist in source array. + * + * @param data the data to copy + * @param index the index of the item to remove + * @return the new array with the omitted + * item + */ + public static int[] removeIndex(int[] data, int... index) { + if (index.length >= data.length) { + throw new IllegalStateException("Illegal remove: indexes.length > data.length (index.length=" + + index.length + ", data.length=" + data.length + ")"); + } + int offset = 0; + /* + workaround for non-existent indexes (such as Integer.MAX_VALUE) + + + for (int i = 0; i < index.length; i ++) { + if (index[i] >= data.length || index[i] < 0) offset++; + } + */ + + int[] ret = new int[data.length - index.length + offset]; + int count = 0; + for (int i = 0; i < data.length; i++) + if (!Ints.contains(index, i)) { + ret[count++] = data[i]; + } + + return ret; + } + + public static long[] removeIndex(long[] data, int... index) { + if (index.length >= data.length) { + throw new IllegalStateException("Illegal remove: indexes.length >= data.length (index.length=" + + index.length + ", data.length=" + data.length + ")"); + } + int offset = 0; + /* + workaround for non-existent indexes (such as Integer.MAX_VALUE) + + + for (int i = 0; i < index.length; i ++) { + if (index[i] >= data.length || index[i] < 0) offset++; + } + */ + + long[] ret = new long[data.length - index.length + offset]; + int count = 0; + for (int i = 0; i < data.length; i++) + if (!Ints.contains(index, i)) { + ret[count++] = data[i]; + } + + return ret; + } + + + + /** + * Zip 2 arrays in to: + * + * @param as + * @param bs + * @return + */ + public static int[][] zip(int[] as, int[] bs) { + int[][] result = new int[as.length][2]; + for (int i = 0; i < result.length; i++) { + result[i] = new int[] {as[i], bs[i]}; + } + + return result; + } + + /** + * Get the tensor matrix multiply shape + * @param aShape the shape of the first array + * @param bShape the shape of the second array + * @param axes the axes to do the multiply + * @return the shape for tensor matrix multiply + */ + public static long[] getTensorMmulShape(long[] aShape, long[] bShape, int[][] axes) { + + int validationLength = Math.min(axes[0].length, axes[1].length); + for (int i = 0; i < validationLength; i++) { + if (aShape[axes[0][i]] != bShape[axes[1][i]]) + throw new IllegalArgumentException( + "Size of the given axes a" + " t each dimension must be the same size."); + if (axes[0][i] < 0) + axes[0][i] += aShape.length; + if (axes[1][i] < 0) + axes[1][i] += bShape.length; + + } + + List listA = new ArrayList<>(); + for (int i = 0; i < aShape.length; i++) { + if (!Ints.contains(axes[0], i)) + listA.add(i); + } + + + + List listB = new ArrayList<>(); + for (int i = 0; i < bShape.length; i++) { + if (!Ints.contains(axes[1], i)) + listB.add(i); + } + + + int n2 = 1; + int aLength = Math.min(aShape.length, axes[0].length); + for (int i = 0; i < aLength; i++) { + n2 *= aShape[axes[0][i]]; + } + + //if listA and listB are empty these donot initialize. + //so initializing with {1} which will then get overriden if not empty + long[] oldShapeA; + if (listA.size() == 0) { + oldShapeA = new long[] {1}; + } else { + oldShapeA = Longs.toArray(listA); + for (int i = 0; i < oldShapeA.length; i++) + oldShapeA[i] = aShape[(int) oldShapeA[i]]; + } + + int n3 = 1; + int bNax = Math.min(bShape.length, axes[1].length); + for (int i = 0; i < bNax; i++) { + n3 *= bShape[axes[1][i]]; + } + + + long[] oldShapeB; + if (listB.isEmpty()) { + oldShapeB = new long[] {1}; + } else { + oldShapeB = Longs.toArray(listB); + for (int i = 0; i < oldShapeB.length; i++) + oldShapeB[i] = bShape[(int) oldShapeB[i]]; + } + + + long[] aPlusB = Longs.concat(oldShapeA, oldShapeB); + return aPlusB; + } + + /** + * Permute the given input + * switching the dimensions of the input shape + * array with in the order of the specified + * dimensions + * @param shape the shape to permute + * @param dimensions the dimensions + * @return + */ + public static int[] permute(int[] shape, int[] dimensions) { + int[] ret = new int[shape.length]; + for (int i = 0; i < shape.length; i++) { + ret[i] = shape[dimensions[i]]; + } + + return ret; + } + + + public static long[] permute(long[] shape, int[] dimensions) { + val ret = new long[shape.length]; + for (int i = 0; i < shape.length; i++) { + ret[i] = shape[dimensions[i]]; + } + + return ret; + } + + + /** + * Original credit: https://github.com/alberts/array4j/blob/master/src/main/java/net/lunglet/util/ArrayUtils.java + * @param a + * @return + */ + public static int[] argsort(int[] a) { + return argsort(a, true); + } + + + /** + * + * @param a + * @param ascending + * @return + */ + public static int[] argsort(final int[] a, final boolean ascending) { + Integer[] indexes = new Integer[a.length]; + for (int i = 0; i < indexes.length; i++) { + indexes[i] = i; + } + Arrays.sort(indexes, new Comparator() { + @Override + public int compare(final Integer i1, final Integer i2) { + return (ascending ? 1 : -1) * Ints.compare(a[i1], a[i2]); + } + }); + + int[] ret = new int[indexes.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = indexes[i]; + + return ret; + } + + + + /** + * Convert all dimensions in the specified + * axes array to be positive + * based on the specified range of values + * @param range + * @param axes + * @return + */ + public static int[] convertNegativeIndices(int range, int[] axes) { + int[] axesRet = ArrayUtil.range(0, range); + int[] newAxes = ArrayUtil.copy(axes); + for (int i = 0; i < axes.length; i++) { + newAxes[i] = axes[axesRet[i]]; + } + + return newAxes; + } + + + + /** + * Generate an array from 0 to length + * and generate take a subset + * @param length the length to generate to + * @param from the begin of the interval to take + * @param to the end of the interval to take + * @return the generated array + */ + public static int[] copyOfRangeFrom(int length, int from, int to) { + return Arrays.copyOfRange(ArrayUtil.range(0, length), from, to); + + } + + //Credit: https://stackoverflow.com/questions/15533854/converting-byte-array-to-double-array + + /** + * + * @param doubleArray + * @return + */ + public static byte[] toByteArray(double[] doubleArray) { + int times = Double.SIZE / Byte.SIZE; + byte[] bytes = new byte[doubleArray.length * times]; + for (int i = 0; i < doubleArray.length; i++) { + ByteBuffer.wrap(bytes, i * times, times).putDouble(doubleArray[i]); + } + return bytes; + } + + /** + * + * @param byteArray + * @return + */ + public static double[] toDoubleArray(byte[] byteArray) { + int times = Double.SIZE / Byte.SIZE; + double[] doubles = new double[byteArray.length / times]; + for (int i = 0; i < doubles.length; i++) { + doubles[i] = ByteBuffer.wrap(byteArray, i * times, times).getDouble(); + } + return doubles; + } + + + /** + * + * @param doubleArray + * @return + */ + public static byte[] toByteArray(float[] doubleArray) { + int times = Float.SIZE / Byte.SIZE; + byte[] bytes = new byte[doubleArray.length * times]; + for (int i = 0; i < doubleArray.length; i++) { + ByteBuffer.wrap(bytes, i * times, times).putFloat(doubleArray[i]); + } + return bytes; + } + + public static long[] toLongArray(int[] intArray) { + long[] ret = new long[intArray.length]; + for (int i = 0; i < intArray.length; i++) { + ret[i] = intArray[i]; + } + return ret; + } + + public static long[] toLongArray(float[] array) { + val ret = new long[array.length]; + for (int i = 0; i < array.length; i++) { + ret[i] = (long) array[i]; + } + return ret; + } + + /** + * + * @param byteArray + * @return + */ + public static float[] toFloatArray(byte[] byteArray) { + int times = Float.SIZE / Byte.SIZE; + float[] doubles = new float[byteArray.length / times]; + for (int i = 0; i < doubles.length; i++) { + doubles[i] = ByteBuffer.wrap(byteArray, i * times, times).getFloat(); + } + return doubles; + } + + /** + * + * @param intArray + * @return + */ + public static byte[] toByteArray(int[] intArray) { + int times = Integer.SIZE / Byte.SIZE; + byte[] bytes = new byte[intArray.length * times]; + for (int i = 0; i < intArray.length; i++) { + ByteBuffer.wrap(bytes, i * times, times).putInt(intArray[i]); + } + return bytes; + } + + /** + * + * @param byteArray + * @return + */ + public static int[] toIntArray(byte[] byteArray) { + int times = Integer.SIZE / Byte.SIZE; + int[] ints = new int[byteArray.length / times]; + for (int i = 0; i < ints.length; i++) { + ints[i] = ByteBuffer.wrap(byteArray, i * times, times).getInt(); + } + return ints; + } + + + /** + * Return a copy of this array with the + * given index omitted + * + * @param data the data to copy + * @param index the index of the item to remove + * @return the new array with the omitted + * item + */ + public static int[] removeIndex(int[] data, int index) { + if (data == null) + return null; + + if (index >= data.length) + throw new IllegalArgumentException("Unable to remove index " + index + " was >= data.length"); + if (data.length < 1) + return data; + if (index < 0) + return data; + + int len = data.length; + int[] result = new int[len - 1]; + System.arraycopy(data, 0, result, 0, index); + System.arraycopy(data, index + 1, result, index, len - index - 1); + return result; + } + + public static long[] removeIndex(long[] data, int index) { + if (data == null) + return null; + + if (index >= data.length) + throw new IllegalArgumentException("Unable to remove index " + index + " was >= data.length"); + if (data.length < 1) + return data; + if (index < 0) + return data; + + int len = data.length; + long[] result = new long[len - 1]; + System.arraycopy(data, 0, result, 0, index); + System.arraycopy(data, index + 1, result, index, len - index - 1); + return result; + } + + + /** + * Create a copy of the given array + * starting at the given index with the given length. + * + * The intent here is for striding. + * + * For example in slicing, you want the major stride to be first. + * You achieve this by taking the last index + * of the matrix's stride and putting + * this as the first stride of the new ndarray + * for slicing. + * + * All of the elements except the copied elements are + * initialized as the given value + * @param valueStarting the starting value + * @param copy the array to copy + * @param idxFrom the index to start at in the from array + * @param idxAt the index to start at in the return array + * @param length the length of the array to create + * @return the given array + */ + public static int[] valueStartingAt(int valueStarting, int[] copy, int idxFrom, int idxAt, int length) { + int[] ret = new int[length]; + Arrays.fill(ret, valueStarting); + for (int i = 0; i < length; i++) { + if (i + idxFrom >= copy.length || i + idxAt >= ret.length) + break; + ret[i + idxAt] = copy[i + idxFrom]; + } + + return ret; + } + + + + /** + * Returns the array with the item in index + * removed, if the array is empty it will return the array itself + * + * @param data the data to remove data from + * @param index the index of the item to remove + * @return a copy of the array with the removed item, + * or the array itself if empty + */ + public static Integer[] removeIndex(Integer[] data, int index) { + if (data == null) + return null; + if (data.length < 1) + return data; + int len = data.length; + Integer[] result = new Integer[len - 1]; + System.arraycopy(data, 0, result, 0, index); + System.arraycopy(data, index + 1, result, index, len - index - 1); + return result; + } + + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @param startNum the start number for the strides + * @return the strides for a matrix of n dimensions + */ + public static int[] calcStridesFortran(int[] shape, int startNum) { + if (shape.length == 2 && (shape[0] == 1 || shape[1] == 1)) { + int[] ret = new int[2]; + Arrays.fill(ret, startNum); + return ret; + } + + int dimensions = shape.length; + int[] stride = new int[dimensions]; + int st = startNum; + for (int j = 0; j < stride.length; j++) { + stride[j] = st; + st *= shape[j]; + } + + return stride; + } + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @param startNum the start number for the strides + * @return the strides for a matrix of n dimensions + */ + public static long[] calcStridesFortran(long[] shape, int startNum) { + if (shape.length == 2 && (shape[0] == 1 || shape[1] == 1)) { + long[] ret = new long[2]; + Arrays.fill(ret, startNum); + return ret; + } + + int dimensions = shape.length; + long[] stride = new long[dimensions]; + int st = startNum; + for (int j = 0; j < stride.length; j++) { + stride[j] = st; + st *= shape[j]; + } + + return stride; + } + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @return the strides for a matrix of n dimensions + */ + public static int[] calcStridesFortran(int[] shape) { + return calcStridesFortran(shape, 1); + } + + public static long[] calcStridesFortran(long[] shape) { + return calcStridesFortran(shape, 1); + } + + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @param startValue the startValue for the strides + * @return the strides for a matrix of n dimensions + */ + public static int[] calcStrides(int[] shape, int startValue) { + if (shape.length == 2 && (shape[0] == 1 || shape[1] == 1)) { + int[] ret = new int[2]; + Arrays.fill(ret, startValue); + return ret; + } + + + int dimensions = shape.length; + int[] stride = new int[dimensions]; + + int st = startValue; + for (int j = dimensions - 1; j >= 0; j--) { + stride[j] = st; + st *= shape[j]; + } + + return stride; + } + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @param startValue the startValue for the strides + * @return the strides for a matrix of n dimensions + */ + public static long[] calcStrides(long[] shape, int startValue) { + if (shape.length == 2 && (shape[0] == 1 || shape[1] == 1)) { + long[] ret = new long[2]; + Arrays.fill(ret, startValue); + return ret; + } + + + int dimensions = shape.length; + long[] stride = new long[dimensions]; + + int st = startValue; + for (int j = dimensions - 1; j >= 0; j--) { + stride[j] = st; + st *= shape[j]; + } + + return stride; + } + + + /** + * Returns true if the given + * two arrays are reverse copies of each other + * @param first + * @param second + * @return + */ + public static boolean isInverse(int[] first, int[] second) { + int backWardCount = second.length - 1; + for (int i = 0; i < first.length; i++) { + if (first[i] != second[backWardCount--]) + return false; + } + return true; + } + + public static int[] plus(int[] ints, int mult) { + int[] ret = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = ints[i] + mult; + return ret; + } + + + public static int[] plus(int[] ints, int[] mult) { + if (ints.length != mult.length) + throw new IllegalArgumentException("Both arrays must have the same length"); + int[] ret = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = ints[i] + mult[i]; + return ret; + } + + public static int[] times(int[] ints, int mult) { + int[] ret = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = ints[i] * mult; + return ret; + } + + public static int[] times(int[] ints, int[] mult) { + Preconditions.checkArgument(ints.length == mult.length, "Ints and mult must be the same length"); + int[] ret = new int[ints.length]; + for (int i = 0; i < ints.length; i++) + ret[i] = ints[i] * mult[i]; + return ret; + } + + + + /** + * For use with row vectors to ensure consistent strides + * with varying offsets + * + * @param arr the array to get the stride for + * @return the stride + */ + public static int nonOneStride(int[] arr) { + for (int i = 0; i < arr.length; i++) + if (arr[i] != 1) + return arr[i]; + return 1; + } + + + /** + * Computes the standard packed array strides for a given shape. + * + * @param shape the shape of a matrix: + * @return the strides for a matrix of n dimensions + */ + public static int[] calcStrides(int[] shape) { + return calcStrides(shape, 1); + } + + public static long[] calcStrides(long[] shape) { + return calcStrides(shape, 1); + } + + + /** + * Create a backwards copy of the given array + * + * @param e the array to createComplex a reverse clone of + * @return the reversed copy + */ + public static int[] reverseCopy(int[] e) { + if (e.length < 1) + return e; + + int[] copy = new int[e.length]; + for (int i = 0; i <= e.length / 2; i++) { + int temp = e[i]; + copy[i] = e[e.length - i - 1]; + copy[e.length - i - 1] = temp; + } + return copy; + } + + public static long[] reverseCopy(long[] e) { + if (e.length < 1) + return e; + + long[] copy = new long[e.length]; + for (int i = 0; i <= e.length / 2; i++) { + long temp = e[i]; + copy[i] = e[e.length - i - 1]; + copy[e.length - i - 1] = temp; + } + return copy; + } + + + public static double[] read(int length, DataInputStream dis) throws IOException { + double[] ret = new double[length]; + for (int i = 0; i < length; i++) + ret[i] = dis.readDouble(); + return ret; + } + + + public static void write(double[] data, DataOutputStream dos) throws IOException { + for (int i = 0; i < data.length; i++) + dos.writeDouble(data[i]); + } + + public static double[] readDouble(int length, DataInputStream dis) throws IOException { + double[] ret = new double[length]; + for (int i = 0; i < length; i++) + ret[i] = dis.readDouble(); + return ret; + } + + + public static float[] readFloat(int length, DataInputStream dis) throws IOException { + float[] ret = new float[length]; + for (int i = 0; i < length; i++) + ret[i] = dis.readFloat(); + return ret; + } + + + public static void write(float[] data, DataOutputStream dos) throws IOException { + for (int i = 0; i < data.length; i++) + dos.writeFloat(data[i]); + } + + + public static void assertSquare(double[]... d) { + if (d.length > 2) { + for (int i = 0; i < d.length; i++) { + assertSquare(d[i]); + } + } else { + int firstLength = d[0].length; + for (int i = 1; i < d.length; i++) { + Preconditions.checkState(d[i].length == firstLength); + } + } + } + + + /** + * Multiply the given array + * by the given scalar + * @param arr the array to multily + * @param mult the scalar to multiply by + */ + public static void multiplyBy(int[] arr, int mult) { + for (int i = 0; i < arr.length; i++) + arr[i] *= mult; + + } + + /** + * Reverse the passed in array in place + * + * @param e the array to reverse + */ + public static void reverse(int[] e) { + for (int i = 0; i <= e.length / 2; i++) { + int temp = e[i]; + e[i] = e[e.length - i - 1]; + e[e.length - i - 1] = temp; + } + } + + public static void reverse(long[] e) { + for (int i = 0; i <= e.length / 2; i++) { + long temp = e[i]; + e[i] = e[e.length - i - 1]; + e[e.length - i - 1] = temp; + } + } + + + public static List zerosMatrix(long... dimensions) { + List ret = new ArrayList<>(); + for (int i = 0; i < dimensions.length; i++) { + ret.add(new double[(int) dimensions[i]]); + } + return ret; + } + + public static List zerosMatrix(int... dimensions) { + List ret = new ArrayList<>(); + for (int i = 0; i < dimensions.length; i++) { + ret.add(new double[dimensions[i]]); + } + return ret; + } + + + public static float[] reverseCopy(float[] e) { + float[] copy = new float[e.length]; + for (int i = 0; i <= e.length / 2; i++) { + float temp = e[i]; + copy[i] = e[e.length - i - 1]; + copy[e.length - i - 1] = temp; + } + return copy; + + } + + + public static E[] reverseCopy(E[] e) { + E[] copy = (E[]) new Object[e.length]; + for (int i = 0; i <= e.length / 2; i++) { + E temp = e[i]; + copy[i] = e[e.length - i - 1]; + copy[e.length - i - 1] = temp; + } + return copy; + + } + + public static void reverse(E[] e) { + for (int i = 0; i <= e.length / 2; i++) { + E temp = e[i]; + e[i] = e[e.length - i - 1]; + e[e.length - i - 1] = temp; + } + } + + public static boolean[] flatten(boolean[][] arr) { + if(arr.length == 0 || arr[0].length == 0) + return new boolean[0]; + boolean[] ret = new boolean[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static boolean[] flatten(boolean[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new boolean[0]; + boolean[] ret = new boolean[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static float[] flatten(float[][] arr) { + if(arr.length == 0 || arr[0].length == 0) + return new float[0]; + float[] ret = new float[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + + public static float[] flatten(float[][][] arr) { + if (arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new float[0]; + float[] ret = new float[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + + return ret; + } + + public static double[] flatten(double[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new double[0]; + double[] ret = new double[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static int[] flatten(int[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new int[0]; + int[] ret = new int[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static short[] flatten(short[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new short[0]; + val ret = new short[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static byte[] flatten(byte[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new byte[0]; + val ret = new byte[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + public static long[] flatten(long[][][][] arr) { + val ret = new long[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static short[] flatten(short[][][][] arr) { + val ret = new short[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static byte[] flatten(byte[][][][] arr) { + val ret = new byte[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static boolean[] flatten(boolean[][][][] arr) { + val ret = new boolean[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static float[] flatten(float[][][][] arr) { + float[] ret = new float[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static double[] flatten(double[][][][] arr) { + double[] ret = new double[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + public static int[] flatten(int[][][][] arr) { + int[] ret = new int[arr.length * arr[0].length * arr[0][0].length * arr[0][0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + for (int k = 0; k < arr[0][0].length; k++) { + System.arraycopy(arr[i][j][k], 0, ret, count, arr[0][0][0].length); + count += arr[0][0][0].length; + } + } + } + + return ret; + } + + + public static int[] flatten(int[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new int[0]; + int[] ret = new int[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static short[] flatten(short[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new short[0]; + val ret = new short[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static byte[] flatten(byte[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new byte[0]; + val ret = new byte[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static long[] flatten(long[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new long[0]; + long[] ret = new long[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + public static long[] flatten(long[][][] arr) { + if(arr.length == 0 || arr[0].length == 0 || arr[0][0].length == 0) + return new long[0]; + long[] ret = new long[arr.length * arr[0].length * arr[0][0].length]; + + int count = 0; + for (int i = 0; i < arr.length; i++) { + for (int j = 0; j < arr[0].length; j++) { + System.arraycopy(arr[i][j], 0, ret, count, arr[0][0].length); + count += arr[0][0].length; + } + } + return ret; + } + + + /** + * Convert a 2darray in to a flat + * array (row wise) + * @param arr the array to flatten + * @return a flattened representation of the array + */ + public static double[] flatten(double[][] arr) { + if(arr.length == 0 || arr[0].length == 0 ) + return new double[0]; + double[] ret = new double[arr.length * arr[0].length]; + int count = 0; + for (int i = 0; i < arr.length; i++) { + System.arraycopy(arr[i], 0, ret, count, arr[i].length); + count += arr[i].length; + } + return ret; + } + + /** + * Convert a 2darray in to a flat + * array (row wise) + * @param arr the array to flatten + * @return a flattened representation of the array + */ + public static double[] flattenF(double[][] arr) { + double[] ret = new double[arr.length * arr[0].length]; + int count = 0; + for (int j = 0; j < arr[0].length; j++) + for (int i = 0; i < arr.length; i++) + ret[count++] = arr[i][j]; + return ret; + } + + public static float[] flattenF(float[][] arr) { + float[] ret = new float[arr.length * arr[0].length]; + int count = 0; + for (int j = 0; j < arr[0].length; j++) + for (int i = 0; i < arr.length; i++) + ret[count++] = arr[i][j]; + return ret; + } + + public static int[] flattenF(int[][] arr) { + int[] ret = new int[arr.length * arr[0].length]; + int count = 0; + for (int j = 0; j < arr[0].length; j++) + for (int i = 0; i < arr.length; i++) + ret[count++] = arr[i][j]; + return ret; + } + + + public static long[] flattenF(long[][] arr) { + long[] ret = new long[arr.length * arr[0].length]; + int count = 0; + for (int j = 0; j < arr[0].length; j++) + for (int i = 0; i < arr.length; i++) + ret[count++] = arr[i][j]; + return ret; + } + + public static int[][] reshapeInt(int[] in, int rows, int cols){ + int[][] out = new int[rows][cols]; + int x = 0; + for(int i=0; i T[][] reshapeObject(T[] in, int rows, int cols){ + Object[][] out = new Object[rows][cols]; + int x = 0; + for(int i=0; i T[][][] reshapeObject(T[] in, int d0, int d1, int d2){ + Object[][][] out = new Object[d0][d1][d2]; + int x = 0; + for(int i=0; i nums) { + int length = 0; + for (int i = 0; i < nums.size(); i++) + length += nums.get(i).length; + float[] ret = new float[length]; + int count = 0; + for (float[] i : nums) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + + /** + * Combines a apply of int arrays in to one flat int array + * + * @param nums the int arrays to combineDouble + * @return one combined int array + */ + public static float[] combine(List nums) { + int length = 0; + for (int i = 0; i < nums.size(); i++) + length += nums.get(i).length; + float[] ret = new float[length]; + int count = 0; + for (float[] i : nums) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + /** + * Combines a apply of int arrays in to one flat int array + * + * @param nums the int arrays to combineDouble + * @return one combined int array + */ + public static double[] combineDouble(List nums) { + int length = 0; + for (int i = 0; i < nums.size(); i++) + length += nums.get(i).length; + double[] ret = new double[length]; + int count = 0; + for (double[] i : nums) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + /** + * Combines a apply of int arrays in to one flat int array + * + * @param ints the int arrays to combineDouble + * @return one combined int array + */ + public static double[] combine(float[]... ints) { + int length = 0; + for (int i = 0; i < ints.length; i++) + length += ints[i].length; + double[] ret = new double[length]; + int count = 0; + for (float[] i : ints) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + /** + * Combines a apply of int arrays in to one flat int array + * + * @param ints the int arrays to combineDouble + * @return one combined int array + */ + public static int[] combine(int[]... ints) { + int length = 0; + for (int i = 0; i < ints.length; i++) + length += ints[i].length; + int[] ret = new int[length]; + int count = 0; + for (int[] i : ints) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + /** + * Combines a apply of long arrays in to one flat long array + * + * @param ints the int arrays to combineDouble + * @return one combined int array + */ + public static long[] combine(long[]... ints) { + int length = 0; + for (int i = 0; i < ints.length; i++) + length += ints[i].length; + long[] ret = new long[length]; + int count = 0; + for (long[] i : ints) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + + public static E[] combine(E[]... arrs) { + int length = 0; + for (int i = 0; i < arrs.length; i++) + length += arrs[i].length; + + E[] ret = (E[]) Array.newInstance(arrs[0][0].getClass(), length); + int count = 0; + for (E[] i : arrs) { + for (int j = 0; j < i.length; j++) { + ret[count++] = i[j]; + } + } + + return ret; + } + + + public static int[] toOutcomeArray(int outcome, int numOutcomes) { + int[] nums = new int[numOutcomes]; + nums[outcome] = 1; + return nums; + } + + public static double[] toDouble(int[] data) { + double[] ret = new double[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = data[i]; + return ret; + } + + public static double[] toDouble(long[] data) { + double[] ret = new double[data.length]; + for (int i = 0; i < ret.length; i++) + ret[i] = data[i]; + return ret; + } + + public static float[] copy(float[] data) { + float[] result = new float[data.length]; + System.arraycopy(data, 0, result, 0, data.length); + return result; + } + + public static double[] copy(double[] data) { + double[] result = new double[data.length]; + System.arraycopy(data, 0, result, 0, data.length); + return result; + } + + + /** Convert an arbitrary-dimensional rectangular double array to flat vector.
+ * Can pass double[], double[][], double[][][], etc. + */ + public static double[] flattenDoubleArray(Object doubleArray) { + if (doubleArray instanceof double[]) + return (double[]) doubleArray; + + LinkedList stack = new LinkedList<>(); + stack.push(doubleArray); + + int[] shape = arrayShape(doubleArray); + int length = ArrayUtil.prod(shape); + double[] flat = new double[length]; + int count = 0; + + while (!stack.isEmpty()) { + Object current = stack.pop(); + if (current instanceof double[]) { + double[] arr = (double[]) current; + for (int i = 0; i < arr.length; i++) + flat[count++] = arr[i]; + } else if (current instanceof Object[]) { + Object[] o = (Object[]) current; + for (int i = o.length - 1; i >= 0; i--) + stack.push(o[i]); + } else + throw new IllegalArgumentException("Base array is not double[]"); + } + + if (count != flat.length) + throw new IllegalArgumentException("Fewer elements than expected. Array is ragged?"); + return flat; + } + + /** Convert an arbitrary-dimensional rectangular float array to flat vector.
+ * Can pass float[], float[][], float[][][], etc. + */ + public static float[] flattenFloatArray(Object floatArray) { + if (floatArray instanceof float[]) + return (float[]) floatArray; + + LinkedList stack = new LinkedList<>(); + stack.push(floatArray); + + int[] shape = arrayShape(floatArray); + int length = ArrayUtil.prod(shape); + float[] flat = new float[length]; + int count = 0; + + while (!stack.isEmpty()) { + Object current = stack.pop(); + if (current instanceof float[]) { + float[] arr = (float[]) current; + for (int i = 0; i < arr.length; i++) + flat[count++] = arr[i]; + } else if (current instanceof Object[]) { + Object[] o = (Object[]) current; + for (int i = o.length - 1; i >= 0; i--) + stack.push(o[i]); + } else + throw new IllegalArgumentException("Base array is not float[]"); + } + + if (count != flat.length) + throw new IllegalArgumentException("Fewer elements than expected. Array is ragged?"); + return flat; + } + + /** Calculate the shape of an arbitrary multi-dimensional array. Assumes:
+ * (a) array is rectangular (not ragged) and first elements (i.e., array[0][0][0]...) are non-null
+ * (b) First elements have > 0 length. So array[0].length > 0, array[0][0].length > 0, etc.
+ * Can pass any Java array opType: double[], Object[][][], float[][], etc.
+ * Length of returned array is number of dimensions; returned[i] is size of ith dimension. + */ + public static int[] arrayShape(Object array) { + return arrayShape(array, false); + } + + /** Calculate the shape of an arbitrary multi-dimensional array.
+ * Note that the method assumes the array is rectangular (not ragged) and first elements (i.e., array[0][0][0]...) are non-null
+ * Note also that if allowSize0Dims is true, any elements are length 0, all subsequent dimensions will be reported as 0. + * i.e., a double[3][0][2] would be reported as shape [3,0,0]. If allowSize0Dims is false, an exception will be thrown for this case instead. + * Can pass any Java array opType: double[], Object[][][], float[][], etc.
+ * Length of returned array is number of dimensions; returned[i] is size of ith dimension. + */ + public static int[] arrayShape(Object array, boolean allowSize0Dims) { + int nDimensions = 0; + Class c = array.getClass().getComponentType(); + while (c != null) { + nDimensions++; + c = c.getComponentType(); + } + + int[] shape = new int[nDimensions]; + Object current = array; + for (int i = 0; i < shape.length - 1; i++) { + shape[i] = ((Object[]) current).length; + if(shape[i] == 0){ + if(allowSize0Dims){ + return shape; + } + throw new IllegalStateException("Cannot calculate array shape: Array has size 0 for dimension " + i ); + } + current = ((Object[]) current)[0]; + } + + if (current instanceof Object[]) { + shape[shape.length - 1] = ((Object[]) current).length; + } else if (current instanceof double[]) { + shape[shape.length - 1] = ((double[]) current).length; + } else if (current instanceof float[]) { + shape[shape.length - 1] = ((float[]) current).length; + } else if (current instanceof long[]) { + shape[shape.length - 1] = ((long[]) current).length; + } else if (current instanceof int[]) { + shape[shape.length - 1] = ((int[]) current).length; + } else if (current instanceof byte[]) { + shape[shape.length - 1] = ((byte[]) current).length; + } else if (current instanceof char[]) { + shape[shape.length - 1] = ((char[]) current).length; + } else if (current instanceof boolean[]) { + shape[shape.length - 1] = ((boolean[]) current).length; + } else if (current instanceof short[]) { + shape[shape.length - 1] = ((short[]) current).length; + } else + throw new IllegalStateException("Unknown array type"); //Should never happen + return shape; + } + + + /** Returns the maximum value in the array */ + public static int max(int[] in) { + int max = Integer.MIN_VALUE; + for (int i = 0; i < in.length; i++) + if (in[i] > max) + max = in[i]; + return max; + } + + /** Returns the minimum value in the array */ + public static int min(int[] in) { + int min = Integer.MAX_VALUE; + for (int i = 0; i < in.length; i++) + if (in[i] < min) + min = in[i]; + return min; + } + + /** Returns the index of the maximum value in the array. + * If two entries have same maximum value, index of the first one is returned. */ + public static int argMax(int[] in) { + int maxIdx = 0; + for (int i = 1; i < in.length; i++) + if (in[i] > in[maxIdx]) + maxIdx = i; + return maxIdx; + } + + /** Returns the index of the minimum value in the array. + * If two entries have same minimum value, index of the first one is returned. */ + public static int argMin(int[] in) { + int minIdx = 0; + for (int i = 1; i < in.length; i++) + if (in[i] < in[minIdx]) + minIdx = i; + return minIdx; + } + + /** Returns the index of the maximum value in the array. + * If two entries have same maximum value, index of the first one is returned. */ + public static int argMax(long[] in) { + int maxIdx = 0; + for (int i = 1; i < in.length; i++) + if (in[i] > in[maxIdx]) + maxIdx = i; + return maxIdx; + } + + /** Returns the index of the minimum value in the array. + * If two entries have same minimum value, index of the first one is returned. */ + public static int argMin(long[] in) { + int minIdx = 0; + for (int i = 1; i < in.length; i++) + if (in[i] < in[minIdx]) + minIdx = i; + return minIdx; + } + + /** + * + * @return + */ + public static int[] buildHalfVector(Random rng, int length) { + int[] result = new int[length]; + List indexes = new ArrayList<>(); + + // we add indexes from second half only + for (int i = result.length - 1; i >= result.length / 2; i--) { + indexes.add(i); + } + + Collections.shuffle(indexes, rng); + + for (int i = 0; i < result.length; i++) { + if (i < result.length / 2) { + result[i] = indexes.get(0); + indexes.remove(0); + } else + result[i] = -1; + } + + return result; + } + + public static int[] buildInterleavedVector(Random rng, int length) { + int[] result = new int[length]; + + List indexes = new ArrayList<>(); + List odds = new ArrayList<>(); + + // we add odd indexes only to list + for (int i = 1; i < result.length; i += 2) { + indexes.add(i); + odds.add(i - 1); + } + + Collections.shuffle(indexes, rng); + + // now all even elements will be interleaved with odd elements + for (int i = 0; i < result.length; i++) { + if (i % 2 == 0 && !indexes.isEmpty()) { + int idx = indexes.get(0); + indexes.remove(0); + result[i] = idx; + } else + result[i] = -1; + } + + // for odd tad numbers, we add special random clause for last element + if (length % 2 != 0) { + int rndClause = odds.get(rng.nextInt(odds.size())); + int tmp = result[rndClause]; + result[rndClause] = result[result.length - 1]; + result[result.length - 1] = tmp; + } + + + return result; + } + + public static long[] buildInterleavedVector(Random rng, long length) { + if (length > Integer.MAX_VALUE) { + throw new RuntimeException("Integer overflow"); + } + val result = new long[(int) length]; + + List indexes = new ArrayList<>(); + List odds = new ArrayList<>(); + + // we add odd indexes only to list + for (int i = 1; i < result.length; i += 2) { + indexes.add(i); + odds.add(i - 1); + } + + Collections.shuffle(indexes, rng); + + // now all even elements will be interleaved with odd elements + for (int i = 0; i < result.length; i++) { + if (i % 2 == 0 && !indexes.isEmpty()) { + int idx = indexes.get(0); + indexes.remove(0); + result[i] = idx; + } else + result[i] = -1; + } + + // for odd tad numbers, we add special random clause for last element + if (length % 2 != 0) { + int rndClause = odds.get(rng.nextInt(odds.size())); + long tmp = result[rndClause]; + result[rndClause] = result[result.length - 1]; + result[result.length - 1] = tmp; + } + + + return result; + } + + protected static void swap(List objects, int idxA, int idxB) { + T tmpA = objects.get(idxA); + T tmpB = objects.get(idxB); + objects.set(idxA, tmpB); + objects.set(idxB, tmpA); + } + + public static void shuffleWithMap(List objects, int[] map) { + for (int i = 0; i < map.length; i++) { + if (map[i] >= 0) { + swap(objects, i, map[i]); + } + } + } + + public static int argMinOfMax(int[] first, int[] second) { + int minIdx = 0; + int maxAtMinIdx = Math.max(first[0], second[0]); + for (int i = 1; i < first.length; i++) { + int maxAtIndex = Math.max(first[i], second[i]); + if (maxAtMinIdx > maxAtIndex) { + maxAtMinIdx = maxAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static long argMinOfMax(long[] first, long[] second) { + long minIdx = 0; + long maxAtMinIdx = Math.max(first[0], second[0]); + for (int i = 1; i < first.length; i++) { + long maxAtIndex = Math.max(first[i], second[i]); + if (maxAtMinIdx > maxAtIndex) { + maxAtMinIdx = maxAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static int argMinOfMax(int[]... arrays) { + int minIdx = 0; + int maxAtMinIdx = Integer.MAX_VALUE; + + for (int i = 0; i < arrays[0].length; i++) { + int maxAtIndex = Integer.MIN_VALUE; + for (int j = 0; j < arrays.length; j++) { + maxAtIndex = Math.max(maxAtIndex, arrays[j][i]); + } + + if (maxAtMinIdx > maxAtIndex) { + maxAtMinIdx = maxAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static long argMinOfMax(long[]... arrays) { + int minIdx = 0; + long maxAtMinIdx = Long.MAX_VALUE; + + for (int i = 0; i < arrays[0].length; i++) { + long maxAtIndex = Long.MIN_VALUE; + for (int j = 0; j < arrays.length; j++) { + maxAtIndex = Math.max(maxAtIndex, arrays[j][i]); + } + + if (maxAtMinIdx > maxAtIndex) { + maxAtMinIdx = maxAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static int argMinOfSum(int[] first, int[] second) { + int minIdx = 0; + int sumAtMinIdx = first[0] + second[0]; + for (int i = 1; i < first.length; i++) { + int sumAtIndex = first[i] + second[i]; + if (sumAtMinIdx > sumAtIndex) { + sumAtMinIdx = sumAtIndex; + minIdx = i; + } + } + return minIdx; + } + + public static > Map sortMapByValue(Map map) { + List> list = new LinkedList<>(map.entrySet()); + Collections.sort(list, new Comparator>() { + @Override + public int compare(Map.Entry o1, Map.Entry o2) { + return (o1.getValue()).compareTo(o2.getValue()); + } + }); + + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : list) { + result.put(entry.getKey(), entry.getValue()); + } + return result; + } + + + public static T getRandomElement(List list) { + if (list.isEmpty()) + return null; + + return list.get(RandomUtils.nextInt(0, list.size())); + } + + /** + * Convert an int + * @param bool + * @return + */ + public static int fromBoolean(boolean bool) { + return bool ? 1 : 0; + } + + public static long[] toPrimitives(Long[] array) { + val res = new long[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static int[] toPrimitives(Integer[] array) { + val res = new int[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static short[] toPrimitives(Short[] array) { + val res = new short[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static byte[] toPrimitives(Byte[] array) { + val res = new byte[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static float[] toPrimitives(Float[] array) { + val res = new float[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static double[] toPrimitives(Double[] array) { + val res = new double[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static boolean[] toPrimitives(Boolean[] array) { + val res = new boolean[array.length]; + for (int e = 0; e < array.length; e++) + res[e] = array[e]; + + return res; + } + + public static long[][] toPrimitives(Long[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new long[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static int[][] toPrimitives(Integer[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new int[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static short[][] toPrimitives(Short[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new short[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static byte[][] toPrimitives(Byte[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new byte[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static double[][] toPrimitives(Double[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new double[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static float[][] toPrimitives(Float[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new float[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static boolean [][] toPrimitives(Boolean[][] array) { + ArrayUtil.assertNotRagged(array); + val res = new boolean[array.length][array[0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + res[i][j] = array[i][j]; + + return res; + } + + public static long[][][] toPrimitives(Long[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new long[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static int[][][] toPrimitives(Integer[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new int[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static short[][][] toPrimitives(Short[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new short[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static byte[][][] toPrimitives(Byte[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new byte[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static double[][][] toPrimitives(Double[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new double[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static float[][][] toPrimitives(Float[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new float[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static boolean[][][] toPrimitives(Boolean[][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new boolean[array.length][array[0].length][array[0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + res[i][j][k] = array[i][j][k]; + + return res; + } + + public static long[][][][] toPrimitives(Long[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new long[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static int[][][][] toPrimitives(Integer[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new int[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static short[][][][] toPrimitives(Short[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new short[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static byte[][][][] toPrimitives(Byte[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new byte[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static double[][][][] toPrimitives(Double[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new double[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static float[][][][] toPrimitives(Float[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new float[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + public static boolean[][][][] toPrimitives(Boolean[][][][] array) { + ArrayUtil.assertNotRagged(array); + val res = new boolean[array.length][array[0].length][array[0][0].length][array[0][0][0].length]; + for (int i = 0; i < array.length; i++) + for (int j = 0; j < array[0].length; j++) + for (int k = 0; j < array[0][0].length; k++) + for (int l = 0; l < array[0][0][0].length; l++) + res[i][j][k][l] = array[i][j][k][l]; + + return res; + } + + + /** + * Assert that the specified array is not ragged (i.e., is rectangular).
+ * Can be used to check Object arrays with any number of dimensions (up to rank 4), or primitive arrays with rank 2 or higher
+ * An IllegalStateException is thrown if the array is ragged + * + * @param array Array to check + */ + public static void assertNotRagged(T[] array){ + Class c = array.getClass().getComponentType(); + int[] arrayShape = ArrayUtil.arrayShape(array, true); + int rank = arrayShape.length; + + if(rank == 1){ + //Rank 1 cannot be ragged + return; + } + + if(rank >= 2){ + for( int i=1; i= 3){ + + for( int i=0; i= 4){ + for( int i=0; i + * Example: if input is [2, 0, 1] then output is [1, 2, 0]
+ * The idea is that x.permute(input).permute(invertPermutation(input)) == x + * + * @param input 1D indices for permutation + * @return 1D inverted permutation + */ + public static int[] invertPermutation(int... input){ + int[] target = new int[input.length]; + + for(int i = 0 ; i < input.length ; i++){ + target[input[i]] = i; + } + + return target; + } + + /** + * @see #invertPermutation(int...) + * + * @param input 1D indices for permutation + * @return 1D inverted permutation + */ + public static long[] invertPermutation(long... input){ + long[] target = new long[input.length]; + + for(int i = 0 ; i < input.length ; i++){ + target[(int) input[i]] = i; + } + + return target; + } + + /** + * Is this shape an empty shape? + * Shape is considered to be an empty shape if it contains any zeros. + * Note: a length 0 shape is NOT considered empty (it's rank 0 scalar) + * @param shape Shape to check + * @return True if shape contains zeros + */ + public static boolean isEmptyShape(long[] shape){ + for( long l : shape){ + if(l == 0) + return true; + } + return false; + } + + /** + * Is this shape an empty shape? + * Shape is considered to be an empty shape if it contains any zeros. + * Note: a length 0 shape is NOT considered empty (it's rank 0 scalar) + * @param shape Shape to check + * @return True if shape contains zeros + */ + public static boolean isEmptyShape(int[] shape){ + for( int i : shape){ + if(i == 0) + return true; + } + return false; + } + + public static T[] filterNull(T... in){ + int count = 0; + for( int i=0; i a = new ArrayList(); + + public Bernoulli() { + if (a.isEmpty()) { + a.add(Rational.ONE); + a.add(new Rational(1, 6)); + } + } + + /** + * Set a coefficient in the internal table. + * + * @param n the zero-based index of the coefficient. n=0 for the constant term. + * @param value the new value of the coefficient. + */ + protected void set(final int n, final Rational value) { + final int nindx = n / 2; + if (nindx < a.size()) { + a.set(nindx, value); + } else { + while (a.size() < nindx) { + a.add(Rational.ZERO); + } + a.add(value); + } + } + + /** + * The Bernoulli number at the index provided. + * + * @param n the index, non-negative. + * @return the B_0=1 for n=0, B_1=-1/2 for n=1, B_2=1/6 for n=2 etc + */ + public Rational at(int n) { + if (n == 1) { + return (new Rational(-1, 2)); + } else if (n % 2 != 0) { + return Rational.ZERO; + } else { + final int nindx = n / 2; + if (a.size() <= nindx) { + for (int i = 2 * a.size(); i <= n; i += 2) { + set(i, doubleSum(i)); + } + } + return a.get(nindx); + } + } + /* Generate a new B_n by a standard double sum. + * @param n The index of the Bernoulli number. + * @return The Bernoulli number at n. + */ + + private Rational doubleSum(int n) { + Rational resul = Rational.ZERO; + for (int k = 0; k <= n; k++) { + Rational jsum = Rational.ZERO; + BigInteger bin = BigInteger.ONE; + for (int j = 0; j <= k; j++) { + BigInteger jpown = BigInteger.valueOf(j).pow(n); + if (j % 2 == 0) { + jsum = jsum.add(bin.multiply(jpown)); + } else { + jsum = jsum.subtract(bin.multiply(jpown)); + } + /* update binomial(k,j) recursively + */ + bin = bin.multiply(BigInteger.valueOf(k - j)).divide(BigInteger.valueOf(j + 1)); + } + resul = resul.add(jsum.divide(BigInteger.valueOf(k + 1))); + } + return resul; + } +} /* Bernoulli */ diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Factorial.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Factorial.java new file mode 100644 index 000000000..87efeffea --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Factorial.java @@ -0,0 +1,64 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +/** + * + * @author dmtrl + */ + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +class Factorial { + + /** + * The list of all factorials as a vector. + */ + static List a = new ArrayList<>(); + + /** + * ctor(). + * Initialize the vector of the factorials with 0!=1 and 1!=1. + */ + public Factorial() { + if (a.isEmpty()) { + a.add(BigInteger.ONE); + a.add(BigInteger.ONE); + } + } + + /** + * Compute the factorial of the non-negative integer. + * + * @param n the argument to the factorial, non-negative. + * @return the factorial of n. + */ + public BigInteger at(int n) { + while (a.size() <= n) { + final int lastn = a.size() - 1; + final BigInteger nextn = BigInteger.valueOf(lastn + 1); + a.add(a.get(lastn).multiply(nextn)); + } + return a.get(n); + } +} /* Factorial */ diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Index.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Index.java new file mode 100644 index 000000000..cc64e145d --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Index.java @@ -0,0 +1,118 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + + +import java.io.Serializable; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@SuppressWarnings({"rawtypes", "unchecked"}) +public class Index implements Serializable { + + private static final long serialVersionUID = 1160629777026141078L; + private Map objects = new ConcurrentHashMap<>(); + private Map indexes = new ConcurrentHashMap<>(); + + public synchronized boolean add(Object o, int idx) { + if (o instanceof String && o.toString().isEmpty()) { + throw new IllegalArgumentException("Unable to add the empty string"); + } + + Integer index = indexes.get(o); + if (index == null) { + index = idx; + objects.put(idx, o); + indexes.put(o, index); + return true; + } + return false; + } + + public synchronized boolean add(Object o) { + if (o instanceof String && o.toString().isEmpty()) { + throw new IllegalArgumentException("Unable to add the empty string"); + } + Integer index = indexes.get(o); + if (index == null) { + index = objects.size(); + objects.put(index, o); + indexes.put(o, index); + return true; + } + return false; + } + + public synchronized int indexOf(Object o) { + Integer index = indexes.get(o); + if (index == null) { + return -1; + } else { + return index; + } + } + + public synchronized Object get(int i) { + return objects.get(i); + } + + public int size() { + return objects.size(); + } + + @Override + public String toString() { + StringBuilder buff = new StringBuilder("["); + int sz = objects.size(); + int i; + for (i = 0; i < sz; i++) { + Object e = objects.get(i); + buff.append(e); + if (i < (sz - 1)) + buff.append(" , "); + } + buff.append("]"); + return buff.toString(); + + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Index index = (Index) o; + + if (objects != null ? !objects.equals(index.objects) : index.objects != null) + return false; + return !(indexes != null ? !indexes.equals(index.indexes) : index.indexes != null); + + } + + @Override + public int hashCode() { + int result = objects != null ? objects.hashCode() : 0; + result = 31 * result + (indexes != null ? indexes.hashCode() : 0); + return result; + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/InputStreamUtil.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/InputStreamUtil.java new file mode 100644 index 000000000..f8bf6a9c4 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/InputStreamUtil.java @@ -0,0 +1,73 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class InputStreamUtil { + /** + * Count number of lines in a file + * + * @param is + * @return + * @throws IOException + */ + public static int countLines(InputStream is) throws IOException { + try { + byte[] c = new byte[1024]; + int count = 0; + int readChars = 0; + boolean empty = true; + while ((readChars = is.read(c)) != -1) { + empty = false; + for (int i = 0; i < readChars; ++i) { + if (c[i] == '\n') { + ++count; + } + } + } + return (count == 0 && !empty) ? 1 : count; + } finally { + is.close(); + } + + + } + + /** + * Count number of lines in a file + * + * @param filename + * @return + * @throws IOException + */ + public static int countLines(String filename) throws IOException { + FileInputStream fis = new FileInputStream(filename); + try { + return countLines(new BufferedInputStream(fis)); + } finally { + fis.close(); + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/LinkedMultiValueMap.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/LinkedMultiValueMap.java new file mode 100644 index 000000000..3faf22375 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/LinkedMultiValueMap.java @@ -0,0 +1,144 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import java.io.Serializable; +import java.util.*; + +public class LinkedMultiValueMap implements MultiValueMap, Serializable { + private static final long serialVersionUID = 3801124242820219131L; + private final Map> targetMap; + + public LinkedMultiValueMap() { + this.targetMap = new LinkedHashMap(); + } + + public LinkedMultiValueMap(int initialCapacity) { + this.targetMap = new LinkedHashMap(initialCapacity); + } + + public LinkedMultiValueMap(Map> otherMap) { + this.targetMap = new LinkedHashMap(otherMap); + } + + public void add(K key, V value) { + List values = this.targetMap.get(key); + if (values == null) { + values = new LinkedList<>(); + this.targetMap.put(key, values); + } + + values.add(value); + } + + public V getFirst(K key) { + List values = this.targetMap.get(key); + return values != null ? values.get(0) : null; + } + + public void set(K key, V value) { + LinkedList values = new LinkedList(); + values.add(value); + this.targetMap.put(key, values); + } + + public void setAll(Map values) { + Iterator i$ = values.entrySet().iterator(); + + while (i$.hasNext()) { + Entry entry = (Entry) i$.next(); + this.set(entry.getKey(), entry.getValue()); + } + + } + + public Map toSingleValueMap() { + LinkedHashMap singleValueMap = new LinkedHashMap(this.targetMap.size()); + Iterator i$ = this.targetMap.entrySet().iterator(); + + while (i$.hasNext()) { + Entry entry = (Entry) i$.next(); + singleValueMap.put(entry.getKey(), ((List) entry.getValue()).get(0)); + } + + return singleValueMap; + } + + public int size() { + return this.targetMap.size(); + } + + public boolean isEmpty() { + return this.targetMap.isEmpty(); + } + + public boolean containsKey(Object key) { + return this.targetMap.containsKey(key); + } + + public boolean containsValue(Object value) { + return this.targetMap.containsValue(value); + } + + public List get(Object key) { + return this.targetMap.get(key); + } + + public List put(K key, List value) { + return this.targetMap.put(key, value); + } + + public List remove(Object key) { + return this.targetMap.remove(key); + } + + public void putAll(Map> m) { + this.targetMap.putAll(m); + } + + public void clear() { + this.targetMap.clear(); + } + + public Set keySet() { + return this.targetMap.keySet(); + } + + public Collection> values() { + return this.targetMap.values(); + } + + public Set>> entrySet() { + return this.targetMap.entrySet(); + } + + public boolean equals(Object obj) { + return this.targetMap.equals(obj); + } + + public int hashCode() { + return this.targetMap.hashCode(); + } + + public String toString() { + return this.targetMap.toString(); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/MathUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/MathUtils.java new file mode 100644 index 000000000..58d72eace --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/MathUtils.java @@ -0,0 +1,1401 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + + +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.commons.math3.util.FastMath; +import org.nd4j.common.primitives.Counter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.Set; + + +public class MathUtils { + + + + /** + * The natural logarithm of 2. + */ + public static final double log2 = Math.log(2); + /** + * The small deviation allowed in double comparisons. + */ + public static final double SMALL = 1e-6; + + + public static double pow(double base, double exponent) { + double result = 1; + + if (exponent == 0) { + return result; + } + if (exponent < 0) { + return 1 / pow(base, exponent * -1); + } + + return FastMath.pow(base, exponent); + } + + /** + * Normalize a value + * (val - min) / (max - min) + * + * @param val value to normalize + * @param max max value + * @param min min value + * @return the normalized value + */ + public static double normalize(double val, double min, double max) { + if (max < min) + throw new IllegalArgumentException("Max must be greather than min"); + + return (val - min) / (max - min); + } + + /** + * Clamps the value to a discrete value + * + * @param value the value to clamp + * @param min min for the probability distribution + * @param max max for the probability distribution + * @return the discrete value + */ + public static int clamp(int value, int min, int max) { + if (value < min) + value = min; + if (value > max) + value = max; + return value; + } + + /** + * Discretize the given value + * + * @param value the value to discretize + * @param min the min of the distribution + * @param max the max of the distribution + * @param binCount the number of bins + * @return the discretized value + */ + public static int discretize(double value, double min, double max, int binCount) { + int discreteValue = (int) (binCount * normalize(value, min, max)); + return clamp(discreteValue, 0, binCount - 1); + } + + /** + * See: https://stackoverflow.com/questions/466204/rounding-off-to-nearest-power-of-2 + * + * @param v the number to getFromOrigin the next power of 2 for + * @return the next power of 2 for the passed in value + */ + public static long nextPowOf2(long v) { + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v++; + return v; + + } + + /** + * Generates a binomial distributed number using + * the given rng + * + * @param rng + * @param n + * @param p + * @return + */ + public static int binomial(RandomGenerator rng, int n, double p) { + if ((p < 0) || (p > 1)) { + return 0; + } + int c = 0; + for (int i = 0; i < n; i++) { + if (rng.nextDouble() < p) { + c++; + } + } + return c; + } + + /** + * Generate a uniform random number from the given rng + * + * @param rng the rng to use + * @param min the min num + * @param max the max num + * @return a number uniformly distributed between min and max + */ + public static double uniform(Random rng, double min, double max) { + return rng.nextDouble() * (max - min) + min; + } + + /** + * Returns the correlation coefficient of two double vectors. + * + * @param residuals residuals + * @param targetAttribute target attribute vector + * @return the correlation coefficient or r + */ + public static double correlation(double[] residuals, double targetAttribute[]) { + double[] predictedValues = new double[residuals.length]; + for (int i = 0; i < predictedValues.length; i++) { + predictedValues[i] = targetAttribute[i] - residuals[i]; + } + double ssErr = ssError(predictedValues, targetAttribute); + double total = ssTotal(residuals, targetAttribute); + return 1 - (ssErr / total); + }//end correlation + + /** + * 1 / 1 + exp(-x) + * + * @param x + * @return + */ + public static double sigmoid(double x) { + return 1.0 / (1.0 + Math.pow(Math.E, -x)); + } + + /** + * How much of the variance is explained by the regression + * + * @param residuals error + * @param targetAttribute data for target attribute + * @return the sum squares of regression + */ + public static double ssReg(double[] residuals, double[] targetAttribute) { + double mean = sum(targetAttribute) / targetAttribute.length; + double ret = 0; + for (int i = 0; i < residuals.length; i++) { + ret += Math.pow(residuals[i] - mean, 2); + } + return ret; + } + + /** + * How much of the variance is NOT explained by the regression + * + * @param predictedValues predicted values + * @param targetAttribute data for target attribute + * @return the sum squares of regression + */ + public static double ssError(double[] predictedValues, double[] targetAttribute) { + double ret = 0; + for (int i = 0; i < predictedValues.length; i++) { + ret += Math.pow(targetAttribute[i] - predictedValues[i], 2); + } + return ret; + } + + /** + * Calculate string similarity with tfidf weights relative to each character + * frequency and how many times a character appears in a given string + * @param strings the strings to calculate similarity for + * @return the cosine similarity between the strings + */ + public static double stringSimilarity(String... strings) { + if (strings == null) + return 0; + Counter counter = new Counter<>(); + Counter counter2 = new Counter<>(); + + for (int i = 0; i < strings[0].length(); i++) + counter.incrementCount(String.valueOf(strings[0].charAt(i)), 1.0f); + + for (int i = 0; i < strings[1].length(); i++) + counter2.incrementCount(String.valueOf(strings[1].charAt(i)), 1.0f); + Set v1 = counter.keySet(); + Set v2 = counter2.keySet(); + + + Set both = SetUtils.intersection(v1, v2); + + double sclar = 0, norm1 = 0, norm2 = 0; + for (String k : both) + sclar += counter.getCount(k) * counter2.getCount(k); + for (String k : v1) + norm1 += counter.getCount(k) * counter.getCount(k); + for (String k : v2) + norm2 += counter2.getCount(k) * counter2.getCount(k); + return sclar / Math.sqrt(norm1 * norm2); + } + + /** + * Returns the vector length (sqrt(sum(x_i)) + * + * @param vector the vector to return the vector length for + * @return the vector length of the passed in array + */ + public static double vectorLength(double[] vector) { + double ret = 0; + if (vector == null) + return ret; + else { + for (int i = 0; i < vector.length; i++) { + ret += Math.pow(vector[i], 2); + } + + } + return ret; + } + + /** + * Inverse document frequency: the total docs divided by the number of times the word + * appeared in a document + * + * @param totalDocs the total documents for the data applyTransformToDestination + * @param numTimesWordAppearedInADocument the number of times the word occurred in a document + * @return log(10) (totalDocs/numTImesWordAppearedInADocument) + */ + public static double idf(double totalDocs, double numTimesWordAppearedInADocument) { + return totalDocs > 0 ? Math.log10(totalDocs / numTimesWordAppearedInADocument) : 0; + } + + /** + * Term frequency: 1+ log10(count) + * + * @param count the count of a word or character in a given string or document + * @return 1+ log(10) count + */ + public static double tf(int count) { + return count > 0 ? 1 + Math.log10(count) : 0; + } + + /** + * Return td * idf + * + * @param td the term frequency (assumed calculated) + * @param idf inverse document frequency (assumed calculated) + * @return td * idf + */ + public static double tfidf(double td, double idf) { + return td * idf; + } + + private static int charForLetter(char c) { + char[] chars = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + for (int i = 0; i < chars.length; i++) + if (chars[i] == c) + return i; + return -1; + + } + + /** + * Total variance in target attribute + * + * @param residuals error + * @param targetAttribute data for target attribute + * @return Total variance in target attribute + */ + public static double ssTotal(double[] residuals, double[] targetAttribute) { + return ssReg(residuals, targetAttribute) + ssError(residuals, targetAttribute); + } + + /** + * This returns the sum of the given array. + * + * @param nums the array of numbers to sum + * @return the sum of the given array + */ + public static double sum(double[] nums) { + + double ret = 0; + for (double d : nums) + ret += d; + + return ret; + }//end sum + + /** + * This will merge the coordinates of the given coordinate system. + * + * @param x the x coordinates + * @param y the y coordinates + * @return a vector such that each (x,y) pair is at ret[i],ret[i+1] + */ + public static double[] mergeCoords(double[] x, double[] y) { + if (x.length != y.length) + throw new IllegalArgumentException( + "Sample sizes must be the same for each data applyTransformToDestination."); + double[] ret = new double[x.length + y.length]; + + for (int i = 0; i < x.length; i++) { + ret[i] = x[i]; + ret[i + 1] = y[i]; + } + return ret; + }//end mergeCoords + + /** + * This will merge the coordinates of the given coordinate system. + * + * @param x the x coordinates + * @param y the y coordinates + * @return a vector such that each (x,y) pair is at ret[i],ret[i+1] + */ + public static List mergeCoords(List x, List y) { + if (x.size() != y.size()) + throw new IllegalArgumentException( + "Sample sizes must be the same for each data applyTransformToDestination."); + + List ret = new ArrayList(); + + for (int i = 0; i < x.size(); i++) { + ret.add(x.get(i)); + ret.add(y.get(i)); + } + return ret; + }//end mergeCoords + + /** + * This returns the minimized loss values for a given vector. + * It is assumed that the x, y pairs are at + * vector[i], vector[i+1] + * + * @param vector the vector of numbers to getFromOrigin the weights for + * @return a double array with w_0 and w_1 are the associated indices. + */ + public static double[] weightsFor(List vector) { + /* split coordinate system */ + List coords = coordSplit(vector); + /* x vals */ + double[] x = coords.get(0); + /* y vals */ + double[] y = coords.get(1); + + + double meanX = sum(x) / x.length; + double meanY = sum(y) / y.length; + + double sumOfMeanDifferences = sumOfMeanDifferences(x, y); + double xDifferenceOfMean = sumOfMeanDifferencesOnePoint(x); + + double w_1 = sumOfMeanDifferences / xDifferenceOfMean; + + double w_0 = meanY - (w_1) * meanX; + + //double w_1=(n*sumOfProducts(x,y) - sum(x) * sum(y))/(n*sumOfSquares(x) - Math.pow(sum(x),2)); + + // double w_0=(sum(y) - (w_1 * sum(x)))/n; + + double[] ret = new double[vector.size()]; + ret[0] = w_0; + ret[1] = w_1; + + return ret; + }//end weightsFor + + /** + * This will return the squared loss of the given + * points + * + * @param x the x coordinates to use + * @param y the y coordinates to use + * @param w_0 the first weight + * @param w_1 the second weight + * @return the squared loss of the given points + */ + public static double squaredLoss(double[] x, double[] y, double w_0, double w_1) { + double sum = 0; + for (int j = 0; j < x.length; j++) { + sum += Math.pow((y[j] - (w_1 * x[j] + w_0)), 2); + } + return sum; + }//end squaredLoss + + public static double w_1(double[] x, double[] y, int n) { + return (n * sumOfProducts(x, y) - sum(x) * sum(y)) / (n * sumOfSquares(x) - Math.pow(sum(x), 2)); + } + + public static double w_0(double[] x, double[] y, int n) { + double weight1 = w_1(x, y, n); + + return (sum(y) - (weight1 * sum(x))) / n; + } + + /** + * This returns the minimized loss values for a given vector. + * It is assumed that the x, y pairs are at + * vector[i], vector[i+1] + * + * @param vector the vector of numbers to getFromOrigin the weights for + * @return a double array with w_0 and w_1 are the associated indices. + */ + public static double[] weightsFor(double[] vector) { + + /* split coordinate system */ + List coords = coordSplit(vector); + /* x vals */ + double[] x = coords.get(0); + /* y vals */ + double[] y = coords.get(1); + + + double meanX = sum(x) / x.length; + double meanY = sum(y) / y.length; + + double sumOfMeanDifferences = sumOfMeanDifferences(x, y); + double xDifferenceOfMean = sumOfMeanDifferencesOnePoint(x); + + double w_1 = sumOfMeanDifferences / xDifferenceOfMean; + + double w_0 = meanY - (w_1) * meanX; + + + double[] ret = new double[vector.length]; + ret[0] = w_0; + ret[1] = w_1; + + return ret; + }//end weightsFor + + public static double errorFor(double actual, double prediction) { + return actual - prediction; + } + + /** + * Used for calculating top part of simple regression for + * beta 1 + * + * @param vector the x coordinates + * @param vector2 the y coordinates + * @return the sum of mean differences for the input vectors + */ + public static double sumOfMeanDifferences(double[] vector, double[] vector2) { + double mean = sum(vector) / vector.length; + double mean2 = sum(vector2) / vector2.length; + double ret = 0; + for (int i = 0; i < vector.length; i++) { + double vec1Diff = vector[i] - mean; + double vec2Diff = vector2[i] - mean2; + ret += vec1Diff * vec2Diff; + } + return ret; + }//end sumOfMeanDifferences + + /** + * Used for calculating top part of simple regression for + * beta 1 + * + * @param vector the x coordinates + * @return the sum of mean differences for the input vectors + */ + public static double sumOfMeanDifferencesOnePoint(double[] vector) { + double mean = sum(vector) / vector.length; + double ret = 0; + for (int i = 0; i < vector.length; i++) { + double vec1Diff = Math.pow(vector[i] - mean, 2); + ret += vec1Diff; + } + return ret; + }//end sumOfMeanDifferences + + /** + * This returns the product of all numbers in the given array. + * + * @param nums the numbers to multiply over + * @return the product of all numbers in the array, or 0 + * if the length is or or nums i null + */ + public static double times(double[] nums) { + if (nums == null || nums.length == 0) + return 0; + double ret = 1; + for (int i = 0; i < nums.length; i++) + ret *= nums[i]; + return ret; + }//end times + + /** + * This returns the sum of products for the given + * numbers. + * + * @param nums the sum of products for the give numbers + * @return the sum of products for the given numbers + */ + public static double sumOfProducts(double[]... nums) { + if (nums == null || nums.length < 1) + return 0; + double sum = 0; + + for (int i = 0; i < nums.length; i++) { + /* The ith column for all of the rows */ + double[] column = column(i, nums); + sum += times(column); + + } + return sum; + }//end sumOfProducts + + /** + * This returns the given column over an n arrays + * + * @param column the column to getFromOrigin values for + * @param nums the arrays to extract values from + * @return a double array containing all of the numbers in that column + * for all of the arrays. + * @throws IllegalArgumentException if the index is < 0 + */ + private static double[] column(int column, double[]... nums) throws IllegalArgumentException { + + double[] ret = new double[nums.length]; + + for (int i = 0; i < nums.length; i++) { + double[] curr = nums[i]; + ret[i] = curr[column]; + } + return ret; + }//end column + + /** + * This returns the coordinate split in a list of coordinates + * such that the values for ret[0] are the x values + * and ret[1] are the y values + * + * @param vector the vector to split with x and y values/ + * @return a coordinate split for the given vector of values. + * if null, is passed in null is returned + */ + public static List coordSplit(double[] vector) { + + if (vector == null) + return null; + List ret = new ArrayList(); + /* x coordinates */ + double[] xVals = new double[vector.length / 2]; + /* y coordinates */ + double[] yVals = new double[vector.length / 2]; + /* current points */ + int xTracker = 0; + int yTracker = 0; + for (int i = 0; i < vector.length; i++) { + //even value, x coordinate + if (i % 2 == 0) + xVals[xTracker++] = vector[i]; + //y coordinate + else + yVals[yTracker++] = vector[i]; + } + ret.add(xVals); + ret.add(yVals); + + return ret; + }//end coordSplit + + /** + * This will partition the given whole variable data applyTransformToDestination in to the specified chunk number. + * + * @param arr the data applyTransformToDestination to pass in + * @param chunk the number to separate by + * @return a partition data applyTransformToDestination relative to the passed in chunk number + */ + public static List> partitionVariable(List arr, int chunk) { + int count = 0; + List> ret = new ArrayList>(); + + + while (count < arr.size()) { + + List sublist = arr.subList(count, count + chunk); + count += chunk; + ret.add(sublist); + + } + //All data sets must be same size + for (List lists : ret) { + if (lists.size() < chunk) + ret.remove(lists); + } + return ret; + }//end partitionVariable + + /** + * This returns the coordinate split in a list of coordinates + * such that the values for ret[0] are the x values + * and ret[1] are the y values + * + * @param vector the vector to split with x and y values + * Note that the list will be more stable due to the size operator. + * The array version will have extraneous values if not monitored + * properly. + * @return a coordinate split for the given vector of values. + * if null, is passed in null is returned + */ + public static List coordSplit(List vector) { + + if (vector == null) + return null; + List ret = new ArrayList(); + /* x coordinates */ + double[] xVals = new double[vector.size() / 2]; + /* y coordinates */ + double[] yVals = new double[vector.size() / 2]; + /* current points */ + int xTracker = 0; + int yTracker = 0; + for (int i = 0; i < vector.size(); i++) { + //even value, x coordinate + if (i % 2 == 0) + xVals[xTracker++] = vector.get(i); + //y coordinate + else + yVals[yTracker++] = vector.get(i); + } + ret.add(xVals); + ret.add(yVals); + + return ret; + }//end coordSplit + + /** + * This returns the x values of the given vector. + * These are assumed to be the even values of the vector. + * + * @param vector the vector to getFromOrigin the values for + * @return the x values of the given vector + */ + public static double[] xVals(double[] vector) { + + + if (vector == null) + return null; + double[] x = new double[vector.length / 2]; + int count = 0; + for (int i = 0; i < vector.length; i++) { + if (i % 2 != 0) + x[count++] = vector[i]; + } + return x; + }//end xVals + + /** + * This returns the odd indexed values for the given vector + * + * @param vector the odd indexed values of rht egiven vector + * @return the y values of the given vector + */ + public static double[] yVals(double[] vector) { + double[] y = new double[vector.length / 2]; + int count = 0; + for (int i = 0; i < vector.length; i++) { + if (i % 2 == 0) + y[count++] = vector[i]; + } + return y; + }//end yVals + + /** + * This returns the sum of squares for the given vector. + * + * @param vector the vector to obtain the sum of squares for + * @return the sum of squares for this vector + */ + public static double sumOfSquares(double[] vector) { + double ret = 0; + for (double d : vector) + ret += Math.pow(d, 2); + return ret; + } + + /** + * This returns the determination coefficient of two vectors given a length + * + * @param y1 the first vector + * @param y2 the second vector + * @param n the length of both vectors + * @return the determination coefficient or r^2 + */ + public static double determinationCoefficient(double[] y1, double[] y2, int n) { + return Math.pow(correlation(y1, y2), 2); + } + + /** + * Returns the logarithm of a for base 2. + * + * @param a a double + * @return the logarithm for base 2 + */ + public static double log2(double a) { + if (a == 0) + return 0.0; + return Math.log(a) / log2; + } + + /** + * This returns the root mean squared error of two data sets + * + * @param real the realComponent values + * @param predicted the predicted values + * @return the root means squared error for two data sets + */ + public static double rootMeansSquaredError(double[] real, double[] predicted) { + double ret = 1 / real.length; + for (int i = 0; i < real.length; i++) { + ret += Math.pow((real[i] - predicted[i]), 2); + } + return Math.sqrt(ret); + }//end rootMeansSquaredError + + /** + * This returns the entropy (information gain, or uncertainty of a random variable): -sum(x*log(x)) + * + * @param vector the vector of values to getFromOrigin the entropy for + * @return the entropy of the given vector + */ + public static double entropy(double[] vector) { + if (vector == null || vector.length == 0) + return 0; + else { + double ret = 0; + for (double d : vector) + ret += d * Math.log(d); + return -ret; + + } + }//end entropy + + /** + * This returns the kronecker delta of two doubles. + * + * @param i the first number to compare + * @param j the second number to compare + * @return 1 if they are equal, 0 otherwise + */ + public static int kroneckerDelta(double i, double j) { + return (i == j) ? 1 : 0; + } + + /** + * This calculates the adjusted r^2 including degrees of freedom. + * Also known as calculating "strength" of a regression + * + * @param rSquared the r squared value to calculate + * @param numRegressors number of variables + * @param numDataPoints size of the data applyTransformToDestination + * @return an adjusted r^2 for degrees of freedom + */ + public static double adjustedrSquared(double rSquared, int numRegressors, int numDataPoints) { + double divide = (numDataPoints - 1) / (numDataPoints - numRegressors - 1); + double rSquaredDiff = 1 - rSquared; + return 1 - (rSquaredDiff * divide); + } + + + public static double[] normalizeToOne(double[] doubles) { + normalize(doubles, sum(doubles)); + return doubles; + } + + public static double min(double[] doubles) { + double ret = doubles[0]; + for (double d : doubles) + if (d < ret) + ret = d; + return ret; + } + + public static double max(double[] doubles) { + double ret = doubles[0]; + for (double d : doubles) + if (d > ret) + ret = d; + return ret; + } + + /** + * Normalizes the doubles in the array using the given value. + * + * @param doubles the array of double + * @param sum the value by which the doubles are to be normalized + * @throws IllegalArgumentException if sum is zero or NaN + */ + public static void normalize(double[] doubles, double sum) { + + if (Double.isNaN(sum)) { + throw new IllegalArgumentException("Can't normalize array. Sum is NaN."); + } + if (sum == 0) { + // Maybe this should just be a return. + throw new IllegalArgumentException("Can't normalize array. Sum is zero."); + } + for (int i = 0; i < doubles.length; i++) { + doubles[i] /= sum; + } + }//end normalize + + /** + * Converts an array containing the natural logarithms of + * probabilities stored in a vector back into probabilities. + * The probabilities are assumed to sum to one. + * + * @param a an array holding the natural logarithms of the probabilities + * @return the converted array + */ + public static double[] logs2probs(double[] a) { + + double max = a[maxIndex(a)]; + double sum = 0.0; + + double[] result = new double[a.length]; + for (int i = 0; i < a.length; i++) { + result[i] = Math.exp(a[i] - max); + sum += result[i]; + } + + normalize(result, sum); + + return result; + }//end logs2probs + + /** + * This returns the entropy for a given vector of probabilities. + * + * @param probabilities the probabilities to getFromOrigin the entropy for + * @return the entropy of the given probabilities. + */ + public static double information(double[] probabilities) { + double total = 0.0; + for (double d : probabilities) { + total += (-1.0 * log2(d) * d); + } + return total; + }//end information + + /** + * Returns index of maximum element in a given + * array of doubles. First maximum is returned. + * + * @param doubles the array of doubles + * @return the index of the maximum element + */ + public static /*@pure@*/ int maxIndex(double[] doubles) { + + double maximum = 0; + int maxIndex = 0; + + for (int i = 0; i < doubles.length; i++) { + if ((i == 0) || (doubles[i] > maximum)) { + maxIndex = i; + maximum = doubles[i]; + } + } + + return maxIndex; + }//end maxIndex + + /** + * This will return the factorial of the given number n. + * + * @param n the number to getFromOrigin the factorial for + * @return the factorial for this number + */ + public static double factorial(double n) { + if (n == 1 || n == 0) + return 1; + for (double i = n; i > 0; i--, n *= (i > 0 ? i : 1)) { + } + return n; + }//end factorial + + /** + * Returns the log-odds for a given probability. + * + * @param prob the probability + * @return the log-odds after the probability has been mapped to + * [Utils.SMALL, 1-Utils.SMALL] + */ + public static /*@pure@*/ double probToLogOdds(double prob) { + + if (gr(prob, 1) || (sm(prob, 0))) { + throw new IllegalArgumentException("probToLogOdds: probability must " + "be in [0,1] " + prob); + } + double p = SMALL + (1.0 - 2 * SMALL) * prob; + return Math.log(p / (1 - p)); + } + + /** + * Rounds a double to the next nearest integer value. The JDK version + * of it doesn't work properly. + * + * @param value the double value + * @return the resulting integer value + */ + public static /*@pure@*/ int round(double value) { + + int roundedValue = value > 0 ? (int) (value + 0.5) : -(int) (Math.abs(value) + 0.5); + + return roundedValue; + }//end round + + /** + * This returns the permutation of n choose r. + * + * @param n the n to choose + * @param r the number of elements to choose + * @return the permutation of these numbers + */ + public static double permutation(double n, double r) { + double nFac = MathUtils.factorial(n); + double nMinusRFac = MathUtils.factorial((n - r)); + return nFac / nMinusRFac; + }//end permutation + + /** + * This returns the combination of n choose r + * + * @param n the number of elements overall + * @param r the number of elements to choose + * @return the amount of possible combinations for this applyTransformToDestination of elements + */ + public static double combination(double n, double r) { + double nFac = MathUtils.factorial(n); + double rFac = MathUtils.factorial(r); + double nMinusRFac = MathUtils.factorial((n - r)); + + return nFac / (rFac * nMinusRFac); + }//end combination + + /** + * sqrt(a^2 + b^2) without under/overflow. + */ + public static double hypotenuse(double a, double b) { + double r; + if (Math.abs(a) > Math.abs(b)) { + r = b / a; + r = Math.abs(a) * Math.sqrt(1 + r * r); + } else if (b != 0) { + r = a / b; + r = Math.abs(b) * Math.sqrt(1 + r * r); + } else { + r = 0.0; + } + return r; + }//end hypotenuse + + /** + * Rounds a double to the next nearest integer value in a probabilistic + * fashion (e.g. 0.8 has a 20% chance of being rounded down to 0 and a + * 80% chance of being rounded up to 1). In the limit, the average of + * the rounded numbers generated by this procedure should converge to + * the original double. + * + * @param value the double value + * @param rand the random number generator + * @return the resulting integer value + */ + public static int probRound(double value, Random rand) { + + if (value >= 0) { + double lower = Math.floor(value); + double prob = value - lower; + if (rand.nextDouble() < prob) { + return (int) lower + 1; + } else { + return (int) lower; + } + } else { + double lower = Math.floor(Math.abs(value)); + double prob = Math.abs(value) - lower; + if (rand.nextDouble() < prob) { + return -((int) lower + 1); + } else { + return -(int) lower; + } + } + }//end probRound + + /** + * Rounds a double to the given number of decimal places. + * + * @param value the double value + * @param afterDecimalPoint the number of digits after the decimal point + * @return the double rounded to the given precision + */ + public static /*@pure@*/ double roundDouble(double value, int afterDecimalPoint) { + + double mask = Math.pow(10.0, (double) afterDecimalPoint); + + return (double) (Math.round(value * mask)) / mask; + }//end roundDouble + + /** + * Rounds a double to the given number of decimal places. + * + * @param value the double value + * @param afterDecimalPoint the number of digits after the decimal point + * @return the double rounded to the given precision + */ + public static /*@pure@*/ float roundFloat(float value, int afterDecimalPoint) { + + float mask = (float) Math.pow(10, (float) afterDecimalPoint); + + return (float) (Math.round(value * mask)) / mask; + }//end roundDouble + + /** + * This will return the bernoulli trial for the given event. + * A bernoulli trial is a mechanism for detecting the probability + * of a given event occurring k times in n independent trials + * + * @param n the number of trials + * @param k the number of times the target event occurs + * @param successProb the probability of the event happening + * @return the probability of the given event occurring k times. + */ + public static double bernoullis(double n, double k, double successProb) { + + double combo = MathUtils.combination(n, k); + double p = successProb; + double q = 1 - successProb; + return combo * Math.pow(p, k) * Math.pow(q, n - k); + }//end bernoullis + + /** + * Tests if a is smaller than b. + * + * @param a a double + * @param b a double + */ + public static /*@pure@*/ boolean sm(double a, double b) { + + return (b - a > SMALL); + } + + /** + * Tests if a is greater than b. + * + * @param a a double + * @param b a double + */ + public static /*@pure@*/ boolean gr(double a, double b) { + + return (a - b > SMALL); + } + + /** + * This will take a given string and separator and convert it to an equivalent + * double array. + * + * @param data the data to separate + * @param separator the separator to use + * @return the new double array based on the given data + */ + public static double[] fromString(String data, String separator) { + String[] split = data.split(separator); + double[] ret = new double[split.length]; + for (int i = 0; i < split.length; i++) { + ret[i] = Double.parseDouble(split[i]); + } + return ret; + }//end fromString + + /** + * Computes the mean for an array of doubles. + * + * @param vector the array + * @return the mean + */ + public static /*@pure@*/ double mean(double[] vector) { + + double sum = 0; + + if (vector.length == 0) { + return 0; + } + for (int i = 0; i < vector.length; i++) { + sum += vector[i]; + } + return sum / (double) vector.length; + }//end mean + + /** + * This will convert the given binary string to a decimal based + * integer + * + * @param binary the binary string to convert + * @return an equivalent base 10 number + */ + public static int toDecimal(String binary) { + long num = Long.parseLong(binary); + long rem; + /* Use the remainder method to ensure validity */ + while (num > 0) { + rem = num % 10; + num = num / 10; + if (rem != 0 && rem != 1) { + System.out.println("This is not a binary number."); + System.out.println("Please try once again."); + return -1; + } + } + int i = Integer.parseInt(binary, 2); + return i; + }//end toDecimal + + /** + * This will translate a vector in to an equivalent integer + * + * @param vector the vector to translate + * @return a z value such that the value is the interleaved lsd to msd for each + * double in the vector + */ + public static int distanceFinderZValue(double[] vector) { + StringBuilder binaryBuffer = new StringBuilder(); + List binaryReps = new ArrayList(vector.length); + for (int i = 0; i < vector.length; i++) { + double d = vector[i]; + int j = (int) d; + String binary = Integer.toBinaryString(j); + binaryReps.add(binary); + } + //append from left to right, the least to the most significant bit + //till all strings are empty + while (!binaryReps.isEmpty()) { + for (int j = 0; j < binaryReps.size(); j++) { + String curr = binaryReps.get(j); + if (!curr.isEmpty()) { + char first = curr.charAt(0); + binaryBuffer.append(first); + curr = curr.substring(1); + binaryReps.set(j, curr); + } else + binaryReps.remove(j); + } + } + return Integer.parseInt(binaryBuffer.toString(), 2); + + }//end distanceFinderZValue + + /** + * This returns the euclidean distance of two vectors + * sum(i=1,n) (q_i - p_i)^2 + * + * @param p the first vector + * @param q the second vector + * @return the euclidean distance between two vectors + */ + public static double euclideanDistance(double[] p, double[] q) { + + double ret = 0; + for (int i = 0; i < p.length; i++) { + double diff = (q[i] - p[i]); + double sq = Math.pow(diff, 2); + ret += sq; + } + return ret; + + }//end euclideanDistance + + /** + * This returns the euclidean distance of two vectors + * sum(i=1,n) (q_i - p_i)^2 + * + * @param p the first vector + * @param q the second vector + * @return the euclidean distance between two vectors + */ + public static double euclideanDistance(float[] p, float[] q) { + + double ret = 0; + for (int i = 0; i < p.length; i++) { + double diff = (q[i] - p[i]); + double sq = Math.pow(diff, 2); + ret += sq; + } + return ret; + + }//end euclideanDistance + + /** + * This will generate a series of uniformally distributed + * numbers between l times + * + * @param l the number of numbers to generate + * @return l uniformally generated numbers + */ + public static double[] generateUniform(int l) { + double[] ret = new double[l]; + Random rgen = new Random(); + for (int i = 0; i < l; i++) { + ret[i] = rgen.nextDouble(); + } + return ret; + }//end generateUniform + + /** + * This will calculate the Manhattan distance between two sets of points. + * The Manhattan distance is equivalent to: + * 1_sum_n |p_i - q_i| + * + * @param p the first point vector + * @param q the second point vector + * @return the Manhattan distance between two object + */ + public static double manhattanDistance(double[] p, double[] q) { + + double ret = 0; + for (int i = 0; i < p.length; i++) { + double difference = p[i] - q[i]; + ret += Math.abs(difference); + } + return ret; + }//end manhattanDistance + + public static double[] sampleDoublesInInterval(double[][] doubles, int l) { + double[] sample = new double[l]; + for (int i = 0; i < l; i++) { + int rand1 = randomNumberBetween(0, doubles.length - 1); + int rand2 = randomNumberBetween(0, doubles[i].length); + sample[i] = doubles[rand1][rand2]; + } + + return sample; + } + + /** + * Generates a random integer between the specified numbers + * + * @param begin the begin of the interval + * @param end the end of the interval + * @param anchor the base number (assuming to be generated from an external rng) + * @return an int between begin and end + */ + public static int randomNumberBetween(double begin, double end,double anchor) { + if (begin > end) + throw new IllegalArgumentException("Begin must not be less than end"); + return (int) begin + (int) (anchor * ((end - begin) + 1)); + } + + + /** + * Generates a random integer between the specified numbers + * + * @param begin the begin of the interval + * @param end the end of the interval + * @return an int between begin and end + */ + public static int randomNumberBetween(double begin, double end) { + if (begin > end) + throw new IllegalArgumentException("Begin must not be less than end"); + return (int) begin + (int) (Math.random() * ((end - begin) + 1)); + } + + /** + * Generates a random integer between the specified numbers + * + * @param begin the begin of the interval + * @param end the end of the interval + * @return an int between begin and end + */ + public static int randomNumberBetween(double begin, double end, RandomGenerator rng) { + if (begin > end) + throw new IllegalArgumentException("Begin must not be less than end"); + return (int) begin + (int) (rng.nextDouble() * ((end - begin) + 1)); + } + + public static float randomFloatBetween(float begin, float end) { + float rand = (float) Math.random(); + return begin + (rand * (end - begin)); + } + + public static double randomDoubleBetween(double begin, double end) { + return begin + (Math.random() * (end - begin)); + } + + /** + * This returns the slope of the given points. + * + * @param x1 the first x to use + * @param x2 the end x to use + * @param y1 the begin y to use + * @param y2 the end y to use + * @return the slope of the given points + */ + public double slope(double x1, double x2, double y1, double y2) { + return (y2 - y1) / (x2 - x1); + }//end slope + + /** + * Shuffle the array elements using the specified RNG seed. + * Uses Fisher Yates shuffle internally: + * https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm + * + * @param array Array to shuffle + * @param rngSeed RNG seed to use for shuffling + */ + public static void shuffleArray(int[] array, long rngSeed) { + shuffleArray(array, new Random(rngSeed)); + } + + /** + * Shuffle the array elements using the specified Random instance + * Uses Fisher Yates shuffle internally: + * https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm + * + * @param array Array to shuffle + * @param rng Random instance to use for shuffling + */ + public static void shuffleArray(int[] array, Random rng) { + shuffleArraySubset(array, array.length, rng); + } + + /** + * Shuffle the first N elements of the array using the specified Random instance.
+ * If shuffleFirst < array.length, only the elements 0 to shuffleFirst-1 are modified; values at indices shuffleFirst to + * array.length-1 are not changed. + * Uses Fisher Yates shuffle internally: + * https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm + * + * @param array Array to shuffle first N elements of + * + * @param rng Random instance to use for shuffling + */ + public static void shuffleArraySubset(int[] array, int shuffleFirst, Random rng) { + for (int i = shuffleFirst-1; i > 0; i--) { + int j = rng.nextInt(i + 1); + int temp = array[j]; + array[j] = array[i]; + array[i] = temp; + } + } + + /** + * hashCode method, taken from Java 1.8 Double.hashCode(double) method + * + * @param value Double value to hash + * @return Hash code for the double value + */ + public static int hashCode(double value) { + long bits = Double.doubleToLongBits(value); + return (int) (bits ^ (bits >>> 32)); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/MultiValueMap.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/MultiValueMap.java new file mode 100644 index 000000000..750f5820a --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/MultiValueMap.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import java.util.List; +import java.util.Map; + +public interface MultiValueMap extends Map> { + V getFirst(K var1); + + void add(K var1, V var2); + + void set(K var1, V var2); + + void setAll(Map var1); + + Map toSingleValueMap(); +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ND4JFileUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ND4JFileUtils.java new file mode 100644 index 000000000..d6bc2f3f6 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ND4JFileUtils.java @@ -0,0 +1,66 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import org.nd4j.common.config.ND4JSystemProperties; + +import java.io.File; +import java.io.IOException; + +public class ND4JFileUtils { + + private ND4JFileUtils(){ } + + /** + * Create a temporary file in the location specified by {@link ND4JSystemProperties#ND4J_TEMP_DIR_PROPERTY} if set, + * or the default temporary directory (usually specified by java.io.tmpdir system property) + * @param prefix Prefix for generating file's name; must be at least 3 characeters + * @param suffix Suffix for generating file's name; may be null (".tmp" will be used if null) + * @return A temporary file + */ + public static File createTempFile(String prefix, String suffix) { + String p = System.getProperty(ND4JSystemProperties.ND4J_TEMP_DIR_PROPERTY); + try { + if (p == null || p.isEmpty()) { + return File.createTempFile(prefix, suffix); + } else { + return File.createTempFile(prefix, suffix, new File(p)); + } + } catch (IOException e){ + throw new RuntimeException("Error creating temporary file", e); + } + } + + /** + * Get the temporary directory. This is the location specified by {@link ND4JSystemProperties#ND4J_TEMP_DIR_PROPERTY} if set, + * or the default temporary directory (usually specified by java.io.tmpdir system property) + * @return Temporary directory + */ + public static File getTempDir(){ + String p = System.getProperty(ND4JSystemProperties.ND4J_TEMP_DIR_PROPERTY); + if(p == null || p.isEmpty()){ + return new File(System.getProperty("java.io.tmpdir")); + } else { + return new File(p); + } + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/NioUtil.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/NioUtil.java new file mode 100644 index 000000000..3f11e4d59 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/NioUtil.java @@ -0,0 +1,91 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import java.nio.*; + +/** + * NioUtils for operations on + * nio buffers + * @author Adam Gibson + */ +public class NioUtil { + + private NioUtil() {} + + public enum BufferType { + INT, FLOAT, DOUBLE + } + + /** + * Copy from the given from buffer + * to the to buffer at the specified + * offsets and strides + * @param n + * @param bufferType + * @param from the origin buffer + * @param fromOffset the starting offset + * @param fromStride the stride at which to copy from the origin + * @param to the destination buffer + * @param toOffset the starting point + * @param toStride the to stride + */ + public static void copyAtStride(int n, BufferType bufferType, ByteBuffer from, int fromOffset, int fromStride, + ByteBuffer to, int toOffset, int toStride) { + // TODO: implement shape copy for cases where stride == 1 + ByteBuffer fromView = from; + ByteBuffer toView = to; + fromView.order(ByteOrder.nativeOrder()); + toView.order(ByteOrder.nativeOrder()); + switch (bufferType) { + case INT: + IntBuffer fromInt = fromView.asIntBuffer(); + IntBuffer toInt = toView.asIntBuffer(); + for (int i = 0; i < n; i++) { + int put = fromInt.get(fromOffset + i * fromStride); + toInt.put(toOffset + i * toStride, put); + } + break; + case FLOAT: + FloatBuffer fromFloat = fromView.asFloatBuffer(); + FloatBuffer toFloat = toView.asFloatBuffer(); + for (int i = 0; i < n; i++) { + float put = fromFloat.get(fromOffset + i * fromStride); + toFloat.put(toOffset + i * toStride, put); + } + break; + case DOUBLE: + DoubleBuffer fromDouble = fromView.asDoubleBuffer(); + DoubleBuffer toDouble = toView.asDoubleBuffer(); + for (int i = 0; i < n; i++) { + toDouble.put(toOffset + i * toStride, fromDouble.get(fromOffset + i * fromStride)); + + } + break; + default: + throw new IllegalArgumentException("Only floats and double supported"); + + } + + + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/OneTimeLogger.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/OneTimeLogger.java new file mode 100644 index 000000000..6d673fb3c --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/OneTimeLogger.java @@ -0,0 +1,91 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; + +import java.util.HashSet; +import java.util.Queue; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +@Slf4j +public class OneTimeLogger { + protected static HashSet hashSet = new HashSet<>(); + protected static final Queue buffer = new LinkedTransferQueue<>(); + + private static final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + + protected static boolean isEligible(String message) { + + try { + lock.readLock().lock(); + + if (hashSet.contains(message)) + return false; + + } finally { + lock.readLock().unlock(); + } + + try { + lock.writeLock().lock(); + + if (buffer.size() >= 100) { + String rem = buffer.remove(); + hashSet.remove(rem); + } + + buffer.add(message); + hashSet.add(message); + + return true; + } finally { + lock.writeLock().unlock(); + } + } + + public static void info(Logger logger, String format, Object... arguments) { + if (!isEligible(format)) + return; + + logger.info(format, arguments); + } + + public static void warn(Logger logger, String format, Object... arguments) { + if (!isEligible(format)) + return; + + logger.warn(format, arguments); + } + + public static void error(Logger logger, String format, Object... arguments) { + if (!isEligible(format)) + return; + + logger.error(format, arguments); + } + + public static void reset() { + buffer.clear(); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Paths.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Paths.java new file mode 100644 index 000000000..e00b74894 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Paths.java @@ -0,0 +1,70 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.util.Iterator; + +/** + * Path Utilities + * + * @author Adam Gibson + */ +public class Paths { + + public final static String PATH_ENV_VARIABLE = "PATH"; + + private Paths() {} + + /** + * Check if a file exists in the path + * @param name the name of the file + * @return true if the name exists + * false otherwise + */ + public static boolean nameExistsInPath(String name) { + String path = System.getenv(PATH_ENV_VARIABLE); + String[] dirs = path.split(File.pathSeparator); + for (String dir : dirs) { + File dirFile = new File(dir); + if (!dirFile.exists()) + continue; + + if (dirFile.isFile() && dirFile.getName().equals(name)) + return true; + else { + Iterator files = FileUtils.iterateFiles(dirFile, null, false); + while (files.hasNext()) { + File curr = files.next(); + if (curr.getName().equals(name)) + return true; + } + + } + } + + return false; + } + + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Rational.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Rational.java new file mode 100644 index 000000000..404874016 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/Rational.java @@ -0,0 +1,597 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.math.RoundingMode; + +class Rational implements Cloneable { + + /* The maximum and minimum value of a standard Java integer, 2^31. + */ + static BigInteger MAX_INT = BigInteger.valueOf(Integer.MAX_VALUE); + static BigInteger MIN_INT = BigInteger.valueOf(Integer.MIN_VALUE); + static Rational ONE = new Rational(1, 1); + static Rational ZERO = new Rational(); + /** + * numerator + */ + BigInteger a; + /** + * denominator + */ + BigInteger b; + + /** + * Default ctor, which represents the zero. + */ + public Rational() { + a = BigInteger.ZERO; + b = BigInteger.ONE; + } + + /** + * ctor from a numerator and denominator. + * + * @param a the numerator. + * @param b the denominator. + */ + public Rational(BigInteger a, BigInteger b) { + this.a = a; + this.b = b; + normalize(); + } + + /** + * ctor from a numerator. + * + * @param a the BigInteger. + */ + public Rational(BigInteger a) { + this.a = a; + b = BigInteger.valueOf(1); + } + + /** + * ctor from a numerator and denominator. + * + * @param a the numerator. + * @param b the denominator. + */ + public Rational(int a, int b) { + this(BigInteger.valueOf(a), BigInteger.valueOf(b)); + } + + /** + * ctor from a string representation. + * + * @param str the string. + * This either has a slash in it, separating two integers, or, if there is no slash, + * is representing the numerator with implicit denominator equal to 1. + * @warning this does not yet test for a denominator equal to zero + */ + public Rational(String str) throws NumberFormatException { + this(str, 10); + } + + /** + * ctor from a string representation in a specified base. + * + * @param str the string. + * This either has a slash in it, separating two integers, or, if there is no slash, + * is just representing the numerator. + * @param radix the number base for numerator and denominator + * @warning this does not yet test for a denominator equal to zero + * 5 + */ + public Rational(String str, int radix) throws NumberFormatException { + int hasslah = str.indexOf("/"); + if (hasslah == -1) { + a = new BigInteger(str, radix); + b = new BigInteger("1", radix); + /* no normalization necessary here */ + } else { + /* create numerator and denominator separately + */ + a = new BigInteger(str.substring(0, hasslah), radix); + b = new BigInteger(str.substring(hasslah + 1), radix); + normalize(); + } + } + + /** + * binomial (n choose m). + * + * @param n the numerator. Equals the size of the set to choose from. + * @param m the denominator. Equals the number of elements to select. + * @return the binomial coefficient. + */ + public static Rational binomial(Rational n, BigInteger m) { + if (m.compareTo(BigInteger.ZERO) == 0) { + return Rational.ONE; + } + Rational bin = n; + for (BigInteger i = BigInteger.valueOf(2); i.compareTo(m) != 1; i = i.add(BigInteger.ONE)) { + bin = bin.multiply(n.subtract(i.subtract(BigInteger.ONE))).divide(i); + } + return bin; + } /* Rational.binomial */ + + /** + * binomial (n choose m). + * + * @param n the numerator. Equals the size of the set to choose from. + * @param m the denominator. Equals the number of elements to select. + * @return the binomial coefficient. + */ + public static Rational binomial(Rational n, int m) { + if (m == 0) { + return Rational.ONE; + } + Rational bin = n; + for (int i = 2; i <= m; i++) { + bin = bin.multiply(n.subtract(i - 1)).divide(i); + } + return bin; + } /* Rational.binomial */ + + /** + * Create a copy. + */ + @Override + public Rational clone() { + /* protected access means this does not work + * return new Rational(a.clone(), b.clone()) ; + */ + BigInteger aclon = new BigInteger("" + a); + BigInteger bclon = new BigInteger("" + b); + return new Rational(aclon, bclon); + } /* Rational.clone */ + + /** + * Multiply by another fraction. + * + * @param val a second rational number. + * @return the product of this with the val. + */ + public Rational multiply(final Rational val) { + BigInteger num = a.multiply(val.a); + BigInteger deno = b.multiply(val.b); + /* Normalization to an coprime format will be done inside + * the ctor() and is not duplicated here. + */ + return (new Rational(num, deno)); + } /* Rational.multiply */ + + /** + * Multiply by a BigInteger. + * + * @param val a second number. + * @return the product of this with the value. + */ + public Rational multiply(final BigInteger val) { + Rational val2 = new Rational(val, BigInteger.ONE); + return (multiply(val2)); + } /* Rational.multiply */ + + /** + * Multiply by an integer. + * + * @param val a second number. + * @return the product of this with the value. + */ + public Rational multiply(final int val) { + BigInteger tmp = BigInteger.valueOf(val); + return multiply(tmp); + } /* Rational.multiply */ + + /** + * Power to an integer. + * + * @param exponent the exponent. + * @return this value raised to the power given by the exponent. + * If the exponent is 0, the value 1 is returned. + */ + public Rational pow(int exponent) { + if (exponent == 0) { + return new Rational(1, 1); + } + BigInteger num = a.pow(Math.abs(exponent)); + BigInteger deno = b.pow(Math.abs(exponent)); + if (exponent > 0) { + return (new Rational(num, deno)); + } else { + return (new Rational(deno, num)); + } + } /* Rational.pow */ + + /** + * Power to an integer. + * + * @param exponent the exponent. + * @return this value raised to the power given by the exponent. + * If the exponent is 0, the value 1 is returned. + */ + public Rational pow(BigInteger exponent) throws NumberFormatException { + /* test for overflow */ + if (exponent.compareTo(MAX_INT) == 1) { + throw new NumberFormatException("Exponent " + exponent.toString() + " too large."); + } + if (exponent.compareTo(MIN_INT) == -1) { + throw new NumberFormatException("Exponent " + exponent.toString() + " too small."); + } + /* promote to the simpler interface above */ + return pow(exponent.intValue()); + } /* Rational.pow */ + + /** + * Divide by another fraction. + * + * @param val A second rational number. + * @return The value of this/val + */ + public Rational divide(final Rational val) { + BigInteger num = a.multiply(val.b); + BigInteger deno = b.multiply(val.a); + /* Reduction to a coprime format is done inside the ctor, + * and not repeated here. + */ + return (new Rational(num, deno)); + } /* Rational.divide */ + + /** + * Divide by an integer. + * + * @param val a second number. + * @return the value of this/val + */ + public Rational divide(BigInteger val) { + Rational val2 = new Rational(val, BigInteger.ONE); + return (divide(val2)); + } /* Rational.divide */ + + /** + * Divide by an integer. + * + * @param val A second number. + * @return The value of this/val + */ + public Rational divide(int val) { + Rational val2 = new Rational(val, 1); + return (divide(val2)); + } /* Rational.divide */ + + /** + * Add another fraction. + * + * @param val The number to be added + * @return this+val. + */ + public Rational add(Rational val) { + BigInteger num = a.multiply(val.b).add(b.multiply(val.a)); + BigInteger deno = b.multiply(val.b); + return (new Rational(num, deno)); + } /* Rational.add */ + + /** + * Add another integer. + * + * @param val The number to be added + * @return this+val. + */ + public Rational add(BigInteger val) { + Rational val2 = new Rational(val, BigInteger.ONE); + return (add(val2)); + } /* Rational.add */ + + /** + * Compute the negative. + * + * @return -this. + */ + public Rational negate() { + return (new Rational(a.negate(), b)); + } /* Rational.negate */ + + /** + * Subtract another fraction. + * 7 + * + * @param val the number to be subtracted from this + * @return this - val. + */ + public Rational subtract(Rational val) { + Rational val2 = val.negate(); + return (add(val2)); + } /* Rational.subtract */ + + /** + * Subtract an integer. + * + * @param val the number to be subtracted from this + * @return this - val. + */ + public Rational subtract(BigInteger val) { + Rational val2 = new Rational(val, BigInteger.ONE); + return (subtract(val2)); + } /* Rational.subtract */ + + /** + * Subtract an integer. + * + * @param val the number to be subtracted from this + * @return this - val. + */ + public Rational subtract(int val) { + Rational val2 = new Rational(val, 1); + return (subtract(val2)); + } /* Rational.subtract */ + + /** + * Get the numerator. + * + * @return The numerator of the reduced fraction. + */ + public BigInteger numer() { + return a; + } + + /** + * Get the denominator. + * + * @return The denominator of the reduced fraction. + */ + public BigInteger denom() { + return b; + } + + /** + * Absolute value. + * + * @return The absolute (non-negative) value of this. + */ + public Rational abs() { + return (new Rational(a.abs(), b.abs())); + } + + /** + * floor(): the nearest integer not greater than this. + * + * @return The integer rounded towards negative infinity. + */ + public BigInteger floor() { + /* is already integer: return the numerator + */ + if (b.compareTo(BigInteger.ONE) == 0) { + return a; + } else if (a.compareTo(BigInteger.ZERO) > 0) { + return a.divide(b); + } else { + return a.divide(b).subtract(BigInteger.ONE); + } + } /* Rational.floor */ + + + /** + * Remove the fractional part. + * + * @return The integer rounded towards zero. + */ + public BigInteger trunc() { + /* is already integer: return the numerator + */ + if (b.compareTo(BigInteger.ONE) == 0) { + return a; + } else { + return a.divide(b); + } + } /* Rational.trunc */ + + + /** + * Compares the value of this with another constant. + * + * @param val the other constant to compare with + * @return -1, 0 or 1 if this number is numerically less than, equal to, + * or greater than val. + */ + public int compareTo(final Rational val) { + /* Since we have always kept the denominators positive, + * simple cross-multiplying works without changing the sign. + */ + final BigInteger left = a.multiply(val.b); + final BigInteger right = val.a.multiply(b); + return left.compareTo(right); + } /* Rational.compareTo */ + + + /** + * Compares the value of this with another constant. + * + * @param val the other constant to compare with + * @return -1, 0 or 1 if this number is numerically less than, equal to, + * or greater than val. + */ + public int compareTo(final BigInteger val) { + final Rational val2 = new Rational(val, BigInteger.ONE); + return (compareTo(val2)); + } /* Rational.compareTo */ + + + /** + * Return a string in the format number/denom. + * If the denominator equals 1, print just the numerator without a slash. + * + * @return the human-readable version in base 10 + */ + @Override + public String toString() { + if (b.compareTo(BigInteger.ONE) != 0) { + return (a.toString() + "/" + b.toString()); + } else { + return a.toString(); + } + } /* Rational.toString */ + + + /** + * Return a double value representation. + * + * @return The value with double precision. + */ + public double doubleValue() { + /* To meet the risk of individual overflows of the exponents of + * a separate invocation a.doubleValue() or b.doubleValue(), we divide first + * in a BigDecimal environment and converst the result. + */ + BigDecimal adivb = (new BigDecimal(a)).divide(new BigDecimal(b), MathContext.DECIMAL128); + return adivb.doubleValue(); + } /* Rational.doubleValue */ + + + /** + * Return a float value representation. + * + * @return The value with single precision. + */ + public float floatValue() { + BigDecimal adivb = (new BigDecimal(a)).divide(new BigDecimal(b), MathContext.DECIMAL128); + return adivb.floatValue(); + } /* Rational.floatValue */ + + + /** + * Return a representation as BigDecimal. + * + * @param mc the mathematical context which determines precision, rounding mode etc + * @return A representation as a BigDecimal floating point number. + */ + public BigDecimal BigDecimalValue(MathContext mc) { + /* numerator and denominator individually rephrased + */ + BigDecimal n = new BigDecimal(a); + BigDecimal d = new BigDecimal(b); + return n.divide(d, mc); + } /* Rational.BigDecimnalValue */ + + + /** + * Return a string in floating point format. + * + * @param digits The precision (number of digits) + * @return The human-readable version in base 10. + */ + public String toFString(int digits) { + if (b.compareTo(BigInteger.ONE) != 0) { + MathContext mc = new MathContext(digits, RoundingMode.DOWN); + BigDecimal f = (new BigDecimal(a)).divide(new BigDecimal(b), mc); + return (f.toString()); + } else { + return a.toString(); + } + } /* Rational.toFString */ + + + /** + * Compares the value of this with another constant. + * + * @param val The other constant to compare with + * @return The arithmetic maximum of this and val. + */ + public Rational max(final Rational val) { + if (compareTo(val) > 0) { + return this; + } else { + return val; + } + } /* Rational.max */ + + + /** + * Compares the value of this with another constant. + * + * @param val The other constant to compare with + * @return The arithmetic minimum of this and val. + */ + public Rational min(final Rational val) { + if (compareTo(val) < 0) { + return this; + } else { + return val; + } + } /* Rational.min */ + + + /** + * Compute Pochhammer's symbol (this)_n. + * + * @param n The number of product terms in the evaluation. + * @return Gamma(this+n)/Gamma(this) = this*(this+1)*...*(this+n-1). + */ + public Rational Pochhammer(final BigInteger n) { + if (n.compareTo(BigInteger.ZERO) < 0) { + return null; + } else if (n.compareTo(BigInteger.ZERO) == 0) { + return Rational.ONE; + } else { + /* initialize results with the current value + */ + Rational res = new Rational(a, b); + BigInteger i = BigInteger.ONE; + for (; i.compareTo(n) < 0; i = i.add(BigInteger.ONE)) { + res = res.multiply(add(i)); + } + return res; + } + } /* Rational.pochhammer */ + + + /** + * Compute pochhammer's symbol (this)_n. + * + * @param n The number of product terms in the evaluation. + * @return Gamma(this+n)/GAMMA(this). + */ + public Rational Pochhammer(int n) { + return Pochhammer(BigInteger.valueOf(n)); + } /* Rational.pochhammer */ + + + /** + * Normalize to coprime numerator and denominator. + * Also copy a negative sign of the denominator to the numerator. + */ + protected void normalize() { + /* compute greatest common divisor of numerator and denominator + */ + final BigInteger g = a.gcd(b); + if (g.compareTo(BigInteger.ONE) > 0) { + a = a.divide(g); + b = b.divide(g); + } + if (b.compareTo(BigInteger.ZERO) == -1) { + a = a.negate(); + b = b.negate(); + } + } /* Rational.normalize */ + +} /* Rational */ diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SerializationUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SerializationUtils.java new file mode 100644 index 000000000..40dd9ea25 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SerializationUtils.java @@ -0,0 +1,150 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import org.apache.commons.io.FileUtils; + +import java.io.*; + +public class SerializationUtils { + + protected SerializationUtils() {} + + @SuppressWarnings("unchecked") + public static T readObject(File file) { + try { + ObjectInputStream ois = new ObjectInputStream(FileUtils.openInputStream(file)); + T ret = (T) ois.readObject(); + ois.close(); + return ret; + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + /** + * Reads an object from the given input stream + * @param is the input stream to read from + * @return the read object + */ + @SuppressWarnings("unchecked") + public static T readObject(InputStream is) { + try { + ObjectInputStream ois = new ObjectInputStream(is); + T ret = (T) ois.readObject(); + ois.close(); + return ret; + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + + + /** + * Converts the given object to a byte array + * @param toSave the object to save + */ + public static byte[] toByteArray(Serializable toSave) { + try { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream os = new ObjectOutputStream(bos); + os.writeObject(toSave); + byte[] ret = bos.toByteArray(); + os.close(); + return ret; + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + /** + * Deserializes object from byte array + * @param bytes + * @param + * @return + */ + public static T fromByteArray(byte[] bytes) { + return readObject(new ByteArrayInputStream(bytes)); + } + + /** + * Deserializes object from byte array + * @param bytes + * @param + * @return + */ + public static T deserialize(byte[] bytes) { + return fromByteArray(bytes); + } + + /** + * Deserializes object from InputStream + * @param bytes + * @param + * @return + */ + public static T deserialize(InputStream is) { + return readObject(is); + } + + /** + * Writes the object to the output stream + * THIS DOES NOT FLUSH THE STREAM + * @param toSave the object to save + * @param writeTo the output stream to write to + */ + public static void writeObject(Serializable toSave, OutputStream writeTo) { + try { + ObjectOutputStream os = new ObjectOutputStream(writeTo); + os.writeObject(toSave); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Writes the object to the output stream + * THIS DOES NOT FLUSH THE STREAM + * @param toSave the object to save + * @param writeTo the output stream to write to + */ + public static void serialize(Serializable object, OutputStream os) { + writeObject(object, os); + } + + public static void saveObject(Object toSave, File saveTo) { + try { + OutputStream os1 = FileUtils.openOutputStream(saveTo); + ObjectOutputStream os = new ObjectOutputStream(os1); + os.writeObject(toSave); + os.flush(); + os.close(); + os1.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SetUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SetUtils.java new file mode 100644 index 000000000..6e520e098 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SetUtils.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +public class SetUtils { + protected SetUtils() {} + + // Set specific operations + + public static Set intersection(Collection parentCollection, Collection removeFromCollection) { + Set results = new HashSet<>(parentCollection); + results.retainAll(removeFromCollection); + return results; + } + + public static boolean intersectionP(Set s1, Set s2) { + for (T elt : s1) { + if (s2.contains(elt)) + return true; + } + return false; + } + + public static Set union(Set s1, Set s2) { + Set s3 = new HashSet<>(s1); + s3.addAll(s2); + return s3; + } + + /** Return is s1 \ s2 */ + + public static Set difference(Collection s1, Collection s2) { + Set s3 = new HashSet<>(s1); + s3.removeAll(s2); + return s3; + } +} + + diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SynchronizedTable.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SynchronizedTable.java new file mode 100644 index 000000000..ace0bf5f1 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/SynchronizedTable.java @@ -0,0 +1,130 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +import com.google.common.collect.Table; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +public class SynchronizedTable implements Table { + private Table wrapped; + + public SynchronizedTable(Table wrapped) { + this.wrapped = wrapped; + } + + @Override + public synchronized boolean contains(Object rowKey, Object columnKey) { + return wrapped.contains(rowKey, columnKey); + } + + @Override + public synchronized boolean containsRow(Object rowKey) { + return wrapped.containsRow(rowKey); + } + + @Override + public synchronized boolean containsColumn(Object columnKey) { + return wrapped.containsColumn(columnKey); + } + + @Override + public synchronized boolean containsValue(Object value) { + return wrapped.containsValue(value); + } + + @Override + public synchronized V get(Object rowKey, Object columnKey) { + return wrapped.get(rowKey, columnKey); + } + + @Override + public synchronized boolean isEmpty() { + return wrapped.isEmpty(); + } + + @Override + public int size() { + return wrapped.size(); + } + + @Override + public synchronized void clear() { + wrapped.clear(); + } + + @Override + public synchronized V put(R rowKey, C columnKey, V value) { + return wrapped.put(rowKey, columnKey, value); + } + + @Override + public synchronized void putAll(Table table) { + wrapped.putAll(table); + } + + @Override + public synchronized V remove(Object rowKey, Object columnKey) { + return wrapped.remove(rowKey, columnKey); + } + + @Override + public synchronized Map row(R rowKey) { + return wrapped.row(rowKey); + } + + @Override + public synchronized Map column(C columnKey) { + return wrapped.column(columnKey); + } + + @Override + public synchronized Set> cellSet() { + return wrapped.cellSet(); + } + + @Override + public synchronized Set rowKeySet() { + return wrapped.rowKeySet(); + } + + @Override + public synchronized Set columnKeySet() { + return wrapped.columnKeySet(); + } + + @Override + public synchronized Collection values() { + return wrapped.values(); + } + + @Override + public synchronized Map> rowMap() { + return wrapped.rowMap(); + } + + @Override + public synchronized Map> columnMap() { + return wrapped.columnMap(); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ThreadUtils.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ThreadUtils.java new file mode 100644 index 000000000..ba6b65ade --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/util/ThreadUtils.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.util; + +public class ThreadUtils { + + private ThreadUtils(){ } + + public static void uncheckedSleep(long sleepTimeMs){ + try{ + Thread.sleep(sleepTimeMs); + } catch (InterruptedException e){ } + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/validation/Nd4jCommonValidator.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/validation/Nd4jCommonValidator.java new file mode 100644 index 000000000..94da93db1 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/validation/Nd4jCommonValidator.java @@ -0,0 +1,292 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.validation; + +import lombok.NonNull; +import org.apache.commons.io.FileUtils; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class Nd4jCommonValidator { + + private Nd4jCommonValidator() { + } + + /** + * Validate whether the specified file is a valid file (must exist and be non-empty) + * + * @param f File to check + * @return Result of validation + */ + public static ValidationResult isValidFile(@NonNull File f) { + ValidationResult vr = isValidFile(f, "File", false); + if (vr != null) + return vr; + return ValidationResult.builder() + .valid(true) + .formatType("File") + .path(getPath(f)) + .build(); + } + + /** + * Validate whether the specified file is a valid file + * + * @param f File to check + * @param formatType Name of the file format to include in validation results + * @param allowEmpty If true: allow empty files to pass. False: empty files will fail validation + * @return Result of validation + */ + public static ValidationResult isValidFile(@NonNull File f, String formatType, boolean allowEmpty) { + String path; + try { + path = f.getAbsolutePath(); //Very occasionally: getAbsolutePath not possible (files in JARs etc) + } catch (Throwable t) { + path = f.getPath(); + } + + if (f.exists() && !f.isFile()) { + return ValidationResult.builder() + .valid(false) + .formatType(formatType) + .path(path) + .issues(Collections.singletonList(f.isDirectory() ? "Specified path is a directory" : "Specified path is not a file")) + .build(); + } + + if (!f.exists() || !f.isFile()) { + return ValidationResult.builder() + .valid(false) + .formatType(formatType) + .path(path) + .issues(Collections.singletonList("File does not exist")) + .build(); + } + + if (!allowEmpty && f.length() <= 0) { + return ValidationResult.builder() + .valid(false) + .formatType(formatType) + .path(path) + .issues(Collections.singletonList("File is empty (length 0)")) + .build(); + } + + return null; //OK + } + + public static ValidationResult isValidJsonUTF8(@NonNull File f) { + return isValidJson(f, StandardCharsets.UTF_8); + } + + /** + * Validate whether the specified file is a valid JSON file. Note that this does not match the JSON content against a specific schema + * + * @param f File to check + * @param charset Character set for file + * @return Result of validation + */ + public static ValidationResult isValidJson(@NonNull File f, Charset charset) { + + ValidationResult vr = isValidFile(f, "JSON", false); + if (vr != null) + return vr; + + String content; + try { + content = FileUtils.readFileToString(f, charset); + } catch (IOException e) { + return ValidationResult.builder() + .valid(false) + .formatType("JSON") + .path(getPath(f)) + .issues(Collections.singletonList("Unable to read file (IOException)")) + .exception(e) + .build(); + } + + + return isValidJson(content, f); + } + + /** + * Validate whether the specified String is valid JSON. Note that this does not match the JSON content against a specific schema + * + * @param s JSON String to check + * @return Result of validation + */ + public static ValidationResult isValidJSON(String s) { + return isValidJson(s, null); + } + + + protected static ValidationResult isValidJson(String content, File f) { + try { + ObjectMapper om = new ObjectMapper(); + JavaType javaType = om.getTypeFactory().constructMapType(Map.class, String.class, Object.class); + om.readValue(content, javaType); //Don't care about result, just that it can be parsed successfully + } catch (Throwable t) { + //Jackson should tell us specifically where error occurred also + return ValidationResult.builder() + .valid(false) + .formatType("JSON") + .path(getPath(f)) + .issues(Collections.singletonList("File does not appear to be valid JSON")) + .exception(t) + .build(); + } + + + return ValidationResult.builder() + .valid(true) + .formatType("JSON") + .path(getPath(f)) + .build(); + } + + + /** + * Validate whether the specified file is a valid Zip file + * + * @param f File to check + * @param allowEmpty If true: allow empty zip files to pass validation. False: empty zip files will fail validation. + * @return Result of validation + */ + public static ValidationResult isValidZipFile(@NonNull File f, boolean allowEmpty) { + return isValidZipFile(f, allowEmpty, (List) null); + } + + /** + * Validate whether the specified file is a valid Zip file + * + * @param f File to check + * @param allowEmpty If true: allow empty zip files to pass validation. False: empty zip files will fail validation. + * @return Result of validation + */ + public static ValidationResult isValidZipFile(@NonNull File f, boolean allowEmpty, String... requiredEntries) { + return isValidZipFile(f, allowEmpty, requiredEntries == null ? null : Arrays.asList(requiredEntries)); + } + + /** + * Validate whether the specified file is a valid Zip file, and contains all of the required entries + * + * @param f File to check + * @param allowEmpty If true: allow empty zip files to pass validation. False: empty zip files will fail validation. + * @param requiredEntries If non-null, all of the specified entries must be present for the file to pass validation + * @return Result of validation + */ + public static ValidationResult isValidZipFile(@NonNull File f, boolean allowEmpty, List requiredEntries) { + ValidationResult vr = isValidFile(f, "Zip File", false); + if (vr != null) + return vr; + + ZipFile zf; + try { + zf = new ZipFile(f); + } catch (Throwable e) { + return ValidationResult.builder() + .valid(false) + .formatType("Zip File") + .path(getPath(f)) + .issues(Collections.singletonList("File does not appear to be valid zip file (not a zip file or content is corrupt)")) + .exception(e) + .build(); + } + + try { + int numEntries = zf.size(); + if (!allowEmpty && numEntries <= 0) { + return ValidationResult.builder() + .valid(false) + .formatType("Zip File") + .path(getPath(f)) + .issues(Collections.singletonList("Zip file is empty")) + .build(); + } + + if (requiredEntries != null && !requiredEntries.isEmpty()) { + List missing = null; + for (String s : requiredEntries) { + ZipEntry ze = zf.getEntry(s); + if (ze == null) { + if (missing == null) + missing = new ArrayList<>(); + missing.add(s); + } + } + + if (missing != null) { + String s = "Zip file is missing " + missing.size() + " of " + requiredEntries.size() + " required entries: " + missing; + return ValidationResult.builder() + .valid(false) + .formatType("Zip File") + .path(getPath(f)) + .issues(Collections.singletonList(s)) + .build(); + } + } + + } catch (Throwable t) { + return ValidationResult.builder() + .valid(false) + .formatType("Zip File") + .path(getPath(f)) + .issues(Collections.singletonList("Error reading zip file")) + .exception(t) + .build(); + } finally { + try { + zf.close(); + } catch (IOException e) { + } //Ignore, can't do anything about it... + } + + return ValidationResult.builder() + .valid(true) + .formatType("Zip File") + .path(getPath(f)) + .build(); + } + + + /** + * Null-safe and "no absolute path exists" safe method for getting the path of a file for validation purposes + */ + public static String getPath(File f) { + if (f == null) + return null; + try { + return f.getAbsolutePath(); //Very occasionally: getAbsolutePath not possible (files in JARs etc) + } catch (Throwable t) { + return f.getPath(); + } + } + + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/validation/ValidationResult.java b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/validation/ValidationResult.java new file mode 100644 index 000000000..569b4a661 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/main/java/org/nd4j/common/validation/ValidationResult.java @@ -0,0 +1,97 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.validation; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Data +public class ValidationResult implements Serializable { + + private String formatType; //Human readable format/model type + private Class formatClass; //Actual class the format/model is (or should be) + private String path; //Path of file (if applicable) + private boolean valid; //Whether the file/model is valid + private List issues; //List of issues (generally only present if not valid) + private Throwable exception; //Exception, if applicable + + + + @Override + public String toString(){ + List lines = new ArrayList<>(); + if(formatType != null) { + lines.add("Format type: " + formatType); + } + if(formatClass != null){ + lines.add("Format class: " + formatClass.getName()); + } + if(path != null){ + lines.add("Path: " + path); + } + lines.add("Format valid: " + valid); + if(issues != null && !issues.isEmpty()){ + if(issues.size() == 1){ + addWithIndent(issues.get(0), lines, "Issue: ", " "); + } else { + lines.add("Issues:"); + for (String s : issues) { + addWithIndent(s, lines, "- ", " "); + } + } + } + if(exception != null){ + String ex = ExceptionUtils.getStackTrace(exception); + lines.add("Stack Trace:"); + addWithIndent(ex, lines, " ", " "); + } + //Would use String.join but that's Java 8... + StringBuilder sb = new StringBuilder(); + boolean first = true; + for(String s : lines){ + if(!first) + sb.append("\n"); + sb.append(s); + first = false; + } + return sb.toString(); + } + + protected static void addWithIndent(String toAdd, List list, String firstLineIndent, String laterLineIndent){ + String[] split = toAdd.split("\n"); + boolean first = true; + for(String issueLine : split){ + list.add((first ? firstLineIndent : laterLineIndent) + issueLine); + first = false; + } + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/base/TestPreconditions.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/base/TestPreconditions.java new file mode 100644 index 000000000..45d16becf --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/base/TestPreconditions.java @@ -0,0 +1,300 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.base; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class TestPreconditions { + + @Test + public void testPreconditions(){ + + Preconditions.checkArgument(true); + try{ + Preconditions.checkArgument(false); + } catch (IllegalArgumentException e){ + assertNull(e.getMessage()); + } + + Preconditions.checkArgument(true, "Message %s here", 10); + try{ + Preconditions.checkArgument(false, "Message %s here", 10); + } catch (IllegalArgumentException e){ + assertEquals("Message 10 here", e.getMessage()); + } + + Preconditions.checkArgument(true, "Message %s here %s there", 10, 20); + try{ + Preconditions.checkArgument(false, "Message %s here %s there", 10, 20); + } catch (IllegalArgumentException e){ + assertEquals("Message 10 here 20 there", e.getMessage()); + } + + Preconditions.checkArgument(true, "Message %s here %s there %s more", 10, 20, 30); + try{ + Preconditions.checkArgument(false, "Message %s here %s there %s more", 10, 20, 30); + } catch (IllegalArgumentException e){ + assertEquals("Message 10 here 20 there 30 more", e.getMessage()); + } + + Preconditions.checkArgument(true, "Message %s here", 10L); + try{ + Preconditions.checkArgument(false, "Message %s here", 10L); + } catch (IllegalArgumentException e){ + assertEquals("Message 10 here", e.getMessage()); + } + + Preconditions.checkArgument(true, "Message %s here %s there", 10L, 20L); + try{ + Preconditions.checkArgument(false, "Message %s here %s there", 10L, 20L); + } catch (IllegalArgumentException e){ + assertEquals("Message 10 here 20 there", e.getMessage()); + } + + Preconditions.checkArgument(true, "Message %s here %s there %s more", 10L, 20L, 30L); + try{ + Preconditions.checkArgument(false, "Message %s here %s there %s more", 10L, 20L, 30L); + } catch (IllegalArgumentException e){ + assertEquals("Message 10 here 20 there 30 more", e.getMessage()); + } + + Preconditions.checkArgument(true, "Message %s here %s there %s more", "A", "B", "C"); + try{ + Preconditions.checkArgument(false, "Message %s here %s there %s more", "A", "B", "C"); + } catch (IllegalArgumentException e){ + assertEquals("Message A here B there C more", e.getMessage()); + } + + + } + + @Test + public void testPreconditionsMalformed(){ + + //No %s: + Preconditions.checkArgument(true, "This is malformed", "A", "B", "C"); + try{ + Preconditions.checkArgument(false, "This is malformed", "A", "B", "C"); + } catch (IllegalArgumentException e){ + assertEquals("This is malformed [A,B,C]", e.getMessage()); + } + + //More args than %s: + Preconditions.checkArgument(true, "This is %s malformed", "A", "B", "C"); + try{ + Preconditions.checkArgument(false, "This is %s malformed", "A", "B", "C"); + } catch (IllegalArgumentException e){ + assertEquals("This is A malformed [B,C]", e.getMessage()); + } + + //No args + Preconditions.checkArgument(true, "This is %s %s malformed"); + try{ + Preconditions.checkArgument(false, "This is %s %s malformed"); + } catch (IllegalArgumentException e){ + assertEquals("This is %s %s malformed", e.getMessage()); + } + + //More %s than args + Preconditions.checkArgument(true, "This is %s %s malformed", "A"); + try{ + Preconditions.checkArgument(false, "This is %s %s malformed", "A"); + } catch (IllegalArgumentException e){ + assertEquals("This is A %s malformed", e.getMessage()); + } + } + + + @Test + public void testPreconditionsState(){ + + Preconditions.checkState(true); + try{ + Preconditions.checkState(false); + } catch (IllegalStateException e){ + assertNull(e.getMessage()); + } + + Preconditions.checkState(true, "Message %s here", 10); + try{ + Preconditions.checkState(false, "Message %s here", 10); + } catch (IllegalStateException e){ + assertEquals("Message 10 here", e.getMessage()); + } + + Preconditions.checkState(true, "Message %s here %s there", 10, 20); + try{ + Preconditions.checkState(false, "Message %s here %s there", 10, 20); + } catch (IllegalStateException e){ + assertEquals("Message 10 here 20 there", e.getMessage()); + } + + Preconditions.checkState(true, "Message %s here %s there %s more", 10, 20, 30); + try{ + Preconditions.checkState(false, "Message %s here %s there %s more", 10, 20, 30); + } catch (IllegalStateException e){ + assertEquals("Message 10 here 20 there 30 more", e.getMessage()); + } + + Preconditions.checkState(true, "Message %s here", 10L); + try{ + Preconditions.checkState(false, "Message %s here", 10L); + } catch (IllegalStateException e){ + assertEquals("Message 10 here", e.getMessage()); + } + + Preconditions.checkState(true, "Message %s here %s there", 10L, 20L); + try{ + Preconditions.checkState(false, "Message %s here %s there", 10L, 20L); + } catch (IllegalStateException e){ + assertEquals("Message 10 here 20 there", e.getMessage()); + } + + Preconditions.checkState(true, "Message %s here %s there %s more", 10L, 20L, 30L); + try{ + Preconditions.checkState(false, "Message %s here %s there %s more", 10L, 20L, 30L); + } catch (IllegalStateException e){ + assertEquals("Message 10 here 20 there 30 more", e.getMessage()); + } + + Preconditions.checkState(true, "Message %s here %s there %s more", "A", "B", "C"); + try{ + Preconditions.checkState(false, "Message %s here %s there %s more", "A", "B", "C"); + } catch (IllegalStateException e){ + assertEquals("Message A here B there C more", e.getMessage()); + } + } + + @Test + public void testPreconditionsMalformedState(){ + + //No %s: + Preconditions.checkState(true, "This is malformed", "A", "B", "C"); + try{ + Preconditions.checkState(false, "This is malformed", "A", "B", "C"); + } catch (IllegalStateException e){ + assertEquals("This is malformed [A,B,C]", e.getMessage()); + } + + //More args than %s: + Preconditions.checkState(true, "This is %s malformed", "A", "B", "C"); + try{ + Preconditions.checkState(false, "This is %s malformed", "A", "B", "C"); + } catch (IllegalStateException e){ + assertEquals("This is A malformed [B,C]", e.getMessage()); + } + + //No args + Preconditions.checkState(true, "This is %s %s malformed"); + try{ + Preconditions.checkState(false, "This is %s %s malformed"); + } catch (IllegalStateException e){ + assertEquals("This is %s %s malformed", e.getMessage()); + } + + //More %s than args + Preconditions.checkState(true, "This is %s %s malformed", "A"); + try{ + Preconditions.checkState(false, "This is %s %s malformed", "A"); + } catch (IllegalStateException e){ + assertEquals("This is A %s malformed", e.getMessage()); + } + } + + + @Test + public void testPreconditionsNull(){ + + Preconditions.checkNotNull(""); + try{ + Preconditions.checkNotNull(null); + } catch (NullPointerException e){ + assertNull(e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here", 10); + try{ + Preconditions.checkNotNull(null, "Message %s here", 10); + } catch (NullPointerException e){ + assertEquals("Message 10 here", e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here %s there", 10, 20); + try{ + Preconditions.checkNotNull(null, "Message %s here %s there", 10, 20); + } catch (NullPointerException e){ + assertEquals("Message 10 here 20 there", e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here %s there %s more", 10, 20, 30); + try{ + Preconditions.checkNotNull(null, "Message %s here %s there %s more", 10, 20, 30); + } catch (NullPointerException e){ + assertEquals("Message 10 here 20 there 30 more", e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here", 10L); + try{ + Preconditions.checkNotNull(null, "Message %s here", 10L); + } catch (NullPointerException e){ + assertEquals("Message 10 here", e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here %s there", 10L, 20L); + try{ + Preconditions.checkNotNull(null, "Message %s here %s there", 10L, 20L); + } catch (NullPointerException e){ + assertEquals("Message 10 here 20 there", e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here %s there %s more", 10L, 20L, 30L); + try{ + Preconditions.checkNotNull(null, "Message %s here %s there %s more", 10L, 20L, 30L); + } catch (NullPointerException e){ + assertEquals("Message 10 here 20 there 30 more", e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here %s there %s more", "A", "B", "C"); + try{ + Preconditions.checkNotNull(null, "Message %s here %s there %s more", "A", "B", "C"); + } catch (NullPointerException e){ + assertEquals("Message A here B there C more", e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here %s there %s more", new int[]{0,1}, new double[]{2.0, 3.0}, new boolean[]{true, false}); + try{ + Preconditions.checkNotNull(null, "Message %s here %s there %s more", new int[]{0,1}, new double[]{2.0, 3.0}, new boolean[]{true, false}); + } catch (NullPointerException e){ + assertEquals("Message [0, 1] here [2.0, 3.0] there [true, false] more", e.getMessage()); + } + + Preconditions.checkNotNull("", "Message %s here %s there", new String[]{"A", "B"}, new Object[]{1.0, "C"}); + try{ + Preconditions.checkNotNull(null, "Message %s here %s there", new String[]{"A", "B"}, new Object[]{1.0, "C"}); + } catch (NullPointerException e){ + assertEquals("Message [A, B] here [1.0, C] there", e.getMessage()); + } + } + +} diff --git a/nd4j/nd4j-common/src/test/java/org/nd4j/common/function/FunctionalUtilsTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/function/FunctionalUtilsTest.java similarity index 100% rename from nd4j/nd4j-common/src/test/java/org/nd4j/common/function/FunctionalUtilsTest.java rename to cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/function/FunctionalUtilsTest.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java new file mode 100644 index 000000000..9e1591f82 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/io/ClassPathResourceTest.java @@ -0,0 +1,53 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.io; + + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +import org.nd4j.common.io.ClassPathResource; + +import java.io.File; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ClassPathResourceTest { + + + + @Test + public void testDirExtractingIntelliJ() throws Exception { + //https://github.com/deeplearning4j/deeplearning4j/issues/6483 + + ClassPathResource cpr = new ClassPathResource("somedir"); + + File f = new File(FileUtils.getTempDirectoryPath()+File.separatorChar+ UUID.randomUUID().toString()); + FileUtils.forceMkdir(f); + cpr.copyDirectory(f); + + File[] files = f.listFiles(); + assertEquals(1, files.length); + assertEquals("afile.txt", files[0].getName()); + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/loader/TestFileBatch.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/loader/TestFileBatch.java new file mode 100644 index 000000000..b3c924919 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/loader/TestFileBatch.java @@ -0,0 +1,107 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.loader; + +import org.apache.commons.io.FileUtils; + +import org.junit.jupiter.api.Test; + +import org.nd4j.common.loader.FileBatch; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestFileBatch { + + + @Test + public void testFileBatch() throws Exception { + File baseDir = FileUtils.getTempDirectory(); + + List fileList = new ArrayList<>(); + for( int i=0; i<10; i++ ){ + String s = "File contents - file " + i; + File f = new File(baseDir, "origFile" + i + ".txt"); + FileUtils.writeStringToFile(f, s, StandardCharsets.UTF_8); + fileList.add(f); + } + + FileBatch fb = FileBatch.forFiles(fileList); + + assertEquals(10, fb.getFileBytes().size()); + assertEquals(10, fb.getOriginalUris().size()); + for( int i=0; i<10; i++ ){ + byte[] expBytes = ("File contents - file " + i).getBytes(StandardCharsets.UTF_8); + byte[] actBytes = fb.getFileBytes().get(i); + assertArrayEquals(expBytes, actBytes); + + String expPath = fileList.get(i).toURI().toString(); + String actPath = fb.getOriginalUris().get(i); + assertEquals(expPath, actPath); + } + + //Save and load: + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + fb.writeAsZip(baos); + byte[] asBytes = baos.toByteArray(); + + FileBatch fb2; + try(ByteArrayInputStream bais = new ByteArrayInputStream(asBytes)){ + fb2 = FileBatch.readFromZip(bais); + } + + assertEquals(fb.getOriginalUris(), fb2.getOriginalUris()); + assertEquals(10, fb2.getFileBytes().size()); + for( int i=0; i<10; i++ ){ + assertArrayEquals(fb.getFileBytes().get(i), fb2.getFileBytes().get(i)); + } + + //Check that it is indeed a valid zip file: + + File f = new File(FileUtils.getTempDirectoryPath()+"/"+UUID.randomUUID().toString()); + f.delete(); + fb.writeAsZip(f); + + ZipFile zf = new ZipFile(f); + Enumeration e = zf.entries(); + int count = 0; + Set names = new HashSet<>(); + while(e.hasMoreElements()){ + ZipEntry entry = e.nextElement(); + names.add(entry.getName()); + } + + assertEquals(11, names.size()); //10 files, 1 "original file names" file + assertTrue(names.contains(FileBatch.ORIGINAL_PATHS_FILENAME)); + for( int i=0; i<10; i++ ){ + String n = "file_" + i + ".txt"; + assertTrue(names.contains(n), n); + } + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/primitives/AtomicTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/primitives/AtomicTest.java new file mode 100644 index 000000000..c3ebbfa60 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/primitives/AtomicTest.java @@ -0,0 +1,76 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.val; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.primitives.Atomic; +import org.nd4j.common.util.SerializationUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + + + +public class AtomicTest { + + @Test + public void testEquality_1() { + val v0 = new Atomic(1327541); + val v1 = new Atomic(1327541); + val v3 = new Atomic(1327542); + + Assertions.assertEquals(v0, v1); + Assertions.assertNotEquals(v0, v3); + } + + @Test + public void testSerialization_1() throws Exception { + val v0 = new Atomic(1327541); + + try (val baos = new ByteArrayOutputStream()) { + SerializationUtils.serialize(v0, baos); + + try (val bais = new ByteArrayInputStream(baos.toByteArray())) { + Atomic v1 = SerializationUtils.deserialize(bais); + + Assertions.assertEquals(v1, v0); + } + } + } + + @Test + public void testCas_1() throws Exception { + val v0 = new Atomic(); + + v0.cas(null, "alpha"); + Assertions.assertEquals("alpha", v0.get()); + } + + @Test + public void testCas_2() throws Exception { + val v0 = new Atomic("beta"); + + v0.cas(null, "alpha"); + Assertions.assertEquals("beta", v0.get()); + } +} \ No newline at end of file diff --git a/nd4j/nd4j-common/src/test/java/org/nd4j/common/primitives/CounterMapTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/primitives/CounterMapTest.java similarity index 100% rename from nd4j/nd4j-common/src/test/java/org/nd4j/common/primitives/CounterMapTest.java rename to cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/primitives/CounterMapTest.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/primitives/CounterTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/primitives/CounterTest.java new file mode 100644 index 000000000..ec8b3e6d1 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/primitives/CounterTest.java @@ -0,0 +1,131 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.primitives; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.primitives.Counter; + +import java.util.List; + +@Slf4j +public class CounterTest { + + @Test + public void testCounterIncrementAll1() { + Counter counterA = new Counter<>(); + + counterA.incrementCount("A", 1); + counterA.incrementCount("A", 1); + counterA.incrementCount("A", 1); + + + + Counter counterB = new Counter<>(); + counterB.incrementCount("B", 2); + counterB.incrementCount("B", 2); + + Assertions.assertEquals(3.0, counterA.getCount("A"), 1e-5); + Assertions.assertEquals(4.0, counterB.getCount("B"), 1e-5); + + counterA.incrementAll(counterB); + + Assertions.assertEquals(3.0, counterA.getCount("A"), 1e-5); + Assertions.assertEquals(4.0, counterA.getCount("B"), 1e-5); + + counterA.setCount("B", 234); + + Assertions.assertEquals(234.0, counterA.getCount("B"), 1e-5); + } + + + + @Test + public void testCounterTopN1() { + Counter counterA = new Counter<>(); + + counterA.incrementCount("A", 1); + counterA.incrementCount("B", 2); + counterA.incrementCount("C", 3); + counterA.incrementCount("D", 4); + counterA.incrementCount("E", 5); + + counterA.keepTopNElements(4); + + Assertions.assertEquals(4,counterA.size()); + + // we expect element A to be gone + Assertions.assertEquals(0.0, counterA.getCount("A"), 1e-5); + Assertions.assertEquals(2.0, counterA.getCount("B"), 1e-5); + Assertions.assertEquals(3.0, counterA.getCount("C"), 1e-5); + Assertions.assertEquals(4.0, counterA.getCount("D"), 1e-5); + Assertions.assertEquals(5.0, counterA.getCount("E"), 1e-5); + } + + @Test + public void testKeysSorted1() throws Exception { + Counter counterA = new Counter<>(); + + counterA.incrementCount("A", 1); + counterA.incrementCount("B", 2); + counterA.incrementCount("C", 3); + counterA.incrementCount("D", 4); + counterA.incrementCount("E", 5); + + Assertions.assertEquals("E", counterA.argMax()); + + List list = counterA.keySetSorted(); + + Assertions.assertEquals(5, list.size()); + + Assertions.assertEquals("E", list.get(0)); + Assertions.assertEquals("D", list.get(1)); + Assertions.assertEquals("C", list.get(2)); + Assertions.assertEquals("B", list.get(3)); + Assertions.assertEquals("A", list.get(4)); + } + + @Test + public void testCounterTotal() { + Counter counter = new Counter<>(); + + counter.incrementCount("A", 1); + counter.incrementCount("B", 1); + counter.incrementCount("C", 1); + + Assertions.assertEquals(3.0, counter.totalCount(), 1e-5); + + counter.setCount("B", 234); + + Assertions.assertEquals(236.0, counter.totalCount(), 1e-5); + + counter.setCount("D", 1); + + Assertions.assertEquals(237.0, counter.totalCount(), 1e-5); + + counter.removeKey("B"); + + Assertions.assertEquals(3.0, counter.totalCount(), 1e-5); + + } + +} \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/resources/TestArchiveUtils.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/resources/TestArchiveUtils.java new file mode 100644 index 000000000..83ab40816 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/resources/TestArchiveUtils.java @@ -0,0 +1,68 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import org.nd4j.common.util.ArchiveUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class TestArchiveUtils { + + + @Test + public void testUnzipFileTo() throws IOException { + //random txt file + File dir = FileUtils.getTempDirectory(); + String content = "test file content"; + String path = "myDir/myTestFile.txt"; + File testFile = new File(dir, path); + testFile.getParentFile().mkdir(); + FileUtils.writeStringToFile(testFile, content, StandardCharsets.UTF_8); + + //zip it as test.zip + File zipFile = new File(testFile.getParentFile(),"test.zip"); + FileOutputStream fos = new FileOutputStream(zipFile); + ZipOutputStream zipOut = new ZipOutputStream(fos); + FileInputStream fis = new FileInputStream(testFile); + ZipEntry zipEntry = new ZipEntry(testFile.getName()); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + zipOut.close(); + fis.close(); + fos.close(); + + //now unzip to a directory that doesn't previously exist + File unzipDir = new File(testFile.getParentFile(),"unzipTo"); + ArchiveUtils.unzipFileTo(zipFile.getAbsolutePath(),unzipDir.getAbsolutePath()); + } +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/resources/TestStrumpf.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/resources/TestStrumpf.java new file mode 100644 index 000000000..5b23cec0b --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/resources/TestStrumpf.java @@ -0,0 +1,102 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.resources; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.LineIterator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.nd4j.common.config.ND4JSystemProperties; +import org.nd4j.common.resources.Resources; +import org.nd4j.common.resources.strumpf.StrumpfResolver; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; + +public class TestStrumpf { + + public File testDir = FileUtils.getTempDirectory(); + + @Test + public void testResolvingReference() throws Exception { + + File f = Resources.asFile("big/raw_sentences.txt"); + Assertions.assertTrue(f.exists()); + + System.out.println(f.getAbsolutePath()); + try(Reader r = new BufferedReader(new FileReader(f))){ + LineIterator iter = IOUtils.lineIterator(r); + for( int i=0; i<5 && iter.hasNext(); i++ ){ + System.out.println("LINE " + i + ": " + iter.next()); + } + } + } + + @Test + public void testResolvingActual() throws Exception { + File f = Resources.asFile("data/irisSvmLight.txt"); + Assertions.assertTrue(f.exists()); + + //System.out.println(f.getAbsolutePath()); + int count = 0; + try(Reader r = new BufferedReader(new FileReader(f))){ + LineIterator iter = IOUtils.lineIterator(r); + while(iter.hasNext()){ + String line = iter.next(); + //System.out.println("LINE " + i + ": " + line); + count++; + } + } + + Assertions.assertEquals(12, count); //Iris normally has 150 examples; this is subset with 12 + } + + @Test + public void testResolveLocal() throws Exception { + + File dir = testDir; + + String content = "test file content"; + String path = "myDir/myTestFile.txt"; + File testFile = new File(dir, path); + testFile.getParentFile().mkdir(); + FileUtils.writeStringToFile(testFile, content, StandardCharsets.UTF_8); + + System.setProperty(ND4JSystemProperties.RESOURCES_LOCAL_DIRS, dir.getAbsolutePath()); + + try{ + StrumpfResolver r = new StrumpfResolver(); + Assertions.assertTrue(r.exists(path)); + File f = r.asFile(path); + Assertions.assertTrue(f.exists()); + Assertions.assertEquals(testFile.getAbsolutePath(), f.getAbsolutePath()); + String s = FileUtils.readFileToString(f, StandardCharsets.UTF_8); + Assertions.assertEquals(content, s); + } finally { + System.setProperty(ND4JSystemProperties.RESOURCES_LOCAL_DIRS, ""); + } + } + +} diff --git a/nd4j/nd4j-common/src/test/java/org/nd4j/common/tools/BToolsTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/BToolsTest.java similarity index 100% rename from nd4j/nd4j-common/src/test/java/org/nd4j/common/tools/BToolsTest.java rename to cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/BToolsTest.java diff --git a/nd4j/nd4j-common/src/test/java/org/nd4j/common/tools/InfoLineTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/InfoLineTest.java similarity index 100% rename from nd4j/nd4j-common/src/test/java/org/nd4j/common/tools/InfoLineTest.java rename to cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/InfoLineTest.java diff --git a/nd4j/nd4j-common/src/test/java/org/nd4j/common/tools/InfoValuesTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/InfoValuesTest.java similarity index 100% rename from nd4j/nd4j-common/src/test/java/org/nd4j/common/tools/InfoValuesTest.java rename to cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/InfoValuesTest.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/PropertyParserTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/PropertyParserTest.java new file mode 100644 index 000000000..fd9475edd --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/PropertyParserTest.java @@ -0,0 +1,1331 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import org.junit.jupiter.api.*; +import org.nd4j.common.tools.PropertyParser; + +/** + * Tests for PropertyParser + * + * @author gagatust + */ +public class PropertyParserTest { + + public PropertyParserTest() { + } + + @BeforeAll + public static void setUpClass() { + } + + @AfterAll + public static void tearDownClass() { + } + + @BeforeEach + public void setUp() { + } + + @AfterEach + public void tearDown() { + } + + /** + * Test of getProperties method, of class PropertyParser. + */ + @Test + public void testGetProperties() { + + } + + /** + * Test of setProperties method, of class PropertyParser. + */ + @Test + public void testSetProperties() { + + } + + /** + * Test of parseString method, of class PropertyParser. + */ + @Test + public void testParseString() { + System.out.println("parseString"); + String expResult; + String result; + + Properties props = new Properties(); + props.put("value1", "sTr1"); + props.put("value2", "str_2"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = "sTr1"; + result = instance.parseString("value1"); + assertEquals(expResult, result); + + expResult = "str_2"; + result = instance.parseString("value2"); + assertEquals(expResult, result); + + expResult = ""; + result = instance.parseString("empty"); + assertEquals(expResult, result); + + expResult = "abc"; + result = instance.parseString("str"); + assertEquals(expResult, result); + + expResult = "true"; + result = instance.parseString("boolean"); + assertEquals(expResult, result); + + expResult = "24.98"; + result = instance.parseString("float"); + assertEquals(expResult, result); + + expResult = "12"; + result = instance.parseString("int"); + assertEquals(expResult, result); + + expResult = "a"; + result = instance.parseString("char"); + assertEquals(expResult, result); + + try { + instance.parseString("nonexistent"); + fail("no exception"); + } catch (NullPointerException e) { + } + } + + /** + * Test of parseInt method, of class PropertyParser. + */ + @Test + public void testParseInt() { + System.out.println("parseInt"); + int expResult; + int result; + + Properties props = new Properties(); + props.put("value1", "432"); + props.put("value2", "-242"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 432; + result = instance.parseInt("value1"); + assertEquals(expResult, result); + + expResult = -242; + result = instance.parseInt("value2"); + assertEquals(expResult, result); + + try { + instance.parseInt("empty"); + fail("no exception"); + } catch (NumberFormatException e) { + } + + try { + instance.parseInt("str"); + fail("no exception"); + } catch (NumberFormatException e) { + } + + try { + instance.parseInt("boolean"); + assertEquals(expResult, result); + fail("no exception"); + } catch (NumberFormatException e) { + } + + try { + instance.parseInt("float"); + fail("no exception"); + } catch (NumberFormatException e) { + } + + expResult = 12; + result = instance.parseInt("int"); + assertEquals(expResult, result); + + try { + instance.parseInt("char"); + fail("no exception"); + } catch (NumberFormatException e) { + } + + try { + expResult = 0; + result = instance.parseInt("nonexistent"); + fail("no exception"); + assertEquals(expResult, result); + } catch (IllegalArgumentException e) { + } + } + + /** + * Test of parseBoolean method, of class PropertyParser. + */ + @Test + public void testParseBoolean() { + System.out.println("parseBoolean"); + boolean expResult; + boolean result; + + Properties props = new Properties(); + props.put("value1", "true"); + props.put("value2", "false"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = true; + result = instance.parseBoolean("value1"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("value2"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("empty"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("str"); + assertEquals(expResult, result); + + expResult = true; + result = instance.parseBoolean("boolean"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("float"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("int"); + assertEquals(expResult, result); + + expResult = false; + result = instance.parseBoolean("char"); + assertEquals(expResult, result); + + try { + expResult = false; + result = instance.parseBoolean("nonexistent"); + fail("no exception"); + assertEquals(expResult, result); + } catch (IllegalArgumentException e) { + } + } + + /** + * Test of parseDouble method, of class PropertyParser. + */ + @Test + public void testParseFloat() { + System.out.println("parseFloat"); + double expResult; + double result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789f; + result = instance.parseFloat("value1"); + assertEquals(expResult, result, 0); + + expResult = -9000.001f; + result = instance.parseFloat("value2"); + assertEquals(expResult, result, 0); + + try { + instance.parseFloat("empty"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseFloat("str"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseFloat("boolean"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + expResult = 24.98f; + result = instance.parseFloat("float"); + assertEquals(expResult, result, 0); + + expResult = 12f; + result = instance.parseFloat("int"); + assertEquals(expResult, result, 0); + + try { + instance.parseFloat("char"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseFloat("nonexistent"); + fail("no exception"); + } catch (NullPointerException e) { + } + } + + /** + * Test of parseDouble method, of class PropertyParser. + */ + @Test + public void testParseDouble() { + System.out.println("parseDouble"); + double expResult; + double result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789; + result = instance.parseDouble("value1"); + assertEquals(expResult, result, 0); + + expResult = -9000.001; + result = instance.parseDouble("value2"); + assertEquals(expResult, result, 0); + + try { + instance.parseDouble("empty"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseDouble("str"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseDouble("boolean"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + expResult = 24.98; + result = instance.parseDouble("float"); + assertEquals(expResult, result, 0); + + expResult = 12; + result = instance.parseDouble("int"); + assertEquals(expResult, result, 0); + + try { + instance.parseDouble("char"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseDouble("nonexistent"); + fail("no exception"); + } catch (NullPointerException e) { + } + } + + /** + * Test of parseLong method, of class PropertyParser. + */ + @Test + public void testParseLong() { + System.out.println("parseLong"); + long expResult; + long result; + + Properties props = new Properties(); + props.put("value1", "12345678900"); + props.put("value2", "-9000001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345678900L; + result = instance.parseLong("value1"); + assertEquals(expResult, result); + + expResult = -9000001L; + result = instance.parseLong("value2"); + assertEquals(expResult, result); + + try { + instance.parseLong("empty"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseLong("str"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseLong("boolean"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseLong("float"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + expResult = 12L; + result = instance.parseLong("int"); + assertEquals(expResult, result); + + try { + instance.parseLong("char"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseLong("nonexistent"); + fail("no exception"); + } catch (IllegalArgumentException e) { + } + } + + /** + * Test of parseChar method, of class PropertyParser. + */ + @Test + public void testParseChar() { + System.out.println("parseChar"); + char expResult; + char result; + + Properties props = new Properties(); + props.put("value1", "b"); + props.put("value2", "c"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 'b'; + result = instance.parseChar("value1"); + assertEquals(expResult, result); + + expResult = 'c'; + result = instance.parseChar("value2"); + assertEquals(expResult, result); + + try { + instance.parseChar("empty"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseChar("str"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseChar("boolean"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseChar("float"); + } catch (IllegalArgumentException e) { + } + + try { + instance.parseChar("int"); + } catch (IllegalArgumentException e) { + } + + expResult = 'a'; + result = instance.parseChar("char"); + assertEquals(expResult, result); + + try { + instance.parseChar("nonexistent"); + fail("no exception"); + assertEquals(expResult, result); + } catch (NullPointerException e) { + } + } + + /** + * Test of toString method, of class PropertyParser. + */ + @Test + public void testToString_String() { + System.out.println("toString"); + String expResult; + String result; + + Properties props = new Properties(); + props.put("value1", "sTr1"); + props.put("value2", "str_2"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = "sTr1"; + result = instance.toString("value1"); + assertEquals(expResult, result); + + expResult = "str_2"; + result = instance.toString("value2"); + assertEquals(expResult, result); + + expResult = ""; + result = instance.toString("empty"); + assertEquals(expResult, result); + + expResult = "abc"; + result = instance.toString("str"); + assertEquals(expResult, result); + + expResult = "true"; + result = instance.toString("boolean"); + assertEquals(expResult, result); + + expResult = "24.98"; + result = instance.toString("float"); + assertEquals(expResult, result); + + expResult = "12"; + result = instance.toString("int"); + assertEquals(expResult, result); + + expResult = "a"; + result = instance.toString("char"); + assertEquals(expResult, result); + + expResult = ""; + result = instance.toString("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toInt method, of class PropertyParser. + */ + @Test + public void testToInt_String() { + System.out.println("toInt"); + int expResult; + int result; + + Properties props = new Properties(); + props.put("value1", "123"); + props.put("value2", "-54"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 123; + result = instance.toInt("value1"); + assertEquals(expResult, result); + + expResult = -54; + result = instance.toInt("value2"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("empty"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("str"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("boolean"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("float"); + assertEquals(expResult, result); + + expResult = 12; + result = instance.toInt("int"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("char"); + assertEquals(expResult, result); + + expResult = 0; + result = instance.toInt("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toBoolean method, of class PropertyParser. + */ + @Test + public void testToBoolean_String() { + System.out.println("toBoolean"); + boolean expResult; + boolean result; + + Properties props = new Properties(); + props.put("value1", "true"); + props.put("value2", "false"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = true; + result = instance.toBoolean("value1"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("value2"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("empty"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("str"); + assertEquals(expResult, result); + + expResult = true; + result = instance.toBoolean("boolean"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("float"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("int"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("char"); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toDouble method, of class PropertyParser. + */ + @Test + public void testToFloat_String() { + System.out.println("toFloat"); + float expResult; + float result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789f; + result = instance.toFloat("value1"); + assertEquals(expResult, result, 0f); + + expResult = -9000.001f; + result = instance.toFloat("value2"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("empty"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("str"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("boolean"); + assertEquals(expResult, result, 0f); + + expResult = 24.98f; + result = instance.toFloat("float"); + assertEquals(expResult, result, 0f); + + expResult = 12f; + result = instance.toFloat("int"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("char"); + assertEquals(expResult, result, 0f); + + expResult = 0f; + result = instance.toFloat("nonexistent"); + assertEquals(expResult, result, 0f); + } + + /** + * Test of toDouble method, of class PropertyParser. + */ + @Test + public void testToDouble_String() { + System.out.println("toDouble"); + double expResult; + double result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789; + result = instance.toDouble("value1"); + assertEquals(expResult, result, 0); + + expResult = -9000.001; + result = instance.toDouble("value2"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("empty"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("str"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("boolean"); + assertEquals(expResult, result, 0); + + expResult = 24.98; + result = instance.toDouble("float"); + assertEquals(expResult, result, 0); + + expResult = 12; + result = instance.toDouble("int"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("char"); + assertEquals(expResult, result, 0); + + expResult = 0; + result = instance.toDouble("nonexistent"); + assertEquals(expResult, result, 0); + } + + /** + * Test of toLong method, of class PropertyParser. + */ + @Test + public void testToLong_String() { + System.out.println("toLong"); + long expResult; + long result; + + Properties props = new Properties(); + props.put("value1", "12345678900"); + props.put("value2", "-9000001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345678900L; + result = instance.toLong("value1"); + assertEquals(expResult, result); + + expResult = -9000001L; + result = instance.toLong("value2"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("empty"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("str"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("boolean"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("float"); + assertEquals(expResult, result); + + expResult = 12L; + result = instance.toLong("int"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("char"); + assertEquals(expResult, result); + + expResult = 0L; + result = instance.toLong("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toChar method, of class PropertyParser. + */ + @Test + public void testToChar_String() { + System.out.println("toChar"); + char expResult; + char result; + + Properties props = new Properties(); + props.put("value1", "f"); + props.put("value2", "w"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 'f'; + result = instance.toChar("value1"); + assertEquals(expResult, result); + + expResult = 'w'; + result = instance.toChar("value2"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("empty"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("str"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("boolean"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("float"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("int"); + assertEquals(expResult, result); + + expResult = 'a'; + result = instance.toChar("char"); + assertEquals(expResult, result); + + expResult = '\u0000'; + result = instance.toChar("nonexistent"); + assertEquals(expResult, result); + } + + /** + * Test of toString method, of class PropertyParser. + */ + @Test + public void testToString_String_String() { + System.out.println("toString"); + String expResult; + String result; + + Properties props = new Properties(); + props.put("value1", "sTr1"); + props.put("value2", "str_2"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = "sTr1"; + result = instance.toString("value1", "defStr"); + assertEquals(expResult, result); + + expResult = "str_2"; + result = instance.toString("value2", "defStr"); + assertEquals(expResult, result); + + expResult = ""; + result = instance.toString("empty", "defStr"); + assertEquals(expResult, result); + + expResult = "abc"; + result = instance.toString("str", "defStr"); + assertEquals(expResult, result); + + expResult = "true"; + result = instance.toString("boolean", "defStr"); + assertEquals(expResult, result); + + expResult = "24.98"; + result = instance.toString("float", "defStr"); + assertEquals(expResult, result); + + expResult = "12"; + result = instance.toString("int", "defStr"); + assertEquals(expResult, result); + + expResult = "a"; + result = instance.toString("char", "defStr"); + assertEquals(expResult, result); + + expResult = "defStr"; + result = instance.toString("nonexistent", "defStr"); + assertEquals(expResult, result); + } + + /** + * Test of toInt method, of class PropertyParser. + */ + @Test + public void testToInt_String_int() { + System.out.println("toInt"); + int expResult; + int result; + + Properties props = new Properties(); + props.put("value1", "123"); + props.put("value2", "-54"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 123; + result = instance.toInt("value1", 17); + assertEquals(expResult, result); + + expResult = -54; + result = instance.toInt("value2", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("empty", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("str", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("boolean", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("float", 17); + assertEquals(expResult, result); + + expResult = 12; + result = instance.toInt("int", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("char", 17); + assertEquals(expResult, result); + + expResult = 17; + result = instance.toInt("nonexistent", 17); + assertEquals(expResult, result); + } + + /** + * Test of toBoolean method, of class PropertyParser. + */ + @Test + public void testToBoolean_String_boolean() { + System.out.println("toBoolean"); + + boolean expResult; + boolean result; + + Properties props = new Properties(); + props.put("value1", "true"); + props.put("value2", "false"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = true; + result = instance.toBoolean("value1", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("value2", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("empty", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("str", true); + assertEquals(expResult, result); + + expResult = true; + result = instance.toBoolean("boolean", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("float", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("int", true); + assertEquals(expResult, result); + + expResult = false; + result = instance.toBoolean("char", true); + assertEquals(expResult, result); + + expResult = true; + result = instance.toBoolean("nonexistent", true); + assertEquals(expResult, result); + } + + /** + * Test of toDouble method, of class PropertyParser. + */ + @Test + public void testToFloat_String_float() { + System.out.println("toFloat"); + float expResult; + float result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789f; + result = instance.toFloat("value1", 0.123f); + assertEquals(expResult, result, 0); + + expResult = -9000.001f; + result = instance.toFloat("value2", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("empty", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("str", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("boolean", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 24.98f; + result = instance.toFloat("float", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 12; + result = instance.toFloat("int", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("char", 0.123f); + assertEquals(expResult, result, 0); + + expResult = 0.123f; + result = instance.toFloat("nonexistent", 0.123f); + assertEquals(expResult, result, 0); + } + + /** + * Test of toDouble method, of class PropertyParser. + */ + @Test + public void testToDouble_String_double() { + System.out.println("toDouble"); + double expResult; + double result; + + Properties props = new Properties(); + props.put("value1", "12345.6789"); + props.put("value2", "-9000.001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345.6789; + result = instance.toDouble("value1", 0.123); + assertEquals(expResult, result, 0); + + expResult = -9000.001; + result = instance.toDouble("value2", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("empty", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("str", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("boolean", 0.123); + assertEquals(expResult, result, 0); + + expResult = 24.98; + result = instance.toDouble("float", 0.123); + assertEquals(expResult, result, 0); + + expResult = 12; + result = instance.toDouble("int", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("char", 0.123); + assertEquals(expResult, result, 0); + + expResult = 0.123; + result = instance.toDouble("nonexistent", 0.123); + assertEquals(expResult, result, 0); + } + + /** + * Test of toLong method, of class PropertyParser. + */ + @Test + public void testToLong_String_long() { + System.out.println("toLong"); + long expResult; + long result; + + Properties props = new Properties(); + props.put("value1", "12345678900"); + props.put("value2", "-9000001"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 12345678900L; + result = instance.toLong("value1", 3L); + assertEquals(expResult, result); + + expResult = -9000001L; + result = instance.toLong("value2", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("empty", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("str", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("boolean", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("float", 3L); + assertEquals(expResult, result); + + expResult = 12L; + result = instance.toLong("int", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("char", 3L); + assertEquals(expResult, result); + + expResult = 3L; + result = instance.toLong("nonexistent", 3L); + assertEquals(expResult, result); + } + + /** + * Test of toChar method, of class PropertyParser. + */ + @Test + public void testToChar_String_char() { + System.out.println("toChar"); + char expResult; + char result; + + Properties props = new Properties(); + props.put("value1", "f"); + props.put("value2", "w"); + props.put("empty", ""); + props.put("str", "abc"); + props.put("boolean", "true"); + props.put("float", "24.98"); + props.put("int", "12"); + props.put("char", "a"); + PropertyParser instance = new PropertyParser(props); + + expResult = 'f'; + result = instance.toChar("value1", 't'); + assertEquals(expResult, result); + + expResult = 'w'; + result = instance.toChar("value2", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("empty", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("str", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("boolean", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("float", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("int", 't'); + assertEquals(expResult, result); + + expResult = 'a'; + result = instance.toChar("char", 't'); + assertEquals(expResult, result); + + expResult = 't'; + result = instance.toChar("nonexistent", 't'); + assertEquals(expResult, result); + } + +} diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/SISTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/SISTest.java new file mode 100644 index 000000000..e89fdd324 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/tools/SISTest.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.common.tools; + + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import org.nd4j.common.tools.SIS; + +import static org.junit.jupiter.api.Assertions.*; + +public class SISTest { + // + + private SIS sis; + // + + @Test + public void testAll() throws Exception { + // + sis = new SIS(); + // + int mtLv = 0; + // + sis.initValues( mtLv, "TEST", System.out, System.err, FileUtils.getTempDirectory().getAbsolutePath(), "Test", "ABC", true, true ); + // + String fFName = sis.getfullFileName(); + sis.info( fFName ); + sis.info( "aaabbbcccdddeefff" ); + // + assertEquals( 33, fFName.length() ); + assertEquals( "Z", fFName.substring( 0, 1 ) ); + assertEquals( "_Test_ABC.txt", fFName.substring( fFName.length() - 13, fFName.length() ) ); + // assertEquals( "", fFName ); + // assertEquals( "", tmpFld.getRoot().getAbsolutePath() ); + // + } + + @AfterEach + public void after() { + // + int mtLv = 0; + if ( sis != null ) sis.onStop( mtLv ); + // + // tmpFld.delete(); + // + } + + + +} \ No newline at end of file diff --git a/nd4j/nd4j-common/src/test/java/org/nd4j/common/util/ArrayUtilTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/util/ArrayUtilTest.java similarity index 100% rename from nd4j/nd4j-common/src/test/java/org/nd4j/common/util/ArrayUtilTest.java rename to cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/util/ArrayUtilTest.java diff --git a/nd4j/nd4j-common/src/test/java/org/nd4j/common/util/OneTimeLoggerTest.java b/cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/util/OneTimeLoggerTest.java similarity index 100% rename from nd4j/nd4j-common/src/test/java/org/nd4j/common/util/OneTimeLoggerTest.java rename to cavis-nd4j/cavis-nd4j-common/src/test/java/org/nd4j/common/util/OneTimeLoggerTest.java diff --git a/cavis-nd4j/cavis-nd4j-common/src/test/resources/somedir/afile.txt b/cavis-nd4j/cavis-nd4j-common/src/test/resources/somedir/afile.txt new file mode 100644 index 000000000..94f6db57d --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-common/src/test/resources/somedir/afile.txt @@ -0,0 +1 @@ +This file is to test ClassPathResource directory extracting in IntelliJ \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle new file mode 100644 index 000000000..af5e0aa84 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/build.gradle @@ -0,0 +1,38 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation 'com.mashape.unirest:unirest-java:1.4.9' + implementation "io.aeron:aeron-all:1.32.0" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "org.slf4j:slf4j-api" + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerModel + implementation projects.cavisNd4j.cavisNd4jAeron + implementation projects.cavisDnn.cavisDnnApi + + testImplementation 'org.zeroturnaround:zt-exec:1.9' + testImplementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore + testImplementation projects.cavisNd4j.cavisNd4jCommonTests +} + diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java similarity index 99% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java index da853bcfb..b1b2d6abc 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/main/java/org/nd4j/parameterserver/client/ParameterServerClient.java @@ -33,7 +33,7 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.model.MasterStatus; import org.nd4j.parameterserver.model.ServerTypeJson; import org.nd4j.parameterserver.model.SubscriberState; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/BackgroundDaemonStarter.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/BackgroundDaemonStarter.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/BackgroundDaemonStarter.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/BackgroundDaemonStarter.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java similarity index 94% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java index 795309263..62175a7fa 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/background/RemoteParameterServerClientTests.java @@ -26,11 +26,13 @@ import io.aeron.driver.ThreadingMode; import lombok.extern.slf4j.Slf4j; import org.agrona.CloseHelper; import org.agrona.concurrent.BusySpinIdleStrategy; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.aeron.ipc.AeronUtil; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.client.ParameterServerClient; @@ -40,10 +42,6 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; @Slf4j -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class RemoteParameterServerClientTests extends BaseND4JTest { private int parameterLength = 1000; private Aeron.Context ctx; @@ -95,9 +93,7 @@ public class RemoteParameterServerClientTests extends BaseND4JTest { CloseHelper.close(aeron); } - @Test() - @Timeout(60000L) - @Disabled //AB 20200425 https://github.com/eclipse/deeplearning4j/issues/8882 + @Test @Timeout(30) //@Ignore //AB 20200425 https://github.com/eclipse/deeplearning4j/issues/8882 public void remoteTests() throws Exception { if (masterStatus.get() != 0 || slaveStatus.get() != 0) throw new IllegalStateException("Master or slave failed to start. Exiting"); diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientPartialTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientPartialTest.java new file mode 100644 index 000000000..b57618211 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientPartialTest.java @@ -0,0 +1,150 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.client; + +import io.aeron.Aeron; +import io.aeron.driver.MediaDriver; +import io.aeron.driver.ThreadingMode; +import lombok.extern.slf4j.Slf4j; +import org.agrona.concurrent.BusySpinIdleStrategy; +import org.junit.jupiter.api.BeforeAll; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.aeron.ipc.AeronUtil; +import org.nd4j.aeron.ipc.NDArrayMessage; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.parameterserver.ParameterServerListener; +import org.nd4j.parameterserver.ParameterServerSubscriber; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Slf4j +public class ParameterServerClientPartialTest extends BaseND4JTest { + private static MediaDriver mediaDriver; + private static Aeron.Context ctx; + private static ParameterServerSubscriber masterNode, slaveNode; + private int[] shape = {2, 2}; + private static Aeron aeron; + + @BeforeAll + public static void beforeClass() throws Exception { + final MediaDriver.Context ctx = + new MediaDriver.Context().threadingMode(ThreadingMode.SHARED).dirDeleteOnStart(true) + .termBufferSparseFile(false).conductorIdleStrategy(new BusySpinIdleStrategy()) + .receiverIdleStrategy(new BusySpinIdleStrategy()) + .senderIdleStrategy(new BusySpinIdleStrategy()); + + mediaDriver = MediaDriver.launchEmbedded(ctx); + aeron = Aeron.connect(getContext()); + masterNode = new ParameterServerSubscriber(mediaDriver); + masterNode.setAeron(aeron); + int masterPort = 40223 + new java.util.Random().nextInt(13000); + int masterStatusPort = masterPort - 2000; + masterNode.run(new String[] {"-m", "true", "-p", String.valueOf(masterPort), "-h", "localhost", "-id", "11", + "-md", mediaDriver.aeronDirectoryName(), "-sp", String.valueOf(masterStatusPort), "-s", "2,2", + "-u", String.valueOf(1) + + }); + + assertTrue(masterNode.isMaster()); + assertEquals(masterPort, masterNode.getPort()); + assertEquals("localhost", masterNode.getHost()); + assertEquals(11, masterNode.getStreamId()); + assertEquals(12, masterNode.getResponder().getStreamId()); + assertEquals(masterNode.getMasterArray(), Nd4j.create(new int[] {2, 2})); + + slaveNode = new ParameterServerSubscriber(mediaDriver); + slaveNode.setAeron(aeron); + int slavePort = masterPort + 100; + int slaveStatusPort = slavePort - 2000; + slaveNode.run(new String[] {"-p", String.valueOf(slavePort), "-h", "localhost", "-id", "10", "-pm", + masterNode.getSubscriber().connectionUrl(), "-md", mediaDriver.aeronDirectoryName(), "-sp", + String.valueOf(slaveStatusPort), "-u", String.valueOf(1) + + }); + + assertFalse(slaveNode.isMaster()); + assertEquals(slavePort, slaveNode.getPort()); + assertEquals("localhost", slaveNode.getHost()); + assertEquals(10, slaveNode.getStreamId()); + + int tries = 10; + while (!masterNode.subscriberLaunched() && !slaveNode.subscriberLaunched() && tries < 10) { + Thread.sleep(10000); + tries++; + } + + if (!masterNode.subscriberLaunched() && !slaveNode.subscriberLaunched()) { + throw new IllegalStateException("Failed to start master and slave node"); + } + + log.info("Using media driver directory " + mediaDriver.aeronDirectoryName()); + log.info("Launched media driver"); + } + + + @Test + @Timeout(30) + //@Ignore("AB 2019/06/01 - Intermittent failures - see issue 7657") + public void testServer() throws Exception { + ParameterServerClient client = ParameterServerClient.builder().aeron(aeron) + .ndarrayRetrieveUrl(masterNode.getResponder().connectionUrl()) + .ndarraySendUrl(slaveNode.getSubscriber().connectionUrl()).subscriberHost("localhost") + .subscriberPort(40325).subscriberStream(12).build(); + assertEquals("localhost:40325:12", client.connectionUrl()); + //flow 1: + /** + * Client (40125:12): sends array to listener on slave(40126:10) + * which publishes to master (40123:11) + * which adds the array for parameter averaging. + * In this case totalN should be 1. + */ + client.pushNDArrayMessage(NDArrayMessage.of(Nd4j.ones(2), new int[] {0}, 0)); + log.info("Pushed ndarray"); + Thread.sleep(30000); + ParameterServerListener listener = (ParameterServerListener) masterNode.getCallback(); + assertEquals(1, listener.getUpdater().numUpdates()); + INDArray assertion = Nd4j.create(new int[] {2, 2}); + assertion.getColumn(0).addi(1.0); + assertEquals(assertion, listener.getUpdater().ndArrayHolder().get()); + INDArray arr = client.getArray(); + assertEquals(assertion, arr); + } + + + + private static Aeron.Context getContext() { + if (ctx == null) + ctx = new Aeron.Context().driverTimeoutMs(Long.MAX_VALUE) + .availableImageHandler(AeronUtil::printAvailableImage) + .unavailableImageHandler(AeronUtil::printUnavailableImage) + .aeronDirectoryName(mediaDriver.aeronDirectoryName()).keepAliveIntervalNs(10000) + .errorHandler(e -> log.error(e.toString(), e)); + return ctx; + } + + +} diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientTest.java similarity index 76% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientTest.java index ecb798cc9..985c77ec8 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/java/org/nd4j/parameterserver/client/ParameterServerClientTest.java @@ -22,11 +22,12 @@ package org.nd4j.parameterserver.client; import io.aeron.Aeron; import io.aeron.driver.MediaDriver; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.BeforeAll; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.aeron.ipc.AeronUtil; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.ParameterServerListener; @@ -34,12 +35,10 @@ import org.nd4j.parameterserver.ParameterServerSubscriber; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class ParameterServerClientTest extends BaseND4JTest { private static MediaDriver mediaDriver; private static Logger log = LoggerFactory.getLogger(ParameterServerClientTest.class); @@ -56,8 +55,8 @@ public class ParameterServerClientTest extends BaseND4JTest { masterNode.setAeron(aeron); int masterPort = 40323 + new java.util.Random().nextInt(3000); masterNode.run(new String[] {"-m", "true", "-s", "1," + String.valueOf(parameterLength), "-p", - String.valueOf(masterPort), "-h", "localhost", "-id", "11", "-md", - mediaDriver.aeronDirectoryName(), "-sp", "33000", "-u", String.valueOf(1)}); + String.valueOf(masterPort), "-h", "localhost", "-id", "11", "-md", + mediaDriver.aeronDirectoryName(), "-sp", "33000", "-u", String.valueOf(1)}); assertTrue(masterNode.isMaster()); assertEquals(masterPort, masterNode.getPort()); @@ -68,15 +67,15 @@ public class ParameterServerClientTest extends BaseND4JTest { slaveNode = new ParameterServerSubscriber(mediaDriver); slaveNode.setAeron(aeron); slaveNode.run(new String[] {"-p", String.valueOf(masterPort + 100), "-h", "localhost", "-id", "10", "-pm", - masterNode.getSubscriber().connectionUrl(), "-md", mediaDriver.aeronDirectoryName(), "-sp", - "31000", "-u", String.valueOf(1)}); + masterNode.getSubscriber().connectionUrl(), "-md", mediaDriver.aeronDirectoryName(), "-sp", + "31000", "-u", String.valueOf(1)}); assertFalse(slaveNode.isMaster()); assertEquals(masterPort + 100, slaveNode.getPort()); assertEquals("localhost", slaveNode.getHost()); assertEquals(10, slaveNode.getStreamId()); - int tries = 10; + int tries = 1; while (!masterNode.subscriberLaunched() && !slaveNode.subscriberLaunched() && tries < 10) { Thread.sleep(10000); tries++; @@ -92,15 +91,15 @@ public class ParameterServerClientTest extends BaseND4JTest { - @Test() - @Timeout(60000L) - @Disabled("AB 2019/05/31 - Intermittent failures on CI - see issue 7657") + @Test + @Timeout(30) + //@Ignore("AB 2019/05/31 - Intermittent failures on CI - see issue 7657") public void testServer() throws Exception { int subscriberPort = 40625 + new java.util.Random().nextInt(100); ParameterServerClient client = ParameterServerClient.builder().aeron(aeron) - .ndarrayRetrieveUrl(masterNode.getResponder().connectionUrl()) - .ndarraySendUrl(slaveNode.getSubscriber().connectionUrl()).subscriberHost("localhost") - .subscriberPort(subscriberPort).subscriberStream(12).build(); + .ndarrayRetrieveUrl(masterNode.getResponder().connectionUrl()) + .ndarraySendUrl(slaveNode.getSubscriber().connectionUrl()).subscriberHost("localhost") + .subscriberPort(subscriberPort).subscriberStream(12).build(); assertEquals(String.format("localhost:%d:12", subscriberPort), client.connectionUrl()); //flow 1: /** @@ -123,10 +122,10 @@ public class ParameterServerClientTest extends BaseND4JTest { private static Aeron.Context getContext() { return new Aeron.Context().driverTimeoutMs(Long.MAX_VALUE) - .availableImageHandler(AeronUtil::printAvailableImage) - .unavailableImageHandler(AeronUtil::printUnavailableImage) - .aeronDirectoryName(mediaDriver.aeronDirectoryName()).keepAliveIntervalNs(100000) - .errorHandler(e -> log.error(e.toString(), e)); + .availableImageHandler(AeronUtil::printAvailableImage) + .unavailableImageHandler(AeronUtil::printUnavailableImage) + .aeronDirectoryName(mediaDriver.aeronDirectoryName()).keepAliveIntervalNs(100000) + .errorHandler(e -> log.error(e.toString(), e)); } diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/resources/aeron.properties b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/resources/aeron.properties similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/resources/aeron.properties rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/resources/aeron.properties diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/resources/log4j.properties b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/resources/log4j.properties new file mode 100644 index 000000000..877c4ea9c --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/resources/log4j.properties @@ -0,0 +1,43 @@ +# +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + + +log4j.rootLogger=INFO, Console +log4j.logger.play=DEBUG +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n + +log4j.appender.org.springframework=DEBUG +log4j.appender.org.nd4j=INFO +log4j.logger.org.nd4j.aeron.ipc=INFO +log4j.appender.org.canova=INFO +log4j.appender.org.deeplearning4j=INFO +log4j.appender.opennlp.uima=OFF +log4j.appender.org.apache.uima=OFF +log4j.appender.org.cleartk=OFF + +log4j.logger.org.springframework=INFO +log4j.logger.org.nd4j=DEBUG +log4j.logger.org.deeplearning4j=INFO +log4j.logger.opennlp.uima.util=OFF +log4j.logger.org.apache.uima=OFF +log4j.logger.org.cleartk=OFF + diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/resources/logback.xml b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/resources/logback.xml similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-client/src/test/resources/logback.xml rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-client/src/test/resources/logback.xml diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/build.gradle b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/build.gradle new file mode 100644 index 000000000..ce69834b3 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/build.gradle @@ -0,0 +1,42 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerModel + implementation projects.cavisNd4j.cavisNd4jAeron + + testImplementation 'org.slf4j:slf4j-log4j12:1.7.30' + + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCommon + implementation "com.google.guava:guava" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "io.aeron:aeron-all:1.32.0" + implementation "org.slf4j:slf4j-api" + + implementation "com.beust:jcommander:1.27" + implementation 'com.mashape.unirest:unirest-java:1.4.9' + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests +} \ No newline at end of file diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/ParameterServerListener.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerListener.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/ParameterServerListener.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerListener.java diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java new file mode 100644 index 000000000..6aaca2e49 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/ParameterServerSubscriber.java @@ -0,0 +1,419 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.common.config.ND4JClassLoading; +import org.nd4j.common.io.ReflectionUtils; +import com.google.common.primitives.Ints; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.mashape.unirest.http.Unirest; +import io.aeron.Aeron; +import io.aeron.driver.MediaDriver; +import io.aeron.driver.ThreadingMode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.val; +import org.agrona.CloseHelper; +import org.agrona.concurrent.BusySpinIdleStrategy; +import org.json.JSONObject; +import org.nd4j.aeron.ipc.AeronNDArraySubscriber; +import org.nd4j.aeron.ipc.AeronUtil; +import org.nd4j.aeron.ipc.NDArrayCallback; +import org.nd4j.aeron.ipc.NDArrayHolder; +import org.nd4j.aeron.ipc.response.AeronNDArrayResponder; +import org.nd4j.aeron.ndarrayholder.InMemoryNDArrayHolder; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.common.util.ArrayUtil; +import org.nd4j.parameterserver.model.MasterConnectionInfo; +import org.nd4j.parameterserver.model.ServerState; +import org.nd4j.parameterserver.model.SlaveConnectionInfo; +import org.nd4j.parameterserver.model.SubscriberState; +import org.nd4j.parameterserver.updater.ParameterServerUpdater; +import org.nd4j.parameterserver.updater.SoftSyncParameterUpdater; +import org.nd4j.parameterserver.updater.SynchronousParameterUpdater; +import org.nd4j.parameterserver.updater.storage.InMemoryUpdateStorage; +import org.nd4j.parameterserver.util.CheckSocket; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; + +@NoArgsConstructor +@Data +@Parameters(separators = ",") +public class ParameterServerSubscriber implements AutoCloseable { + + private static Logger log = LoggerFactory.getLogger(ParameterServerSubscriber.class); + + @Parameter(names = {"-p", "--port"}, description = "The port to listen on for the daemon", arity = 1) + private int port = 40123; + @Parameter(names = {"-id", "--streamId"}, description = "The stream id to listen on", arity = 1) + private int streamId = 10; + @Parameter(names = {"-h", "--host"}, description = "Host for the server to bind to", arity = 1) + private String host = "localhost"; + @Parameter(names = {"-d", "--deleteDirectoryOnStart"}, description = "Delete aeron directory on startup.", + arity = 1) + private boolean deleteDirectoryOnStart = true; + @Parameter(names = {"-m", "--master"}, description = "Whether this subscriber is a master node or not.", arity = 1) + private boolean master = false; + @Parameter(names = {"-pm", "--publishmaster"}, + description = "Publish master url: host:port - this is for peer nodes needing to publish to another peer.", + arity = 1) + private String publishMasterUrl = "localhost:40123"; + @Parameter(names = {"-md", "--mediadriverdirectory"}, + description = "The media driver directory opName. This is for when the media driver is started as a separate process.", + arity = 1) + private String mediaDriverDirectoryName; + @Parameter(names = {"-sp", "--statusserverport"}, description = "The status server port, defaults to 9000.", + arity = 1) + private int statusServerPort = 9000; + @Parameter(names = {"-sh", "--statusserverhost"}, description = "The status host, defaults to localhost.", + arity = 1) + private String statusServerHost = "localhost"; + @Parameter(names = {"-up", "--update"}, + description = "The update opType for this parameter server. Defaults to sync. You can specify custom and use a jvm argument -Dorg.nd4j.parameterserver.updatetype=your.fully.qualified.class if you want to use a custom class. This must be able to be instantiated from an empty constructor though.", + arity = 1) + private String updateTypeString = UpdateType.SYNC.toString().toLowerCase(); + + private UpdateType updateType = UpdateType.SYNC; + + @Parameter(names = {"-s", "--shape"}, description = "The shape of the ndarray", arity = 1) + private List shape; + @Parameter(names = {"-hbi", "--heartbeatinterval"}, description = "Heartbeat interval in ms", arity = 1) + private int heartbeatMs = 1000; + private ObjectMapper objectMapper = new ObjectMapper(); + private ScheduledExecutorService scheduledExecutorService; + @Parameter(names = {"-u", "--updatesPerEpoch"}, description = "The number of updates per epoch", arity = 1, + required = true) + private int updatesPerEpoch; + + + /** + * Specify a custom class as a jvm arg. + * Note that this class must be a fully qualified classname + */ + public final static String CUSTOM_UPDATE_TYPE = "org.nd4j.parameterserver.updatetype"; + + /** + * Update types are for + * instantiating various kinds of update types + */ + public enum UpdateType { + HOGWILD, SYNC, TIME_DELAYED, SOFTSYNC, CUSTOM + } + + + + private MediaDriver mediaDriver; + private AeronNDArrayResponder responder; + private AeronNDArraySubscriber subscriber; + private NDArrayCallback callback; + //alias for the callback where relevant + private ParameterServerListener parameterServerListener; + private Aeron aeron; + private ScheduledExecutorService heartbeat; + + /** + * Allow passing in a + * media driver that already exists + * + * @param mediaDriver + */ + public ParameterServerSubscriber(MediaDriver mediaDriver) { + Preconditions.checkNotNull(mediaDriver); + this.mediaDriver = mediaDriver; + } + + + + /** + * Return the current {@link SubscriberState} + * of this subscriber + * + * @return the current state of this subscriber + */ + public SubscriberState asState() { + return SubscriberState.builder() + .parameterUpdaterStatus(parameterServerListener == null ? Collections.emptyMap() + : parameterServerListener.getUpdater().status()) + .isMaster(isMaster()) + .connectionInfo(isMaster() ? masterConnectionInfo().toString() + : slaveConnectionInfo().toString()) + .isAsync(parameterServerListener.getUpdater().isAsync()) + .isReady(parameterServerListener.getUpdater().isReady()) + .totalUpdates(getResponder().getNdArrayHolder().totalUpdates()).streamId(streamId) + .serverState(subscriberLaunched() ? ServerState.STARTED.name().toLowerCase() + : ServerState.STOPPED.name().toLowerCase()) + .build(); + } + + /** + * When this is a slave node + * it returns the connection url for this node + * and the associated master connection urls in the form of: + * host:port:streamId + * + * @return the slave connection info + */ + public SlaveConnectionInfo slaveConnectionInfo() { + if (isMaster()) + throw new IllegalStateException("Unable to determine slave connection info. This is a master node"); + return SlaveConnectionInfo.builder().connectionUrl(subscriber.connectionUrl()).masterUrl(publishMasterUrl) + .build(); + + } + + + /** + * When this is a master node, + * it returns the connection url for this node, + * it's slaves (if any exist) and the responder + * connection url in the form of: + * host:port:streamId + * + * @return the master connection info + */ + public MasterConnectionInfo masterConnectionInfo() { + if (!isMaster()) + throw new IllegalStateException("Unable to determine master connection info. This is a slave node"); + return MasterConnectionInfo.builder().connectionUrl(subscriber.connectionUrl()) + .responderUrl(responder.connectionUrl()).slaveUrls(new ArrayList<>()).build(); + } + + /** + * @param args + */ + public void run(String[] args) throws Exception { + JCommander jcmdr = new JCommander(this); + + try { + jcmdr.parse(args); + } catch (ParameterException e) { + log.error("",e); + //User provides invalid input -> print the usage info + jcmdr.usage(); + try { + Thread.sleep(500); + } catch (Exception e2) { + } + System.exit(1); + } + + + //ensure that the update opType is configured from the command line args + updateType = UpdateType.valueOf(updateTypeString.toUpperCase()); + + + + if (publishMasterUrl == null && !master) + throw new IllegalStateException("Please specify a master url or set master to true"); + + //allows passing in a media driver for things like unit tests + //also ensure we don't use a media driver when a directory is specified + //for a remote one + if (mediaDriver == null && mediaDriverDirectoryName == null) { + //length of array * sizeof(float) + int ipcLength = ArrayUtil.prod(Ints.toArray(shape)) * 4; + //must be a power of 2 + ipcLength *= 2; + //padding for NDArrayMessage + ipcLength += 64; + //Length in bytes for the SO_RCVBUF, 0 means use OS default. This needs to be larger than Receiver Window. + System.setProperty("aeron.socket.so_rcvbuf", String.valueOf(ipcLength)); + final MediaDriver.Context mediaDriverCtx = new MediaDriver.Context().threadingMode(ThreadingMode.DEDICATED) + .dirDeleteOnStart(deleteDirectoryOnStart).termBufferSparseFile(false) + .ipcTermBufferLength(ipcLength).publicationTermBufferLength(ipcLength) + .conductorIdleStrategy(new BusySpinIdleStrategy()) + .receiverIdleStrategy(new BusySpinIdleStrategy()) + .senderIdleStrategy(new BusySpinIdleStrategy()); + AeronUtil.setDaemonizedThreadFactories(mediaDriverCtx); + + mediaDriver = MediaDriver.launchEmbedded(mediaDriverCtx); + //set the variable since we are using a media driver directly + mediaDriverDirectoryName = mediaDriver.aeronDirectoryName(); + log.info("Using media driver directory " + mediaDriver.aeronDirectoryName()); + } + + if (aeron == null) + this.aeron = Aeron.connect(getContext()); + + + if (master) { + if (this.callback == null) { + ParameterServerUpdater updater = null; + //instantiate with shape instead of just length + switch (updateType) { + case HOGWILD: + break; + case SYNC: + updater = new SynchronousParameterUpdater(new InMemoryUpdateStorage(), + new InMemoryNDArrayHolder(Ints.toArray(shape)), updatesPerEpoch); + break; + case SOFTSYNC: + updater = new SoftSyncParameterUpdater(); + break; + case TIME_DELAYED: + break; + case CUSTOM: + String parameterServerUpdateType = System.getProperty(CUSTOM_UPDATE_TYPE); + Class updaterClass = ND4JClassLoading + .loadClassByName(parameterServerUpdateType); + updater = ReflectionUtils.newInstance(updaterClass); + break; + default: + throw new IllegalStateException("Illegal opType of updater"); + } + + callback = new ParameterServerListener(Ints.toArray(shape), updater); + parameterServerListener = (ParameterServerListener) callback; + + } + //start an extra daemon for responding to get queries + ParameterServerListener cast = (ParameterServerListener) callback; + responder = AeronNDArrayResponder.startSubscriber(aeron, host, port + 1, cast.getUpdater().ndArrayHolder(), + streamId + 1); + log.info("Started responder on master node " + responder.connectionUrl()); + } else { + String[] publishMasterUrlArr = publishMasterUrl.split(":"); + if (publishMasterUrlArr == null || publishMasterUrlArr.length < 2) + throw new IllegalStateException("Please specify publish master url as host:port"); + + callback = new PublishingListener( + String.format("aeron:udp?endpoint=%s:%s", publishMasterUrlArr[0], publishMasterUrlArr[1]), + Integer.parseInt(publishMasterUrlArr[2]), getContext()); + } + + log.info("Starting subscriber on " + host + ":" + port + " and stream " + streamId); + AtomicBoolean running = new AtomicBoolean(true); + + //start a node + subscriber = AeronNDArraySubscriber.startSubscriber(aeron, host, port, callback, streamId, running); + + int tries=0; + while (!subscriber.launched() && tries<12) { + tries++; + LockSupport.parkNanos(100000); + } + if(!subscriber.launched()) { + throw new Exception("Subscriber did not start in time."); + } + + //send heartbeat to a status server. There will usually be 1 status server per master. + //Only schedule this if a remote server is available. + if (CheckSocket.remotePortTaken(statusServerHost, statusServerPort, 10000)) { + scheduledExecutorService = Executors.newScheduledThreadPool(1); + final AtomicInteger failCount = new AtomicInteger(0); + scheduledExecutorService.scheduleAtFixedRate(() -> { + try { + // + if (failCount.get() >= 3) + return; + SubscriberState subscriberState = asState(); + JSONObject jsonObject = new JSONObject(objectMapper.writeValueAsString(subscriberState)); + String url = String.format("http://%s:%d/updatestatus/%d", statusServerHost, statusServerPort, + streamId); + val entity = Unirest.post(url).header("Content-Type", "application/json") + .body(jsonObject).asString(); + } catch (Exception e) { + failCount.incrementAndGet(); + if (failCount.get() >= 3) { + log.warn("Failed to send update, shutting down likely?", e); + } + } + }, 0, heartbeatMs, TimeUnit.MILLISECONDS); + + } else { + log.info("No status server found. Will not send heartbeats. Specified host was " + statusServerHost + + " and port was " + statusServerPort); + } + + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + close(); + + })); + + //set the server for the status of the master and slave nodes + } + + + @Override + public void close() { + if (subscriber != null) + CloseHelper.quietClose(subscriber); + if (responder != null) + CloseHelper.quietClose(responder); + if (scheduledExecutorService != null) + scheduledExecutorService.shutdown(); + } + + + + //get a context + public Aeron.Context getContext() { + Aeron.Context ctx = new Aeron.Context().driverTimeoutMs(Long.MAX_VALUE) + .availableImageHandler(AeronUtil::printAvailableImage) + .unavailableImageHandler(AeronUtil::printUnavailableImage) + .aeronDirectoryName(mediaDriverDirectoryName).keepAliveIntervalNs(1000000) + .errorHandler(e -> log.error(e.toString(), e)); + AeronUtil.setDaemonizedThreadFactories(ctx); + return ctx; + } + + /** + * Get the master ndarray from the + * internal {@link NDArrayHolder} + * + * @return the master ndarray + */ + public INDArray getMasterArray() { + return parameterServerListener.getUpdater().ndArrayHolder().get(); + } + + + /** + * Returns true if the subscriber is launched + * + * @return + */ + public boolean subscriberLaunched() { + return subscriber.launched(); + } + + public static void main(String[] args) throws Exception { + new ParameterServerSubscriber().run(args); + } +} diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/PublishingListener.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/PublishingListener.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/PublishingListener.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/PublishingListener.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/BaseParameterUpdater.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/BaseParameterUpdater.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/BaseParameterUpdater.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/BaseParameterUpdater.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/ParameterServerUpdater.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/ParameterServerUpdater.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/ParameterServerUpdater.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/ParameterServerUpdater.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/SoftSyncParameterUpdater.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/SoftSyncParameterUpdater.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/SoftSyncParameterUpdater.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/SoftSyncParameterUpdater.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/SynchronousParameterUpdater.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/SynchronousParameterUpdater.java similarity index 97% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/SynchronousParameterUpdater.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/SynchronousParameterUpdater.java index 5a0963ead..9ebf5bcbd 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/SynchronousParameterUpdater.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/SynchronousParameterUpdater.java @@ -24,8 +24,8 @@ import org.nd4j.aeron.ipc.NDArrayHolder; import org.nd4j.aeron.ipc.NDArrayMessage; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.parameterserver.updater.storage.UpdateStorage; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.Map; diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/TimeDelayedParameterUpdater.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/TimeDelayedParameterUpdater.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/TimeDelayedParameterUpdater.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/TimeDelayedParameterUpdater.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/storage/BaseUpdateStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/storage/BaseUpdateStorage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/storage/BaseUpdateStorage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/storage/BaseUpdateStorage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/storage/InMemoryUpdateStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/storage/InMemoryUpdateStorage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/storage/InMemoryUpdateStorage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/storage/InMemoryUpdateStorage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/storage/NoUpdateStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/storage/NoUpdateStorage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/storage/NoUpdateStorage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/storage/NoUpdateStorage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/storage/UpdateStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/storage/UpdateStorage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/updater/storage/UpdateStorage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/updater/storage/UpdateStorage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/util/CheckSocket.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/util/CheckSocket.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/main/java/org/nd4j/parameterserver/util/CheckSocket.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/main/java/org/nd4j/parameterserver/util/CheckSocket.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/test/java/org/nd4j/parameterserver/updater/ParameterServerUpdaterTests.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/java/org/nd4j/parameterserver/updater/ParameterServerUpdaterTests.java similarity index 83% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/test/java/org/nd4j/parameterserver/updater/ParameterServerUpdaterTests.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/java/org/nd4j/parameterserver/updater/ParameterServerUpdaterTests.java index e68b23a2f..e8377ee74 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/test/java/org/nd4j/parameterserver/updater/ParameterServerUpdaterTests.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/java/org/nd4j/parameterserver/updater/ParameterServerUpdaterTests.java @@ -20,26 +20,22 @@ package org.nd4j.parameterserver.updater; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; import org.nd4j.aeron.ipc.NDArrayMessage; import org.nd4j.aeron.ndarrayholder.InMemoryNDArrayHolder; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.updater.storage.NoUpdateStorage; -import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +@Timeout(30) public class ParameterServerUpdaterTests extends BaseND4JTest { - @Test() - @Timeout(30000L) + @Test public void synchronousTest() { int cores = Runtime.getRuntime().availableProcessors(); ParameterServerUpdater updater = new SynchronousParameterUpdater(new NoUpdateStorage(), @@ -51,7 +47,7 @@ public class ParameterServerUpdaterTests extends BaseND4JTest { assertTrue(updater.shouldReplicate()); updater.reset(); assertFalse(updater.shouldReplicate()); - assertNotNull(updater.toJson()); + assumeTrue(updater.toJson()!=null); } diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/java/org/nd4j/parameterserver/updater/storage/UpdaterStorageTests.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/java/org/nd4j/parameterserver/updater/storage/UpdaterStorageTests.java new file mode 100644 index 000000000..ff220dd90 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/java/org/nd4j/parameterserver/updater/storage/UpdaterStorageTests.java @@ -0,0 +1,60 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.updater.storage; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.aeron.ipc.NDArrayMessage; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class UpdaterStorageTests extends BaseND4JTest { + + + @Test + public void testNone() { + assertThrows(UnsupportedOperationException.class, () -> { + UpdateStorage updateStorage = new NoUpdateStorage(); + NDArrayMessage message = NDArrayMessage.wholeArrayUpdate(Nd4j.scalar(1.0)); + updateStorage.addUpdate(message); + assertEquals(1, updateStorage.numUpdates()); + assertEquals(message, updateStorage.getUpdate(0)); + updateStorage.close(); + }); + } + + @Test + @Timeout(30) + public void testInMemory() { + UpdateStorage updateStorage = new InMemoryUpdateStorage(); + NDArrayMessage message = NDArrayMessage.wholeArrayUpdate(Nd4j.scalar(1.0)); + updateStorage.addUpdate(message); + assertEquals(1, updateStorage.numUpdates()); + assertEquals(message, updateStorage.getUpdate(0)); + updateStorage.clear(); + assertEquals(0, updateStorage.numUpdates()); + updateStorage.close(); + } + +} diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/test/resources/log4j.properties b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/resources/log4j.properties similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/test/resources/log4j.properties rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/resources/log4j.properties diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/test/resources/logback.xml b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/resources/logback.xml similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server/src/test/resources/logback.xml rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-core/src/test/resources/logback.xml diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/build.gradle b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/build.gradle new file mode 100644 index 000000000..e69de29bb diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/MasterConnectionInfo.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/MasterConnectionInfo.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/MasterConnectionInfo.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/MasterConnectionInfo.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/MasterStatus.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/MasterStatus.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/MasterStatus.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/MasterStatus.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerState.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerState.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerState.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerState.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerType.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerType.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerType.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerType.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerTypeJson.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerTypeJson.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerTypeJson.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/ServerTypeJson.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SlaveConnectionInfo.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SlaveConnectionInfo.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SlaveConnectionInfo.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SlaveConnectionInfo.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SlaveStatus.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SlaveStatus.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SlaveStatus.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SlaveStatus.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SubscriberState.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SubscriberState.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SubscriberState.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-model/src/main/java/org/nd4j/parameterserver/model/SubscriberState.java diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/build.gradle b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/build.gradle new file mode 100644 index 000000000..4bd4196f7 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/build.gradle @@ -0,0 +1,43 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + + implementation "commons-io:commons-io" + implementation "org.apache.commons:commons-lang3" + implementation "org.apache.commons:commons-math3" + implementation "commons-net:commons-net" + implementation "com.typesafe.play:play-server_2.13:2.7.3" + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerStatus + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore + implementation projects.cavisNd4j.cavisNd4jAeron + implementation 'io.reactivex.rxjava2:rxjava:2.2.21' + implementation "org.slf4j:slf4j-api" + implementation "org.agrona:agrona" + implementation "io.aeron:aeron-all:1.32.0" + implementation "com.google.guava:guava" + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testImplementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerClient +} \ No newline at end of file diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/VoidParameterServer.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/VoidParameterServer.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/VoidParameterServer.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/VoidParameterServer.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/conf/VoidConfiguration.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/conf/VoidConfiguration.java similarity index 99% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/conf/VoidConfiguration.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/conf/VoidConfiguration.java index ac190f08a..d18a9f897 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/conf/VoidConfiguration.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/conf/VoidConfiguration.java @@ -317,7 +317,7 @@ public class VoidConfiguration implements Serializable { throw new UnsupportedOperationException("Not supported. Use portSupplier method instead"); } - private VoidConfigurationBuilder faultToleranceStrategy(FaultToleranceStrategy faultToleranceStrategy) { + private VoidConfigurationBuilder faultToleranceStrategy(FaultToleranceStrategy faultToleranceStrategy){ throw new UnsupportedOperationException("Reserved for future use"); } diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/ExecutionMode.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/ExecutionMode.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/ExecutionMode.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/ExecutionMode.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/FaultToleranceStrategy.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/FaultToleranceStrategy.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/FaultToleranceStrategy.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/FaultToleranceStrategy.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/NodeRole.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/NodeRole.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/NodeRole.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/NodeRole.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/NodeStatus.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/NodeStatus.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/NodeStatus.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/NodeStatus.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/TransportType.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/TransportType.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/TransportType.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/enums/TransportType.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/ClientRouter.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/ClientRouter.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/ClientRouter.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/ClientRouter.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/RetransmissionHandler.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/RetransmissionHandler.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/RetransmissionHandler.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/RetransmissionHandler.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/SequenceProvider.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/SequenceProvider.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/SequenceProvider.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/SequenceProvider.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/Storage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/Storage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/Storage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/Storage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/Clipboard.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/Clipboard.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/Clipboard.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/Clipboard.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/FrameCompletionHandler.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/FrameCompletionHandler.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/FrameCompletionHandler.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/FrameCompletionHandler.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/RequestDescriptor.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/RequestDescriptor.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/RequestDescriptor.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/completion/RequestDescriptor.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/retransmission/DefaultRetransmissionHandler.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/retransmission/DefaultRetransmissionHandler.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/retransmission/DefaultRetransmissionHandler.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/retransmission/DefaultRetransmissionHandler.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/BaseRouter.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/BaseRouter.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/BaseRouter.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/BaseRouter.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouter.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouter.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouter.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouter.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/RandomRouter.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/RandomRouter.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/RandomRouter.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/RandomRouter.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/StaticRouter.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/StaticRouter.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/StaticRouter.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/routing/StaticRouter.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/sequence/BasicSequenceProvider.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/sequence/BasicSequenceProvider.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/sequence/BasicSequenceProvider.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/sequence/BasicSequenceProvider.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/storage/BaseStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/storage/BaseStorage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/storage/BaseStorage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/storage/BaseStorage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/storage/WordVectorStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/storage/WordVectorStorage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/storage/WordVectorStorage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/logic/storage/WordVectorStorage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/BaseVoidMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/BaseVoidMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/BaseVoidMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/BaseVoidMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/Chain.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/Chain.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/Chain.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/Chain.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/DistributedMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/DistributedMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/DistributedMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/DistributedMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/Frame.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/Frame.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/Frame.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/Frame.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/MeaningfulMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/MeaningfulMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/MeaningfulMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/MeaningfulMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/RequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/RequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/RequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/RequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/TrainingMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/TrainingMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/TrainingMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/TrainingMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/VoidAggregation.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/VoidAggregation.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/VoidAggregation.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/VoidAggregation.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/VoidMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/VoidMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/VoidMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/VoidMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/BaseAggregation.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/BaseAggregation.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/BaseAggregation.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/BaseAggregation.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/DotAggregation.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/DotAggregation.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/DotAggregation.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/DotAggregation.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/InitializationAggregation.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/InitializationAggregation.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/InitializationAggregation.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/InitializationAggregation.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/VectorAggregation.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/VectorAggregation.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/VectorAggregation.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/aggregations/VectorAggregation.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/BaseCompleteMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/BaseCompleteMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/BaseCompleteMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/BaseCompleteMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/FrameCompleteMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/FrameCompleteMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/FrameCompleteMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/FrameCompleteMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/InitializationCompleteMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/InitializationCompleteMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/InitializationCompleteMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/InitializationCompleteMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/IntroductionCompleteMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/IntroductionCompleteMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/IntroductionCompleteMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/IntroductionCompleteMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/VectorCompleteMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/VectorCompleteMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/VectorCompleteMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/complete/VectorCompleteMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedAssignMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedAssignMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedAssignMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedAssignMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedCbowDotMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedCbowDotMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedCbowDotMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedCbowDotMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedInitializationMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedInitializationMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedInitializationMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedInitializationMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedIntroductionMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedIntroductionMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedIntroductionMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedIntroductionMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSgDotMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSgDotMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSgDotMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSgDotMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedShutdownMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedShutdownMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedShutdownMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedShutdownMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSkipGramMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSkipGramMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSkipGramMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSkipGramMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSolidMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSolidMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSolidMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedSolidMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedVectorMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedVectorMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedVectorMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/intercom/DistributedVectorMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/AssignRequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/AssignRequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/AssignRequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/AssignRequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/CbowRequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/CbowRequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/CbowRequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/CbowRequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/InitializationRequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/InitializationRequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/InitializationRequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/InitializationRequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/IntroductionRequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/IntroductionRequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/IntroductionRequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/IntroductionRequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/ShutdownRequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/ShutdownRequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/ShutdownRequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/ShutdownRequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/SkipGramRequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/SkipGramRequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/SkipGramRequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/SkipGramRequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/VectorRequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/VectorRequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/VectorRequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/messages/requests/VectorRequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/BaseTrainer.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/BaseTrainer.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/BaseTrainer.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/BaseTrainer.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/TrainerProvider.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/TrainerProvider.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/TrainerProvider.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/TrainerProvider.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/TrainingDriver.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/TrainingDriver.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/TrainingDriver.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/TrainingDriver.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/chains/CbowChain.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/chains/CbowChain.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/chains/CbowChain.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/chains/CbowChain.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/chains/SkipGramChain.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/chains/SkipGramChain.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/chains/SkipGramChain.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/chains/SkipGramChain.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/impl/CbowTrainer.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/impl/CbowTrainer.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/impl/CbowTrainer.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/impl/CbowTrainer.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/impl/SkipGramTrainer.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/impl/SkipGramTrainer.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/impl/SkipGramTrainer.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/training/impl/SkipGramTrainer.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/BaseTransport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/BaseTransport.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/BaseTransport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/BaseTransport.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/LocalTransport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/LocalTransport.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/LocalTransport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/LocalTransport.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/MulticastTransport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/MulticastTransport.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/MulticastTransport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/MulticastTransport.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/RoutedTransport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/RoutedTransport.java similarity index 99% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/RoutedTransport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/RoutedTransport.java index 6ab7f1544..7ee969015 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/RoutedTransport.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/RoutedTransport.java @@ -20,7 +20,7 @@ package org.nd4j.parameterserver.distributed.transport; -import org.nd4j.shade.guava.math.IntMath; +import com.google.common.math.IntMath; import io.aeron.Aeron; import io.aeron.FragmentAssembler; import io.aeron.Publication; diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/Transport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/Transport.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/Transport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/transport/Transport.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/util/NetworkInformation.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/util/NetworkInformation.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/util/NetworkInformation.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/util/NetworkInformation.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizer.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizer.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizer.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizer.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServer.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServer.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServer.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServer.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/ChunksTracker.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/ChunksTracker.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/ChunksTracker.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/ChunksTracker.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/VoidChunk.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/VoidChunk.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/VoidChunk.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/VoidChunk.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTracker.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTracker.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTracker.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTracker.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTracker.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTracker.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTracker.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTracker.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/MeshBuildMode.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/MeshBuildMode.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/MeshBuildMode.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/MeshBuildMode.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/PropagationMode.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/PropagationMode.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/PropagationMode.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/PropagationMode.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/TransmissionStatus.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/TransmissionStatus.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/TransmissionStatus.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/enums/TransmissionStatus.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/BroadcastableMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/BroadcastableMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/BroadcastableMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/BroadcastableMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/INDArrayMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/INDArrayMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/INDArrayMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/INDArrayMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/MessagesHistoryHolder.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/MessagesHistoryHolder.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/MessagesHistoryHolder.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/MessagesHistoryHolder.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/RequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/RequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/RequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/RequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/ResponseMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/ResponseMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/ResponseMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/ResponseMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/VoidMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/VoidMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/VoidMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/VoidMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolder.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolder.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolder.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolder.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/GradientsUpdateMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/GradientsUpdateMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/GradientsUpdateMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/GradientsUpdateMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/MeshUpdateMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/MeshUpdateMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/MeshUpdateMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/MeshUpdateMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseINDArrayMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseINDArrayMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseINDArrayMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseINDArrayMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseRequestMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseRequestMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseRequestMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseRequestMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseResponseMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseResponseMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseResponseMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseResponseMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseVoidMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseVoidMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseVoidMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/impl/base/BaseVoidMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/handshake/HandshakeRequest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/handshake/HandshakeRequest.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/handshake/HandshakeRequest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/handshake/HandshakeRequest.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/handshake/HandshakeResponse.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/handshake/HandshakeResponse.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/handshake/HandshakeResponse.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/handshake/HandshakeResponse.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/ModelParametersMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/ModelParametersMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/ModelParametersMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/ModelParametersMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/ModelParametersRequest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/ModelParametersRequest.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/ModelParametersRequest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/ModelParametersRequest.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/UpdaterParametersMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/UpdaterParametersMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/UpdaterParametersMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/UpdaterParametersMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/UpdaterParametersRequest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/UpdaterParametersRequest.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/UpdaterParametersRequest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/params/UpdaterParametersRequest.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/ping/PingMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/ping/PingMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/ping/PingMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/ping/PingMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/ping/PongMessage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/ping/PongMessage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/ping/PongMessage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/messages/pairs/ping/PongMessage.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/MessageCallable.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/MessageCallable.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/MessageCallable.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/MessageCallable.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/PortSupplier.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/PortSupplier.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/PortSupplier.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/PortSupplier.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/RestartCallback.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/RestartCallback.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/RestartCallback.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/RestartCallback.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/Transport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/Transport.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/Transport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/Transport.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/UpdaterParametersProvider.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/UpdaterParametersProvider.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/UpdaterParametersProvider.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/UpdaterParametersProvider.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/UpdatesHandler.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/UpdatesHandler.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/UpdatesHandler.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/UpdatesHandler.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransport.java similarity index 98% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransport.java index d7fe68345..aa3410dbe 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransport.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransport.java @@ -20,7 +20,7 @@ package org.nd4j.parameterserver.distributed.v2.transport.impl; -import org.nd4j.shade.guava.math.IntMath; +import com.google.common.math.IntMath; import io.aeron.Aeron; import io.aeron.FragmentAssembler; import io.aeron.Publication; @@ -119,7 +119,8 @@ public class AeronUdpTransport extends BaseTransport implements AutoCloseable { Preconditions.checkArgument(ownPort > 0 && ownPort < 65536, "Own UDP port should be positive value in range of 1 and 65536"); Preconditions.checkArgument(rootPort > 0 && rootPort < 65536, "Master node UDP port should be positive value in range of 1 and 65536"); - //setProperty("aeron.client.liveness.timeout", "30000000000"); + // comment this out, put back to default + // setProperty("aeron.client.liveness.timeout", "30000000000"); // setting this property to try to increase maxmessage length, not sure if it still works though //Term buffer length: must be power of 2 and in range 64kB to 1GB: https://github.com/real-logic/aeron/wiki/Configuration-Options @@ -130,8 +131,11 @@ public class AeronUdpTransport extends BaseTransport implements AutoCloseable { splitter = MessageSplitter.getInstance(); + /* context = new Aeron.Context().driverTimeoutMs(30000) .keepAliveIntervalNs(100000000); + */ + context = new Aeron.Context(); AeronUtil.setDaemonizedThreadFactories(context); final MediaDriver.Context mediaDriverCtx = new MediaDriver.Context(); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/BaseTransport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/BaseTransport.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/BaseTransport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/BaseTransport.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DelayedDummyTransport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DelayedDummyTransport.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DelayedDummyTransport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DelayedDummyTransport.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransport.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransport.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransport.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransport.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/EnvironmentVarPortSupplier.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/EnvironmentVarPortSupplier.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/EnvironmentVarPortSupplier.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/EnvironmentVarPortSupplier.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/StaticPortSupplier.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/StaticPortSupplier.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/StaticPortSupplier.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/transport/impl/StaticPortSupplier.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/AbstractSubscriber.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/AbstractSubscriber.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/AbstractSubscriber.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/AbstractSubscriber.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/AbstractUpdatesHandler.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/AbstractUpdatesHandler.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/AbstractUpdatesHandler.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/AbstractUpdatesHandler.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizer.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizer.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizer.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizer.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitter.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitter.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitter.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitter.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/UpdaterParametersHolder.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/UpdaterParametersHolder.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/UpdaterParametersHolder.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/distributed/v2/util/UpdaterParametersHolder.java diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/node/ParameterServerNode.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/node/ParameterServerNode.java similarity index 91% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/node/ParameterServerNode.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/node/ParameterServerNode.java index 75b4861cc..7d32ac8b4 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/node/ParameterServerNode.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/java/org/nd4j/parameterserver/node/ParameterServerNode.java @@ -32,6 +32,7 @@ import org.nd4j.parameterserver.ParameterServerListener; import org.nd4j.parameterserver.ParameterServerSubscriber; import org.nd4j.parameterserver.status.play.InMemoryStatusStorage; import org.nd4j.parameterserver.status.play.StatusServer; +import play.server.Server; import java.util.ArrayList; import java.util.Arrays; @@ -41,6 +42,7 @@ import java.util.List; @NoArgsConstructor @Data public class ParameterServerNode implements AutoCloseable { + private Server server; private ParameterServerSubscriber[] subscriber; private MediaDriver mediaDriver; private Aeron aeron; @@ -89,6 +91,7 @@ public class ParameterServerNode implements AutoCloseable { * @param args the arguments for the {@link ParameterServerSubscriber} */ public void runMain(String[] args) { + server = StatusServer.startServer(new InMemoryStatusStorage(), statusPort); if (mediaDriver == null) mediaDriver = MediaDriver.launchEmbedded(); log.info("Started media driver with aeron directory " + mediaDriver.aeronDirectoryName()); @@ -118,7 +121,11 @@ public class ParameterServerNode implements AutoCloseable { if (i == 0) { - subscriber[i].run(multiArgs.toArray(new String[args.length])); + try { + subscriber[i].run(multiArgs.toArray(new String[args.length])); + } catch (Exception e) { + e.printStackTrace(); + } parameterServerListener = subscriber[i].getCallback(); cast = subscriber[i].getParameterServerListener(); } else { @@ -128,7 +135,11 @@ public class ParameterServerNode implements AutoCloseable { //now run the callback initialized with this callback instead //in the run method it will use this reference instead of creating it //itself - subscriber[i].run(multiArgs.toArray(new String[args.length])); + try { + subscriber[i].run(multiArgs.toArray(new String[args.length])); + } catch (Exception e) { + log.error(e.getMessage(), e); + } } @@ -166,7 +177,8 @@ public class ParameterServerNode implements AutoCloseable { } } } - + if (server != null) + server.stop(); if (mediaDriver != null) CloseHelper.quietClose(mediaDriver); if (aeron != null) diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/resources/META-INF/services/org.nd4j.parameterserver.distributed.training.TrainingDriver b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/resources/META-INF/services/org.nd4j.parameterserver.distributed.training.TrainingDriver similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/resources/META-INF/services/org.nd4j.parameterserver.distributed.training.TrainingDriver rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/resources/META-INF/services/org.nd4j.parameterserver.distributed.training.TrainingDriver diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/resources/aeron.properties b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/resources/aeron.properties similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/main/resources/aeron.properties rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/main/resources/aeron.properties diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerStressTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerStressTest.java similarity index 98% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerStressTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerStressTest.java index dac164997..8f95c5ff9 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerStressTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerStressTest.java @@ -22,10 +22,12 @@ package org.nd4j.parameterserver.distributed; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomUtils; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; import org.nd4j.parameterserver.distributed.enums.NodeRole; @@ -51,11 +53,8 @@ import java.util.concurrent.atomic.AtomicLong; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled +//@Ignore @Deprecated -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class VoidParameterServerStressTest extends BaseND4JTest { private static final int NUM_WORDS = 100000; @@ -73,7 +72,7 @@ public class VoidParameterServerStressTest extends BaseND4JTest { * This test measures performance of blocking messages processing, VectorRequestMessage in this case */ @Test - @Disabled + //@Ignore public void testPerformanceStandalone1() { VoidConfiguration voidConfiguration = VoidConfiguration.builder().networkMask("192.168.0.0/16").numberOfShards(1).build(); @@ -134,7 +133,7 @@ public class VoidParameterServerStressTest extends BaseND4JTest { * This test measures performance of non-blocking messages processing, SkipGramRequestMessage in this case */ @Test - @Disabled + //@Ignore public void testPerformanceStandalone2() { VoidConfiguration voidConfiguration = VoidConfiguration.builder().networkMask("192.168.0.0/16").numberOfShards(1).build(); @@ -195,7 +194,7 @@ public class VoidParameterServerStressTest extends BaseND4JTest { @Test - @Disabled + //@Ignore public void testPerformanceMulticast1() throws Exception { VoidConfiguration voidConfiguration = VoidConfiguration.builder().networkMask("192.168.0.0/16").numberOfShards(1).build(); @@ -290,8 +289,7 @@ public class VoidParameterServerStressTest extends BaseND4JTest { /** * This is one of the MOST IMPORTANT tests */ - @Test() - @Timeout(60000L) + @Test @Timeout(60) public void testPerformanceUnicast1() { List list = new ArrayList<>(); for (int t = 0; t < 1; t++) { @@ -389,7 +387,7 @@ public class VoidParameterServerStressTest extends BaseND4JTest { * Here we send non-blocking messages */ @Test - @Disabled + //@Ignore public void testPerformanceUnicast2() { List list = new ArrayList<>(); for (int t = 0; t < 5; t++) { @@ -491,8 +489,7 @@ public class VoidParameterServerStressTest extends BaseND4JTest { * * @throws Exception */ - @Test() - @Timeout(60000L) + @Test @Timeout(60) public void testPerformanceUnicast3() throws Exception { VoidConfiguration voidConfiguration = VoidConfiguration.builder().numberOfShards(1) .shardAddresses(Arrays.asList("127.0.0.1:49823")).build(); @@ -538,8 +535,7 @@ public class VoidParameterServerStressTest extends BaseND4JTest { * * @throws Exception */ - @Test() - @Timeout(60000L) + @Test @Timeout(60) public void testPerformanceUnicast4() throws Exception { VoidConfiguration voidConfiguration = VoidConfiguration.builder().numberOfShards(1) .shardAddresses(Arrays.asList("127.0.0.1:49823")).build(); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerTest.java similarity index 97% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerTest.java index 3f13b51a2..c29f7d6e3 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/VoidParameterServerTest.java @@ -21,10 +21,12 @@ package org.nd4j.parameterserver.distributed; import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; @@ -53,11 +55,8 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled +//@Ignore @Deprecated -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class VoidParameterServerTest extends BaseND4JTest { private static List localIPs; private static List badIPs; @@ -77,8 +76,7 @@ public class VoidParameterServerTest extends BaseND4JTest { } - @Test() - @Timeout(30000L) + @Test @Timeout(30) public void testNodeRole1() throws Exception { final VoidConfiguration conf = VoidConfiguration.builder().multicastPort(45678) .numberOfShards(10).multicastNetwork("224.0.1.1").shardAddresses(localIPs).ttl(4).build(); @@ -91,8 +89,7 @@ public class VoidParameterServerTest extends BaseND4JTest { node.shutdown(); } - @Test() - @Timeout(30000L) + @Test @Timeout(30) public void testNodeRole2() throws Exception { final VoidConfiguration conf = VoidConfiguration.builder().multicastPort(45678) .numberOfShards(10).shardAddresses(badIPs).backupAddresses(localIPs) @@ -106,8 +103,7 @@ public class VoidParameterServerTest extends BaseND4JTest { node.shutdown(); } - @Test() - @Timeout(30000L) + @Test @Timeout(30) public void testNodeRole3() throws Exception { final VoidConfiguration conf = VoidConfiguration.builder().multicastPort(45678) .numberOfShards(10).shardAddresses(badIPs).backupAddresses(badIPs).multicastNetwork("224.0.1.1") @@ -121,8 +117,7 @@ public class VoidParameterServerTest extends BaseND4JTest { node.shutdown(); } - @Test() - @Timeout(60000L) + @Test @Timeout(60) public void testNodeInitialization1() throws Exception { final AtomicInteger failCnt = new AtomicInteger(0); final AtomicInteger passCnt = new AtomicInteger(0); @@ -168,8 +163,7 @@ public class VoidParameterServerTest extends BaseND4JTest { * * @throws Exception */ - @Test() - @Timeout(60000L) + @Test @Timeout(60) public void testNodeInitialization2() throws Exception { final AtomicInteger failCnt = new AtomicInteger(0); final AtomicInteger passCnt = new AtomicInteger(0); @@ -258,8 +252,8 @@ public class VoidParameterServerTest extends BaseND4JTest { // now we check message queue within Shards for (int t = 0; t < threads.length; t++) { VoidMessage incMessage = shards[t].getTransport().takeMessage(); - assertNotEquals( null, incMessage,"Failed for shard " + t); - assertEquals(message.getMessageType(), incMessage.getMessageType(),"Failed for shard " + t); + assertNotEquals(null, incMessage, "Failed for shard " + t); + assertEquals( message.getMessageType(), incMessage.getMessageType(), "Failed for shard " + t); // we should put message back to corresponding shards[t].getTransport().putMessage(incMessage); @@ -276,7 +270,7 @@ public class VoidParameterServerTest extends BaseND4JTest { for (int t = 0; t < threads.length; t++) { VoidMessage incMessage = shards[t].getTransport().takeMessage(); - assertNotEquals(null, incMessage,"Failed for shard " + t); + assertNotEquals( null, incMessage, "Failed for shard " + t); shards[t].handleMessage(message); /** @@ -422,8 +416,7 @@ public class VoidParameterServerTest extends BaseND4JTest { * * @throws Exception */ - @Test - @Timeout(60000L) + @Test @Timeout(60) public void testNodeInitialization3() throws Exception { final AtomicInteger failCnt = new AtomicInteger(0); final AtomicInteger passCnt = new AtomicInteger(0); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/conf/VoidConfigurationTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/conf/VoidConfigurationTest.java similarity index 88% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/conf/VoidConfigurationTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/conf/VoidConfigurationTest.java index b8cab1742..da891a436 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/conf/VoidConfigurationTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/conf/VoidConfigurationTest.java @@ -20,26 +20,20 @@ package org.nd4j.parameterserver.distributed.conf; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; - +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.exception.ND4JIllegalStateException; import static org.junit.jupiter.api.Assertions.*; -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +//@Ignore +@Timeout(30) public class VoidConfigurationTest extends BaseND4JTest { - @Test public void testNetworkMask1() throws Exception { VoidConfiguration configuration = new VoidConfiguration(); @@ -73,26 +67,25 @@ public class VoidConfigurationTest extends BaseND4JTest { assertEquals("192.168.0.0/8", configuration.getNetworkMask()); } - @Test() + @Test public void testNetworkMask3() throws Exception { - assertThrows(ND4JIllegalStateException.class,() -> { + assertThrows(ND4JIllegalStateException.class, () -> { VoidConfiguration configuration = new VoidConfiguration(); configuration.setNetworkMask("192.256.1.1/24"); assertEquals("192.168.1.0/24", configuration.getNetworkMask()); }); - } - @Test() + @Test public void testNetworkMask4() throws Exception { - assertThrows(ND4JIllegalStateException.class,() -> { + assertThrows(ND4JIllegalStateException.class, () -> { VoidConfiguration configuration = new VoidConfiguration(); + configuration.setNetworkMask("0.0.0.0/8"); assertEquals("192.168.1.0/24", configuration.getNetworkMask()); }); - } @Override diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/ClipboardTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/ClipboardTest.java similarity index 95% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/ClipboardTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/ClipboardTest.java index c24a6e22c..48b024e38 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/ClipboardTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/ClipboardTest.java @@ -21,10 +21,12 @@ package org.nd4j.parameterserver.distributed.logic; import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.*; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.distributed.logic.completion.Clipboard; import org.nd4j.parameterserver.distributed.messages.aggregations.InitializationAggregation; @@ -36,11 +38,9 @@ import java.util.*; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled +//@Ignore @Deprecated -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(30) public class ClipboardTest extends BaseND4JTest { @BeforeEach public void setUp() throws Exception { @@ -53,7 +53,6 @@ public class ClipboardTest extends BaseND4JTest { } - @Test public void testPin1() throws Exception { Clipboard clipboard = new Clipboard(); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/FrameCompletionHandlerTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/FrameCompletionHandlerTest.java similarity index 90% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/FrameCompletionHandlerTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/FrameCompletionHandlerTest.java index 22698318b..16638a32f 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/FrameCompletionHandlerTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/FrameCompletionHandlerTest.java @@ -21,23 +21,18 @@ package org.nd4j.parameterserver.distributed.logic; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; - +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.parameterserver.distributed.logic.completion.FrameCompletionHandler; import static org.junit.jupiter.api.Assertions.*; -@Disabled +//@Ignore @Deprecated -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(30) public class FrameCompletionHandlerTest extends BaseND4JTest { @BeforeEach public void setUp() throws Exception { @@ -45,7 +40,6 @@ public class FrameCompletionHandlerTest extends BaseND4JTest { } - /** * This test emulates 2 frames being processed at the same time * @throws Exception diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouterTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouterTest.java similarity index 94% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouterTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouterTest.java index 66e8ab085..2890ed7d2 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouterTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/logic/routing/InterleavedRouterTest.java @@ -21,14 +21,11 @@ package org.nd4j.parameterserver.distributed.logic.routing; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; - +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.util.HashUtil; import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; import org.nd4j.parameterserver.distributed.messages.VoidMessage; @@ -41,11 +38,9 @@ import java.util.Arrays; import static org.junit.jupiter.api.Assertions.*; -@Disabled +//@Ignore @Deprecated -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(30) public class InterleavedRouterTest extends BaseND4JTest { VoidConfiguration configuration; Transport transport; @@ -62,8 +57,6 @@ public class InterleavedRouterTest extends BaseND4JTest { originator = HashUtil.getLongHash(transport.getIp() + ":" + transport.getPort()); } - - /** * Testing default assignment for everything, but training requests * diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/FrameTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/FrameTest.java similarity index 95% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/FrameTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/FrameTest.java index 14a26b3c8..c26ccb38e 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/FrameTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/FrameTest.java @@ -21,10 +21,11 @@ package org.nd4j.parameterserver.distributed.messages; import org.agrona.concurrent.UnsafeBuffer; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; import org.nd4j.parameterserver.distributed.enums.NodeRole; import org.nd4j.parameterserver.distributed.logic.completion.Clipboard; @@ -37,11 +38,8 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; -@Disabled +//@Ignore @Deprecated -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class FrameTest extends BaseND4JTest { @BeforeEach public void setUp() throws Exception { @@ -51,8 +49,7 @@ public class FrameTest extends BaseND4JTest { /** * Simple test for Frame functionality */ - @Test() - @Timeout(30000L) + @Test @Timeout(30) public void testFrame1() { final AtomicInteger count = new AtomicInteger(0); @@ -167,8 +164,7 @@ public class FrameTest extends BaseND4JTest { } - @Test() - @Timeout(30000L) + @Test @Timeout(30) public void testJoin1() throws Exception { SkipGramRequestMessage sgrm = new SkipGramRequestMessage(0, 1, new int[] {3, 4, 5}, new byte[] {0, 1, 0}, (short) 0, 0.01, 119L); diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/VoidMessageTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/VoidMessageTest.java new file mode 100644 index 000000000..23b94d76f --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/VoidMessageTest.java @@ -0,0 +1,63 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.distributed.messages; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.parameterserver.distributed.messages.requests.SkipGramRequestMessage; + +import static org.junit.jupiter.api.Assertions.*; + +//@Ignore +@Deprecated +@Timeout(30) +public class VoidMessageTest extends BaseND4JTest { + @BeforeEach + public void setUp() throws Exception { + + } + + @AfterEach + public void tearDown() throws Exception { + + } + + @Test + public void testSerDe1() throws Exception { + SkipGramRequestMessage message = new SkipGramRequestMessage(10, 12, new int[] {10, 20, 30, 40}, + new byte[] {(byte) 0, (byte) 0, (byte) 1, (byte) 0}, (short) 0, 0.0, 117L); + + byte[] bytes = message.asBytes(); + + SkipGramRequestMessage restored = (SkipGramRequestMessage) VoidMessage.fromBytes(bytes); + + assertNotEquals(null, restored); + + assertEquals(message, restored); + assertArrayEquals(message.getPoints(), restored.getPoints()); + assertArrayEquals(message.getCodes(), restored.getCodes()); + } + +} diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/aggregations/VoidAggregationTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/aggregations/VoidAggregationTest.java similarity index 96% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/aggregations/VoidAggregationTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/aggregations/VoidAggregationTest.java index 139d1a39d..4456c6d04 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/aggregations/VoidAggregationTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/messages/aggregations/VoidAggregationTest.java @@ -21,10 +21,12 @@ package org.nd4j.parameterserver.distributed.messages.aggregations; import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.*; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -34,11 +36,9 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled +//@Ignore @Deprecated -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Timeout(30) public class VoidAggregationTest extends BaseND4JTest { private static final short NODES = 100; private static final int ELEMENTS_PER_NODE = 3; @@ -53,8 +53,6 @@ public class VoidAggregationTest extends BaseND4JTest { } - - /** * In this test we check for aggregation of sample vector. * diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/transport/RoutedTransportTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/transport/RoutedTransportTest.java similarity index 95% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/transport/RoutedTransportTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/transport/RoutedTransportTest.java index e36c32312..f977912ec 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/transport/RoutedTransportTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/transport/RoutedTransportTest.java @@ -20,10 +20,12 @@ package org.nd4j.parameterserver.distributed.transport; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; import org.nd4j.parameterserver.distributed.enums.NodeRole; import org.nd4j.parameterserver.distributed.logic.ClientRouter; @@ -38,11 +40,8 @@ import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.*; -@Disabled +//@Ignore @Deprecated -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class RoutedTransportTest extends BaseND4JTest { @BeforeEach public void setUp() throws Exception { @@ -60,8 +59,7 @@ public class RoutedTransportTest extends BaseND4JTest { * * @throws Exception */ - @Test() - @Timeout(30000) + @Test @Timeout(30) public void testMessaging1() throws Exception { List list = new ArrayList<>(); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizerTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizerTest.java similarity index 98% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizerTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizerTest.java index 9f27b49c0..24b8915e9 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizerTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/util/NetworkOrganizerTest.java @@ -22,20 +22,20 @@ package org.nd4j.parameterserver.distributed.util; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomUtils; -import org.junit.jupiter.api.*; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.*; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +//@Ignore +@Timeout(20) public class NetworkOrganizerTest extends BaseND4JTest { @BeforeEach public void setUp() throws Exception { @@ -387,7 +387,7 @@ public class NetworkOrganizerTest extends BaseND4JTest { } @Test - @Disabled("AB 2019/05/30 - Intermittent issue or flaky test - see issue #7657") + //@Ignore("AB 2019/05/30 - Intermittent issue or flaky test - see issue #7657") public void testNetTree6() throws Exception { List ips = new ArrayList<>(); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/DelayedModelParameterServerTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/DelayedModelParameterServerTest.java similarity index 95% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/DelayedModelParameterServerTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/DelayedModelParameterServerTest.java index 4790ea252..cf6258aba 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/DelayedModelParameterServerTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/DelayedModelParameterServerTest.java @@ -24,10 +24,12 @@ import io.reactivex.functions.Consumer; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.lang3.RandomUtils; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.AtomicBoolean; @@ -53,10 +55,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +//@Ignore public class DelayedModelParameterServerTest extends BaseND4JTest { private static final String rootId = "ROOT_NODE"; @@ -70,8 +69,7 @@ public class DelayedModelParameterServerTest extends BaseND4JTest { MessageSplitter.getInstance().reset(); } - @Test() - @Timeout(20000L) + @Test @Timeout(20) public void testBasicInitialization_1() throws Exception { val connector = new DummyTransport.Connector(); val rootTransport = new DelayedDummyTransport(rootId, connector); @@ -86,8 +84,7 @@ public class DelayedModelParameterServerTest extends BaseND4JTest { rootServer.shutdown(); } - @Test() - @Timeout(40000L) + @Test @Timeout(40) public void testBasicInitialization_2() throws Exception { for (int e = 0; e < 100; e++) { val connector = new DummyTransport.Connector(); @@ -111,17 +108,17 @@ public class DelayedModelParameterServerTest extends BaseND4JTest { val meshA = clientTransportA.getMesh(); val meshB = clientTransportB.getMesh(); - assertEquals(3, meshR.totalNodes(),"Root node failed"); - assertEquals(3, meshB.totalNodes(),"B node failed"); - assertEquals(3, meshA.totalNodes(),"A node failed"); + assertEquals(3, meshR.totalNodes(), "Root node failed"); + assertEquals( 3, meshB.totalNodes(), "B node failed"); + assertEquals( 3, meshA.totalNodes(), "A node failed"); assertEquals(meshR, meshA); assertEquals(meshA, meshB); log.info("Iteration [{}] finished", e); } } - @Test() - @Timeout(180000L) + + @Test @Timeout(180) public void testUpdatesPropagation_1() throws Exception { val conf = VoidConfiguration.builder().meshBuildMode(MeshBuildMode.PLAIN).build(); val array = Nd4j.ones(10, 10); @@ -175,12 +172,11 @@ public class DelayedModelParameterServerTest extends BaseND4JTest { for (int e = 0; e < servers.size(); e++) { val s = servers.get(e); - assertEquals(1, s.getUpdates().size(),"Failed at node [" + e + "]"); + assertEquals( 1, s.getUpdates().size(), "Failed at node [" + e + "]"); } } - @Test() - @Timeout(180000L) + @Test @Timeout(180) public void testModelAndUpdaterParamsUpdate_1() throws Exception { val config = VoidConfiguration.builder().meshBuildMode(MeshBuildMode.PLAIN).build(); val connector = new DummyTransport.Connector(); @@ -305,7 +301,7 @@ public class DelayedModelParameterServerTest extends BaseND4JTest { // we're skipping node 23 since it was reconnected, and has different MPS instance // and node 96, since it sends update if (e != 23 && e != 96) - assertEquals(1, counters[e].get(),"Failed at node: [" + e + "]"); + assertEquals( 1, counters[e].get(), "Failed at node: [" + e + "]"); } assertTrue(updatedModel.get()); @@ -313,8 +309,7 @@ public class DelayedModelParameterServerTest extends BaseND4JTest { assertTrue(gotGradients.get()); } - @Test() - @Timeout(180000L) + @Test @Timeout(180) public void testMeshConsistency_1() throws Exception { Nd4j.create(1); final int numMessages = 500; @@ -389,13 +384,12 @@ public class DelayedModelParameterServerTest extends BaseND4JTest { // now we're checking all nodes, they should get numMessages - messages that were sent through them for (int e = 0; e < servers.size(); e++) { val server = servers.get(e); - assertEquals(numMessages - deductions[e], counters[e].get(),"Failed at node: [" + e + "]"); + assertEquals( numMessages - deductions[e], counters[e].get(), "Failed at node: [" + e + "]"); } } - @Test() - @Timeout(180000L) + @Test @Timeout(180) public void testMeshConsistency_2() throws Exception { Nd4j.create(1); final int numMessages = 100; @@ -475,7 +469,7 @@ public class DelayedModelParameterServerTest extends BaseND4JTest { // now we're checking all nodes, they should get numMessages - messages that were sent through them for (int e = 0; e < servers.size(); e++) { val server = servers.get(e); - assertEquals( numMessages - deductions[e], counters[e].get(),"Failed at node: [" + e + "]"); + assertEquals( numMessages - deductions[e], counters[e].get(), "Failed at node: [" + e + "]"); } } } diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServerTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServerTest.java similarity index 98% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServerTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServerTest.java index d334c2a06..2c168c251 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServerTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/ModelParameterServerTest.java @@ -23,13 +23,9 @@ package org.nd4j.parameterserver.distributed.v2; import io.reactivex.functions.Consumer; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.AtomicBoolean; @@ -52,15 +48,10 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class ModelParameterServerTest extends BaseND4JTest { private static final String rootId = "ROOT_NODE"; - @Test() - @Timeout(20000L) + @Test @Timeout(20) public void testBasicInitialization_1() throws Exception { val connector = new DummyTransport.Connector(); val rootTransport = new DummyTransport(rootId, connector); @@ -75,8 +66,7 @@ public class ModelParameterServerTest extends BaseND4JTest { rootServer.shutdown(); } - @Test() - @Timeout(20000L) + @Test @Timeout(20) public void testBasicInitialization_2() throws Exception { val connector = new DummyTransport.Connector(); val rootTransport = new DummyTransport(rootId, connector); @@ -131,7 +121,7 @@ public class ModelParameterServerTest extends BaseND4JTest { assertEquals(0, updatesA.size()); } - @Test// (timeout = 30000L) + @Test// @Timeout(30) public void testReconnectPropagation_1() throws Exception { val config = VoidConfiguration.builder().meshBuildMode(MeshBuildMode.MESH).build(); val connector = new DummyTransport.Connector(); @@ -447,7 +437,7 @@ public class ModelParameterServerTest extends BaseND4JTest { failedCnt++; } - assertEquals(0, failedCnt,"Some nodes got no updates:"); + assertEquals( 0, failedCnt, "Some nodes got no updates:"); assertTrue(updatedModel.get()); assertTrue(gotGradients.get()); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTrackerTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTrackerTest.java similarity index 90% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTrackerTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTrackerTest.java index 17a42fc84..d753ecf21 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTrackerTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/FileChunksTrackerTest.java @@ -22,12 +22,9 @@ package org.nd4j.parameterserver.distributed.v2.chunks.impl; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.distributed.v2.chunks.VoidChunk; import org.nd4j.parameterserver.distributed.v2.messages.impl.GradientsUpdateMessage; @@ -38,10 +35,7 @@ import java.util.ArrayList; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +//@Ignore public class FileChunksTrackerTest extends BaseND4JTest { @Test public void testTracker_1() throws Exception { @@ -49,7 +43,7 @@ public class FileChunksTrackerTest extends BaseND4JTest { val splitter = MessageSplitter.getInstance(); val message = new GradientsUpdateMessage("123", array); - val messages = new ArrayList<>(splitter.split(message, 16384)); + val messages = new ArrayList(splitter.split(message, 16384)); val tracker = new FileChunksTracker(messages.get(0)); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTrackerTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTrackerTest.java similarity index 90% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTrackerTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTrackerTest.java index db98c1bf3..2c13b629b 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTrackerTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/chunks/impl/InmemoryChunksTrackerTest.java @@ -21,12 +21,9 @@ package org.nd4j.parameterserver.distributed.v2.chunks.impl; import lombok.val; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.distributed.v2.chunks.VoidChunk; import org.nd4j.parameterserver.distributed.v2.messages.impl.GradientsUpdateMessage; @@ -35,12 +32,10 @@ import org.nd4j.parameterserver.distributed.v2.util.MessageSplitter; import java.util.ArrayList; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class InmemoryChunksTrackerTest extends BaseND4JTest { @Test - @Disabled + //@Ignore public void testTracker_1() throws Exception { val array = Nd4j.linspace(1, 100000, 10000).reshape(-1, 1000); val splitter = MessageSplitter.getInstance(); diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/VoidMessageTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/VoidMessageTest.java new file mode 100644 index 000000000..fc7b5d496 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/VoidMessageTest.java @@ -0,0 +1,57 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.distributed.v2.messages; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.common.util.SerializationUtils; +import org.nd4j.parameterserver.distributed.v2.messages.pairs.handshake.HandshakeRequest; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +public class VoidMessageTest extends BaseND4JTest { + @Test + public void testHandshakeSerialization_1() throws Exception { + val req = new HandshakeRequest(); + req.setOriginatorId("1234"); + + val bytes = SerializationUtils.toByteArray(req); + + VoidMessage res = SerializationUtils.deserialize(bytes); + + assertEquals(req.getOriginatorId(), res.getOriginatorId()); + } + + @Test + public void testHandshakeSerialization_2() throws Exception { + val req = new HandshakeRequest(); + req.setOriginatorId("1234"); + + val bytes = SerializationUtils.toByteArray(req); + + VoidMessage res = VoidMessage.fromBytes(bytes); + + assertEquals(req.getOriginatorId(), res.getOriginatorId()); + } +} \ No newline at end of file diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolderTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolderTest.java similarity index 92% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolderTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolderTest.java index 9c8b98ddc..b9b93e299 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolderTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/messages/history/HashHistoryHolderTest.java @@ -22,18 +22,12 @@ package org.nd4j.parameterserver.distributed.v2.messages.history; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class HashHistoryHolderTest extends BaseND4JTest { @Test diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransportTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransportTest.java similarity index 90% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransportTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransportTest.java index 24ee33a84..5a599b238 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransportTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/AeronUdpTransportTest.java @@ -22,20 +22,14 @@ package org.nd4j.parameterserver.distributed.v2.transport.impl; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class AeronUdpTransportTest extends BaseND4JTest { private static final String IP = "127.0.0.1"; private static final int ROOT_PORT = 40781; @@ -46,7 +40,7 @@ public class AeronUdpTransportTest extends BaseND4JTest { } @Test - @Disabled + //@Ignore public void testBasic_Connection_1() throws Exception { // we definitely want to shutdown all transports after test, to avoid issues with shmem try(val transportA = new AeronUdpTransport(IP, ROOT_PORT, IP, ROOT_PORT, VoidConfiguration.builder().build()); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransportTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransportTest.java similarity index 97% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransportTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransportTest.java index 31550981d..4fed8645c 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransportTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/transport/impl/DummyTransportTest.java @@ -22,11 +22,8 @@ package org.nd4j.parameterserver.distributed.v2.transport.impl; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.parameterserver.distributed.v2.enums.PropagationMode; import org.nd4j.parameterserver.distributed.v2.messages.impl.GradientsUpdateMessage; @@ -40,9 +37,6 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class DummyTransportTest extends BaseND4JTest { @Test diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizerTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizerTest.java similarity index 98% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizerTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizerTest.java index e7feac050..92a6a668c 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizerTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MeshOrganizerTest.java @@ -22,12 +22,9 @@ package org.nd4j.parameterserver.distributed.v2.util; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.common.util.SerializationUtils; import org.nd4j.parameterserver.distributed.v2.enums.MeshBuildMode; @@ -38,13 +35,10 @@ import java.util.ArrayList; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class MeshOrganizerTest extends BaseND4JTest { - @Test() - @Timeout(1000L) + @Test + @Timeout(10) public void testDescendantsCount_1() { val node = MeshOrganizer.Node.builder().build(); diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitterTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitterTest.java similarity index 95% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitterTest.java rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitterTest.java index c5567be4f..17e9dd7aa 100644 --- a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitterTest.java +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/distributed/v2/util/MessageSplitterTest.java @@ -22,11 +22,8 @@ package org.nd4j.parameterserver.distributed.v2.util; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Atomic; import org.nd4j.common.primitives.Optional; @@ -37,9 +34,6 @@ import java.util.ArrayList; import static org.junit.jupiter.api.Assertions.*; @Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class MessageSplitterTest extends BaseND4JTest { @Test diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/node/ParameterServerNodeTest.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/node/ParameterServerNodeTest.java new file mode 100644 index 000000000..7fee8e9c0 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/java/org/nd4j/parameterserver/node/ParameterServerNodeTest.java @@ -0,0 +1,129 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.node; + +import io.aeron.Aeron; +import io.aeron.driver.MediaDriver; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeAll; + +import org.junit.jupiter.api.Test; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.aeron.ipc.AeronUtil; +import org.nd4j.aeron.ipc.NDArrayMessage; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.parameterserver.client.ParameterServerClient; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +//@Ignore +@Deprecated +public class ParameterServerNodeTest extends BaseND4JTest { + private static MediaDriver mediaDriver; + private static Aeron aeron; + private static ParameterServerNode parameterServerNode; + private static int parameterLength = 4; + private static int masterStatusPort = 40323 + new java.util.Random().nextInt(15999); + private static int statusPort = masterStatusPort - 1299; + + @BeforeAll + public static void before() throws Exception { + mediaDriver = MediaDriver.launchEmbedded(AeronUtil.getMediaDriverContext(parameterLength)); + System.setProperty("play.server.dir", "/tmp"); + aeron = Aeron.connect(getContext()); + parameterServerNode = new ParameterServerNode(mediaDriver, statusPort); + parameterServerNode.runMain(new String[] {"-m", "true", "-s", "1," + String.valueOf(parameterLength), "-p", + String.valueOf(masterStatusPort), "-h", "localhost", "-id", "11", "-md", + mediaDriver.aeronDirectoryName(), "-sp", String.valueOf(statusPort), "-sh", "localhost", "-u", + String.valueOf(Runtime.getRuntime().availableProcessors())}); + + while (!parameterServerNode.subscriberLaunched()) { + Thread.sleep(10000); + } + + } + + @Test + public void testSimulateRun() throws Exception { + int numCores = Runtime.getRuntime().availableProcessors(); + ExecutorService executorService = Executors.newFixedThreadPool(numCores); + ParameterServerClient[] clients = new ParameterServerClient[numCores]; + String host = "localhost"; + for (int i = 0; i < numCores; i++) { + clients[i] = ParameterServerClient.builder().aeron(aeron).masterStatusHost(host) + .masterStatusPort(statusPort).subscriberHost(host).subscriberPort(40325 + i) + .subscriberStream(10 + i) + .ndarrayRetrieveUrl(parameterServerNode.getSubscriber()[i].getResponder().connectionUrl()) + .ndarraySendUrl(parameterServerNode.getSubscriber()[i].getSubscriber().connectionUrl()) + .build(); + } + + Thread.sleep(60000); + + //no arrays have been sent yet + for (int i = 0; i < numCores; i++) { + assertFalse(clients[i].isReadyForNext()); + } + + //send "numCores" arrays, the default parameter server updater + //is synchronous so it should be "ready" when number of updates == number of workers + for (int i = 0; i < numCores; i++) { + clients[i].pushNDArrayMessage(NDArrayMessage.wholeArrayUpdate(Nd4j.ones(parameterLength))); + } + + Thread.sleep(10000); + + //all arrays should have been sent + for (int i = 0; i < numCores; i++) { + assertTrue(clients[i].isReadyForNext()); + } + + Thread.sleep(10000); + + for (int i = 0; i < 1; i++) { + assertEquals(Nd4j.valueArrayOf(1, parameterLength, numCores), clients[i].getArray()); + Thread.sleep(1000); + } + + executorService.shutdown(); + + Thread.sleep(60000); + + parameterServerNode.close(); + + + } + + + private static Aeron.Context getContext() { + return new Aeron.Context().driverTimeoutMs(10000) + .availableImageHandler(AeronUtil::printAvailableImage) + .unavailableImageHandler(AeronUtil::printUnavailableImage) + .aeronDirectoryName(mediaDriver.aeronDirectoryName()).keepAliveIntervalNs(100000) + .errorHandler(e -> log.error(e.toString(), e)); + } + + +} diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/resources/aeron.properties b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/resources/aeron.properties similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/resources/aeron.properties rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/resources/aeron.properties diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/resources/log4j.properties b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/resources/log4j.properties similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/resources/log4j.properties rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/resources/log4j.properties diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/resources/logback.xml b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/resources/logback.xml similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-node/src/test/resources/logback.xml rename to cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-node/src/test/resources/logback.xml diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/build.gradle b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/build.gradle new file mode 100644 index 000000000..6b1ce4fab --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/build.gradle @@ -0,0 +1,42 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation "org.mapdb:mapdb:3.0.5" + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerCore + implementation projects.cavisNd4j.cavisNd4jParameterServer.cavisNd4jParameterServerModel + implementation "com.typesafe.play:play-netty-server_2.12:2.7.3" + implementation("com.typesafe.play:play-java_2.12:2.7.3") { + exclude group: 'ch.qos.logback', module: 'logback-core' + exclude group: 'ch.qos.logback', module: 'logback-classic' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + exclude group: 'org.slf4j', module: 'jul-to-slf4j' + exclude group: 'org.slf4j', module: 'jcl-over-slf4j' + exclude group: 'org.apache.tomcat', module: 'tomcat-servlet-api' + exclude group: 'net.jodah', module: 'typetools' + } + implementation "io.aeron:aeron-all:1.32.0" + + testImplementation projects.cavisNd4j.cavisNd4jCommonTests + testRuntimeOnly "net.brutex.ai:dl4j-test-resources:1.0.1-SNAPSHOT" + +} \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/BaseStatusStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/BaseStatusStorage.java new file mode 100644 index 000000000..6eef3cd86 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/BaseStatusStorage.java @@ -0,0 +1,152 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.status.play; + +import io.aeron.driver.MediaDriver; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.parameterserver.ParameterServerSubscriber; +import org.nd4j.parameterserver.model.SubscriberState; + +import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@Slf4j +public abstract class BaseStatusStorage implements StatusStorage { + protected Map statusStorageMap = createMap(); + private ScheduledExecutorService executorService; + protected Map updated; + private long heartBeatEjectionMilliSeconds = 1000; + private long checkInterval = 1000; + + public BaseStatusStorage() { + this(1000, 1000); + } + + /** + * The list of state ids + * for the given {@link SubscriberState} + * + * @return the list of ids for the given state + */ + @Override + public List ids() { + return new ArrayList<>(statusStorageMap.keySet()); + } + + /** + * Returns the number of states + * held by this storage + * + * @return + */ + @Override + public int numStates() { + return statusStorageMap.size(); + } + + /** + * + * @param heartBeatEjectionMilliSeconds the amount of time before + * ejecting a given subscriber as failed + * @param checkInterval the interval to check for + */ + public BaseStatusStorage(long heartBeatEjectionMilliSeconds, long checkInterval) { + this.heartBeatEjectionMilliSeconds = heartBeatEjectionMilliSeconds; + this.checkInterval = checkInterval; + init(); + } + + + private void init() { + updated = createUpdatedMap(); + executorService = Executors.newScheduledThreadPool(1); + //eject values that haven't checked in a while + executorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + long curr = System.currentTimeMillis(); + Set remove = new HashSet<>(); + for (Map.Entry entry : updated.entrySet()) { + long val = entry.getValue(); + long diff = Math.abs(curr - val); + if (diff > heartBeatEjectionMilliSeconds) { + remove.add(entry.getKey()); + } + } + + if (!remove.isEmpty()) + log.info("Removing " + remove.size() + " entries"); + //purge removed values + for (Integer i : remove) { + updated.remove(i); + statusStorageMap.remove(i); + } + + } + }, 30000, checkInterval, TimeUnit.MILLISECONDS); + } + + + /** + * Create the storage map + * @return + */ + public abstract Map createUpdatedMap(); + + /** + * Create the storage map + * @return + */ + public abstract Map createMap(); + + /** + * Get the state given an id. + * The integer represents a stream id + * for a given {@link ParameterServerSubscriber}. + *

+ * A {@link SubscriberState} is supposed to be 1 to 1 mapping + * for a stream and a {@link MediaDriver}. + * + * @param id the id of the state to get + * @return the subscriber state for the given id or none + * if it doesn't exist + */ + @Override + public SubscriberState getState(int id) { + if (!statusStorageMap.containsKey(id)) + return SubscriberState.empty(); + return statusStorageMap.get(id); + } + + /** + * Update the state for storage + * + * @param subscriberState the subscriber state to update + */ + @Override + public void updateState(SubscriberState subscriberState) { + updated.put(subscriberState.getStreamId(), System.currentTimeMillis()); + statusStorageMap.put(subscriberState.getStreamId(), subscriberState); + } + +} diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/InMemoryStatusStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/InMemoryStatusStorage.java new file mode 100644 index 000000000..87ec983e4 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/InMemoryStatusStorage.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.status.play; + + +import org.nd4j.parameterserver.model.SubscriberState; + +import java.util.HashMap; +import java.util.Map; + +public class InMemoryStatusStorage extends BaseStatusStorage { + + /** + * Create the storage map + * + * @return + */ + @Override + public Map createUpdatedMap() { + return new HashMap<>(); + } + + @Override + public Map createMap() { + return new HashMap<>(); + } +} diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/MapDbStatusStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/MapDbStatusStorage.java new file mode 100644 index 000000000..f8377f244 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/MapDbStatusStorage.java @@ -0,0 +1,130 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.status.play; + +import io.aeron.driver.MediaDriver; +import lombok.NonNull; +import org.mapdb.*; +import org.nd4j.parameterserver.ParameterServerSubscriber; +import org.nd4j.parameterserver.model.SubscriberState; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +/** + * MapDB status storage + * + * @author Adam Gibson + */ +public class MapDbStatusStorage extends BaseStatusStorage { + private DB db; + private File storageFile; + + /** + * @param heartBeatEjectionMilliSeconds the amount of time before + * ejecting a given subscriber as failed + * @param checkInterval the interval to check for + */ + public MapDbStatusStorage(long heartBeatEjectionMilliSeconds, long checkInterval) { + super(heartBeatEjectionMilliSeconds, checkInterval); + } + + public MapDbStatusStorage() { + this(1000, 1000); + } + + /** + * Create the storage map + * + * @return + */ + @Override + public Map createUpdatedMap() { + if (storageFile == null) { + //In-Memory Stats Storage + db = DBMaker.memoryDB().make(); + } else { + db = DBMaker.fileDB(storageFile).closeOnJvmShutdown().transactionEnable() //Default to Write Ahead Log - lower performance, but has crash protection + .make(); + } + + updated = db.hashMap("updated").keySerializer(Serializer.INTEGER).valueSerializer(Serializer.LONG) + .createOrOpen(); + return updated; + } + + + + @Override + public Map createMap() { + if (storageFile == null) { + //In-Memory Stats Storage + db = DBMaker.memoryDB().make(); + } else { + db = DBMaker.fileDB(storageFile).closeOnJvmShutdown().transactionEnable() //Default to Write Ahead Log - lower performance, but has crash protection + .make(); + } + + statusStorageMap = db.hashMap("statusStorageMap").keySerializer(Serializer.INTEGER) + .valueSerializer(new StatusStorageSerializer()).createOrOpen(); + return statusStorageMap; + } + + /** + * Get the state given an id. + * The integer represents a stream id + * for a given {@link ParameterServerSubscriber}. + *

+ * A {@link SubscriberState} is supposed to be 1 to 1 mapping + * for a stream and a {@link MediaDriver}. + * + * @param id the id of the state to get + * @return the subscriber state for the given id or none + * if it doesn't exist + */ + @Override + public SubscriberState getState(int id) { + if (!statusStorageMap.containsKey(id)) + return SubscriberState.empty(); + return statusStorageMap.get(id); + } + + + + private class StatusStorageSerializer implements Serializer { + + @Override + public void serialize(@NonNull DataOutput2 out, @NonNull SubscriberState value) throws IOException { + value.write(out); + } + + @Override + public SubscriberState deserialize(@NonNull DataInput2 input, int available) throws IOException { + return SubscriberState.read(input); + } + + @Override + public int compare(SubscriberState p1, SubscriberState p2) { + return p1.compareTo(p2); + } + } +} diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/StatusServer.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/StatusServer.java new file mode 100644 index 000000000..ef3806ffa --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/StatusServer.java @@ -0,0 +1,92 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.status.play; + + +import lombok.extern.slf4j.Slf4j; +import org.nd4j.parameterserver.model.MasterStatus; +import org.nd4j.parameterserver.model.ServerTypeJson; +import org.nd4j.parameterserver.model.SlaveStatus; +import org.nd4j.parameterserver.model.SubscriberState; +import play.BuiltInComponents; +import play.Mode; +import play.libs.Json; +import play.routing.Router; +import play.routing.RoutingDsl; +import play.server.Server; + +import static play.libs.Json.toJson; +import static play.mvc.Results.ok; + + +@Slf4j +public class StatusServer { + + /** + * Start a server based on the given subscriber. + * Note that for the port to start the server on, you should + * set the statusServerPortField on the subscriber + * either manually or via command line. The + * server defaults to port 9000. + * + * The end points are: + * /opType: returns the opType information (master/slave) + * /started: if it's a master node, it returns master:started/stopped and responder:started/stopped + * /connectioninfo: See the SlaveConnectionInfo and MasterConnectionInfo classes for fields. + * /ids: the list of ids for all of the subscribers + * @param statusStorage the subscriber to base + * the status server on + * @return the started server + */ + public static Server startServer(StatusStorage statusStorage, int statusServerPort) { + log.info("Starting server on port " + statusServerPort); + return Server.forRouter(Mode.PROD, statusServerPort, builtInComponents -> createRouter(statusStorage, builtInComponents)); + } + + protected static Router createRouter(StatusStorage statusStorage, BuiltInComponents builtInComponents){ + RoutingDsl dsl = RoutingDsl.fromComponents(builtInComponents); + dsl.GET("/ids/").routingTo(request -> ok(toJson(statusStorage.ids()))); + dsl.GET("/state/:id").routingTo((request, id) -> ok(toJson(statusStorage.getState(Integer.parseInt(id.toString()))))); + dsl.GET("/opType/:id").routingTo((request, id) -> ok(toJson(ServerTypeJson.builder() + .type(statusStorage.getState(Integer.parseInt(id.toString())).serverType())))); + dsl.GET("/started/:id").routingTo((request, id) -> { + boolean isMaster = statusStorage.getState(Integer.parseInt(id.toString())).isMaster(); + if(isMaster){ + return ok(toJson(MasterStatus.builder().master(statusStorage.getState(Integer.parseInt(id.toString())).getServerState()) + //note here that a responder is id + 1 + .responder(statusStorage.getState(Integer.parseInt(id.toString()) + 1).getServerState()) + .responderN(statusStorage.getState(Integer.parseInt(id.toString())).getTotalUpdates()) + .build())); + } else { + return ok(toJson(SlaveStatus.builder().slave(statusStorage.getState(Integer.parseInt(id.toString())).serverType()).build())); + } + }); + dsl.GET("/connectioninfo/:id").routingTo((request, id) -> ok(toJson(statusStorage.getState(Integer.parseInt(id.toString())).getConnectionInfo()))); + + dsl.POST("/updatestatus/:id").routingTo((request, id) -> { + SubscriberState subscriberState = Json.fromJson(request.body().asJson(), SubscriberState.class); + statusStorage.updateState(subscriberState); + return ok(toJson(subscriberState)); + }); + + return dsl.build(); + } +} diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/StatusStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/StatusStorage.java new file mode 100644 index 000000000..7cecb1735 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/main/java/org/nd4j/parameterserver/status/play/StatusStorage.java @@ -0,0 +1,61 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.status.play; + +import org.nd4j.parameterserver.model.SubscriberState; + +import java.util.List; + +public interface StatusStorage { + + /** + * The list of state ids + * for the given {@link SubscriberState} + * @return the list of ids for the given state + */ + List ids(); + + /** + * Returns the number of states + * held by this storage + * @return + */ + int numStates(); + + /** + * Get the state given an id. + * The integer represents a stream id + * for a given {@link org.nd4j.parameterserver.ParameterServerSubscriber}. + * + * A {@link SubscriberState} is supposed to be 1 to 1 mapping + * for a stream and a {@link io.aeron.driver.MediaDriver}. + * @param id the id of the state to get + * @return the subscriber state for the given id or none + * if it doesn't exist + */ + SubscriberState getState(int id); + + /** + * Update the state for storage + * @param subscriberState the subscriber state to update + */ + void updateState(SubscriberState subscriberState); +} diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/java/org/nd4j/parameterserver/status/play/StatusServerTests.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/java/org/nd4j/parameterserver/status/play/StatusServerTests.java new file mode 100644 index 000000000..d104e0f0e --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/java/org/nd4j/parameterserver/status/play/StatusServerTests.java @@ -0,0 +1,37 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.status.play; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.common.tests.BaseND4JTest; +import play.server.Server; + +@Timeout(20) +public class StatusServerTests extends BaseND4JTest { + + @Test + public void runStatusServer() { + Server server = StatusServer.startServer(new InMemoryStatusStorage(), 65236); + server.stop(); + } + +} diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/java/org/nd4j/parameterserver/status/play/StorageTests.java b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/java/org/nd4j/parameterserver/status/play/StorageTests.java new file mode 100644 index 000000000..9d49e454f --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/java/org/nd4j/parameterserver/status/play/StorageTests.java @@ -0,0 +1,64 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.status.play; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.parameterserver.model.SubscriberState; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Timeout(20) +public class StorageTests extends BaseND4JTest { + + @Test + public void testMapStorage() throws Exception { + StatusStorage mapDb = new MapDbStatusStorage(); + assertEquals(SubscriberState.empty(), mapDb.getState(-1)); + + + SubscriberState noEmpty = SubscriberState.builder().isMaster(true).serverState("master").streamId(1).build(); + mapDb.updateState(noEmpty); + assertEquals(noEmpty, mapDb.getState(1)); + + Thread.sleep(10000); + assertTrue(mapDb.numStates() == 0); + + } + + @Test + public void testStorage() throws Exception { + StatusStorage statusStorage = new InMemoryStatusStorage(); + assertEquals(SubscriberState.empty(), statusStorage.getState(-1)); + + + SubscriberState noEmpty = SubscriberState.builder().isMaster(true).serverState("master").streamId(1).build(); + statusStorage.updateState(noEmpty); + assertEquals(noEmpty, statusStorage.getState(1)); + + Thread.sleep(10000); + assertTrue(statusStorage.numStates() == 0); + + } + +} diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/resources/log4j.properties b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/resources/log4j.properties new file mode 100644 index 000000000..0b53faa91 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/resources/log4j.properties @@ -0,0 +1,44 @@ +# +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + + +log4j.rootLogger=ERROR, Console +log4j.logger.play=DEBUG +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n + +log4j.appender.org.springframework=DEBUG +log4j.appender.org.nd4j=INFO +log4j.logger.org.nd4j.aeron.ipc=INFO +log4j.appender.org.canova=INFO +log4j.appender.org.deeplearning4j=INFO +log4j.appender.opennlp.uima=OFF +log4j.appender.org.apache.uima=OFF +log4j.appender.org.cleartk=OFF + +log4j.logger.org.springframework=INFO +log4j.logger.org.nd4j=DEBUG +log4j.logger.org.canova=INFO +log4j.logger.org.deeplearning4j=INFO +log4j.logger.opennlp.uima.util=OFF +log4j.logger.org.apache.uima=OFF +log4j.logger.org.cleartk=OFF + diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/resources/logback.xml b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/resources/logback.xml new file mode 100644 index 000000000..18c64d888 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/cavis-nd4j-parameter-server-status/src/test/resources/logback.xml @@ -0,0 +1,56 @@ + + + + + + + + logs/application.log + + %logger{15} - %message%n%xException{5} + + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/nd4j-parameter-server-rocksdb-storage/pom.xml b/cavis-nd4j/cavis-nd4j-parameter-server/nd4j-parameter-server-rocksdb-storage/pom.xml new file mode 100644 index 000000000..f62cbb920 --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/nd4j-parameter-server-rocksdb-storage/pom.xml @@ -0,0 +1,53 @@ + + + + + + 4.0.0 + + + net.brutex.ai + nd4j-parameter-server-parent + 1.0.0-SNAPSHOT + + + nd4j-parameter-server-rocksdb-storage + + nd4j-parameter-server-rocksdb-storage + + + + org.rocksdb + rocksdbjni + 4.11.2 + + + net.brutex.ai + nd4j-parameter-server + + + net.brutex.ai + nd4j-common-tests + + + diff --git a/nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-rocksdb-storage/src/main/java/org/nd4j/parameterserver/updater/storage/RocksDbStorage.java b/cavis-nd4j/cavis-nd4j-parameter-server/nd4j-parameter-server-rocksdb-storage/src/main/java/org/nd4j/parameterserver/updater/storage/RocksDbStorage.java similarity index 100% rename from nd4j/nd4j-parameter-server-parent/nd4j-parameter-server-rocksdb-storage/src/main/java/org/nd4j/parameterserver/updater/storage/RocksDbStorage.java rename to cavis-nd4j/cavis-nd4j-parameter-server/nd4j-parameter-server-rocksdb-storage/src/main/java/org/nd4j/parameterserver/updater/storage/RocksDbStorage.java diff --git a/cavis-nd4j/cavis-nd4j-parameter-server/nd4j-parameter-server-rocksdb-storage/src/test/java/org/nd4j/parameterserver/updater/storage/UpdaterStorageTests.java b/cavis-nd4j/cavis-nd4j-parameter-server/nd4j-parameter-server-rocksdb-storage/src/test/java/org/nd4j/parameterserver/updater/storage/UpdaterStorageTests.java new file mode 100644 index 000000000..65651bf8a --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-parameter-server/nd4j-parameter-server-rocksdb-storage/src/test/java/org/nd4j/parameterserver/updater/storage/UpdaterStorageTests.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.nd4j.parameterserver.updater.storage; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.common.tests.BaseND4JTest; +import org.nd4j.aeron.ipc.NDArrayMessage; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Timeout(30) +public class UpdaterStorageTests extends BaseND4JTest { + + @Test + public void testInMemory() { + UpdateStorage updateStorage = new RocksDbStorage("/tmp/rocksdb"); + NDArrayMessage message = NDArrayMessage.wholeArrayUpdate(Nd4j.scalar(1.0)); + updateStorage.addUpdate(message); + assertEquals(1, updateStorage.numUpdates()); + assertEquals(message, updateStorage.getUpdate(0)); + updateStorage.clear(); + assertEquals(0, updateStorage.numUpdates()); + updateStorage.close(); + } +} diff --git a/cavis-nd4j/cavis-nd4j-tensorflow/build.gradle b/cavis-nd4j/cavis-nd4j-tensorflow/build.gradle new file mode 100644 index 000000000..74c5fed5a --- /dev/null +++ b/cavis-nd4j/cavis-nd4j-tensorflow/build.gradle @@ -0,0 +1,38 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +ext { + buildTarget = rootProject.ext.buildTarget +} + +dependencies { + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisNative.cavisNativeBlas + implementation group: "org.bytedeco", name: "tensorflow" + testRuntimeOnly group: "org.bytedeco", name: "tensorflow", classifier: buildTarget + if(buildTarget.contains("windows") || buildTarget.contains("linux")) { + testRuntimeOnly group: "org.bytedeco", name: "tensorflow", classifier: "${buildTarget}-gpu" + } + implementation "commons-io:commons-io" + implementation "com.google.code.gson:gson" + implementation "com.google.protobuf:protobuf-java" + implementation "com.google.protobuf:protobuf-java-util" + implementation "org.slf4j:slf4j-api" +} \ No newline at end of file diff --git a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/DummyDeAllocator.java b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/DummyDeAllocator.java similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/DummyDeAllocator.java rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/DummyDeAllocator.java diff --git a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/ProtoBufToFlatBufConversion.java b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/ProtoBufToFlatBufConversion.java similarity index 99% rename from nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/ProtoBufToFlatBufConversion.java rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/ProtoBufToFlatBufConversion.java index 0167645e9..36dbd5977 100644 --- a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/ProtoBufToFlatBufConversion.java +++ b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/ProtoBufToFlatBufConversion.java @@ -59,7 +59,7 @@ public class ProtoBufToFlatBufConversion { throws IOException, org.nd4j.linalg.exception.ND4JIllegalStateException { // // Working around some issues in the BERT model's execution. See file: - // nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/TFGraphs/BERTGraphTest.java + // nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/tfgraphs/BERTGraphTest.java // for details. int minibatchSize = 4; diff --git a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorDataType.java b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorDataType.java similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorDataType.java rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorDataType.java diff --git a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorflowConversion.java b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorflowConversion.java similarity index 99% rename from nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorflowConversion.java rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorflowConversion.java index 3ac06c695..7d9d9cb59 100644 --- a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorflowConversion.java +++ b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/TensorflowConversion.java @@ -20,7 +20,7 @@ package org.nd4j.tensorflow.conversion; -import org.nd4j.shade.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.InvalidProtocolBufferException; import org.bytedeco.javacpp.*; import org.bytedeco.javacpp.indexer.*; import org.nd4j.linalg.api.buffer.DataBuffer; @@ -392,7 +392,7 @@ public class TensorflowConversion { MetaGraphDef metaGraphDef; try { - metaGraphDef = MetaGraphDef.parseFrom(metaGraph.data().capacity(metaGraph.length()).asByteBuffer()); + metaGraphDef = MetaGraphDef.parseFrom(metaGraph.data().capacity(metaGraph.length()).asByteBuffer().array()); } catch (InvalidProtocolBufferException ex) { throw new IllegalStateException("ERROR: Unable to import model " + ex); } diff --git a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunner.java b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunner.java similarity index 97% rename from nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunner.java rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunner.java index 27f71a63f..11e76c519 100644 --- a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunner.java +++ b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunner.java @@ -26,9 +26,9 @@ import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.io.ClassPathResource; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.protobuf.ByteString; -import org.nd4j.shade.protobuf.InvalidProtocolBufferException; -import org.nd4j.shade.protobuf.util.JsonFormat; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; import lombok.extern.slf4j.Slf4j; import org.nd4j.tensorflow.conversion.TensorDataType; import org.apache.commons.io.IOUtils; @@ -533,7 +533,7 @@ public class GraphRunner implements Closeable { //use the protobuf api to load the graph definition and load the node metadata org.tensorflow.framework.GraphDef graphDef1 = org.tensorflow.framework.GraphDef.parseFrom(graphToUse); initSessionAndStatusIfNeeded(graphDef1); - } catch (org.nd4j.shade.protobuf.InvalidProtocolBufferException e) { + } catch (com.google.protobuf.InvalidProtocolBufferException e) { log.error("",e); } } @@ -541,7 +541,7 @@ public class GraphRunner implements Closeable { /** * Convert a json string written out - * by {@link org.nd4j.shade.protobuf.util.JsonFormat} + * by {@link com.google.protobuf.util.JsonFormat} * to a {@link org.bytedeco.tensorflow.ConfigProto} * @param json the json to read * @return the config proto to use @@ -549,9 +549,9 @@ public class GraphRunner implements Closeable { public static org.tensorflow.framework.ConfigProto fromJson(String json) { org.tensorflow.framework.ConfigProto.Builder builder = org.tensorflow.framework.ConfigProto.newBuilder(); try { - org.nd4j.shade.protobuf.util.JsonFormat.parser().merge(json,builder); + com.google.protobuf.util.JsonFormat.parser().merge(json,builder); org.tensorflow.framework.ConfigProto build = builder.build(); - org.nd4j.shade.protobuf.ByteString serialized = build.toByteString(); + com.google.protobuf.ByteString serialized = build.toByteString(); byte[] binaryString = serialized.toByteArray(); org.tensorflow.framework.ConfigProto configProto = org.tensorflow.framework.ConfigProto.parseFrom(binaryString); return configProto; @@ -626,14 +626,14 @@ public class GraphRunner implements Closeable { * Write out the session options used * by this {@link org.nd4j.tensorflow.conversion.graphrunner.GraphRunner} * a s a json string using the - * {@link org.nd4j.shade.protobuf.util.JsonFormat} + * {@link com.google.protobuf.util.JsonFormat} * @return the session options as json (mainly for debugging) */ public String sessionOptionsToJson() { if(sessionOptionsConfigProto == null) return null; try { - return org.nd4j.shade.protobuf.util.JsonFormat.printer().print(sessionOptionsConfigProto); + return com.google.protobuf.util.JsonFormat.printer().print(sessionOptionsConfigProto); } catch (Exception e) { log.error("",e); } diff --git a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunnerServiceProvider.java b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunnerServiceProvider.java similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunnerServiceProvider.java rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunnerServiceProvider.java diff --git a/nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/SavedModelConfig.java b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/SavedModelConfig.java similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/SavedModelConfig.java rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/SavedModelConfig.java diff --git a/nd4j/nd4j-tensorflow/src/main/resources/META-INF/services/org.nd4j.TFGraphRunnerService b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/META-INF/services/org.nd4j.TFGraphRunnerService similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/META-INF/services/org.nd4j.TFGraphRunnerService rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/META-INF/services/org.nd4j.TFGraphRunnerService diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/ai/konduit/__init__.py b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/ai/__init__.py similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/ai/konduit/__init__.py rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/ai/__init__.py diff --git a/python4j/python4j-core/src/main/resources/org/nd4j/python4j/pythonexec/__init__.py b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/ai/konduit/__init__.py similarity index 100% rename from python4j/python4j-core/src/main/resources/org/nd4j/python4j/pythonexec/__init__.py rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/ai/konduit/__init__.py diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/ai/konduit/casting.py b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/ai/konduit/casting.py similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/ai/konduit/casting.py rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/ai/konduit/casting.py diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float16_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float32_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_float64_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int16_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int32_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int64_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_int8_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint16_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint32_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint64_uint8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_float64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_int8.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint16.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint16.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint16.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint16.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint32.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint32.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint32.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint32.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint64.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint64.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint64.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint64.pb diff --git a/nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint8.pb b/cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint8.pb similarity index 100% rename from nd4j/nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint8.pb rename to cavis-nd4j/cavis-nd4j-tensorflow/src/main/resources/cast_graph/cast_uint8_uint8.pb diff --git a/cavis-ui/build.gradle b/cavis-ui/build.gradle new file mode 100644 index 000000000..545eb46da --- /dev/null +++ b/cavis-ui/build.gradle @@ -0,0 +1,32 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + + +subprojects { + group = "net.brutex.cavis-ui" + + apply plugin: "java-library" + apply plugin: "maven-publish" + apply plugin: "signing" + + + +} \ No newline at end of file diff --git a/cavis-ui/cavis-ui-common/build.gradle b/cavis-ui/cavis-ui-common/build.gradle new file mode 100644 index 000000000..be91ea669 --- /dev/null +++ b/cavis-ui/cavis-ui-common/build.gradle @@ -0,0 +1,43 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + + +dependencies { + //implementation projects.cavisUi.cavisUiVertx + //implementation projects.cavisUi.cavisUiModel + // implementation projects.cavisDnn.cavisDnnNlp + + implementation "commons-io:commons-io" + implementation "ch.qos.logback:logback-classic" + implementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnNlp + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisUi.cavisUiVertx + implementation projects.cavisUi.cavisUiModel + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatavecIterators + +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/main/java/org/deeplearning4j/ui/weights/ConvolutionalIterationListener.java b/cavis-ui/cavis-ui-common/src/main/java/org/deeplearning4j/ui/weights/ConvolutionalIterationListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/main/java/org/deeplearning4j/ui/weights/ConvolutionalIterationListener.java rename to cavis-ui/cavis-ui-common/src/main/java/org/deeplearning4j/ui/weights/ConvolutionalIterationListener.java diff --git a/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/ApiTest.java b/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/ApiTest.java new file mode 100644 index 000000000..a002603d5 --- /dev/null +++ b/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/ApiTest.java @@ -0,0 +1,42 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.ui; + +import org.apache.commons.io.IOUtils; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.common.resources.Resources; + +import java.io.File; +import java.util.List; + +/** + * @author Adam Gibson + */ +public class ApiTest { + + +} diff --git a/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/ManualTests.java b/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/ManualTests.java new file mode 100644 index 000000000..81cd4e5b1 --- /dev/null +++ b/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/ManualTests.java @@ -0,0 +1,348 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.ui; + +import lombok.extern.slf4j.Slf4j; +import org.datavec.image.loader.LFWLoader; +import org.deeplearning4j.datasets.iterator.impl.LFWDataSetIterator; +import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; +import org.deeplearning4j.eval.Evaluation; +import org.deeplearning4j.models.embeddings.reader.impl.BasicModelUtils; +import org.deeplearning4j.models.word2vec.VocabWord; +import org.deeplearning4j.models.word2vec.Word2Vec; +import org.deeplearning4j.nn.conf.GradientNormalization; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.inputs.InputType; +import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; +import org.deeplearning4j.nn.conf.layers.DenseLayer; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; +import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer; +import org.deeplearning4j.nn.conf.weightnoise.DropConnect; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.deeplearning4j.text.sentenceiterator.BasicLineIterator; +import org.deeplearning4j.text.sentenceiterator.SentenceIterator; +import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; +import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; +import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; +import org.deeplearning4j.ui.weights.ConvolutionalIterationListener; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.nd4j.common.resources.Resources; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.SplitTestAndTrain; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.AdaGrad; +import org.nd4j.linalg.learning.config.Nesterovs; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.fail; + +//@Ignore +@Slf4j +@Disabled +public class ManualTests { + + + @Test + @Disabled + public void testLaunch() throws Exception { + + // UiServer server = UiServer.getInstance(); + // + // System.out.println("http://localhost:" + server.getPort()+ "/"); + + Thread.sleep(10000000000L); + + new ScoreIterationListener(100); + fail("not implemneted"); + } + + + + + /** + * This test is for manual execution only, since it's here just to get working CNN and visualize it's layers + * + * @throws Exception + */ + @Test + @Tag("manual") + public void testCNNActivationsVisualization() throws Exception { + final int numRows = 40; + final int numColumns = 40; + int nChannels = 3; + int outputNum = LFWLoader.NUM_LABELS; + int numSamples = LFWLoader.NUM_IMAGES; + boolean useSubset = false; + int batchSize = 200;// numSamples/10; + int iterations = 5; + int splitTrainNum = (int) (batchSize * .8); + int seed = 123; + int listenerFreq = iterations / 5; + DataSet lfwNext; + SplitTestAndTrain trainTest; + DataSet trainInput; + List testInput = new ArrayList<>(); + List testLabels = new ArrayList<>(); + + log.info("Load data...."); + DataSetIterator lfw = new LFWDataSetIterator(batchSize, numSamples, new int[] {numRows, numColumns, nChannels}, + outputNum, useSubset, true, 1.0, new Random(seed)); + + log.info("Build model...."); + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .activation(Activation.RELU).weightInit(WeightInit.XAVIER) + .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) + .updater(new AdaGrad(0.01)).weightNoise(new DropConnect(0.5)).list() + .layer(0, new ConvolutionLayer.Builder(4, 4).name("cnn1").nIn(nChannels).stride(1, 1).nOut(20) + .build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .name("pool1").build()) + .layer(2, new ConvolutionLayer.Builder(3, 3).name("cnn2").stride(1, 1).nOut(40).build()) + .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .name("pool2").build()) + .layer(4, new ConvolutionLayer.Builder(3, 3).name("cnn3").stride(1, 1).nOut(60).build()) + .layer(5, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2}) + .name("pool3").build()) + .layer(6, new ConvolutionLayer.Builder(2, 2).name("cnn3").stride(1, 1).nOut(80).build()) + .layer(7, new DenseLayer.Builder().name("ffn1").nOut(160).dropOut(0.5).build()) + .layer(8, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).activation(Activation.SOFTMAX).build()) + + .setInputType(InputType.convolutional(numRows, numColumns, nChannels)); + + MultiLayerNetwork model = new MultiLayerNetwork(builder.build()); + model.init(); + + log.info("Train model...."); + + model.setListeners(new ScoreIterationListener(listenerFreq), new ConvolutionalIterationListener(listenerFreq)); + + while (lfw.hasNext()) { + lfwNext = lfw.next(); + lfwNext.scale(); + trainTest = lfwNext.splitTestAndTrain(splitTrainNum, new Random(seed)); // train set that is the result + trainInput = trainTest.getTrain(); // get feature matrix and labels for training + testInput.add(trainTest.getTest().getFeatures()); + testLabels.add(trainTest.getTest().getLabels()); + model.fit(trainInput); + } + + log.info("Evaluate model...."); + Evaluation eval = new Evaluation(lfw.getLabels()); + for (int i = 0; i < testInput.size(); i++) { + INDArray output = model.output(testInput.get(i)); + eval.eval(testLabels.get(i), output); + } + INDArray output = model.output(testInput.get(0)); + eval.eval(testLabels.get(0), output); + log.info(eval.stats()); + log.info("****************Example finished********************"); + + } + + @Test + @Timeout(50) + public void testWord2VecPlot() throws Exception { + File inputFile = Resources.asFile("big/raw_sentences.txt"); + SentenceIterator iter = new BasicLineIterator(inputFile.getAbsolutePath()); + + TokenizerFactory t = new DefaultTokenizerFactory(); + t.setTokenPreProcessor(new CommonPreprocessor()); + + Word2Vec vec = new Word2Vec.Builder().minWordFrequency(5).iterations(2).batchSize(1000).learningRate(0.025) + .layerSize(100).seed(42).sampling(0).negativeSample(0).windowSize(5) + .modelUtils(new BasicModelUtils()).useAdaGrad(false).iterate(iter).workers(10) + .tokenizerFactory(t).build(); + + vec.fit(); + + //UiConnectionInfo connectionInfo = UiServer.getInstance().getConnectionInfo(); + + //vec.getLookupTable().plotVocab(100, connectionInfo); + + Thread.sleep(10000000000L); + fail("Not implemented"); + } + + @Test + public void testImage() throws Exception { + INDArray array = Nd4j.create(11, 13); + for (int i = 0; i < array.rows(); i++) { + array.putRow(i, Nd4j.create(new double[] {0.0f, 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.0f, + 1.2f, 1.3f})); + } + writeImage(array, new File("test.png")); + } + + private void writeImage(INDArray array, File file) { + // BufferedImage image = ImageLoader.toImage(array); + + log.info("Array.rank(): " + array.rank()); + log.info("Size(-1): " + array.size(-1)); + log.info("Size(-2): " + array.size(-2)); + BufferedImage imageToRender = new BufferedImage(array.columns(), array.rows(), BufferedImage.TYPE_BYTE_GRAY); + for (int x = 0; x < array.columns(); x++) { + for (int y = 0; y < array.rows(); y++) { + log.info("x: " + (x) + " y: " + y); + imageToRender.getRaster().setSample(x, y, 0, (int) (255 * array.getRow(y).getDouble(x))); + } + } + + try { + ImageIO.write(imageToRender, "png", file); + } catch (IOException e) { + log.error("",e); + } + + } + + @Test + public void testCNNActivations2() throws Exception { + + int nChannels = 1; + int outputNum = 10; + int batchSize = 64; + int nEpochs = 10; + int seed = 123; + + log.info("Load data...."); + DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345); + DataSetIterator mnistTest = new MnistDataSetIterator(batchSize, false, 12345); + + log.info("Build model...."); + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .l2(0.0005) + .weightInit(WeightInit.XAVIER) + .updater(new Nesterovs(0.01, 0.9)).list() + .layer(0, new ConvolutionLayer.Builder(5, 5) + //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied + .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()) + .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2) + .stride(2, 2).build()) + .layer(2, new ConvolutionLayer.Builder(5, 5) + //Note that nIn needed be specified in later layers + .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()) + .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2) + .stride(2, 2).build()) + .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()) + .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutional(28, 28, nChannels)); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + /* + ParallelWrapper wrapper = new ParallelWrapper.Builder(model) + .averagingFrequency(1) + .prefetchBuffer(12) + .workers(2) + .reportScoreAfterAveraging(false) + .useLegacyAveraging(false) + .build(); + */ + + log.info("Train model...."); + model.setListeners(new ConvolutionalIterationListener(1)); + + //((NativeOpExecutioner) Nd4j.getExecutioner()).getLoop().setOmpNumThreads(8); + + long timeX = System.currentTimeMillis(); + // nEpochs = 2; + for (int i = 0; i < nEpochs; i++) { + long time1 = System.currentTimeMillis(); + model.fit(mnistTrain); + //wrapper.fit(mnistTrain); + long time2 = System.currentTimeMillis(); + log.info("*** Completed epoch {}, Time elapsed: {} ***", i, (time2 - time1)); + } + long timeY = System.currentTimeMillis(); + + log.info("Evaluate model...."); + Evaluation eval = new Evaluation(outputNum); + while (mnistTest.hasNext()) { + DataSet ds = mnistTest.next(); + INDArray output = model.output(ds.getFeatures(), false); + eval.eval(ds.getLabels(), output); + } + log.info(eval.stats()); + mnistTest.reset(); + + log.info("****************Example finished********************"); + } + + @Test + public void testCNNActivationsFrozen() throws Exception { + + int nChannels = 1; + int outputNum = 10; + int batchSize = 64; + int nEpochs = 10; + int seed = 123; + + log.info("Load data...."); + DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345); + + log.info("Build model...."); + MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed) + .l2(0.0005) + .weightInit(WeightInit.XAVIER) + .updater(new Nesterovs(0.01, 0.9)).list() + .layer(0, new FrozenLayer(new ConvolutionLayer.Builder(5, 5) + //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied + .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build())) + .layer(1, new FrozenLayer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2) + .stride(2, 2).build())) + .layer(2, new FrozenLayer(new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build())) + .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nOut(outputNum).activation(Activation.SOFTMAX).build()) + .setInputType(InputType.convolutionalFlat(28, 28, nChannels)); + + MultiLayerConfiguration conf = builder.build(); + MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + log.info("Train model...."); + model.setListeners(new ConvolutionalIterationListener(1)); + + for (int i = 0; i < nEpochs; i++) { + model.fit(mnistTrain); + } + } +} diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/java/org/deeplearning4j/ui/weights/HistogramBinTest.java b/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/weights/HistogramBinTest.java similarity index 94% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/java/org/deeplearning4j/ui/weights/HistogramBinTest.java rename to cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/weights/HistogramBinTest.java index 11d58d775..9b6d31400 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/java/org/deeplearning4j/ui/weights/HistogramBinTest.java +++ b/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/weights/HistogramBinTest.java @@ -21,22 +21,21 @@ package org.deeplearning4j.ui.weights; import org.deeplearning4j.ui.model.weights.HistogramBin; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import java.math.BigDecimal; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class HistogramBinTest { + @BeforeEach + public void setUp() throws Exception { + + } @Test public void testGetBins() throws Exception { diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/java/org/deeplearning4j/ui/weights/TestConvolutionalListener.java b/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/weights/TestConvolutionalListener.java similarity index 96% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/java/org/deeplearning4j/ui/weights/TestConvolutionalListener.java rename to cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/weights/TestConvolutionalListener.java index 0daf0c21d..442f3bf01 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/java/org/deeplearning4j/ui/weights/TestConvolutionalListener.java +++ b/cavis-ui/cavis-ui-common/src/test/java/org/deeplearning4j/ui/weights/TestConvolutionalListener.java @@ -36,20 +36,16 @@ import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.learning.config.Nesterovs; import org.nd4j.linalg.lossfunctions.LossFunctions; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestConvolutionalListener { @Test - @Disabled + @Tag("manual") + @Disabled//Should be run manually public void testUI() throws Exception { int nChannels = 1; // Number of input channels diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/resources/log4j.properties b/cavis-ui/cavis-ui-common/src/test/resources/log4j.properties similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/resources/log4j.properties rename to cavis-ui/cavis-ui-common/src/test/resources/log4j.properties diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/resources/logback.xml b/cavis-ui/cavis-ui-common/src/test/resources/logback.xml similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/resources/logback.xml rename to cavis-ui/cavis-ui-common/src/test/resources/logback.xml diff --git a/cavis-ui/cavis-ui-components/build.gradle b/cavis-ui/cavis-ui-components/build.gradle new file mode 100644 index 000000000..432a168fe --- /dev/null +++ b/cavis-ui/cavis-ui-components/build.gradle @@ -0,0 +1,31 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" + implementation "org.freemarker:freemarker:2.3.23" + implementation "commons-io:commons-io" + implementation projects.cavisDnn.cavisDnnCommon + testImplementation projects.cavisDnn.cavisDnnCommonTests +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/developerReadme.md b/cavis-ui/cavis-ui-components/developerReadme.md similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/developerReadme.md rename to cavis-ui/cavis-ui-components/developerReadme.md diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/Component.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/Component.java similarity index 96% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/Component.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/Component.java index 93e53b758..75586155c 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/Component.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/Component.java @@ -26,8 +26,8 @@ import org.deeplearning4j.ui.components.component.ComponentDiv; import org.deeplearning4j.ui.components.decorator.DecoratorAccordion; import org.deeplearning4j.ui.components.table.ComponentTable; import org.deeplearning4j.ui.components.text.ComponentText; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) @JsonSubTypes(value = {@JsonSubTypes.Type(value = ChartHistogram.class, name = "ChartHistogram"), diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/LengthUnit.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/LengthUnit.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/LengthUnit.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/LengthUnit.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/Style.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/Style.java similarity index 97% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/Style.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/Style.java index e8a49d842..226378e24 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/Style.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/Style.java @@ -28,8 +28,8 @@ import org.deeplearning4j.ui.components.component.style.StyleDiv; import org.deeplearning4j.ui.components.decorator.style.StyleAccordion; import org.deeplearning4j.ui.components.table.style.StyleTable; import org.deeplearning4j.ui.components.text.style.StyleText; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.awt.*; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/Utils.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/Utils.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/api/Utils.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/api/Utils.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/Chart.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/Chart.java similarity index 99% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/Chart.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/Chart.java index fc15d4e4b..d6fc0f165 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/Chart.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/Chart.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import org.deeplearning4j.ui.api.Component; import org.deeplearning4j.ui.components.chart.style.StyleChart; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; @Data @EqualsAndHashCode(callSuper = true) diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHistogram.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHistogram.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHistogram.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHistogram.java index adaab695c..dee8b4f9f 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHistogram.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHistogram.java @@ -23,7 +23,7 @@ package org.deeplearning4j.ui.components.chart; import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.components.chart.style.StyleChart; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.List; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHorizontalBar.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHorizontalBar.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHorizontalBar.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHorizontalBar.java index 836f0cab7..bfc15f9e4 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHorizontalBar.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartHorizontalBar.java @@ -23,7 +23,7 @@ package org.deeplearning4j.ui.components.chart; import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.components.chart.style.StyleChart; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.List; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartLine.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartLine.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartLine.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartLine.java index a2325f938..d40b63682 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartLine.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartLine.java @@ -23,7 +23,7 @@ package org.deeplearning4j.ui.components.chart; import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.components.chart.style.StyleChart; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartScatter.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartScatter.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartScatter.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartScatter.java index db32abf0a..ae79a8c77 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartScatter.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartScatter.java @@ -23,7 +23,7 @@ package org.deeplearning4j.ui.components.chart; import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.components.chart.style.StyleChart; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartStackedArea.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartStackedArea.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartStackedArea.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartStackedArea.java index 553324448..199357c26 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartStackedArea.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartStackedArea.java @@ -23,7 +23,7 @@ package org.deeplearning4j.ui.components.chart; import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.components.chart.style.StyleChart; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.Arrays; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartTimeline.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartTimeline.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartTimeline.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartTimeline.java index 09022e422..57b2bf4f6 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartTimeline.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/ChartTimeline.java @@ -26,7 +26,7 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.deeplearning4j.ui.api.Utils; import org.deeplearning4j.ui.components.chart.style.StyleChart; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.awt.*; import java.util.ArrayList; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/style/StyleChart.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/style/StyleChart.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/style/StyleChart.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/style/StyleChart.java index 2ce3d3bc9..3f9706a37 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/style/StyleChart.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/chart/style/StyleChart.java @@ -27,7 +27,7 @@ import lombok.NoArgsConstructor; import org.deeplearning4j.ui.api.Style; import org.deeplearning4j.ui.api.Utils; import org.deeplearning4j.ui.components.text.style.StyleText; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.awt.*; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/component/ComponentDiv.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/component/ComponentDiv.java similarity index 97% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/component/ComponentDiv.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/component/ComponentDiv.java index 26eb68943..ce2790ff1 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/component/ComponentDiv.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/component/ComponentDiv.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.api.Component; import org.deeplearning4j.ui.api.Style; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.Collection; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/component/style/StyleDiv.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/component/style/StyleDiv.java similarity index 97% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/component/style/StyleDiv.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/component/style/StyleDiv.java index 655894927..040a682f6 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/component/style/StyleDiv.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/component/style/StyleDiv.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.deeplearning4j.ui.api.Style; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; @NoArgsConstructor @Data diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.java index e69ecae43..543282d74 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.api.Component; import org.deeplearning4j.ui.components.decorator.style.StyleAccordion; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.Collections; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.java similarity index 96% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.java index c11ba4e5c..de73d91ba 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.deeplearning4j.ui.api.Style; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; @NoArgsConstructor diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/table/ComponentTable.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/table/ComponentTable.java similarity index 97% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/table/ComponentTable.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/table/ComponentTable.java index 04c539efe..aebb3d0e6 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/table/ComponentTable.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/table/ComponentTable.java @@ -25,7 +25,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.api.Component; import org.deeplearning4j.ui.components.table.style.StyleTable; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; @EqualsAndHashCode(callSuper = true) @Data diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/table/style/StyleTable.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/table/style/StyleTable.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/table/style/StyleTable.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/table/style/StyleTable.java index cb37f27ff..c3453cb1b 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/table/style/StyleTable.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/table/style/StyleTable.java @@ -26,7 +26,7 @@ import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.api.LengthUnit; import org.deeplearning4j.ui.api.Style; import org.deeplearning4j.ui.api.Utils; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.awt.*; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/text/ComponentText.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/text/ComponentText.java similarity index 97% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/text/ComponentText.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/text/ComponentText.java index 2a71b59d3..fc5ee7d7a 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/text/ComponentText.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/text/ComponentText.java @@ -24,7 +24,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.deeplearning4j.ui.api.Component; import org.deeplearning4j.ui.components.text.style.StyleText; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; @EqualsAndHashCode(callSuper = true) @Data diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/text/style/StyleText.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/text/style/StyleText.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/text/style/StyleText.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/text/style/StyleText.java index 778fc8b4a..15c590244 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/components/text/style/StyleText.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/components/text/style/StyleText.java @@ -26,7 +26,7 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import org.deeplearning4j.ui.api.Style; import org.deeplearning4j.ui.api.Utils; -import org.nd4j.shade.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import java.awt.*; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/standalone/ComponentObject.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/standalone/ComponentObject.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/standalone/ComponentObject.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/standalone/ComponentObject.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/standalone/StaticPageUtil.java b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/standalone/StaticPageUtil.java similarity index 95% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/standalone/StaticPageUtil.java rename to cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/standalone/StaticPageUtil.java index f66eae54c..0ee0d2527 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/java/org/deeplearning4j/ui/standalone/StaticPageUtil.java +++ b/cavis-ui/cavis-ui-components/src/main/java/org/deeplearning4j/ui/standalone/StaticPageUtil.java @@ -28,10 +28,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.deeplearning4j.ui.api.Component; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import java.io.File; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/resources/assets/dl4j-ui.d.ts b/cavis-ui/cavis-ui-components/src/main/resources/assets/dl4j-ui.d.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/resources/assets/dl4j-ui.d.ts rename to cavis-ui/cavis-ui-components/src/main/resources/assets/dl4j-ui.d.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/resources/assets/dl4j-ui.js b/cavis-ui/cavis-ui-components/src/main/resources/assets/dl4j-ui.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/resources/assets/dl4j-ui.js rename to cavis-ui/cavis-ui-components/src/main/resources/assets/dl4j-ui.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/resources/assets/dl4j-ui.js.map b/cavis-ui/cavis-ui-components/src/main/resources/assets/dl4j-ui.js.map similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/resources/assets/dl4j-ui.js.map rename to cavis-ui/cavis-ui-components/src/main/resources/assets/dl4j-ui.js.map diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/resources/org/deeplearning4j/ui/standalone/staticpage.ftl b/cavis-ui/cavis-ui-components/src/main/resources/org/deeplearning4j/ui/standalone/staticpage.ftl similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/resources/org/deeplearning4j/ui/standalone/staticpage.ftl rename to cavis-ui/cavis-ui-components/src/main/resources/org/deeplearning4j/ui/standalone/staticpage.ftl diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Component.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Component.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Component.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Component.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/ComponentType.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/ComponentType.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/ComponentType.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/ComponentType.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Constants.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Constants.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Constants.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Constants.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Margin.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Margin.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Margin.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Margin.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Renderable.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Renderable.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Renderable.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Renderable.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Style.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Style.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Style.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/api/Style.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/Chart.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/Chart.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/Chart.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/Chart.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartHistogram.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartHistogram.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartHistogram.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartHistogram.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartLine.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartLine.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartLine.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartLine.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartScatter.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartScatter.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartScatter.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartScatter.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartStackedArea.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartStackedArea.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartStackedArea.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartStackedArea.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartTimeline.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartTimeline.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartTimeline.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/ChartTimeline.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/Legend.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/Legend.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/Legend.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/Legend.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/style/StyleChart.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/style/StyleChart.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/style/StyleChart.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/chart/style/StyleChart.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/component/ComponentDiv.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/component/ComponentDiv.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/component/ComponentDiv.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/component/ComponentDiv.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/component/style/StyleDiv.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/component/style/StyleDiv.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/component/style/StyleDiv.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/component/style/StyleDiv.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/decorator/DecoratorAccordion.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/decorator/style/StyleAccordion.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/table/ComponentTable.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/table/ComponentTable.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/table/ComponentTable.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/table/ComponentTable.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/table/style/StyleTable.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/table/style/StyleTable.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/table/style/StyleTable.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/table/style/StyleTable.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/text/ComponentText.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/text/ComponentText.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/text/ComponentText.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/text/ComponentText.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/text/style/StyleText.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/text/style/StyleText.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/components/text/style/StyleText.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/components/text/style/StyleText.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/tsconfig.json b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/tsconfig.json similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/tsconfig.json rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/tsconfig.json diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/d3.d.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/d3.d.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/d3.d.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/d3.d.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/jquery.d.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/jquery.d.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/jquery.d.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/jquery.d.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/jqueryui.d.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/jqueryui.d.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/jqueryui.d.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/typedefs/jqueryui.d.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/util/TSUtils.ts b/cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/util/TSUtils.ts similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/main/typescript/org/deeplearning4j/ui/util/TSUtils.ts rename to cavis-ui/cavis-ui-components/src/main/typescript/org/deeplearning4j/ui/util/TSUtils.ts diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestComponentSerialization.java b/cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestComponentSerialization.java similarity index 96% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestComponentSerialization.java rename to cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestComponentSerialization.java index 5e67cb09d..e5695e51e 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestComponentSerialization.java +++ b/cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestComponentSerialization.java @@ -34,21 +34,15 @@ import org.deeplearning4j.ui.components.table.ComponentTable; import org.deeplearning4j.ui.components.table.style.StyleTable; import org.deeplearning4j.ui.components.text.ComponentText; import org.deeplearning4j.ui.components.text.style.StyleText; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.awt.*; import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestComponentSerialization extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestRendering.java b/cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestRendering.java similarity index 96% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestRendering.java rename to cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestRendering.java index 5e369b45d..95afce9bb 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestRendering.java +++ b/cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestRendering.java @@ -35,12 +35,9 @@ import org.deeplearning4j.ui.components.table.ComponentTable; import org.deeplearning4j.ui.components.table.style.StyleTable; import org.deeplearning4j.ui.components.text.ComponentText; import org.deeplearning4j.ui.components.text.style.StyleText; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.awt.*; import java.io.File; @@ -48,13 +45,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestRendering extends BaseDL4JTest { - @Disabled + //@Ignore @Test public void test() throws Exception { diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestStandAlone.java b/cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestStandAlone.java similarity index 92% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestStandAlone.java rename to cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestStandAlone.java index 7b4a23ae9..e8ce7ecf3 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/src/test/java/org/deeplearning4j/ui/TestStandAlone.java +++ b/cavis-ui/cavis-ui-components/src/test/java/org/deeplearning4j/ui/TestStandAlone.java @@ -28,17 +28,11 @@ import org.deeplearning4j.ui.components.chart.style.StyleChart; import org.deeplearning4j.ui.components.table.ComponentTable; import org.deeplearning4j.ui.components.table.style.StyleTable; import org.deeplearning4j.ui.standalone.StaticPageUtil; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.awt.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestStandAlone extends BaseDL4JTest { @Test diff --git a/cavis-ui/cavis-ui-model/build.gradle b/cavis-ui/cavis-ui-model/build.gradle new file mode 100644 index 000000000..768e29693 --- /dev/null +++ b/cavis-ui/cavis-ui-model/build.gradle @@ -0,0 +1,45 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +ext { + buildTarget = rootProject.ext.buildTarget +} +dependencies { + testImplementation 'ch.qos.logback:logback-classic' + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisDnn.cavisDnnApi + implementation "org.agrona:agrona" + implementation "org.mapdb:mapdb:3.0.5" + implementation "org.xerial:sqlite-jdbc:3.15.1" + compileOnly "javax.annotation:javax.annotation-api:1.2" + testImplementation projects.cavisDnn.cavisDnnCommonTests + implementation group:"org.bytedeco", name:"javacpp" + implementation group:"org.bytedeco", name:"javacpp", classifier: buildTarget + implementation "commons-io:commons-io" + implementation "org.apache.commons:commons-compress" + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisNative.cavisNativeBlas + implementation "org.slf4j:slf4j-api" + implementation "it.unimi.dsi:fastutil:8.1.1" + implementation "com.fasterxml.jackson.core:jackson-annotations" + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/activation/PathUpdate.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/activation/PathUpdate.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/activation/PathUpdate.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/activation/PathUpdate.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/nearestneighbors/word2vec/NearestNeighborsQuery.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/nearestneighbors/word2vec/NearestNeighborsQuery.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/nearestneighbors/word2vec/NearestNeighborsQuery.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/nearestneighbors/word2vec/NearestNeighborsQuery.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/renders/PathUpdate.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/renders/PathUpdate.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/renders/PathUpdate.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/renders/PathUpdate.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/BaseStatsListener.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/BaseStatsListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/BaseStatsListener.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/BaseStatsListener.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/J7StatsListener.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/J7StatsListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/J7StatsListener.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/J7StatsListener.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/StatsListener.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/StatsListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/StatsListener.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/StatsListener.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/Histogram.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/Histogram.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/Histogram.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/Histogram.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationConfiguration.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationConfiguration.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationConfiguration.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationReport.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationReport.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationReport.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsInitializationReport.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsReport.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsReport.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsReport.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsReport.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsType.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsType.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsType.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsType.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsUpdateConfiguration.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsUpdateConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsUpdateConfiguration.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/StatsUpdateConfiguration.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/SummaryType.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/SummaryType.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/SummaryType.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/api/SummaryType.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsInitializationConfiguration.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsInitializationConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsInitializationConfiguration.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsInitializationConfiguration.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsUpdateConfiguration.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsUpdateConfiguration.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsUpdateConfiguration.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/DefaultStatsUpdateConfiguration.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsInitializationReport.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsInitializationReport.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsInitializationReport.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsInitializationReport.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsReport.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsReport.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsReport.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeStatsReport.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeUtil.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeUtil.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeUtil.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/SbeUtil.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsInitializationReport.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsInitializationReport.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsInitializationReport.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsInitializationReport.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsReport.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsReport.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsReport.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/impl/java/JavaStatsReport.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingDecoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingDecoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingDecoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingDecoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingEncoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingEncoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingEncoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/GroupSizeEncodingEncoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentDecoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentDecoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentDecoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentDecoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentEncoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentEncoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentEncoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/InitFieldsPresentEncoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MemoryType.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MemoryType.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MemoryType.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MemoryType.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderDecoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderDecoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderDecoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderDecoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderEncoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderEncoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderEncoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MessageHeaderEncoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MetaAttribute.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MetaAttribute.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MetaAttribute.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/MetaAttribute.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatSource.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatSource.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatSource.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatSource.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatType.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatType.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatType.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatType.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StaticInfoDecoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StaticInfoDecoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StaticInfoDecoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StaticInfoDecoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StaticInfoEncoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StaticInfoEncoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StaticInfoEncoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StaticInfoEncoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatsType.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatsType.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatsType.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StatsType.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataDecoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataDecoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataDecoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataDecoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataEncoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataEncoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataEncoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/StorageMetaDataEncoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/SummaryType.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/SummaryType.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/SummaryType.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/SummaryType.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateDecoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateDecoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateDecoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateDecoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateEncoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateEncoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateEncoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateEncoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentDecoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentDecoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentDecoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentDecoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentEncoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentEncoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentEncoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/UpdateFieldsPresentEncoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Decoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Decoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Decoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Decoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Encoder.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Encoder.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Encoder.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/stats/sbe/VarDataUTF8Encoder.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/AgronaPersistable.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/AgronaPersistable.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/AgronaPersistable.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/AgronaPersistable.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/BaseCollectionStatsStorage.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/BaseCollectionStatsStorage.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/BaseCollectionStatsStorage.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/BaseCollectionStatsStorage.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/FileStatsStorage.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/FileStatsStorage.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/FileStatsStorage.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/FileStatsStorage.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/InMemoryStatsStorage.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/InMemoryStatsStorage.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/InMemoryStatsStorage.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/InMemoryStatsStorage.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/JavaStorageMetaData.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/JavaStorageMetaData.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/JavaStorageMetaData.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/JavaStorageMetaData.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueuePairStatsStorageListener.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueuePairStatsStorageListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueuePairStatsStorageListener.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueuePairStatsStorageListener.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueueStatsStorageListener.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueueStatsStorageListener.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueueStatsStorageListener.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/QueueStatsStorageListener.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/SbeStorageMetaData.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/SbeStorageMetaData.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/SbeStorageMetaData.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/impl/SbeStorageMetaData.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/mapdb/MapDBStatsStorage.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/mapdb/MapDBStatsStorage.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/mapdb/MapDBStatsStorage.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/mapdb/MapDBStatsStorage.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/sqlite/J7FileStatsStorage.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/sqlite/J7FileStatsStorage.java similarity index 99% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/sqlite/J7FileStatsStorage.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/sqlite/J7FileStatsStorage.java index 06a03ecb6..07dee8b04 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/sqlite/J7FileStatsStorage.java +++ b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/storage/sqlite/J7FileStatsStorage.java @@ -56,7 +56,7 @@ public class J7FileStatsStorage implements StatsStorage { try { connection = DriverManager.getConnection("jdbc:sqlite:" + file.getAbsolutePath()); } catch (Exception e) { - throw new RuntimeException("Error ninializing J7FileStatsStorage instance", e); + throw new RuntimeException("Error initializing J7FileStatsStorage instance", e); } try { diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/ConvolutionListenerPersistable.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/ConvolutionListenerPersistable.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/ConvolutionListenerPersistable.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/ConvolutionListenerPersistable.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/HistogramBin.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/HistogramBin.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/HistogramBin.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/HistogramBin.java index 106a72057..a8f4dffc3 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/HistogramBin.java +++ b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/HistogramBin.java @@ -23,7 +23,7 @@ package org.deeplearning4j.ui.model.weights; import lombok.Data; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/beans/CompactModelAndGradient.java b/cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/beans/CompactModelAndGradient.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/beans/CompactModelAndGradient.java rename to cavis-ui/cavis-ui-model/src/main/java/org/deeplearning4j/ui/model/weights/beans/CompactModelAndGradient.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/resources/StatsListenerSchemas.xml b/cavis-ui/cavis-ui-model/src/main/resources/StatsListenerSchemas.xml similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/main/resources/StatsListenerSchemas.xml rename to cavis-ui/cavis-ui-model/src/main/resources/StatsListenerSchemas.xml diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/TestStorageMetaData.java b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/TestStorageMetaData.java similarity index 92% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/TestStorageMetaData.java rename to cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/TestStorageMetaData.java index 2a69cbdc8..e31fc1541 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/TestStorageMetaData.java +++ b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/TestStorageMetaData.java @@ -23,18 +23,12 @@ package org.deeplearning4j.ui; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.core.storage.StorageMetaData; import org.deeplearning4j.ui.model.storage.impl.SbeStorageMetaData; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.Serializable; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestStorageMetaData extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsClasses.java b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsClasses.java similarity index 86% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsClasses.java rename to cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsClasses.java index 784246107..2ec36472c 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsClasses.java +++ b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsClasses.java @@ -25,11 +25,9 @@ import org.deeplearning4j.ui.model.stats.api.*; import org.deeplearning4j.ui.model.stats.impl.SbeStatsInitializationReport; import org.deeplearning4j.ui.model.stats.impl.SbeStatsReport; import org.deeplearning4j.ui.model.stats.impl.java.JavaStatsInitializationReport; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.nd4j.common.primitives.Pair; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.*; import java.util.ArrayList; @@ -38,10 +36,7 @@ import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestStatsClasses extends BaseDL4JTest { @Test @@ -552,9 +547,9 @@ public class TestStatsClasses extends BaseDL4JTest { assertEquals(perfTotalMB, report2.getTotalMinibatches()); assertEquals(perfEPS, report2.getExamplesPerSecond(), 0.0); assertEquals(perfMBPS, report2.getMinibatchesPerSecond(), 0.0); - assertTrue(report2.hasPerformance()); + Assertions.assertTrue(report2.hasPerformance()); } else { - assertFalse(report2.hasPerformance()); + Assertions.assertFalse(report2.hasPerformance()); } if (collectMemoryStats) { @@ -565,30 +560,30 @@ public class TestStatsClasses extends BaseDL4JTest { assertArrayEquals(memDC, report2.getDeviceCurrentBytes()); assertArrayEquals(memDM, report2.getDeviceMaxBytes()); - assertTrue(report2.hasMemoryUse()); + Assertions.assertTrue(report2.hasMemoryUse()); } else { - assertFalse(report2.hasMemoryUse()); + Assertions.assertFalse(report2.hasMemoryUse()); } if (collectGCStats) { List> gcs = report2.getGarbageCollectionStats(); - assertEquals(2, gcs.size()); - assertEquals(gc1Name, gcs.get(0).getFirst()); + Assertions.assertEquals(2, gcs.size()); + Assertions.assertEquals(gc1Name, gcs.get(0).getFirst()); assertArrayEquals(new int[] {gcdc1, gcdt1}, gcs.get(0).getSecond()); - assertEquals(gc2Name, gcs.get(1).getFirst()); + Assertions.assertEquals(gc2Name, gcs.get(1).getFirst()); assertArrayEquals(new int[] {gcdc2, gcdt2}, gcs.get(1).getSecond()); - assertTrue(report2.hasGarbageCollection()); + Assertions.assertTrue(report2.hasGarbageCollection()); } else { - assertFalse(report2.hasGarbageCollection()); + Assertions.assertFalse(report2.hasGarbageCollection()); } if (collectScore) { assertEquals(score, report2.getScore(), 0.0); - assertTrue(report2.hasScore()); + Assertions.assertTrue(report2.hasScore()); } else { - assertFalse(report2.hasScore()); + Assertions.assertFalse(report2.hasScore()); } if (collectLearningRates) { @@ -597,9 +592,9 @@ public class TestStatsClasses extends BaseDL4JTest { assertEquals(lrByParam.get(s), report2.getLearningRates().get(s), 1e-6); } - assertTrue(report2.hasLearningRates()); + Assertions.assertTrue(report2.hasLearningRates()); } else { - assertFalse(report2.hasLearningRates()); + Assertions.assertFalse(report2.hasLearningRates()); } if (collectMetaData) { @@ -614,112 +609,112 @@ public class TestStatsClasses extends BaseDL4JTest { if (collectHistograms[0]) { assertEquals(pHist, report2.getHistograms(StatsType.Parameters)); - assertTrue(report2.hasHistograms(StatsType.Parameters)); + Assertions.assertTrue(report2.hasHistograms(StatsType.Parameters)); } else { - assertFalse(report2.hasHistograms(StatsType.Parameters)); + Assertions.assertFalse(report2.hasHistograms(StatsType.Parameters)); } if (collectHistograms[1]) { assertEquals(gHist, report2.getHistograms(StatsType.Gradients)); - assertTrue(report2.hasHistograms(StatsType.Gradients)); + Assertions.assertTrue(report2.hasHistograms(StatsType.Gradients)); } else { - assertFalse(report2.hasHistograms(StatsType.Gradients)); + Assertions.assertFalse(report2.hasHistograms(StatsType.Gradients)); } if (collectHistograms[2]) { assertEquals(uHist, report2.getHistograms(StatsType.Updates)); - assertTrue(report2.hasHistograms(StatsType.Updates)); + Assertions.assertTrue(report2.hasHistograms(StatsType.Updates)); } else { - assertFalse(report2.hasHistograms(StatsType.Updates)); + Assertions.assertFalse(report2.hasHistograms(StatsType.Updates)); } if (collectHistograms[3]) { assertEquals(aHist, report2.getHistograms(StatsType.Activations)); - assertTrue(report2.hasHistograms(StatsType.Activations)); + Assertions.assertTrue(report2.hasHistograms(StatsType.Activations)); } else { - assertFalse(report2.hasHistograms(StatsType.Activations)); + Assertions.assertFalse(report2.hasHistograms(StatsType.Activations)); } if (collectMeanStdev[0]) { assertEquals(pMean, report2.getMean(StatsType.Parameters)); assertEquals(pStd, report2.getStdev(StatsType.Parameters)); - assertTrue(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Parameters, SummaryType.Mean)); - assertTrue(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Parameters, SummaryType.Stdev)); } else { - assertFalse(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Parameters, SummaryType.Mean)); - assertFalse(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Parameters, SummaryType.Stdev)); } if (collectMeanStdev[1]) { assertEquals(gMean, report2.getMean(StatsType.Gradients)); assertEquals(gStd, report2.getStdev(StatsType.Gradients)); - assertTrue(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Gradients, SummaryType.Mean)); - assertTrue(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Gradients, SummaryType.Stdev)); } else { - assertFalse(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Gradients, SummaryType.Mean)); - assertFalse(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Gradients, SummaryType.Stdev)); } if (collectMeanStdev[2]) { assertEquals(uMean, report2.getMean(StatsType.Updates)); assertEquals(uStd, report2.getStdev(StatsType.Updates)); - assertTrue(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Updates, SummaryType.Mean)); - assertTrue(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Updates, SummaryType.Stdev)); } else { - assertFalse(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Updates, SummaryType.Mean)); - assertFalse(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Updates, SummaryType.Stdev)); } if (collectMeanStdev[3]) { assertEquals(aMean, report2.getMean(StatsType.Activations)); assertEquals(aStd, report2.getStdev(StatsType.Activations)); - assertTrue(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Activations, SummaryType.Mean)); - assertTrue(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Activations, SummaryType.Stdev)); } else { - assertFalse(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Activations, SummaryType.Mean)); - assertFalse(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Activations, SummaryType.Stdev)); } if (collectMM[0]) { assertEquals(pMM, report2.getMeanMagnitudes(StatsType.Parameters)); - assertTrue(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Parameters, SummaryType.MeanMagnitudes)); } else { - assertFalse(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Parameters, SummaryType.MeanMagnitudes)); } if (collectMM[1]) { assertEquals(gMM, report2.getMeanMagnitudes(StatsType.Gradients)); - assertTrue(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Gradients, SummaryType.MeanMagnitudes)); } else { - assertFalse(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Gradients, SummaryType.MeanMagnitudes)); } if (collectMM[2]) { assertEquals(uMM, report2.getMeanMagnitudes(StatsType.Updates)); - assertTrue(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Updates, SummaryType.MeanMagnitudes)); } else { - assertFalse(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Updates, SummaryType.MeanMagnitudes)); } if (collectMM[3]) { assertEquals(aMM, report2.getMeanMagnitudes(StatsType.Activations)); - assertTrue(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertTrue(report2.hasSummaryStats(StatsType.Activations, SummaryType.MeanMagnitudes)); } else { - assertFalse(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Activations, SummaryType.MeanMagnitudes)); } @@ -747,7 +742,7 @@ public class TestStatsClasses extends BaseDL4JTest { } } - assertEquals(13824, testCount); + Assertions.assertEquals(13824, testCount); } @Test @@ -908,9 +903,9 @@ public class TestStatsClasses extends BaseDL4JTest { assertEquals(perfTotalMB, report2.getTotalMinibatches()); assertEquals(perfEPS, report2.getExamplesPerSecond(), 0.0); assertEquals(perfMBPS, report2.getMinibatchesPerSecond(), 0.0); - assertTrue(report2.hasPerformance()); + Assertions.assertTrue(report2.hasPerformance()); } else { - assertFalse(report2.hasPerformance()); + Assertions.assertFalse(report2.hasPerformance()); } if (collectMemoryStats) { @@ -921,23 +916,23 @@ public class TestStatsClasses extends BaseDL4JTest { assertArrayEquals(memDC, report2.getDeviceCurrentBytes()); assertArrayEquals(memDM, report2.getDeviceMaxBytes()); - assertTrue(report2.hasMemoryUse()); + Assertions.assertTrue(report2.hasMemoryUse()); } else { - assertFalse(report2.hasMemoryUse()); + Assertions.assertFalse(report2.hasMemoryUse()); } if (collectGCStats) { List> gcs = report2.getGarbageCollectionStats(); - assertEquals(2, gcs.size()); + Assertions.assertEquals(2, gcs.size()); assertNullOrZeroLength(gcs.get(0).getFirst()); assertArrayEquals(new int[] {gcdc1, gcdt1}, gcs.get(0).getSecond()); assertNullOrZeroLength(gcs.get(1).getFirst()); assertArrayEquals(new int[] {gcdc2, gcdt2}, gcs.get(1).getSecond()); - assertTrue(report2.hasGarbageCollection()); + Assertions.assertTrue(report2.hasGarbageCollection()); } else { - assertFalse(report2.hasGarbageCollection()); + Assertions.assertFalse(report2.hasGarbageCollection()); } if (collectDataSetMetaData) { @@ -946,71 +941,71 @@ public class TestStatsClasses extends BaseDL4JTest { if (collectScore) { assertEquals(score, report2.getScore(), 0.0); - assertTrue(report2.hasScore()); + Assertions.assertTrue(report2.hasScore()); } else { - assertFalse(report2.hasScore()); + Assertions.assertFalse(report2.hasScore()); } if (collectLearningRates) { assertNull(report2.getLearningRates()); } else { - assertFalse(report2.hasLearningRates()); + Assertions.assertFalse(report2.hasLearningRates()); } assertNull(report2.getHistograms(StatsType.Parameters)); - assertFalse(report2.hasHistograms(StatsType.Parameters)); + Assertions.assertFalse(report2.hasHistograms(StatsType.Parameters)); assertNull(report2.getHistograms(StatsType.Gradients)); - assertFalse(report2.hasHistograms(StatsType.Gradients)); + Assertions.assertFalse(report2.hasHistograms(StatsType.Gradients)); assertNull(report2.getHistograms(StatsType.Updates)); - assertFalse(report2.hasHistograms(StatsType.Updates)); + Assertions.assertFalse(report2.hasHistograms(StatsType.Updates)); assertNull(report2.getHistograms(StatsType.Activations)); - assertFalse(report2.hasHistograms(StatsType.Activations)); + Assertions.assertFalse(report2.hasHistograms(StatsType.Activations)); assertNull(report2.getMean(StatsType.Parameters)); assertNull(report2.getStdev(StatsType.Parameters)); - assertFalse(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Parameters, SummaryType.Mean)); - assertFalse(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Parameters, SummaryType.Stdev)); assertNull(report2.getMean(StatsType.Gradients)); assertNull(report2.getStdev(StatsType.Gradients)); - assertFalse(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Gradients, SummaryType.Mean)); - assertFalse(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Gradients, SummaryType.Stdev)); assertNull(report2.getMean(StatsType.Updates)); assertNull(report2.getStdev(StatsType.Updates)); - assertFalse(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Updates, SummaryType.Mean)); - assertFalse(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Updates, SummaryType.Stdev)); assertNull(report2.getMean(StatsType.Activations)); assertNull(report2.getStdev(StatsType.Activations)); - assertFalse(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Activations, SummaryType.Mean)); - assertFalse(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Activations, SummaryType.Stdev)); assertNull(report2.getMeanMagnitudes(StatsType.Parameters)); - assertFalse(report2.hasSummaryStats(StatsType.Parameters, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Parameters, SummaryType.MeanMagnitudes)); assertNull(report2.getMeanMagnitudes(StatsType.Gradients)); - assertFalse(report2.hasSummaryStats(StatsType.Gradients, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Gradients, SummaryType.MeanMagnitudes)); assertNull(report2.getMeanMagnitudes(StatsType.Updates)); - assertFalse(report2.hasSummaryStats(StatsType.Updates, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Updates, SummaryType.MeanMagnitudes)); assertNull(report2.getMeanMagnitudes(StatsType.Activations)); - assertFalse(report2.hasSummaryStats(StatsType.Activations, + Assertions.assertFalse(report2.hasSummaryStats(StatsType.Activations, SummaryType.MeanMagnitudes)); //Check standard Java serialization @@ -1037,7 +1032,7 @@ public class TestStatsClasses extends BaseDL4JTest { } } - assertEquals(13824, testCount); + Assertions.assertEquals(13824, testCount); } } diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsListener.java b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsListener.java similarity index 95% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsListener.java rename to cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsListener.java index c1a44f8bd..56952d870 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsListener.java +++ b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestStatsListener.java @@ -32,10 +32,7 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.ui.model.stats.J7StatsListener; import org.deeplearning4j.ui.model.stats.StatsListener; import org.deeplearning4j.ui.model.storage.mapdb.MapDBStatsStorage; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.lossfunctions.LossFunctions; @@ -44,10 +41,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestStatsListener extends BaseDL4JTest { @Test diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestTransferStatsCollection.java b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestTransferStatsCollection.java similarity index 90% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestTransferStatsCollection.java rename to cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestTransferStatsCollection.java index c647c3e89..5736cdb7a 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestTransferStatsCollection.java +++ b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/stats/TestTransferStatsCollection.java @@ -20,6 +20,7 @@ package org.deeplearning4j.ui.stats; +import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.nn.conf.MultiLayerConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.DenseLayer; @@ -27,27 +28,16 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; import org.deeplearning4j.nn.transferlearning.TransferLearning; -import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.ui.model.stats.StatsListener; -import org.deeplearning4j.ui.model.storage.FileStatsStorage; import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; - -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; - -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.learning.config.Sgd; -import java.io.File; import java.io.IOException; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestTransferStatsCollection extends BaseDL4JTest { @Override diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/storage/TestStatsStorage.java b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/storage/TestStatsStorage.java similarity index 90% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/storage/TestStatsStorage.java rename to cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/storage/TestStatsStorage.java index 008915ec6..0c7fb5ed0 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/src/test/java/org/deeplearning4j/ui/storage/TestStatsStorage.java +++ b/cavis-ui/cavis-ui-model/src/test/java/org/deeplearning4j/ui/storage/TestStatsStorage.java @@ -36,33 +36,25 @@ import org.deeplearning4j.ui.model.stats.impl.java.JavaStatsReport; import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; import org.deeplearning4j.ui.model.storage.mapdb.MapDBStatsStorage; import org.deeplearning4j.ui.model.storage.sqlite.J7FileStatsStorage; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.*; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestStatsStorage extends BaseDL4JTest { - + @TempDir + public File testDir; @Test - @Disabled("AB 2019/05/21 - Failing on linux-x86_64-cuda-9.2 only - Issue #7657") - public void testStatsStorage(@TempDir Path testDir) throws IOException { + //@Ignore("AB 2019/05/21 - Failing on linux-x86_64-cuda-9.2 only - Issue #7657") + public void testStatsStorage() throws IOException { for (boolean useJ7Storage : new boolean[] {false, true}) { for (int i = 0; i < 3; i++) { @@ -70,12 +62,12 @@ public class TestStatsStorage extends BaseDL4JTest { StatsStorage ss; switch (i) { case 0: - File f = createTempFile(testDir,"TestMapDbStatsStore", ".db"); + File f = createTempFile("TestMapDbStatsStore", ".db"); f.delete(); //Don't want file to exist... ss = new MapDBStatsStorage.Builder().file(f).build(); break; case 1: - File f2 = createTempFile(testDir,"TestJ7FileStatsStore", ".db"); + File f2 = createTempFile("TestJ7FileStatsStore", ".db"); f2.delete(); //Don't want file to exist... ss = new J7FileStatsStorage(f2); break; @@ -127,7 +119,7 @@ public class TestStatsStorage extends BaseDL4JTest { assertEquals(Collections.singletonList("wid0"), ss.listWorkerIDsForSession("sid0")); assertEquals(Collections.singletonList("wid0"), ss.listWorkerIDsForSessionAndType("sid0", "tid0")); assertEquals(Collections.singletonList(getReport(0, 0, 0, 12345, useJ7Storage)), - ss.getAllUpdatesAfter("sid0", "tid0", "wid0", 0)); + ss.getAllUpdatesAfter("sid0", "tid0", "wid0", 0)); assertEquals(1, ss.getNumUpdateRecordsFor("sid0")); assertEquals(1, ss.getNumUpdateRecordsFor("sid0", "tid0", "wid0")); @@ -165,17 +157,17 @@ public class TestStatsStorage extends BaseDL4JTest { ss.putUpdate(getReport(100, 200, 300, 12346, useJ7Storage)); assertEquals(Collections.singletonList(getReport(100, 200, 300, 12346, useJ7Storage)), - ss.getLatestUpdateAllWorkers("sid100", "tid200")); + ss.getLatestUpdateAllWorkers("sid100", "tid200")); assertEquals(Collections.singletonList("tid200"), ss.listTypeIDsForSession("sid100")); List temp = ss.listWorkerIDsForSession("sid100"); System.out.println("temp: " + temp); assertEquals(Collections.singletonList("wid300"), ss.listWorkerIDsForSession("sid100")); assertEquals(Collections.singletonList("wid300"), - ss.listWorkerIDsForSessionAndType("sid100", "tid200")); + ss.listWorkerIDsForSessionAndType("sid100", "tid200")); assertEquals(getReport(100, 200, 300, 12346, useJ7Storage), - ss.getLatestUpdate("sid100", "tid200", "wid300")); + ss.getLatestUpdate("sid100", "tid200", "wid300")); assertEquals(getReport(100, 200, 300, 12346, useJ7Storage), - ss.getUpdate("sid100", "tid200", "wid300", 12346)); + ss.getUpdate("sid100", "tid200", "wid300", 12346)); assertEquals(2, l.countNewSession); assertEquals(3, l.countNewWorkerId); @@ -216,16 +208,16 @@ public class TestStatsStorage extends BaseDL4JTest { @Test - @Disabled("AB 2019/05/21 - Failing on linux-x86_64-cuda-9.2 only - Issue #7657") - public void testFileStatsStore(@TempDir Path testDir) throws IOException { + //@Ignore("AB 2019/05/21 - Failing on linux-x86_64-cuda-9.2 only - Issue #7657") + public void testFileStatsStore() throws IOException { for (boolean useJ7Storage : new boolean[] {false, true}) { for (int i = 0; i < 2; i++) { File f; if (i == 0) { - f = createTempFile(testDir,"TestMapDbStatsStore", ".db"); + f = createTempFile("TestMapDbStatsStore", ".db"); } else { - f = createTempFile(testDir,"TestSqliteStatsStore", ".db"); + f = createTempFile("TestSqliteStatsStore", ".db"); } f.delete(); //Don't want file to exist... @@ -277,7 +269,7 @@ public class TestStatsStorage extends BaseDL4JTest { assertEquals(Collections.singletonList("wid0"), ss.listWorkerIDsForSession("sid0")); assertEquals(Collections.singletonList("wid0"), ss.listWorkerIDsForSessionAndType("sid0", "tid0")); assertEquals(Collections.singletonList(getReport(0, 0, 0, 12345, useJ7Storage)), - ss.getAllUpdatesAfter("sid0", "tid0", "wid0", 0)); + ss.getAllUpdatesAfter("sid0", "tid0", "wid0", 0)); assertEquals(1, ss.getNumUpdateRecordsFor("sid0")); assertEquals(1, ss.getNumUpdateRecordsFor("sid0", "tid0", "wid0")); @@ -315,17 +307,17 @@ public class TestStatsStorage extends BaseDL4JTest { ss.putUpdate(getReport(100, 200, 300, 12346, useJ7Storage)); assertEquals(Collections.singletonList(getReport(100, 200, 300, 12346, useJ7Storage)), - ss.getLatestUpdateAllWorkers("sid100", "tid200")); + ss.getLatestUpdateAllWorkers("sid100", "tid200")); assertEquals(Collections.singletonList("tid200"), ss.listTypeIDsForSession("sid100")); List temp = ss.listWorkerIDsForSession("sid100"); System.out.println("temp: " + temp); assertEquals(Collections.singletonList("wid300"), ss.listWorkerIDsForSession("sid100")); assertEquals(Collections.singletonList("wid300"), - ss.listWorkerIDsForSessionAndType("sid100", "tid200")); + ss.listWorkerIDsForSessionAndType("sid100", "tid200")); assertEquals(getReport(100, 200, 300, 12346, useJ7Storage), - ss.getLatestUpdate("sid100", "tid200", "wid300")); + ss.getLatestUpdate("sid100", "tid200", "wid300")); assertEquals(getReport(100, 200, 300, 12346, useJ7Storage), - ss.getUpdate("sid100", "tid200", "wid300", 12346)); + ss.getUpdate("sid100", "tid200", "wid300", 12346)); assertEquals(2, l.countNewSession); assertEquals(3, l.countNewWorkerId); @@ -356,11 +348,11 @@ public class TestStatsStorage extends BaseDL4JTest { assertEquals(Collections.singletonList("tid200"), ss.listTypeIDsForSession("sid100")); assertEquals(Collections.singletonList("wid300"), ss.listWorkerIDsForSession("sid100")); assertEquals(Collections.singletonList("wid300"), - ss.listWorkerIDsForSessionAndType("sid100", "tid200")); + ss.listWorkerIDsForSessionAndType("sid100", "tid200")); assertEquals(getReport(100, 200, 300, 12346, useJ7Storage), - ss.getLatestUpdate("sid100", "tid200", "wid300")); + ss.getLatestUpdate("sid100", "tid200", "wid300")); assertEquals(getReport(100, 200, 300, 12346, useJ7Storage), - ss.getUpdate("sid100", "tid200", "wid300", 12346)); + ss.getUpdate("sid100", "tid200", "wid300", 12346)); } } } @@ -380,7 +372,7 @@ public class TestStatsStorage extends BaseDL4JTest { envInfo.put("envInfo0", "value0"); envInfo.put("envInfo1", "value1"); rep.reportSoftwareInfo("arch", "osName", "jvmName", "jvmVersion", "1.8", "backend", "dtype", "hostname", - "jvmuid", envInfo); + "jvmuid", envInfo); return rep; } @@ -435,11 +427,8 @@ public class TestStatsStorage extends BaseDL4JTest { } } - private File createTempFile(Path testDir, String prefix, String suffix) throws IOException { - File newFile = new File(testDir.toFile(),prefix + "-" + System.nanoTime() + suffix); - newFile.createNewFile(); - newFile.deleteOnExit(); - return newFile; + private File createTempFile(String prefix, String suffix) throws IOException { + return new File(testDir, prefix + "-" + System.nanoTime() + suffix); } } diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/resources/log4j.properties b/cavis-ui/cavis-ui-model/src/test/resources/log4j.properties similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/resources/log4j.properties rename to cavis-ui/cavis-ui-model/src/test/resources/log4j.properties diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/resources/logback.xml b/cavis-ui/cavis-ui-model/src/test/resources/logback.xml similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/src/test/resources/logback.xml rename to cavis-ui/cavis-ui-model/src/test/resources/logback.xml diff --git a/cavis-ui/cavis-ui-standalone/build.gradle b/cavis-ui/cavis-ui-standalone/build.gradle new file mode 100644 index 000000000..b9783b97e --- /dev/null +++ b/cavis-ui/cavis-ui-standalone/build.gradle @@ -0,0 +1,29 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + + +dependencies { + implementation projects.cavisUi.cavisUiCommon + implementation projects.cavisUi.cavisUiModel + implementation projects.cavisUi.cavisUiComponents +// implementation projects.cavisUi.cavisUiVertex + testImplementation projects.cavisDnn.cavisDnnCommonTests +} \ No newline at end of file diff --git a/cavis-ui/cavis-ui-standalone/src/main/java/net/brutex/ai/Dummy.java b/cavis-ui/cavis-ui-standalone/src/main/java/net/brutex/ai/Dummy.java new file mode 100644 index 000000000..3179dc3e5 --- /dev/null +++ b/cavis-ui/cavis-ui-standalone/src/main/java/net/brutex/ai/Dummy.java @@ -0,0 +1,25 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + +package net.brutex.ai; + +public class Dummy { +} diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-standalone/src/main/resources/logback.xml b/cavis-ui/cavis-ui-standalone/src/main/resources/logback.xml similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-standalone/src/main/resources/logback.xml rename to cavis-ui/cavis-ui-standalone/src/main/resources/logback.xml diff --git a/cavis-ui/cavis-ui-vertx/build.gradle b/cavis-ui/cavis-ui-vertx/build.gradle new file mode 100644 index 000000000..e4fb8b77d --- /dev/null +++ b/cavis-ui/cavis-ui-vertx/build.gradle @@ -0,0 +1,89 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation "io.vertx:vertx-core:4.0.2" + implementation "io.vertx:vertx-web:4.0.2" + implementation projects.cavisDnn.cavisDnnCore + implementation projects.cavisDnn.cavisDnnCommon + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisUi.cavisUiModel + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisNative.cavisNativeCommon + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets + + implementation "org.slf4j:slf4j-api" + implementation "it.unimi.dsi:fastutil:8.1.1" + testImplementation 'ch.qos.logback:logback-classic' + implementation "org.freemarker:freemarker:2.3.23" + implementation "com.beust:jcommander:1.27" + implementation 'jakarta.xml.bind:jakarta.xml.bind-api:2.3.2' + testImplementation projects.cavisDnn.cavisDnnCommonTests + implementation 'org.webjars.npm:babel__polyfill:7.4.4' + implementation('org.webjars.npm:coreui__coreui:2.1.9') { + exclude group: 'org.webjars.npm', module: 'coreui__coreui-plugin-npm-postinstall' + } + implementation "commons-io:commons-io" + implementation 'org.webjars.npm:coreui__icons:0.3.0' + implementation 'org.webjars.npm:jquery:3.4.1' + implementation 'org.webjars.bower:popper.js:1.12.9' + implementation 'org.webjars.npm:bootstrap:5.1.2' + implementation 'org.webjars:jquery:2.2.0' + implementation 'org.webjars:jquery-migrate:1.2.1' + implementation 'org.webjars:jquery-ui:1.10.2' + implementation 'org.webjars:modernizr:2.8.3-1' + implementation 'org.webjars:jquery-cookie:1.4.1-1' + implementation 'org.webjars:fullcalendar:1.6.4' + implementation 'org.webjars:excanvas:3' + implementation 'org.webjars.npm:cytoscape:3.3.3' + implementation 'org.webjars.bower:cytoscape-dagre:2.1.0' + implementation 'org.webjars.npm:dagre:0.8.4' + implementation 'org.webjars.npm:cytoscape-cola:2.3.0' + implementation 'org.webjars.npm:cytoscape-cose-bilkent:4.0.0' + implementation 'org.webjars.npm:cytoscape-euler:1.2.1' + implementation 'org.webjars.npm:cytoscape-klay:3.1.2' + implementation 'org.webjars.npm:klayjs:0.4.1' + implementation 'org.webjars.npm:cytoscape-spread:3.0.0' + implementation 'org.webjars.npm:weaverjs:1.2.0' + implementation 'org.webjars:retinajs:0.0.2' + implementation 'org.webjars:flot:0.8.3' + implementation 'org.webjars:chosen:0.9.8' + implementation 'org.webjars:uniform:2.1.2-1' + implementation 'org.webjars:noty:2.2.2' + implementation 'org.webjars:jquery-raty:2.5.2' + implementation 'org.webjars:imagesloaded:2.1.1' + implementation 'org.webjars:masonry:3.1.5' + implementation 'org.webjars:jquery.sparkline:2.1.2' + implementation 'org.webjars:jquery-knob:1.2.2' + implementation 'org.webjars:datatables:1.9.4' + implementation 'org.webjars:jquery-ui-touch-punch:0.2.2' + implementation 'org.webjars:d3js:3.3.5' + implementation 'org.webjars:bootstrap-notify:3.1.3-1' + implementation 'org.webjars.npm:github-com-jboesch-Gritter:1.7.4' + implementation 'org.webjars.bowergithub.stenin-nikita:open-sans:0.1.3' + implementation 'org.webjars:font-awesome:3.0.2' + implementation 'org.webjars:bootstrap-glyphicons:bdd2cbfba0' + implementation 'org.webjars.npm:flatbuffers:1.9.0' + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/VertxUIServer.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/VertxUIServer.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/VertxUIServer.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/VertxUIServer.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/HttpMethod.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/HttpMethod.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/HttpMethod.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/HttpMethod.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/I18N.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/I18N.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/I18N.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/I18N.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/Route.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/Route.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/Route.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/Route.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/UIModule.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/UIModule.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/UIModule.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/UIModule.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/UIServer.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/UIServer.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/api/UIServer.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/api/UIServer.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/i18n/DefaultI18N.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/i18n/DefaultI18N.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/i18n/DefaultI18N.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/i18n/DefaultI18N.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/i18n/I18NProvider.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/i18n/I18NProvider.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/i18n/I18NProvider.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/i18n/I18NProvider.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/i18n/I18NResource.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/i18n/I18NResource.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/i18n/I18NResource.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/i18n/I18NResource.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/SameDiffModule.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/SameDiffModule.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/SameDiffModule.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/SameDiffModule.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/convolutional/ConvolutionalListenerModule.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/convolutional/ConvolutionalListenerModule.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/convolutional/ConvolutionalListenerModule.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/convolutional/ConvolutionalListenerModule.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/defaultModule/DefaultModule.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/defaultModule/DefaultModule.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/defaultModule/DefaultModule.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/defaultModule/DefaultModule.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/remote/RemoteReceiverModule.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/remote/RemoteReceiverModule.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/remote/RemoteReceiverModule.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/remote/RemoteReceiverModule.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java similarity index 99% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java index 3accef9a2..975d78a3f 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java +++ b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModule.java @@ -62,7 +62,7 @@ import org.nd4j.linalg.learning.config.IUpdater; import org.nd4j.common.primitives.Pair; import org.nd4j.common.primitives.Triple; import org.nd4j.common.resources.Resources; -import org.nd4j.shade.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.File; import java.io.StringReader; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModuleUtils.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModuleUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModuleUtils.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/train/TrainModuleUtils.java diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/tsne/TsneModule.java b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/tsne/TsneModule.java similarity index 98% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/tsne/TsneModule.java rename to cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/tsne/TsneModule.java index 4dfce73db..b7602ecb4 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/java/org/deeplearning4j/ui/module/tsne/TsneModule.java +++ b/cavis-ui/cavis-ui-vertx/src/main/java/org/deeplearning4j/ui/module/tsne/TsneModule.java @@ -31,7 +31,7 @@ import org.deeplearning4j.ui.api.HttpMethod; import org.deeplearning4j.ui.api.Route; import org.deeplearning4j.ui.api.UIModule; import org.deeplearning4j.ui.i18n.I18NResource; -import org.nd4j.shade.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.File; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/META-INF/services/org.deeplearning4j.ui.api.UIModule b/cavis-ui/cavis-ui-vertx/src/main/resources/META-INF/services/org.deeplearning4j.ui.api.UIModule similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/META-INF/services/org.deeplearning4j.ui.api.UIModule rename to cavis-ui/cavis-ui-vertx/src/main/resources/META-INF/services/org.deeplearning4j.ui.api.UIModule diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/css/samediff/samediff.css b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/css/samediff/samediff.css similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/css/samediff/samediff.css rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/css/samediff/samediff.css diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/css/style.css b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/css/style.css similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/css/style.css rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/css/style.css diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/img/favicon.ico b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/img/favicon.ico similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/img/favicon.ico rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/img/favicon.ico diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/counter.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/counter.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/counter.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/counter.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/cytoscape-style.json b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/cytoscape-style.json similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/cytoscape-style.json rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/cytoscape-style.json diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/flatbuffers-utils.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/flatbuffers-utils.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/flatbuffers-utils.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/flatbuffers-utils.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/array_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/array_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/array_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/array_generated.js diff --git a/libnd4j/include/graph/generated/config_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/config_generated.js similarity index 100% rename from libnd4j/include/graph/generated/config_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/config_generated.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/graph_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/graph_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/graph_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/graph_generated.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/node_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/node_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/node_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/node_generated.js diff --git a/libnd4j/include/graph/generated/properties_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/properties_generated.js similarity index 100% rename from libnd4j/include/graph/generated/properties_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/properties_generated.js diff --git a/libnd4j/include/graph/generated/request_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/request_generated.js similarity index 100% rename from libnd4j/include/graph/generated/request_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/request_generated.js diff --git a/libnd4j/include/graph/generated/result_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/result_generated.js similarity index 100% rename from libnd4j/include/graph/generated/result_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/result_generated.js diff --git a/libnd4j/include/graph/generated/uigraphevents_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/uigraphevents_generated.js similarity index 100% rename from libnd4j/include/graph/generated/uigraphevents_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/uigraphevents_generated.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/uigraphstatic_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/uigraphstatic_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/uigraphstatic_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/uigraphstatic_generated.js diff --git a/libnd4j/include/graph/generated/utils_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/utils_generated.js similarity index 100% rename from libnd4j/include/graph/generated/utils_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/utils_generated.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/variable_generated.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/variable_generated.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/variable_generated.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/generated/variable_generated.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-graph.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-graph.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-graph.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-graph.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-plots.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-plots.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-plots.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-plots.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-ui.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-ui.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-ui.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/samediff/samediff-ui.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model-graph.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model-graph.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model-graph.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model-graph.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model-layers.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model-layers.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model-layers.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model-layers.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/model.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/overview.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/overview.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/overview.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/overview.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/system.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/system.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/system.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/system.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/train.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/train.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/js/train/train.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/js/train/train.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/common.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/common.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/common.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/common.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/deeplearning4j.img b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/deeplearning4j.img similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/deeplearning4j.img rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/deeplearning4j.img diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/jquery-fileupload.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/jquery-fileupload.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/jquery-fileupload.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/jquery-fileupload.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/render.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/render.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/render.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/render.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/renderTsne.js b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/renderTsne.js similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/renderTsne.js rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/renderTsne.js diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/roboto.css b/cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/roboto.css similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/deeplearning4jUiAssets/legacy/roboto.css rename to cavis-ui/cavis-ui-vertx/src/main/resources/deeplearning4jUiAssets/legacy/roboto.css diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.de b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.de similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.de rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.de diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.en b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.en similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.en rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.en diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.ja b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.ja similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.ja rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.ja diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.ko b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.ko similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.ko rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.ko diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.de b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.de similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.de rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.de diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.en b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.en similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.en rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.en diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.ja b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.ja similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.ja rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.ja diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.ko b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.ko similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.ko rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.ko diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.ru b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.ru similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.ru rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.ru diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.zh b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.zh similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.model.zh rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.model.zh diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.de b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.de similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.de rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.de diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.en b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.en similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.en rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.en diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.ja b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.ja similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.ja rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.ja diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.ko b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.ko similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.ko rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.ko diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.ru b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.ru similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.ru rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.ru diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.zh b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.zh similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.overview.zh rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.overview.zh diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.ru b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.ru similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.ru rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.ru diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.de b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.de similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.de rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.de diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.en b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.en similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.en rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.en diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.ja b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.ja similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.ja rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.ja diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.ko b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.ko similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.ko rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.ko diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.ru b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.ru similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.ru rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.ru diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.zh b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.zh similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.system.zh rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.system.zh diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.zh b/cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.zh similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/dl4j_i18n/train.zh rename to cavis-ui/cavis-ui-vertx/src/main/resources/dl4j_i18n/train.zh diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/Activations.html b/cavis-ui/cavis-ui-vertx/src/main/resources/templates/Activations.html similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/Activations.html rename to cavis-ui/cavis-ui-vertx/src/main/resources/templates/Activations.html diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/SameDiffUI.html b/cavis-ui/cavis-ui-vertx/src/main/resources/templates/SameDiffUI.html similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/SameDiffUI.html rename to cavis-ui/cavis-ui-vertx/src/main/resources/templates/SameDiffUI.html diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/TrainingModel.html.ftl b/cavis-ui/cavis-ui-vertx/src/main/resources/templates/TrainingModel.html.ftl similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/TrainingModel.html.ftl rename to cavis-ui/cavis-ui-vertx/src/main/resources/templates/TrainingModel.html.ftl diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/TrainingOverview.html.ftl b/cavis-ui/cavis-ui-vertx/src/main/resources/templates/TrainingOverview.html.ftl similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/TrainingOverview.html.ftl rename to cavis-ui/cavis-ui-vertx/src/main/resources/templates/TrainingOverview.html.ftl diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/TrainingSystem.html.ftl b/cavis-ui/cavis-ui-vertx/src/main/resources/templates/TrainingSystem.html.ftl similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/TrainingSystem.html.ftl rename to cavis-ui/cavis-ui-vertx/src/main/resources/templates/TrainingSystem.html.ftl diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/Tsne.html b/cavis-ui/cavis-ui-vertx/src/main/resources/templates/Tsne.html similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/main/resources/templates/Tsne.html rename to cavis-ui/cavis-ui-vertx/src/main/resources/templates/Tsne.html diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestRemoteReceiver.java b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestRemoteReceiver.java similarity index 95% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestRemoteReceiver.java rename to cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestRemoteReceiver.java index 442895092..37a7aab14 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestRemoteReceiver.java +++ b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestRemoteReceiver.java @@ -38,11 +38,9 @@ import org.deeplearning4j.ui.model.stats.StatsListener; import org.deeplearning4j.ui.model.stats.impl.SbeStatsInitializationReport; import org.deeplearning4j.ui.model.stats.impl.SbeStatsReport; import org.deeplearning4j.ui.model.storage.impl.SbeStorageMetaData; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.lossfunctions.LossFunctions; @@ -54,15 +52,16 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +//@Ignore public class TestRemoteReceiver extends BaseDL4JTest { + @AfterAll + public void shutdownServer() throws InterruptedException { + UIServer.getInstance().stop(); + } + @Test - @Disabled + //@Ignore public void testRemoteBasic() throws Exception { List updates = new ArrayList<>(); @@ -130,7 +129,7 @@ public class TestRemoteReceiver extends BaseDL4JTest { @Test - @Disabled + //@Ignore public void testRemoteFull() throws Exception { //Use this in conjunction with startRemoteUI() @@ -157,7 +156,7 @@ public class TestRemoteReceiver extends BaseDL4JTest { } @Test - @Disabled + //@Ignore public void startRemoteUI() throws Exception { UIServer s = UIServer.getInstance(); diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestSameDiffUI.java b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestSameDiffUI.java similarity index 87% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestSameDiffUI.java rename to cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestSameDiffUI.java index 14ba10f66..1e59c08e8 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestSameDiffUI.java +++ b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestSameDiffUI.java @@ -20,39 +20,43 @@ package org.deeplearning4j.ui; +import lombok.extern.slf4j.Slf4j; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.ui.api.UIServer; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + + +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.io.TempDir; import org.nd4j.autodiff.samediff.SDVariable; import org.nd4j.autodiff.samediff.SameDiff; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.graph.ui.LogFileWriter; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; -import java.nio.file.Path; import java.util.Arrays; -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +//@Ignore +@Slf4j public class TestSameDiffUI extends BaseDL4JTest { - private static Logger log = LoggerFactory.getLogger(TestSameDiffUI.class.getName()); - @Disabled + @TempDir + public File testDir; + + + @AfterAll + public void shutdownServer() throws InterruptedException { + UIServer.getInstance().stop(); + } + + + //@Ignore @Test - public void testSameDiff(@TempDir Path testDir) throws Exception { - File dir = testDir.toFile(); + public void testSameDiff() throws Exception { + File dir = testDir; File f = new File(dir, "ui_data.bin"); log.info("File path: {}", f.getAbsolutePath()); diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUI.java b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUI.java similarity index 96% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUI.java rename to cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUI.java index 4919322b4..988b9b502 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUI.java +++ b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUI.java @@ -23,6 +23,7 @@ package org.deeplearning4j.ui; import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.Vertx; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.core.storage.StatsStorage; @@ -44,42 +45,39 @@ import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.deeplearning4j.ui.api.UIServer; import org.deeplearning4j.ui.model.stats.StatsListener; import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.common.function.Function; import org.nd4j.linalg.learning.config.Sgd; import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.*; -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Slf4j +//@Ignore public class TestVertxUI extends BaseDL4JTest { - private static Logger log = LoggerFactory.getLogger(TestVertxUI.class.getName()); - @BeforeEach public void setUp() throws Exception { UIServer.stopInstance(); } + @AfterAll + public void shutdownServer() throws InterruptedException { + UIServer.getInstance().stop(); + } + @Test public void testUI() throws Exception { VertxUIServer uiServer = (VertxUIServer) UIServer.getInstance(); @@ -315,15 +313,17 @@ public class TestVertxUI extends BaseDL4JTest { uiServer.stop(); } - @Test () + @Test public void testUIStartPortAlreadyBound() throws InterruptedException { - assertThrows(DL4JException.class,() -> { + assertThrows(DL4JException.class, () -> { CountDownLatch latch = new CountDownLatch(1); + //Create HttpServer that binds the same port int port = VertxUIServer.DEFAULT_UI_PORT; Vertx vertx = Vertx.vertx(); vertx.createHttpServer() - .requestHandler(event -> {}) + .requestHandler(event -> { + }) .listen(port, result -> latch.countDown()); latch.await(); @@ -334,7 +334,6 @@ public class TestVertxUI extends BaseDL4JTest { vertx.close(); } }); - } @Test diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIManual.java b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIManual.java similarity index 95% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIManual.java rename to cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIManual.java index cf457071c..eb3d19c51 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIManual.java +++ b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIManual.java @@ -38,18 +38,14 @@ import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.deeplearning4j.ui.api.UIServer; import org.deeplearning4j.ui.model.stats.StatsListener; import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.common.function.Function; import org.nd4j.linalg.learning.config.Sgd; import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; @@ -60,23 +56,22 @@ import java.util.concurrent.CountDownLatch; import static org.junit.jupiter.api.Assertions.*; -@Disabled -@Tag(TagNames.FILE_IO) -@Tag(TagNames.UI) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +@Slf4j +//@Ignore public class TestVertxUIManual extends BaseDL4JTest { - private static Logger log = LoggerFactory.getLogger(TestVertxUIManual.class.getName()); - - @Override public long getTimeoutMilliseconds() { return 3600_000L; } + @AfterAll + public void shutdownServer() throws InterruptedException { + UIServer.getInstance().stop(); + } + @Test - @Disabled + //@Ignore public void testUI() throws Exception { VertxUIServer uiServer = (VertxUIServer) UIServer.getInstance(); assertEquals(9000, uiServer.getPort()); @@ -86,7 +81,7 @@ public class TestVertxUIManual extends BaseDL4JTest { } @Test - @Disabled + //@Ignore public void testUISequentialSessions() throws Exception { UIServer uiServer = UIServer.getInstance(); StatsStorage ss = null; @@ -129,7 +124,7 @@ public class TestVertxUIManual extends BaseDL4JTest { } @Test - @Disabled + //@Ignore public void testUIServerStop() throws Exception { UIServer uiServer = UIServer.getInstance(true, null); assertTrue(uiServer.isMultiSession()); @@ -155,7 +150,7 @@ public class TestVertxUIManual extends BaseDL4JTest { @Test - @Disabled + //@Ignore public void testUIServerStopAsync() throws Exception { UIServer uiServer = UIServer.getInstance(true, null); assertTrue(uiServer.isMultiSession()); @@ -187,7 +182,7 @@ public class TestVertxUIManual extends BaseDL4JTest { } @Test - @Disabled + //@Ignore public void testUIAutoAttachDetach() throws Exception { long detachTimeoutMillis = 15_000; AutoDetachingStatsStorageProvider statsProvider = new AutoDetachingStatsStorageProvider(detachTimeoutMillis); diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java similarity index 91% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java rename to cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java index 2a6f70c8b..5a774dceb 100644 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java +++ b/cavis-ui/cavis-ui-vertx/src/test/java/org/deeplearning4j/ui/TestVertxUIMultiSession.java @@ -36,19 +36,15 @@ import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.deeplearning4j.ui.api.UIServer; import org.deeplearning4j.ui.model.stats.StatsListener; import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.common.function.Function; import org.nd4j.linalg.learning.config.Adam; import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -62,19 +58,19 @@ import static org.junit.jupiter.api.Assertions.*; /** * @author Tamas Fenyvesi */ - @Disabled //https://github.com/eclipse/deeplearning4j/issues/8891 - @Tag(TagNames.FILE_IO) - @Tag(TagNames.UI) - @Tag(TagNames.DIST_SYSTEMS) - @NativeTag +@Slf4j //@Ignore //https://github.com/eclipse/deeplearning4j/issues/8891 public class TestVertxUIMultiSession extends BaseDL4JTest { - private static Logger log = LoggerFactory.getLogger(TestVertxUIMultiSession.class.getName()); @BeforeEach public void setUp() throws Exception { UIServer.stopInstance(); } + @AfterAll + public void shutdownServer() throws InterruptedException { + UIServer.getInstance().stop(); + } + @Test public void testUIMultiSessionParallelTraining() throws Exception { UIServer uIServer = UIServer.getInstance(true, null); @@ -194,26 +190,23 @@ public class TestVertxUIMultiSession extends BaseDL4JTest { } } - @Test () + @Test public void testUIServerGetInstanceMultipleCalls1() { - assertThrows(DL4JException.class,() -> { - UIServer uiServer = UIServer.getInstance(); - assertFalse(uiServer.isMultiSession()); - UIServer.getInstance(true, null); - }); - - + assertThrows(DL4JException.class, () -> { + UIServer uiServer = UIServer.getInstance(); + assertFalse(uiServer.isMultiSession()); + UIServer.getInstance(true, null); + }); } - @Test () + @Test public void testUIServerGetInstanceMultipleCalls2() { - assertThrows(DL4JException.class,() -> { + assertThrows(DL4JException.class, () -> { UIServer uiServer = UIServer.getInstance(true, null); assertTrue(uiServer.isMultiSession()); UIServer.getInstance(false, null); }); - } /** diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/resources/logback.xml b/cavis-ui/cavis-ui-vertx/src/test/resources/logback.xml similarity index 100% rename from deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/src/test/resources/logback.xml rename to cavis-ui/cavis-ui-vertx/src/test/resources/logback.xml diff --git a/cavis-zoo/build.gradle b/cavis-zoo/build.gradle new file mode 100644 index 000000000..a4eb88732 --- /dev/null +++ b/cavis-zoo/build.gradle @@ -0,0 +1,10 @@ +subprojects { + group = "net.brutex.cavis-zoo" + + apply plugin: "java-library" + apply plugin: "maven-publish" + apply plugin: "signing" + + + +} \ No newline at end of file diff --git a/cavis-zoo/cavis-zoo-models/build.gradle b/cavis-zoo/cavis-zoo-models/build.gradle new file mode 100644 index 000000000..56b674d37 --- /dev/null +++ b/cavis-zoo/cavis-zoo-models/build.gradle @@ -0,0 +1,41 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +apply from: "${project.rootProject.projectDir}/createTestBackends.gradle" + +dependencies { + implementation 'org.slf4j:slf4j-api' + implementation projects.cavisDnn.cavisDnnApi + implementation projects.cavisDnn.cavisDnnNn + implementation projects.cavisDnn.cavisDnnCommon + testImplementation 'ch.qos.logback:logback-classic' + testImplementation projects.cavisDnn.cavisDnnCore + testImplementation projects.cavisDnn.cavisDnnCommonTests + testImplementation projects.cavisDatavec.cavisDatavecData.cavisDatavecDataImage + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataDatasets + testImplementation projects.cavisDnn.cavisDnnData.cavisDnnDataUtilityIterators + implementation "commons-io:commons-io" + testImplementation "org.apache.commons:commons-compress" + implementation "com.fasterxml.jackson.core:jackson-core" + implementation "com.fasterxml.jackson.core:jackson-annotations" + implementation "com.fasterxml.jackson.core:jackson-databind" + testImplementation "org.bytedeco:opencv" + testImplementation "org.bytedeco:javacv" +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/InstantiableModel.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/InstantiableModel.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/InstantiableModel.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/InstantiableModel.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/ModelMetaData.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/ModelMetaData.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/ModelMetaData.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/ModelMetaData.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/PretrainedType.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/PretrainedType.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/PretrainedType.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/PretrainedType.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/ZooModel.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/ZooModel.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/ZooModel.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/ZooModel.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/ZooType.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/ZooType.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/ZooType.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/ZooType.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/AlexNet.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/AlexNet.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/AlexNet.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/AlexNet.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/Darknet19.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/Darknet19.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/Darknet19.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/Darknet19.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/FaceNetNN4Small2.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/FaceNetNN4Small2.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/FaceNetNN4Small2.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/FaceNetNN4Small2.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/InceptionResNetV1.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/InceptionResNetV1.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/InceptionResNetV1.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/InceptionResNetV1.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/LeNet.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/LeNet.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/LeNet.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/LeNet.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/NASNet.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/NASNet.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/NASNet.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/NASNet.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/ResNet50.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/ResNet50.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/ResNet50.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/ResNet50.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/SimpleCNN.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/SimpleCNN.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/SimpleCNN.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/SimpleCNN.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/SqueezeNet.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/SqueezeNet.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/SqueezeNet.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/SqueezeNet.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/TextGenerationLSTM.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/TextGenerationLSTM.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/TextGenerationLSTM.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/TextGenerationLSTM.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/TinyYOLO.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/TinyYOLO.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/TinyYOLO.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/TinyYOLO.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/UNet.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/UNet.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/UNet.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/UNet.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/VGG16.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/VGG16.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/VGG16.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/VGG16.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/VGG19.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/VGG19.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/VGG19.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/VGG19.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/Xception.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/Xception.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/Xception.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/Xception.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/YOLO2.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/YOLO2.java old mode 100755 new mode 100644 similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/YOLO2.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/YOLO2.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/helper/DarknetHelper.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/helper/DarknetHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/helper/DarknetHelper.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/helper/DarknetHelper.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/helper/FaceNetHelper.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/helper/FaceNetHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/helper/FaceNetHelper.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/helper/FaceNetHelper.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/helper/InceptionResNetHelper.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/helper/InceptionResNetHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/helper/InceptionResNetHelper.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/helper/InceptionResNetHelper.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/helper/NASNetHelper.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/helper/NASNetHelper.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/model/helper/NASNetHelper.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/model/helper/NASNetHelper.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/BaseLabels.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/BaseLabels.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/BaseLabels.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/BaseLabels.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/ClassPrediction.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/ClassPrediction.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/ClassPrediction.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/ClassPrediction.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/Labels.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/Labels.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/Labels.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/Labels.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/darknet/COCOLabels.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/darknet/COCOLabels.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/darknet/COCOLabels.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/darknet/COCOLabels.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/darknet/DarknetLabels.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/darknet/DarknetLabels.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/darknet/DarknetLabels.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/darknet/DarknetLabels.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/darknet/VOCLabels.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/darknet/VOCLabels.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/darknet/VOCLabels.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/darknet/VOCLabels.java diff --git a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/imagenet/ImageNetLabels.java b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/imagenet/ImageNetLabels.java similarity index 98% rename from deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/imagenet/ImageNetLabels.java rename to cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/imagenet/ImageNetLabels.java index 95bd3ef57..1064ad999 100644 --- a/deeplearning4j/deeplearning4j-zoo/src/main/java/org/deeplearning4j/zoo/util/imagenet/ImageNetLabels.java +++ b/cavis-zoo/cavis-zoo-models/src/main/java/org/deeplearning4j/zoo/util/imagenet/ImageNetLabels.java @@ -25,7 +25,7 @@ import org.deeplearning4j.zoo.util.BaseLabels; import org.nd4j.common.base.Preconditions; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; diff --git a/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/MiscTests.java b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/MiscTests.java new file mode 100644 index 000000000..0bc1572e7 --- /dev/null +++ b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/MiscTests.java @@ -0,0 +1,74 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.zoo; + +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.layers.OutputLayer; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.transferlearning.TransferLearning; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.zoo.model.VGG16; + +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.lossfunctions.LossFunctions; + +import java.io.File; +//@Ignore("Times out too often") +public class MiscTests extends BaseDL4JTest { + + @Override + public long getTimeoutMilliseconds() { + return Long.MAX_VALUE; + } + + @Test + public void testTransferVGG() throws Exception { + DataSet ds = new DataSet(); + ds.setFeatures(Nd4j.create(1, 3, 224, 224)); + ds.setLabels(Nd4j.create(1, 2)); + + ComputationGraph model = (ComputationGraph)( + VGG16.builder().build() + .initPretrained(PretrainedType.IMAGENET)); +// System.out.println(model.summary()); + + ComputationGraph transferModel = new TransferLearning.GraphBuilder(model) + .setFeatureExtractor("fc2") + .removeVertexKeepConnections("predictions") + .addLayer("predictions", + new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nIn(4096).nOut(2) + .weightInit(WeightInit.XAVIER) + .activation(Activation.SOFTMAX).build(), "fc2") + .build(); + +// System.out.println(transferModel.summary()); +// System.out.println("Fitting"); + transferModel.fit(ds); + + ComputationGraph g2 = TestUtils.testModelSerialization(transferModel); + g2.fit(ds); + } + +} diff --git a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestDownload.java b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestDownload.java similarity index 87% rename from deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestDownload.java rename to cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestDownload.java index 308fd0e95..41c8410ea 100644 --- a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestDownload.java +++ b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestDownload.java @@ -31,14 +31,16 @@ import org.deeplearning4j.zoo.model.UNet; import org.deeplearning4j.zoo.util.darknet.COCOLabels; import org.deeplearning4j.zoo.util.darknet.DarknetLabels; import org.deeplearning4j.zoo.util.imagenet.ImageNetLabels; -import org.junit.jupiter.api.*; + + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; import java.io.File; -import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -47,25 +49,18 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) +//@Ignore("Times out too often") +@Timeout(300) public class TestDownload extends BaseDL4JTest { + @TempDir - static Path sharedTempDir; - - @Override - public long getTimeoutMilliseconds() { - return isIntegrationTests() ? 480000L : 60000L; - } - - + public static File testDir; + private static File f; @BeforeAll public static void before() throws Exception { - DL4JResources.setBaseDirectory(sharedTempDir.toFile()); + f = testDir; + DL4JResources.setBaseDirectory(f); } @AfterAll diff --git a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestImageNet.java b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestImageNet.java similarity index 95% rename from deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestImageNet.java rename to cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestImageNet.java index 584204f0e..8be3a65cc 100644 --- a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestImageNet.java +++ b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestImageNet.java @@ -37,11 +37,8 @@ import org.deeplearning4j.zoo.util.darknet.COCOLabels; import org.deeplearning4j.zoo.util.darknet.DarknetLabels; import org.deeplearning4j.zoo.util.darknet.VOCLabels; import org.deeplearning4j.zoo.util.imagenet.ImageNetLabels; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization; @@ -57,11 +54,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) +//@Ignore("Times out too often") public class TestImageNet extends BaseDL4JTest { @Override @@ -99,7 +92,7 @@ public class TestImageNet extends BaseDL4JTest { } @Test - @Disabled("AB 2019/05/30 - Failing (intermittently?) on CI linux - see issue 7657") + //@Ignore("AB 2019/05/30 - Failing (intermittently?) on CI linux - see issue 7657") public void testDarknetLabels() throws IOException { // set up model ZooModel model = Darknet19.builder().numClasses(0).build(); //num labels doesn't matter since we're getting pretrained imagenet diff --git a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestInstantiation.java b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestInstantiation.java similarity index 94% rename from deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestInstantiation.java rename to cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestInstantiation.java index a1d0f003c..9abe0b848 100644 --- a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestInstantiation.java +++ b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestInstantiation.java @@ -35,11 +35,8 @@ import org.deeplearning4j.nn.transferlearning.TransferLearningHelper; import org.deeplearning4j.zoo.model.*; import org.deeplearning4j.zoo.model.helper.DarknetHelper; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -50,16 +47,12 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; import java.io.IOException; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; @Slf4j -@Disabled("Times out too often") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -@Tag(TagNames.LONG_TEST) +////@Ignore("Times out too often") public class TestInstantiation extends BaseDL4JTest { protected static void ignoreIfCuda(){ @@ -93,7 +86,7 @@ public class TestInstantiation extends BaseDL4JTest { runTest(TinyYOLO.builder().numClasses(10).build(), "TinyYOLO", 10); } - @Test @Disabled("AB 2019/05/28 - Crashing on CI linux-x86-64 CPU only - Issue #7657") + @Test //@Ignore("AB 2019/05/28 - Crashing on CI linux-x86-64 CPU only - Issue #7657") public void testCnnTrainingYOLO2() throws Exception { runTest(YOLO2.builder().numClasses(10).build(), "YOLO2", 10); } @@ -169,12 +162,12 @@ public class TestInstantiation extends BaseDL4JTest { testInitPretrained(VGG19.builder().numClasses(0).build(), new long[]{1,3,224,224}, new long[]{1,1000}); } - @Test @Disabled("AB 2019/05/28 - JVM crash on linux CUDA CI machines - Issue 7657") + @Test //@Ignore("AB 2019/05/28 - JVM crash on linux CUDA CI machines - Issue 7657") public void testInitPretrainedDarknet19() throws Exception { testInitPretrained(Darknet19.builder().numClasses(0).build(), new long[]{1,3,224,224}, new long[]{1,1000}); } - @Test @Disabled("AB 2019/05/28 - JVM crash on linux CUDA CI machines - Issue 7657") + @Test //@Ignore("AB 2019/05/28 - JVM crash on linux CUDA CI machines - Issue 7657") public void testInitPretrainedDarknet19S2() throws Exception { testInitPretrained(Darknet19.builder().numClasses(0).inputShape(new int[]{3,448,448}).build(), new long[]{1,3,448,448}, new long[]{1,1000}); } @@ -247,7 +240,7 @@ public class TestInstantiation extends BaseDL4JTest { testInitRandomModel(Xception.builder().numClasses(1000).build(), new long[]{1,3,299,299}, new long[]{1,1000}); } - @Test @Disabled("AB - 2019/05/28 - JVM crash on CI - intermittent? Issue 7657") + @Test //@Ignore("AB - 2019/05/28 - JVM crash on CI - intermittent? Issue 7657") public void testInitRandomModelSqueezenet() throws IOException { testInitRandomModel(SqueezeNet.builder().numClasses(1000).build(), new long[]{1,3,227,227}, new long[]{1,1000}); } @@ -257,7 +250,7 @@ public class TestInstantiation extends BaseDL4JTest { testInitRandomModel(FaceNetNN4Small2.builder().embeddingSize(100).numClasses(10).build(), new long[]{1,3,64,64}, new long[]{1,10}); } - @Test @Disabled("AB 2019/05/29 - Crashing on CI linux-x86-64 CPU only - Issue #7657") + @Test //@Ignore("AB 2019/05/29 - Crashing on CI linux-x86-64 CPU only - Issue #7657") public void testInitRandomModelUNet() throws IOException { testInitRandomModel(UNet.builder().build(), new long[]{1,3,512,512}, new long[]{1,1,512,512}); } @@ -288,7 +281,7 @@ public class TestInstantiation extends BaseDL4JTest { @Test public void testYolo4635() throws Exception { ignoreIfCuda(); - //https://github.com/eclipse/deeplearning4j/issues/4635 + //https://github.com/deeplearning4j/deeplearning4j/issues/4635 int nClasses = 10; TinyYOLO model = TinyYOLO.builder().numClasses(nClasses).build(); @@ -299,7 +292,7 @@ public class TestInstantiation extends BaseDL4JTest { @Test public void testTransferLearning() throws Exception { ignoreIfCuda(); - //https://github.com/eclipse/deeplearning4j/issues/7193 + //https://github.com/deeplearning4j/deeplearning4j/issues/7193 ComputationGraph cg = (ComputationGraph) ResNet50.builder().build().initPretrained(); diff --git a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestUtils.java b/cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestUtils.java similarity index 100% rename from deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/TestUtils.java rename to cavis-zoo/cavis-zoo-models/src/test/java/org/deeplearning4j/zoo/TestUtils.java diff --git a/change-cuda-versions.sh b/change-cuda-versions.sh old mode 100755 new mode 100644 index 23fc2850b..f402b01e4 --- a/change-cuda-versions.sh +++ b/change-cuda-versions.sh @@ -55,7 +55,7 @@ check_cuda_version "$VERSION" case $VERSION in 11.2) VERSION2="8.1" - VERSION3="1.5.5" + VERSION3="1.5.5-SNAPSHOT" ;; 11.1) VERSION2="8.0" diff --git a/change-scala-versions.sh b/change-scala-versions.sh old mode 100755 new mode 100644 diff --git a/chooseBackend.gradle b/chooseBackend.gradle new file mode 100644 index 000000000..258650a05 --- /dev/null +++ b/chooseBackend.gradle @@ -0,0 +1,31 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ +ext { + chip = (properties.CAVIS_CHIP ?: "cuda,cpu").toLowerCase() + chipList = chip.split(",") + + withCuda = { -> + return chip.contains("cuda") + } + withCpu = { -> + return chip.contains("cpu") + } +} diff --git a/contrib/codegen-tools/codegen/README.md b/contrib/codegen-tools/codegen/README.md deleted file mode 100644 index b2bc4f915..000000000 --- a/contrib/codegen-tools/codegen/README.md +++ /dev/null @@ -1,614 +0,0 @@ -# ND4J Op Definitions and Code Generation -This project contains the ND4J Op definitions, the DSL (Domain Specific Language) that is used for those definitions and -code generators that use those definitions to create the actual Java code that is used to use the defined operations. - - -## Why define ops externally? -As we started to support SameDiff, we also started to introduce inconsistencies between SameDiff and ND4J. Even though -both of those libraries use the same underlying implementations for operations, there are both small and large -differences in the API that we provide for them. Sometimes, we have provided an official API only for one usage, and not -the other. And very often the documentation for a single op is in many different places. - -In the future we want to support other programming languages with libnd4j, and provide more ways to use our C++ backend. -This would only increase the aforementioned problems. - -The root of all of those problems, is that Ops are used across different environments, and there is no single way of -defining them with an enforced interface. - - -## How does this project help with enforcing a single consistent interface for ops? -The solution we propose, is to define the operations separately, and then generate the necessary API code for them. All -of the generated code is to be considered untouchable, editing it will result in the changes being overwritten sooner -rather than later. - -The combination of external op definition and code generation, opens up many opportunities for us. The first one being -that we can easily create consistent APIs for both ND4J and SameDiff in Java. But, looking into the future, we can also -create those APIs for other programming languages like Python, Swift, or even C#. We can even go beyond programming -languages, and use the op definitions to create better documentation than what JavaDoc or similar might support out of -the box. - -## Maintenance -This project is currently maintained by Paul Dubs, with feedback often collected from raver119 and Alex Black. - -## Current Status -At the moment we still focus on nailing down an easily readable and contribution friendly DSL for op definition and code -generation that can replace namespace definitions. This means that at the moment we still rely on the pre-existing Op -definition classes that already exist in ND4J. - -## Roadmap -* Replace Bitwise and Random namespaces with autogenerated code – In progress. -* Implement a convenient CLI tool. -* Define all Ops using the DSL. -* Automatically generate derivative op declarations from existing ops -* Replace all namespace definitions in ND4J / SameDiff with automatically generated ones -* Replace all Op classes with automatically generated ones. - -# Usage -Pre-requisites: -* JDK 8 or higher -* Maven 3.3 or higher - -TODO: Show usage output of the project itself - -TODO: Show how to use from mvn - - -## Generating Code - ND4J Namespaces - -A script - `generate.sh` - is provided in the project root. This can be used (at present) to generate ND4J namespace classes. -It is assumed that the deeplearning4j mono repo and the dl4j-dev-tools repo both exist and have a common parent directory -i.e., `somedir/deeplearning4j` and `somedir/dl4j-dev-tools` both exist. - -The script takes as argument the name (or names) of the ND4J namespaces to generate (not case sensitive) and projects (supported -projects are nd4j, sd and both by default). - -As of 26/11, namespaces names (and hence valid args) include: `bitwise`, `neuralnetwork`, `random`, and `math` -Note also that `all` may be passed to the script to generate all namespaces. - -For example, to generate both bitwise and random namespaces for both nd4j and SameDiff: -``` -./generate.sh bitwise,random -``` -Or to generate all namespaces for both nd4j and SameDiff, use: -``` -./generate.sh all -``` -To generate namespaces for one project only, use: -``` -./generate.sh linalg -projects sd -``` -or: -``` -./generate.sh linalg -projects nd4j -``` -The script will first compile the project, before running. -Internally, the `org.nd4j.codegen.cli.CLI` class is used. -Classes are written to `deeplearning4j/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/ops/` - -## Generating documentation. -It is possible to use generate.sh for generation of code only, docs in markdown format only, or both docs and code. -To generate docs only and store them to new folder "docs" for all namespaces: -``` -./generate.sh all -docsdir ../../docs -``` -Generation for selected namespaces works in the same way as for code: -``` -./generate.sh -docsdir ../../docs bitwise,linalg -``` - -# Code structure -The project is implemented using a mix of Java and Kotlin. The DSL definition and the accompanying data structures are -implemented in Kotlin. At the moment the code generators are implemented in Java, in order to allow people who are not -fluent in Kotlin, but know Java to be able to contribute to the code generators. - -The source code for this project is structured a bit different that what you would typically see in a Java or Kotlin -project. When you take a look inside the `src/main` directory, you will find 4 sub-directories. - -The `java` and `kotlin` directories contain Java and Kotlin code respectively. - -In order to not confuse op definitions with the machinery that allows them to be defined in that way, ops are kept in a -separate folder called `ops`. - -Because we use JavaPoet for Java code generator implementation, we also have yet another folder called `stubs`. That -folder contains stub classes, that are used to reference other classes available in ND4J. These stub classes are -intentionally left empty, as JavaPoet only requires them for naming and automatically creating proper imports. We use -stub classes instead of depending on the actual nd4j API in order to break a cyclic dependency that would otherwise be -created (i.e. in order to be able to generate code for ND4J, we would need an already compiled nd4j to be available). -**Note:** If something is stubbed here and is moved in ND4J, then it also has to be moved to the appropriate place here, -otherwise the generated code will be wrong. - -The `adr` folder contains "Architecture Decision Records". These files give you more insight into the "why" of some of -the bigger decisions within this project. - -# DSL for Op Definition -Ops are defined using a DSL that is implemented in Kotlin. This means that other than the DSL, as defined in the -following, you can also use all of Kotlin when defining Ops. However, doing things the obvious and clearly -understandable way is better than coming up with a clever way, so prefer to use the DSL as described if unsure. - -```kotlin -val mathNs = Namespace("math") { - Op("add") { - javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic" - - Input(NUMERIC, "x") { description = "First input to add" } - Input(NUMERIC,"y") { count = AtLeast(1); description = "Second input to add" } - Arg(INT,"shape") { count = AtLeast(1); description = "shape" } - - - Output(NUMERIC, "z") { description = "Output (x+y)" } - - Doc(Language.ANY, DocScope.ALL) { - """ - (From AddOp) Add op doc text that will appear everywhere - classes, constructors, op creators - """.trimIndent() - } - Doc(Language.ANY, DocScope.CLASS_DOC_ONLY) { - "Add op doc text that will appear in all class docs (javadoc etc)" - } - Doc(Language.ANY, DocScope.CONSTRUCTORS_ONLY) { - "Add op doc text for constructors only" - } - - } -} -``` - -This example shows how a namespace is defined. Namespaces are at the top layer, and ops can only be defined within the -context of a namespace. This example namespace contains only a single op, called "add". If we wanted to add another op, -we would simply add it below the first. - -As you can see, every op has to have a name, if you try to create one without a name, you will get a compile error. -Within the context of the op, we first set in which java package the op class can be found in, then define its inputs, -arguments and outputs and finally add some free form documentation about what that op is doing. - -Like with the op itself, the inputs, arguments and outputs all have to have a name, but unlike the op, they also require -a type. Within their context, you can set a description and a count of how many parameters they can take respectively. - -If an input, argument or output take anything else than exactly 1, they will be treated as arrays. Typically you would -use this to define ops like `concat` which can take multiple input tensors or ops that might take shape arguments. - -## Examples -The following shows how a typical op definition looks like and how the generated Java code may look. - -An op might be defined like this: - -```kotlin -Op("binomial") { - javaPackage = "org.nd4j.linalg.api.ops.random.custom" - Arg(INT, "nTrials") { description = "Number of trials parameter for the binomial distribution" } - Arg(FLOATING_POINT, "p") { description = "Probability of success for each trial" } - Arg(INT, "shape") { count = AtLeast(1); description = "Shape of the new random SDVariable, as a 1D array" } - - Output(NUMERIC, "output") { description = "new random SDVariable, where values are randomly sampled according to a Binomial distribution" } - - Doc(Language.ANY, DocScope.ALL) { - """ - Generate a new random SDVariable, where values are randomly sampled according to a Binomial distribution, - with the specified number of trials and probability. - """.trimIndent() - } -} -``` - -The java code generator will create a method like the following for it: -```java - /** - * Generate a new random SDVariable, where values are randomly sampled according to a Binomial distribution, - * with the specified number of trials and probability. - * - * @param nTrials Number of trials parameter for the binomial distribution - * @param p Probability of success for each trial - * @param shape Shape of the new random SDVariable, as a 1D array (Size: AtLeast(min=1)) - * @return output new random SDVariable, where values are randomly sampled according to a Binomial distribution (NUMERIC type) - */ - public static INDArray binomial(long nTrials, double p, long... shape) { - Preconditions.checkArgument(shape.length >= 1, "shape has incorrect count. Expected: AtLeast(min=1)"); - return Nd4j.exec(new org.nd4j.linalg.api.ops.random.custom.BinomialOp(nTrials, p, shape))[0]; - } -``` - -Or an op with some more constraints: - -```kotlin -Op("and") { - javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom" - val x = Input(INT, "x") { description = "First input array" } - val y = Input(INT, "y") { description = "Second input array" } - Constraint("Must be same types"){ sameType(x, y) } - Constraint("Must have broadcastable shapes"){ broadcastableShapes(x, y) } - - Output(INT, "output"){ description = "Bitwise AND array" } - - Doc(Language.ANY, DocScope.ALL){ - """ - Bitwise AND operation. Supports broadcasting. - """.trimIndent() - } -} -``` - -will be converted to java like this: - -```java - /** - * Bitwise AND operation. Supports broadcasting. - * - * Inputs must satisfy the following constraints: - * Must be same types: isSameType(x, y) - * Must have broadcastable shapes: isBroadcastableShapes(x, y) - * - * @param x First input array (INT type) - * @param y Second input array (INT type) - * @return output Bitwise AND array (INT type) - */ - public static INDArray and(INDArray x, INDArray y) { - NDValidation.validateInteger("and", x); - NDValidation.validateInteger("and", y); - Preconditions.checkArgument(isSameType(x, y), "Must be same types"); - Preconditions.checkArgument(isBroadcastableShapes(x, y), "Must have broadcastable shapes"); - return Nd4j.exec(new org.nd4j.linalg.api.ops.impl.transforms.custom.AndOp(x, y))[0]; - } -``` - -# Full DSL Description -## Namespace - - fun NamespaceName() = Namespace("name"){ /* Op definitions in namespace context */} - -Defines a namespace. - -## Op -Only available within a namespace context - - Op("opName") { /* op properties in op context */ } - Op("anotherOp", mixin) { /* op properties in op context */ } - Op("anotherOp2", mixin, keepInputs=false) { /* op properties in op context */ } - -Every op requires a namespace unique op name. - -When defining an op, you can also pass a mixin that it should inherit initial properties from. This has the same effect -as using `useMixin(mixin)` as the very first thing in the op definition. If you don't want to inherit all of the -parameters of the mixin, you can pass the same additional configuration as you would pass to -`useMixin(mixin, ...options..)`. See [Mixin](#mixin) for more information. - -### Op properties -* `javaPackage` (String): Package where the op is to be found in the java implementation. -* `javaOpClass` (String): Name of java op class if inconsistent with opName. Default: same as opName -* `libnd4jName` (String): The name the op has in libnd4j. Default: same as opName - - -## Mixin -Available in global context. - - val mixin = Mixin("name"){ /* op properties in op context */ } - // within an op context: - useMixin(mixin) - useMixin(mixin, ...options...) - - // When needing to access something from within the mixin - mixin.input("name") - mixin.arg("name") - mixin.config("name") - mixin.output("name") - -Mixins provide the facility to share commonalities between Ops. You can think of it like inheritance, especially when -you declare the use of a mixin on Op definition. In contrast to normal (single) inheritance where only a single super -class is possible, the mixin mechanism allows to "inherit" from multiple sources. - -You can define almost all the same things within a mixin that you can within an Op. The only things that *can not* be -configured within a mixin are Op `name`, `libnd4jName` and `javaOpClass`. - -As mixins can be configured within the global context, you can share them across namespaces by defining them in their -own file. If a mixin is namespace specific, you can also define it within the namespace context. - -Mixins are used either on definition as a parameter `Op("opname", mixin){...}`, or with `useMixin(mixin)` within the op -definition. While the former version only supports a single mixin, the latter version allows you to use as many mixins -as are required. - -You can also build up mixins by using `useMixin(mixin)` inside a Mixin itself. - -`useMixin(mixin, ...options...)` supports a few additional options: `keepInputs`, `keepArgs`, `keepConfigs`, -`keepOutputs`, `keepSignatures`, `keepDoc`, `keepConstraints`. They default to `true`. If you want to skip including -some of them, you simply set the parameter for it to `false`, e.g. `useMixin(mixin, keepDoc=false)`. - -When using `useMixin(mixin)`, all definitions within the mixin are applied as if this invocation was replaced with the -content of the mixin itself. This means, that if you have already defined anything prior to using a mixin, the mixin's -definitions will be **after** the previously defined things. This can be very useful if the commonality between ops is -that they have a few trailing options. - -If a named property or section is defined in both a mixin (or multiple mixins) and the op, then the **last** to define it will -win. Named properties are `legacy`, `javaPackage`, named sections are `Input`, `Arg`, `Output`, `Config`. - -For example, assume you have `javaPackage` defined in both an op and a mixin. Then you can have the following two -cases: - -First case: -```kotlin - Op("foo"){ - useMixin(exampleMixin) - javaPackage = "some.example.package" - } -``` - -Second case: -```kotlin - Op("foo"){ - javaPackage = "some.example.package" - useMixin(exampleMixin) - } -``` - -In the first case, the op will have the `javaPackage` value that is defined within the op. In the second case it will -have the `javaPackage` value defined in the mixin. - -For inputs, args, outputs, it works similarly. Assume you have `Input(dataType, "a")` defined in both the mixin and the -op. Again you can have two cases: - -First case: -```kotlin - Op("foo"){ - useMixin(exampleMixin) - Input(NUMERIC, "a") - } -``` - -Second case: -```kotlin - Op("foo"){ - Input(NUMERIC, "a") - useMixin(exampleMixin) - } -``` - -In the first case, it will overwrite the input from the mixin. In the second case, the mixin will overwrite that the -input from the op. - -## Config -Only available within a namespace context - - val nameConfig = Config("Name"){ - /* input, arg, constraint, doc properties */ - } - -Every config requires a namespace unique name. - -A config allows to define a configuration class, that can be used as a holder for complex properties of specific ops -which will be passed to an op as a parameter. - -Similar to an op itself, it supports `Input`, `Arg`, `Constraint` and `Doc` definitions. - -in order to use the config within an op you either use `useConfig(cfg)` or `val configRef = useConfig(cfg)`. The second -form allows you to reference the config. - -Referencing the config allows to you reference its inputs and args by name: `configRef.input("name")` and -`configRef.arg("name")`. Also it allows you to use a config in a signature `Signature(a, b, c, configRef)`. - -When default and shorthand signatures are used, configs will be always placed at the end. - -If a config is defined but not used, an `IllegalStateException` will be thrown. - -See also [ADR 0007 "Configuration Objects"](adr/0007-configuration_objects.md). - - -## Input -Available within an op, mixin and a config context - - Input(FLOATING_POINT, "b"){ /* input properties in input context */ } - val a = Input(INT, "a"){ /* input properties in input context */ } - -Inputs represent tensors. They are what the op will work on. - -Every input requires a data type (either `INT`, `FLOATING_POINT`, `NUMERIC` or `BOOLEAN`) and an op unique name. - -When defining an input, you can assign it to a variable in order to be able to reference it later on. You might want to -do this when defining constraints. - -If you want an input to represent an array, you will have to set a count accordingly. If no count is set, it is assumed -that the count is meant to be `Exactly(1)`. - -### Input properties -* `description` (String): A short description what this input represents. Setting this is recommended. -* `count` (Count): Can take one of `Exactly(n)`; `AtLeast(n)`; `AtMost(n)`; `Range(from, to)` -* `defaultValue` (Input): use another input as the default if this isn't set explicitly. The data type of the other - input has to match the data type of this input. The other input may also have a default value. - -## Argument -Available within an op, mixin and config context - - Arg(FLOATING_POINT, "b"){ /* Arg properties in arg context */ } - val a = Arg(INT, "a"){ /* Arg properties in arg context */ } - -Args represent arguments. They modify how the op works on its inputs. - -Every arg requires a data type (either `INT`, `FLOATING_POINT`, `NUMERIC` or `BOOLEAN`) and an op unique name. - -When defining an arg, you can assign it to a variable in order to be able to reference it later on. You might want to do -this when defining constraints. - -If you want an arg to represent an array, you will have to set a count accordingly. If no count is set, it is assumed -that the count is meant to be `Exactly(1)`. - -Note (Java specific): If the last arg is defined to represent an array, it will be translated to a vararg parameter, e.g. -`Arg(INT, "a"){ count = AtLeast(1); description = "..." }` will be turned into `long... a`. - -### Argument properties -* `description` (String): A short description what this argument represents. Setting this is recommended. -* `count` (Count): Can take one of `Exactly(n)`; `AtLeast(n)`; `AtMost(n)`; `Range(from, to)` -* `defaultValue` (null|Number|Boolean|int[]|double[]|boolean[]|Arg|TensorShapeValue|TensorDataTypeValue|String): - Use given value as default value, if this isn't explicitly set. Can refer to *inputs* and *outputs* using `x.shape()` - and `x.dataType()`. The given default values has to match the data type for this argument. May also refer to another - Arg, and that Arg may also have a default value. Default values based on outputs are treated like without a default - in SameDiff mode. -* `possibleValues` (String[]): only available when ENUM data type is used for the argument. Takes a list of possible - values for the Enum. If used in in abstract base op, the enum will only be created once. See also - [ADR 0006 "Op specific enums"](adr/0006-op_specific_enums.md). - - -## Output -Only available within an op and mixin context - - Output(FLOATING_POINT, "b"){ /* Arg properties in arg context */ } - -Every output requires a data type (either `INT`, `FLOATING_POINT`, `NUMERIC` or `BOOLEAN`) and an op unique name. - -While outputs can be assigned to a variable, there is no intended use-case for it. In contrast to inputs and args, -outputs can not be used in constraints. - -### Output properties -* `description` (String): A short description what this argument represents. Setting this is recommended. - - -## Signature -Only available within an op and mixin context - - Signature(a,b,c) - Signature(a,b,c) { "Some Documentation" } - AllParamSignature() - AllDefaultParamSignature() - -For some ops only specific signatures make sense, as for example some optional parameters may become required in the -presence of other optional parameters. This feature is mainly meant to help with the fact that not all programming -languages (e.g. Java) support default parameters. Each signature is meant to describe one overload in those languages. - -See also [ADR 0005 "Optional parameters and signatures"](adr/0005-optional_parameters_and_signatures.md). - -Signatures can also reference the output(s) of an op. Those signatures are only relevant in NDArray programming mode. -They are not to be generated in SameDiff mode. - -`AllParamSignature()` and `AllDefaultParamSignature()` are short hands for `Signature(...all parameters...)` and -`Signature(...only parameters with no default values...)`. Their parameters include references to outputs unless -disabled using `withOutput=false` (e.g. `AllParamSignature(withOutput=false)`). - -If no signature is specified for an op, it is treated as if `AllParamSignature()` and `AllDefaultParamSignature()` are -both specified. - -Each signature must satisfy the condition, that all required parameters are listed there. If this condition is not -satisfied, an `IllegalStateException` will be thrown on construction. - - -## Documentation -Only available within an op and mixin context - - Doc(Language.ANY, DocScope.ALL){ - """ Some documentation - It can be multiline. And indented. - """.trimIndent() - } - -Documentation can be language specific, and can be set to be only given at specific places. The documentation itself is -given as a string. Because Kotlin supports multiline strings along with proper indentation, we are using them directly -here. - -Note: At the moment we are only creating java code, so the documentation can use JavaDoc syntax. - -You can have multiple Doc definitions; they are treated as additive. - -Any instances of the following values will be replaced when generating code: - -* `%OPNAME%` -> operation name ("Add", "Sub", etc) -* `%LIBND4J_OPNAME%` -> libnd4j op name ("add", "sub", etc) -* `%INPUT_TYPE%` -> input / output type depending on the generated api, i.e. `SDVariable` for SameDiff and `INDArray` - for ND4J - -See `DocTokens` class for more details. - -## Constraints -Available within an op, mixin and a config context. - - Constraint("Error Message if constraint isn't satisfied"){ /* constraint definition */ } - BackendConstraint("Error Message if constraint isn't satisfied"){ /* constraint definition */ } - -Many ops expect their inputs and arguments to satisfy some specific rules. Those rules can be expressed with the -constraint system. - -Constraints are to be enforced within the frontend language, while BackendConstraints are currently only to be used as -a part of the documentation. They will be enforced within the C++ backend, so there is no point in double checking them. - -There is a system in place to define even complex constraints for inputs and arguments. - -In a constraint definition, you can reference inputs and arguments directly, if they are previously assigned to -a variable using `val name = Input(...)`. Inside the Constraint block, you can use the following operations: - -* `eq`: Compare equality (applicable to numbers and booleans), e.g. `x eq 7`, `x eq true` -* `neq`: Compare inequality (applicable to numbers and booleans), e.g. `x neq 3`, `x neq true` -* `lt`, `lte`: less than, less than equal (applicable to numbers), e.g. `x lt 3`, `x lte 4` -* `gt`, `gte`: greater than, grater than equal (applicable to numbers), e.g. `x gt 5`, `x gte 6` -* `and`: combine two comparisons where both have to be true, e.g. `(x eq 8) and (y lt 3)` -* `or`: combine two comparisons where one has to be true, e.g. `(x eq 8) or (y eq true)` -* `all`: combine N comparisons where all have to be true, e.g. `all(x eq 8, y lt 3, z eq true)` -* `some`: combine N comparisons where at least one has to be true, e.g. `some(x eq 8, y lt 3, z eq true)` -* `not`: negates a comparison, e.g. `not(x eq 3)` - -In addition to those operations, you also get access to some more complex constraints: -* `sameType(...)`: true if all given inputs are the same type, e.g. `sameType(x,y,z)` -* `sameShape(...)`: true if all given inputs have the same shape, e.g. `sameShape(x,y,z)` -* `broadcastableShapes(...)`: true if all given inputs have broadcast compatible shapes, e.g. `broadcastableShapes(x,y,z)` - -Inputs also get some additional methods on them to define useful constraints: -* `input.rank()`: Rank of the given input -* `input.sizeAt(i)`: size of the given input at the i-th dimension -* `input.isScalar()`: Short hand for `x.rank() == 1` - -### Examples -Some examples of constraints, and what they evaluate to. The example code contains a little bit of context. - -```kotlin -val x = Input(INT, "x") { description = "First input array" } -val y = Input(INT, "y") { description = "Second input array" } -Constraint("foo bar"){ - x.sizeAt(7) eq 7 and y.isScalar() -} -``` - -will evaluate to: -```java - Preconditions.checkArgument((x.sizeAt(7) == 7) && (y.rank() == 1), "foo bar"); -``` - -More examples (only the constraint itself, without context code): - -#### Some -```kotlin -some(input.rank() eq 3, input.sizeAt(2) gte 7, input.sizeAt(4) lt 5) -``` -turns to: -```java -((x.rank() == 3) || (x.sizeAt(2) >= 7)) || (x.sizeAt(4) < 5) -``` - -# Contributing to this project -If you want to contribute to this project other than by adding or improving op definitions, the following sections might -be of special interest to you. - -## Extending the DSL -The DSL is implemented using Kotlin’s type-safe builders feature -(see https://kotlinlang.org/docs/reference/type-safe-builders.html). The basic principle is that functions calls can -receive blocks that can be executed in a specified context. When combined with the fact that we are just looking to -create an object graph that is then going to be used as input to the code generators, this allows us to create a very -feature rich DSL without actually having to write a lot of code to support it. - -Most of the DSL specific code can be found in `src/kotlin/org/nd4j/codegen/dsl/OpBuilder.kt`. The actual class -definitions for the object graph we are building, can be found in `src/kotlin/org/nd4j/codegen/api`. - -If you want to add just a simple field to one of the objects, it is usually enough to directly add it to the particular -class. - -If you want to add a specific section to the op definition, i.e. a section like Input or Doc, you will have to add both -the class for the object that it is going to be creating, as well as a function within OpBuilder.kt to create and -register that section within the op. - -**Note:** When you extend the DSL you will most likely also have to update all code generators to support the feature -you have added. - -## Adding / extending code generators -Code generators can be written in either Java or Kotlin. Java has the advantage that more people will have experience in -using it. Kotlin has the advantage of more convenient syntax, especially for plain string manipulation and when dealing -with Enums and fixed sets of subclasses (called sealed classes in Kotlin). - -All generators have to implement the `org.nd4j.codegen.api.generator.Generator` interface. For automatic detection by -the CLI tool, they should also be within the `org.nd4j.codegen.impl.LANGUAGE` package, where `LANGUAGE` is the actual -language that they generate. - -Code generators can also use an auxiliary generator for constraint generation. Those auxiliary generators, have to -implement ` org.nd4j.codegen.api.generator.ConstraintCodeGenerator` interface. - diff --git a/contrib/codegen-tools/codegen/generate.sh b/contrib/codegen-tools/codegen/generate.sh deleted file mode 100644 index e254ba92c..000000000 --- a/contrib/codegen-tools/codegen/generate.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if test "$#" -eq 0; then - echo "No namespaces were specified. One or more namespaces must be provided as an argument" - echo "Usage example 1 (single namespace): ./generate.sh math" - echo "Usage example 2 (multiple namespaces): ./generate.sh math,random" - echo "Usage example 2 (all namespaces): ./generate.sh all" -else - mvn clean package -DskipTests - java -cp target/codegen-1.0.0-SNAPSHOT-shaded.jar org.nd4j.codegen.cli.CLI -dir ../../../ -namespaces "$@" -fi \ No newline at end of file diff --git a/contrib/codegen-tools/codegen/pom.xml b/contrib/codegen-tools/codegen/pom.xml deleted file mode 100644 index 7fa91d8fe..000000000 --- a/contrib/codegen-tools/codegen/pom.xml +++ /dev/null @@ -1,267 +0,0 @@ - - - 4.0.0 - - org.nd4j - codegen - 1.0.0-SNAPSHOT - - - UTF-8 - 2.5 - 3.12.0 - 1.7 - 1.18.8 - 1.1.7 - 5.8.0-M1 - 5.4.2 - 1.8 - 3.1.1 - 1.3.50 - 1.8 - true - - - - - org.slf4j - slf4j-api - 1.7.28 - - - - ch.qos.logback - logback-classic - ${logback.version} - - - - commons-io - commons-io - ${commonsio.version} - - - - org.projectlombok - lombok - ${lombok.version} - - - - org.apache.commons - commons-lang3 - ${commons.lang.version} - - - - com.squareup - javapoet - 1.11.1 - - - - - - org.junit.jupiter - junit-jupiter-api - ${junit-jupiter.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit-jupiter.version} - test - - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - ${kotlin.version} - - - org.jetbrains.kotlin - kotlin-test - ${kotlin.version} - test - - - - com.fasterxml.jackson.module - jackson-module-kotlin - 2.9.9 - - - - com.beust - jcommander - 1.78 - - - org.nd4j - nd4j-api - ${project.version} - - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.0.0 - - - add-source - generate-sources - add-source - - - src/main/stubs - - - - - get-cpu-count - - cpu-count - - - cpu.core.count - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - true - false - - - *:* - - org/datanucleus/** - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - package - - shade - - - - - reference.conf - - - - - - - - - - - org.jetbrains.kotlin - kotlin-maven-plugin - ${kotlin.version} - - - -Xjsr305=strict - - - spring - jpa - - - - - org.jetbrains.kotlin - kotlin-maven-allopen - ${kotlin.version} - - - org.jetbrains.kotlin - kotlin-maven-noarg - ${kotlin.version} - - - - - compile - compile - - - ${project.basedir}/src/main/stubs - ${project.basedir}/src/main/kotlin - ${project.basedir}/src/main/java - ${project.basedir}/src/main/ops - - - - - test-compile - test-compile - - - ${project.basedir}/src/test/stubs - ${project.basedir}/src/test/kotlin - ${project.basedir}/src/test/java - ${project.basedir}/src/test/ops - - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - - - default-compile - none - - - - default-testCompile - none - - - java-compile - compile - compile - - - java-test-compile - test-compile - testCompile - - - - ${java.version} - ${java.version} - - - - - - \ No newline at end of file diff --git a/contrib/codegen-tools/codegen/src/main/java/org/nd4j/codegen/impl/java/JavaPoetGenerator.java b/contrib/codegen-tools/codegen/src/main/java/org/nd4j/codegen/impl/java/JavaPoetGenerator.java index e50ef4651..7f445ed4b 100644 --- a/contrib/codegen-tools/codegen/src/main/java/org/nd4j/codegen/impl/java/JavaPoetGenerator.java +++ b/contrib/codegen-tools/codegen/src/main/java/org/nd4j/codegen/impl/java/JavaPoetGenerator.java @@ -19,13 +19,17 @@ */ package org.nd4j.codegen.impl.java; + import org.apache.commons.lang3.StringUtils; import org.nd4j.codegen.api.Language; +import org.nd4j.codegen.api.Namespace; import org.nd4j.codegen.api.NamespaceOps; +import org.nd4j.codegen.api.Op; import org.nd4j.codegen.api.generator.Generator; import org.nd4j.codegen.api.generator.GeneratorConfig; import java.io.File; +import java.io.IOException; public class JavaPoetGenerator implements Generator { @@ -36,12 +40,12 @@ public class JavaPoetGenerator implements Generator { } @Override - public void generateNamespaceNd4j(NamespaceOps namespace, GeneratorConfig config, File directory, String className) throws java.io.IOException { + public void generateNamespaceNd4j(NamespaceOps namespace, GeneratorConfig config, File directory, String className) throws IOException { Nd4jNamespaceGenerator.generate(namespace, config, directory, className, "org.nd4j.linalg.factory", StringUtils.EMPTY); } @Override - public void generateNamespaceSameDiff(NamespaceOps namespace, GeneratorConfig config, File directory, String className) throws java.io.IOException { + public void generateNamespaceSameDiff(NamespaceOps namespace, GeneratorConfig config, File directory, String className) throws IOException { //throw new UnsupportedOperationException("Not yet implemented"); Nd4jNamespaceGenerator.generate(namespace, config, directory, className, "org.nd4j.autodiff.samediff", StringUtils.EMPTY); } diff --git a/contrib/codegen-tools/codegen/src/main/java/org/nd4j/codegen/ir/SerializationTest.java b/contrib/codegen-tools/codegen/src/main/java/org/nd4j/codegen/ir/SerializationTest.java index cbe4e265c..f41bd93a6 100644 --- a/contrib/codegen-tools/codegen/src/main/java/org/nd4j/codegen/ir/SerializationTest.java +++ b/contrib/codegen-tools/codegen/src/main/java/org/nd4j/codegen/ir/SerializationTest.java @@ -17,14 +17,13 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.nd4j.codegen.ir; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; +public class SerializationTest { -@DisplayName("Serialization Test") -class SerializationTest { + public static void main(String...args) { - public static void main(String... args) { } + } diff --git a/contrib/codegen-tools/codegen/src/main/ops/org/nd4j/codegen/ops/SDBaseOps.kt b/contrib/codegen-tools/codegen/src/main/ops/org/nd4j/codegen/ops/SDBaseOps.kt index 862b951e3..28a4d6887 100644 --- a/contrib/codegen-tools/codegen/src/main/ops/org/nd4j/codegen/ops/SDBaseOps.kt +++ b/contrib/codegen-tools/codegen/src/main/ops/org/nd4j/codegen/ops/SDBaseOps.kt @@ -29,6 +29,7 @@ import org.nd4j.codegen.api.doc.DocScope import org.nd4j.codegen.dsl.* import org.nd4j.codegen.api.DataType.* import org.nd4j.codegen.mixins.* +import org.nd4j.linalg.api.buffer.DataType import java.lang.Boolean.FALSE fun SDBaseOps() = Namespace("BaseOps"){ @@ -593,7 +594,7 @@ fun SDBaseOps() = Namespace("BaseOps"){ legacy = true Input(NUMERIC, "x") { description = "Input variable" } Arg(BOOL, "keepDims") { description = "If true: keep the dimensions that are reduced on (as size 1). False: remove the reduction dimensions" - ; defaultValue=FALSE } + ; defaultValue=FALSE } Arg(INT, "dimensions") { count = AtLeast(0); description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" } Output(NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" } Doc(Language.ANY, DocScope.ALL){ @@ -772,19 +773,6 @@ fun SDBaseOps() = Namespace("BaseOps"){ useMixin(keepDimsDoc) } - Op("split") { - javaPackage = "org.nd4j.linalg.api.ops.impl.shape" - javaOpClass = "Split" - Input(NUMERIC,"input") {description = "Input to split"} - Arg(INT, "numSplit") { description = "Number of splits" } - Arg(INT, "splitDim") { description = "The dimension to split on" } - Doc(Language.ANY, DocScope.ALL){ - """ - Split a value in to a list of ndarrays. - """.trimIndent() - } - } - Op("oneHot") { javaPackage = "org.nd4j.linalg.api.ops.impl.shape" Input(NUMERIC, "indices") { description = "Indices - value 0 to depth-1" } @@ -792,7 +780,7 @@ fun SDBaseOps() = Namespace("BaseOps"){ Arg(INT, "axis") { description = "" } Arg(NUMERIC, "on") { description = "" } Arg(NUMERIC, "off") { description = "" } - Arg(DATA_TYPE, "dataType") { description = "Output data type"; defaultValue = org.nd4j.linalg.api.buffer.DataType.FLOAT } + Arg(DATA_TYPE, "dataType") { description = "Output data type"; defaultValue = DataType.FLOAT } Output(NUMERIC, "output"){ description = "Output variable" } Doc(Language.ANY, DocScope.ALL){ diff --git a/contrib/codegen-tools/codegen/src/main/ops/org/nd4j/codegen/ops/SDLoss.kt b/contrib/codegen-tools/codegen/src/main/ops/org/nd4j/codegen/ops/SDLoss.kt index b6db6f887..d8706bd02 100644 --- a/contrib/codegen-tools/codegen/src/main/ops/org/nd4j/codegen/ops/SDLoss.kt +++ b/contrib/codegen-tools/codegen/src/main/ops/org/nd4j/codegen/ops/SDLoss.kt @@ -47,23 +47,6 @@ fun SDLoss() = Namespace("Loss"){ } } - - Op("ctcLoss") { - javaPackage = "org.nd4j.linalg.api.ops.impl.loss" - javaOpClass = "CtcLoss" - Input(NUMERIC, "targetLabels") { description = "Label array" } - Input(NUMERIC, "logitInput") { description = "Inputs" } - Input(NUMERIC, "targetLabelLengths") { description = "Length of the target label" } - Input(NUMERIC, "logitInputLengths") { description = "Length of the input"} - Output(NUMERIC, "output"){ description = "Ctc loss " } - Doc(Language.ANY, DocScope.ALL){ - """ - CTC Loss: Connectionist Temporal Classification Loss. See: - https://dl.acm.org/citation.cfm?id=1143891 - """.trimIndent() - } - } - Op("cosineDistance") { javaPackage = "org.nd4j.linalg.api.ops.impl.loss" javaOpClass = "CosineDistanceLoss" diff --git a/contrib/codegen-tools/codegen/src/main/resources/logback.xml b/contrib/codegen-tools/codegen/src/main/resources/logback.xml index ad6e8c561..e3d6a28cf 100644 --- a/contrib/codegen-tools/codegen/src/main/resources/logback.xml +++ b/contrib/codegen-tools/codegen/src/main/resources/logback.xml @@ -39,8 +39,8 @@ - - + + diff --git a/contrib/codegen-tools/codegen/src/main/resources/onnx.pbtxt b/contrib/codegen-tools/codegen/src/main/resources/onnx.pbtxt index 5fa814d06..c4385573a 100644 --- a/contrib/codegen-tools/codegen/src/main/resources/onnx.pbtxt +++ b/contrib/codegen-tools/codegen/src/main/resources/onnx.pbtxt @@ -1,6004 +1,6004 @@ -input: "X" -output: "Y" -name: "Abs" -op_type: "Abs" -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nAbsolute takes one input data (Tensor) and produces one output data\n(Tensor) where the absolute is, y = abs(x), is applied to\nthe tensor elementwise.\n" -----f -input: "input" -output: "output" -name: "Acos" -op_type: "Acos" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the arccosine (inverse of cosine) of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Acosh" -op_type: "Acosh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic arccosine of the given input tensor element-wise.\n" -----f -input: "R" -input: "T" -input: "inputs" -output: "outputs" -name: "Adagrad" -op_type: "Adagrad" -attribute { - name: "decay_factor" - f: 0.0 - type: FLOAT -} -attribute { - name: "epsilon" - f: 1e-06 - type: FLOAT -} -attribute { - name: "norm_coefficient" - f: 0.0 - type: FLOAT -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - type: STRINGS -} -attribute { - name: "T-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "inputs-types" - strings: "float" - strings: "double" - type: STRINGS -} -doc_string: "\n Compute one iteration of ADAGRAD, a stochastic gradient based optimization\n algorithm. This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. As you can imagine, ADAGRAD requires\n some parameters:\n \n - The initial learning-rate \"R\".\n - The update count \"T\". That is, the number of training iterations conducted.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A learning-rate decay factor \"decay_factor\".\n - A small constant \"epsilon\" to avoid dividing-by-zero. \n\n At each ADAGRAD iteration, the optimized tensors are moved along a direction\n computed based on their estimated gradient and accumulated squared gradient. Assume\n that only a single tensor \"X\" is updated by this operator. We need the value of \"X\",\n its gradient \"G\", and its accumulated squared gradient \"H\". Therefore, variables in\n this operator\'s input list are sequentially \"R\", \"T\", \"X\", \"G\", and \"H\". Other\n parameters are given as attributes because they are usually constants. Also, the\n corresponding output tensors are the new value of \"X\" (called \"X_new\"), and then\n the new accumulated squared gradient (called \"H_new\"). Those outputs are computed\n from the given inputs following the pseudo code below.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise arithmetic operations with\n numpy-style broadcasting support. The pseudo code to compute those outputs is:\n\n // Compute a scalar learning-rate factor. At the first update of X, T is generally\n // 0 (0-based update index) or 1 (1-based update index).\n r = R / (1 + T * decay_factor);\n\n // Add gradient of 0.5 * norm_coefficient * ||X||_2^2, where ||X||_2 is the 2-norm.\n G_regularized = norm_coefficient * X + G;\n\n // Compute new accumulated squared gradient.\n H_new = H + G_regularized * G_regularized;\n\n // Compute the adaptive part of per-coordinate learning rate. Note that Sqrt(...)\n // computes element-wise square-root.\n H_adaptive = Sqrt(H_new) + epsilon\n\n // Compute the new value of \"X\".\n X_new = X - r * G_regularized / H_adaptive;\n\n If one assign this operators to optimize multiple inputs, for example, \"X_1\" and \"X_2\", the same\n pseudo code may be extended to handle all tensors jointly. More specifically, we can view \"X\" as a\n concatenation of \"X_1\" and \"X_2\" (of course, their gradient and accumulate gradient should\n be concatenated too) and then just reuse the entire pseudo code.\n\n Note that ADAGRAD was first proposed in http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf.\n In that reference paper, this operator is a special case of the Figure 1\'s composite mirror\n descent update.\n" -----f -input: "R" -input: "T" -input: "inputs" -output: "outputs" -name: "Adam" -op_type: "Adam" -attribute { - name: "alpha" - f: 0.9 - type: FLOAT -} -attribute { - name: "beta" - f: 0.999 - type: FLOAT -} -attribute { - name: "epsilon" - f: 1e-06 - type: FLOAT -} -attribute { - name: "norm_coefficient" - f: 0.0 - type: FLOAT -} -attribute { - name: "norm_coefficient_post" - f: 0.0 - type: FLOAT -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - type: STRINGS -} -attribute { - name: "T-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "inputs-types" - strings: "float" - strings: "double" - type: STRINGS -} -doc_string: "\n Compute one iteration of Adam, a stochastic gradient based optimization\n algorithm. This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. First of all, Adam requires\n some parameters:\n \n - The learning-rate \"R\".\n - The update count \"T\". That is, the number of training iterations conducted.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A small constant \"epsilon\" to avoid dividing-by-zero. \n - Two coefficients, \"alpha\" and \"beta\".\n\n At each Adam iteration, the optimized tensors are moved along a direction\n computed based on their exponentially-averaged historical gradient and\n exponentially-averaged historical squared gradient. Assume that only a tensor\n \"X\" is being optimized. The rest of required information is\n \n - the value of \"X\",\n - \"X\"\'s gradient (denoted by \"G\"),\n - \"X\"\'s exponentially-averaged historical gradient (denoted by \"V\"), and\n - \"X\"\'s exponentially-averaged historical squared gradient (denoted by \"H\").\n\n Some of those parameters are passed into this operator as input tensors and others\n are stored as this operator\'s attributes. Specifically, this operator\'s input tensor\n list is [\"R\", \"T\", \"X\", \"G\", \"V\", \"H\"]. That is, \"R\" is the first input, \"T\" is\n the second input, and so on. Other parameters are given as attributes because they\n are constants. Moreover, the corresponding output tensors are \n \n - the new value of \"X\" (called \"X_new\"),\n - the new exponentially-averaged historical gradient (denoted by \"V_new\"), and\n - the new exponentially-averaged historical squared gradient (denoted by \"H_new\").\n\n Those outputs are computed following the pseudo code below.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise arithmetic operations with\n numpy-style broadcasting support. The pseudo code to compute those outputs is:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||_2^2, where ||X||_2 is the 2-norm.\n G_regularized = norm_coefficient * X + G\n\n // Update exponentially-averaged historical gradient.\n V_new = alpha * V + (1 - alpha) * G_regularized\n\n // Update exponentially-averaged historical squared gradient.\n H_new = beta * H + (1 - beta) * G_regularized * G_regularized\n\n // Compute the element-wise square-root of H_new. V_new will be element-wisely\n // divided by H_sqrt for a better update direction.\n H_sqrt = Sqrt(H_new) + epsilon\n\n // Compute learning-rate. Note that \"alpha**T\"/\"beta**T\" is alpha\'s/beta\'s T-th power.\n R_adjusted = T > 0 ? R * Sqrt(1 - beta**T) / (1 - alpha**T) : R\n\n // Compute new value of \"X\".\n X_new = X - R_adjusted * V_new / H_sqrt\n\n // Post-update regularization.\n X_final = (1 - norm_coefficient_post) * X_new \n\n If there are multiple inputs to be optimized, the pseudo code will be applied\n independently to each of them.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Add" -op_type: "Add" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPerforms element-wise binary addition (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "A" -input: "B" -output: "C" -name: "And" -op_type: "And" -attribute { - name: "A-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "B-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `and` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "data" -output: "reduced" -name: "ArgMax" -op_type: "ArgMax" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "select_last_index" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the indices of the max elements of the input tensor\'s element along the \nprovided axis. The resulting tensor has the same rank as the input if keepdims equal 1. \nIf keepdims equal 0, then the resulting tensor have the reduced dimension pruned. \nIf select_last_index is True (default False), the index of the last occurrence of the max \nis selected if the max appears more than once in the input. Otherwise the index of the \nfirst occurrence is selected.\nThe type of the output tensor is integer." -----f -input: "data" -output: "reduced" -name: "ArgMin" -op_type: "ArgMin" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "select_last_index" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the indices of the min elements of the input tensor\'s element along the \nprovided axis. The resulting tensor has the same rank as the input if keepdims equal 1. \nIf keepdims equal 0, then the resulting tensor have the reduced dimension pruned. \nIf select_last_index is True (default False), the index of the last occurrence of the min \nis selected if the min appears more than once in the input. Otherwise the index of the \nfirst occurrence is selected.\nThe type of the output tensor is integer." -----f -input: "X" -input: "Y" -output: "Z" -name: "ArrayFeatureExtractor" -op_type: "ArrayFeatureExtractor" -attribute { - name: "X-types" - strings: "int32" - strings: "string" - strings: "double" - strings: "int64" - strings: "float" - type: STRINGS -} -attribute { - name: "Y-types" - strings: "int64" - type: STRINGS -} -doc_string: "\n Select elements of the input tensor based on the indices passed.
\n The indices are applied to the last axes of the tensor.\n" -----f -input: "input" -output: "output" -name: "Asin" -op_type: "Asin" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the arcsine (inverse of sine) of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Asinh" -op_type: "Asinh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic arcsine of the given input tensor element-wise.\n" -----f -input: "input" -output: "output" -name: "Atan" -op_type: "Atan" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the arctangent (inverse of tangent) of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Atanh" -op_type: "Atanh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic arctangent of the given input tensor element-wise.\n" -----f -input: "X" -output: "Y" -name: "AveragePool" -op_type: "AveragePool" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "ceil_mode" - i: 0 - type: INT -} -attribute { - name: "count_include_pad" - i: 0 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n AveragePool consumes an input tensor X and applies average pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n average pooling consisting of computing the average on all values of a\n subset of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing. The output spatial shape will be following:\n ```\n output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1)\n ```\n or\n ```\n output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1)\n ```\n if ceil_mode is enabled\n\n ```\n * pad_shape[i] is sum of pads along axis i\n ```\n\n `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following:\n ```\n VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - kernel_spatial_shape[i] + 1) / strides_spatial_shape[i])\n SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i])\n ```\n And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`:\n ```\n pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + kernel_spatial_shape[i] - input_spatial_shape[i]\n ```\n The output of each pooling window is divided by the number of elements (exclude pad when attribute count_include_pad is zero).\n " -----f -input: "X" -input: "scale" -input: "B" -input: "mean" -input: "var" -output: "Y" -output: "mean" -output: "var" -output: "saved_mean" -output: "saved_var" -name: "BatchNormalization" -op_type: "BatchNormalization" -attribute { - name: "epsilon" - f: 1e-05 - type: FLOAT -} -attribute { - name: "momentum" - f: 0.9 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "scale-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "mean-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "var-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCarries out batch normalization as described in the paper\nhttps://arxiv.org/abs/1502.03167. Depending on the mode it is being run,\nthere are multiple cases for the number of outputs, which we list below:\n\nOutput case #1: Y, mean, var, saved_mean, saved_var (training mode)\nOutput case #2: Y (test mode)\n\nFor previous (depreciated) non-spatial cases, implementors are suggested\nto flatten the input shape to (N x C*D1*D2 ..*Dn) before a BatchNormalization Op.\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "X" -output: "Y" -name: "Binarizer" -op_type: "Binarizer" -attribute { - name: "threshold" - f: 0.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Maps the values of the input tensor to either 0 or 1, element-wise, based on the outcome of a comparison against a threshold value.\n" -----f -input: "X" -input: "Y" -output: "Z" -name: "BitShift" -op_type: "BitShift" -attribute { - name: "direction" - s: "" - type: STRING -} -attribute { - name: "X-types" - strings: "uint32" - strings: "uint16" - strings: "uint8" - strings: "uint64" - type: STRINGS -} -attribute { - name: "Y-types" - strings: "uint32" - strings: "uint16" - strings: "uint8" - strings: "uint64" - type: STRINGS -} -doc_string: "\nBitwise shift operator performs element-wise operation. For each input element, if the\n attribute \"direction\" is \"RIGHT\", this operator moves its binary representation toward\n the right side so that the input value is effectively decreased. If the attribute \"direction\"\n is \"LEFT\", bits of binary representation moves toward the left side, which results the\n increase of its actual value. The input X is the tensor to be shifted and another input\n Y specifies the amounts of shifting. For example, if \"direction\" is \"Right\", X is [1, 4],\n and S is [1, 1], the corresponding output Z would be [0, 2]. If \"direction\" is \"LEFT\" with\n X=[1, 2] and S=[1, 2], the corresponding output Y would be [2, 8].\n \n Because this operator supports Numpy-style broadcasting, X\'s and Y\'s shapes are\n not necessarily identical.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md)." -----f -input: "input" -output: "output" -name: "Cast" -op_type: "Cast" -attribute { - name: "to" - s: "" - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "float16" - strings: "int32" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nThe operator casts the elements of a given input tensor to a data type\nspecified by the \'to\' argument and returns an output tensor of the same size in\nthe converted type. The \'to\' argument must be one of the data types specified\nin the \'DataType\' enum field in the TensorProto message.\n\nCasting from string tensor in plain (e.g., \"3.14\" and \"1000\") and scientific numeric representations\n(e.g., \"1e-5\" and \"1E8\") to float types is supported. For example, converting string \"100.5\" to an integer may\nresult 100. There are some string literals reserved for special floating-point values;\n\"+INF\" (and \"INF\"), \"-INF\", and \"NaN\" are positive infinity, negative infinity, and not-a-number, respectively.\nAny string which can exactly match \"+INF\" in a case-insensitive way would be mapped to positive infinite. Similarly,\nthis case-insensitive rule is applied to \"INF\" and \"NaN\". When casting from numeric tensors\nto string tensors, plain floating-point representation (such as \"314.15926\") would be used. \nConverting non-numerical-literal string such as \"Hello World!\" is an undefined behavior. Cases \nof converting string representing floating-point arithmetic value, such as \"2.718\", to INT is an undefined behavior.\n\nConversion from a numerical type to any numerical type is always allowed.\nUser must be aware of precision loss and value change caused by range difference between two types.\nFor example, a 64-bit float 3.1415926459 may be round to a 32-bit float 3.141592. Similarly, converting\nan integer 36 to Boolean may produce 1 because we truncate bits which can\'t be stored in the targeted type.\n" -----f -input: "X" -output: "Y" -name: "CastMap" -op_type: "CastMap" -attribute { - name: "cast_to" - s: "TO_FLOAT" - type: STRING -} -attribute { - name: "map_form" - s: "DENSE" - type: STRING -} -attribute { - name: "max_map" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "map(int64,string" - strings: "map(int64,float" - type: STRINGS -} -doc_string: "\n Converts a map to a tensor.
The map key must be an int64 and the values will be ordered\n in ascending order based on this key.
The operator supports dense packing or sparse packing.\n If using sparse packing, the key cannot exceed the max_map-1 value.\n" -----f -input: "X" -output: "Y" -name: "CategoryMapper" -op_type: "CategoryMapper" -attribute { - name: "cats_int64s" - s: "" - type: INTS -} -attribute { - name: "cats_strings" - s: "" - type: STRINGS -} -attribute { - name: "default_int64" - i: -1 - type: INT -} -attribute { - name: "default_string" - s: "_Unused" - type: STRING -} -attribute { - name: "X-types" - strings: "string" - strings: "int64" - type: STRINGS -} -doc_string: "\n Converts strings to integers and vice versa.
\n Two sequences of equal length are used to map between integers and strings,\n with strings and integers at the same index detailing the mapping.
\n Each operator converts either integers to strings or strings to integers, depending \n on which default value attribute is provided. Only one default value attribute\n should be defined.
\n If the string default value is set, it will convert integers to strings.\n If the int default value is set, it will convert strings to integers.\n" -----f -input: "X" -output: "Y" -name: "Ceil" -op_type: "Ceil" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCeil takes one input data (Tensor) and produces one output data\n(Tensor) where the ceil is, y = ceil(x), is applied to\nthe tensor elementwise.\n" -----f -input: "X" -output: "Y" -name: "Celu" -op_type: "Celu" -attribute { - name: "alpha" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - type: STRINGS -} -doc_string: "\nContinuously Differentiable Exponential Linear Units:\nPerform the linear unit element-wise on the input tensor X\nusing formula: \n\n```\nmax(0,x) + min(0,alpha*(exp(x/alpha)-1))\n```\n" -----f -input: "input" -input: "min" -input: "max" -output: "output" -name: "Clip" -op_type: "Clip" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "min-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "max-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nClip operator limits the given input within an interval. The interval is\nspecified by the inputs \'min\' and \'max\'. They default to\nnumeric_limits::lowest() and numeric_limits::max(), respectively.\n" -----f -input: "input" -input: "condition" -output: "output" -name: "Compress" -op_type: "Compress" -attribute { - name: "axis" - s: "" - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "condition-types" - strings: "bool" - type: STRINGS -} -doc_string: "\n Selects slices from an input tensor along a given axis where condition evaluates to True for each axis index.\n In case axis is not provided, input is flattened before elements are selected.\n Compress behaves like numpy.compress: https://docs.scipy.org/doc/numpy/reference/generated/numpy.compress.html\n " -----f -input: "inputs" -output: "concat_result" -name: "Concat" -op_type: "Concat" -attribute { - name: "axis" - s: "" - type: INT -} -attribute { - name: "inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "Concatenate a list of tensors into a single tensor. All input tensors must have the same shape, except for the dimension size of the axis to concatenate on." -----f -input: "input_sequence" -output: "concat_result" -name: "ConcatFromSequence" -op_type: "ConcatFromSequence" -attribute { - name: "axis" - s: "" - type: INT -} -attribute { - name: "new_axis" - i: 0 - type: INT -} -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -doc_string: "\nConcatenate a sequence of tensors into a single tensor.\nAll input tensors must have the same shape, except for the dimension size of the axis to concatenate on.\nBy default \'new_axis\' is 0, the behavior is similar to numpy.concatenate.\nWhen \'new_axis\' is 1, the behavior is similar to numpy.stack.\n" -----f -output: "output" -name: "Constant" -op_type: "Constant" -attribute { - name: "sparse_value" - s: "" - type: SPARSE_TENSOR -} -attribute { - name: "value" - s: "" - type: TENSOR -} -attribute { - name: "value_float" - s: "" - type: FLOAT -} -attribute { - name: "value_floats" - s: "" - type: FLOATS -} -attribute { - name: "value_int" - s: "" - type: INT -} -attribute { - name: "value_ints" - s: "" - type: INTS -} -attribute { - name: "value_string" - s: "" - type: STRING -} -attribute { - name: "value_strings" - s: "" - type: STRINGS -} -doc_string: "\nThis operator produces a constant tensor. Exactly one of the provided attributes, either value, sparse_value,\nor value_* must be specified.\n" -----f -input: "input" -output: "output" -name: "ConstantOfShape" -op_type: "ConstantOfShape" -attribute { - name: "value" - s: "" - type: TENSOR -} -attribute { - name: "input-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nGenerate a tensor with given value and shape.\n" -----f -input: "X" -input: "W" -input: "B" -output: "Y" -name: "Conv" -op_type: "Conv" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "group" - i: 1 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe convolution operator consumes an input tensor and a filter, and\ncomputes the output." -----f -input: "x" -input: "w" -input: "x_zero_point" -input: "w_zero_point" -output: "y" -name: "ConvInteger" -op_type: "ConvInteger" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "group" - i: 1 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "x-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "w-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "x_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "w_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nThe integer convolution operator consumes an input tensor, its zero-point, a filter, and its zero-point,\nand computes the output. The production MUST never overflow. The accumulation may overflow if and only if in 32 bits.\n" -----f -input: "X" -input: "W" -input: "B" -output: "Y" -name: "ConvTranspose" -op_type: "ConvTranspose" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "group" - i: 1 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "output_padding" - s: "" - type: INTS -} -attribute { - name: "output_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe convolution transpose operator consumes an input tensor and a filter,\nand computes the output.\n\nIf the pads parameter is provided the shape of the output is calculated via the following equation:\n\n output_shape[i] = stride[i] * (input_size[i] - 1) + output_padding[i] + ((kernel_shape[i] - 1) * dilations[i] + 1) - pads[start_i] - pads[end_i]\n\noutput_shape can also be explicitly specified in which case pads values are auto generated using these equations:\n\n total_padding[i] = stride[i] * (input_size[i] - 1) + output_padding[i] + ((kernel_shape[i] - 1) * dilations[i] + 1) - output_shape[i]\n If (auto_pads != SAME_UPPER): pads[start_i] = total_padding[i]/2; pads[end_i] = total_padding[i] - (total_padding[i]/2)\n Else: pads[start_i] = total_padding[i] - (total_padding[i]/2); pads[end_i] = (total_padding[i]/2).\n\n " -----f -input: "input" -output: "output" -name: "Cos" -op_type: "Cos" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the cosine of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Cosh" -op_type: "Cosh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic cosine of the given input tensor element-wise.\n" -----f -input: "x" -input: "axis" -output: "y" -name: "CumSum" -op_type: "CumSum" -attribute { - name: "exclusive" - i: 0 - type: INT -} -attribute { - name: "reverse" - i: 0 - type: INT -} -attribute { - name: "x-types" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "axis-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nPerforms cumulative sum of the input elements along the given axis.\nBy default, it will do the sum inclusively meaning the first element is copied as is.\nThrough an `exclusive` attribute, this behavior can change to exclude the first element.\nIt can also perform summation in the opposite direction of the axis. For that, set `reverse` attribute to 1.\n\nExample:\n```\ninput_x = [1, 2, 3]\naxis=0\noutput = [1, 3, 6]\nexclusive=1\noutput = [0, 1, 3]\nexclusive=0\nreverse=1\noutput = [6, 5, 3]\nexclusive=1\nreverse=1\noutput = [5, 3, 0]\n```\n " -----f -input: "input" -output: "output" -name: "DepthToSpace" -op_type: "DepthToSpace" -attribute { - name: "blocksize" - s: "" - type: INT -} -attribute { - name: "mode" - s: "DCR" - type: STRING -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "DepthToSpace rearranges (permutes) data from depth into blocks of spatial data.\nThis is the reverse transformation of SpaceToDepth. More specifically, this op outputs a copy of\nthe input tensor where values from the depth dimension are moved in spatial blocks to the height\nand width dimensions. By default, `mode` = `DCR`.\nIn the DCR mode, elements along the depth dimension from the input tensor are rearranged in the\nfollowing order: depth, column, and then row. The output y is computed from the input x as below:\n\nb, c, h, w = x.shape\n\ntmp = np.reshape(x, [b, blocksize, blocksize, c // (blocksize**2), h, w])\n\ntmp = np.transpose(tmp, [0, 3, 4, 1, 5, 2])\n\ny = np.reshape(tmp, [b, c // (blocksize**2), h * blocksize, w * blocksize])\n\n\nIn the CRD mode, elements along the depth dimension from the input tensor are rearranged in the\nfollowing order: column, row, and the depth. The output y is computed from the input x as below:\n\nb, c, h, w = x.shape\n\ntmp = np.reshape(x, [b, c // (blocksize ** 2), blocksize, blocksize, h, w])\n\ntmp = np.transpose(tmp, [0, 1, 4, 2, 5, 3])\n\ny = np.reshape(tmp, [b, c // (blocksize ** 2), h * blocksize, w * blocksize])\n\n" -----f -input: "x" -input: "x_scale" -input: "x_zero_point" -output: "y" -name: "DequantizeLinear" -op_type: "DequantizeLinear" -attribute { - name: "x-types" - strings: "int32" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "x_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "x_zero_point-types" - strings: "int32" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nThe linear dequantization operator. It consumes a quantized tensor, a scale, a zero point to compute the full precision tensor.\nThe dequantization formula is y = (x - x_zero_point) * x_scale. \'x_scale\' and \'x_zero_point\' must have same shape.\n\'x_zero_point\' and \'x\' must have same type. \'x\' and \'y\' must have same shape. In the case of dequantizing int32,\nthere\'s no zero point (zero point is supposed to be 0).\n" -----f -input: "X" -output: "Y" -name: "Det" -op_type: "Det" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nDet calculates determinant of a square matrix or batches of square matrices.\nDet takes one input tensor of shape `[*, M, M]`, where `*` is zero or more batch dimensions,\nand the inner-most 2 dimensions form square matrices.\nThe output is a tensor of shape `[*]`, containing the determinants of all input submatrices.\ne.g., When the input is 2-D, the output is a scalar(shape is empty: `[]`).\n" -----f -input: "X" -output: "Y" -name: "DictVectorizer" -op_type: "DictVectorizer" -attribute { - name: "int64_vocabulary" - s: "" - type: INTS -} -attribute { - name: "string_vocabulary" - s: "" - type: STRINGS -} -attribute { - name: "X-types" - strings: "map(int64,float" - strings: "map(int64,string" - strings: "map(string,int64" - strings: "map(string,float" - strings: "map(string,double" - strings: "map(int64,double" - type: STRINGS -} -doc_string: "\n Uses an index mapping to convert a dictionary to an array.
\n Given a dictionary, each key is looked up in the vocabulary attribute corresponding to\n the key type. The index into the vocabulary array at which the key is found is then\n used to index the output 1-D tensor \'Y\' and insert into it the value found in the dictionary \'X\'.
\n The key type of the input map must correspond to the element type of the defined vocabulary attribute.\n Therefore, the output array will be equal in length to the index mapping vector parameter.\n All keys in the input dictionary must be present in the index mapping vector.\n For each item in the input dictionary, insert its value in the output array.\n Any keys not present in the input dictionary, will be zero in the output array.
\n For example: if the ``string_vocabulary`` parameter is set to ``[\"a\", \"c\", \"b\", \"z\"]``,\n then an input of ``{\"a\": 4, \"c\": 8}`` will produce an output of ``[4, 8, 0, 0]``.\n " -----f -input: "A" -input: "B" -output: "C" -name: "Div" -op_type: "Div" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPerforms element-wise binary division (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "data" -input: "ratio" -input: "training_mode" -output: "output" -output: "mask" -name: "Dropout" -op_type: "Dropout" -attribute { - name: "seed" - s: "" - type: INT -} -attribute { - name: "data-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "ratio-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "training_mode-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nDropout takes an input floating-point tensor, an optional input ratio (floating-point scalar) and an optional input training_mode (boolean scalar). It produces two tensor outputs,\noutput (floating-point tensor) and mask (optional `Tensor`). If `training_mode` is true then the output Y will be a random dropout;\nNote that this Dropout scales the masked input data by the following equation, so to convert the trained model into inference mode,\nthe user can simply not pass `training_mode` input or set it to false.\n```\noutput = scale * data * mask,\n```\nwhere\n```\nscale = 1. / (1. - ratio).\n```\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "x" -output: "y" -output: "y_scale" -output: "y_zero_point" -name: "DynamicQuantizeLinear" -op_type: "DynamicQuantizeLinear" -attribute { - name: "x-types" - strings: "float" - type: STRINGS -} -doc_string: "\nA Function to fuse calculation for Scale, Zero Point and FP32->8Bit convertion of FP32 Input data.\nOutputs Scale, ZeroPoint and Quantized Input for a given FP32 Input.\nScale is calculated as:\n```\n y_scale = (max(x) - min(x))/(qmax - qmin)\n * where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8\n * data range is adjusted to include 0.\n```\nZero point is calculated as:\n```\nintermediate_zero_point = qmin - min(x)/y_scale\ny_zero_point = cast(round(saturate(itermediate_zero_point)))\n* where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8\n* for saturation, it saturates to [0, 255] if it\'s uint8, or [-127, 127] if it\'s int8. Right now only uint8 is supported.\n* rounding to nearest ties to even.\n```\nData quantization formula is:\n```\ny = saturate (round (x / y_scale) + y_zero_point)\n* for saturation, it saturates to [0, 255] if it\'s uint8, or [-127, 127] if it\'s int8. Right now only uint8 is supported.\n* rounding to nearest ties to even.\n```\n" -----f -input: "Inputs" -output: "Output" -name: "Einsum" -op_type: "Einsum" -attribute { - name: "equation" - s: "" - type: STRING -} -attribute { - name: "Inputs-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nAn einsum of the form ```term1, term2 -> output-term``` produces an output tensor using the following equation\n\n```output[output-term] = reduce-sum( input1[term1] * input2[term] )```\n\nwhere the reduce-sum performs a summation over all the indices occurring in in the input terms (term1, term2)\nthat do not occur in the output-term.\n\nThe Einsum operator evaluates algebraic tensor operations on a sequence of tensors, using the Einstein summation\nconvention. The equation string contains a comma-separated sequence of lower case letters. Each term corresponds to\nan operand tensor, and the characters within the terms correspond to operands dimensions.\n\nThis sequence may be followed by \"->\" to separate the left and right hand side of the equation.\nIf the equation contains \"->\" followed by the right-hand side, the explicit (not classical) form of the Einstein\nsummation is performed, and the right-hand side indices indicate output tensor dimensions. In other cases,\noutput indices are (implicitly) set to the alphabetically sorted sequence of indices appearing exactly once in the\nequation.\n\nWhen a dimension character is repeated in the left-hand side, it represents summation along the dimension.\n\nThe equation may contain ellipsis (\"...\") to enable broadcasting. Ellipsis must indicate a fixed number of dimensions.\nSpecifically, every occurrence of ellipsis in the equation must represent the same number of dimensions.\nThe right-hand side may contain exactly one ellipsis. In implicit mode, the ellipsis dimensions are set to the\nbeginning of the output. The equation string may contain space (U+0020) character.\n" -----f -input: "X" -output: "Y" -name: "Elu" -op_type: "Elu" -attribute { - name: "alpha" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nElu takes one input data (Tensor) and produces one output data\n(Tensor) where the function `f(x) = alpha * (exp(x) - 1.) for x <\n0`, `f(x) = x for x >= 0`., is applied to the tensor elementwise.\n\n" -----f -input: "A" -input: "B" -output: "C" -name: "Equal" -op_type: "Equal" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "input" -output: "output" -name: "Erf" -op_type: "Erf" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the error function of the given input tensor element-wise.\n" -----f -input: "input" -output: "output" -name: "Exp" -op_type: "Exp" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the exponential of the given input tensor, element-wise.\n" -----f -input: "input" -input: "shape" -output: "output" -name: "Expand" -op_type: "Expand" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "shape-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nBroadcast the input tensor following the given shape and the broadcast rule.\nThe broadcast rule is similar to numpy.array(input) * numpy.ones(shape):\nDimensions are right alignment;\nTwo corresponding dimension must have the same value, or one of them is equal to 1.\nAlso, this operator is similar to numpy.broadcast_to(input, shape),\nbut the major difference is numpy.broadcast_to() does not allow shape to be smaller than input.size().\nIt is possible that the output.shape is not equal to shape, when some dimensions in shape is equal to 1,\nor the shape.ndim < input.shape.ndim.\n" -----f -input: "input" -output: "output" -name: "EyeLike" -op_type: "EyeLike" -attribute { - name: "dtype" - s: "" - type: INT -} -attribute { - name: "k" - i: 0 - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "float16" - strings: "int32" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGenerate a 2D tensor (matrix) with ones on the diagonal and zeros everywhere else. Only 2D\ntensors are supported, i.e. input T1 must be of rank 2. The shape of the output tensor is the\nsame as the input tensor. The data type can be specified by the \'dtype\' argument. If\n\'dtype\' is not specified, then the type of input tensor is used. By default, the main diagonal\nis populated with ones, but attribute \'k\' can be used to populate upper or lower diagonals.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message and be valid as an output type.\n" -----f -input: "X" -output: "Y" -name: "FeatureVectorizer" -op_type: "FeatureVectorizer" -attribute { - name: "inputdimensions" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Concatenates input tensors into one continuous output.
\n All input shapes are 2-D and are concatenated along the second dimention. 1-D tensors are treated as [1,C].\n Inputs are copied to the output maintaining the order of the input arguments.
\n All inputs must be integers or floats, while the output will be all floating point values.\n" -----f -input: "input" -output: "output" -name: "Flatten" -op_type: "Flatten" -attribute { - name: "axis" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nFlattens the input tensor into a 2D matrix. If input tensor has shape\n(d_0, d_1, ... d_n) then the output will have shape\n(d_0 X d_1 ... d_(axis-1), d_axis X d_(axis+1) ... X dn).\n" -----f -input: "X" -output: "Y" -name: "Floor" -op_type: "Floor" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nFloor takes one input data (Tensor) and produces one output data\n(Tensor) where the floor is, y = floor(x), is applied to\nthe tensor elementwise.\n" -----f -input: "X" -input: "W" -input: "R" -input: "B" -input: "sequence_lens" -input: "initial_h" -output: "Y" -output: "Y_h" -name: "GRU" -op_type: "GRU" -attribute { - name: "activation_alpha" - s: "" - type: FLOATS -} -attribute { - name: "activation_beta" - s: "" - type: FLOATS -} -attribute { - name: "activations" - s: "" - type: STRINGS -} -attribute { - name: "clip" - s: "" - type: FLOAT -} -attribute { - name: "direction" - s: "forward" - type: STRING -} -attribute { - name: "hidden_size" - s: "" - type: INT -} -attribute { - name: "linear_before_reset" - i: 0 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "sequence_lens-types" - strings: "int32" - type: STRINGS -} -attribute { - name: "initial_h-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nComputes an one-layer GRU. This operator is usually supported via some custom\nimplementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`z` - update gate\n\n`r` - reset gate\n\n`h` - hidden gate\n\n`t` - time step (t-1 means previous time step)\n\n`W[zrh]` - W parameter weight matrix for update, reset, and hidden gates\n\n`R[zrh]` - R recurrence weight matrix for update, reset, and hidden gates\n\n`Wb[zrh]` - W bias vectors for update, reset, and hidden gates\n\n`Rb[zrh]` - R bias vectors for update, reset, and hidden gates\n\n`WB[zrh]` - W parameter weight matrix for backward update, reset, and hidden gates\n\n`RB[zrh]` - R recurrence weight matrix for backward update, reset, and hidden gates\n\n`WBb[zrh]` - W bias vectors for backward update, reset, and hidden gates\n\n`RBb[zrh]` - R bias vectors for backward update, reset, and hidden gates\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Sigmoid, g=Tanh):\n\n - zt = f(Xt*(Wz^T) + Ht-1*(Rz^T) + Wbz + Rbz)\n\n - rt = f(Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr)\n\n - ht = g(Xt*(Wh^T) + (rt (.) Ht-1)*(Rh^T) + Rbh + Wbh) # default, when linear_before_reset = 0\n\n - ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) + Wbh) # when linear_before_reset != 0\n\n - Ht = (1 - zt) (.) ht + zt (.) Ht-1\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "data" -input: "indices" -output: "output" -name: "Gather" -op_type: "Gather" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nGiven `data` tensor of rank r >= 1, and `indices` tensor of rank q, gather\nentries of the axis dimension of `data` (by default outer-most one as axis=0) indexed by `indices`, and concatenates\nthem in an output tensor of rank q + (r - 1).\n\naxis = 0 :\n\nLet\nk = indices[i_{0}, ..., i_{q-1}]\nThen\noutput[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[k , j_{0}, ..., j_{r-2}]\n\n```\n data = [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ]\n indices = [\n [0, 1],\n [1, 2],\n ]\n output = [\n [\n [1.0, 1.2],\n [2.3, 3.4],\n ],\n [\n [2.3, 3.4],\n [4.5, 5.7],\n ],\n ]\n```\naxis = 1 :\n\nLet\nk = indices[i_{0}, ..., i_{q-1}]\nThen\noutput[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[j_{0}, k, j_{1}, ..., j_{r-2}]\n\n```\n data = [\n [1.0, 1.2, 1.9],\n [2.3, 3.4, 3.9],\n [4.5, 5.7, 5.9],\n ]\n indices = [\n [0, 2],\n ]\n axis = 1,\n output = [\n [\n [1.0, 1.9],\n [2.3, 3.9],\n [4.5, 5.9],\n ],\n ]\n```\n" -----f -input: "data" -input: "indices" -output: "output" -name: "GatherElements" -op_type: "GatherElements" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n\nGatherElements takes two inputs `data` and `indices` of the same rank r >= 1\nand an optional attribute `axis` that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). It is an indexing operation\nthat produces its output by indexing into the input data tensor at index\npositions determined by elements of the `indices` tensor.\nIts output shape is the same as the shape of `indices` and consists of one value\n(gathered from the `data`) for each element in `indices`.\n\nFor instance, in the 3-D case (r = 3), the output produced is determined\nby the following equations: \n```\n out[i][j][k] = input[index[i][j][k]][j][k] if axis = 0,\n out[i][j][k] = input[i][index[i][j][k]][k] if axis = 1,\n out[i][j][k] = input[i][j][index[i][j][k]] if axis = 2,\n```\n\nThis operator is also the inverse of ScatterElements. It is similar to Torch\'s gather operation.\n\nExample 1:\n```\n data = [\n [1, 2],\n [3, 4],\n ]\n indices = [\n [0, 0],\n [1, 0],\n ]\n axis = 1\n output = [\n [\n [1, 1],\n [4, 3],\n ],\n ]\n```\nExample 2:\n```\n data = [\n [1, 2, 3],\n [4, 5, 6],\n [7, 8, 9],\n ]\n indices = [\n [1, 2, 0],\n [2, 0, 0],\n ]\n axis = 0\n output = [\n [\n [4, 8, 3],\n [7, 2, 3],\n ],\n ]\n```\n" -----f -input: "data" -input: "indices" -output: "output" -name: "GatherND" -op_type: "GatherND" -attribute { - name: "batch_dims" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nGiven `data` tensor of rank `r` >= 1, `indices` tensor of rank `q` >= 1, and `batch_dims` integer `b`, this operator gathers \nslices of `data` into an output tensor of rank `q + r - indices_shape[-1] - 1 - b`.\n\n`indices` is an q-dimensional integer tensor, best thought of as a `(q-1)`-dimensional tensor of index-tuples into `data`, \nwhere each element defines a slice of `data`\n\n`batch_dims` (denoted as `b`) is an integer indicating the number of batch dimensions, i.e the leading `b` number of dimensions of \n`data` tensor and `indices` are representing the batches, and the gather starts from the `b+1` dimension. \n\nSome salient points about the inputs\' rank and shape:\n \n1) r >= 1 and q >= 1 are to be honored. There is no dependency condition to be met between ranks `r` and `q`\n\n2) The first `b` dimensions of the shape of `indices` tensor and `data` tensor must be equal.\n\n3) b < min(q, r) is to be honored.\n\n4) The `indices_shape[-1]` should have a value between 1 (inclusive) and rank `r-b` (inclusive) \n\n5) All values in `indices` are expected to be within bounds [-s, s-1] along axis of size `s` (i.e.) `-data_shape[i] <= indices[...,i] <= data_shape[i] - 1`.\n It is an error if any of the index values are out of bounds.\n\nThe output is computed as follows:\n\nThe output tensor is obtained by mapping each index-tuple in the `indices` tensor to the corresponding slice of the input `data`.\n \n1) If `indices_shape[-1] > r-b` => error condition\n\n2) If `indices_shape[-1] == r-b`, since the rank of `indices` is `q`, `indices` can be thought of as `N` `(q-b-1)`-dimensional tensors\n containing 1-D tensors of dimension `r-b`, where `N` is an integer equals to the product of 1 and all the elements in the batch dimensions \n of the indices_shape. Let us think of each such `r-b` ranked tensor as `indices_slice`. Each *scalar value* corresponding to `data[0:b-1,indices_slice]` \n is filled into the corresponding location of the `(q-b-1)`-dimensional tensor to form the `output` tensor (Example 1 below)\n\n3) If `indices_shape[-1] < r-b`, since the rank of `indices` is `q`, `indices` can be thought of as `N` `(q-b-1)`-dimensional tensor\n containing 1-D tensors of dimension `< r-b`. Let us think of each such tensors as `indices_slice`. Each *tensor slice* corresponding \n to `data[0:b-1, indices_slice , :]` is filled into the corresponding location of the `(q-b-1)`-dimensional tensor \n to form the `output` tensor (Examples 2, 3, 4 and 5 below)\n\nThis operator is the inverse of `ScatterND`.\n\n`Example 1`\n\n batch_dims = 0\n\n data = [[0,1],[2,3]] # data_shape = [2, 2]\n\n indices = [[0,0],[1,1]] # indices_shape = [2, 2]\n\n output = [0,3] # output_shape = [2]\n\n`Example 2`\n\n batch_dims = 0\n\n data = [[0,1],[2,3]] # data_shape = [2, 2]\n\n indices = [[1],[0]] # indices_shape = [2, 1]\n\n output = [[2,3],[0,1]] # output_shape = [2, 2]\n\n`Example 3`\n\n batch_dims = 0\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[0,1],[1,0]] # indices_shape = [2, 2]\n\n output = [[2,3],[4,5]] # output_shape = [2, 2] \n\n`Example 4`\n\n batch_dims = 0\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[[0,1]],[[1,0]]] # indices_shape = [2, 1, 2]\n\n output = [[[2,3]],[[4,5]]] # output_shape = [2, 1, 2] \n\n`Example 5`\n\n batch_dims = 1\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[1],[0]] # indices_shape = [2, 1]\n\n output = [[2,3],[4,5]] # output_shape = [2, 2] \n\n\n" -----f -input: "A" -input: "B" -input: "C" -output: "Y" -name: "Gemm" -op_type: "Gemm" -attribute { - name: "alpha" - f: 1.0 - type: FLOAT -} -attribute { - name: "beta" - f: 1.0 - type: FLOAT -} -attribute { - name: "transA" - i: 0 - type: INT -} -attribute { - name: "transB" - i: 0 - type: INT -} -attribute { - name: "A-types" - strings: "int32" - strings: "float16" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "int32" - strings: "float16" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "C-types" - strings: "int32" - strings: "float16" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "General Matrix multiplication:\nhttps://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms#Level_3\n\nA\' = transpose(A) if transA else A\n\nB\' = transpose(B) if transB else B\n\nCompute Y = alpha * A\' * B\' + beta * C, where input tensor A has shape (M, K) or (K, M),\ninput tensor B has shape (K, N) or (N, K), input tensor C is broadcastable to shape (M, N),\nand output tensor Y has shape (M, N). A will be transposed before doing the\ncomputation if attribute transA is non-zero, same for B and transB.\nThis operator supports **unidirectional broadcasting** (tensor C should be unidirectional broadcastable to tensor A * B); for more details please check [the doc](Broadcasting.md).\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "X" -output: "Y" -name: "GlobalAveragePool" -op_type: "GlobalAveragePool" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n GlobalAveragePool consumes an input tensor X and applies average pooling across\n the values in the same channel. This is equivalent to AveragePool with kernel size\n equal to the spatial dimension of input tensor." -----f -input: "X" -output: "Y" -name: "GlobalLpPool" -op_type: "GlobalLpPool" -attribute { - name: "p" - i: 2 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n GlobalLpPool consumes an input tensor X and applies lp pool pooling across\n the values in the same channel. This is equivalent to LpPool with kernel size\n equal to the spatial dimension of input tensor." -----f -input: "X" -output: "Y" -name: "GlobalMaxPool" -op_type: "GlobalMaxPool" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n GlobalMaxPool consumes an input tensor X and applies max pooling across\n the values in the same channel. This is equivalent to MaxPool with kernel size\n equal to the spatial dimension of input tensor." -----f -input: "Inputs" -output: "Outputs" -name: "Gradient" -op_type: "Gradient" -attribute { - name: "xs" - s: "" - type: STRINGS -} -attribute { - name: "y" - s: "" - type: STRING -} -attribute { - name: "zs" - s: "" - type: STRINGS -} -attribute { - name: "Inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGradient operator computes the partial derivatives of a specific tensor w.r.t.\nsome other tensors. This operator is widely used in gradient-based training\nalgorithms. To illustrate its use, let\'s consider a computation graph,\n\n```\nX -----.\n |\n v\nW --> Conv --> H --> Gemm --> Y\n ^\n |\n Z\n```\n\n, where W and Z are trainable tensors. Note that operators\' attributes are\nomitted for the sake of simplicity. Let dY/dW (dY/dZ) be the gradient of\nY with respect to W (Z). The user can compute gradient by inserting Gradient\noperator to form another graph shown below.\n\n```\nW --> Conv --> H --> Gemm --> Y\n| ^ ^\n| | |\n| X Z\n| | |\n| | .----------\'\n| | | (W/Z/X is the 1st/2nd/3rd input of Gradient as shown in\n| | | \"xs\" followed by \"zs\")\n| v v\n\'---> Gradient(xs=[\"W\", \"Z\"], zs=[\"X\"], y=\"Y\")\n | |\n | \'-----------------------------------> dY/dW (1st output of Gradient)\n |\n \'---------------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nBy definition, the tensor \"y\" is a function of independent variables in \"xs\"\nand \"zs\". Since we only compute the gradient of \"y\" w.r.t. the differentiable\nvariables in \"xs\", this Gradient only outputs dY/dW and dY/dZ. Note that \"H\"\ncannot appear in \"xs\" and \"zs\". The reason is that \"H\" can be determined by\ntensors \"W\" and \"X\" and therefore \"H\" is not an independent variable.\n\nAll outputs are optional. If needed, for example, user can assign an empty\nstring to the 1st output name of that Gradient to skip the generation of dY/dW.\nNote that the concept of optional outputs can also be found in ONNX\'s RNN, GRU,\nand LSTM.\n\nGradient operator can compute derivative against intermediate tensors. For\nexample, the gradient of Y with respect to H can be done via\n\n```\nW --> Conv --> H --> Gemm --> Y\n ^ | ^\n | | |\n X | Z\n .-------\' |\n | .----------\'\n | | (H/Z is the 1st/2nd input of Gradient as shown in \"xs\")\n v v\n Gradient(xs=[\"H\", \"Z\"], y=\"Y\")\n | |\n | \'-----------------------------------> dY/dH (1st output of Gradient)\n |\n \'---------------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nIt is possible to represent high-order differentiation using Gradient operators.\nFor example, given the following linear model:\n\n```\nW --> Gemm --> Y --> Loss --> O\n ^ ^\n | |\n X L\n```\n\nTo compute the 2nd order derivative of O with respect to W (denoted by\nd^2O/dW^2), one can do\n\n```\nW --> Gemm --> Y --> Loss --> O\n| ^ ^\n| | |\n| X .------------L\n| | | |\n| | | v\n+------+-+> Gradient(xs=[\"X\", \"W\"], zs=[\"L\"], y=\"O\") ---> dO/dX (1st output of Gradient)\n| | | |\n| | | \'---> dO/dW (2nd output of Gradient)\n| v v\n\'---> Gradient(xs=[\"X\", \"W\"], zs=[\"L\"], y=\"dO/dW\") ---> d(dO/dW)dX (1st output of\n | Gradient)\n |\n |\n \'---> d^2O/dW^2 (2nd output of Gradient)\n```\n\nThe tensors named in attributes \"xs\", \"zs\", and \"y\" define the differentiated\ncomputation graph, and the inputs to Gradient node define the values at\nwhich the gradient is computed. We can feed different tensors to the identified\ngraph. For example, one can compute the gradient of Y with respect to H at \na specific value of H, H_1, by providing that value as an input to the Gradient\nnode.\n\n```\nW --> Conv --> H --> Gemm --> Y\n ^ ^\n | |\n X Z\n\n Z_1 (2nd input of Gradient)\n |\n v\nH_1 --> Gradient(xs=[\"H\", \"Z\"], y=\"Y\") ---> dY/dH when H = H_1 and Y = Y_1.\n |\n \'------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nWhen the inputs of Gradient are the tensors named in \"xs\" and \"zs\", the\ncomputation can be optimized. More specifically, intermediate variables in\nforward pass can be reused if the gradient is computed via reverse-mode\nauto-differentiation.\n\n" -----f -input: "Inputs" -output: "Outputs" -name: "GraphCall" -op_type: "GraphCall" -attribute { - name: "graph_name" - s: "" - type: STRING -} -attribute { - name: "Inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nThe GraphCall operator invokes a graph inside TrainingInfoProto\'s\nalgorithm field. The GraphCall inputs and outputs are bound to those of\ninvoked graph by position. If a graph input has an initializer, that input\nis considered optional. All graph outputs are optional.\n\nBelow Python syntax is used for describing dictionary and list.\n\nAssume that ModelProto\'s graph field has\n- name: \"MyInferenceGraph\"\n- input: [\"X\", \"W\", \"Z\"]\n- initializer: [W]\n- output: [\"Y\"]\n\nas visualized below for inference.\n\n```\nX -----.\n |\n v\nW --> Conv --> H --> Gemm --> Y\n ^\n |\n Z\n```\n\nAssume that the training algorithm contains\n\n- inputs: [\"X_1\", \"Z_1\", \"C\"]\n- initializer: [T]\n- outputs: [\"W_new\"]\n\nwith a dictionary\n\n- update_binding: {\"W\": \"W_new\", \"T\": \"T_new\"}\n\nInside the training algorithm graph, one can invoke the inference\ngraph via adding a GraphCall node with\n\n- inputs: [\"X_1\", \"W\", Z_1\"]\n- outputs: [\"Y_1\"]\n- an attribute graph_name=\"MyInferenceGraph\",\n\nThe initializers, \"W\" and \"T\" in this case, in update_binding\nare considered globally-visible and mutable variables, which\ncan be used as inputs of operators in the training graph.\n\nAn example training algorithm graph may look like\n\n```\n.-------- W (a global and mutable variable from\n| | the inference graph)\n| |\n| .-----\'-----------.\n| | |\n| | v\n| | .-- X_1 --> GraphCall(graph_name=\"MyInferenceGraph\")\n| | | | |\n| | | | |\n| | | Z_1 -----\' |\n| | | | V\n| | | | Y_1 ---> Loss ---> O\n| | | | ^\n| | | | |\n| | `--. | C\n| | | | |\n| | | | .----------------\'\n| | | | |\n| | v v v\n| `--> Gradient(xs=[\"W\"], zs=[\"X_1\", \"Z_1\", \"C\"], y=\"O\")\n| |\n| v\n| dO_dW (gradient of W) 1 (a scalar one)\n| | |\n| V v\n| Div <--- T ------------> Add ---> T_new\n| | (T is the number of training iterations.\n| | T is also globally visible and mutable.)\n| v\n`-----> Sub ----> W_new\n```\n\nwhere Loss is a dummy node which computes the minimized objective function.\n\nThe variable \"W\" is an optional input in the called graph.\nIf the user omits it, the input list of GraphCall becomes [\"X_1\", \"\", \"Z_1\"].\nIn this case, from the view of computation graph, the Conv operator invoked by\nGraphCall\'s may be still connected the global \"W\" variable and therefore the\nstructure of the computation graph is unchanged.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Greater" -op_type: "Greater" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `greater` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "A" -input: "B" -output: "C" -name: "GreaterOrEqual" -op_type: "GreaterOrEqual" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `greater_equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Y" -name: "HardSigmoid" -op_type: "HardSigmoid" -attribute { - name: "alpha" - f: 0.2 - type: FLOAT -} -attribute { - name: "beta" - f: 0.5 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nHardSigmoid takes one input data (Tensor) and produces one output data\n(Tensor) where the HardSigmoid function, y = max(0, min(1, alpha * x + beta)),\nis applied to the tensor elementwise.\n" -----f -input: "input" -output: "output" -name: "Hardmax" -op_type: "Hardmax" -attribute { - name: "axis" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe operator computes the hardmax (1 for the first maximum value, and 0 for all others) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the hardmax values of the corresponding input.\n" -----f -input: "input" -output: "output" -name: "Identity" -op_type: "Identity" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "Identity operator" -----f -input: "cond" -output: "outputs" -name: "If" -op_type: "If" -attribute { - name: "else_branch" - s: "" - type: GRAPH -} -attribute { - name: "then_branch" - s: "" - type: GRAPH -} -attribute { - name: "cond-types" - strings: "bool" - type: STRINGS -} -doc_string: "If conditional" -----f -input: "X" -output: "Y" -name: "Imputer" -op_type: "Imputer" -attribute { - name: "imputed_value_floats" - s: "" - type: FLOATS -} -attribute { - name: "imputed_value_int64s" - s: "" - type: INTS -} -attribute { - name: "replaced_value_float" - f: 0.0 - type: FLOAT -} -attribute { - name: "replaced_value_int64" - i: 0 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Replaces inputs that equal one value with another, leaving all other elements alone.
\n This operator is typically used to replace missing values in situations where they have a canonical\n representation, such as -1, 0, NaN, or some extreme value.
\n One and only one of imputed_value_floats or imputed_value_int64s should be defined -- floats if the input tensor\n holds floats, integers if the input tensor holds integers. The imputed values must all fit within the\n width of the tensor element type. One and only one of the replaced_value_float or replaced_value_int64 should be defined,\n which one depends on whether floats or integers are being processed.
\n The imputed_value attribute length can be 1 element, or it can have one element per input feature.
In other words, if the input tensor has the shape [*,F], then the length of the attribute array may be 1 or F. If it is 1, then it is broadcast along the last dimension and applied to each feature.\n" -----f -input: "input" -input: "scale" -input: "B" -output: "output" -name: "InstanceNormalization" -op_type: "InstanceNormalization" -attribute { - name: "epsilon" - f: 1e-05 - type: FLOAT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "scale-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCarries out instance normalization as described in the paper\nhttps://arxiv.org/abs/1607.08022.\n\ny = scale * (x - mean) / sqrt(variance + epsilon) + B,\nwhere mean and variance are computed per instance per channel.\n\n" -----f -input: "X" -output: "Y" -name: "IsInf" -op_type: "IsInf" -attribute { - name: "detect_negative" - i: 1 - type: INT -} -attribute { - name: "detect_positive" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - type: STRINGS -} -doc_string: "Map infinity to true and other values to false." -----f -input: "X" -output: "Y" -name: "IsNaN" -op_type: "IsNaN" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "Returns which elements of the input are NaN." -----f -input: "X" -output: "Y" -name: "LRN" -op_type: "LRN" -attribute { - name: "alpha" - f: 0.0001 - type: FLOAT -} -attribute { - name: "beta" - f: 0.75 - type: FLOAT -} -attribute { - name: "bias" - f: 1.0 - type: FLOAT -} -attribute { - name: "size" - s: "" - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nLocal Response Normalization proposed in the [AlexNet paper](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf).\nIt normalizes over local input regions.\nThe local region is defined across the channels. For an element X[n, c, d1, ..., dk] in a tensor\nof shape (N x C x D1 x D2, ..., Dk), its region is\n{X[n, i, d1, ..., dk] | max(0, c - floor((size - 1) / 2)) <= i <= min(C - 1, c + ceil((size - 1) / 2))}.\n\nsquare_sum[n, c, d1, ..., dk] = sum(X[n, i, d1, ..., dk] ^ 2),\nwhere max(0, c - floor((size - 1) / 2)) <= i <= min(C - 1, c + ceil((size - 1) / 2)).\n\nY[n, c, d1, ..., dk] = X[n, c, d1, ..., dk] / (bias + alpha / size * square_sum[n, c, d1, ..., dk] ) ^ beta\n" -----f -input: "X" -input: "W" -input: "R" -input: "B" -input: "sequence_lens" -input: "initial_h" -input: "initial_c" -input: "P" -output: "Y" -output: "Y_h" -output: "Y_c" -name: "LSTM" -op_type: "LSTM" -attribute { - name: "activation_alpha" - s: "" - type: FLOATS -} -attribute { - name: "activation_beta" - s: "" - type: FLOATS -} -attribute { - name: "activations" - s: "" - type: STRINGS -} -attribute { - name: "clip" - s: "" - type: FLOAT -} -attribute { - name: "direction" - s: "forward" - type: STRING -} -attribute { - name: "hidden_size" - s: "" - type: INT -} -attribute { - name: "input_forget" - i: 0 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "sequence_lens-types" - strings: "int32" - type: STRINGS -} -attribute { - name: "initial_h-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "initial_c-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "P-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nComputes an one-layer LSTM. This operator is usually supported via some\ncustom implementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`i` - input gate\n\n`o` - output gate\n\n`f` - forget gate\n\n`c` - cell gate\n\n`t` - time step (t-1 means previous time step)\n\n`W[iofc]` - W parameter weight matrix for input, output, forget, and cell gates\n\n`R[iofc]` - R recurrence weight matrix for input, output, forget, and cell gates\n\n`Wb[iofc]` - W bias vectors for input, output, forget, and cell gates\n\n`Rb[iofc]` - R bias vectors for input, output, forget, and cell gates\n\n`P[iof]` - P peephole weight vector for input, output, and forget gates\n\n`WB[iofc]` - W parameter weight matrix for backward input, output, forget, and cell gates\n\n`RB[iofc]` - R recurrence weight matrix for backward input, output, forget, and cell gates\n\n`WBb[iofc]` - W bias vectors for backward input, output, forget, and cell gates\n\n`RBb[iofc]` - R bias vectors for backward input, output, forget, and cell gates\n\n`PB[iof]` - P peephole weight vector for backward input, output, and forget gates\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Sigmoid, g=Tanh, h=Tanh):\n\n - it = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Pi (.) Ct-1 + Wbi + Rbi)\n\n - ft = f(Xt*(Wf^T) + Ht-1*(Rf^T) + Pf (.) Ct-1 + Wbf + Rbf)\n\n - ct = g(Xt*(Wc^T) + Ht-1*(Rc^T) + Wbc + Rbc)\n\n - Ct = ft (.) Ct-1 + it (.) ct\n\n - ot = f(Xt*(Wo^T) + Ht-1*(Ro^T) + Po (.) Ct + Wbo + Rbo)\n\n - Ht = ot (.) h(Ct)\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "X" -output: "Y" -name: "LabelEncoder" -op_type: "LabelEncoder" -attribute { - name: "default_float" - f: -0.0 - type: FLOAT -} -attribute { - name: "default_int64" - i: -1 - type: INT -} -attribute { - name: "default_string" - s: "_Unused" - type: STRING -} -attribute { - name: "keys_floats" - s: "" - type: FLOATS -} -attribute { - name: "keys_int64s" - s: "" - type: INTS -} -attribute { - name: "keys_strings" - s: "" - type: STRINGS -} -attribute { - name: "values_floats" - s: "" - type: FLOATS -} -attribute { - name: "values_int64s" - s: "" - type: INTS -} -attribute { - name: "values_strings" - s: "" - type: STRINGS -} -attribute { - name: "X-types" - strings: "string" - strings: "float" - strings: "int64" - type: STRINGS -} -doc_string: "\n Maps each element in the input tensor to another value.
\n The mapping is determined by the two parallel attributes, \'keys_*\' and\n \'values_*\' attribute. The i-th value in the specified \'keys_*\' attribute\n would be mapped to the i-th value in the specified \'values_*\' attribute. It\n implies that input\'s element type and the element type of the specified\n \'keys_*\' should be identical while the output type is identical to the\n specified \'values_*\' attribute. If an input element can not be found in the\n specified \'keys_*\' attribute, the \'default_*\' that matches the specified\n \'values_*\' attribute may be used as its output value.
\n Let\'s consider an example which maps a string tensor to an integer tensor.\n Assume and \'keys_strings\' is [\"Amy\", \"Sally\"], \'values_int64s\' is [5, 6],\n and \'default_int64\' is \'-1\'. The input [\"Dori\", \"Amy\", \"Amy\", \"Sally\",\n \"Sally\"] would be mapped to [-1, 5, 5, 6, 6].
\n Since this operator is an one-to-one mapping, its input and output shapes\n are the same. Notice that only one of \'keys_*\'/\'values_*\' can be set.
\n For key look-up, bit-wise comparison is used so even a float NaN can be\n mapped to a value in \'values_*\' attribute.
\n" -----f -input: "X" -output: "Y" -name: "LeakyRelu" -op_type: "LeakyRelu" -attribute { - name: "alpha" - f: 0.01 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nLeakyRelu takes input data (Tensor) and an argument alpha, and produces one\noutput data (Tensor) where the function `f(x) = alpha * x for x < 0`,\n`f(x) = x for x >= 0`, is applied to the data tensor elementwise.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Less" -op_type: "Less" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `less` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "A" -input: "B" -output: "C" -name: "LessOrEqual" -op_type: "LessOrEqual" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `less_equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Y" -output: "Z" -name: "LinearClassifier" -op_type: "LinearClassifier" -attribute { - name: "classlabels_ints" - s: "" - type: INTS -} -attribute { - name: "classlabels_strings" - s: "" - type: STRINGS -} -attribute { - name: "coefficients" - s: "" - type: FLOATS -} -attribute { - name: "intercepts" - s: "" - type: FLOATS -} -attribute { - name: "multi_class" - i: 0 - type: INT -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Linear classifier\n" -----f -input: "X" -output: "Y" -name: "LinearRegressor" -op_type: "LinearRegressor" -attribute { - name: "coefficients" - s: "" - type: FLOATS -} -attribute { - name: "intercepts" - s: "" - type: FLOATS -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "targets" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Generalized linear regression evaluation.
\n If targets is set to 1 (default) then univariate regression is performed.
\n If targets is set to M then M sets of coefficients must be passed in as a sequence\n and M results will be output for each input n in N.
\n The coefficients array is of length n, and the coefficients for each target are contiguous.\n Intercepts are optional but if provided must match the number of targets.\n" -----f -input: "input" -output: "output" -name: "Log" -op_type: "Log" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the natural log of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "LogSoftmax" -op_type: "LogSoftmax" -attribute { - name: "axis" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe operator computes the logsoftmax (log of softmax) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the logsoftmax values of the corresponding input.\n" -----f -input: "M" -input: "cond" -input: "v_initial" -output: "v_final_and_scan_outputs" -name: "Loop" -op_type: "Loop" -attribute { - name: "body" - s: "" - type: GRAPH -} -attribute { - name: "M-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "cond-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "v_initial-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGeneric Looping construct. This loop has multiple termination conditions:\n\n1) Trip count. Iteration count specified at runtime. Set by\n specifying the input M. Optional. Set to empty string to omit.\n Note that a static trip count (specified at graph construction time) can be\n specified by passing in a constant node for input M.\n2) Loop termination condition. This is an input to the op that determines\n whether to run the first iteration and also a loop-carried dependency for\n the body graph. The body graph must yield a value for the condition variable,\n whether this input is provided or not.\n\nThis table summarizes the operating modes of this operator with equivalent\nC-style code:\n\n Operator inputs defined as (max_trip_count, condition_var).\n\n input (\"\", \"\"):\n for (int i=0; ; ++i) {\n cond = ... // Note this value is ignored, but is required in the body\n }\n\n input (\"\", cond) // Note this is analogous to a while loop\n bool cond = ...;\n for (int i=0; cond; ++i) {\n cond = ...;\n }\n\n input (\"\", 1) // Note this is analogous to a do-while loop\n bool cond = true\n for (int i=0; cond; ++i) {\n cond = ...;\n }\n\n input (trip_count, \"\") // Note this is analogous to a for loop\n int trip_count = ...\n for (int i=0; i < trip_count; ++i) {\n cond = ...; // ignored\n }\n\n input (trip_count, cond)\n int trip_count = ...;\n bool cond = ...;\n for (int i=0; i < trip_count && cond; ++i) {\n cond = ...;\n }\n\n\n*Sample usage - cond as well as trip count*\n\n graph predict-net {\n %a = Constant[value = ]()\n %b = Constant[value = ]()\n %keepgoing = Constant[value = ]()\n %max_trip_count = Constant[value = ]()\n %keepgoing_out, %b_out, %user_defined_vals = Loop[body = ](%max_trip_count, %keepgoing, %b)\n return\n }\n\n graph body-net (\n %i[INT32, scalar] // iteration number\n %keepgoing_in[BOOL, scalar] // incoming loop-termination-condition; not used\n %b_in[INT32, scalar] // incoming value of loop-carried-dependency b\n ) {\n %my_local = Add(%a, %b_in)\n %b_out = Sub(%a, %b_in) // outgoing value of loop-carried-dependency b\n %keepgoing_out = Greater(%my_local, %b_out) // outgoing loop-termination-condition\n %user_defined_val = Add(%b_in, %b_in) // scan-output value to be accumulated\n return %keepgoing_out, %b_out, %user_defined_val\n }\n\n*Sample equivalent C code*\n\n {\n /* User-defined code (enclosing scope) */\n int a = 3, b = 6;\n bool keepgoing = true; // Analogous to input cond\n /* End user-defined code */\n\n /* Implicitly-defined code */\n const int max_trip_count = 10; // Analogous to input M\n int user_defined_vals[]; // Imagine this is resizable\n /* End implicitly-defined code */\n /* initialize loop-carried variables and scan-output variables */\n bool keepgoing_out = keepgoing\n int b_out = b\n\n for (int i=0; i < max_trip_count && keepgoing_out; ++i) {\n /* Implicitly-defined code: bind actual parameter values\n to formal parameter variables of loop-body */\n bool keepgoing_in = keepgoing_out; \n bool b_in = b_out;\n\n /* User-defined code (loop body) */\n int my_local = a + b_in; // Reading value \"a\" from the enclosing scope is fine\n b_out = a - b_in;\n keepgoing_out = my_local > b_out; \n user_defined_val = b_in + b_in; // b_in and b_out are different variables\n /* End user-defined code */\n\n /* Implicitly defined-code */\n user_defined_vals[i] = user_defined_val // accumulate scan-output values\n }\n // int t = my_local; // Can\'t do this. my_local is not accessible here.\n\n // The values below are bound to the output variables of the loop and therefore accessible\n // b_out; user_defined_vals; keepgoing_out;\n }\n\nThere are several things of note in this code snippet:\n\n1) Values from the enclosing scope (i.e. variable \"a\" here) are in scope and can\n be referenced in the inputs of the loop.\n2) Any values computed in the loop body that needs to be used in a subsequent\n iteration or after the loop are modelled using a pair of variables in the loop-body,\n consisting of an input variable (eg., b_in) and an output variable (eg., b_out).\n These are referred to as loop-carried dependences. The loop operation node\n supplies the input value of the input variable for the first iteration, and\n returns the output value of the output variable produced by the final\n iteration.\n3) Scan_output variables are used to implicitly concatenate values computed across\n all the iterations. In the above example, the value of user_defined_val computed\n over all iterations are concatenated and returned as the value of user_defined_vals\n after the loop.\n4) Values created in the body cannot be accessed in the enclosing scope,\n except using the mechanism described above.\n\nNote that the semantics of this op support \"diagonal\" or \"wavefront\" execution.\n(See Step 3 here for an example:\nhttps://devblogs.nvidia.com/optimizing-recurrent-neural-networks-cudnn-5/).\nFrontends should emit multi-layer RNNs as a series of While operators (with\ntime being the inner looping dimension), with each successive layer consuming\nthe scan_outputs from the previous layer, possibly going through several\npoint-wise operators (e.g. dropout, residual connections, linear layer).\n" -----f -input: "input" -output: "output" -name: "LpNormalization" -op_type: "LpNormalization" -attribute { - name: "axis" - i: -1 - type: INT -} -attribute { - name: "p" - i: 2 - type: INT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nGiven a matrix, apply Lp-normalization along the provided axis.\n" -----f -input: "X" -output: "Y" -name: "LpPool" -op_type: "LpPool" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "p" - i: 2 - type: INT -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n LpPool consumes an input tensor X and applies Lp pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n Lp pooling consisting of computing the Lp norm on all values of a subset\n of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing." -----f -input: "A" -input: "B" -output: "Y" -name: "MatMul" -op_type: "MatMul" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html\n" -----f -input: "A" -input: "B" -input: "a_zero_point" -input: "b_zero_point" -output: "Y" -name: "MatMulInteger" -op_type: "MatMulInteger" -attribute { - name: "A-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "B-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "a_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "b_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html.\nThe production MUST never overflow. The accumulation may overflow if and only if in 32 bits.\n" -----f -input: "data_0" -output: "max" -name: "Max" -op_type: "Max" -attribute { - name: "data_0-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nElement-wise max of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Y" -output: "Indices" -name: "MaxPool" -op_type: "MaxPool" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "ceil_mode" - i: 0 - type: INT -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "storage_order" - i: 0 - type: INT -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "int8" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "float" - type: STRINGS -} -doc_string: "\n MaxPool consumes an input tensor X and applies max pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n max pooling consisting of computing the max on all values of a\n subset of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing. The output spatial shape will be following:\n ```\n output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1)) / strides_spatial_shape[i] + 1)\n ```\n or\n ```\n output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1)) / strides_spatial_shape[i] + 1)\n ```\n if ceil_mode is enabled\n\n ```\n * pad_shape[i] is sum of pads along axis i\n ```\n\n `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following:\n ```\n VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1) + 1) / strides_spatial_shape[i])\n SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i])\n ```\n And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`:\n ```\n pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + ((kernel_spatial_shape[i] - 1) * dilations[i] + 1) - input_spatial_shape[i]\n ```\n The output of each pooling window is maximum number of elements exclude pad. \n " -----f -input: "X" -input: "rois" -output: "Y" -name: "MaxRoiPool" -op_type: "MaxRoiPool" -attribute { - name: "pooled_shape" - s: "" - type: INTS -} -attribute { - name: "spatial_scale" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "rois-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n ROI max pool consumes an input tensor X and region of interests (RoIs) to\n apply max pooling across each RoI, to produce output 4-D tensor of shape\n (num_rois, channels, pooled_shape[0], pooled_shape[1])." -----f -input: "X" -input: "I" -input: "output_shape" -output: "output" -name: "MaxUnpool" -op_type: "MaxUnpool" -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "I-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "output_shape-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nMaxUnpool essentially computes the partial inverse of the MaxPool op.\n The input information to this op is typically the the output information from a MaxPool op. The first\n input tensor X is the tensor that needs to be unpooled, which is typically the pooled tensor (first output)\n from MaxPool. The second input tensor, I, contains the indices to the (locally maximal) elements corrsponding\n to the elements in the first input tensor X. Input tensor I is typically the second output of the MaxPool op.\n The third (optional) input is a tensor that specifies the output size of the unpooling operation.\n\nMaxUnpool is intended to do \'partial\' inverse of the MaxPool op. \'Partial\' because all the non-maximal\n values from the original input to MaxPool are set to zero in the output of the MaxUnpool op. Pooling\n the result of an unpooling operation should give back the original input to the unpooling op.\n\nMaxUnpool can produce the same output size for several input sizes, which makes unpooling op ambiguous.\n The third input argument, output_size, is meant to disambiguate the op and produce output tensor of\n known/predictable size.\n\nIn addition to the inputs, MaxUnpool takes three attributes, namely kernel_shape, strides, and pads,\n which define the exact unpooling op. The attributes typically have the same values as the corrsponding\n pooling op that the unpooling op is trying to invert.\n" -----f -input: "data_0" -output: "mean" -name: "Mean" -op_type: "Mean" -attribute { - name: "data_0-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nElement-wise mean of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Y" -name: "MeanVarianceNormalization" -op_type: "MeanVarianceNormalization" -attribute { - name: "axes" - ints: 0 - ints: 2 - ints: 3 - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n A MeanVarianceNormalization Function: Perform mean variance normalization\n on the input tensor X using formula:
``` (X-EX)/sqrt(E(X-EX)^2) ```\n" -----f -input: "data_0" -output: "min" -name: "Min" -op_type: "Min" -attribute { - name: "data_0-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nElement-wise min of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "A" -input: "B" -output: "C" -name: "Mod" -op_type: "Mod" -attribute { - name: "fmod" - i: 0 - type: INT -} -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\n Performs element-wise binary modulus (with Numpy-style broadcasting support). \n The sign of the remainder is the same as that of the Divisor.\n \n Mod operator can also behave like C fmod() or numpy.fmod. In this case, the sign of the remainder however, will be the same as the Dividend \n (in contrast to integer mod). To force a behavior like numpy.fmod() an \'fmod\' Attribute is provided.\n This attribute is set to 0 by default causing the behavior to be like integer mod. \n Setting this attribute to 1 causes the remainder to be calculated similar to that of numpy.fmod().\n\n If the input type is floating point, then `fmod` attribute must be set to 1.\n \n In case of dividend being zero, the results will be platform dependent.\n\n This operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "R" -input: "T" -input: "inputs" -output: "outputs" -name: "Momentum" -op_type: "Momentum" -attribute { - name: "alpha" - s: "" - type: FLOAT -} -attribute { - name: "beta" - s: "" - type: FLOAT -} -attribute { - name: "mode" - s: "" - type: STRING -} -attribute { - name: "norm_coefficient" - s: "" - type: FLOAT -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - type: STRINGS -} -attribute { - name: "T-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "inputs-types" - strings: "float" - strings: "double" - type: STRINGS -} -doc_string: "\n Compute one iteration of stochastic gradient update with momentum.\n This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. As you can imagine, SG with momentum requires\n several parameters:\n \n - The learning-rate \"R\".\n - The update count \"T\". That is, the number of conducted training iterations. It should\n be zero in the first training iteration.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A decay coefficient of previous accumulated gradient (i.e., momentum) \"alpha\".\n - The scaling coefficient of current gradient \"beta\".\n - An attribute to choose either standard momentum or Nesterov\'s momentum \"mode\" should\n be used.\n\n For the sake of simplicity, assume that there is only one tensor (called \"X\") to be optimized.\n Other necessary inputs are \"X\"\'s gradient (called \"G\") and \"X\"\'s momentum (called \"V\"). This\n Momentum operator maps all these inputs to the new value of \"X\" (called \"X_new\") and its new\n momentum (called \"V_new\").\n \n This operator supports two different momentum algorithms. Set the attribute \"mode\" to\n \"nesterov\" if Nesterov\'s momentum is desired. Otherwise, set the attribute \"model\" to\n \"standard\" to use standard momentum. Computation details are described subsequently.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise operations with numpy-style broadcasting.\n\n Pseudo code for SG with standard momentum:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||^2, where ||X|| is the sum of squared\n // values of all elements in X.\n G_regularized = norm_coefficient * X + G\n\n // In the first training iteration, beta should always be 1.\n beta_adjusted = T > 0 ? beta : 1\n\n // Compute the current momentum based on previous momentum and the current gradient.\n V_new = alpha * V + beta_adjusted * G_regularized\n\n // Update X.\n X_new = X - R * V_new\n\n Pseudo code for SG with Nesterov\'s momentum:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||^2, where ||X|| is the sum of squared\n // values of all elements in X.\n G_regularized = norm_coefficient * X + G;\n\n // In the first training iteration, beta should always be 1.\n beta_adjusted = T > 0 ? beta : 1\n\n // Compute the current momentum based on previous momentum and the current gradient.\n V_new = alpha * V + beta_adjusted * G_regularized;\n\n // Compute final update direction and then update X.\n X_new = X - R * (G_regularized + alpha * V_new)\n\n If one assign this operators to optimize multiple inputs, for example, \"X_1\" and \"X_2\". The same\n pseudo code would be extended to handle all tensors jointly. More specifically, we can view \"X\" as a\n concatenation of \"X_1\" and \"X_2\" (of course, their gradient and accumulate gradient should\n be concatenated too) and then our pseudo code becomes applicable.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Mul" -op_type: "Mul" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPerforms element-wise binary multiplication (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "input" -output: "output" -name: "Multinomial" -op_type: "Multinomial" -attribute { - name: "dtype" - i: 6 - type: INT -} -attribute { - name: "sample_size" - i: 1 - type: INT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nGenerate a tensor of samples from a multinomial distribution according to the probabilities\nof each of the possible outcomes.\n" -----f -input: "X" -output: "Y" -name: "Neg" -op_type: "Neg" -attribute { - name: "X-types" - strings: "int8" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - strings: "int16" - type: STRINGS -} -doc_string: "\nNeg takes one input data (Tensor) and produces one output data\n(Tensor) where each element flipped sign, y = -x, is applied to\nthe tensor elementwise.\n" -----f -input: "input" -input: "target" -input: "weight" -output: "loss" -name: "NegativeLogLikelihoodLoss" -op_type: "NegativeLogLikelihoodLoss" -attribute { - name: "ignore_index" - s: "" - type: INT -} -attribute { - name: "reduction" - s: "mean" - type: STRING -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "target-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "weight-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nA NegativeLogLikelihoodLoss operator computes (weighted) negative log likelihood loss.\nIts \"input\" tensor has the shape of (N, C, d1, d2, ..., dk) where k >= 0.\nThe \"input\" tensor contains log-probabilities for input[n, :, d_1, d_2,..., d_k] being in a class of [0, C).\nThe operator\'s \"target\" input tensor has the shape of (N, d1, d2, ..., dk). It encodes class labels (one of C classes)\nor it may contain a special value (indicated by an attribute ignore_index) for N x d1 x d2 x ... x dk samples.\nThe loss value for input[n, :, d_1, d_2,...d_k] being classified as class c = target[n][d_1][d_2]...[d_k] is computed as:\n\n loss[n][d_1][d_2]...[d_k] = -input[n][c][d_1][d_2]...[d_k].\n\nWhen an optional \"weight\" is provided, the sample loss is calculated as:\n\n loss[n][d_1][d_2]...[d_k] = -input[n][c][d_1][d_2]...[d_k] * weight[c].\n\nloss is zero for the case when target-value equals ignore_index.\n \n loss[n][d_1][d_2]...[d_k] = 0, when target[n][d_1][d_2]...[d_k] = ignore_index\n\nIf \"reduction\" attribute is set to \"none\", the operator\'s output will be the above loss with shape (N, d1, d2, ..., dk).\nIf \"reduction\" attribute is set to \"mean\" (the default attribute value), the output loss is (weight) averaged:\n\n mean(loss), if \"weight\" is not provided,\n\nor if weight is provided,\n\n sum(loss) / sum(weight[target[n][d_1][d_2]...[d_k]]]), for all samples.\n\nIf \"reduction\" attribute is set to \"sum\", the output is a scalar:\n sum(loss).\n\nSee also https://pytorch.org/docs/stable/nn.html#torch.nn.NLLLoss.\n\nExample 1:\n\n // negative log likelihood loss, \"none\" reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n\n loss = np.zeros((N, d1))\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1]\n\n // print(loss)\n // [[-3. -2.]\n // [-0. -2.]]\n\nExample 2:\n\n // weighted negative log likelihood loss, sum reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n weight = [0.2, 0.3, 0.1]\n loss = np.zeros((N, d1))\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1] * weight[c]\n\n loss = np.sum(loss)\n // print(loss)\n // -1.1\n\nExample 3:\n\n // weighted negative log likelihood loss, mean reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n weight = [0.2, 0.3, 0.1]\n loss = np.zeros((N, d1))\n weight_total = 0\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1] * weight[c]\n weight_total = weight_total + weight[c]\n\n loss = np.sum(loss) / weight_total\n // print(loss)\n // -1.57\n" -----f -input: "boxes" -input: "scores" -input: "max_output_boxes_per_class" -input: "iou_threshold" -input: "score_threshold" -output: "selected_indices" -name: "NonMaxSuppression" -op_type: "NonMaxSuppression" -attribute { - name: "center_point_box" - i: 0 - type: INT -} -attribute { - name: "boxes-types" - strings: "float" - type: STRINGS -} -attribute { - name: "scores-types" - strings: "float" - type: STRINGS -} -attribute { - name: "max_output_boxes_per_class-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "iou_threshold-types" - strings: "float" - type: STRINGS -} -attribute { - name: "score_threshold-types" - strings: "float" - type: STRINGS -} -doc_string: "\nFilter out boxes that have high intersection-over-union (IOU) overlap with previously selected boxes.\nBounding boxes with score less than score_threshold are removed. Bounding box format is indicated by attribute center_point_box.\nNote that this algorithm is agnostic to where the origin is in the coordinate system and more generally is invariant to\northogonal transformations and translations of the coordinate system; thus translating or reflections of the coordinate system\nresult in the same boxes being selected by the algorithm.\nThe selected_indices output is a set of integers indexing into the input collection of bounding boxes representing the selected boxes.\nThe bounding box coordinates corresponding to the selected indices can then be obtained using the Gather or GatherND operation.\n" -----f -input: "X" -output: "Y" -name: "NonZero" -op_type: "NonZero" -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\n Returns the indices of the elements that are non-zero\n (in row-major order - by dimension).\n NonZero behaves similar to numpy.nonzero:\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.nonzero.html\n" -----f -input: "X" -output: "Y" -name: "Normalizer" -op_type: "Normalizer" -attribute { - name: "norm" - s: "MAX" - type: STRING -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Normalize the input. There are three normalization modes, which have the corresponding formulas,\n defined using element-wise infix operators \'/\' and \'^\' and tensor-wide functions \'max\' and \'sum\':
\n
\n Max: Y = X / max(X)
\n L1: Y = X / sum(X)
\n L2: Y = sqrt(X^2 / sum(X^2)}
\n In all modes, if the divisor is zero, Y == X.\n
\n For batches, that is, [N,C] tensors, normalization is done along the C axis. In other words, each row\n of the batch is normalized independently.\n" -----f -input: "X" -output: "Y" -name: "Not" -op_type: "Not" -attribute { - name: "X-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nReturns the negation of the input tensor element-wise.\n" -----f -input: "indices" -input: "depth" -input: "values" -output: "output" -name: "OneHot" -op_type: "OneHot" -attribute { - name: "axis" - i: -1 - type: INT -} -attribute { - name: "indices-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "depth-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "values-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\n Produces a one-hot tensor based on inputs.\n The locations represented by the index values in the \'indices\' input tensor will have \'on_value\'\n and the other locations will have \'off_value\' in the output tensor, where \'on_value\' and \'off_value\'\n are specified as part of required input argument \'values\', which is a two-element tensor of format\n [off_value, on_value]. The rank of the output tensor will be one greater than the rank of the\n input tensor. The additional dimension is for one-hot representation. The additional dimension will\n be inserted at the position specified by \'axis\'. If \'axis\' is not specified then then additional\n dimension will be inserted as the innermost dimension, i.e. axis=-1. The size of the additional\n dimension is specified by required scalar input \'depth\'. The type of the output tensor is the same\n as the type of the \'values\' input. Any entries in the \'indices\' input tensor with values outside\n the range [-depth, depth-1] will result in one-hot representation with all \'off_value\' values in the\n output tensor.\n\n when axis = 0:\n output[input[i, j, k], i, j, k] = 1 for all i, j, k and 0 otherwise.\n\n when axis = -1:\n output[i, j, k, input[i, j, k]] = 1 for all i, j, k and 0 otherwise.\n\n" -----f -input: "X" -output: "Y" -name: "OneHotEncoder" -op_type: "OneHotEncoder" -attribute { - name: "cats_int64s" - s: "" - type: INTS -} -attribute { - name: "cats_strings" - s: "" - type: STRINGS -} -attribute { - name: "zeros" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "int32" - strings: "string" - strings: "double" - strings: "int64" - strings: "float" - type: STRINGS -} -doc_string: "\n Replace each input element with an array of ones and zeros, where a single\n one is placed at the index of the category that was passed in. The total category count \n will determine the size of the extra dimension of the output array Y.
\n For example, if we pass a tensor with a single value of 4, and a category count of 8, \n the output will be a tensor with ``[0,0,0,0,1,0,0,0]``.
\n This operator assumes every input feature is from the same set of categories.
\n If the input is a tensor of float, int32, or double, the data will be cast\n to integers and the cats_int64s category list will be used for the lookups.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Or" -op_type: "Or" -attribute { - name: "A-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "B-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `or` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -input: "slope" -output: "Y" -name: "PRelu" -op_type: "PRelu" -attribute { - name: "X-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "slope-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPRelu takes input data (Tensor) and slope tensor as input, and produces one\noutput data (Tensor) where the function `f(x) = slope * x for x < 0`,\n`f(x) = x for x >= 0`., is applied to the data tensor elementwise.\nThis operator supports **unidirectional broadcasting** (tensor slope should be unidirectional broadcastable to input tensor X); for more details please check [the doc](Broadcasting.md)." -----f -input: "data" -input: "pads" -input: "constant_value" -output: "output" -name: "Pad" -op_type: "Pad" -attribute { - name: "mode" - s: "constant" - type: STRING -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "pads-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "constant_value-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGiven a tensor containing the data to be padded (`data`), a tensor containing the number of start and end pad values for axis (`pads`), (optionally) a `mode`, and (optionally) `constant_value`, \na padded tensor (`output`) is generated.\n\nThe three supported `modes` are (similar to corresponding modes supported by `numpy.pad`):\n\n1) `constant`(default) - pads with a given constant value as specified by `constant_value` (which defaults to 0)\n\n2) `reflect` - pads with the reflection of the vector mirrored on the first and last values of the vector along each axis\n\n3) `edge` - pads with the edge values of array\n\n\nExample 1 (`constant` mode):\n Insert 0 pads to the beginning of the second dimension.\n\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'constant\'\n\n constant_value = 0.0\n\n output = \n [\n [\n [0.0, 0.0, 1.0, 1.2],\n [0.0, 0.0, 2.3, 3.4],\n [0.0, 0.0, 4.5, 5.7],\n ],\n ]\n\n\nExample 2 (`reflect` mode):\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'reflect\'\n\n output = \n [\n [\n [1.0, 1.2, 1.0, 1.2],\n [2.3, 3.4, 2.3, 3.4],\n [4.5, 5.7, 4.5, 5.7],\n ],\n ]\n\n\nExample 3 (`edge` mode):\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'edge\'\n\n output = \n [\n [\n [1.0, 1.0, 1.0, 1.2],\n [2.3, 2.3, 2.3, 3.4],\n [4.5, 4.5, 4.5, 5.7],\n ],\n ]\n\n" -----f -input: "X" -input: "Y" -output: "Z" -name: "Pow" -op_type: "Pow" -attribute { - name: "X-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - type: STRINGS -} -attribute { - name: "Y-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPow takes input data (Tensor) and exponent Tensor, and\nproduces one output data (Tensor) where the function `f(x) = x^exponent`,\nis applied to the data tensor elementwise.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md)." -----f -input: "x" -input: "x_scale" -input: "x_zero_point" -input: "w" -input: "w_scale" -input: "w_zero_point" -input: "y_scale" -input: "y_zero_point" -input: "B" -output: "y" -name: "QLinearConv" -op_type: "QLinearConv" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "group" - i: 1 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "x-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "x_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "x_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "w-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "w_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "w_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "y_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "y_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "B-types" - strings: "int32" - type: STRINGS -} -doc_string: "\nThe convolution operator consumes a quantized input tensor, its scale and zero point,\na quantized filter, its scale and zero point, and output\'s scale and zero point,\nand computes the quantized output. Each scale and zero-point pair must have same shape.\nIt means they must be either scalars (per tensor) or 1-D tensors (per output channel).\nEach input or output and its related zero point must have same type.\nWhen bias is present it must be quantized using scale = input scale * weight scale and \nzero point as 0.\n" -----f -input: "a" -input: "a_scale" -input: "a_zero_point" -input: "b" -input: "b_scale" -input: "b_zero_point" -input: "y_scale" -input: "y_zero_point" -output: "y" -name: "QLinearMatMul" -op_type: "QLinearMatMul" -attribute { - name: "a-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "a_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "a_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "b-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "b_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "b_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "y_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "y_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html.\nIt consumes two quantized input tensors, their scales and zero points, scale and zero point of output, and computes the quantized output.\nThe quantization formula is y = saturate((x / y_scale) + y_zero_point). For (x / y_scale), it is rounding to nearest ties to even.\nRefer to https://en.wikipedia.org/wiki/Rounding for details. Scale and zero point must have same shape.\nThey must be either scalar (per tensor) or 1-D tensor (per row for \'a\' and per column for \'b\'). If scale and zero point are 1-D tensor,\nthe number of elements of scale and zero point tensor of input \'a\' and output \'y\' should be equal to the number of rows of input \'a\',\nand the number of elements of scale and zero point tensor of input \'b\' should be equal to the number of columns of input \'b\'.\nProduction must never overflow, and accumulation may overflow if and only if in 32 bits.\n" -----f -input: "x" -input: "y_scale" -input: "y_zero_point" -output: "y" -name: "QuantizeLinear" -op_type: "QuantizeLinear" -attribute { - name: "x-types" - strings: "float" - strings: "int32" - type: STRINGS -} -attribute { - name: "y_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "y_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nThe linear per-tensor/layer quantization operator. It consumes a high precision tensor, a scale, a zero point to compute the low precision / quantized tensor.\nThe quantization formula is y = saturate ((x / y_scale) + y_zero_point). For saturation, it saturates to [0, 255] if it\'s uint8, or [-128, 127] if it\'s int8.\nFor (x / y_scale), it\'s rounding to nearest ties to even. Refer to https://en.wikipedia.org/wiki/Rounding for details. \'y_zero_point\' and \'y\' must have same type.\n" -----f -input: "X" -input: "W" -input: "R" -input: "B" -input: "sequence_lens" -input: "initial_h" -output: "Y" -output: "Y_h" -name: "RNN" -op_type: "RNN" -attribute { - name: "activation_alpha" - s: "" - type: FLOATS -} -attribute { - name: "activation_beta" - s: "" - type: FLOATS -} -attribute { - name: "activations" - strings: "Tanh" - strings: "Tanh" - type: STRINGS -} -attribute { - name: "clip" - s: "" - type: FLOAT -} -attribute { - name: "direction" - s: "forward" - type: STRING -} -attribute { - name: "hidden_size" - s: "" - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "sequence_lens-types" - strings: "int32" - type: STRINGS -} -attribute { - name: "initial_h-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nComputes an one-layer simple RNN. This operator is usually supported\nvia some custom implementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`i` - input gate\n\n`t` - time step (t-1 means previous time step)\n\n`Wi` - W parameter weight matrix for input gate\n\n`Ri` - R recurrence weight matrix for input gate\n\n`Wbi` - W parameter bias vector for input gate\n\n`Rbi` - R parameter bias vector for input gate\n\n`WBi` - W parameter weight matrix for backward input gate\n\n`RBi` - R recurrence weight matrix for backward input gate\n\n`WBbi` - WR bias vectors for backward input gate\n\n`RBbi` - RR bias vectors for backward input gate\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Tanh):\n\n - Ht = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -output: "output" -name: "RandomNormal" -op_type: "RandomNormal" -attribute { - name: "dtype" - i: 1 - type: INT -} -attribute { - name: "mean" - f: 0.0 - type: FLOAT -} -attribute { - name: "scale" - f: 1.0 - type: FLOAT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "shape" - s: "" - type: INTS -} -doc_string: "\nGenerate a tensor with random values drawn from a normal distribution. The shape\nof the tensor is specified by the `shape` argument and the parameter of the normal distribution\nspecified by `mean` and `scale`.\n\nThe data type is specified by the \'dtype\' argument. The \'dtype\' argument must\nbe one of the data types specified in the \'DataType\' enum field in the\nTensorProto message.\n" -----f -input: "input" -output: "output" -name: "RandomNormalLike" -op_type: "RandomNormalLike" -attribute { - name: "dtype" - s: "" - type: INT -} -attribute { - name: "mean" - f: 0.0 - type: FLOAT -} -attribute { - name: "scale" - f: 1.0 - type: FLOAT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGenerate a tensor with random values drawn from a normal distribution.\nThe shape of the output tensor is copied from the shape of the input tensor,\nand the parameters of the normal distribution are specified by `mean` and `scale`.\n\nThe data type is specified by the \'dtype\' argument, or copied from the input tensor if not provided.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message, and be valid as an output type.\n" -----f -output: "output" -name: "RandomUniform" -op_type: "RandomUniform" -attribute { - name: "dtype" - i: 1 - type: INT -} -attribute { - name: "high" - f: 1.0 - type: FLOAT -} -attribute { - name: "low" - f: 0.0 - type: FLOAT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "shape" - s: "" - type: INTS -} -doc_string: "\nGenerate a tensor with random values drawn from a uniform distribution. The shape\nof the tensor is specified by the `shape` argument and the range by `low` and `high`.\n\nThe data type is specified by the \'dtype\' argument. The \'dtype\' argument must\nbe one of the data types specified in the \'DataType\' enum field in the\nTensorProto message.\n" -----f -input: "input" -output: "output" -name: "RandomUniformLike" -op_type: "RandomUniformLike" -attribute { - name: "dtype" - s: "" - type: INT -} -attribute { - name: "high" - f: 1.0 - type: FLOAT -} -attribute { - name: "low" - f: 0.0 - type: FLOAT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGenerate a tensor with random values drawn from a uniform distribution.\nThe shape of the output tensor is copied from the shape of the input tensor,\nand the parameters of the uniform distribution are specified by `low` and `high`.\n\nThe data type is specified by the \'dtype\' argument, or copied from the input tensor if not provided.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message and be valid as an output type.\n" -----f -input: "start" -input: "limit" -input: "delta" -output: "output" -name: "Range" -op_type: "Range" -attribute { - name: "start-types" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - strings: "int16" - type: STRINGS -} -attribute { - name: "limit-types" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - strings: "int16" - type: STRINGS -} -attribute { - name: "delta-types" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - strings: "int16" - type: STRINGS -} -doc_string: "\nGenerate a tensor containing a sequence of numbers that begin at `start` and extends by increments of `delta`\nup to `limit` (exclusive).\n\nThe number of elements in the output of range is computed as below-\n\n`number_of_elements = max( ceil( (limit - start) / delta ) , 0 )`\n\nThe pseudocode determining the contents of the output is shown below-\n\n`for(int i=0; i) and produces one output data\n(Tensor) where the reciprocal is, y = 1/x, is applied to\nthe tensor elementwise.\n" -----f -input: "data" -output: "reduced" -name: "ReduceL1" -op_type: "ReduceL1" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the L1 norm of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceL2" -op_type: "ReduceL2" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the L2 norm of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceLogSum" -op_type: "ReduceLogSum" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the log sum of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceLogSumExp" -op_type: "ReduceLogSumExp" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the log sum exponent of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceMax" -op_type: "ReduceMax" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "int8" - strings: "float16" - strings: "int32" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the max of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceMean" -op_type: "ReduceMean" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the mean of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceMin" -op_type: "ReduceMin" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "int8" - strings: "float16" - strings: "int32" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the min of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceProd" -op_type: "ReduceProd" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the product of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceSum" -op_type: "ReduceSum" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the sum of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceSumSquare" -op_type: "ReduceSumSquare" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the sum square of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "X" -output: "Y" -name: "Relu" -op_type: "Relu" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nRelu takes one input data (Tensor) and produces one output data\n(Tensor) where the rectified linear function, y = max(0, x), is applied to\nthe tensor elementwise.\n" -----f -input: "data" -input: "shape" -output: "reshaped" -name: "Reshape" -op_type: "Reshape" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "shape-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nReshape the input tensor similar to numpy.reshape.\nFirst input is the data tensor, second input is a shape tensor which specifies the output shape. It outputs the reshaped tensor.\nAt most one dimension of the new shape can be -1. In this case, the value is\ninferred from the size of the tensor and the remaining dimensions. A dimension\ncould also be 0, in which case the actual dimension value is unchanged (i.e. taken\nfrom the input tensor)." -----f -input: "X" -input: "roi" -input: "scales" -input: "sizes" -output: "Y" -name: "Resize" -op_type: "Resize" -attribute { - name: "coordinate_transformation_mode" - s: "half_pixel" - type: STRING -} -attribute { - name: "cubic_coeff_a" - f: -0.75 - type: FLOAT -} -attribute { - name: "exclude_outside" - i: 0 - type: INT -} -attribute { - name: "extrapolation_value" - f: 0.0 - type: FLOAT -} -attribute { - name: "mode" - s: "nearest" - type: STRING -} -attribute { - name: "nearest_mode" - s: "round_prefer_floor" - type: STRING -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "roi-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "scales-types" - strings: "float" - type: STRINGS -} -attribute { - name: "sizes-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nResize the input tensor. In general, it calculates every value in the output tensor as a weighted average of neighborhood (a.k.a. sampling locations) in the input tensor.\nEach dimension value of the output tensor is:\n output_dimension = floor(input_dimension * (roi_end - roi_start) * scale) if input \\\"sizes\\\" is not specified.\n" -----f -input: "input" -input: "sequence_lens" -output: "Y" -name: "ReverseSequence" -op_type: "ReverseSequence" -attribute { - name: "batch_axis" - i: 1 - type: INT -} -attribute { - name: "time_axis" - i: 0 - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "sequence_lens-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nReverse batch of sequences having different lengths specified by `sequence_lens`.\n\nFor each slice i iterating on batch axis, the operator reverses the first sequence_lens[i] elements on time axis,\nand copies elements whose index\'s beyond sequence_lens[i] to the output. So the output slice i contains reversed\nsequences on the first sequence_lens[i] elements, then have original values copied for the other elements.\n\nExample 1:\n input = [[0.0, 4.0, 8.0, 12.0],\n [1.0, 5.0, 9.0, 13.0],\n [2.0, 6.0, 10.0, 14.0],\n [3.0, 7.0, 11.0, 15.0]]\n sequence_lens = [4, 3, 2, 1]\n time_axis = 0\n batch_axis = 1\n\n output = [[3.0, 6.0, 9.0, 12.0],\n [2.0, 5.0, 8.0, 13.0],\n [1.0, 4.0, 10.0, 14.0],\n [0.0, 7.0, 11.0, 15.0]]\n\nExample 2:\n input = [[0.0, 1.0, 2.0, 3.0 ],\n [4.0, 5.0, 6.0, 7.0 ],\n [8.0, 9.0, 10.0, 11.0],\n [12.0, 13.0, 14.0, 15.0]]\n sequence_lens = [1, 2, 3, 4]\n time_axis = 1\n batch_axis = 0\n\n output = [[0.0, 1.0, 2.0, 3.0 ],\n [5.0, 4.0, 6.0, 7.0 ],\n [10.0, 9.0, 8.0, 11.0],\n [15.0, 14.0, 13.0, 12.0]]\n" -----f -input: "X" -input: "rois" -input: "batch_indices" -output: "Y" -name: "RoiAlign" -op_type: "RoiAlign" -attribute { - name: "mode" - s: "avg" - type: STRING -} -attribute { - name: "output_height" - i: 1 - type: INT -} -attribute { - name: "output_width" - i: 1 - type: INT -} -attribute { - name: "sampling_ratio" - i: 0 - type: INT -} -attribute { - name: "spatial_scale" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "rois-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "batch_indices-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nRegion of Interest (RoI) align operation described in the\n[Mask R-CNN paper](https://arxiv.org/abs/1703.06870).\nRoiAlign consumes an input tensor X and region of interests (rois)\nto apply pooling across each RoI; it produces a 4-D tensor of shape\n(num_rois, C, output_height, output_width).\n\nRoiAlign is proposed to avoid the misalignment by removing\nquantizations while converting from original image into feature\nmap and from feature map into RoI feature; in each ROI bin,\nthe value of the sampled locations are computed directly\nthrough bilinear interpolation.\n" -----f -input: "X" -output: "Y" -name: "Round" -op_type: "Round" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nRound takes one input Tensor and rounds the values, element-wise, meaning\nit finds the nearest integer for each value.\nIn case of halfs, the rule is to round them to the nearest even integer.\nThe output tensor has the same shape and type as the input.\n\nExamples:\n```\nround([0.9]) = [1.0]\nround([2.5]) = [2.0]\nround([2.3]) = [2.0]\nround([1.5]) = [2.0]\nround([-4.5]) = [-4.0]\n```\n" -----f -input: "X" -output: "Y" -output: "Z" -name: "SVMClassifier" -op_type: "SVMClassifier" -attribute { - name: "classlabels_ints" - s: "" - type: INTS -} -attribute { - name: "classlabels_strings" - s: "" - type: STRINGS -} -attribute { - name: "coefficients" - s: "" - type: FLOATS -} -attribute { - name: "kernel_params" - s: "" - type: FLOATS -} -attribute { - name: "kernel_type" - s: "LINEAR" - type: STRING -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "prob_a" - s: "" - type: FLOATS -} -attribute { - name: "prob_b" - s: "" - type: FLOATS -} -attribute { - name: "rho" - s: "" - type: FLOATS -} -attribute { - name: "support_vectors" - s: "" - type: FLOATS -} -attribute { - name: "vectors_per_class" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Support Vector Machine classifier\n" -----f -input: "X" -output: "Y" -name: "SVMRegressor" -op_type: "SVMRegressor" -attribute { - name: "coefficients" - s: "" - type: FLOATS -} -attribute { - name: "kernel_params" - s: "" - type: FLOATS -} -attribute { - name: "kernel_type" - s: "LINEAR" - type: STRING -} -attribute { - name: "n_supports" - i: 0 - type: INT -} -attribute { - name: "one_class" - i: 0 - type: INT -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "rho" - s: "" - type: FLOATS -} -attribute { - name: "support_vectors" - s: "" - type: FLOATS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Support Vector Machine regression prediction and one-class SVM anomaly detection.\n" -----f -input: "X" -output: "Y" -name: "Scaler" -op_type: "Scaler" -attribute { - name: "offset" - s: "" - type: FLOATS -} -attribute { - name: "scale" - s: "" - type: FLOATS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Rescale input data, for example to standardize features by removing the mean and scaling to unit variance.\n" -----f -input: "initial_state_and_scan_inputs" -output: "final_state_and_scan_outputs" -name: "Scan" -op_type: "Scan" -attribute { - name: "body" - s: "" - type: GRAPH -} -attribute { - name: "num_scan_inputs" - s: "" - type: INT -} -attribute { - name: "scan_input_axes" - s: "" - type: INTS -} -attribute { - name: "scan_input_directions" - s: "" - type: INTS -} -attribute { - name: "scan_output_axes" - s: "" - type: INTS -} -attribute { - name: "scan_output_directions" - s: "" - type: INTS -} -attribute { - name: "initial_state_and_scan_inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nScan can be used to iterate over one or more scan_input tensors,\nconstructing zero or more scan_output tensors. It combines ideas from general recurrences,\nfunctional programming constructs such as scan, fold, map, and zip and is intended to enable\ngeneralizations of RNN-like constructs for sequence-to-sequence processing.\nOther tensors (referred to as state_variables here) can be used to carry a state\nwhen iterating from one element to another (similar to hidden-state in RNNs, also referred\nto as loop-carried dependences in the context of loops).\nMany common usages involve a single scan_input tensor (where functionality\nsimilar to scan, fold and map can be obtained). When more than one scan_input is used,\na behavior similar to zip is obtained.\n\nThe attribute body must be a graph, specifying the computation to be performed in\nevery iteration. It takes as input the current values of the state_variables and\nthe current iterated element of the scan_inputs. It must return the (updated) values\nof the state_variables and zero or more scan_output_element tensors. The values of the\nscan_output_element tensors are concatenated over all the iterations to produce the\nscan_output values of the scan construct (similar to the concatenated intermediate\nhidden-state values of RNN-like constructs). All the output tensors (state_variables as\nwell as scan_output_element tensors) are required to have the same shape in each iteration\nof the loop (a restriction imposed to enable efficient memory allocation).\n\nNote that the iterated element passed to the body subgraph does not have a sequence\naxis. It will have a rank one less than the rank of the corresponding scan_input.\n\nThe scan operation returns the final values of the state_variables as well as the\nscan_outputs.\n\nThe optional attribute scan_input_directions specifies the direction (forward or backward)\nfor each scan input. If this attribute is omitted, all sequences are scanned in the forward\ndirection. A bidirectional scan may be performed by specifying the same tensor input twice\nin the scan_inputs, once with a forward direction, and once with a backward direction.\n\nThe scan_output of the operation is produced by concatenating the scan_output_element\nvalues produced by the body in each iteration. The optional attribute scan_output_directions\nspecifies the direction in which scan_output is constructed (by appending or prepending the\nscan_output_element to scan_output in each iteration) for each scan_output. If this attribute\nis omitted, the scan_output_element is appended to the scan_output in each iteration.\n\nThe optional attribute scan_input_axes specifies the axis to be scanned for each scan_input.\nIf omitted, every scan_input will be scanned in axis 0. For example, if axis 0 is the\nbatch axis and axis 1 is the time axis (to be scanned), specify an axis value of 1.\nNote that scanning a non-zero axis may be less efficient than scanning axis zero.\n\nThe optional attribute scan_output_axes specifies the axis along which the scan_outputs\nare accumulated for each scan_output. For example, if axis 1 is the time axis (to be\nscanned) for both inputs and outputs, specify a scan_input axis and scan_output axis\nvalue of 1.\n\nNote that because of the ONNX restriction that only the last parameter of an operator can\nbe variadic, the initial-states and scan-inputs are listed together as one input parameter.\nSimilarly, the final-states and scan-outputs are listed together as one output parameter.\nThe attribute num_scan_inputs indicates the number M of scan-inputs.\n\nThe behavior of\n\n Scan <\n num_scan_inputs = m,\n body = loop-body,\n scan_input_axes = [axis_1, ..., axis_m]\n > (init_1, ..., init_n, scan_1, ..., scan_m)\n\nis equivalent to the following pseudo-code:\n\n // scan_i.shape[axis_i] denotes the (max) sequence-length of scan_i\n // scan_i.shape[axis_i] is required to be equal to scan_j.shape[axis_j] for all i,j.\n sequence_length = scan_1.shape[axis_1];\n\n // initialize state-variables\n st_1 = init_1; ... st_n = init_n;\n // initialize scan-output variables: [] denotes an empty tensor\n scan_out_1 = []; ...; scan_out_k = [];\n // identify number of iterations:\n\n // execute loop\n for (int t = 0; t < sequence_length; ++t) {\n // generate the scan-input elements: the notation T[t] indicates the sub-tensor\n // of rank one less than T obtained by indexing T at position t along axis k.\n si_1 = scan_1[t];\n ... ;\n si_m = scan_m[t];\n // execute loop-body\n st_1, ..., st_n, so_1, ..., so_k = loop-body(st_1, ..., st_n, si_1, ..., si_m)\n // accumulate the scan-output elements\n scan_out_1 = Concat(scan_out_1, so_1); ... ; scan_out_k = Concat(scan_out_k, so_k);\n }\n\n return st_1, ..., st_n, scan_out_1, ..., scan_out_k;\n\n*Sample usage: Encoding RNN using a Scan*\n\nThe following example shows how a simple RNN over an input tensor %X, with weight tensor %Wi,\nrecurrence weight tensor %Ri, bias tensors %Wbi and %Rbi, and initial hidden-state %H_0 can\nbe encoded as a ScanLoop. Note that the loop-body is a nested graph, and it directly computes\n%Wi, %Ri, %Wbi, and %Rbi (typically constants or initializers in the body graph). If these\nvalues are computed in the outer graph, they need to be passed in as extra state_variables.\n\n graph rnn-encoding {\n %H_0 = ... \n %X = ...\n %Y_h, %Y = Scan[body = , num_scan_inputs=1](%H_0, %X)\n return %Y, %Y_h\n }\n\n graph rnn-cell-1 (\n %H_tminus1[FLOAT, tensor]\n %X_t[FLOAT, tensor]\n ) {\n %Wi = ...\n %Ri = ...\n %Wbi = ...\n %Rbi = ...\n %t1 = X_t * (Wi^T)\n %t2 = H_tminus1*(Ri^T)\n %t3 = Add(%t1, %t2)\n %t4 = Add(%t3, %Wbi)\n %t5 = Add(%t4, %Rbi)\n %Ht = Tanh(%t5)\n %Accumulate = Identity(%Ht)\n return %Ht, %Accumulate\n }\n\n" -----f -input: "data" -input: "indices" -input: "updates" -output: "output" -name: "Scatter" -op_type: "Scatter" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "updates-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nThis operator is deprecated. Please use ScatterElements, which provides the same functionality.\n\nScatter takes three inputs `data`, `updates`, and `indices` of the same\nrank r >= 1 and an optional attribute axis that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value\nto values specified by `updates` at specific index positions specified by\n`indices`. Its output shape is the same as the shape of `data`.\n\nFor each entry in `updates`, the target index in `data` is obtained by combining\nthe corresponding entry in `indices` with the index of the entry itself: the\nindex-value for dimension = axis is obtained from the value of the corresponding\nentry in `indices` and the index-value for dimension != axis is obtained from the\nindex of the entry itself.\n\nFor instance, in a 2-D tensor case, the update corresponding to the [i][j] entry\nis performed as below:\n```\n output[indices[i][j]][j] = updates[i][j] if axis = 0, \n output[i][indices[i][j]] = updates[i][j] if axis = 1,\n```\n\nThis operator is the inverse of GatherElements. It is similar to Torch\'s Scatter operation.\n\nExample 1:\n```\n data = [\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n ]\n indices = [\n [1, 0, 2],\n [0, 2, 1],\n ]\n updates = [\n [1.0, 1.1, 1.2],\n [2.0, 2.1, 2.2],\n ]\n output = [\n [2.0, 1.1, 0.0]\n [1.0, 0.0, 2.2]\n [0.0, 2.1, 1.2]\n ]\n```\nExample 2:\n```\n data = [[1.0, 2.0, 3.0, 4.0, 5.0]]\n indices = [[1, 3]]\n updates = [[1.1, 2.1]]\n axis = 1\n output = [[1.0, 1.1, 3.0, 2.1, 5.0]]\n```\n" -----f -input: "data" -input: "indices" -input: "updates" -output: "output" -name: "ScatterElements" -op_type: "ScatterElements" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "updates-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nScatterElements takes three inputs `data`, `updates`, and `indices` of the same\nrank r >= 1 and an optional attribute axis that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value\nto values specified by `updates` at specific index positions specified by\n`indices`. Its output shape is the same as the shape of `data`.\n\nFor each entry in `updates`, the target index in `data` is obtained by combining\nthe corresponding entry in `indices` with the index of the entry itself: the\nindex-value for dimension = axis is obtained from the value of the corresponding\nentry in `indices` and the index-value for dimension != axis is obtained from the\nindex of the entry itself.\n\nFor instance, in a 2-D tensor case, the update corresponding to the [i][j] entry\nis performed as below:\n```\n output[indices[i][j]][j] = updates[i][j] if axis = 0, \n output[i][indices[i][j]] = updates[i][j] if axis = 1,\n```\n\nThis operator is the inverse of GatherElements. It is similar to Torch\'s Scatter operation.\n\nExample 1:\n```\n data = [\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n ]\n indices = [\n [1, 0, 2],\n [0, 2, 1],\n ]\n updates = [\n [1.0, 1.1, 1.2],\n [2.0, 2.1, 2.2],\n ]\n output = [\n [2.0, 1.1, 0.0]\n [1.0, 0.0, 2.2]\n [0.0, 2.1, 1.2]\n ]\n```\nExample 2:\n```\n data = [[1.0, 2.0, 3.0, 4.0, 5.0]]\n indices = [[1, 3]]\n updates = [[1.1, 2.1]]\n axis = 1\n output = [[1.0, 1.1, 3.0, 2.1, 5.0]]\n```\n" -----f -input: "data" -input: "indices" -input: "updates" -output: "output" -name: "ScatterND" -op_type: "ScatterND" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "updates-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nScatterND takes three inputs `data` tensor of rank r >= 1, `indices` tensor of rank q >= 1,\nand `updates` tensor of rank q + r - indices.shape[-1] - 1. The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value to values\nspecified by `updates` at specific index positions specified by `indices`. Its output shape\nis the same as the shape of `data`. Note that `indices` should not have duplicate entries.\nThat is, two or more `updates` for the same index-location is not supported.\n\n`indices` is an integer tensor. Let k denote indices.shape[-1], the last dimension in the shape of `indices`.\n `indices` is treated as a (q-1)-dimensional tensor of k-tuples, where each k-tuple is a partial-index into `data`.\nHence, k can be a value at most the rank of `data`. When k equals rank(data), each update entry specifies an\nupdate to a single element of the tensor. When k is less than rank(data) each update entry specifies an\nupdate to a slice of the tensor.\n\n`updates` is treated as a (q-1)-dimensional tensor of replacement-slice-values. Thus, the\nfirst (q-1) dimensions of updates.shape must match the first (q-1) dimensions of indices.shape.\nThe remaining dimensions of `updates` correspond to the dimensions of the\nreplacement-slice-values. Each replacement-slice-value is a (r-k) dimensional tensor,\ncorresponding to the trailing (r-k) dimensions of `data`. Thus, the shape of `updates`\nmust equal indices.shape[0:q-1] ++ data.shape[k:r-1], where ++ denotes the concatenation\nof shapes.\n\nThe `output` is calculated via the following equation:\n\n output = np.copy(data)\n update_indices = indices.shape[:-1]\n for idx in np.ndindex(update_indices):\n output[indices[idx]] = updates[idx]\n\nThe order of iteration in the above loop is not specified.\nIn particular, indices should not have duplicate entries: that is, if idx1 != idx2, then indices[idx1] != indices[idx2].\nThis ensures that the output value does not depend on the iteration order.\n\nThis operator is the inverse of GatherND.\n\nExample 1:\n```\n data = [1, 2, 3, 4, 5, 6, 7, 8]\n indices = [[4], [3], [1], [7]]\n updates = [9, 10, 11, 12]\n output = [1, 11, 3, 10, 9, 6, 7, 12]\n```\n\nExample 2:\n```\n data = [[[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]]]\n indices = [[0], [2]]\n updates = [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],\n [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]]\n output = [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],\n [[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]]]\n```\n" -----f -input: "X" -output: "Y" -name: "Selu" -op_type: "Selu" -attribute { - name: "alpha" - f: 1.6732632 - type: FLOAT -} -attribute { - name: "gamma" - f: 1.050701 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nSelu takes one input data (Tensor) and produces one output data\n(Tensor) where the scaled exponential linear unit function,\n`y = gamma * (alpha * e^x - alpha) for x <= 0`, `y = gamma * x for x > 0`,\nis applied to the tensor elementwise.\n" -----f -input: "input_sequence" -input: "position" -output: "tensor" -name: "SequenceAt" -op_type: "SequenceAt" -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -attribute { - name: "position-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nOutputs a tensor copy from the tensor at \'position\' in \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n - 1]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n" -----f -input: "inputs" -output: "output_sequence" -name: "SequenceConstruct" -op_type: "SequenceConstruct" -attribute { - name: "inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nConstruct a tensor sequence containing \'inputs\' tensors.\nAll tensors in \'inputs\' must have the same data type.\n" -----f -output: "output" -name: "SequenceEmpty" -op_type: "SequenceEmpty" -attribute { - name: "dtype" - s: "" - type: INT -} -doc_string: "\nConstruct an empty tensor sequence, with given data type.\n" -----f -input: "input_sequence" -input: "position" -output: "output_sequence" -name: "SequenceErase" -op_type: "SequenceErase" -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -attribute { - name: "position-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nOutputs a tensor sequence that removes the tensor at \'position\' from \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n - 1]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n\'position\' is optional, by default it erases the last tensor from \'input_sequence\'.\n" -----f -input: "input_sequence" -input: "tensor" -input: "position" -output: "output_sequence" -name: "SequenceInsert" -op_type: "SequenceInsert" -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -attribute { - name: "tensor-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "position-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nOutputs a tensor sequence that inserts \'tensor\' into \'input_sequence\' at \'position\'.\n\'tensor\' must have the same data type as \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n\'position\' is optional, by default it inserts \'tensor\' to the back of \'input_sequence\'.\n" -----f -input: "input_sequence" -output: "length" -name: "SequenceLength" -op_type: "SequenceLength" -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -doc_string: "\nProduces a scalar(tensor of empty shape) containing the number of tensors in \'input_sequence\'.\n" -----f -input: "data" -output: "shape" -name: "Shape" -op_type: "Shape" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nTakes a tensor as input and outputs an 1D int64 tensor containing the shape of the input tensor.\n" -----f -input: "input" -output: "output" -name: "Shrink" -op_type: "Shrink" -attribute { - name: "bias" - f: 0.0 - type: FLOAT -} -attribute { - name: "lambd" - f: 0.5 - type: FLOAT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nShrink takes one input data (Tensor) and produces one Tensor output,\nhaving same datatype and shape with input. It has two attributes, lambd and\nbias. The formula of this operator is: If x < -lambd, y = x + bias;\nIf x > lambd, y = x - bias; Otherwise, y = 0.\n" -----f -input: "X" -output: "Y" -name: "Sigmoid" -op_type: "Sigmoid" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nSigmoid takes one input data (Tensor) and produces one output data\n(Tensor) where the sigmoid function, y = 1 / (1 + exp(-x)), is applied to the\ntensor elementwise.\n" -----f -input: "input" -output: "output" -name: "Sign" -op_type: "Sign" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nCalculate the sign of the given input tensor element-wise.\nIf input > 0, output 1. if input < 0, output -1. if input == 0, output 0.\n" -----f -input: "input" -output: "output" -name: "Sin" -op_type: "Sin" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the sine of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Sinh" -op_type: "Sinh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic sine of the given input tensor element-wise.\n" -----f -input: "data" -output: "size" -name: "Size" -op_type: "Size" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nTakes a tensor as input and outputs a int64 scalar that equals to the total number of elements of the input tensor.\n" -----f -input: "data" -input: "starts" -input: "ends" -input: "axes" -input: "steps" -output: "output" -name: "Slice" -op_type: "Slice" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "starts-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "ends-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "axes-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "steps-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nProduces a slice of the input tensor along multiple axes. Similar to numpy:\nhttps://docs.scipy.org/doc/numpy/reference/arrays.indexing.html\nSlices uses `starts`, `ends`, `axes` and `steps` inputs to specify the start and end\ndimension and step for each axis in the list of axes, it uses this information to\nslice the input `data` tensor. If a negative value is passed for any of the\nstart or end indices, it represents number of elements before the end of that\ndimension. If the value passed to start or end is larger than the `n` (the\nnumber of elements in this dimension), it represents `n`. For slicing to the\nend of a dimension with unknown size, it is recommended to pass in `INT_MAX` \nwhen sclicing forward and \'INT_MIN\' when slicing backward.\nIf a negative value is passed for step, it represents slicing backward. \nHowever step value cannot be 0.\nIf `axes` are omitted, they are set to `[0, ..., ndim-1]`.\nIf `steps` are omitted, they are set to `[1, ..., 1]` of length `len(starts)`\nExample 1:\n data = [\n [1, 2, 3, 4],\n [5, 6, 7, 8],\n ]\n axes = [0, 1]\n starts = [1, 0]\n ends = [2, 3]\n steps = [1, 2]\n result = [\n [5, 7],\n ]\nExample 2:\n data = [\n [1, 2, 3, 4],\n [5, 6, 7, 8],\n ]\n starts = [0, 1]\n ends = [-1, 1000]\n result = [\n [2, 3, 4],\n ]\n" -----f -input: "input" -output: "output" -name: "Softmax" -op_type: "Softmax" -attribute { - name: "axis" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe operator computes the softmax (normalized exponential) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the softmax values of the corresponding input.\n" -----f -input: "scores" -input: "labels" -input: "weights" -output: "output" -output: "log_prob" -name: "SoftmaxCrossEntropyLoss" -op_type: "SoftmaxCrossEntropyLoss" -attribute { - name: "ignore_index" - s: "" - type: INT -} -attribute { - name: "reduction" - s: "mean" - type: STRING -} -attribute { - name: "scores-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "labels-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "weights-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "Loss function that measures the softmax cross entropy\nbetween \'scores\' and \'labels\'.\nThis operator first computes a loss tensor whose shape is identical to the labels input.\nIf the input is 2-D with shape (N, C), the loss tensor may be a N-element vector L = (l_1, l_2, ..., l_N).\nIf the input is N-D tensor with shape (N, C, D1, D2, ..., Dk),\nthe loss tensor L may have (N, D1, D2, ..., Dk) as its shape and L[i,][j_1][j_2]...[j_k] denotes a scalar element in L.\nAfter L is available, this operator can optionally do a reduction operator.\n\nshape(scores): (N, C) where C is the number of classes, or (N, C, D1, D2,..., Dk),\n with K >= 1 in case of K-dimensional loss.\nshape(labels): (N) where each value is 0 <= labels[i] <= C-1, or (N, D1, D2,..., Dk),\n with K >= 1 in case of K-dimensional loss.\n\nThe loss for one sample, l_i, can caculated as follows:\n l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk], where i is the index of classes.\nor\n l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk] * weights[c], if \'weights\' is provided.\n\nloss is zero for the case when label-value equals ignore_index.\n l[i][d1][d2]...[dk] = 0, when labels[n][d1][d2]...[dk] = ignore_index\n\nwhere:\n p = Softmax(scores)\n y = Log(p)\n c = labels[i][d1][d2]...[dk]\n\nFinally, L is optionally reduced:\nIf reduction = \'none\', the output is L with shape (N, D1, D2, ..., Dk).\nIf reduction = \'sum\', the output is scalar: Sum(L).\nIf reduction = \'mean\', the output is scalar: ReduceMean(L), or if weight is provided: ReduceSum(L) / ReduceSum(W),\nwhere tensor W is of shape (N, D1, D2, ..., Dk) and W[n][d1][d2]...[dk] = weights[labels[i][d1][d2]...[dk]].\n" -----f -input: "X" -output: "Y" -name: "Softplus" -op_type: "Softplus" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nSoftplus takes one input data (Tensor) and produces one output data\n(Tensor) where the softplus function, y = ln(exp(x) + 1), is applied to\nthe tensor elementwise.\n" -----f -input: "input" -output: "output" -name: "Softsign" -op_type: "Softsign" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the softsign (x/(1+|x|)) of the given input tensor element-wise.\n" -----f -input: "input" -output: "output" -name: "SpaceToDepth" -op_type: "SpaceToDepth" -attribute { - name: "blocksize" - s: "" - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "SpaceToDepth rearranges blocks of spatial data into depth. More specifically,\nthis op outputs a copy of the input tensor where values from the height and width dimensions\nare moved to the depth dimension.\n" -----f -input: "input" -output: "outputs" -name: "Split" -op_type: "Split" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "split" - s: "" - type: INTS -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "Split a tensor into a list of tensors, along the specified\n\'axis\'. Lengths of the parts can be specified using argument \'split\'.\nOtherwise, the tensor is split to equal sized parts.\n" -----f -input: "input" -input: "split" -output: "output_sequence" -name: "SplitToSequence" -op_type: "SplitToSequence" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "split-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "Split a tensor into a sequence of tensors, along the specified\n\'axis\'. Lengths of the parts can be specified using argument \'split\'.\n\'split\' must contain only positive numbers.\n\'split\' is either a scalar (tensor of empty shape), or a 1-D tensor.\nIf \'split\' is a scalar, then \'input\' will be split into equally sized chunks(if possible).\nLast chunk will be smaller if the \'input\' size along the given axis \'axis\' is not divisible\nby \'split\'.\nOtherwise, the tensor is split into \'size(split)\' chunks, with lengths of the parts on \'axis\'\nspecified in \'split\'. In this scenario, the sum of entries in \'split\' must be equal to the\ndimension size of input tensor on \'axis\'.\n" -----f -input: "X" -output: "Y" -name: "Sqrt" -op_type: "Sqrt" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nSquare root takes one input data (Tensor) and produces one output data\n(Tensor) where the square root is, y = x^0.5, is applied to\nthe tensor elementwise. If x is negative, then it will return NaN.\n" -----f -input: "data" -output: "squeezed" -name: "Squeeze" -op_type: "Squeeze" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nRemove single-dimensional entries from the shape of a tensor.\nTakes a parameter `axes` with a list of axes to squeeze.\nIf `axes` is not provided, all the single dimensions will be removed from\nthe shape. If an axis is selected with shape entry not equal to one, an error is raised.\n" -----f -input: "X" -output: "Y" -name: "StringNormalizer" -op_type: "StringNormalizer" -attribute { - name: "case_change_action" - s: "NONE" - type: STRING -} -attribute { - name: "is_case_sensitive" - i: 0 - type: INT -} -attribute { - name: "locale" - s: "" - type: STRING -} -attribute { - name: "stopwords" - s: "" - type: STRINGS -} -attribute { - name: "X-types" - strings: "string" - type: STRINGS -} -doc_string: "\nStringNormalization performs string operations for basic cleaning.\nThis operator has only one input (denoted by X) and only one output\n(denoted by Y). This operator first examines the elements in the X,\nand removes elements specified in \"stopwords\" attribute.\nAfter removing stop words, the intermediate result can be further lowercased,\nuppercased, or just returned depending the \"case_change_action\" attribute.\nThis operator only accepts [C]- and [1, C]-tensor.\nIf all elements in X are dropped, the output will be the empty value of string tensor with shape [1]\nif input shape is [C] and shape [1, 1] if input shape is [1, C].\n" -----f -input: "A" -input: "B" -output: "C" -name: "Sub" -op_type: "Sub" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPerforms element-wise binary subtraction (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "data_0" -output: "sum" -name: "Sum" -op_type: "Sum" -attribute { - name: "data_0-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nElement-wise sum of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "input" -output: "output" -name: "Tan" -op_type: "Tan" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the tangent of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Tanh" -op_type: "Tanh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic tangent of the given input tensor element-wise.\n" -----f -input: "X" -output: "Y" -name: "TfIdfVectorizer" -op_type: "TfIdfVectorizer" -attribute { - name: "max_gram_length" - s: "" - type: INT -} -attribute { - name: "max_skip_count" - s: "" - type: INT -} -attribute { - name: "min_gram_length" - s: "" - type: INT -} -attribute { - name: "mode" - s: "" - type: STRING -} -attribute { - name: "ngram_counts" - s: "" - type: INTS -} -attribute { - name: "ngram_indexes" - s: "" - type: INTS -} -attribute { - name: "pool_int64s" - s: "" - type: INTS -} -attribute { - name: "pool_strings" - s: "" - type: STRINGS -} -attribute { - name: "weights" - s: "" - type: FLOATS -} -attribute { - name: "X-types" - strings: "string" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nThis transform extracts n-grams from the input sequence and save them as a vector. Input can\nbe either a 1-D or 2-D tensor. For 1-D input, output is the n-gram representation of that input.\nFor 2-D input, the output is also a 2-D tensor whose i-th row is the n-gram representation of the i-th input row.\nMore specifically, if input shape is [C], the corresponding output shape would be [max(ngram_indexes) + 1].\nIf input shape is [N, C], this operator produces a [N, max(ngram_indexes) + 1]-tensor.\n\nIn contrast to standard n-gram extraction, here, the indexes of extracting an n-gram from the original\nsequence are not necessarily consecutive numbers. The discontinuity between indexes are controlled by the number of skips.\nIf the number of skips is 2, we should skip two tokens when scanning through the original sequence.\nLet\'s consider an example. Assume that input sequence is [94, 17, 36, 12, 28] and the number of skips is 2.\nThe associated 2-grams are [94, 12] and [17, 28] respectively indexed by [0, 3] and [1, 4].\nIf the number of skips becomes 0, the 2-grams generated are [94, 17], [17, 36], [36, 12], [12, 28]\nindexed by [0, 1], [1, 2], [2, 3], [3, 4], respectively.\n\nThe output vector (denoted by Y) stores the count of each n-gram;\nY[ngram_indexes[i]] indicates the times that the i-th n-gram is found. The attribute ngram_indexes is used to determine the mapping\nbetween index i and the corresponding n-gram\'s output coordinate. If pool_int64s is [94, 17, 17, 36], ngram_indexes is [1, 0],\nngram_counts=[0, 0], then the Y[0] (first element in Y) and Y[1] (second element in Y) are the counts of [17, 36] and [94, 17],\nrespectively. An n-gram which cannot be found in pool_strings/pool_int64s should be ignored and has no effect on the output.\nNote that we may consider all skips up to S when generating the n-grams.\n\nThe examples used above are true if mode is \"TF\". If mode is \"IDF\", all the counts larger than 1 would be truncated to 1 and\nthe i-th element in weights would be used to scale (by multiplication) the count of the i-th n-gram in pool. If mode is \"TFIDF\",\nthis operator first computes the counts of all n-grams and then scale them by the associated values in the weights attribute.\n\nOnly one of pool_strings and pool_int64s can be set. If pool_int64s is set, the input should be an integer tensor.\nIf pool_strings is set, the input must be a string tensor.\n" -----f -input: "X" -output: "Y" -name: "ThresholdedRelu" -op_type: "ThresholdedRelu" -attribute { - name: "alpha" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThresholdedRelu takes one input data (Tensor) and produces one output data\n(Tensor) where the rectified linear function, y = x for x > alpha, y = 0 otherwise,\nis applied to the tensor elementwise.\n" -----f -input: "input" -input: "repeats" -output: "output" -name: "Tile" -op_type: "Tile" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "repeats-types" - strings: "int64" - type: STRINGS -} -doc_string: "Constructs a tensor by tiling a given tensor.\nThis is the same as function `tile` in Numpy, but no broadcast.\nFor example A = [[1, 2], [3, 4]], B = [1, 2], tile(A, B) = [[1, 2, 1, 2], [3, 4, 3, 4]]\n" -----f -input: "X" -input: "K" -output: "Values" -output: "Indices" -name: "TopK" -op_type: "TopK" -attribute { - name: "axis" - i: -1 - type: INT -} -attribute { - name: "largest" - i: 1 - type: INT -} -attribute { - name: "sorted" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "K-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nRetrieve the top-K largest or smallest elements along a specified axis. Given an input tensor of\nshape [a_1, a_2, ..., a_n, r] and integer argument k, return two outputs:\n -Value tensor of shape [a_1, a_2, ..., a_{axis-1}, k, a_{axis+1}, ... a_n]\n which contains the values of the top k elements along the specified axis\n -Index tensor of shape [a_1, a_2, ..., a_{axis-1}, k, a_{axis+1}, ... a_n] which\n contains the indices of the top k elements (original indices from the input\n tensor).\n\nIf \"largest\" is 1 (the default value) then the k largest elements are returned.\nIf \"sorted\" is 1 (the default value) then the resulting k elements will be sorted.\nIf \"sorted\" is 0, order of returned \'Values\' and \'Indices\' are undefined.\n\nGiven two equivalent values, this operator uses the indices along the axis as\n a tiebreaker. That is, the element with the lower index will appear first.\n" -----f -input: "data" -output: "transposed" -name: "Transpose" -op_type: "Transpose" -attribute { - name: "perm" - s: "" - type: INTS -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nTranspose the input tensor similar to numpy.transpose. For example, when\nperm=(1, 0, 2), given an input tensor of shape (1, 2, 3), the output shape\nwill be (2, 1, 3).\n" -----f -input: "X" -output: "Y" -output: "Z" -name: "TreeEnsembleClassifier" -op_type: "TreeEnsembleClassifier" -attribute { - name: "base_values" - s: "" - type: FLOATS -} -attribute { - name: "class_ids" - s: "" - type: INTS -} -attribute { - name: "class_nodeids" - s: "" - type: INTS -} -attribute { - name: "class_treeids" - s: "" - type: INTS -} -attribute { - name: "class_weights" - s: "" - type: FLOATS -} -attribute { - name: "classlabels_int64s" - s: "" - type: INTS -} -attribute { - name: "classlabels_strings" - s: "" - type: STRINGS -} -attribute { - name: "nodes_falsenodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_featureids" - s: "" - type: INTS -} -attribute { - name: "nodes_hitrates" - s: "" - type: FLOATS -} -attribute { - name: "nodes_missing_value_tracks_true" - s: "" - type: INTS -} -attribute { - name: "nodes_modes" - s: "" - type: STRINGS -} -attribute { - name: "nodes_nodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_treeids" - s: "" - type: INTS -} -attribute { - name: "nodes_truenodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_values" - s: "" - type: FLOATS -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Tree Ensemble classifier. Returns the top class for each of N inputs.
\n The attributes named \'nodes_X\' form a sequence of tuples, associated by \n index into the sequences, which must all be of equal length. These tuples\n define the nodes.
\n Similarly, all fields prefixed with \'class_\' are tuples of votes at the leaves.\n A leaf may have multiple votes, where each vote is weighted by\n the associated class_weights index.
\n One and only one of classlabels_strings or classlabels_int64s\n will be defined. The class_ids are indices into this list.\n" -----f -input: "X" -output: "Y" -name: "TreeEnsembleRegressor" -op_type: "TreeEnsembleRegressor" -attribute { - name: "aggregate_function" - s: "SUM" - type: STRING -} -attribute { - name: "base_values" - s: "" - type: FLOATS -} -attribute { - name: "n_targets" - s: "" - type: INT -} -attribute { - name: "nodes_falsenodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_featureids" - s: "" - type: INTS -} -attribute { - name: "nodes_hitrates" - s: "" - type: FLOATS -} -attribute { - name: "nodes_missing_value_tracks_true" - s: "" - type: INTS -} -attribute { - name: "nodes_modes" - s: "" - type: STRINGS -} -attribute { - name: "nodes_nodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_treeids" - s: "" - type: INTS -} -attribute { - name: "nodes_truenodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_values" - s: "" - type: FLOATS -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "target_ids" - s: "" - type: INTS -} -attribute { - name: "target_nodeids" - s: "" - type: INTS -} -attribute { - name: "target_treeids" - s: "" - type: INTS -} -attribute { - name: "target_weights" - s: "" - type: FLOATS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Tree Ensemble regressor. Returns the regressed values for each input in N.
\n All args with nodes_ are fields of a tuple of tree nodes, and\n it is assumed they are the same length, and an index i will decode the\n tuple across these inputs. Each node id can appear only once\n for each tree id.
\n All fields prefixed with target_ are tuples of votes at the leaves.
\n A leaf may have multiple votes, where each vote is weighted by\n the associated target_weights index.
\n All trees must have their node ids start at 0 and increment by 1.
\n Mode enum is BRANCH_LEQ, BRANCH_LT, BRANCH_GTE, BRANCH_GT, BRANCH_EQ, BRANCH_NEQ, LEAF\n" -----f -input: "X" -output: "Y" -output: "indices" -output: "inverse_indices" -output: "counts" -name: "Unique" -op_type: "Unique" -attribute { - name: "axis" - s: "" - type: INT -} -attribute { - name: "sorted" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nFind the unique elements of a tensor. When an optional attribute \'axis\' is provided, unique subtensors sliced along the \'axis\' are returned. \nOtherwise the input tensor is flattened and unique values of the flattened tensor are returned. \n\nThis operator returns the unique values or sliced unique subtensors of the input tensor and three optional outputs. \nThe first output tensor \'Y\' contains all unique values or subtensors of the input. \nThe second optional output tensor \'indices\' contains indices of \'Y\' elements\' first occurance in \'X\'.. \nThe third optional output tensor \'inverse_indices\' contains, for elements of \'X\', its corresponding indices in \'Y\'. \". \nThe fourth optional output tensor \'counts\' contains the count of each element of \'Y\' in the input. \n\nOutputs are either sorted in ascending order or optionally in the order of the first occurrence of the values in the input. \n\nhttps://docs.scipy.org/doc/numpy/reference/generated/numpy.unique.html\n\nExample 1:\n input_X = [2, 1, 1, 3, 4, 3]\n attribute_sorted = 0\n attribute_axis = None\n output_Y = [2, 1, 3, 4]\n output_indices = [0, 1, 3, 4]\n output_inverse_indices = [0, 1, 1, 2, 3, 2]\n output_counts = [1, 2, 2, 1]\n\nExample 2:\n input_X = [[1, 3], [2, 3]]\n attribute_sorted = 1\n attribute_axis = None\n output_Y = [1, 2, 3]\n output_indices = [0, 2, 1]\n output_inverse_indices = [0, 2, 1, 2]\n output_counts = [1, 1, 2]\n\nExample 3:\n input_X = [[1, 0, 0], [1, 0, 0], [2, 3, 4]]\n attribute_sorted = 1\n attribute_axis = 0\n output_Y = [[1, 0, 0], [2, 3, 4]]\n output_indices = [0, 2]\n output_inverse_indices = [0, 0, 1]\n output_counts = [2, 1]\n\nExample 4:\n input_x = [[[1., 1.], [0., 1.], [2., 1.], [0., 1.]], \n [[1., 1.], [0., 1.], [2., 1.], [0., 1.]]]\n attribute_sorted = 1\n attribute_axis = 1\n\n intermediate data are presented below for better understanding: \n \n there are 4 subtensors sliced along axis 1 of input_x (shape = (2, 4, 2)):\n A: [[1, 1], [1, 1]], \n [[0, 1], [0, 1]], \n [[2, 1], [2, 1]], \n [[0, 1], [0, 1]].\n \n there are 3 unique subtensors: \n [[1, 1], [1, 1]], \n [[0, 1], [0, 1]], \n [[2, 1], [2, 1]].\n \n sorted unique subtensors:\n B: [[0, 1], [0, 1]], \n [[1, 1], [1, 1]], \n [[2, 1], [2, 1]].\n \n output_Y is constructed from B:\n [[[0. 1.], [1. 1.], [2. 1.]], \n [[0. 1.], [1. 1.], [2. 1.]]]\n\n output_indices is to map from B to A:\n [1, 0, 2]\n \n output_inverse_indices is to map from A to B:\n [1, 0, 2, 0]\n\n output_counts = [2 1 1]\n" -----f -input: "data" -output: "expanded" -name: "Unsqueeze" -op_type: "Unsqueeze" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nInsert single-dimensional entries to the shape of an input tensor (`data`).\nTakes one required argument `axes` - which contains a list of dimension indices and this operator will insert a dimension of value `1` into the corresponding index of the output tensor (`expanded`).\n\nFor example:\n Given an input tensor (`data`) of shape [3, 4, 5], then\n Unsqueeze(data, axes=[0, 4]) outputs a tensor (`expanded`) containing same data as `data` but with shape [1, 3, 4, 5, 1].\n\nThe attribute `axes` should not contain any duplicate entries. It is an error if it contains duplicates.\nThe rank of the output tensor (`output_rank`) is the rank of the input tensor (`data`) plus the number of values in `axes`.\nEach value in `axes` should be within the (inclusive) range [-output_rank , output_rank - 1]. \nThe order of values in `axes` does not matter and can come in any order. \n\n" -----f -input: "X" -input: "scales" -output: "Y" -name: "Upsample" -op_type: "Upsample" -attribute { - name: "mode" - s: "nearest" - type: STRING -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "scales-types" - strings: "float" - type: STRINGS -} -doc_string: "\nUpsample the input tensor.\nEach dimension value of the output tensor is:\n output_dimension = floor(input_dimension * scale).\n" -----f -input: "condition" -input: "X" -input: "Y" -output: "output" -name: "Where" -op_type: "Where" -attribute { - name: "condition-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "Y-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\n Return elements, either from X or Y, depending on condition\n (with Numpy-style broadcasting support).\n Where behaves like numpy.where with three parameters:\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.where.html\n" -----f -input: "A" -input: "B" -output: "C" -name: "Xor" -op_type: "Xor" -attribute { - name: "A-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "B-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `xor` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Z" -name: "ZipMap" -op_type: "ZipMap" -attribute { - name: "classlabels_int64s" - s: "" - type: INTS -} -attribute { - name: "classlabels_strings" - s: "" - type: STRINGS -} -attribute { - name: "X-types" - strings: "float" - type: STRINGS -} -doc_string: "\n Creates a map from the input and the attributes.
\n The values are provided by the input tensor, while the keys are specified by the attributes.\n Must provide keys in either classlabels_strings or classlabels_int64s (but not both).
\n The columns of the tensor correspond one-by-one to the keys specified by the attributes. There must be as many columns as keys.
\n" -----f +input: "X" +output: "Y" +name: "Abs" +op_type: "Abs" +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nAbsolute takes one input data (Tensor) and produces one output data\n(Tensor) where the absolute is, y = abs(x), is applied to\nthe tensor elementwise.\n" +----f +input: "input" +output: "output" +name: "Acos" +op_type: "Acos" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the arccosine (inverse of cosine) of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Acosh" +op_type: "Acosh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic arccosine of the given input tensor element-wise.\n" +----f +input: "R" +input: "T" +input: "inputs" +output: "outputs" +name: "Adagrad" +op_type: "Adagrad" +attribute { + name: "decay_factor" + f: 0.0 + type: FLOAT +} +attribute { + name: "epsilon" + f: 1e-06 + type: FLOAT +} +attribute { + name: "norm_coefficient" + f: 0.0 + type: FLOAT +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + type: STRINGS +} +attribute { + name: "T-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "inputs-types" + strings: "float" + strings: "double" + type: STRINGS +} +doc_string: "\n Compute one iteration of ADAGRAD, a stochastic gradient based optimization\n algorithm. This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. As you can imagine, ADAGRAD requires\n some parameters:\n \n - The initial learning-rate \"R\".\n - The update count \"T\". That is, the number of training iterations conducted.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A learning-rate decay factor \"decay_factor\".\n - A small constant \"epsilon\" to avoid dividing-by-zero. \n\n At each ADAGRAD iteration, the optimized tensors are moved along a direction\n computed based on their estimated gradient and accumulated squared gradient. Assume\n that only a single tensor \"X\" is updated by this operator. We need the value of \"X\",\n its gradient \"G\", and its accumulated squared gradient \"H\". Therefore, variables in\n this operator\'s input list are sequentially \"R\", \"T\", \"X\", \"G\", and \"H\". Other\n parameters are given as attributes because they are usually constants. Also, the\n corresponding output tensors are the new value of \"X\" (called \"X_new\"), and then\n the new accumulated squared gradient (called \"H_new\"). Those outputs are computed\n from the given inputs following the pseudo code below.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise arithmetic operations with\n numpy-style broadcasting support. The pseudo code to compute those outputs is:\n\n // Compute a scalar learning-rate factor. At the first update of X, T is generally\n // 0 (0-based update index) or 1 (1-based update index).\n r = R / (1 + T * decay_factor);\n\n // Add gradient of 0.5 * norm_coefficient * ||X||_2^2, where ||X||_2 is the 2-norm.\n G_regularized = norm_coefficient * X + G;\n\n // Compute new accumulated squared gradient.\n H_new = H + G_regularized * G_regularized;\n\n // Compute the adaptive part of per-coordinate learning rate. Note that Sqrt(...)\n // computes element-wise square-root.\n H_adaptive = Sqrt(H_new) + epsilon\n\n // Compute the new value of \"X\".\n X_new = X - r * G_regularized / H_adaptive;\n\n If one assign this operators to optimize multiple inputs, for example, \"X_1\" and \"X_2\", the same\n pseudo code may be extended to handle all tensors jointly. More specifically, we can view \"X\" as a\n concatenation of \"X_1\" and \"X_2\" (of course, their gradient and accumulate gradient should\n be concatenated too) and then just reuse the entire pseudo code.\n\n Note that ADAGRAD was first proposed in http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf.\n In that reference paper, this operator is a special case of the Figure 1\'s composite mirror\n descent update.\n" +----f +input: "R" +input: "T" +input: "inputs" +output: "outputs" +name: "Adam" +op_type: "Adam" +attribute { + name: "alpha" + f: 0.9 + type: FLOAT +} +attribute { + name: "beta" + f: 0.999 + type: FLOAT +} +attribute { + name: "epsilon" + f: 1e-06 + type: FLOAT +} +attribute { + name: "norm_coefficient" + f: 0.0 + type: FLOAT +} +attribute { + name: "norm_coefficient_post" + f: 0.0 + type: FLOAT +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + type: STRINGS +} +attribute { + name: "T-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "inputs-types" + strings: "float" + strings: "double" + type: STRINGS +} +doc_string: "\n Compute one iteration of Adam, a stochastic gradient based optimization\n algorithm. This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. First of all, Adam requires\n some parameters:\n \n - The learning-rate \"R\".\n - The update count \"T\". That is, the number of training iterations conducted.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A small constant \"epsilon\" to avoid dividing-by-zero. \n - Two coefficients, \"alpha\" and \"beta\".\n\n At each Adam iteration, the optimized tensors are moved along a direction\n computed based on their exponentially-averaged historical gradient and\n exponentially-averaged historical squared gradient. Assume that only a tensor\n \"X\" is being optimized. The rest of required information is\n \n - the value of \"X\",\n - \"X\"\'s gradient (denoted by \"G\"),\n - \"X\"\'s exponentially-averaged historical gradient (denoted by \"V\"), and\n - \"X\"\'s exponentially-averaged historical squared gradient (denoted by \"H\").\n\n Some of those parameters are passed into this operator as input tensors and others\n are stored as this operator\'s attributes. Specifically, this operator\'s input tensor\n list is [\"R\", \"T\", \"X\", \"G\", \"V\", \"H\"]. That is, \"R\" is the first input, \"T\" is\n the second input, and so on. Other parameters are given as attributes because they\n are constants. Moreover, the corresponding output tensors are \n \n - the new value of \"X\" (called \"X_new\"),\n - the new exponentially-averaged historical gradient (denoted by \"V_new\"), and\n - the new exponentially-averaged historical squared gradient (denoted by \"H_new\").\n\n Those outputs are computed following the pseudo code below.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise arithmetic operations with\n numpy-style broadcasting support. The pseudo code to compute those outputs is:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||_2^2, where ||X||_2 is the 2-norm.\n G_regularized = norm_coefficient * X + G\n\n // Update exponentially-averaged historical gradient.\n V_new = alpha * V + (1 - alpha) * G_regularized\n\n // Update exponentially-averaged historical squared gradient.\n H_new = beta * H + (1 - beta) * G_regularized * G_regularized\n\n // Compute the element-wise square-root of H_new. V_new will be element-wisely\n // divided by H_sqrt for a better update direction.\n H_sqrt = Sqrt(H_new) + epsilon\n\n // Compute learning-rate. Note that \"alpha**T\"/\"beta**T\" is alpha\'s/beta\'s T-th power.\n R_adjusted = T > 0 ? R * Sqrt(1 - beta**T) / (1 - alpha**T) : R\n\n // Compute new value of \"X\".\n X_new = X - R_adjusted * V_new / H_sqrt\n\n // Post-update regularization.\n X_final = (1 - norm_coefficient_post) * X_new \n\n If there are multiple inputs to be optimized, the pseudo code will be applied\n independently to each of them.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Add" +op_type: "Add" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPerforms element-wise binary addition (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "A" +input: "B" +output: "C" +name: "And" +op_type: "And" +attribute { + name: "A-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "B-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `and` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "data" +output: "reduced" +name: "ArgMax" +op_type: "ArgMax" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "select_last_index" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the indices of the max elements of the input tensor\'s element along the \nprovided axis. The resulting tensor has the same rank as the input if keepdims equal 1. \nIf keepdims equal 0, then the resulting tensor have the reduced dimension pruned. \nIf select_last_index is True (default False), the index of the last occurrence of the max \nis selected if the max appears more than once in the input. Otherwise the index of the \nfirst occurrence is selected.\nThe type of the output tensor is integer." +----f +input: "data" +output: "reduced" +name: "ArgMin" +op_type: "ArgMin" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "select_last_index" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the indices of the min elements of the input tensor\'s element along the \nprovided axis. The resulting tensor has the same rank as the input if keepdims equal 1. \nIf keepdims equal 0, then the resulting tensor have the reduced dimension pruned. \nIf select_last_index is True (default False), the index of the last occurrence of the min \nis selected if the min appears more than once in the input. Otherwise the index of the \nfirst occurrence is selected.\nThe type of the output tensor is integer." +----f +input: "X" +input: "Y" +output: "Z" +name: "ArrayFeatureExtractor" +op_type: "ArrayFeatureExtractor" +attribute { + name: "X-types" + strings: "int32" + strings: "string" + strings: "double" + strings: "int64" + strings: "float" + type: STRINGS +} +attribute { + name: "Y-types" + strings: "int64" + type: STRINGS +} +doc_string: "\n Select elements of the input tensor based on the indices passed.
\n The indices are applied to the last axes of the tensor.\n" +----f +input: "input" +output: "output" +name: "Asin" +op_type: "Asin" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the arcsine (inverse of sine) of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Asinh" +op_type: "Asinh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic arcsine of the given input tensor element-wise.\n" +----f +input: "input" +output: "output" +name: "Atan" +op_type: "Atan" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the arctangent (inverse of tangent) of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Atanh" +op_type: "Atanh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic arctangent of the given input tensor element-wise.\n" +----f +input: "X" +output: "Y" +name: "AveragePool" +op_type: "AveragePool" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "ceil_mode" + i: 0 + type: INT +} +attribute { + name: "count_include_pad" + i: 0 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n AveragePool consumes an input tensor X and applies average pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n average pooling consisting of computing the average on all values of a\n subset of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing. The output spatial shape will be following:\n ```\n output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1)\n ```\n or\n ```\n output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1)\n ```\n if ceil_mode is enabled\n\n ```\n * pad_shape[i] is sum of pads along axis i\n ```\n\n `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following:\n ```\n VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - kernel_spatial_shape[i] + 1) / strides_spatial_shape[i])\n SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i])\n ```\n And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`:\n ```\n pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + kernel_spatial_shape[i] - input_spatial_shape[i]\n ```\n The output of each pooling window is divided by the number of elements (exclude pad when attribute count_include_pad is zero).\n " +----f +input: "X" +input: "scale" +input: "B" +input: "mean" +input: "var" +output: "Y" +output: "mean" +output: "var" +output: "saved_mean" +output: "saved_var" +name: "BatchNormalization" +op_type: "BatchNormalization" +attribute { + name: "epsilon" + f: 1e-05 + type: FLOAT +} +attribute { + name: "momentum" + f: 0.9 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "scale-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "mean-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "var-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCarries out batch normalization as described in the paper\nhttps://arxiv.org/abs/1502.03167. Depending on the mode it is being run,\nthere are multiple cases for the number of outputs, which we list below:\n\nOutput case #1: Y, mean, var, saved_mean, saved_var (training mode)\nOutput case #2: Y (test mode)\n\nFor previous (depreciated) non-spatial cases, implementors are suggested\nto flatten the input shape to (N x C*D1*D2 ..*Dn) before a BatchNormalization Op.\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "X" +output: "Y" +name: "Binarizer" +op_type: "Binarizer" +attribute { + name: "threshold" + f: 0.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Maps the values of the input tensor to either 0 or 1, element-wise, based on the outcome of a comparison against a threshold value.\n" +----f +input: "X" +input: "Y" +output: "Z" +name: "BitShift" +op_type: "BitShift" +attribute { + name: "direction" + s: "" + type: STRING +} +attribute { + name: "X-types" + strings: "uint32" + strings: "uint16" + strings: "uint8" + strings: "uint64" + type: STRINGS +} +attribute { + name: "Y-types" + strings: "uint32" + strings: "uint16" + strings: "uint8" + strings: "uint64" + type: STRINGS +} +doc_string: "\nBitwise shift operator performs element-wise operation. For each input element, if the\n attribute \"direction\" is \"RIGHT\", this operator moves its binary representation toward\n the right side so that the input value is effectively decreased. If the attribute \"direction\"\n is \"LEFT\", bits of binary representation moves toward the left side, which results the\n increase of its actual value. The input X is the tensor to be shifted and another input\n Y specifies the amounts of shifting. For example, if \"direction\" is \"Right\", X is [1, 4],\n and S is [1, 1], the corresponding output Z would be [0, 2]. If \"direction\" is \"LEFT\" with\n X=[1, 2] and S=[1, 2], the corresponding output Y would be [2, 8].\n \n Because this operator supports Numpy-style broadcasting, X\'s and Y\'s shapes are\n not necessarily identical.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md)." +----f +input: "input" +output: "output" +name: "Cast" +op_type: "Cast" +attribute { + name: "to" + s: "" + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "float16" + strings: "int32" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nThe operator casts the elements of a given input tensor to a data type\nspecified by the \'to\' argument and returns an output tensor of the same size in\nthe converted type. The \'to\' argument must be one of the data types specified\nin the \'DataType\' enum field in the TensorProto message.\n\nCasting from string tensor in plain (e.g., \"3.14\" and \"1000\") and scientific numeric representations\n(e.g., \"1e-5\" and \"1E8\") to float types is supported. For example, converting string \"100.5\" to an integer may\nresult 100. There are some string literals reserved for special floating-point values;\n\"+INF\" (and \"INF\"), \"-INF\", and \"NaN\" are positive infinity, negative infinity, and not-a-number, respectively.\nAny string which can exactly match \"+INF\" in a case-insensitive way would be mapped to positive infinite. Similarly,\nthis case-insensitive rule is applied to \"INF\" and \"NaN\". When casting from numeric tensors\nto string tensors, plain floating-point representation (such as \"314.15926\") would be used. \nConverting non-numerical-literal string such as \"Hello World!\" is an undefined behavior. Cases \nof converting string representing floating-point arithmetic value, such as \"2.718\", to INT is an undefined behavior.\n\nConversion from a numerical type to any numerical type is always allowed.\nUser must be aware of precision loss and value change caused by range difference between two types.\nFor example, a 64-bit float 3.1415926459 may be round to a 32-bit float 3.141592. Similarly, converting\nan integer 36 to Boolean may produce 1 because we truncate bits which can\'t be stored in the targeted type.\n" +----f +input: "X" +output: "Y" +name: "CastMap" +op_type: "CastMap" +attribute { + name: "cast_to" + s: "TO_FLOAT" + type: STRING +} +attribute { + name: "map_form" + s: "DENSE" + type: STRING +} +attribute { + name: "max_map" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "map(int64,string" + strings: "map(int64,float" + type: STRINGS +} +doc_string: "\n Converts a map to a tensor.
The map key must be an int64 and the values will be ordered\n in ascending order based on this key.
The operator supports dense packing or sparse packing.\n If using sparse packing, the key cannot exceed the max_map-1 value.\n" +----f +input: "X" +output: "Y" +name: "CategoryMapper" +op_type: "CategoryMapper" +attribute { + name: "cats_int64s" + s: "" + type: INTS +} +attribute { + name: "cats_strings" + s: "" + type: STRINGS +} +attribute { + name: "default_int64" + i: -1 + type: INT +} +attribute { + name: "default_string" + s: "_Unused" + type: STRING +} +attribute { + name: "X-types" + strings: "string" + strings: "int64" + type: STRINGS +} +doc_string: "\n Converts strings to integers and vice versa.
\n Two sequences of equal length are used to map between integers and strings,\n with strings and integers at the same index detailing the mapping.
\n Each operator converts either integers to strings or strings to integers, depending \n on which default value attribute is provided. Only one default value attribute\n should be defined.
\n If the string default value is set, it will convert integers to strings.\n If the int default value is set, it will convert strings to integers.\n" +----f +input: "X" +output: "Y" +name: "Ceil" +op_type: "Ceil" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCeil takes one input data (Tensor) and produces one output data\n(Tensor) where the ceil is, y = ceil(x), is applied to\nthe tensor elementwise.\n" +----f +input: "X" +output: "Y" +name: "Celu" +op_type: "Celu" +attribute { + name: "alpha" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + type: STRINGS +} +doc_string: "\nContinuously Differentiable Exponential Linear Units:\nPerform the linear unit element-wise on the input tensor X\nusing formula: \n\n```\nmax(0,x) + min(0,alpha*(exp(x/alpha)-1))\n```\n" +----f +input: "input" +input: "min" +input: "max" +output: "output" +name: "Clip" +op_type: "Clip" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "min-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "max-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nClip operator limits the given input within an interval. The interval is\nspecified by the inputs \'min\' and \'max\'. They default to\nnumeric_limits::lowest() and numeric_limits::max(), respectively.\n" +----f +input: "input" +input: "condition" +output: "output" +name: "Compress" +op_type: "Compress" +attribute { + name: "axis" + s: "" + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "condition-types" + strings: "bool" + type: STRINGS +} +doc_string: "\n Selects slices from an input tensor along a given axis where condition evaluates to True for each axis index.\n In case axis is not provided, input is flattened before elements are selected.\n Compress behaves like numpy.compress: https://docs.scipy.org/doc/numpy/reference/generated/numpy.compress.html\n " +----f +input: "inputs" +output: "concat_result" +name: "Concat" +op_type: "Concat" +attribute { + name: "axis" + s: "" + type: INT +} +attribute { + name: "inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "Concatenate a list of tensors into a single tensor. All input tensors must have the same shape, except for the dimension size of the axis to concatenate on." +----f +input: "input_sequence" +output: "concat_result" +name: "ConcatFromSequence" +op_type: "ConcatFromSequence" +attribute { + name: "axis" + s: "" + type: INT +} +attribute { + name: "new_axis" + i: 0 + type: INT +} +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +doc_string: "\nConcatenate a sequence of tensors into a single tensor.\nAll input tensors must have the same shape, except for the dimension size of the axis to concatenate on.\nBy default \'new_axis\' is 0, the behavior is similar to numpy.concatenate.\nWhen \'new_axis\' is 1, the behavior is similar to numpy.stack.\n" +----f +output: "output" +name: "Constant" +op_type: "Constant" +attribute { + name: "sparse_value" + s: "" + type: SPARSE_TENSOR +} +attribute { + name: "value" + s: "" + type: TENSOR +} +attribute { + name: "value_float" + s: "" + type: FLOAT +} +attribute { + name: "value_floats" + s: "" + type: FLOATS +} +attribute { + name: "value_int" + s: "" + type: INT +} +attribute { + name: "value_ints" + s: "" + type: INTS +} +attribute { + name: "value_string" + s: "" + type: STRING +} +attribute { + name: "value_strings" + s: "" + type: STRINGS +} +doc_string: "\nThis operator produces a constant tensor. Exactly one of the provided attributes, either value, sparse_value,\nor value_* must be specified.\n" +----f +input: "input" +output: "output" +name: "ConstantOfShape" +op_type: "ConstantOfShape" +attribute { + name: "value" + s: "" + type: TENSOR +} +attribute { + name: "input-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nGenerate a tensor with given value and shape.\n" +----f +input: "X" +input: "W" +input: "B" +output: "Y" +name: "Conv" +op_type: "Conv" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "group" + i: 1 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe convolution operator consumes an input tensor and a filter, and\ncomputes the output." +----f +input: "x" +input: "w" +input: "x_zero_point" +input: "w_zero_point" +output: "y" +name: "ConvInteger" +op_type: "ConvInteger" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "group" + i: 1 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "x-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "w-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "x_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "w_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nThe integer convolution operator consumes an input tensor, its zero-point, a filter, and its zero-point,\nand computes the output. The production MUST never overflow. The accumulation may overflow if and only if in 32 bits.\n" +----f +input: "X" +input: "W" +input: "B" +output: "Y" +name: "ConvTranspose" +op_type: "ConvTranspose" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "group" + i: 1 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "output_padding" + s: "" + type: INTS +} +attribute { + name: "output_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe convolution transpose operator consumes an input tensor and a filter,\nand computes the output.\n\nIf the pads parameter is provided the shape of the output is calculated via the following equation:\n\n output_shape[i] = stride[i] * (input_size[i] - 1) + output_padding[i] + ((kernel_shape[i] - 1) * dilations[i] + 1) - pads[start_i] - pads[end_i]\n\noutput_shape can also be explicitly specified in which case pads values are auto generated using these equations:\n\n total_padding[i] = stride[i] * (input_size[i] - 1) + output_padding[i] + ((kernel_shape[i] - 1) * dilations[i] + 1) - output_shape[i]\n If (auto_pads != SAME_UPPER): pads[start_i] = total_padding[i]/2; pads[end_i] = total_padding[i] - (total_padding[i]/2)\n Else: pads[start_i] = total_padding[i] - (total_padding[i]/2); pads[end_i] = (total_padding[i]/2).\n\n " +----f +input: "input" +output: "output" +name: "Cos" +op_type: "Cos" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the cosine of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Cosh" +op_type: "Cosh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic cosine of the given input tensor element-wise.\n" +----f +input: "x" +input: "axis" +output: "y" +name: "CumSum" +op_type: "CumSum" +attribute { + name: "exclusive" + i: 0 + type: INT +} +attribute { + name: "reverse" + i: 0 + type: INT +} +attribute { + name: "x-types" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "axis-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nPerforms cumulative sum of the input elements along the given axis.\nBy default, it will do the sum inclusively meaning the first element is copied as is.\nThrough an `exclusive` attribute, this behavior can change to exclude the first element.\nIt can also perform summation in the opposite direction of the axis. For that, set `reverse` attribute to 1.\n\nExample:\n```\ninput_x = [1, 2, 3]\naxis=0\noutput = [1, 3, 6]\nexclusive=1\noutput = [0, 1, 3]\nexclusive=0\nreverse=1\noutput = [6, 5, 3]\nexclusive=1\nreverse=1\noutput = [5, 3, 0]\n```\n " +----f +input: "input" +output: "output" +name: "DepthToSpace" +op_type: "DepthToSpace" +attribute { + name: "blocksize" + s: "" + type: INT +} +attribute { + name: "mode" + s: "DCR" + type: STRING +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "DepthToSpace rearranges (permutes) data from depth into blocks of spatial data.\nThis is the reverse transformation of SpaceToDepth. More specifically, this op outputs a copy of\nthe input tensor where values from the depth dimension are moved in spatial blocks to the height\nand width dimensions. By default, `mode` = `DCR`.\nIn the DCR mode, elements along the depth dimension from the input tensor are rearranged in the\nfollowing order: depth, column, and then row. The output y is computed from the input x as below:\n\nb, c, h, w = x.shape\n\ntmp = np.reshape(x, [b, blocksize, blocksize, c // (blocksize**2), h, w])\n\ntmp = np.transpose(tmp, [0, 3, 4, 1, 5, 2])\n\ny = np.reshape(tmp, [b, c // (blocksize**2), h * blocksize, w * blocksize])\n\n\nIn the CRD mode, elements along the depth dimension from the input tensor are rearranged in the\nfollowing order: column, row, and the depth. The output y is computed from the input x as below:\n\nb, c, h, w = x.shape\n\ntmp = np.reshape(x, [b, c // (blocksize ** 2), blocksize, blocksize, h, w])\n\ntmp = np.transpose(tmp, [0, 1, 4, 2, 5, 3])\n\ny = np.reshape(tmp, [b, c // (blocksize ** 2), h * blocksize, w * blocksize])\n\n" +----f +input: "x" +input: "x_scale" +input: "x_zero_point" +output: "y" +name: "DequantizeLinear" +op_type: "DequantizeLinear" +attribute { + name: "x-types" + strings: "int32" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "x_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "x_zero_point-types" + strings: "int32" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nThe linear dequantization operator. It consumes a quantized tensor, a scale, a zero point to compute the full precision tensor.\nThe dequantization formula is y = (x - x_zero_point) * x_scale. \'x_scale\' and \'x_zero_point\' must have same shape.\n\'x_zero_point\' and \'x\' must have same type. \'x\' and \'y\' must have same shape. In the case of dequantizing int32,\nthere\'s no zero point (zero point is supposed to be 0).\n" +----f +input: "X" +output: "Y" +name: "Det" +op_type: "Det" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nDet calculates determinant of a square matrix or batches of square matrices.\nDet takes one input tensor of shape `[*, M, M]`, where `*` is zero or more batch dimensions,\nand the inner-most 2 dimensions form square matrices.\nThe output is a tensor of shape `[*]`, containing the determinants of all input submatrices.\ne.g., When the input is 2-D, the output is a scalar(shape is empty: `[]`).\n" +----f +input: "X" +output: "Y" +name: "DictVectorizer" +op_type: "DictVectorizer" +attribute { + name: "int64_vocabulary" + s: "" + type: INTS +} +attribute { + name: "string_vocabulary" + s: "" + type: STRINGS +} +attribute { + name: "X-types" + strings: "map(int64,float" + strings: "map(int64,string" + strings: "map(string,int64" + strings: "map(string,float" + strings: "map(string,double" + strings: "map(int64,double" + type: STRINGS +} +doc_string: "\n Uses an index mapping to convert a dictionary to an array.
\n Given a dictionary, each key is looked up in the vocabulary attribute corresponding to\n the key type. The index into the vocabulary array at which the key is found is then\n used to index the output 1-D tensor \'Y\' and insert into it the value found in the dictionary \'X\'.
\n The key type of the input map must correspond to the element type of the defined vocabulary attribute.\n Therefore, the output array will be equal in length to the index mapping vector parameter.\n All keys in the input dictionary must be present in the index mapping vector.\n For each item in the input dictionary, insert its value in the output array.\n Any keys not present in the input dictionary, will be zero in the output array.
\n For example: if the ``string_vocabulary`` parameter is set to ``[\"a\", \"c\", \"b\", \"z\"]``,\n then an input of ``{\"a\": 4, \"c\": 8}`` will produce an output of ``[4, 8, 0, 0]``.\n " +----f +input: "A" +input: "B" +output: "C" +name: "Div" +op_type: "Div" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPerforms element-wise binary division (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "data" +input: "ratio" +input: "training_mode" +output: "output" +output: "mask" +name: "Dropout" +op_type: "Dropout" +attribute { + name: "seed" + s: "" + type: INT +} +attribute { + name: "data-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "ratio-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "training_mode-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nDropout takes an input floating-point tensor, an optional input ratio (floating-point scalar) and an optional input training_mode (boolean scalar). It produces two tensor outputs,\noutput (floating-point tensor) and mask (optional `Tensor`). If `training_mode` is true then the output Y will be a random dropout;\nNote that this Dropout scales the masked input data by the following equation, so to convert the trained model into inference mode,\nthe user can simply not pass `training_mode` input or set it to false.\n```\noutput = scale * data * mask,\n```\nwhere\n```\nscale = 1. / (1. - ratio).\n```\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "x" +output: "y" +output: "y_scale" +output: "y_zero_point" +name: "DynamicQuantizeLinear" +op_type: "DynamicQuantizeLinear" +attribute { + name: "x-types" + strings: "float" + type: STRINGS +} +doc_string: "\nA Function to fuse calculation for Scale, Zero Point and FP32->8Bit convertion of FP32 Input data.\nOutputs Scale, ZeroPoint and Quantized Input for a given FP32 Input.\nScale is calculated as:\n```\n y_scale = (max(x) - min(x))/(qmax - qmin)\n * where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8\n * data range is adjusted to include 0.\n```\nZero point is calculated as:\n```\nintermediate_zero_point = qmin - min(x)/y_scale\ny_zero_point = cast(round(saturate(itermediate_zero_point)))\n* where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8\n* for saturation, it saturates to [0, 255] if it\'s uint8, or [-127, 127] if it\'s int8. Right now only uint8 is supported.\n* rounding to nearest ties to even.\n```\nData quantization formula is:\n```\ny = saturate (round (x / y_scale) + y_zero_point)\n* for saturation, it saturates to [0, 255] if it\'s uint8, or [-127, 127] if it\'s int8. Right now only uint8 is supported.\n* rounding to nearest ties to even.\n```\n" +----f +input: "Inputs" +output: "Output" +name: "Einsum" +op_type: "Einsum" +attribute { + name: "equation" + s: "" + type: STRING +} +attribute { + name: "Inputs-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nAn einsum of the form ```term1, term2 -> output-term``` produces an output tensor using the following equation\n\n```output[output-term] = reduce-sum( input1[term1] * input2[term] )```\n\nwhere the reduce-sum performs a summation over all the indices occurring in in the input terms (term1, term2)\nthat do not occur in the output-term.\n\nThe Einsum operator evaluates algebraic tensor operations on a sequence of tensors, using the Einstein summation\nconvention. The equation string contains a comma-separated sequence of lower case letters. Each term corresponds to\nan operand tensor, and the characters within the terms correspond to operands dimensions.\n\nThis sequence may be followed by \"->\" to separate the left and right hand side of the equation.\nIf the equation contains \"->\" followed by the right-hand side, the explicit (not classical) form of the Einstein\nsummation is performed, and the right-hand side indices indicate output tensor dimensions. In other cases,\noutput indices are (implicitly) set to the alphabetically sorted sequence of indices appearing exactly once in the\nequation.\n\nWhen a dimension character is repeated in the left-hand side, it represents summation along the dimension.\n\nThe equation may contain ellipsis (\"...\") to enable broadcasting. Ellipsis must indicate a fixed number of dimensions.\nSpecifically, every occurrence of ellipsis in the equation must represent the same number of dimensions.\nThe right-hand side may contain exactly one ellipsis. In implicit mode, the ellipsis dimensions are set to the\nbeginning of the output. The equation string may contain space (U+0020) character.\n" +----f +input: "X" +output: "Y" +name: "Elu" +op_type: "Elu" +attribute { + name: "alpha" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nElu takes one input data (Tensor) and produces one output data\n(Tensor) where the function `f(x) = alpha * (exp(x) - 1.) for x <\n0`, `f(x) = x for x >= 0`., is applied to the tensor elementwise.\n\n" +----f +input: "A" +input: "B" +output: "C" +name: "Equal" +op_type: "Equal" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "input" +output: "output" +name: "Erf" +op_type: "Erf" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the error function of the given input tensor element-wise.\n" +----f +input: "input" +output: "output" +name: "Exp" +op_type: "Exp" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the exponential of the given input tensor, element-wise.\n" +----f +input: "input" +input: "shape" +output: "output" +name: "Expand" +op_type: "Expand" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "shape-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nBroadcast the input tensor following the given shape and the broadcast rule.\nThe broadcast rule is similar to numpy.array(input) * numpy.ones(shape):\nDimensions are right alignment;\nTwo corresponding dimension must have the same value, or one of them is equal to 1.\nAlso, this operator is similar to numpy.broadcast_to(input, shape),\nbut the major difference is numpy.broadcast_to() does not allow shape to be smaller than input.size().\nIt is possible that the output.shape is not equal to shape, when some dimensions in shape is equal to 1,\nor the shape.ndim < input.shape.ndim.\n" +----f +input: "input" +output: "output" +name: "EyeLike" +op_type: "EyeLike" +attribute { + name: "dtype" + s: "" + type: INT +} +attribute { + name: "k" + i: 0 + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "float16" + strings: "int32" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGenerate a 2D tensor (matrix) with ones on the diagonal and zeros everywhere else. Only 2D\ntensors are supported, i.e. input T1 must be of rank 2. The shape of the output tensor is the\nsame as the input tensor. The data type can be specified by the \'dtype\' argument. If\n\'dtype\' is not specified, then the type of input tensor is used. By default, the main diagonal\nis populated with ones, but attribute \'k\' can be used to populate upper or lower diagonals.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message and be valid as an output type.\n" +----f +input: "X" +output: "Y" +name: "FeatureVectorizer" +op_type: "FeatureVectorizer" +attribute { + name: "inputdimensions" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Concatenates input tensors into one continuous output.
\n All input shapes are 2-D and are concatenated along the second dimention. 1-D tensors are treated as [1,C].\n Inputs are copied to the output maintaining the order of the input arguments.
\n All inputs must be integers or floats, while the output will be all floating point values.\n" +----f +input: "input" +output: "output" +name: "Flatten" +op_type: "Flatten" +attribute { + name: "axis" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nFlattens the input tensor into a 2D matrix. If input tensor has shape\n(d_0, d_1, ... d_n) then the output will have shape\n(d_0 X d_1 ... d_(axis-1), d_axis X d_(axis+1) ... X dn).\n" +----f +input: "X" +output: "Y" +name: "Floor" +op_type: "Floor" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nFloor takes one input data (Tensor) and produces one output data\n(Tensor) where the floor is, y = floor(x), is applied to\nthe tensor elementwise.\n" +----f +input: "X" +input: "W" +input: "R" +input: "B" +input: "sequence_lens" +input: "initial_h" +output: "Y" +output: "Y_h" +name: "GRU" +op_type: "GRU" +attribute { + name: "activation_alpha" + s: "" + type: FLOATS +} +attribute { + name: "activation_beta" + s: "" + type: FLOATS +} +attribute { + name: "activations" + s: "" + type: STRINGS +} +attribute { + name: "clip" + s: "" + type: FLOAT +} +attribute { + name: "direction" + s: "forward" + type: STRING +} +attribute { + name: "hidden_size" + s: "" + type: INT +} +attribute { + name: "linear_before_reset" + i: 0 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "sequence_lens-types" + strings: "int32" + type: STRINGS +} +attribute { + name: "initial_h-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nComputes an one-layer GRU. This operator is usually supported via some custom\nimplementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`z` - update gate\n\n`r` - reset gate\n\n`h` - hidden gate\n\n`t` - time step (t-1 means previous time step)\n\n`W[zrh]` - W parameter weight matrix for update, reset, and hidden gates\n\n`R[zrh]` - R recurrence weight matrix for update, reset, and hidden gates\n\n`Wb[zrh]` - W bias vectors for update, reset, and hidden gates\n\n`Rb[zrh]` - R bias vectors for update, reset, and hidden gates\n\n`WB[zrh]` - W parameter weight matrix for backward update, reset, and hidden gates\n\n`RB[zrh]` - R recurrence weight matrix for backward update, reset, and hidden gates\n\n`WBb[zrh]` - W bias vectors for backward update, reset, and hidden gates\n\n`RBb[zrh]` - R bias vectors for backward update, reset, and hidden gates\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Sigmoid, g=Tanh):\n\n - zt = f(Xt*(Wz^T) + Ht-1*(Rz^T) + Wbz + Rbz)\n\n - rt = f(Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr)\n\n - ht = g(Xt*(Wh^T) + (rt (.) Ht-1)*(Rh^T) + Rbh + Wbh) # default, when linear_before_reset = 0\n\n - ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) + Wbh) # when linear_before_reset != 0\n\n - Ht = (1 - zt) (.) ht + zt (.) Ht-1\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "data" +input: "indices" +output: "output" +name: "Gather" +op_type: "Gather" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nGiven `data` tensor of rank r >= 1, and `indices` tensor of rank q, gather\nentries of the axis dimension of `data` (by default outer-most one as axis=0) indexed by `indices`, and concatenates\nthem in an output tensor of rank q + (r - 1).\n\naxis = 0 :\n\nLet\nk = indices[i_{0}, ..., i_{q-1}]\nThen\noutput[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[k , j_{0}, ..., j_{r-2}]\n\n```\n data = [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ]\n indices = [\n [0, 1],\n [1, 2],\n ]\n output = [\n [\n [1.0, 1.2],\n [2.3, 3.4],\n ],\n [\n [2.3, 3.4],\n [4.5, 5.7],\n ],\n ]\n```\naxis = 1 :\n\nLet\nk = indices[i_{0}, ..., i_{q-1}]\nThen\noutput[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[j_{0}, k, j_{1}, ..., j_{r-2}]\n\n```\n data = [\n [1.0, 1.2, 1.9],\n [2.3, 3.4, 3.9],\n [4.5, 5.7, 5.9],\n ]\n indices = [\n [0, 2],\n ]\n axis = 1,\n output = [\n [\n [1.0, 1.9],\n [2.3, 3.9],\n [4.5, 5.9],\n ],\n ]\n```\n" +----f +input: "data" +input: "indices" +output: "output" +name: "GatherElements" +op_type: "GatherElements" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n\nGatherElements takes two inputs `data` and `indices` of the same rank r >= 1\nand an optional attribute `axis` that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). It is an indexing operation\nthat produces its output by indexing into the input data tensor at index\npositions determined by elements of the `indices` tensor.\nIts output shape is the same as the shape of `indices` and consists of one value\n(gathered from the `data`) for each element in `indices`.\n\nFor instance, in the 3-D case (r = 3), the output produced is determined\nby the following equations: \n```\n out[i][j][k] = input[index[i][j][k]][j][k] if axis = 0,\n out[i][j][k] = input[i][index[i][j][k]][k] if axis = 1,\n out[i][j][k] = input[i][j][index[i][j][k]] if axis = 2,\n```\n\nThis operator is also the inverse of ScatterElements. It is similar to Torch\'s gather operation.\n\nExample 1:\n```\n data = [\n [1, 2],\n [3, 4],\n ]\n indices = [\n [0, 0],\n [1, 0],\n ]\n axis = 1\n output = [\n [\n [1, 1],\n [4, 3],\n ],\n ]\n```\nExample 2:\n```\n data = [\n [1, 2, 3],\n [4, 5, 6],\n [7, 8, 9],\n ]\n indices = [\n [1, 2, 0],\n [2, 0, 0],\n ]\n axis = 0\n output = [\n [\n [4, 8, 3],\n [7, 2, 3],\n ],\n ]\n```\n" +----f +input: "data" +input: "indices" +output: "output" +name: "GatherND" +op_type: "GatherND" +attribute { + name: "batch_dims" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nGiven `data` tensor of rank `r` >= 1, `indices` tensor of rank `q` >= 1, and `batch_dims` integer `b`, this operator gathers \nslices of `data` into an output tensor of rank `q + r - indices_shape[-1] - 1 - b`.\n\n`indices` is an q-dimensional integer tensor, best thought of as a `(q-1)`-dimensional tensor of index-tuples into `data`, \nwhere each element defines a slice of `data`\n\n`batch_dims` (denoted as `b`) is an integer indicating the number of batch dimensions, i.e the leading `b` number of dimensions of \n`data` tensor and `indices` are representing the batches, and the gather starts from the `b+1` dimension. \n\nSome salient points about the inputs\' rank and shape:\n \n1) r >= 1 and q >= 1 are to be honored. There is no dependency condition to be met between ranks `r` and `q`\n\n2) The first `b` dimensions of the shape of `indices` tensor and `data` tensor must be equal.\n\n3) b < min(q, r) is to be honored.\n\n4) The `indices_shape[-1]` should have a value between 1 (inclusive) and rank `r-b` (inclusive) \n\n5) All values in `indices` are expected to be within bounds [-s, s-1] along axis of size `s` (i.e.) `-data_shape[i] <= indices[...,i] <= data_shape[i] - 1`.\n It is an error if any of the index values are out of bounds.\n\nThe output is computed as follows:\n\nThe output tensor is obtained by mapping each index-tuple in the `indices` tensor to the corresponding slice of the input `data`.\n \n1) If `indices_shape[-1] > r-b` => error condition\n\n2) If `indices_shape[-1] == r-b`, since the rank of `indices` is `q`, `indices` can be thought of as `N` `(q-b-1)`-dimensional tensors\n containing 1-D tensors of dimension `r-b`, where `N` is an integer equals to the product of 1 and all the elements in the batch dimensions \n of the indices_shape. Let us think of each such `r-b` ranked tensor as `indices_slice`. Each *scalar value* corresponding to `data[0:b-1,indices_slice]` \n is filled into the corresponding location of the `(q-b-1)`-dimensional tensor to form the `output` tensor (Example 1 below)\n\n3) If `indices_shape[-1] < r-b`, since the rank of `indices` is `q`, `indices` can be thought of as `N` `(q-b-1)`-dimensional tensor\n containing 1-D tensors of dimension `< r-b`. Let us think of each such tensors as `indices_slice`. Each *tensor slice* corresponding \n to `data[0:b-1, indices_slice , :]` is filled into the corresponding location of the `(q-b-1)`-dimensional tensor \n to form the `output` tensor (Examples 2, 3, 4 and 5 below)\n\nThis operator is the inverse of `ScatterND`.\n\n`Example 1`\n\n batch_dims = 0\n\n data = [[0,1],[2,3]] # data_shape = [2, 2]\n\n indices = [[0,0],[1,1]] # indices_shape = [2, 2]\n\n output = [0,3] # output_shape = [2]\n\n`Example 2`\n\n batch_dims = 0\n\n data = [[0,1],[2,3]] # data_shape = [2, 2]\n\n indices = [[1],[0]] # indices_shape = [2, 1]\n\n output = [[2,3],[0,1]] # output_shape = [2, 2]\n\n`Example 3`\n\n batch_dims = 0\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[0,1],[1,0]] # indices_shape = [2, 2]\n\n output = [[2,3],[4,5]] # output_shape = [2, 2] \n\n`Example 4`\n\n batch_dims = 0\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[[0,1]],[[1,0]]] # indices_shape = [2, 1, 2]\n\n output = [[[2,3]],[[4,5]]] # output_shape = [2, 1, 2] \n\n`Example 5`\n\n batch_dims = 1\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[1],[0]] # indices_shape = [2, 1]\n\n output = [[2,3],[4,5]] # output_shape = [2, 2] \n\n\n" +----f +input: "A" +input: "B" +input: "C" +output: "Y" +name: "Gemm" +op_type: "Gemm" +attribute { + name: "alpha" + f: 1.0 + type: FLOAT +} +attribute { + name: "beta" + f: 1.0 + type: FLOAT +} +attribute { + name: "transA" + i: 0 + type: INT +} +attribute { + name: "transB" + i: 0 + type: INT +} +attribute { + name: "A-types" + strings: "int32" + strings: "float16" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "int32" + strings: "float16" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "C-types" + strings: "int32" + strings: "float16" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "General Matrix multiplication:\nhttps://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms#Level_3\n\nA\' = transpose(A) if transA else A\n\nB\' = transpose(B) if transB else B\n\nCompute Y = alpha * A\' * B\' + beta * C, where input tensor A has shape (M, K) or (K, M),\ninput tensor B has shape (K, N) or (N, K), input tensor C is broadcastable to shape (M, N),\nand output tensor Y has shape (M, N). A will be transposed before doing the\ncomputation if attribute transA is non-zero, same for B and transB.\nThis operator supports **unidirectional broadcasting** (tensor C should be unidirectional broadcastable to tensor A * B); for more details please check [the doc](Broadcasting.md).\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "X" +output: "Y" +name: "GlobalAveragePool" +op_type: "GlobalAveragePool" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n GlobalAveragePool consumes an input tensor X and applies average pooling across\n the values in the same channel. This is equivalent to AveragePool with kernel size\n equal to the spatial dimension of input tensor." +----f +input: "X" +output: "Y" +name: "GlobalLpPool" +op_type: "GlobalLpPool" +attribute { + name: "p" + i: 2 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n GlobalLpPool consumes an input tensor X and applies lp pool pooling across\n the values in the same channel. This is equivalent to LpPool with kernel size\n equal to the spatial dimension of input tensor." +----f +input: "X" +output: "Y" +name: "GlobalMaxPool" +op_type: "GlobalMaxPool" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n GlobalMaxPool consumes an input tensor X and applies max pooling across\n the values in the same channel. This is equivalent to MaxPool with kernel size\n equal to the spatial dimension of input tensor." +----f +input: "Inputs" +output: "Outputs" +name: "Gradient" +op_type: "Gradient" +attribute { + name: "xs" + s: "" + type: STRINGS +} +attribute { + name: "y" + s: "" + type: STRING +} +attribute { + name: "zs" + s: "" + type: STRINGS +} +attribute { + name: "Inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGradient operator computes the partial derivatives of a specific tensor w.r.t.\nsome other tensors. This operator is widely used in gradient-based training\nalgorithms. To illustrate its use, let\'s consider a computation graph,\n\n```\nX -----.\n |\n v\nW --> Conv --> H --> Gemm --> Y\n ^\n |\n Z\n```\n\n, where W and Z are trainable tensors. Note that operators\' attributes are\nomitted for the sake of simplicity. Let dY/dW (dY/dZ) be the gradient of\nY with respect to W (Z). The user can compute gradient by inserting Gradient\noperator to form another graph shown below.\n\n```\nW --> Conv --> H --> Gemm --> Y\n| ^ ^\n| | |\n| X Z\n| | |\n| | .----------\'\n| | | (W/Z/X is the 1st/2nd/3rd input of Gradient as shown in\n| | | \"xs\" followed by \"zs\")\n| v v\n\'---> Gradient(xs=[\"W\", \"Z\"], zs=[\"X\"], y=\"Y\")\n | |\n | \'-----------------------------------> dY/dW (1st output of Gradient)\n |\n \'---------------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nBy definition, the tensor \"y\" is a function of independent variables in \"xs\"\nand \"zs\". Since we only compute the gradient of \"y\" w.r.t. the differentiable\nvariables in \"xs\", this Gradient only outputs dY/dW and dY/dZ. Note that \"H\"\ncannot appear in \"xs\" and \"zs\". The reason is that \"H\" can be determined by\ntensors \"W\" and \"X\" and therefore \"H\" is not an independent variable.\n\nAll outputs are optional. If needed, for example, user can assign an empty\nstring to the 1st output name of that Gradient to skip the generation of dY/dW.\nNote that the concept of optional outputs can also be found in ONNX\'s RNN, GRU,\nand LSTM.\n\nGradient operator can compute derivative against intermediate tensors. For\nexample, the gradient of Y with respect to H can be done via\n\n```\nW --> Conv --> H --> Gemm --> Y\n ^ | ^\n | | |\n X | Z\n .-------\' |\n | .----------\'\n | | (H/Z is the 1st/2nd input of Gradient as shown in \"xs\")\n v v\n Gradient(xs=[\"H\", \"Z\"], y=\"Y\")\n | |\n | \'-----------------------------------> dY/dH (1st output of Gradient)\n |\n \'---------------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nIt is possible to represent high-order differentiation using Gradient operators.\nFor example, given the following linear model:\n\n```\nW --> Gemm --> Y --> Loss --> O\n ^ ^\n | |\n X L\n```\n\nTo compute the 2nd order derivative of O with respect to W (denoted by\nd^2O/dW^2), one can do\n\n```\nW --> Gemm --> Y --> Loss --> O\n| ^ ^\n| | |\n| X .------------L\n| | | |\n| | | v\n+------+-+> Gradient(xs=[\"X\", \"W\"], zs=[\"L\"], y=\"O\") ---> dO/dX (1st output of Gradient)\n| | | |\n| | | \'---> dO/dW (2nd output of Gradient)\n| v v\n\'---> Gradient(xs=[\"X\", \"W\"], zs=[\"L\"], y=\"dO/dW\") ---> d(dO/dW)dX (1st output of\n | Gradient)\n |\n |\n \'---> d^2O/dW^2 (2nd output of Gradient)\n```\n\nThe tensors named in attributes \"xs\", \"zs\", and \"y\" define the differentiated\ncomputation graph, and the inputs to Gradient node define the values at\nwhich the gradient is computed. We can feed different tensors to the identified\ngraph. For example, one can compute the gradient of Y with respect to H at \na specific value of H, H_1, by providing that value as an input to the Gradient\nnode.\n\n```\nW --> Conv --> H --> Gemm --> Y\n ^ ^\n | |\n X Z\n\n Z_1 (2nd input of Gradient)\n |\n v\nH_1 --> Gradient(xs=[\"H\", \"Z\"], y=\"Y\") ---> dY/dH when H = H_1 and Y = Y_1.\n |\n \'------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nWhen the inputs of Gradient are the tensors named in \"xs\" and \"zs\", the\ncomputation can be optimized. More specifically, intermediate variables in\nforward pass can be reused if the gradient is computed via reverse-mode\nauto-differentiation.\n\n" +----f +input: "Inputs" +output: "Outputs" +name: "GraphCall" +op_type: "GraphCall" +attribute { + name: "graph_name" + s: "" + type: STRING +} +attribute { + name: "Inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nThe GraphCall operator invokes a graph inside TrainingInfoProto\'s\nalgorithm field. The GraphCall inputs and outputs are bound to those of\ninvoked graph by position. If a graph input has an initializer, that input\nis considered optional. All graph outputs are optional.\n\nBelow Python syntax is used for describing dictionary and list.\n\nAssume that ModelProto\'s graph field has\n- name: \"MyInferenceGraph\"\n- input: [\"X\", \"W\", \"Z\"]\n- initializer: [W]\n- output: [\"Y\"]\n\nas visualized below for inference.\n\n```\nX -----.\n |\n v\nW --> Conv --> H --> Gemm --> Y\n ^\n |\n Z\n```\n\nAssume that the training algorithm contains\n\n- inputs: [\"X_1\", \"Z_1\", \"C\"]\n- initializer: [T]\n- outputs: [\"W_new\"]\n\nwith a dictionary\n\n- update_binding: {\"W\": \"W_new\", \"T\": \"T_new\"}\n\nInside the training algorithm graph, one can invoke the inference\ngraph via adding a GraphCall node with\n\n- inputs: [\"X_1\", \"W\", Z_1\"]\n- outputs: [\"Y_1\"]\n- an attribute graph_name=\"MyInferenceGraph\",\n\nThe initializers, \"W\" and \"T\" in this case, in update_binding\nare considered globally-visible and mutable variables, which\ncan be used as inputs of operators in the training graph.\n\nAn example training algorithm graph may look like\n\n```\n.-------- W (a global and mutable variable from\n| | the inference graph)\n| |\n| .-----\'-----------.\n| | |\n| | v\n| | .-- X_1 --> GraphCall(graph_name=\"MyInferenceGraph\")\n| | | | |\n| | | | |\n| | | Z_1 -----\' |\n| | | | V\n| | | | Y_1 ---> Loss ---> O\n| | | | ^\n| | | | |\n| | `--. | C\n| | | | |\n| | | | .----------------\'\n| | | | |\n| | v v v\n| `--> Gradient(xs=[\"W\"], zs=[\"X_1\", \"Z_1\", \"C\"], y=\"O\")\n| |\n| v\n| dO_dW (gradient of W) 1 (a scalar one)\n| | |\n| V v\n| Div <--- T ------------> Add ---> T_new\n| | (T is the number of training iterations.\n| | T is also globally visible and mutable.)\n| v\n`-----> Sub ----> W_new\n```\n\nwhere Loss is a dummy node which computes the minimized objective function.\n\nThe variable \"W\" is an optional input in the called graph.\nIf the user omits it, the input list of GraphCall becomes [\"X_1\", \"\", \"Z_1\"].\nIn this case, from the view of computation graph, the Conv operator invoked by\nGraphCall\'s may be still connected the global \"W\" variable and therefore the\nstructure of the computation graph is unchanged.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Greater" +op_type: "Greater" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `greater` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "A" +input: "B" +output: "C" +name: "GreaterOrEqual" +op_type: "GreaterOrEqual" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `greater_equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Y" +name: "HardSigmoid" +op_type: "HardSigmoid" +attribute { + name: "alpha" + f: 0.2 + type: FLOAT +} +attribute { + name: "beta" + f: 0.5 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nHardSigmoid takes one input data (Tensor) and produces one output data\n(Tensor) where the HardSigmoid function, y = max(0, min(1, alpha * x + beta)),\nis applied to the tensor elementwise.\n" +----f +input: "input" +output: "output" +name: "Hardmax" +op_type: "Hardmax" +attribute { + name: "axis" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe operator computes the hardmax (1 for the first maximum value, and 0 for all others) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the hardmax values of the corresponding input.\n" +----f +input: "input" +output: "output" +name: "Identity" +op_type: "Identity" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "Identity operator" +----f +input: "cond" +output: "outputs" +name: "If" +op_type: "If" +attribute { + name: "else_branch" + s: "" + type: GRAPH +} +attribute { + name: "then_branch" + s: "" + type: GRAPH +} +attribute { + name: "cond-types" + strings: "bool" + type: STRINGS +} +doc_string: "If conditional" +----f +input: "X" +output: "Y" +name: "Imputer" +op_type: "Imputer" +attribute { + name: "imputed_value_floats" + s: "" + type: FLOATS +} +attribute { + name: "imputed_value_int64s" + s: "" + type: INTS +} +attribute { + name: "replaced_value_float" + f: 0.0 + type: FLOAT +} +attribute { + name: "replaced_value_int64" + i: 0 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Replaces inputs that equal one value with another, leaving all other elements alone.
\n This operator is typically used to replace missing values in situations where they have a canonical\n representation, such as -1, 0, NaN, or some extreme value.
\n One and only one of imputed_value_floats or imputed_value_int64s should be defined -- floats if the input tensor\n holds floats, integers if the input tensor holds integers. The imputed values must all fit within the\n width of the tensor element type. One and only one of the replaced_value_float or replaced_value_int64 should be defined,\n which one depends on whether floats or integers are being processed.
\n The imputed_value attribute length can be 1 element, or it can have one element per input feature.
In other words, if the input tensor has the shape [*,F], then the length of the attribute array may be 1 or F. If it is 1, then it is broadcast along the last dimension and applied to each feature.\n" +----f +input: "input" +input: "scale" +input: "B" +output: "output" +name: "InstanceNormalization" +op_type: "InstanceNormalization" +attribute { + name: "epsilon" + f: 1e-05 + type: FLOAT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "scale-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCarries out instance normalization as described in the paper\nhttps://arxiv.org/abs/1607.08022.\n\ny = scale * (x - mean) / sqrt(variance + epsilon) + B,\nwhere mean and variance are computed per instance per channel.\n\n" +----f +input: "X" +output: "Y" +name: "IsInf" +op_type: "IsInf" +attribute { + name: "detect_negative" + i: 1 + type: INT +} +attribute { + name: "detect_positive" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + type: STRINGS +} +doc_string: "Map infinity to true and other values to false." +----f +input: "X" +output: "Y" +name: "IsNaN" +op_type: "IsNaN" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "Returns which elements of the input are NaN." +----f +input: "X" +output: "Y" +name: "LRN" +op_type: "LRN" +attribute { + name: "alpha" + f: 0.0001 + type: FLOAT +} +attribute { + name: "beta" + f: 0.75 + type: FLOAT +} +attribute { + name: "bias" + f: 1.0 + type: FLOAT +} +attribute { + name: "size" + s: "" + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nLocal Response Normalization proposed in the [AlexNet paper](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf).\nIt normalizes over local input regions.\nThe local region is defined across the channels. For an element X[n, c, d1, ..., dk] in a tensor\nof shape (N x C x D1 x D2, ..., Dk), its region is\n{X[n, i, d1, ..., dk] | max(0, c - floor((size - 1) / 2)) <= i <= min(C - 1, c + ceil((size - 1) / 2))}.\n\nsquare_sum[n, c, d1, ..., dk] = sum(X[n, i, d1, ..., dk] ^ 2),\nwhere max(0, c - floor((size - 1) / 2)) <= i <= min(C - 1, c + ceil((size - 1) / 2)).\n\nY[n, c, d1, ..., dk] = X[n, c, d1, ..., dk] / (bias + alpha / size * square_sum[n, c, d1, ..., dk] ) ^ beta\n" +----f +input: "X" +input: "W" +input: "R" +input: "B" +input: "sequence_lens" +input: "initial_h" +input: "initial_c" +input: "P" +output: "Y" +output: "Y_h" +output: "Y_c" +name: "LSTM" +op_type: "LSTM" +attribute { + name: "activation_alpha" + s: "" + type: FLOATS +} +attribute { + name: "activation_beta" + s: "" + type: FLOATS +} +attribute { + name: "activations" + s: "" + type: STRINGS +} +attribute { + name: "clip" + s: "" + type: FLOAT +} +attribute { + name: "direction" + s: "forward" + type: STRING +} +attribute { + name: "hidden_size" + s: "" + type: INT +} +attribute { + name: "input_forget" + i: 0 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "sequence_lens-types" + strings: "int32" + type: STRINGS +} +attribute { + name: "initial_h-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "initial_c-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "P-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nComputes an one-layer LSTM. This operator is usually supported via some\ncustom implementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`i` - input gate\n\n`o` - output gate\n\n`f` - forget gate\n\n`c` - cell gate\n\n`t` - time step (t-1 means previous time step)\n\n`W[iofc]` - W parameter weight matrix for input, output, forget, and cell gates\n\n`R[iofc]` - R recurrence weight matrix for input, output, forget, and cell gates\n\n`Wb[iofc]` - W bias vectors for input, output, forget, and cell gates\n\n`Rb[iofc]` - R bias vectors for input, output, forget, and cell gates\n\n`P[iof]` - P peephole weight vector for input, output, and forget gates\n\n`WB[iofc]` - W parameter weight matrix for backward input, output, forget, and cell gates\n\n`RB[iofc]` - R recurrence weight matrix for backward input, output, forget, and cell gates\n\n`WBb[iofc]` - W bias vectors for backward input, output, forget, and cell gates\n\n`RBb[iofc]` - R bias vectors for backward input, output, forget, and cell gates\n\n`PB[iof]` - P peephole weight vector for backward input, output, and forget gates\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Sigmoid, g=Tanh, h=Tanh):\n\n - it = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Pi (.) Ct-1 + Wbi + Rbi)\n\n - ft = f(Xt*(Wf^T) + Ht-1*(Rf^T) + Pf (.) Ct-1 + Wbf + Rbf)\n\n - ct = g(Xt*(Wc^T) + Ht-1*(Rc^T) + Wbc + Rbc)\n\n - Ct = ft (.) Ct-1 + it (.) ct\n\n - ot = f(Xt*(Wo^T) + Ht-1*(Ro^T) + Po (.) Ct + Wbo + Rbo)\n\n - Ht = ot (.) h(Ct)\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "X" +output: "Y" +name: "LabelEncoder" +op_type: "LabelEncoder" +attribute { + name: "default_float" + f: -0.0 + type: FLOAT +} +attribute { + name: "default_int64" + i: -1 + type: INT +} +attribute { + name: "default_string" + s: "_Unused" + type: STRING +} +attribute { + name: "keys_floats" + s: "" + type: FLOATS +} +attribute { + name: "keys_int64s" + s: "" + type: INTS +} +attribute { + name: "keys_strings" + s: "" + type: STRINGS +} +attribute { + name: "values_floats" + s: "" + type: FLOATS +} +attribute { + name: "values_int64s" + s: "" + type: INTS +} +attribute { + name: "values_strings" + s: "" + type: STRINGS +} +attribute { + name: "X-types" + strings: "string" + strings: "float" + strings: "int64" + type: STRINGS +} +doc_string: "\n Maps each element in the input tensor to another value.
\n The mapping is determined by the two parallel attributes, \'keys_*\' and\n \'values_*\' attribute. The i-th value in the specified \'keys_*\' attribute\n would be mapped to the i-th value in the specified \'values_*\' attribute. It\n implies that input\'s element type and the element type of the specified\n \'keys_*\' should be identical while the output type is identical to the\n specified \'values_*\' attribute. If an input element can not be found in the\n specified \'keys_*\' attribute, the \'default_*\' that matches the specified\n \'values_*\' attribute may be used as its output value.
\n Let\'s consider an example which maps a string tensor to an integer tensor.\n Assume and \'keys_strings\' is [\"Amy\", \"Sally\"], \'values_int64s\' is [5, 6],\n and \'default_int64\' is \'-1\'. The input [\"Dori\", \"Amy\", \"Amy\", \"Sally\",\n \"Sally\"] would be mapped to [-1, 5, 5, 6, 6].
\n Since this operator is an one-to-one mapping, its input and output shapes\n are the same. Notice that only one of \'keys_*\'/\'values_*\' can be set.
\n For key look-up, bit-wise comparison is used so even a float NaN can be\n mapped to a value in \'values_*\' attribute.
\n" +----f +input: "X" +output: "Y" +name: "LeakyRelu" +op_type: "LeakyRelu" +attribute { + name: "alpha" + f: 0.01 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nLeakyRelu takes input data (Tensor) and an argument alpha, and produces one\noutput data (Tensor) where the function `f(x) = alpha * x for x < 0`,\n`f(x) = x for x >= 0`, is applied to the data tensor elementwise.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Less" +op_type: "Less" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `less` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "A" +input: "B" +output: "C" +name: "LessOrEqual" +op_type: "LessOrEqual" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `less_equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Y" +output: "Z" +name: "LinearClassifier" +op_type: "LinearClassifier" +attribute { + name: "classlabels_ints" + s: "" + type: INTS +} +attribute { + name: "classlabels_strings" + s: "" + type: STRINGS +} +attribute { + name: "coefficients" + s: "" + type: FLOATS +} +attribute { + name: "intercepts" + s: "" + type: FLOATS +} +attribute { + name: "multi_class" + i: 0 + type: INT +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Linear classifier\n" +----f +input: "X" +output: "Y" +name: "LinearRegressor" +op_type: "LinearRegressor" +attribute { + name: "coefficients" + s: "" + type: FLOATS +} +attribute { + name: "intercepts" + s: "" + type: FLOATS +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "targets" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Generalized linear regression evaluation.
\n If targets is set to 1 (default) then univariate regression is performed.
\n If targets is set to M then M sets of coefficients must be passed in as a sequence\n and M results will be output for each input n in N.
\n The coefficients array is of length n, and the coefficients for each target are contiguous.\n Intercepts are optional but if provided must match the number of targets.\n" +----f +input: "input" +output: "output" +name: "Log" +op_type: "Log" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the natural log of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "LogSoftmax" +op_type: "LogSoftmax" +attribute { + name: "axis" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe operator computes the logsoftmax (log of softmax) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the logsoftmax values of the corresponding input.\n" +----f +input: "M" +input: "cond" +input: "v_initial" +output: "v_final_and_scan_outputs" +name: "Loop" +op_type: "Loop" +attribute { + name: "body" + s: "" + type: GRAPH +} +attribute { + name: "M-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "cond-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "v_initial-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGeneric Looping construct. This loop has multiple termination conditions:\n\n1) Trip count. Iteration count specified at runtime. Set by\n specifying the input M. Optional. Set to empty string to omit.\n Note that a static trip count (specified at graph construction time) can be\n specified by passing in a constant node for input M.\n2) Loop termination condition. This is an input to the op that determines\n whether to run the first iteration and also a loop-carried dependency for\n the body graph. The body graph must yield a value for the condition variable,\n whether this input is provided or not.\n\nThis table summarizes the operating modes of this operator with equivalent\nC-style code:\n\n Operator inputs defined as (max_trip_count, condition_var).\n\n input (\"\", \"\"):\n for (int i=0; ; ++i) {\n cond = ... // Note this value is ignored, but is required in the body\n }\n\n input (\"\", cond) // Note this is analogous to a while loop\n bool cond = ...;\n for (int i=0; cond; ++i) {\n cond = ...;\n }\n\n input (\"\", 1) // Note this is analogous to a do-while loop\n bool cond = true\n for (int i=0; cond; ++i) {\n cond = ...;\n }\n\n input (trip_count, \"\") // Note this is analogous to a for loop\n int trip_count = ...\n for (int i=0; i < trip_count; ++i) {\n cond = ...; // ignored\n }\n\n input (trip_count, cond)\n int trip_count = ...;\n bool cond = ...;\n for (int i=0; i < trip_count && cond; ++i) {\n cond = ...;\n }\n\n\n*Sample usage - cond as well as trip count*\n\n graph predict-net {\n %a = Constant[value = ]()\n %b = Constant[value = ]()\n %keepgoing = Constant[value = ]()\n %max_trip_count = Constant[value = ]()\n %keepgoing_out, %b_out, %user_defined_vals = Loop[body = ](%max_trip_count, %keepgoing, %b)\n return\n }\n\n graph body-net (\n %i[INT32, scalar] // iteration number\n %keepgoing_in[BOOL, scalar] // incoming loop-termination-condition; not used\n %b_in[INT32, scalar] // incoming value of loop-carried-dependency b\n ) {\n %my_local = Add(%a, %b_in)\n %b_out = Sub(%a, %b_in) // outgoing value of loop-carried-dependency b\n %keepgoing_out = Greater(%my_local, %b_out) // outgoing loop-termination-condition\n %user_defined_val = Add(%b_in, %b_in) // scan-output value to be accumulated\n return %keepgoing_out, %b_out, %user_defined_val\n }\n\n*Sample equivalent C code*\n\n {\n /* User-defined code (enclosing scope) */\n int a = 3, b = 6;\n bool keepgoing = true; // Analogous to input cond\n /* End user-defined code */\n\n /* Implicitly-defined code */\n const int max_trip_count = 10; // Analogous to input M\n int user_defined_vals[]; // Imagine this is resizable\n /* End implicitly-defined code */\n /* initialize loop-carried variables and scan-output variables */\n bool keepgoing_out = keepgoing\n int b_out = b\n\n for (int i=0; i < max_trip_count && keepgoing_out; ++i) {\n /* Implicitly-defined code: bind actual parameter values\n to formal parameter variables of loop-body */\n bool keepgoing_in = keepgoing_out; \n bool b_in = b_out;\n\n /* User-defined code (loop body) */\n int my_local = a + b_in; // Reading value \"a\" from the enclosing scope is fine\n b_out = a - b_in;\n keepgoing_out = my_local > b_out; \n user_defined_val = b_in + b_in; // b_in and b_out are different variables\n /* End user-defined code */\n\n /* Implicitly defined-code */\n user_defined_vals[i] = user_defined_val // accumulate scan-output values\n }\n // int t = my_local; // Can\'t do this. my_local is not accessible here.\n\n // The values below are bound to the output variables of the loop and therefore accessible\n // b_out; user_defined_vals; keepgoing_out;\n }\n\nThere are several things of note in this code snippet:\n\n1) Values from the enclosing scope (i.e. variable \"a\" here) are in scope and can\n be referenced in the inputs of the loop.\n2) Any values computed in the loop body that needs to be used in a subsequent\n iteration or after the loop are modelled using a pair of variables in the loop-body,\n consisting of an input variable (eg., b_in) and an output variable (eg., b_out).\n These are referred to as loop-carried dependences. The loop operation node\n supplies the input value of the input variable for the first iteration, and\n returns the output value of the output variable produced by the final\n iteration.\n3) Scan_output variables are used to implicitly concatenate values computed across\n all the iterations. In the above example, the value of user_defined_val computed\n over all iterations are concatenated and returned as the value of user_defined_vals\n after the loop.\n4) Values created in the body cannot be accessed in the enclosing scope,\n except using the mechanism described above.\n\nNote that the semantics of this op support \"diagonal\" or \"wavefront\" execution.\n(See Step 3 here for an example:\nhttps://devblogs.nvidia.com/optimizing-recurrent-neural-networks-cudnn-5/).\nFrontends should emit multi-layer RNNs as a series of While operators (with\ntime being the inner looping dimension), with each successive layer consuming\nthe scan_outputs from the previous layer, possibly going through several\npoint-wise operators (e.g. dropout, residual connections, linear layer).\n" +----f +input: "input" +output: "output" +name: "LpNormalization" +op_type: "LpNormalization" +attribute { + name: "axis" + i: -1 + type: INT +} +attribute { + name: "p" + i: 2 + type: INT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nGiven a matrix, apply Lp-normalization along the provided axis.\n" +----f +input: "X" +output: "Y" +name: "LpPool" +op_type: "LpPool" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "p" + i: 2 + type: INT +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n LpPool consumes an input tensor X and applies Lp pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n Lp pooling consisting of computing the Lp norm on all values of a subset\n of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing." +----f +input: "A" +input: "B" +output: "Y" +name: "MatMul" +op_type: "MatMul" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html\n" +----f +input: "A" +input: "B" +input: "a_zero_point" +input: "b_zero_point" +output: "Y" +name: "MatMulInteger" +op_type: "MatMulInteger" +attribute { + name: "A-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "B-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "a_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "b_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html.\nThe production MUST never overflow. The accumulation may overflow if and only if in 32 bits.\n" +----f +input: "data_0" +output: "max" +name: "Max" +op_type: "Max" +attribute { + name: "data_0-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nElement-wise max of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Y" +output: "Indices" +name: "MaxPool" +op_type: "MaxPool" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "ceil_mode" + i: 0 + type: INT +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "storage_order" + i: 0 + type: INT +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "int8" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "float" + type: STRINGS +} +doc_string: "\n MaxPool consumes an input tensor X and applies max pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n max pooling consisting of computing the max on all values of a\n subset of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing. The output spatial shape will be following:\n ```\n output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1)) / strides_spatial_shape[i] + 1)\n ```\n or\n ```\n output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1)) / strides_spatial_shape[i] + 1)\n ```\n if ceil_mode is enabled\n\n ```\n * pad_shape[i] is sum of pads along axis i\n ```\n\n `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following:\n ```\n VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1) + 1) / strides_spatial_shape[i])\n SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i])\n ```\n And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`:\n ```\n pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + ((kernel_spatial_shape[i] - 1) * dilations[i] + 1) - input_spatial_shape[i]\n ```\n The output of each pooling window is maximum number of elements exclude pad. \n " +----f +input: "X" +input: "rois" +output: "Y" +name: "MaxRoiPool" +op_type: "MaxRoiPool" +attribute { + name: "pooled_shape" + s: "" + type: INTS +} +attribute { + name: "spatial_scale" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "rois-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n ROI max pool consumes an input tensor X and region of interests (RoIs) to\n apply max pooling across each RoI, to produce output 4-D tensor of shape\n (num_rois, channels, pooled_shape[0], pooled_shape[1])." +----f +input: "X" +input: "I" +input: "output_shape" +output: "output" +name: "MaxUnpool" +op_type: "MaxUnpool" +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "I-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "output_shape-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nMaxUnpool essentially computes the partial inverse of the MaxPool op.\n The input information to this op is typically the the output information from a MaxPool op. The first\n input tensor X is the tensor that needs to be unpooled, which is typically the pooled tensor (first output)\n from MaxPool. The second input tensor, I, contains the indices to the (locally maximal) elements corrsponding\n to the elements in the first input tensor X. Input tensor I is typically the second output of the MaxPool op.\n The third (optional) input is a tensor that specifies the output size of the unpooling operation.\n\nMaxUnpool is intended to do \'partial\' inverse of the MaxPool op. \'Partial\' because all the non-maximal\n values from the original input to MaxPool are set to zero in the output of the MaxUnpool op. Pooling\n the result of an unpooling operation should give back the original input to the unpooling op.\n\nMaxUnpool can produce the same output size for several input sizes, which makes unpooling op ambiguous.\n The third input argument, output_size, is meant to disambiguate the op and produce output tensor of\n known/predictable size.\n\nIn addition to the inputs, MaxUnpool takes three attributes, namely kernel_shape, strides, and pads,\n which define the exact unpooling op. The attributes typically have the same values as the corrsponding\n pooling op that the unpooling op is trying to invert.\n" +----f +input: "data_0" +output: "mean" +name: "Mean" +op_type: "Mean" +attribute { + name: "data_0-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nElement-wise mean of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Y" +name: "MeanVarianceNormalization" +op_type: "MeanVarianceNormalization" +attribute { + name: "axes" + ints: 0 + ints: 2 + ints: 3 + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n A MeanVarianceNormalization Function: Perform mean variance normalization\n on the input tensor X using formula:
``` (X-EX)/sqrt(E(X-EX)^2) ```\n" +----f +input: "data_0" +output: "min" +name: "Min" +op_type: "Min" +attribute { + name: "data_0-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nElement-wise min of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "A" +input: "B" +output: "C" +name: "Mod" +op_type: "Mod" +attribute { + name: "fmod" + i: 0 + type: INT +} +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\n Performs element-wise binary modulus (with Numpy-style broadcasting support). \n The sign of the remainder is the same as that of the Divisor.\n \n Mod operator can also behave like C fmod() or numpy.fmod. In this case, the sign of the remainder however, will be the same as the Dividend \n (in contrast to integer mod). To force a behavior like numpy.fmod() an \'fmod\' Attribute is provided.\n This attribute is set to 0 by default causing the behavior to be like integer mod. \n Setting this attribute to 1 causes the remainder to be calculated similar to that of numpy.fmod().\n\n If the input type is floating point, then `fmod` attribute must be set to 1.\n \n In case of dividend being zero, the results will be platform dependent.\n\n This operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "R" +input: "T" +input: "inputs" +output: "outputs" +name: "Momentum" +op_type: "Momentum" +attribute { + name: "alpha" + s: "" + type: FLOAT +} +attribute { + name: "beta" + s: "" + type: FLOAT +} +attribute { + name: "mode" + s: "" + type: STRING +} +attribute { + name: "norm_coefficient" + s: "" + type: FLOAT +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + type: STRINGS +} +attribute { + name: "T-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "inputs-types" + strings: "float" + strings: "double" + type: STRINGS +} +doc_string: "\n Compute one iteration of stochastic gradient update with momentum.\n This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. As you can imagine, SG with momentum requires\n several parameters:\n \n - The learning-rate \"R\".\n - The update count \"T\". That is, the number of conducted training iterations. It should\n be zero in the first training iteration.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A decay coefficient of previous accumulated gradient (i.e., momentum) \"alpha\".\n - The scaling coefficient of current gradient \"beta\".\n - An attribute to choose either standard momentum or Nesterov\'s momentum \"mode\" should\n be used.\n\n For the sake of simplicity, assume that there is only one tensor (called \"X\") to be optimized.\n Other necessary inputs are \"X\"\'s gradient (called \"G\") and \"X\"\'s momentum (called \"V\"). This\n Momentum operator maps all these inputs to the new value of \"X\" (called \"X_new\") and its new\n momentum (called \"V_new\").\n \n This operator supports two different momentum algorithms. Set the attribute \"mode\" to\n \"nesterov\" if Nesterov\'s momentum is desired. Otherwise, set the attribute \"model\" to\n \"standard\" to use standard momentum. Computation details are described subsequently.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise operations with numpy-style broadcasting.\n\n Pseudo code for SG with standard momentum:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||^2, where ||X|| is the sum of squared\n // values of all elements in X.\n G_regularized = norm_coefficient * X + G\n\n // In the first training iteration, beta should always be 1.\n beta_adjusted = T > 0 ? beta : 1\n\n // Compute the current momentum based on previous momentum and the current gradient.\n V_new = alpha * V + beta_adjusted * G_regularized\n\n // Update X.\n X_new = X - R * V_new\n\n Pseudo code for SG with Nesterov\'s momentum:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||^2, where ||X|| is the sum of squared\n // values of all elements in X.\n G_regularized = norm_coefficient * X + G;\n\n // In the first training iteration, beta should always be 1.\n beta_adjusted = T > 0 ? beta : 1\n\n // Compute the current momentum based on previous momentum and the current gradient.\n V_new = alpha * V + beta_adjusted * G_regularized;\n\n // Compute final update direction and then update X.\n X_new = X - R * (G_regularized + alpha * V_new)\n\n If one assign this operators to optimize multiple inputs, for example, \"X_1\" and \"X_2\". The same\n pseudo code would be extended to handle all tensors jointly. More specifically, we can view \"X\" as a\n concatenation of \"X_1\" and \"X_2\" (of course, their gradient and accumulate gradient should\n be concatenated too) and then our pseudo code becomes applicable.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Mul" +op_type: "Mul" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPerforms element-wise binary multiplication (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "input" +output: "output" +name: "Multinomial" +op_type: "Multinomial" +attribute { + name: "dtype" + i: 6 + type: INT +} +attribute { + name: "sample_size" + i: 1 + type: INT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nGenerate a tensor of samples from a multinomial distribution according to the probabilities\nof each of the possible outcomes.\n" +----f +input: "X" +output: "Y" +name: "Neg" +op_type: "Neg" +attribute { + name: "X-types" + strings: "int8" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + strings: "int16" + type: STRINGS +} +doc_string: "\nNeg takes one input data (Tensor) and produces one output data\n(Tensor) where each element flipped sign, y = -x, is applied to\nthe tensor elementwise.\n" +----f +input: "input" +input: "target" +input: "weight" +output: "loss" +name: "NegativeLogLikelihoodLoss" +op_type: "NegativeLogLikelihoodLoss" +attribute { + name: "ignore_index" + s: "" + type: INT +} +attribute { + name: "reduction" + s: "mean" + type: STRING +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "target-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "weight-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nA NegativeLogLikelihoodLoss operator computes (weighted) negative log likelihood loss.\nIts \"input\" tensor has the shape of (N, C, d1, d2, ..., dk) where k >= 0.\nThe \"input\" tensor contains log-probabilities for input[n, :, d_1, d_2,..., d_k] being in a class of [0, C).\nThe operator\'s \"target\" input tensor has the shape of (N, d1, d2, ..., dk). It encodes class labels (one of C classes)\nor it may contain a special value (indicated by an attribute ignore_index) for N x d1 x d2 x ... x dk samples.\nThe loss value for input[n, :, d_1, d_2,...d_k] being classified as class c = target[n][d_1][d_2]...[d_k] is computed as:\n\n loss[n][d_1][d_2]...[d_k] = -input[n][c][d_1][d_2]...[d_k].\n\nWhen an optional \"weight\" is provided, the sample loss is calculated as:\n\n loss[n][d_1][d_2]...[d_k] = -input[n][c][d_1][d_2]...[d_k] * weight[c].\n\nloss is zero for the case when target-value equals ignore_index.\n \n loss[n][d_1][d_2]...[d_k] = 0, when target[n][d_1][d_2]...[d_k] = ignore_index\n\nIf \"reduction\" attribute is set to \"none\", the operator\'s output will be the above loss with shape (N, d1, d2, ..., dk).\nIf \"reduction\" attribute is set to \"mean\" (the default attribute value), the output loss is (weight) averaged:\n\n mean(loss), if \"weight\" is not provided,\n\nor if weight is provided,\n\n sum(loss) / sum(weight[target[n][d_1][d_2]...[d_k]]]), for all samples.\n\nIf \"reduction\" attribute is set to \"sum\", the output is a scalar:\n sum(loss).\n\nSee also https://pytorch.org/docs/stable/nn.html#torch.nn.NLLLoss.\n\nExample 1:\n\n // negative log likelihood loss, \"none\" reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n\n loss = np.zeros((N, d1))\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1]\n\n // print(loss)\n // [[-3. -2.]\n // [-0. -2.]]\n\nExample 2:\n\n // weighted negative log likelihood loss, sum reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n weight = [0.2, 0.3, 0.1]\n loss = np.zeros((N, d1))\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1] * weight[c]\n\n loss = np.sum(loss)\n // print(loss)\n // -1.1\n\nExample 3:\n\n // weighted negative log likelihood loss, mean reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n weight = [0.2, 0.3, 0.1]\n loss = np.zeros((N, d1))\n weight_total = 0\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1] * weight[c]\n weight_total = weight_total + weight[c]\n\n loss = np.sum(loss) / weight_total\n // print(loss)\n // -1.57\n" +----f +input: "boxes" +input: "scores" +input: "max_output_boxes_per_class" +input: "iou_threshold" +input: "score_threshold" +output: "selected_indices" +name: "NonMaxSuppression" +op_type: "NonMaxSuppression" +attribute { + name: "center_point_box" + i: 0 + type: INT +} +attribute { + name: "boxes-types" + strings: "float" + type: STRINGS +} +attribute { + name: "scores-types" + strings: "float" + type: STRINGS +} +attribute { + name: "max_output_boxes_per_class-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "iou_threshold-types" + strings: "float" + type: STRINGS +} +attribute { + name: "score_threshold-types" + strings: "float" + type: STRINGS +} +doc_string: "\nFilter out boxes that have high intersection-over-union (IOU) overlap with previously selected boxes.\nBounding boxes with score less than score_threshold are removed. Bounding box format is indicated by attribute center_point_box.\nNote that this algorithm is agnostic to where the origin is in the coordinate system and more generally is invariant to\northogonal transformations and translations of the coordinate system; thus translating or reflections of the coordinate system\nresult in the same boxes being selected by the algorithm.\nThe selected_indices output is a set of integers indexing into the input collection of bounding boxes representing the selected boxes.\nThe bounding box coordinates corresponding to the selected indices can then be obtained using the Gather or GatherND operation.\n" +----f +input: "X" +output: "Y" +name: "NonZero" +op_type: "NonZero" +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\n Returns the indices of the elements that are non-zero\n (in row-major order - by dimension).\n NonZero behaves similar to numpy.nonzero:\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.nonzero.html\n" +----f +input: "X" +output: "Y" +name: "Normalizer" +op_type: "Normalizer" +attribute { + name: "norm" + s: "MAX" + type: STRING +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Normalize the input. There are three normalization modes, which have the corresponding formulas,\n defined using element-wise infix operators \'/\' and \'^\' and tensor-wide functions \'max\' and \'sum\':
\n
\n Max: Y = X / max(X)
\n L1: Y = X / sum(X)
\n L2: Y = sqrt(X^2 / sum(X^2)}
\n In all modes, if the divisor is zero, Y == X.\n
\n For batches, that is, [N,C] tensors, normalization is done along the C axis. In other words, each row\n of the batch is normalized independently.\n" +----f +input: "X" +output: "Y" +name: "Not" +op_type: "Not" +attribute { + name: "X-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nReturns the negation of the input tensor element-wise.\n" +----f +input: "indices" +input: "depth" +input: "values" +output: "output" +name: "OneHot" +op_type: "OneHot" +attribute { + name: "axis" + i: -1 + type: INT +} +attribute { + name: "indices-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "depth-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "values-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\n Produces a one-hot tensor based on inputs.\n The locations represented by the index values in the \'indices\' input tensor will have \'on_value\'\n and the other locations will have \'off_value\' in the output tensor, where \'on_value\' and \'off_value\'\n are specified as part of required input argument \'values\', which is a two-element tensor of format\n [off_value, on_value]. The rank of the output tensor will be one greater than the rank of the\n input tensor. The additional dimension is for one-hot representation. The additional dimension will\n be inserted at the position specified by \'axis\'. If \'axis\' is not specified then then additional\n dimension will be inserted as the innermost dimension, i.e. axis=-1. The size of the additional\n dimension is specified by required scalar input \'depth\'. The type of the output tensor is the same\n as the type of the \'values\' input. Any entries in the \'indices\' input tensor with values outside\n the range [-depth, depth-1] will result in one-hot representation with all \'off_value\' values in the\n output tensor.\n\n when axis = 0:\n output[input[i, j, k], i, j, k] = 1 for all i, j, k and 0 otherwise.\n\n when axis = -1:\n output[i, j, k, input[i, j, k]] = 1 for all i, j, k and 0 otherwise.\n\n" +----f +input: "X" +output: "Y" +name: "OneHotEncoder" +op_type: "OneHotEncoder" +attribute { + name: "cats_int64s" + s: "" + type: INTS +} +attribute { + name: "cats_strings" + s: "" + type: STRINGS +} +attribute { + name: "zeros" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "int32" + strings: "string" + strings: "double" + strings: "int64" + strings: "float" + type: STRINGS +} +doc_string: "\n Replace each input element with an array of ones and zeros, where a single\n one is placed at the index of the category that was passed in. The total category count \n will determine the size of the extra dimension of the output array Y.
\n For example, if we pass a tensor with a single value of 4, and a category count of 8, \n the output will be a tensor with ``[0,0,0,0,1,0,0,0]``.
\n This operator assumes every input feature is from the same set of categories.
\n If the input is a tensor of float, int32, or double, the data will be cast\n to integers and the cats_int64s category list will be used for the lookups.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Or" +op_type: "Or" +attribute { + name: "A-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "B-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `or` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +input: "slope" +output: "Y" +name: "PRelu" +op_type: "PRelu" +attribute { + name: "X-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "slope-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPRelu takes input data (Tensor) and slope tensor as input, and produces one\noutput data (Tensor) where the function `f(x) = slope * x for x < 0`,\n`f(x) = x for x >= 0`., is applied to the data tensor elementwise.\nThis operator supports **unidirectional broadcasting** (tensor slope should be unidirectional broadcastable to input tensor X); for more details please check [the doc](Broadcasting.md)." +----f +input: "data" +input: "pads" +input: "constant_value" +output: "output" +name: "Pad" +op_type: "Pad" +attribute { + name: "mode" + s: "constant" + type: STRING +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "pads-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "constant_value-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGiven a tensor containing the data to be padded (`data`), a tensor containing the number of start and end pad values for axis (`pads`), (optionally) a `mode`, and (optionally) `constant_value`, \na padded tensor (`output`) is generated.\n\nThe three supported `modes` are (similar to corresponding modes supported by `numpy.pad`):\n\n1) `constant`(default) - pads with a given constant value as specified by `constant_value` (which defaults to 0)\n\n2) `reflect` - pads with the reflection of the vector mirrored on the first and last values of the vector along each axis\n\n3) `edge` - pads with the edge values of array\n\n\nExample 1 (`constant` mode):\n Insert 0 pads to the beginning of the second dimension.\n\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'constant\'\n\n constant_value = 0.0\n\n output = \n [\n [\n [0.0, 0.0, 1.0, 1.2],\n [0.0, 0.0, 2.3, 3.4],\n [0.0, 0.0, 4.5, 5.7],\n ],\n ]\n\n\nExample 2 (`reflect` mode):\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'reflect\'\n\n output = \n [\n [\n [1.0, 1.2, 1.0, 1.2],\n [2.3, 3.4, 2.3, 3.4],\n [4.5, 5.7, 4.5, 5.7],\n ],\n ]\n\n\nExample 3 (`edge` mode):\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'edge\'\n\n output = \n [\n [\n [1.0, 1.0, 1.0, 1.2],\n [2.3, 2.3, 2.3, 3.4],\n [4.5, 4.5, 4.5, 5.7],\n ],\n ]\n\n" +----f +input: "X" +input: "Y" +output: "Z" +name: "Pow" +op_type: "Pow" +attribute { + name: "X-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + type: STRINGS +} +attribute { + name: "Y-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPow takes input data (Tensor) and exponent Tensor, and\nproduces one output data (Tensor) where the function `f(x) = x^exponent`,\nis applied to the data tensor elementwise.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md)." +----f +input: "x" +input: "x_scale" +input: "x_zero_point" +input: "w" +input: "w_scale" +input: "w_zero_point" +input: "y_scale" +input: "y_zero_point" +input: "B" +output: "y" +name: "QLinearConv" +op_type: "QLinearConv" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "group" + i: 1 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "x-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "x_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "x_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "w-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "w_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "w_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "y_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "y_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "B-types" + strings: "int32" + type: STRINGS +} +doc_string: "\nThe convolution operator consumes a quantized input tensor, its scale and zero point,\na quantized filter, its scale and zero point, and output\'s scale and zero point,\nand computes the quantized output. Each scale and zero-point pair must have same shape.\nIt means they must be either scalars (per tensor) or 1-D tensors (per output channel).\nEach input or output and its related zero point must have same type.\nWhen bias is present it must be quantized using scale = input scale * weight scale and \nzero point as 0.\n" +----f +input: "a" +input: "a_scale" +input: "a_zero_point" +input: "b" +input: "b_scale" +input: "b_zero_point" +input: "y_scale" +input: "y_zero_point" +output: "y" +name: "QLinearMatMul" +op_type: "QLinearMatMul" +attribute { + name: "a-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "a_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "a_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "b-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "b_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "b_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "y_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "y_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html.\nIt consumes two quantized input tensors, their scales and zero points, scale and zero point of output, and computes the quantized output.\nThe quantization formula is y = saturate((x / y_scale) + y_zero_point). For (x / y_scale), it is rounding to nearest ties to even.\nRefer to https://en.wikipedia.org/wiki/Rounding for details. Scale and zero point must have same shape.\nThey must be either scalar (per tensor) or 1-D tensor (per row for \'a\' and per column for \'b\'). If scale and zero point are 1-D tensor,\nthe number of elements of scale and zero point tensor of input \'a\' and output \'y\' should be equal to the number of rows of input \'a\',\nand the number of elements of scale and zero point tensor of input \'b\' should be equal to the number of columns of input \'b\'.\nProduction must never overflow, and accumulation may overflow if and only if in 32 bits.\n" +----f +input: "x" +input: "y_scale" +input: "y_zero_point" +output: "y" +name: "QuantizeLinear" +op_type: "QuantizeLinear" +attribute { + name: "x-types" + strings: "float" + strings: "int32" + type: STRINGS +} +attribute { + name: "y_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "y_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nThe linear per-tensor/layer quantization operator. It consumes a high precision tensor, a scale, a zero point to compute the low precision / quantized tensor.\nThe quantization formula is y = saturate ((x / y_scale) + y_zero_point). For saturation, it saturates to [0, 255] if it\'s uint8, or [-128, 127] if it\'s int8.\nFor (x / y_scale), it\'s rounding to nearest ties to even. Refer to https://en.wikipedia.org/wiki/Rounding for details. \'y_zero_point\' and \'y\' must have same type.\n" +----f +input: "X" +input: "W" +input: "R" +input: "B" +input: "sequence_lens" +input: "initial_h" +output: "Y" +output: "Y_h" +name: "RNN" +op_type: "RNN" +attribute { + name: "activation_alpha" + s: "" + type: FLOATS +} +attribute { + name: "activation_beta" + s: "" + type: FLOATS +} +attribute { + name: "activations" + strings: "Tanh" + strings: "Tanh" + type: STRINGS +} +attribute { + name: "clip" + s: "" + type: FLOAT +} +attribute { + name: "direction" + s: "forward" + type: STRING +} +attribute { + name: "hidden_size" + s: "" + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "sequence_lens-types" + strings: "int32" + type: STRINGS +} +attribute { + name: "initial_h-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nComputes an one-layer simple RNN. This operator is usually supported\nvia some custom implementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`i` - input gate\n\n`t` - time step (t-1 means previous time step)\n\n`Wi` - W parameter weight matrix for input gate\n\n`Ri` - R recurrence weight matrix for input gate\n\n`Wbi` - W parameter bias vector for input gate\n\n`Rbi` - R parameter bias vector for input gate\n\n`WBi` - W parameter weight matrix for backward input gate\n\n`RBi` - R recurrence weight matrix for backward input gate\n\n`WBbi` - WR bias vectors for backward input gate\n\n`RBbi` - RR bias vectors for backward input gate\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Tanh):\n\n - Ht = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +output: "output" +name: "RandomNormal" +op_type: "RandomNormal" +attribute { + name: "dtype" + i: 1 + type: INT +} +attribute { + name: "mean" + f: 0.0 + type: FLOAT +} +attribute { + name: "scale" + f: 1.0 + type: FLOAT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "shape" + s: "" + type: INTS +} +doc_string: "\nGenerate a tensor with random values drawn from a normal distribution. The shape\nof the tensor is specified by the `shape` argument and the parameter of the normal distribution\nspecified by `mean` and `scale`.\n\nThe data type is specified by the \'dtype\' argument. The \'dtype\' argument must\nbe one of the data types specified in the \'DataType\' enum field in the\nTensorProto message.\n" +----f +input: "input" +output: "output" +name: "RandomNormalLike" +op_type: "RandomNormalLike" +attribute { + name: "dtype" + s: "" + type: INT +} +attribute { + name: "mean" + f: 0.0 + type: FLOAT +} +attribute { + name: "scale" + f: 1.0 + type: FLOAT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGenerate a tensor with random values drawn from a normal distribution.\nThe shape of the output tensor is copied from the shape of the input tensor,\nand the parameters of the normal distribution are specified by `mean` and `scale`.\n\nThe data type is specified by the \'dtype\' argument, or copied from the input tensor if not provided.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message, and be valid as an output type.\n" +----f +output: "output" +name: "RandomUniform" +op_type: "RandomUniform" +attribute { + name: "dtype" + i: 1 + type: INT +} +attribute { + name: "high" + f: 1.0 + type: FLOAT +} +attribute { + name: "low" + f: 0.0 + type: FLOAT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "shape" + s: "" + type: INTS +} +doc_string: "\nGenerate a tensor with random values drawn from a uniform distribution. The shape\nof the tensor is specified by the `shape` argument and the range by `low` and `high`.\n\nThe data type is specified by the \'dtype\' argument. The \'dtype\' argument must\nbe one of the data types specified in the \'DataType\' enum field in the\nTensorProto message.\n" +----f +input: "input" +output: "output" +name: "RandomUniformLike" +op_type: "RandomUniformLike" +attribute { + name: "dtype" + s: "" + type: INT +} +attribute { + name: "high" + f: 1.0 + type: FLOAT +} +attribute { + name: "low" + f: 0.0 + type: FLOAT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGenerate a tensor with random values drawn from a uniform distribution.\nThe shape of the output tensor is copied from the shape of the input tensor,\nand the parameters of the uniform distribution are specified by `low` and `high`.\n\nThe data type is specified by the \'dtype\' argument, or copied from the input tensor if not provided.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message and be valid as an output type.\n" +----f +input: "start" +input: "limit" +input: "delta" +output: "output" +name: "Range" +op_type: "Range" +attribute { + name: "start-types" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + strings: "int16" + type: STRINGS +} +attribute { + name: "limit-types" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + strings: "int16" + type: STRINGS +} +attribute { + name: "delta-types" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + strings: "int16" + type: STRINGS +} +doc_string: "\nGenerate a tensor containing a sequence of numbers that begin at `start` and extends by increments of `delta`\nup to `limit` (exclusive).\n\nThe number of elements in the output of range is computed as below-\n\n`number_of_elements = max( ceil( (limit - start) / delta ) , 0 )`\n\nThe pseudocode determining the contents of the output is shown below-\n\n`for(int i=0; i) and produces one output data\n(Tensor) where the reciprocal is, y = 1/x, is applied to\nthe tensor elementwise.\n" +----f +input: "data" +output: "reduced" +name: "ReduceL1" +op_type: "ReduceL1" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the L1 norm of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceL2" +op_type: "ReduceL2" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the L2 norm of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceLogSum" +op_type: "ReduceLogSum" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the log sum of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceLogSumExp" +op_type: "ReduceLogSumExp" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the log sum exponent of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceMax" +op_type: "ReduceMax" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "int8" + strings: "float16" + strings: "int32" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the max of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceMean" +op_type: "ReduceMean" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the mean of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceMin" +op_type: "ReduceMin" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "int8" + strings: "float16" + strings: "int32" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the min of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceProd" +op_type: "ReduceProd" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the product of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceSum" +op_type: "ReduceSum" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the sum of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceSumSquare" +op_type: "ReduceSumSquare" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the sum square of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "X" +output: "Y" +name: "Relu" +op_type: "Relu" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nRelu takes one input data (Tensor) and produces one output data\n(Tensor) where the rectified linear function, y = max(0, x), is applied to\nthe tensor elementwise.\n" +----f +input: "data" +input: "shape" +output: "reshaped" +name: "Reshape" +op_type: "Reshape" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "shape-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nReshape the input tensor similar to numpy.reshape.\nFirst input is the data tensor, second input is a shape tensor which specifies the output shape. It outputs the reshaped tensor.\nAt most one dimension of the new shape can be -1. In this case, the value is\ninferred from the size of the tensor and the remaining dimensions. A dimension\ncould also be 0, in which case the actual dimension value is unchanged (i.e. taken\nfrom the input tensor)." +----f +input: "X" +input: "roi" +input: "scales" +input: "sizes" +output: "Y" +name: "Resize" +op_type: "Resize" +attribute { + name: "coordinate_transformation_mode" + s: "half_pixel" + type: STRING +} +attribute { + name: "cubic_coeff_a" + f: -0.75 + type: FLOAT +} +attribute { + name: "exclude_outside" + i: 0 + type: INT +} +attribute { + name: "extrapolation_value" + f: 0.0 + type: FLOAT +} +attribute { + name: "mode" + s: "nearest" + type: STRING +} +attribute { + name: "nearest_mode" + s: "round_prefer_floor" + type: STRING +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "roi-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "scales-types" + strings: "float" + type: STRINGS +} +attribute { + name: "sizes-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nResize the input tensor. In general, it calculates every value in the output tensor as a weighted average of neighborhood (a.k.a. sampling locations) in the input tensor.\nEach dimension value of the output tensor is:\n output_dimension = floor(input_dimension * (roi_end - roi_start) * scale) if input \\\"sizes\\\" is not specified.\n" +----f +input: "input" +input: "sequence_lens" +output: "Y" +name: "ReverseSequence" +op_type: "ReverseSequence" +attribute { + name: "batch_axis" + i: 1 + type: INT +} +attribute { + name: "time_axis" + i: 0 + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "sequence_lens-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nReverse batch of sequences having different lengths specified by `sequence_lens`.\n\nFor each slice i iterating on batch axis, the operator reverses the first sequence_lens[i] elements on time axis,\nand copies elements whose index\'s beyond sequence_lens[i] to the output. So the output slice i contains reversed\nsequences on the first sequence_lens[i] elements, then have original values copied for the other elements.\n\nExample 1:\n input = [[0.0, 4.0, 8.0, 12.0],\n [1.0, 5.0, 9.0, 13.0],\n [2.0, 6.0, 10.0, 14.0],\n [3.0, 7.0, 11.0, 15.0]]\n sequence_lens = [4, 3, 2, 1]\n time_axis = 0\n batch_axis = 1\n\n output = [[3.0, 6.0, 9.0, 12.0],\n [2.0, 5.0, 8.0, 13.0],\n [1.0, 4.0, 10.0, 14.0],\n [0.0, 7.0, 11.0, 15.0]]\n\nExample 2:\n input = [[0.0, 1.0, 2.0, 3.0 ],\n [4.0, 5.0, 6.0, 7.0 ],\n [8.0, 9.0, 10.0, 11.0],\n [12.0, 13.0, 14.0, 15.0]]\n sequence_lens = [1, 2, 3, 4]\n time_axis = 1\n batch_axis = 0\n\n output = [[0.0, 1.0, 2.0, 3.0 ],\n [5.0, 4.0, 6.0, 7.0 ],\n [10.0, 9.0, 8.0, 11.0],\n [15.0, 14.0, 13.0, 12.0]]\n" +----f +input: "X" +input: "rois" +input: "batch_indices" +output: "Y" +name: "RoiAlign" +op_type: "RoiAlign" +attribute { + name: "mode" + s: "avg" + type: STRING +} +attribute { + name: "output_height" + i: 1 + type: INT +} +attribute { + name: "output_width" + i: 1 + type: INT +} +attribute { + name: "sampling_ratio" + i: 0 + type: INT +} +attribute { + name: "spatial_scale" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "rois-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "batch_indices-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nRegion of Interest (RoI) align operation described in the\n[Mask R-CNN paper](https://arxiv.org/abs/1703.06870).\nRoiAlign consumes an input tensor X and region of interests (rois)\nto apply pooling across each RoI; it produces a 4-D tensor of shape\n(num_rois, C, output_height, output_width).\n\nRoiAlign is proposed to avoid the misalignment by removing\nquantizations while converting from original image into feature\nmap and from feature map into RoI feature; in each ROI bin,\nthe value of the sampled locations are computed directly\nthrough bilinear interpolation.\n" +----f +input: "X" +output: "Y" +name: "Round" +op_type: "Round" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nRound takes one input Tensor and rounds the values, element-wise, meaning\nit finds the nearest integer for each value.\nIn case of halfs, the rule is to round them to the nearest even integer.\nThe output tensor has the same shape and type as the input.\n\nExamples:\n```\nround([0.9]) = [1.0]\nround([2.5]) = [2.0]\nround([2.3]) = [2.0]\nround([1.5]) = [2.0]\nround([-4.5]) = [-4.0]\n```\n" +----f +input: "X" +output: "Y" +output: "Z" +name: "SVMClassifier" +op_type: "SVMClassifier" +attribute { + name: "classlabels_ints" + s: "" + type: INTS +} +attribute { + name: "classlabels_strings" + s: "" + type: STRINGS +} +attribute { + name: "coefficients" + s: "" + type: FLOATS +} +attribute { + name: "kernel_params" + s: "" + type: FLOATS +} +attribute { + name: "kernel_type" + s: "LINEAR" + type: STRING +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "prob_a" + s: "" + type: FLOATS +} +attribute { + name: "prob_b" + s: "" + type: FLOATS +} +attribute { + name: "rho" + s: "" + type: FLOATS +} +attribute { + name: "support_vectors" + s: "" + type: FLOATS +} +attribute { + name: "vectors_per_class" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Support Vector Machine classifier\n" +----f +input: "X" +output: "Y" +name: "SVMRegressor" +op_type: "SVMRegressor" +attribute { + name: "coefficients" + s: "" + type: FLOATS +} +attribute { + name: "kernel_params" + s: "" + type: FLOATS +} +attribute { + name: "kernel_type" + s: "LINEAR" + type: STRING +} +attribute { + name: "n_supports" + i: 0 + type: INT +} +attribute { + name: "one_class" + i: 0 + type: INT +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "rho" + s: "" + type: FLOATS +} +attribute { + name: "support_vectors" + s: "" + type: FLOATS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Support Vector Machine regression prediction and one-class SVM anomaly detection.\n" +----f +input: "X" +output: "Y" +name: "Scaler" +op_type: "Scaler" +attribute { + name: "offset" + s: "" + type: FLOATS +} +attribute { + name: "scale" + s: "" + type: FLOATS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Rescale input data, for example to standardize features by removing the mean and scaling to unit variance.\n" +----f +input: "initial_state_and_scan_inputs" +output: "final_state_and_scan_outputs" +name: "Scan" +op_type: "Scan" +attribute { + name: "body" + s: "" + type: GRAPH +} +attribute { + name: "num_scan_inputs" + s: "" + type: INT +} +attribute { + name: "scan_input_axes" + s: "" + type: INTS +} +attribute { + name: "scan_input_directions" + s: "" + type: INTS +} +attribute { + name: "scan_output_axes" + s: "" + type: INTS +} +attribute { + name: "scan_output_directions" + s: "" + type: INTS +} +attribute { + name: "initial_state_and_scan_inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nScan can be used to iterate over one or more scan_input tensors,\nconstructing zero or more scan_output tensors. It combines ideas from general recurrences,\nfunctional programming constructs such as scan, fold, map, and zip and is intended to enable\ngeneralizations of RNN-like constructs for sequence-to-sequence processing.\nOther tensors (referred to as state_variables here) can be used to carry a state\nwhen iterating from one element to another (similar to hidden-state in RNNs, also referred\nto as loop-carried dependences in the context of loops).\nMany common usages involve a single scan_input tensor (where functionality\nsimilar to scan, fold and map can be obtained). When more than one scan_input is used,\na behavior similar to zip is obtained.\n\nThe attribute body must be a graph, specifying the computation to be performed in\nevery iteration. It takes as input the current values of the state_variables and\nthe current iterated element of the scan_inputs. It must return the (updated) values\nof the state_variables and zero or more scan_output_element tensors. The values of the\nscan_output_element tensors are concatenated over all the iterations to produce the\nscan_output values of the scan construct (similar to the concatenated intermediate\nhidden-state values of RNN-like constructs). All the output tensors (state_variables as\nwell as scan_output_element tensors) are required to have the same shape in each iteration\nof the loop (a restriction imposed to enable efficient memory allocation).\n\nNote that the iterated element passed to the body subgraph does not have a sequence\naxis. It will have a rank one less than the rank of the corresponding scan_input.\n\nThe scan operation returns the final values of the state_variables as well as the\nscan_outputs.\n\nThe optional attribute scan_input_directions specifies the direction (forward or backward)\nfor each scan input. If this attribute is omitted, all sequences are scanned in the forward\ndirection. A bidirectional scan may be performed by specifying the same tensor input twice\nin the scan_inputs, once with a forward direction, and once with a backward direction.\n\nThe scan_output of the operation is produced by concatenating the scan_output_element\nvalues produced by the body in each iteration. The optional attribute scan_output_directions\nspecifies the direction in which scan_output is constructed (by appending or prepending the\nscan_output_element to scan_output in each iteration) for each scan_output. If this attribute\nis omitted, the scan_output_element is appended to the scan_output in each iteration.\n\nThe optional attribute scan_input_axes specifies the axis to be scanned for each scan_input.\nIf omitted, every scan_input will be scanned in axis 0. For example, if axis 0 is the\nbatch axis and axis 1 is the time axis (to be scanned), specify an axis value of 1.\nNote that scanning a non-zero axis may be less efficient than scanning axis zero.\n\nThe optional attribute scan_output_axes specifies the axis along which the scan_outputs\nare accumulated for each scan_output. For example, if axis 1 is the time axis (to be\nscanned) for both inputs and outputs, specify a scan_input axis and scan_output axis\nvalue of 1.\n\nNote that because of the ONNX restriction that only the last parameter of an operator can\nbe variadic, the initial-states and scan-inputs are listed together as one input parameter.\nSimilarly, the final-states and scan-outputs are listed together as one output parameter.\nThe attribute num_scan_inputs indicates the number M of scan-inputs.\n\nThe behavior of\n\n Scan <\n num_scan_inputs = m,\n body = loop-body,\n scan_input_axes = [axis_1, ..., axis_m]\n > (init_1, ..., init_n, scan_1, ..., scan_m)\n\nis equivalent to the following pseudo-code:\n\n // scan_i.shape[axis_i] denotes the (max) sequence-length of scan_i\n // scan_i.shape[axis_i] is required to be equal to scan_j.shape[axis_j] for all i,j.\n sequence_length = scan_1.shape[axis_1];\n\n // initialize state-variables\n st_1 = init_1; ... st_n = init_n;\n // initialize scan-output variables: [] denotes an empty tensor\n scan_out_1 = []; ...; scan_out_k = [];\n // identify number of iterations:\n\n // execute loop\n for (int t = 0; t < sequence_length; ++t) {\n // generate the scan-input elements: the notation T[t] indicates the sub-tensor\n // of rank one less than T obtained by indexing T at position t along axis k.\n si_1 = scan_1[t];\n ... ;\n si_m = scan_m[t];\n // execute loop-body\n st_1, ..., st_n, so_1, ..., so_k = loop-body(st_1, ..., st_n, si_1, ..., si_m)\n // accumulate the scan-output elements\n scan_out_1 = Concat(scan_out_1, so_1); ... ; scan_out_k = Concat(scan_out_k, so_k);\n }\n\n return st_1, ..., st_n, scan_out_1, ..., scan_out_k;\n\n*Sample usage: Encoding RNN using a Scan*\n\nThe following example shows how a simple RNN over an input tensor %X, with weight tensor %Wi,\nrecurrence weight tensor %Ri, bias tensors %Wbi and %Rbi, and initial hidden-state %H_0 can\nbe encoded as a ScanLoop. Note that the loop-body is a nested graph, and it directly computes\n%Wi, %Ri, %Wbi, and %Rbi (typically constants or initializers in the body graph). If these\nvalues are computed in the outer graph, they need to be passed in as extra state_variables.\n\n graph rnn-encoding {\n %H_0 = ... \n %X = ...\n %Y_h, %Y = Scan[body = , num_scan_inputs=1](%H_0, %X)\n return %Y, %Y_h\n }\n\n graph rnn-cell-1 (\n %H_tminus1[FLOAT, tensor]\n %X_t[FLOAT, tensor]\n ) {\n %Wi = ...\n %Ri = ...\n %Wbi = ...\n %Rbi = ...\n %t1 = X_t * (Wi^T)\n %t2 = H_tminus1*(Ri^T)\n %t3 = Add(%t1, %t2)\n %t4 = Add(%t3, %Wbi)\n %t5 = Add(%t4, %Rbi)\n %Ht = Tanh(%t5)\n %Accumulate = Identity(%Ht)\n return %Ht, %Accumulate\n }\n\n" +----f +input: "data" +input: "indices" +input: "updates" +output: "output" +name: "Scatter" +op_type: "Scatter" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "updates-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nThis operator is deprecated. Please use ScatterElements, which provides the same functionality.\n\nScatter takes three inputs `data`, `updates`, and `indices` of the same\nrank r >= 1 and an optional attribute axis that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value\nto values specified by `updates` at specific index positions specified by\n`indices`. Its output shape is the same as the shape of `data`.\n\nFor each entry in `updates`, the target index in `data` is obtained by combining\nthe corresponding entry in `indices` with the index of the entry itself: the\nindex-value for dimension = axis is obtained from the value of the corresponding\nentry in `indices` and the index-value for dimension != axis is obtained from the\nindex of the entry itself.\n\nFor instance, in a 2-D tensor case, the update corresponding to the [i][j] entry\nis performed as below:\n```\n output[indices[i][j]][j] = updates[i][j] if axis = 0, \n output[i][indices[i][j]] = updates[i][j] if axis = 1,\n```\n\nThis operator is the inverse of GatherElements. It is similar to Torch\'s Scatter operation.\n\nExample 1:\n```\n data = [\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n ]\n indices = [\n [1, 0, 2],\n [0, 2, 1],\n ]\n updates = [\n [1.0, 1.1, 1.2],\n [2.0, 2.1, 2.2],\n ]\n output = [\n [2.0, 1.1, 0.0]\n [1.0, 0.0, 2.2]\n [0.0, 2.1, 1.2]\n ]\n```\nExample 2:\n```\n data = [[1.0, 2.0, 3.0, 4.0, 5.0]]\n indices = [[1, 3]]\n updates = [[1.1, 2.1]]\n axis = 1\n output = [[1.0, 1.1, 3.0, 2.1, 5.0]]\n```\n" +----f +input: "data" +input: "indices" +input: "updates" +output: "output" +name: "ScatterElements" +op_type: "ScatterElements" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "updates-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nScatterElements takes three inputs `data`, `updates`, and `indices` of the same\nrank r >= 1 and an optional attribute axis that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value\nto values specified by `updates` at specific index positions specified by\n`indices`. Its output shape is the same as the shape of `data`.\n\nFor each entry in `updates`, the target index in `data` is obtained by combining\nthe corresponding entry in `indices` with the index of the entry itself: the\nindex-value for dimension = axis is obtained from the value of the corresponding\nentry in `indices` and the index-value for dimension != axis is obtained from the\nindex of the entry itself.\n\nFor instance, in a 2-D tensor case, the update corresponding to the [i][j] entry\nis performed as below:\n```\n output[indices[i][j]][j] = updates[i][j] if axis = 0, \n output[i][indices[i][j]] = updates[i][j] if axis = 1,\n```\n\nThis operator is the inverse of GatherElements. It is similar to Torch\'s Scatter operation.\n\nExample 1:\n```\n data = [\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n ]\n indices = [\n [1, 0, 2],\n [0, 2, 1],\n ]\n updates = [\n [1.0, 1.1, 1.2],\n [2.0, 2.1, 2.2],\n ]\n output = [\n [2.0, 1.1, 0.0]\n [1.0, 0.0, 2.2]\n [0.0, 2.1, 1.2]\n ]\n```\nExample 2:\n```\n data = [[1.0, 2.0, 3.0, 4.0, 5.0]]\n indices = [[1, 3]]\n updates = [[1.1, 2.1]]\n axis = 1\n output = [[1.0, 1.1, 3.0, 2.1, 5.0]]\n```\n" +----f +input: "data" +input: "indices" +input: "updates" +output: "output" +name: "ScatterND" +op_type: "ScatterND" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "updates-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nScatterND takes three inputs `data` tensor of rank r >= 1, `indices` tensor of rank q >= 1,\nand `updates` tensor of rank q + r - indices.shape[-1] - 1. The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value to values\nspecified by `updates` at specific index positions specified by `indices`. Its output shape\nis the same as the shape of `data`. Note that `indices` should not have duplicate entries.\nThat is, two or more `updates` for the same index-location is not supported.\n\n`indices` is an integer tensor. Let k denote indices.shape[-1], the last dimension in the shape of `indices`.\n `indices` is treated as a (q-1)-dimensional tensor of k-tuples, where each k-tuple is a partial-index into `data`.\nHence, k can be a value at most the rank of `data`. When k equals rank(data), each update entry specifies an\nupdate to a single element of the tensor. When k is less than rank(data) each update entry specifies an\nupdate to a slice of the tensor.\n\n`updates` is treated as a (q-1)-dimensional tensor of replacement-slice-values. Thus, the\nfirst (q-1) dimensions of updates.shape must match the first (q-1) dimensions of indices.shape.\nThe remaining dimensions of `updates` correspond to the dimensions of the\nreplacement-slice-values. Each replacement-slice-value is a (r-k) dimensional tensor,\ncorresponding to the trailing (r-k) dimensions of `data`. Thus, the shape of `updates`\nmust equal indices.shape[0:q-1] ++ data.shape[k:r-1], where ++ denotes the concatenation\nof shapes.\n\nThe `output` is calculated via the following equation:\n\n output = np.copy(data)\n update_indices = indices.shape[:-1]\n for idx in np.ndindex(update_indices):\n output[indices[idx]] = updates[idx]\n\nThe order of iteration in the above loop is not specified.\nIn particular, indices should not have duplicate entries: that is, if idx1 != idx2, then indices[idx1] != indices[idx2].\nThis ensures that the output value does not depend on the iteration order.\n\nThis operator is the inverse of GatherND.\n\nExample 1:\n```\n data = [1, 2, 3, 4, 5, 6, 7, 8]\n indices = [[4], [3], [1], [7]]\n updates = [9, 10, 11, 12]\n output = [1, 11, 3, 10, 9, 6, 7, 12]\n```\n\nExample 2:\n```\n data = [[[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]]]\n indices = [[0], [2]]\n updates = [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],\n [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]]\n output = [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],\n [[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]]]\n```\n" +----f +input: "X" +output: "Y" +name: "Selu" +op_type: "Selu" +attribute { + name: "alpha" + f: 1.6732632 + type: FLOAT +} +attribute { + name: "gamma" + f: 1.050701 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nSelu takes one input data (Tensor) and produces one output data\n(Tensor) where the scaled exponential linear unit function,\n`y = gamma * (alpha * e^x - alpha) for x <= 0`, `y = gamma * x for x > 0`,\nis applied to the tensor elementwise.\n" +----f +input: "input_sequence" +input: "position" +output: "tensor" +name: "SequenceAt" +op_type: "SequenceAt" +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +attribute { + name: "position-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nOutputs a tensor copy from the tensor at \'position\' in \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n - 1]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n" +----f +input: "inputs" +output: "output_sequence" +name: "SequenceConstruct" +op_type: "SequenceConstruct" +attribute { + name: "inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nConstruct a tensor sequence containing \'inputs\' tensors.\nAll tensors in \'inputs\' must have the same data type.\n" +----f +output: "output" +name: "SequenceEmpty" +op_type: "SequenceEmpty" +attribute { + name: "dtype" + s: "" + type: INT +} +doc_string: "\nConstruct an empty tensor sequence, with given data type.\n" +----f +input: "input_sequence" +input: "position" +output: "output_sequence" +name: "SequenceErase" +op_type: "SequenceErase" +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +attribute { + name: "position-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nOutputs a tensor sequence that removes the tensor at \'position\' from \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n - 1]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n\'position\' is optional, by default it erases the last tensor from \'input_sequence\'.\n" +----f +input: "input_sequence" +input: "tensor" +input: "position" +output: "output_sequence" +name: "SequenceInsert" +op_type: "SequenceInsert" +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +attribute { + name: "tensor-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "position-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nOutputs a tensor sequence that inserts \'tensor\' into \'input_sequence\' at \'position\'.\n\'tensor\' must have the same data type as \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n\'position\' is optional, by default it inserts \'tensor\' to the back of \'input_sequence\'.\n" +----f +input: "input_sequence" +output: "length" +name: "SequenceLength" +op_type: "SequenceLength" +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +doc_string: "\nProduces a scalar(tensor of empty shape) containing the number of tensors in \'input_sequence\'.\n" +----f +input: "data" +output: "shape" +name: "Shape" +op_type: "Shape" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nTakes a tensor as input and outputs an 1D int64 tensor containing the shape of the input tensor.\n" +----f +input: "input" +output: "output" +name: "Shrink" +op_type: "Shrink" +attribute { + name: "bias" + f: 0.0 + type: FLOAT +} +attribute { + name: "lambd" + f: 0.5 + type: FLOAT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nShrink takes one input data (Tensor) and produces one Tensor output,\nhaving same datatype and shape with input. It has two attributes, lambd and\nbias. The formula of this operator is: If x < -lambd, y = x + bias;\nIf x > lambd, y = x - bias; Otherwise, y = 0.\n" +----f +input: "X" +output: "Y" +name: "Sigmoid" +op_type: "Sigmoid" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nSigmoid takes one input data (Tensor) and produces one output data\n(Tensor) where the sigmoid function, y = 1 / (1 + exp(-x)), is applied to the\ntensor elementwise.\n" +----f +input: "input" +output: "output" +name: "Sign" +op_type: "Sign" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nCalculate the sign of the given input tensor element-wise.\nIf input > 0, output 1. if input < 0, output -1. if input == 0, output 0.\n" +----f +input: "input" +output: "output" +name: "Sin" +op_type: "Sin" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the sine of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Sinh" +op_type: "Sinh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic sine of the given input tensor element-wise.\n" +----f +input: "data" +output: "size" +name: "Size" +op_type: "Size" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nTakes a tensor as input and outputs a int64 scalar that equals to the total number of elements of the input tensor.\n" +----f +input: "data" +input: "starts" +input: "ends" +input: "axes" +input: "steps" +output: "output" +name: "Slice" +op_type: "Slice" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "starts-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "ends-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "axes-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "steps-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nProduces a slice of the input tensor along multiple axes. Similar to numpy:\nhttps://docs.scipy.org/doc/numpy/reference/arrays.indexing.html\nSlices uses `starts`, `ends`, `axes` and `steps` inputs to specify the start and end\ndimension and step for each axis in the list of axes, it uses this information to\nslice the input `data` tensor. If a negative value is passed for any of the\nstart or end indices, it represents number of elements before the end of that\ndimension. If the value passed to start or end is larger than the `n` (the\nnumber of elements in this dimension), it represents `n`. For slicing to the\nend of a dimension with unknown size, it is recommended to pass in `INT_MAX` \nwhen sclicing forward and \'INT_MIN\' when slicing backward.\nIf a negative value is passed for step, it represents slicing backward. \nHowever step value cannot be 0.\nIf `axes` are omitted, they are set to `[0, ..., ndim-1]`.\nIf `steps` are omitted, they are set to `[1, ..., 1]` of length `len(starts)`\nExample 1:\n data = [\n [1, 2, 3, 4],\n [5, 6, 7, 8],\n ]\n axes = [0, 1]\n starts = [1, 0]\n ends = [2, 3]\n steps = [1, 2]\n result = [\n [5, 7],\n ]\nExample 2:\n data = [\n [1, 2, 3, 4],\n [5, 6, 7, 8],\n ]\n starts = [0, 1]\n ends = [-1, 1000]\n result = [\n [2, 3, 4],\n ]\n" +----f +input: "input" +output: "output" +name: "Softmax" +op_type: "Softmax" +attribute { + name: "axis" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe operator computes the softmax (normalized exponential) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the softmax values of the corresponding input.\n" +----f +input: "scores" +input: "labels" +input: "weights" +output: "output" +output: "log_prob" +name: "SoftmaxCrossEntropyLoss" +op_type: "SoftmaxCrossEntropyLoss" +attribute { + name: "ignore_index" + s: "" + type: INT +} +attribute { + name: "reduction" + s: "mean" + type: STRING +} +attribute { + name: "scores-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "labels-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "weights-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "Loss function that measures the softmax cross entropy\nbetween \'scores\' and \'labels\'.\nThis operator first computes a loss tensor whose shape is identical to the labels input.\nIf the input is 2-D with shape (N, C), the loss tensor may be a N-element vector L = (l_1, l_2, ..., l_N).\nIf the input is N-D tensor with shape (N, C, D1, D2, ..., Dk),\nthe loss tensor L may have (N, D1, D2, ..., Dk) as its shape and L[i,][j_1][j_2]...[j_k] denotes a scalar element in L.\nAfter L is available, this operator can optionally do a reduction operator.\n\nshape(scores): (N, C) where C is the number of classes, or (N, C, D1, D2,..., Dk),\n with K >= 1 in case of K-dimensional loss.\nshape(labels): (N) where each value is 0 <= labels[i] <= C-1, or (N, D1, D2,..., Dk),\n with K >= 1 in case of K-dimensional loss.\n\nThe loss for one sample, l_i, can caculated as follows:\n l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk], where i is the index of classes.\nor\n l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk] * weights[c], if \'weights\' is provided.\n\nloss is zero for the case when label-value equals ignore_index.\n l[i][d1][d2]...[dk] = 0, when labels[n][d1][d2]...[dk] = ignore_index\n\nwhere:\n p = Softmax(scores)\n y = Log(p)\n c = labels[i][d1][d2]...[dk]\n\nFinally, L is optionally reduced:\nIf reduction = \'none\', the output is L with shape (N, D1, D2, ..., Dk).\nIf reduction = \'sum\', the output is scalar: Sum(L).\nIf reduction = \'mean\', the output is scalar: ReduceMean(L), or if weight is provided: ReduceSum(L) / ReduceSum(W),\nwhere tensor W is of shape (N, D1, D2, ..., Dk) and W[n][d1][d2]...[dk] = weights[labels[i][d1][d2]...[dk]].\n" +----f +input: "X" +output: "Y" +name: "Softplus" +op_type: "Softplus" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nSoftplus takes one input data (Tensor) and produces one output data\n(Tensor) where the softplus function, y = ln(exp(x) + 1), is applied to\nthe tensor elementwise.\n" +----f +input: "input" +output: "output" +name: "Softsign" +op_type: "Softsign" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the softsign (x/(1+|x|)) of the given input tensor element-wise.\n" +----f +input: "input" +output: "output" +name: "SpaceToDepth" +op_type: "SpaceToDepth" +attribute { + name: "blocksize" + s: "" + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "SpaceToDepth rearranges blocks of spatial data into depth. More specifically,\nthis op outputs a copy of the input tensor where values from the height and width dimensions\nare moved to the depth dimension.\n" +----f +input: "input" +output: "outputs" +name: "Split" +op_type: "Split" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "split" + s: "" + type: INTS +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "Split a tensor into a list of tensors, along the specified\n\'axis\'. Lengths of the parts can be specified using argument \'split\'.\nOtherwise, the tensor is split to equal sized parts.\n" +----f +input: "input" +input: "split" +output: "output_sequence" +name: "SplitToSequence" +op_type: "SplitToSequence" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "split-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "Split a tensor into a sequence of tensors, along the specified\n\'axis\'. Lengths of the parts can be specified using argument \'split\'.\n\'split\' must contain only positive numbers.\n\'split\' is either a scalar (tensor of empty shape), or a 1-D tensor.\nIf \'split\' is a scalar, then \'input\' will be split into equally sized chunks(if possible).\nLast chunk will be smaller if the \'input\' size along the given axis \'axis\' is not divisible\nby \'split\'.\nOtherwise, the tensor is split into \'size(split)\' chunks, with lengths of the parts on \'axis\'\nspecified in \'split\'. In this scenario, the sum of entries in \'split\' must be equal to the\ndimension size of input tensor on \'axis\'.\n" +----f +input: "X" +output: "Y" +name: "Sqrt" +op_type: "Sqrt" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nSquare root takes one input data (Tensor) and produces one output data\n(Tensor) where the square root is, y = x^0.5, is applied to\nthe tensor elementwise. If x is negative, then it will return NaN.\n" +----f +input: "data" +output: "squeezed" +name: "Squeeze" +op_type: "Squeeze" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nRemove single-dimensional entries from the shape of a tensor.\nTakes a parameter `axes` with a list of axes to squeeze.\nIf `axes` is not provided, all the single dimensions will be removed from\nthe shape. If an axis is selected with shape entry not equal to one, an error is raised.\n" +----f +input: "X" +output: "Y" +name: "StringNormalizer" +op_type: "StringNormalizer" +attribute { + name: "case_change_action" + s: "NONE" + type: STRING +} +attribute { + name: "is_case_sensitive" + i: 0 + type: INT +} +attribute { + name: "locale" + s: "" + type: STRING +} +attribute { + name: "stopwords" + s: "" + type: STRINGS +} +attribute { + name: "X-types" + strings: "string" + type: STRINGS +} +doc_string: "\nStringNormalization performs string operations for basic cleaning.\nThis operator has only one input (denoted by X) and only one output\n(denoted by Y). This operator first examines the elements in the X,\nand removes elements specified in \"stopwords\" attribute.\nAfter removing stop words, the intermediate result can be further lowercased,\nuppercased, or just returned depending the \"case_change_action\" attribute.\nThis operator only accepts [C]- and [1, C]-tensor.\nIf all elements in X are dropped, the output will be the empty value of string tensor with shape [1]\nif input shape is [C] and shape [1, 1] if input shape is [1, C].\n" +----f +input: "A" +input: "B" +output: "C" +name: "Sub" +op_type: "Sub" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPerforms element-wise binary subtraction (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "data_0" +output: "sum" +name: "Sum" +op_type: "Sum" +attribute { + name: "data_0-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nElement-wise sum of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "input" +output: "output" +name: "Tan" +op_type: "Tan" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the tangent of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Tanh" +op_type: "Tanh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic tangent of the given input tensor element-wise.\n" +----f +input: "X" +output: "Y" +name: "TfIdfVectorizer" +op_type: "TfIdfVectorizer" +attribute { + name: "max_gram_length" + s: "" + type: INT +} +attribute { + name: "max_skip_count" + s: "" + type: INT +} +attribute { + name: "min_gram_length" + s: "" + type: INT +} +attribute { + name: "mode" + s: "" + type: STRING +} +attribute { + name: "ngram_counts" + s: "" + type: INTS +} +attribute { + name: "ngram_indexes" + s: "" + type: INTS +} +attribute { + name: "pool_int64s" + s: "" + type: INTS +} +attribute { + name: "pool_strings" + s: "" + type: STRINGS +} +attribute { + name: "weights" + s: "" + type: FLOATS +} +attribute { + name: "X-types" + strings: "string" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nThis transform extracts n-grams from the input sequence and save them as a vector. Input can\nbe either a 1-D or 2-D tensor. For 1-D input, output is the n-gram representation of that input.\nFor 2-D input, the output is also a 2-D tensor whose i-th row is the n-gram representation of the i-th input row.\nMore specifically, if input shape is [C], the corresponding output shape would be [max(ngram_indexes) + 1].\nIf input shape is [N, C], this operator produces a [N, max(ngram_indexes) + 1]-tensor.\n\nIn contrast to standard n-gram extraction, here, the indexes of extracting an n-gram from the original\nsequence are not necessarily consecutive numbers. The discontinuity between indexes are controlled by the number of skips.\nIf the number of skips is 2, we should skip two tokens when scanning through the original sequence.\nLet\'s consider an example. Assume that input sequence is [94, 17, 36, 12, 28] and the number of skips is 2.\nThe associated 2-grams are [94, 12] and [17, 28] respectively indexed by [0, 3] and [1, 4].\nIf the number of skips becomes 0, the 2-grams generated are [94, 17], [17, 36], [36, 12], [12, 28]\nindexed by [0, 1], [1, 2], [2, 3], [3, 4], respectively.\n\nThe output vector (denoted by Y) stores the count of each n-gram;\nY[ngram_indexes[i]] indicates the times that the i-th n-gram is found. The attribute ngram_indexes is used to determine the mapping\nbetween index i and the corresponding n-gram\'s output coordinate. If pool_int64s is [94, 17, 17, 36], ngram_indexes is [1, 0],\nngram_counts=[0, 0], then the Y[0] (first element in Y) and Y[1] (second element in Y) are the counts of [17, 36] and [94, 17],\nrespectively. An n-gram which cannot be found in pool_strings/pool_int64s should be ignored and has no effect on the output.\nNote that we may consider all skips up to S when generating the n-grams.\n\nThe examples used above are true if mode is \"TF\". If mode is \"IDF\", all the counts larger than 1 would be truncated to 1 and\nthe i-th element in weights would be used to scale (by multiplication) the count of the i-th n-gram in pool. If mode is \"TFIDF\",\nthis operator first computes the counts of all n-grams and then scale them by the associated values in the weights attribute.\n\nOnly one of pool_strings and pool_int64s can be set. If pool_int64s is set, the input should be an integer tensor.\nIf pool_strings is set, the input must be a string tensor.\n" +----f +input: "X" +output: "Y" +name: "ThresholdedRelu" +op_type: "ThresholdedRelu" +attribute { + name: "alpha" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThresholdedRelu takes one input data (Tensor) and produces one output data\n(Tensor) where the rectified linear function, y = x for x > alpha, y = 0 otherwise,\nis applied to the tensor elementwise.\n" +----f +input: "input" +input: "repeats" +output: "output" +name: "Tile" +op_type: "Tile" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "repeats-types" + strings: "int64" + type: STRINGS +} +doc_string: "Constructs a tensor by tiling a given tensor.\nThis is the same as function `tile` in Numpy, but no broadcast.\nFor example A = [[1, 2], [3, 4]], B = [1, 2], tile(A, B) = [[1, 2, 1, 2], [3, 4, 3, 4]]\n" +----f +input: "X" +input: "K" +output: "Values" +output: "Indices" +name: "TopK" +op_type: "TopK" +attribute { + name: "axis" + i: -1 + type: INT +} +attribute { + name: "largest" + i: 1 + type: INT +} +attribute { + name: "sorted" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "K-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nRetrieve the top-K largest or smallest elements along a specified axis. Given an input tensor of\nshape [a_1, a_2, ..., a_n, r] and integer argument k, return two outputs:\n -Value tensor of shape [a_1, a_2, ..., a_{axis-1}, k, a_{axis+1}, ... a_n]\n which contains the values of the top k elements along the specified axis\n -Index tensor of shape [a_1, a_2, ..., a_{axis-1}, k, a_{axis+1}, ... a_n] which\n contains the indices of the top k elements (original indices from the input\n tensor).\n\nIf \"largest\" is 1 (the default value) then the k largest elements are returned.\nIf \"sorted\" is 1 (the default value) then the resulting k elements will be sorted.\nIf \"sorted\" is 0, order of returned \'Values\' and \'Indices\' are undefined.\n\nGiven two equivalent values, this operator uses the indices along the axis as\n a tiebreaker. That is, the element with the lower index will appear first.\n" +----f +input: "data" +output: "transposed" +name: "Transpose" +op_type: "Transpose" +attribute { + name: "perm" + s: "" + type: INTS +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nTranspose the input tensor similar to numpy.transpose. For example, when\nperm=(1, 0, 2), given an input tensor of shape (1, 2, 3), the output shape\nwill be (2, 1, 3).\n" +----f +input: "X" +output: "Y" +output: "Z" +name: "TreeEnsembleClassifier" +op_type: "TreeEnsembleClassifier" +attribute { + name: "base_values" + s: "" + type: FLOATS +} +attribute { + name: "class_ids" + s: "" + type: INTS +} +attribute { + name: "class_nodeids" + s: "" + type: INTS +} +attribute { + name: "class_treeids" + s: "" + type: INTS +} +attribute { + name: "class_weights" + s: "" + type: FLOATS +} +attribute { + name: "classlabels_int64s" + s: "" + type: INTS +} +attribute { + name: "classlabels_strings" + s: "" + type: STRINGS +} +attribute { + name: "nodes_falsenodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_featureids" + s: "" + type: INTS +} +attribute { + name: "nodes_hitrates" + s: "" + type: FLOATS +} +attribute { + name: "nodes_missing_value_tracks_true" + s: "" + type: INTS +} +attribute { + name: "nodes_modes" + s: "" + type: STRINGS +} +attribute { + name: "nodes_nodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_treeids" + s: "" + type: INTS +} +attribute { + name: "nodes_truenodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_values" + s: "" + type: FLOATS +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Tree Ensemble classifier. Returns the top class for each of N inputs.
\n The attributes named \'nodes_X\' form a sequence of tuples, associated by \n index into the sequences, which must all be of equal length. These tuples\n define the nodes.
\n Similarly, all fields prefixed with \'class_\' are tuples of votes at the leaves.\n A leaf may have multiple votes, where each vote is weighted by\n the associated class_weights index.
\n One and only one of classlabels_strings or classlabels_int64s\n will be defined. The class_ids are indices into this list.\n" +----f +input: "X" +output: "Y" +name: "TreeEnsembleRegressor" +op_type: "TreeEnsembleRegressor" +attribute { + name: "aggregate_function" + s: "SUM" + type: STRING +} +attribute { + name: "base_values" + s: "" + type: FLOATS +} +attribute { + name: "n_targets" + s: "" + type: INT +} +attribute { + name: "nodes_falsenodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_featureids" + s: "" + type: INTS +} +attribute { + name: "nodes_hitrates" + s: "" + type: FLOATS +} +attribute { + name: "nodes_missing_value_tracks_true" + s: "" + type: INTS +} +attribute { + name: "nodes_modes" + s: "" + type: STRINGS +} +attribute { + name: "nodes_nodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_treeids" + s: "" + type: INTS +} +attribute { + name: "nodes_truenodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_values" + s: "" + type: FLOATS +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "target_ids" + s: "" + type: INTS +} +attribute { + name: "target_nodeids" + s: "" + type: INTS +} +attribute { + name: "target_treeids" + s: "" + type: INTS +} +attribute { + name: "target_weights" + s: "" + type: FLOATS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Tree Ensemble regressor. Returns the regressed values for each input in N.
\n All args with nodes_ are fields of a tuple of tree nodes, and\n it is assumed they are the same length, and an index i will decode the\n tuple across these inputs. Each node id can appear only once\n for each tree id.
\n All fields prefixed with target_ are tuples of votes at the leaves.
\n A leaf may have multiple votes, where each vote is weighted by\n the associated target_weights index.
\n All trees must have their node ids start at 0 and increment by 1.
\n Mode enum is BRANCH_LEQ, BRANCH_LT, BRANCH_GTE, BRANCH_GT, BRANCH_EQ, BRANCH_NEQ, LEAF\n" +----f +input: "X" +output: "Y" +output: "indices" +output: "inverse_indices" +output: "counts" +name: "Unique" +op_type: "Unique" +attribute { + name: "axis" + s: "" + type: INT +} +attribute { + name: "sorted" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nFind the unique elements of a tensor. When an optional attribute \'axis\' is provided, unique subtensors sliced along the \'axis\' are returned. \nOtherwise the input tensor is flattened and unique values of the flattened tensor are returned. \n\nThis operator returns the unique values or sliced unique subtensors of the input tensor and three optional outputs. \nThe first output tensor \'Y\' contains all unique values or subtensors of the input. \nThe second optional output tensor \'indices\' contains indices of \'Y\' elements\' first occurance in \'X\'.. \nThe third optional output tensor \'inverse_indices\' contains, for elements of \'X\', its corresponding indices in \'Y\'. \". \nThe fourth optional output tensor \'counts\' contains the count of each element of \'Y\' in the input. \n\nOutputs are either sorted in ascending order or optionally in the order of the first occurrence of the values in the input. \n\nhttps://docs.scipy.org/doc/numpy/reference/generated/numpy.unique.html\n\nExample 1:\n input_X = [2, 1, 1, 3, 4, 3]\n attribute_sorted = 0\n attribute_axis = None\n output_Y = [2, 1, 3, 4]\n output_indices = [0, 1, 3, 4]\n output_inverse_indices = [0, 1, 1, 2, 3, 2]\n output_counts = [1, 2, 2, 1]\n\nExample 2:\n input_X = [[1, 3], [2, 3]]\n attribute_sorted = 1\n attribute_axis = None\n output_Y = [1, 2, 3]\n output_indices = [0, 2, 1]\n output_inverse_indices = [0, 2, 1, 2]\n output_counts = [1, 1, 2]\n\nExample 3:\n input_X = [[1, 0, 0], [1, 0, 0], [2, 3, 4]]\n attribute_sorted = 1\n attribute_axis = 0\n output_Y = [[1, 0, 0], [2, 3, 4]]\n output_indices = [0, 2]\n output_inverse_indices = [0, 0, 1]\n output_counts = [2, 1]\n\nExample 4:\n input_x = [[[1., 1.], [0., 1.], [2., 1.], [0., 1.]], \n [[1., 1.], [0., 1.], [2., 1.], [0., 1.]]]\n attribute_sorted = 1\n attribute_axis = 1\n\n intermediate data are presented below for better understanding: \n \n there are 4 subtensors sliced along axis 1 of input_x (shape = (2, 4, 2)):\n A: [[1, 1], [1, 1]], \n [[0, 1], [0, 1]], \n [[2, 1], [2, 1]], \n [[0, 1], [0, 1]].\n \n there are 3 unique subtensors: \n [[1, 1], [1, 1]], \n [[0, 1], [0, 1]], \n [[2, 1], [2, 1]].\n \n sorted unique subtensors:\n B: [[0, 1], [0, 1]], \n [[1, 1], [1, 1]], \n [[2, 1], [2, 1]].\n \n output_Y is constructed from B:\n [[[0. 1.], [1. 1.], [2. 1.]], \n [[0. 1.], [1. 1.], [2. 1.]]]\n\n output_indices is to map from B to A:\n [1, 0, 2]\n \n output_inverse_indices is to map from A to B:\n [1, 0, 2, 0]\n\n output_counts = [2 1 1]\n" +----f +input: "data" +output: "expanded" +name: "Unsqueeze" +op_type: "Unsqueeze" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nInsert single-dimensional entries to the shape of an input tensor (`data`).\nTakes one required argument `axes` - which contains a list of dimension indices and this operator will insert a dimension of value `1` into the corresponding index of the output tensor (`expanded`).\n\nFor example:\n Given an input tensor (`data`) of shape [3, 4, 5], then\n Unsqueeze(data, axes=[0, 4]) outputs a tensor (`expanded`) containing same data as `data` but with shape [1, 3, 4, 5, 1].\n\nThe attribute `axes` should not contain any duplicate entries. It is an error if it contains duplicates.\nThe rank of the output tensor (`output_rank`) is the rank of the input tensor (`data`) plus the number of values in `axes`.\nEach value in `axes` should be within the (inclusive) range [-output_rank , output_rank - 1]. \nThe order of values in `axes` does not matter and can come in any order. \n\n" +----f +input: "X" +input: "scales" +output: "Y" +name: "Upsample" +op_type: "Upsample" +attribute { + name: "mode" + s: "nearest" + type: STRING +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "scales-types" + strings: "float" + type: STRINGS +} +doc_string: "\nUpsample the input tensor.\nEach dimension value of the output tensor is:\n output_dimension = floor(input_dimension * scale).\n" +----f +input: "condition" +input: "X" +input: "Y" +output: "output" +name: "Where" +op_type: "Where" +attribute { + name: "condition-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "Y-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\n Return elements, either from X or Y, depending on condition\n (with Numpy-style broadcasting support).\n Where behaves like numpy.where with three parameters:\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.where.html\n" +----f +input: "A" +input: "B" +output: "C" +name: "Xor" +op_type: "Xor" +attribute { + name: "A-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "B-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `xor` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Z" +name: "ZipMap" +op_type: "ZipMap" +attribute { + name: "classlabels_int64s" + s: "" + type: INTS +} +attribute { + name: "classlabels_strings" + s: "" + type: STRINGS +} +attribute { + name: "X-types" + strings: "float" + type: STRINGS +} +doc_string: "\n Creates a map from the input and the attributes.
\n The values are provided by the input tensor, while the keys are specified by the attributes.\n Must provide keys in either classlabels_strings or classlabels_int64s (but not both).
\n The columns of the tensor correspond one-by-one to the keys specified by the attributes. There must be as many columns as keys.
\n" +----f diff --git a/contrib/codegen-tools/codegen/src/test/java/org/nd4j/codegen/dsl/DocsGeneratorTest.java b/contrib/codegen-tools/codegen/src/test/java/org/nd4j/codegen/dsl/DocsGeneratorTest.java index 7eeef5717..5d8e12885 100644 --- a/contrib/codegen-tools/codegen/src/test/java/org/nd4j/codegen/dsl/DocsGeneratorTest.java +++ b/contrib/codegen-tools/codegen/src/test/java/org/nd4j/codegen/dsl/DocsGeneratorTest.java @@ -17,23 +17,29 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.nd4j.codegen.dsl; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Test; import org.nd4j.codegen.impl.java.DocsGenerator; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -@DisplayName("Docs Generator Test") -class DocsGeneratorTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DocsGeneratorTest { @Test - @DisplayName("Test J Dto MD Adapter") - void testJDtoMDAdapter() { - String original = "{@code %INPUT_TYPE% eye = eye(3,2)\n" + " eye:\n" + " [ 1, 0]\n" + " [ 0, 1]\n" + " [ 0, 0]}"; - String expected = "{ INDArray eye = eye(3,2)\n" + " eye:\n" + " [ 1, 0]\n" + " [ 0, 1]\n" + " [ 0, 0]}"; + public void testJDtoMDAdapter() { + String original = "{@code %INPUT_TYPE% eye = eye(3,2)\n" + + " eye:\n" + + " [ 1, 0]\n" + + " [ 0, 1]\n" + + " [ 0, 0]}"; + String expected = "{ INDArray eye = eye(3,2)\n" + + " eye:\n" + + " [ 1, 0]\n" + + " [ 0, 1]\n" + + " [ 0, 0]}"; DocsGenerator.JavaDocToMDAdapter adapter = new DocsGenerator.JavaDocToMDAdapter(original); String out = adapter.filter("@code", StringUtils.EMPTY).filter("%INPUT_TYPE%", "INDArray").toString(); assertEquals(out, expected); diff --git a/contrib/codegen-tools/libnd4j-gen/pom.xml b/contrib/codegen-tools/libnd4j-gen/pom.xml index 267765425..100b31312 100644 --- a/contrib/codegen-tools/libnd4j-gen/pom.xml +++ b/contrib/codegen-tools/libnd4j-gen/pom.xml @@ -1,109 +1,94 @@ - - - - - - 4.0.0 - - org.nd4j - libnd4j-gen - 1.0-SNAPSHOT - - libnd4j-gen - - - - UTF-8 - 1.8 - 1.8 - 1.0.0-SNAPSHOT - - - - - com.codepoetics - protonpack - 1.16 - - - com.github.javaparser - javaparser-core-serialization - 3.17.0 - - - com.github.javaparser - javaparser-symbol-solver-core - 3.17.0 - - - org.apache.commons - commons-text - 1.9 - - - org.apache.commons - commons-collections4 - 4.1 - - - org.reflections - reflections - 0.9.10 - - - - org.nd4j - protobuf - ${nd4j.version} - - - - junit - junit - 4.12 - test - - - - - org.projectlombok - lombok - 1.18.12 - - - - - org.nd4j - nd4j-native - ${nd4j.version} - - - - - org.nd4j - nd4j-api - ${nd4j.version} - - - - - + + + + + + 4.0.0 + + net.brutex.ai + libnd4j-gen + 1.0-SNAPSHOT + + libnd4j-gen + + + + UTF-8 + 1.8 + 1.8 + 1.0.0-SNAPSHOT + + + + + com.codepoetics + protonpack + 1.16 + + + com.github.javaparser + javaparser-core-serialization + 3.17.0 + + + com.github.javaparser + javaparser-symbol-solver-core + 3.17.0 + + + org.apache.commons + commons-text + 1.9 + + + org.apache.commons + commons-collections4 + 4.1 + + + org.reflections + reflections + 0.9.10 + + + + net.brutex.ai + protobuf + ${project.version} + + + + net.brutex.ai + nd4j-native + ${project.version} + + + + + net.brutex.ai + nd4j-api + ${project.version} + + + + + diff --git a/contrib/codegen-tools/libnd4j-gen/src/main/java/org/nd4j/descriptor/ParseOpFile.java b/contrib/codegen-tools/libnd4j-gen/src/main/java/org/nd4j/descriptor/ParseOpFile.java index 61e518201..d33500ea9 100644 --- a/contrib/codegen-tools/libnd4j-gen/src/main/java/org/nd4j/descriptor/ParseOpFile.java +++ b/contrib/codegen-tools/libnd4j-gen/src/main/java/org/nd4j/descriptor/ParseOpFile.java @@ -29,7 +29,7 @@ import org.nd4j.descriptor.proposal.impl.JavaSourceArgDescriptorSource; import org.nd4j.descriptor.proposal.impl.Libnd4jArgDescriptorSource; import org.nd4j.descriptor.proposal.impl.ArgDescriptorParserUtils; import org.nd4j.ir.OpNamespace; -import org.nd4j.shade.protobuf.TextFormat; +import com.google.protobuf.TextFormat; import java.io.File; import java.nio.charset.Charset; diff --git a/contrib/codegen-tools/onnx-def-gen/README.md b/contrib/codegen-tools/onnx-def-gen/README.md index 62a8eb8b0..aba3a7e42 100644 --- a/contrib/codegen-tools/onnx-def-gen/README.md +++ b/contrib/codegen-tools/onnx-def-gen/README.md @@ -1,19 +1,19 @@ -Onnx op definition loading ---------------------------------- - -Setup -------- -Use anaconda and install onnx: -``` -conda install onnx -``` - -Generate a file ---------------------- -``` -python onnx_def_gen.py -``` - -This will generate a file with all op definitions -loadable as NodeProto in onnx serialized as a text file -split by --\n. +Onnx op definition loading +--------------------------------- + +Setup +------- +Use anaconda and install onnx: +``` +conda install onnx +``` + +Generate a file +--------------------- +``` +python onnx_def_gen.py +``` + +This will generate a file with all op definitions +loadable as NodeProto in onnx serialized as a text file +split by --\n. diff --git a/contrib/codegen-tools/onnx-def-gen/onnx.pbtxt b/contrib/codegen-tools/onnx-def-gen/onnx.pbtxt index 5fa814d06..c4385573a 100644 --- a/contrib/codegen-tools/onnx-def-gen/onnx.pbtxt +++ b/contrib/codegen-tools/onnx-def-gen/onnx.pbtxt @@ -1,6004 +1,6004 @@ -input: "X" -output: "Y" -name: "Abs" -op_type: "Abs" -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nAbsolute takes one input data (Tensor) and produces one output data\n(Tensor) where the absolute is, y = abs(x), is applied to\nthe tensor elementwise.\n" -----f -input: "input" -output: "output" -name: "Acos" -op_type: "Acos" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the arccosine (inverse of cosine) of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Acosh" -op_type: "Acosh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic arccosine of the given input tensor element-wise.\n" -----f -input: "R" -input: "T" -input: "inputs" -output: "outputs" -name: "Adagrad" -op_type: "Adagrad" -attribute { - name: "decay_factor" - f: 0.0 - type: FLOAT -} -attribute { - name: "epsilon" - f: 1e-06 - type: FLOAT -} -attribute { - name: "norm_coefficient" - f: 0.0 - type: FLOAT -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - type: STRINGS -} -attribute { - name: "T-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "inputs-types" - strings: "float" - strings: "double" - type: STRINGS -} -doc_string: "\n Compute one iteration of ADAGRAD, a stochastic gradient based optimization\n algorithm. This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. As you can imagine, ADAGRAD requires\n some parameters:\n \n - The initial learning-rate \"R\".\n - The update count \"T\". That is, the number of training iterations conducted.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A learning-rate decay factor \"decay_factor\".\n - A small constant \"epsilon\" to avoid dividing-by-zero. \n\n At each ADAGRAD iteration, the optimized tensors are moved along a direction\n computed based on their estimated gradient and accumulated squared gradient. Assume\n that only a single tensor \"X\" is updated by this operator. We need the value of \"X\",\n its gradient \"G\", and its accumulated squared gradient \"H\". Therefore, variables in\n this operator\'s input list are sequentially \"R\", \"T\", \"X\", \"G\", and \"H\". Other\n parameters are given as attributes because they are usually constants. Also, the\n corresponding output tensors are the new value of \"X\" (called \"X_new\"), and then\n the new accumulated squared gradient (called \"H_new\"). Those outputs are computed\n from the given inputs following the pseudo code below.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise arithmetic operations with\n numpy-style broadcasting support. The pseudo code to compute those outputs is:\n\n // Compute a scalar learning-rate factor. At the first update of X, T is generally\n // 0 (0-based update index) or 1 (1-based update index).\n r = R / (1 + T * decay_factor);\n\n // Add gradient of 0.5 * norm_coefficient * ||X||_2^2, where ||X||_2 is the 2-norm.\n G_regularized = norm_coefficient * X + G;\n\n // Compute new accumulated squared gradient.\n H_new = H + G_regularized * G_regularized;\n\n // Compute the adaptive part of per-coordinate learning rate. Note that Sqrt(...)\n // computes element-wise square-root.\n H_adaptive = Sqrt(H_new) + epsilon\n\n // Compute the new value of \"X\".\n X_new = X - r * G_regularized / H_adaptive;\n\n If one assign this operators to optimize multiple inputs, for example, \"X_1\" and \"X_2\", the same\n pseudo code may be extended to handle all tensors jointly. More specifically, we can view \"X\" as a\n concatenation of \"X_1\" and \"X_2\" (of course, their gradient and accumulate gradient should\n be concatenated too) and then just reuse the entire pseudo code.\n\n Note that ADAGRAD was first proposed in http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf.\n In that reference paper, this operator is a special case of the Figure 1\'s composite mirror\n descent update.\n" -----f -input: "R" -input: "T" -input: "inputs" -output: "outputs" -name: "Adam" -op_type: "Adam" -attribute { - name: "alpha" - f: 0.9 - type: FLOAT -} -attribute { - name: "beta" - f: 0.999 - type: FLOAT -} -attribute { - name: "epsilon" - f: 1e-06 - type: FLOAT -} -attribute { - name: "norm_coefficient" - f: 0.0 - type: FLOAT -} -attribute { - name: "norm_coefficient_post" - f: 0.0 - type: FLOAT -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - type: STRINGS -} -attribute { - name: "T-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "inputs-types" - strings: "float" - strings: "double" - type: STRINGS -} -doc_string: "\n Compute one iteration of Adam, a stochastic gradient based optimization\n algorithm. This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. First of all, Adam requires\n some parameters:\n \n - The learning-rate \"R\".\n - The update count \"T\". That is, the number of training iterations conducted.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A small constant \"epsilon\" to avoid dividing-by-zero. \n - Two coefficients, \"alpha\" and \"beta\".\n\n At each Adam iteration, the optimized tensors are moved along a direction\n computed based on their exponentially-averaged historical gradient and\n exponentially-averaged historical squared gradient. Assume that only a tensor\n \"X\" is being optimized. The rest of required information is\n \n - the value of \"X\",\n - \"X\"\'s gradient (denoted by \"G\"),\n - \"X\"\'s exponentially-averaged historical gradient (denoted by \"V\"), and\n - \"X\"\'s exponentially-averaged historical squared gradient (denoted by \"H\").\n\n Some of those parameters are passed into this operator as input tensors and others\n are stored as this operator\'s attributes. Specifically, this operator\'s input tensor\n list is [\"R\", \"T\", \"X\", \"G\", \"V\", \"H\"]. That is, \"R\" is the first input, \"T\" is\n the second input, and so on. Other parameters are given as attributes because they\n are constants. Moreover, the corresponding output tensors are \n \n - the new value of \"X\" (called \"X_new\"),\n - the new exponentially-averaged historical gradient (denoted by \"V_new\"), and\n - the new exponentially-averaged historical squared gradient (denoted by \"H_new\").\n\n Those outputs are computed following the pseudo code below.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise arithmetic operations with\n numpy-style broadcasting support. The pseudo code to compute those outputs is:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||_2^2, where ||X||_2 is the 2-norm.\n G_regularized = norm_coefficient * X + G\n\n // Update exponentially-averaged historical gradient.\n V_new = alpha * V + (1 - alpha) * G_regularized\n\n // Update exponentially-averaged historical squared gradient.\n H_new = beta * H + (1 - beta) * G_regularized * G_regularized\n\n // Compute the element-wise square-root of H_new. V_new will be element-wisely\n // divided by H_sqrt for a better update direction.\n H_sqrt = Sqrt(H_new) + epsilon\n\n // Compute learning-rate. Note that \"alpha**T\"/\"beta**T\" is alpha\'s/beta\'s T-th power.\n R_adjusted = T > 0 ? R * Sqrt(1 - beta**T) / (1 - alpha**T) : R\n\n // Compute new value of \"X\".\n X_new = X - R_adjusted * V_new / H_sqrt\n\n // Post-update regularization.\n X_final = (1 - norm_coefficient_post) * X_new \n\n If there are multiple inputs to be optimized, the pseudo code will be applied\n independently to each of them.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Add" -op_type: "Add" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPerforms element-wise binary addition (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "A" -input: "B" -output: "C" -name: "And" -op_type: "And" -attribute { - name: "A-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "B-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `and` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "data" -output: "reduced" -name: "ArgMax" -op_type: "ArgMax" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "select_last_index" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the indices of the max elements of the input tensor\'s element along the \nprovided axis. The resulting tensor has the same rank as the input if keepdims equal 1. \nIf keepdims equal 0, then the resulting tensor have the reduced dimension pruned. \nIf select_last_index is True (default False), the index of the last occurrence of the max \nis selected if the max appears more than once in the input. Otherwise the index of the \nfirst occurrence is selected.\nThe type of the output tensor is integer." -----f -input: "data" -output: "reduced" -name: "ArgMin" -op_type: "ArgMin" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "select_last_index" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the indices of the min elements of the input tensor\'s element along the \nprovided axis. The resulting tensor has the same rank as the input if keepdims equal 1. \nIf keepdims equal 0, then the resulting tensor have the reduced dimension pruned. \nIf select_last_index is True (default False), the index of the last occurrence of the min \nis selected if the min appears more than once in the input. Otherwise the index of the \nfirst occurrence is selected.\nThe type of the output tensor is integer." -----f -input: "X" -input: "Y" -output: "Z" -name: "ArrayFeatureExtractor" -op_type: "ArrayFeatureExtractor" -attribute { - name: "X-types" - strings: "int32" - strings: "string" - strings: "double" - strings: "int64" - strings: "float" - type: STRINGS -} -attribute { - name: "Y-types" - strings: "int64" - type: STRINGS -} -doc_string: "\n Select elements of the input tensor based on the indices passed.
\n The indices are applied to the last axes of the tensor.\n" -----f -input: "input" -output: "output" -name: "Asin" -op_type: "Asin" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the arcsine (inverse of sine) of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Asinh" -op_type: "Asinh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic arcsine of the given input tensor element-wise.\n" -----f -input: "input" -output: "output" -name: "Atan" -op_type: "Atan" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the arctangent (inverse of tangent) of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Atanh" -op_type: "Atanh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic arctangent of the given input tensor element-wise.\n" -----f -input: "X" -output: "Y" -name: "AveragePool" -op_type: "AveragePool" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "ceil_mode" - i: 0 - type: INT -} -attribute { - name: "count_include_pad" - i: 0 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n AveragePool consumes an input tensor X and applies average pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n average pooling consisting of computing the average on all values of a\n subset of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing. The output spatial shape will be following:\n ```\n output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1)\n ```\n or\n ```\n output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1)\n ```\n if ceil_mode is enabled\n\n ```\n * pad_shape[i] is sum of pads along axis i\n ```\n\n `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following:\n ```\n VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - kernel_spatial_shape[i] + 1) / strides_spatial_shape[i])\n SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i])\n ```\n And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`:\n ```\n pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + kernel_spatial_shape[i] - input_spatial_shape[i]\n ```\n The output of each pooling window is divided by the number of elements (exclude pad when attribute count_include_pad is zero).\n " -----f -input: "X" -input: "scale" -input: "B" -input: "mean" -input: "var" -output: "Y" -output: "mean" -output: "var" -output: "saved_mean" -output: "saved_var" -name: "BatchNormalization" -op_type: "BatchNormalization" -attribute { - name: "epsilon" - f: 1e-05 - type: FLOAT -} -attribute { - name: "momentum" - f: 0.9 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "scale-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "mean-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "var-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCarries out batch normalization as described in the paper\nhttps://arxiv.org/abs/1502.03167. Depending on the mode it is being run,\nthere are multiple cases for the number of outputs, which we list below:\n\nOutput case #1: Y, mean, var, saved_mean, saved_var (training mode)\nOutput case #2: Y (test mode)\n\nFor previous (depreciated) non-spatial cases, implementors are suggested\nto flatten the input shape to (N x C*D1*D2 ..*Dn) before a BatchNormalization Op.\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "X" -output: "Y" -name: "Binarizer" -op_type: "Binarizer" -attribute { - name: "threshold" - f: 0.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Maps the values of the input tensor to either 0 or 1, element-wise, based on the outcome of a comparison against a threshold value.\n" -----f -input: "X" -input: "Y" -output: "Z" -name: "BitShift" -op_type: "BitShift" -attribute { - name: "direction" - s: "" - type: STRING -} -attribute { - name: "X-types" - strings: "uint32" - strings: "uint16" - strings: "uint8" - strings: "uint64" - type: STRINGS -} -attribute { - name: "Y-types" - strings: "uint32" - strings: "uint16" - strings: "uint8" - strings: "uint64" - type: STRINGS -} -doc_string: "\nBitwise shift operator performs element-wise operation. For each input element, if the\n attribute \"direction\" is \"RIGHT\", this operator moves its binary representation toward\n the right side so that the input value is effectively decreased. If the attribute \"direction\"\n is \"LEFT\", bits of binary representation moves toward the left side, which results the\n increase of its actual value. The input X is the tensor to be shifted and another input\n Y specifies the amounts of shifting. For example, if \"direction\" is \"Right\", X is [1, 4],\n and S is [1, 1], the corresponding output Z would be [0, 2]. If \"direction\" is \"LEFT\" with\n X=[1, 2] and S=[1, 2], the corresponding output Y would be [2, 8].\n \n Because this operator supports Numpy-style broadcasting, X\'s and Y\'s shapes are\n not necessarily identical.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md)." -----f -input: "input" -output: "output" -name: "Cast" -op_type: "Cast" -attribute { - name: "to" - s: "" - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "float16" - strings: "int32" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nThe operator casts the elements of a given input tensor to a data type\nspecified by the \'to\' argument and returns an output tensor of the same size in\nthe converted type. The \'to\' argument must be one of the data types specified\nin the \'DataType\' enum field in the TensorProto message.\n\nCasting from string tensor in plain (e.g., \"3.14\" and \"1000\") and scientific numeric representations\n(e.g., \"1e-5\" and \"1E8\") to float types is supported. For example, converting string \"100.5\" to an integer may\nresult 100. There are some string literals reserved for special floating-point values;\n\"+INF\" (and \"INF\"), \"-INF\", and \"NaN\" are positive infinity, negative infinity, and not-a-number, respectively.\nAny string which can exactly match \"+INF\" in a case-insensitive way would be mapped to positive infinite. Similarly,\nthis case-insensitive rule is applied to \"INF\" and \"NaN\". When casting from numeric tensors\nto string tensors, plain floating-point representation (such as \"314.15926\") would be used. \nConverting non-numerical-literal string such as \"Hello World!\" is an undefined behavior. Cases \nof converting string representing floating-point arithmetic value, such as \"2.718\", to INT is an undefined behavior.\n\nConversion from a numerical type to any numerical type is always allowed.\nUser must be aware of precision loss and value change caused by range difference between two types.\nFor example, a 64-bit float 3.1415926459 may be round to a 32-bit float 3.141592. Similarly, converting\nan integer 36 to Boolean may produce 1 because we truncate bits which can\'t be stored in the targeted type.\n" -----f -input: "X" -output: "Y" -name: "CastMap" -op_type: "CastMap" -attribute { - name: "cast_to" - s: "TO_FLOAT" - type: STRING -} -attribute { - name: "map_form" - s: "DENSE" - type: STRING -} -attribute { - name: "max_map" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "map(int64,string" - strings: "map(int64,float" - type: STRINGS -} -doc_string: "\n Converts a map to a tensor.
The map key must be an int64 and the values will be ordered\n in ascending order based on this key.
The operator supports dense packing or sparse packing.\n If using sparse packing, the key cannot exceed the max_map-1 value.\n" -----f -input: "X" -output: "Y" -name: "CategoryMapper" -op_type: "CategoryMapper" -attribute { - name: "cats_int64s" - s: "" - type: INTS -} -attribute { - name: "cats_strings" - s: "" - type: STRINGS -} -attribute { - name: "default_int64" - i: -1 - type: INT -} -attribute { - name: "default_string" - s: "_Unused" - type: STRING -} -attribute { - name: "X-types" - strings: "string" - strings: "int64" - type: STRINGS -} -doc_string: "\n Converts strings to integers and vice versa.
\n Two sequences of equal length are used to map between integers and strings,\n with strings and integers at the same index detailing the mapping.
\n Each operator converts either integers to strings or strings to integers, depending \n on which default value attribute is provided. Only one default value attribute\n should be defined.
\n If the string default value is set, it will convert integers to strings.\n If the int default value is set, it will convert strings to integers.\n" -----f -input: "X" -output: "Y" -name: "Ceil" -op_type: "Ceil" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCeil takes one input data (Tensor) and produces one output data\n(Tensor) where the ceil is, y = ceil(x), is applied to\nthe tensor elementwise.\n" -----f -input: "X" -output: "Y" -name: "Celu" -op_type: "Celu" -attribute { - name: "alpha" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - type: STRINGS -} -doc_string: "\nContinuously Differentiable Exponential Linear Units:\nPerform the linear unit element-wise on the input tensor X\nusing formula: \n\n```\nmax(0,x) + min(0,alpha*(exp(x/alpha)-1))\n```\n" -----f -input: "input" -input: "min" -input: "max" -output: "output" -name: "Clip" -op_type: "Clip" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "min-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "max-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nClip operator limits the given input within an interval. The interval is\nspecified by the inputs \'min\' and \'max\'. They default to\nnumeric_limits::lowest() and numeric_limits::max(), respectively.\n" -----f -input: "input" -input: "condition" -output: "output" -name: "Compress" -op_type: "Compress" -attribute { - name: "axis" - s: "" - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "condition-types" - strings: "bool" - type: STRINGS -} -doc_string: "\n Selects slices from an input tensor along a given axis where condition evaluates to True for each axis index.\n In case axis is not provided, input is flattened before elements are selected.\n Compress behaves like numpy.compress: https://docs.scipy.org/doc/numpy/reference/generated/numpy.compress.html\n " -----f -input: "inputs" -output: "concat_result" -name: "Concat" -op_type: "Concat" -attribute { - name: "axis" - s: "" - type: INT -} -attribute { - name: "inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "Concatenate a list of tensors into a single tensor. All input tensors must have the same shape, except for the dimension size of the axis to concatenate on." -----f -input: "input_sequence" -output: "concat_result" -name: "ConcatFromSequence" -op_type: "ConcatFromSequence" -attribute { - name: "axis" - s: "" - type: INT -} -attribute { - name: "new_axis" - i: 0 - type: INT -} -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -doc_string: "\nConcatenate a sequence of tensors into a single tensor.\nAll input tensors must have the same shape, except for the dimension size of the axis to concatenate on.\nBy default \'new_axis\' is 0, the behavior is similar to numpy.concatenate.\nWhen \'new_axis\' is 1, the behavior is similar to numpy.stack.\n" -----f -output: "output" -name: "Constant" -op_type: "Constant" -attribute { - name: "sparse_value" - s: "" - type: SPARSE_TENSOR -} -attribute { - name: "value" - s: "" - type: TENSOR -} -attribute { - name: "value_float" - s: "" - type: FLOAT -} -attribute { - name: "value_floats" - s: "" - type: FLOATS -} -attribute { - name: "value_int" - s: "" - type: INT -} -attribute { - name: "value_ints" - s: "" - type: INTS -} -attribute { - name: "value_string" - s: "" - type: STRING -} -attribute { - name: "value_strings" - s: "" - type: STRINGS -} -doc_string: "\nThis operator produces a constant tensor. Exactly one of the provided attributes, either value, sparse_value,\nor value_* must be specified.\n" -----f -input: "input" -output: "output" -name: "ConstantOfShape" -op_type: "ConstantOfShape" -attribute { - name: "value" - s: "" - type: TENSOR -} -attribute { - name: "input-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nGenerate a tensor with given value and shape.\n" -----f -input: "X" -input: "W" -input: "B" -output: "Y" -name: "Conv" -op_type: "Conv" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "group" - i: 1 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe convolution operator consumes an input tensor and a filter, and\ncomputes the output." -----f -input: "x" -input: "w" -input: "x_zero_point" -input: "w_zero_point" -output: "y" -name: "ConvInteger" -op_type: "ConvInteger" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "group" - i: 1 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "x-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "w-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "x_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "w_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nThe integer convolution operator consumes an input tensor, its zero-point, a filter, and its zero-point,\nand computes the output. The production MUST never overflow. The accumulation may overflow if and only if in 32 bits.\n" -----f -input: "X" -input: "W" -input: "B" -output: "Y" -name: "ConvTranspose" -op_type: "ConvTranspose" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "group" - i: 1 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "output_padding" - s: "" - type: INTS -} -attribute { - name: "output_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe convolution transpose operator consumes an input tensor and a filter,\nand computes the output.\n\nIf the pads parameter is provided the shape of the output is calculated via the following equation:\n\n output_shape[i] = stride[i] * (input_size[i] - 1) + output_padding[i] + ((kernel_shape[i] - 1) * dilations[i] + 1) - pads[start_i] - pads[end_i]\n\noutput_shape can also be explicitly specified in which case pads values are auto generated using these equations:\n\n total_padding[i] = stride[i] * (input_size[i] - 1) + output_padding[i] + ((kernel_shape[i] - 1) * dilations[i] + 1) - output_shape[i]\n If (auto_pads != SAME_UPPER): pads[start_i] = total_padding[i]/2; pads[end_i] = total_padding[i] - (total_padding[i]/2)\n Else: pads[start_i] = total_padding[i] - (total_padding[i]/2); pads[end_i] = (total_padding[i]/2).\n\n " -----f -input: "input" -output: "output" -name: "Cos" -op_type: "Cos" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the cosine of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Cosh" -op_type: "Cosh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic cosine of the given input tensor element-wise.\n" -----f -input: "x" -input: "axis" -output: "y" -name: "CumSum" -op_type: "CumSum" -attribute { - name: "exclusive" - i: 0 - type: INT -} -attribute { - name: "reverse" - i: 0 - type: INT -} -attribute { - name: "x-types" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "axis-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nPerforms cumulative sum of the input elements along the given axis.\nBy default, it will do the sum inclusively meaning the first element is copied as is.\nThrough an `exclusive` attribute, this behavior can change to exclude the first element.\nIt can also perform summation in the opposite direction of the axis. For that, set `reverse` attribute to 1.\n\nExample:\n```\ninput_x = [1, 2, 3]\naxis=0\noutput = [1, 3, 6]\nexclusive=1\noutput = [0, 1, 3]\nexclusive=0\nreverse=1\noutput = [6, 5, 3]\nexclusive=1\nreverse=1\noutput = [5, 3, 0]\n```\n " -----f -input: "input" -output: "output" -name: "DepthToSpace" -op_type: "DepthToSpace" -attribute { - name: "blocksize" - s: "" - type: INT -} -attribute { - name: "mode" - s: "DCR" - type: STRING -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "DepthToSpace rearranges (permutes) data from depth into blocks of spatial data.\nThis is the reverse transformation of SpaceToDepth. More specifically, this op outputs a copy of\nthe input tensor where values from the depth dimension are moved in spatial blocks to the height\nand width dimensions. By default, `mode` = `DCR`.\nIn the DCR mode, elements along the depth dimension from the input tensor are rearranged in the\nfollowing order: depth, column, and then row. The output y is computed from the input x as below:\n\nb, c, h, w = x.shape\n\ntmp = np.reshape(x, [b, blocksize, blocksize, c // (blocksize**2), h, w])\n\ntmp = np.transpose(tmp, [0, 3, 4, 1, 5, 2])\n\ny = np.reshape(tmp, [b, c // (blocksize**2), h * blocksize, w * blocksize])\n\n\nIn the CRD mode, elements along the depth dimension from the input tensor are rearranged in the\nfollowing order: column, row, and the depth. The output y is computed from the input x as below:\n\nb, c, h, w = x.shape\n\ntmp = np.reshape(x, [b, c // (blocksize ** 2), blocksize, blocksize, h, w])\n\ntmp = np.transpose(tmp, [0, 1, 4, 2, 5, 3])\n\ny = np.reshape(tmp, [b, c // (blocksize ** 2), h * blocksize, w * blocksize])\n\n" -----f -input: "x" -input: "x_scale" -input: "x_zero_point" -output: "y" -name: "DequantizeLinear" -op_type: "DequantizeLinear" -attribute { - name: "x-types" - strings: "int32" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "x_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "x_zero_point-types" - strings: "int32" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nThe linear dequantization operator. It consumes a quantized tensor, a scale, a zero point to compute the full precision tensor.\nThe dequantization formula is y = (x - x_zero_point) * x_scale. \'x_scale\' and \'x_zero_point\' must have same shape.\n\'x_zero_point\' and \'x\' must have same type. \'x\' and \'y\' must have same shape. In the case of dequantizing int32,\nthere\'s no zero point (zero point is supposed to be 0).\n" -----f -input: "X" -output: "Y" -name: "Det" -op_type: "Det" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nDet calculates determinant of a square matrix or batches of square matrices.\nDet takes one input tensor of shape `[*, M, M]`, where `*` is zero or more batch dimensions,\nand the inner-most 2 dimensions form square matrices.\nThe output is a tensor of shape `[*]`, containing the determinants of all input submatrices.\ne.g., When the input is 2-D, the output is a scalar(shape is empty: `[]`).\n" -----f -input: "X" -output: "Y" -name: "DictVectorizer" -op_type: "DictVectorizer" -attribute { - name: "int64_vocabulary" - s: "" - type: INTS -} -attribute { - name: "string_vocabulary" - s: "" - type: STRINGS -} -attribute { - name: "X-types" - strings: "map(int64,float" - strings: "map(int64,string" - strings: "map(string,int64" - strings: "map(string,float" - strings: "map(string,double" - strings: "map(int64,double" - type: STRINGS -} -doc_string: "\n Uses an index mapping to convert a dictionary to an array.
\n Given a dictionary, each key is looked up in the vocabulary attribute corresponding to\n the key type. The index into the vocabulary array at which the key is found is then\n used to index the output 1-D tensor \'Y\' and insert into it the value found in the dictionary \'X\'.
\n The key type of the input map must correspond to the element type of the defined vocabulary attribute.\n Therefore, the output array will be equal in length to the index mapping vector parameter.\n All keys in the input dictionary must be present in the index mapping vector.\n For each item in the input dictionary, insert its value in the output array.\n Any keys not present in the input dictionary, will be zero in the output array.
\n For example: if the ``string_vocabulary`` parameter is set to ``[\"a\", \"c\", \"b\", \"z\"]``,\n then an input of ``{\"a\": 4, \"c\": 8}`` will produce an output of ``[4, 8, 0, 0]``.\n " -----f -input: "A" -input: "B" -output: "C" -name: "Div" -op_type: "Div" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPerforms element-wise binary division (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "data" -input: "ratio" -input: "training_mode" -output: "output" -output: "mask" -name: "Dropout" -op_type: "Dropout" -attribute { - name: "seed" - s: "" - type: INT -} -attribute { - name: "data-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "ratio-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "training_mode-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nDropout takes an input floating-point tensor, an optional input ratio (floating-point scalar) and an optional input training_mode (boolean scalar). It produces two tensor outputs,\noutput (floating-point tensor) and mask (optional `Tensor`). If `training_mode` is true then the output Y will be a random dropout;\nNote that this Dropout scales the masked input data by the following equation, so to convert the trained model into inference mode,\nthe user can simply not pass `training_mode` input or set it to false.\n```\noutput = scale * data * mask,\n```\nwhere\n```\nscale = 1. / (1. - ratio).\n```\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "x" -output: "y" -output: "y_scale" -output: "y_zero_point" -name: "DynamicQuantizeLinear" -op_type: "DynamicQuantizeLinear" -attribute { - name: "x-types" - strings: "float" - type: STRINGS -} -doc_string: "\nA Function to fuse calculation for Scale, Zero Point and FP32->8Bit convertion of FP32 Input data.\nOutputs Scale, ZeroPoint and Quantized Input for a given FP32 Input.\nScale is calculated as:\n```\n y_scale = (max(x) - min(x))/(qmax - qmin)\n * where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8\n * data range is adjusted to include 0.\n```\nZero point is calculated as:\n```\nintermediate_zero_point = qmin - min(x)/y_scale\ny_zero_point = cast(round(saturate(itermediate_zero_point)))\n* where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8\n* for saturation, it saturates to [0, 255] if it\'s uint8, or [-127, 127] if it\'s int8. Right now only uint8 is supported.\n* rounding to nearest ties to even.\n```\nData quantization formula is:\n```\ny = saturate (round (x / y_scale) + y_zero_point)\n* for saturation, it saturates to [0, 255] if it\'s uint8, or [-127, 127] if it\'s int8. Right now only uint8 is supported.\n* rounding to nearest ties to even.\n```\n" -----f -input: "Inputs" -output: "Output" -name: "Einsum" -op_type: "Einsum" -attribute { - name: "equation" - s: "" - type: STRING -} -attribute { - name: "Inputs-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nAn einsum of the form ```term1, term2 -> output-term``` produces an output tensor using the following equation\n\n```output[output-term] = reduce-sum( input1[term1] * input2[term] )```\n\nwhere the reduce-sum performs a summation over all the indices occurring in in the input terms (term1, term2)\nthat do not occur in the output-term.\n\nThe Einsum operator evaluates algebraic tensor operations on a sequence of tensors, using the Einstein summation\nconvention. The equation string contains a comma-separated sequence of lower case letters. Each term corresponds to\nan operand tensor, and the characters within the terms correspond to operands dimensions.\n\nThis sequence may be followed by \"->\" to separate the left and right hand side of the equation.\nIf the equation contains \"->\" followed by the right-hand side, the explicit (not classical) form of the Einstein\nsummation is performed, and the right-hand side indices indicate output tensor dimensions. In other cases,\noutput indices are (implicitly) set to the alphabetically sorted sequence of indices appearing exactly once in the\nequation.\n\nWhen a dimension character is repeated in the left-hand side, it represents summation along the dimension.\n\nThe equation may contain ellipsis (\"...\") to enable broadcasting. Ellipsis must indicate a fixed number of dimensions.\nSpecifically, every occurrence of ellipsis in the equation must represent the same number of dimensions.\nThe right-hand side may contain exactly one ellipsis. In implicit mode, the ellipsis dimensions are set to the\nbeginning of the output. The equation string may contain space (U+0020) character.\n" -----f -input: "X" -output: "Y" -name: "Elu" -op_type: "Elu" -attribute { - name: "alpha" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nElu takes one input data (Tensor) and produces one output data\n(Tensor) where the function `f(x) = alpha * (exp(x) - 1.) for x <\n0`, `f(x) = x for x >= 0`., is applied to the tensor elementwise.\n\n" -----f -input: "A" -input: "B" -output: "C" -name: "Equal" -op_type: "Equal" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "input" -output: "output" -name: "Erf" -op_type: "Erf" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the error function of the given input tensor element-wise.\n" -----f -input: "input" -output: "output" -name: "Exp" -op_type: "Exp" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the exponential of the given input tensor, element-wise.\n" -----f -input: "input" -input: "shape" -output: "output" -name: "Expand" -op_type: "Expand" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "shape-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nBroadcast the input tensor following the given shape and the broadcast rule.\nThe broadcast rule is similar to numpy.array(input) * numpy.ones(shape):\nDimensions are right alignment;\nTwo corresponding dimension must have the same value, or one of them is equal to 1.\nAlso, this operator is similar to numpy.broadcast_to(input, shape),\nbut the major difference is numpy.broadcast_to() does not allow shape to be smaller than input.size().\nIt is possible that the output.shape is not equal to shape, when some dimensions in shape is equal to 1,\nor the shape.ndim < input.shape.ndim.\n" -----f -input: "input" -output: "output" -name: "EyeLike" -op_type: "EyeLike" -attribute { - name: "dtype" - s: "" - type: INT -} -attribute { - name: "k" - i: 0 - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "float16" - strings: "int32" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGenerate a 2D tensor (matrix) with ones on the diagonal and zeros everywhere else. Only 2D\ntensors are supported, i.e. input T1 must be of rank 2. The shape of the output tensor is the\nsame as the input tensor. The data type can be specified by the \'dtype\' argument. If\n\'dtype\' is not specified, then the type of input tensor is used. By default, the main diagonal\nis populated with ones, but attribute \'k\' can be used to populate upper or lower diagonals.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message and be valid as an output type.\n" -----f -input: "X" -output: "Y" -name: "FeatureVectorizer" -op_type: "FeatureVectorizer" -attribute { - name: "inputdimensions" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Concatenates input tensors into one continuous output.
\n All input shapes are 2-D and are concatenated along the second dimention. 1-D tensors are treated as [1,C].\n Inputs are copied to the output maintaining the order of the input arguments.
\n All inputs must be integers or floats, while the output will be all floating point values.\n" -----f -input: "input" -output: "output" -name: "Flatten" -op_type: "Flatten" -attribute { - name: "axis" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nFlattens the input tensor into a 2D matrix. If input tensor has shape\n(d_0, d_1, ... d_n) then the output will have shape\n(d_0 X d_1 ... d_(axis-1), d_axis X d_(axis+1) ... X dn).\n" -----f -input: "X" -output: "Y" -name: "Floor" -op_type: "Floor" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nFloor takes one input data (Tensor) and produces one output data\n(Tensor) where the floor is, y = floor(x), is applied to\nthe tensor elementwise.\n" -----f -input: "X" -input: "W" -input: "R" -input: "B" -input: "sequence_lens" -input: "initial_h" -output: "Y" -output: "Y_h" -name: "GRU" -op_type: "GRU" -attribute { - name: "activation_alpha" - s: "" - type: FLOATS -} -attribute { - name: "activation_beta" - s: "" - type: FLOATS -} -attribute { - name: "activations" - s: "" - type: STRINGS -} -attribute { - name: "clip" - s: "" - type: FLOAT -} -attribute { - name: "direction" - s: "forward" - type: STRING -} -attribute { - name: "hidden_size" - s: "" - type: INT -} -attribute { - name: "linear_before_reset" - i: 0 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "sequence_lens-types" - strings: "int32" - type: STRINGS -} -attribute { - name: "initial_h-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nComputes an one-layer GRU. This operator is usually supported via some custom\nimplementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`z` - update gate\n\n`r` - reset gate\n\n`h` - hidden gate\n\n`t` - time step (t-1 means previous time step)\n\n`W[zrh]` - W parameter weight matrix for update, reset, and hidden gates\n\n`R[zrh]` - R recurrence weight matrix for update, reset, and hidden gates\n\n`Wb[zrh]` - W bias vectors for update, reset, and hidden gates\n\n`Rb[zrh]` - R bias vectors for update, reset, and hidden gates\n\n`WB[zrh]` - W parameter weight matrix for backward update, reset, and hidden gates\n\n`RB[zrh]` - R recurrence weight matrix for backward update, reset, and hidden gates\n\n`WBb[zrh]` - W bias vectors for backward update, reset, and hidden gates\n\n`RBb[zrh]` - R bias vectors for backward update, reset, and hidden gates\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Sigmoid, g=Tanh):\n\n - zt = f(Xt*(Wz^T) + Ht-1*(Rz^T) + Wbz + Rbz)\n\n - rt = f(Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr)\n\n - ht = g(Xt*(Wh^T) + (rt (.) Ht-1)*(Rh^T) + Rbh + Wbh) # default, when linear_before_reset = 0\n\n - ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) + Wbh) # when linear_before_reset != 0\n\n - Ht = (1 - zt) (.) ht + zt (.) Ht-1\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "data" -input: "indices" -output: "output" -name: "Gather" -op_type: "Gather" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nGiven `data` tensor of rank r >= 1, and `indices` tensor of rank q, gather\nentries of the axis dimension of `data` (by default outer-most one as axis=0) indexed by `indices`, and concatenates\nthem in an output tensor of rank q + (r - 1).\n\naxis = 0 :\n\nLet\nk = indices[i_{0}, ..., i_{q-1}]\nThen\noutput[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[k , j_{0}, ..., j_{r-2}]\n\n```\n data = [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ]\n indices = [\n [0, 1],\n [1, 2],\n ]\n output = [\n [\n [1.0, 1.2],\n [2.3, 3.4],\n ],\n [\n [2.3, 3.4],\n [4.5, 5.7],\n ],\n ]\n```\naxis = 1 :\n\nLet\nk = indices[i_{0}, ..., i_{q-1}]\nThen\noutput[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[j_{0}, k, j_{1}, ..., j_{r-2}]\n\n```\n data = [\n [1.0, 1.2, 1.9],\n [2.3, 3.4, 3.9],\n [4.5, 5.7, 5.9],\n ]\n indices = [\n [0, 2],\n ]\n axis = 1,\n output = [\n [\n [1.0, 1.9],\n [2.3, 3.9],\n [4.5, 5.9],\n ],\n ]\n```\n" -----f -input: "data" -input: "indices" -output: "output" -name: "GatherElements" -op_type: "GatherElements" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n\nGatherElements takes two inputs `data` and `indices` of the same rank r >= 1\nand an optional attribute `axis` that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). It is an indexing operation\nthat produces its output by indexing into the input data tensor at index\npositions determined by elements of the `indices` tensor.\nIts output shape is the same as the shape of `indices` and consists of one value\n(gathered from the `data`) for each element in `indices`.\n\nFor instance, in the 3-D case (r = 3), the output produced is determined\nby the following equations: \n```\n out[i][j][k] = input[index[i][j][k]][j][k] if axis = 0,\n out[i][j][k] = input[i][index[i][j][k]][k] if axis = 1,\n out[i][j][k] = input[i][j][index[i][j][k]] if axis = 2,\n```\n\nThis operator is also the inverse of ScatterElements. It is similar to Torch\'s gather operation.\n\nExample 1:\n```\n data = [\n [1, 2],\n [3, 4],\n ]\n indices = [\n [0, 0],\n [1, 0],\n ]\n axis = 1\n output = [\n [\n [1, 1],\n [4, 3],\n ],\n ]\n```\nExample 2:\n```\n data = [\n [1, 2, 3],\n [4, 5, 6],\n [7, 8, 9],\n ]\n indices = [\n [1, 2, 0],\n [2, 0, 0],\n ]\n axis = 0\n output = [\n [\n [4, 8, 3],\n [7, 2, 3],\n ],\n ]\n```\n" -----f -input: "data" -input: "indices" -output: "output" -name: "GatherND" -op_type: "GatherND" -attribute { - name: "batch_dims" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nGiven `data` tensor of rank `r` >= 1, `indices` tensor of rank `q` >= 1, and `batch_dims` integer `b`, this operator gathers \nslices of `data` into an output tensor of rank `q + r - indices_shape[-1] - 1 - b`.\n\n`indices` is an q-dimensional integer tensor, best thought of as a `(q-1)`-dimensional tensor of index-tuples into `data`, \nwhere each element defines a slice of `data`\n\n`batch_dims` (denoted as `b`) is an integer indicating the number of batch dimensions, i.e the leading `b` number of dimensions of \n`data` tensor and `indices` are representing the batches, and the gather starts from the `b+1` dimension. \n\nSome salient points about the inputs\' rank and shape:\n \n1) r >= 1 and q >= 1 are to be honored. There is no dependency condition to be met between ranks `r` and `q`\n\n2) The first `b` dimensions of the shape of `indices` tensor and `data` tensor must be equal.\n\n3) b < min(q, r) is to be honored.\n\n4) The `indices_shape[-1]` should have a value between 1 (inclusive) and rank `r-b` (inclusive) \n\n5) All values in `indices` are expected to be within bounds [-s, s-1] along axis of size `s` (i.e.) `-data_shape[i] <= indices[...,i] <= data_shape[i] - 1`.\n It is an error if any of the index values are out of bounds.\n\nThe output is computed as follows:\n\nThe output tensor is obtained by mapping each index-tuple in the `indices` tensor to the corresponding slice of the input `data`.\n \n1) If `indices_shape[-1] > r-b` => error condition\n\n2) If `indices_shape[-1] == r-b`, since the rank of `indices` is `q`, `indices` can be thought of as `N` `(q-b-1)`-dimensional tensors\n containing 1-D tensors of dimension `r-b`, where `N` is an integer equals to the product of 1 and all the elements in the batch dimensions \n of the indices_shape. Let us think of each such `r-b` ranked tensor as `indices_slice`. Each *scalar value* corresponding to `data[0:b-1,indices_slice]` \n is filled into the corresponding location of the `(q-b-1)`-dimensional tensor to form the `output` tensor (Example 1 below)\n\n3) If `indices_shape[-1] < r-b`, since the rank of `indices` is `q`, `indices` can be thought of as `N` `(q-b-1)`-dimensional tensor\n containing 1-D tensors of dimension `< r-b`. Let us think of each such tensors as `indices_slice`. Each *tensor slice* corresponding \n to `data[0:b-1, indices_slice , :]` is filled into the corresponding location of the `(q-b-1)`-dimensional tensor \n to form the `output` tensor (Examples 2, 3, 4 and 5 below)\n\nThis operator is the inverse of `ScatterND`.\n\n`Example 1`\n\n batch_dims = 0\n\n data = [[0,1],[2,3]] # data_shape = [2, 2]\n\n indices = [[0,0],[1,1]] # indices_shape = [2, 2]\n\n output = [0,3] # output_shape = [2]\n\n`Example 2`\n\n batch_dims = 0\n\n data = [[0,1],[2,3]] # data_shape = [2, 2]\n\n indices = [[1],[0]] # indices_shape = [2, 1]\n\n output = [[2,3],[0,1]] # output_shape = [2, 2]\n\n`Example 3`\n\n batch_dims = 0\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[0,1],[1,0]] # indices_shape = [2, 2]\n\n output = [[2,3],[4,5]] # output_shape = [2, 2] \n\n`Example 4`\n\n batch_dims = 0\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[[0,1]],[[1,0]]] # indices_shape = [2, 1, 2]\n\n output = [[[2,3]],[[4,5]]] # output_shape = [2, 1, 2] \n\n`Example 5`\n\n batch_dims = 1\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[1],[0]] # indices_shape = [2, 1]\n\n output = [[2,3],[4,5]] # output_shape = [2, 2] \n\n\n" -----f -input: "A" -input: "B" -input: "C" -output: "Y" -name: "Gemm" -op_type: "Gemm" -attribute { - name: "alpha" - f: 1.0 - type: FLOAT -} -attribute { - name: "beta" - f: 1.0 - type: FLOAT -} -attribute { - name: "transA" - i: 0 - type: INT -} -attribute { - name: "transB" - i: 0 - type: INT -} -attribute { - name: "A-types" - strings: "int32" - strings: "float16" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "int32" - strings: "float16" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "C-types" - strings: "int32" - strings: "float16" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "General Matrix multiplication:\nhttps://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms#Level_3\n\nA\' = transpose(A) if transA else A\n\nB\' = transpose(B) if transB else B\n\nCompute Y = alpha * A\' * B\' + beta * C, where input tensor A has shape (M, K) or (K, M),\ninput tensor B has shape (K, N) or (N, K), input tensor C is broadcastable to shape (M, N),\nand output tensor Y has shape (M, N). A will be transposed before doing the\ncomputation if attribute transA is non-zero, same for B and transB.\nThis operator supports **unidirectional broadcasting** (tensor C should be unidirectional broadcastable to tensor A * B); for more details please check [the doc](Broadcasting.md).\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "X" -output: "Y" -name: "GlobalAveragePool" -op_type: "GlobalAveragePool" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n GlobalAveragePool consumes an input tensor X and applies average pooling across\n the values in the same channel. This is equivalent to AveragePool with kernel size\n equal to the spatial dimension of input tensor." -----f -input: "X" -output: "Y" -name: "GlobalLpPool" -op_type: "GlobalLpPool" -attribute { - name: "p" - i: 2 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n GlobalLpPool consumes an input tensor X and applies lp pool pooling across\n the values in the same channel. This is equivalent to LpPool with kernel size\n equal to the spatial dimension of input tensor." -----f -input: "X" -output: "Y" -name: "GlobalMaxPool" -op_type: "GlobalMaxPool" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n GlobalMaxPool consumes an input tensor X and applies max pooling across\n the values in the same channel. This is equivalent to MaxPool with kernel size\n equal to the spatial dimension of input tensor." -----f -input: "Inputs" -output: "Outputs" -name: "Gradient" -op_type: "Gradient" -attribute { - name: "xs" - s: "" - type: STRINGS -} -attribute { - name: "y" - s: "" - type: STRING -} -attribute { - name: "zs" - s: "" - type: STRINGS -} -attribute { - name: "Inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGradient operator computes the partial derivatives of a specific tensor w.r.t.\nsome other tensors. This operator is widely used in gradient-based training\nalgorithms. To illustrate its use, let\'s consider a computation graph,\n\n```\nX -----.\n |\n v\nW --> Conv --> H --> Gemm --> Y\n ^\n |\n Z\n```\n\n, where W and Z are trainable tensors. Note that operators\' attributes are\nomitted for the sake of simplicity. Let dY/dW (dY/dZ) be the gradient of\nY with respect to W (Z). The user can compute gradient by inserting Gradient\noperator to form another graph shown below.\n\n```\nW --> Conv --> H --> Gemm --> Y\n| ^ ^\n| | |\n| X Z\n| | |\n| | .----------\'\n| | | (W/Z/X is the 1st/2nd/3rd input of Gradient as shown in\n| | | \"xs\" followed by \"zs\")\n| v v\n\'---> Gradient(xs=[\"W\", \"Z\"], zs=[\"X\"], y=\"Y\")\n | |\n | \'-----------------------------------> dY/dW (1st output of Gradient)\n |\n \'---------------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nBy definition, the tensor \"y\" is a function of independent variables in \"xs\"\nand \"zs\". Since we only compute the gradient of \"y\" w.r.t. the differentiable\nvariables in \"xs\", this Gradient only outputs dY/dW and dY/dZ. Note that \"H\"\ncannot appear in \"xs\" and \"zs\". The reason is that \"H\" can be determined by\ntensors \"W\" and \"X\" and therefore \"H\" is not an independent variable.\n\nAll outputs are optional. If needed, for example, user can assign an empty\nstring to the 1st output name of that Gradient to skip the generation of dY/dW.\nNote that the concept of optional outputs can also be found in ONNX\'s RNN, GRU,\nand LSTM.\n\nGradient operator can compute derivative against intermediate tensors. For\nexample, the gradient of Y with respect to H can be done via\n\n```\nW --> Conv --> H --> Gemm --> Y\n ^ | ^\n | | |\n X | Z\n .-------\' |\n | .----------\'\n | | (H/Z is the 1st/2nd input of Gradient as shown in \"xs\")\n v v\n Gradient(xs=[\"H\", \"Z\"], y=\"Y\")\n | |\n | \'-----------------------------------> dY/dH (1st output of Gradient)\n |\n \'---------------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nIt is possible to represent high-order differentiation using Gradient operators.\nFor example, given the following linear model:\n\n```\nW --> Gemm --> Y --> Loss --> O\n ^ ^\n | |\n X L\n```\n\nTo compute the 2nd order derivative of O with respect to W (denoted by\nd^2O/dW^2), one can do\n\n```\nW --> Gemm --> Y --> Loss --> O\n| ^ ^\n| | |\n| X .------------L\n| | | |\n| | | v\n+------+-+> Gradient(xs=[\"X\", \"W\"], zs=[\"L\"], y=\"O\") ---> dO/dX (1st output of Gradient)\n| | | |\n| | | \'---> dO/dW (2nd output of Gradient)\n| v v\n\'---> Gradient(xs=[\"X\", \"W\"], zs=[\"L\"], y=\"dO/dW\") ---> d(dO/dW)dX (1st output of\n | Gradient)\n |\n |\n \'---> d^2O/dW^2 (2nd output of Gradient)\n```\n\nThe tensors named in attributes \"xs\", \"zs\", and \"y\" define the differentiated\ncomputation graph, and the inputs to Gradient node define the values at\nwhich the gradient is computed. We can feed different tensors to the identified\ngraph. For example, one can compute the gradient of Y with respect to H at \na specific value of H, H_1, by providing that value as an input to the Gradient\nnode.\n\n```\nW --> Conv --> H --> Gemm --> Y\n ^ ^\n | |\n X Z\n\n Z_1 (2nd input of Gradient)\n |\n v\nH_1 --> Gradient(xs=[\"H\", \"Z\"], y=\"Y\") ---> dY/dH when H = H_1 and Y = Y_1.\n |\n \'------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nWhen the inputs of Gradient are the tensors named in \"xs\" and \"zs\", the\ncomputation can be optimized. More specifically, intermediate variables in\nforward pass can be reused if the gradient is computed via reverse-mode\nauto-differentiation.\n\n" -----f -input: "Inputs" -output: "Outputs" -name: "GraphCall" -op_type: "GraphCall" -attribute { - name: "graph_name" - s: "" - type: STRING -} -attribute { - name: "Inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nThe GraphCall operator invokes a graph inside TrainingInfoProto\'s\nalgorithm field. The GraphCall inputs and outputs are bound to those of\ninvoked graph by position. If a graph input has an initializer, that input\nis considered optional. All graph outputs are optional.\n\nBelow Python syntax is used for describing dictionary and list.\n\nAssume that ModelProto\'s graph field has\n- name: \"MyInferenceGraph\"\n- input: [\"X\", \"W\", \"Z\"]\n- initializer: [W]\n- output: [\"Y\"]\n\nas visualized below for inference.\n\n```\nX -----.\n |\n v\nW --> Conv --> H --> Gemm --> Y\n ^\n |\n Z\n```\n\nAssume that the training algorithm contains\n\n- inputs: [\"X_1\", \"Z_1\", \"C\"]\n- initializer: [T]\n- outputs: [\"W_new\"]\n\nwith a dictionary\n\n- update_binding: {\"W\": \"W_new\", \"T\": \"T_new\"}\n\nInside the training algorithm graph, one can invoke the inference\ngraph via adding a GraphCall node with\n\n- inputs: [\"X_1\", \"W\", Z_1\"]\n- outputs: [\"Y_1\"]\n- an attribute graph_name=\"MyInferenceGraph\",\n\nThe initializers, \"W\" and \"T\" in this case, in update_binding\nare considered globally-visible and mutable variables, which\ncan be used as inputs of operators in the training graph.\n\nAn example training algorithm graph may look like\n\n```\n.-------- W (a global and mutable variable from\n| | the inference graph)\n| |\n| .-----\'-----------.\n| | |\n| | v\n| | .-- X_1 --> GraphCall(graph_name=\"MyInferenceGraph\")\n| | | | |\n| | | | |\n| | | Z_1 -----\' |\n| | | | V\n| | | | Y_1 ---> Loss ---> O\n| | | | ^\n| | | | |\n| | `--. | C\n| | | | |\n| | | | .----------------\'\n| | | | |\n| | v v v\n| `--> Gradient(xs=[\"W\"], zs=[\"X_1\", \"Z_1\", \"C\"], y=\"O\")\n| |\n| v\n| dO_dW (gradient of W) 1 (a scalar one)\n| | |\n| V v\n| Div <--- T ------------> Add ---> T_new\n| | (T is the number of training iterations.\n| | T is also globally visible and mutable.)\n| v\n`-----> Sub ----> W_new\n```\n\nwhere Loss is a dummy node which computes the minimized objective function.\n\nThe variable \"W\" is an optional input in the called graph.\nIf the user omits it, the input list of GraphCall becomes [\"X_1\", \"\", \"Z_1\"].\nIn this case, from the view of computation graph, the Conv operator invoked by\nGraphCall\'s may be still connected the global \"W\" variable and therefore the\nstructure of the computation graph is unchanged.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Greater" -op_type: "Greater" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `greater` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "A" -input: "B" -output: "C" -name: "GreaterOrEqual" -op_type: "GreaterOrEqual" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `greater_equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Y" -name: "HardSigmoid" -op_type: "HardSigmoid" -attribute { - name: "alpha" - f: 0.2 - type: FLOAT -} -attribute { - name: "beta" - f: 0.5 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nHardSigmoid takes one input data (Tensor) and produces one output data\n(Tensor) where the HardSigmoid function, y = max(0, min(1, alpha * x + beta)),\nis applied to the tensor elementwise.\n" -----f -input: "input" -output: "output" -name: "Hardmax" -op_type: "Hardmax" -attribute { - name: "axis" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe operator computes the hardmax (1 for the first maximum value, and 0 for all others) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the hardmax values of the corresponding input.\n" -----f -input: "input" -output: "output" -name: "Identity" -op_type: "Identity" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "Identity operator" -----f -input: "cond" -output: "outputs" -name: "If" -op_type: "If" -attribute { - name: "else_branch" - s: "" - type: GRAPH -} -attribute { - name: "then_branch" - s: "" - type: GRAPH -} -attribute { - name: "cond-types" - strings: "bool" - type: STRINGS -} -doc_string: "If conditional" -----f -input: "X" -output: "Y" -name: "Imputer" -op_type: "Imputer" -attribute { - name: "imputed_value_floats" - s: "" - type: FLOATS -} -attribute { - name: "imputed_value_int64s" - s: "" - type: INTS -} -attribute { - name: "replaced_value_float" - f: 0.0 - type: FLOAT -} -attribute { - name: "replaced_value_int64" - i: 0 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Replaces inputs that equal one value with another, leaving all other elements alone.
\n This operator is typically used to replace missing values in situations where they have a canonical\n representation, such as -1, 0, NaN, or some extreme value.
\n One and only one of imputed_value_floats or imputed_value_int64s should be defined -- floats if the input tensor\n holds floats, integers if the input tensor holds integers. The imputed values must all fit within the\n width of the tensor element type. One and only one of the replaced_value_float or replaced_value_int64 should be defined,\n which one depends on whether floats or integers are being processed.
\n The imputed_value attribute length can be 1 element, or it can have one element per input feature.
In other words, if the input tensor has the shape [*,F], then the length of the attribute array may be 1 or F. If it is 1, then it is broadcast along the last dimension and applied to each feature.\n" -----f -input: "input" -input: "scale" -input: "B" -output: "output" -name: "InstanceNormalization" -op_type: "InstanceNormalization" -attribute { - name: "epsilon" - f: 1e-05 - type: FLOAT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "scale-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCarries out instance normalization as described in the paper\nhttps://arxiv.org/abs/1607.08022.\n\ny = scale * (x - mean) / sqrt(variance + epsilon) + B,\nwhere mean and variance are computed per instance per channel.\n\n" -----f -input: "X" -output: "Y" -name: "IsInf" -op_type: "IsInf" -attribute { - name: "detect_negative" - i: 1 - type: INT -} -attribute { - name: "detect_positive" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - type: STRINGS -} -doc_string: "Map infinity to true and other values to false." -----f -input: "X" -output: "Y" -name: "IsNaN" -op_type: "IsNaN" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "Returns which elements of the input are NaN." -----f -input: "X" -output: "Y" -name: "LRN" -op_type: "LRN" -attribute { - name: "alpha" - f: 0.0001 - type: FLOAT -} -attribute { - name: "beta" - f: 0.75 - type: FLOAT -} -attribute { - name: "bias" - f: 1.0 - type: FLOAT -} -attribute { - name: "size" - s: "" - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nLocal Response Normalization proposed in the [AlexNet paper](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf).\nIt normalizes over local input regions.\nThe local region is defined across the channels. For an element X[n, c, d1, ..., dk] in a tensor\nof shape (N x C x D1 x D2, ..., Dk), its region is\n{X[n, i, d1, ..., dk] | max(0, c - floor((size - 1) / 2)) <= i <= min(C - 1, c + ceil((size - 1) / 2))}.\n\nsquare_sum[n, c, d1, ..., dk] = sum(X[n, i, d1, ..., dk] ^ 2),\nwhere max(0, c - floor((size - 1) / 2)) <= i <= min(C - 1, c + ceil((size - 1) / 2)).\n\nY[n, c, d1, ..., dk] = X[n, c, d1, ..., dk] / (bias + alpha / size * square_sum[n, c, d1, ..., dk] ) ^ beta\n" -----f -input: "X" -input: "W" -input: "R" -input: "B" -input: "sequence_lens" -input: "initial_h" -input: "initial_c" -input: "P" -output: "Y" -output: "Y_h" -output: "Y_c" -name: "LSTM" -op_type: "LSTM" -attribute { - name: "activation_alpha" - s: "" - type: FLOATS -} -attribute { - name: "activation_beta" - s: "" - type: FLOATS -} -attribute { - name: "activations" - s: "" - type: STRINGS -} -attribute { - name: "clip" - s: "" - type: FLOAT -} -attribute { - name: "direction" - s: "forward" - type: STRING -} -attribute { - name: "hidden_size" - s: "" - type: INT -} -attribute { - name: "input_forget" - i: 0 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "sequence_lens-types" - strings: "int32" - type: STRINGS -} -attribute { - name: "initial_h-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "initial_c-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "P-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nComputes an one-layer LSTM. This operator is usually supported via some\ncustom implementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`i` - input gate\n\n`o` - output gate\n\n`f` - forget gate\n\n`c` - cell gate\n\n`t` - time step (t-1 means previous time step)\n\n`W[iofc]` - W parameter weight matrix for input, output, forget, and cell gates\n\n`R[iofc]` - R recurrence weight matrix for input, output, forget, and cell gates\n\n`Wb[iofc]` - W bias vectors for input, output, forget, and cell gates\n\n`Rb[iofc]` - R bias vectors for input, output, forget, and cell gates\n\n`P[iof]` - P peephole weight vector for input, output, and forget gates\n\n`WB[iofc]` - W parameter weight matrix for backward input, output, forget, and cell gates\n\n`RB[iofc]` - R recurrence weight matrix for backward input, output, forget, and cell gates\n\n`WBb[iofc]` - W bias vectors for backward input, output, forget, and cell gates\n\n`RBb[iofc]` - R bias vectors for backward input, output, forget, and cell gates\n\n`PB[iof]` - P peephole weight vector for backward input, output, and forget gates\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Sigmoid, g=Tanh, h=Tanh):\n\n - it = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Pi (.) Ct-1 + Wbi + Rbi)\n\n - ft = f(Xt*(Wf^T) + Ht-1*(Rf^T) + Pf (.) Ct-1 + Wbf + Rbf)\n\n - ct = g(Xt*(Wc^T) + Ht-1*(Rc^T) + Wbc + Rbc)\n\n - Ct = ft (.) Ct-1 + it (.) ct\n\n - ot = f(Xt*(Wo^T) + Ht-1*(Ro^T) + Po (.) Ct + Wbo + Rbo)\n\n - Ht = ot (.) h(Ct)\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -input: "X" -output: "Y" -name: "LabelEncoder" -op_type: "LabelEncoder" -attribute { - name: "default_float" - f: -0.0 - type: FLOAT -} -attribute { - name: "default_int64" - i: -1 - type: INT -} -attribute { - name: "default_string" - s: "_Unused" - type: STRING -} -attribute { - name: "keys_floats" - s: "" - type: FLOATS -} -attribute { - name: "keys_int64s" - s: "" - type: INTS -} -attribute { - name: "keys_strings" - s: "" - type: STRINGS -} -attribute { - name: "values_floats" - s: "" - type: FLOATS -} -attribute { - name: "values_int64s" - s: "" - type: INTS -} -attribute { - name: "values_strings" - s: "" - type: STRINGS -} -attribute { - name: "X-types" - strings: "string" - strings: "float" - strings: "int64" - type: STRINGS -} -doc_string: "\n Maps each element in the input tensor to another value.
\n The mapping is determined by the two parallel attributes, \'keys_*\' and\n \'values_*\' attribute. The i-th value in the specified \'keys_*\' attribute\n would be mapped to the i-th value in the specified \'values_*\' attribute. It\n implies that input\'s element type and the element type of the specified\n \'keys_*\' should be identical while the output type is identical to the\n specified \'values_*\' attribute. If an input element can not be found in the\n specified \'keys_*\' attribute, the \'default_*\' that matches the specified\n \'values_*\' attribute may be used as its output value.
\n Let\'s consider an example which maps a string tensor to an integer tensor.\n Assume and \'keys_strings\' is [\"Amy\", \"Sally\"], \'values_int64s\' is [5, 6],\n and \'default_int64\' is \'-1\'. The input [\"Dori\", \"Amy\", \"Amy\", \"Sally\",\n \"Sally\"] would be mapped to [-1, 5, 5, 6, 6].
\n Since this operator is an one-to-one mapping, its input and output shapes\n are the same. Notice that only one of \'keys_*\'/\'values_*\' can be set.
\n For key look-up, bit-wise comparison is used so even a float NaN can be\n mapped to a value in \'values_*\' attribute.
\n" -----f -input: "X" -output: "Y" -name: "LeakyRelu" -op_type: "LeakyRelu" -attribute { - name: "alpha" - f: 0.01 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nLeakyRelu takes input data (Tensor) and an argument alpha, and produces one\noutput data (Tensor) where the function `f(x) = alpha * x for x < 0`,\n`f(x) = x for x >= 0`, is applied to the data tensor elementwise.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Less" -op_type: "Less" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `less` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "A" -input: "B" -output: "C" -name: "LessOrEqual" -op_type: "LessOrEqual" -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `less_equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Y" -output: "Z" -name: "LinearClassifier" -op_type: "LinearClassifier" -attribute { - name: "classlabels_ints" - s: "" - type: INTS -} -attribute { - name: "classlabels_strings" - s: "" - type: STRINGS -} -attribute { - name: "coefficients" - s: "" - type: FLOATS -} -attribute { - name: "intercepts" - s: "" - type: FLOATS -} -attribute { - name: "multi_class" - i: 0 - type: INT -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Linear classifier\n" -----f -input: "X" -output: "Y" -name: "LinearRegressor" -op_type: "LinearRegressor" -attribute { - name: "coefficients" - s: "" - type: FLOATS -} -attribute { - name: "intercepts" - s: "" - type: FLOATS -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "targets" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Generalized linear regression evaluation.
\n If targets is set to 1 (default) then univariate regression is performed.
\n If targets is set to M then M sets of coefficients must be passed in as a sequence\n and M results will be output for each input n in N.
\n The coefficients array is of length n, and the coefficients for each target are contiguous.\n Intercepts are optional but if provided must match the number of targets.\n" -----f -input: "input" -output: "output" -name: "Log" -op_type: "Log" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the natural log of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "LogSoftmax" -op_type: "LogSoftmax" -attribute { - name: "axis" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe operator computes the logsoftmax (log of softmax) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the logsoftmax values of the corresponding input.\n" -----f -input: "M" -input: "cond" -input: "v_initial" -output: "v_final_and_scan_outputs" -name: "Loop" -op_type: "Loop" -attribute { - name: "body" - s: "" - type: GRAPH -} -attribute { - name: "M-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "cond-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "v_initial-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGeneric Looping construct. This loop has multiple termination conditions:\n\n1) Trip count. Iteration count specified at runtime. Set by\n specifying the input M. Optional. Set to empty string to omit.\n Note that a static trip count (specified at graph construction time) can be\n specified by passing in a constant node for input M.\n2) Loop termination condition. This is an input to the op that determines\n whether to run the first iteration and also a loop-carried dependency for\n the body graph. The body graph must yield a value for the condition variable,\n whether this input is provided or not.\n\nThis table summarizes the operating modes of this operator with equivalent\nC-style code:\n\n Operator inputs defined as (max_trip_count, condition_var).\n\n input (\"\", \"\"):\n for (int i=0; ; ++i) {\n cond = ... // Note this value is ignored, but is required in the body\n }\n\n input (\"\", cond) // Note this is analogous to a while loop\n bool cond = ...;\n for (int i=0; cond; ++i) {\n cond = ...;\n }\n\n input (\"\", 1) // Note this is analogous to a do-while loop\n bool cond = true\n for (int i=0; cond; ++i) {\n cond = ...;\n }\n\n input (trip_count, \"\") // Note this is analogous to a for loop\n int trip_count = ...\n for (int i=0; i < trip_count; ++i) {\n cond = ...; // ignored\n }\n\n input (trip_count, cond)\n int trip_count = ...;\n bool cond = ...;\n for (int i=0; i < trip_count && cond; ++i) {\n cond = ...;\n }\n\n\n*Sample usage - cond as well as trip count*\n\n graph predict-net {\n %a = Constant[value = ]()\n %b = Constant[value = ]()\n %keepgoing = Constant[value = ]()\n %max_trip_count = Constant[value = ]()\n %keepgoing_out, %b_out, %user_defined_vals = Loop[body = ](%max_trip_count, %keepgoing, %b)\n return\n }\n\n graph body-net (\n %i[INT32, scalar] // iteration number\n %keepgoing_in[BOOL, scalar] // incoming loop-termination-condition; not used\n %b_in[INT32, scalar] // incoming value of loop-carried-dependency b\n ) {\n %my_local = Add(%a, %b_in)\n %b_out = Sub(%a, %b_in) // outgoing value of loop-carried-dependency b\n %keepgoing_out = Greater(%my_local, %b_out) // outgoing loop-termination-condition\n %user_defined_val = Add(%b_in, %b_in) // scan-output value to be accumulated\n return %keepgoing_out, %b_out, %user_defined_val\n }\n\n*Sample equivalent C code*\n\n {\n /* User-defined code (enclosing scope) */\n int a = 3, b = 6;\n bool keepgoing = true; // Analogous to input cond\n /* End user-defined code */\n\n /* Implicitly-defined code */\n const int max_trip_count = 10; // Analogous to input M\n int user_defined_vals[]; // Imagine this is resizable\n /* End implicitly-defined code */\n /* initialize loop-carried variables and scan-output variables */\n bool keepgoing_out = keepgoing\n int b_out = b\n\n for (int i=0; i < max_trip_count && keepgoing_out; ++i) {\n /* Implicitly-defined code: bind actual parameter values\n to formal parameter variables of loop-body */\n bool keepgoing_in = keepgoing_out; \n bool b_in = b_out;\n\n /* User-defined code (loop body) */\n int my_local = a + b_in; // Reading value \"a\" from the enclosing scope is fine\n b_out = a - b_in;\n keepgoing_out = my_local > b_out; \n user_defined_val = b_in + b_in; // b_in and b_out are different variables\n /* End user-defined code */\n\n /* Implicitly defined-code */\n user_defined_vals[i] = user_defined_val // accumulate scan-output values\n }\n // int t = my_local; // Can\'t do this. my_local is not accessible here.\n\n // The values below are bound to the output variables of the loop and therefore accessible\n // b_out; user_defined_vals; keepgoing_out;\n }\n\nThere are several things of note in this code snippet:\n\n1) Values from the enclosing scope (i.e. variable \"a\" here) are in scope and can\n be referenced in the inputs of the loop.\n2) Any values computed in the loop body that needs to be used in a subsequent\n iteration or after the loop are modelled using a pair of variables in the loop-body,\n consisting of an input variable (eg., b_in) and an output variable (eg., b_out).\n These are referred to as loop-carried dependences. The loop operation node\n supplies the input value of the input variable for the first iteration, and\n returns the output value of the output variable produced by the final\n iteration.\n3) Scan_output variables are used to implicitly concatenate values computed across\n all the iterations. In the above example, the value of user_defined_val computed\n over all iterations are concatenated and returned as the value of user_defined_vals\n after the loop.\n4) Values created in the body cannot be accessed in the enclosing scope,\n except using the mechanism described above.\n\nNote that the semantics of this op support \"diagonal\" or \"wavefront\" execution.\n(See Step 3 here for an example:\nhttps://devblogs.nvidia.com/optimizing-recurrent-neural-networks-cudnn-5/).\nFrontends should emit multi-layer RNNs as a series of While operators (with\ntime being the inner looping dimension), with each successive layer consuming\nthe scan_outputs from the previous layer, possibly going through several\npoint-wise operators (e.g. dropout, residual connections, linear layer).\n" -----f -input: "input" -output: "output" -name: "LpNormalization" -op_type: "LpNormalization" -attribute { - name: "axis" - i: -1 - type: INT -} -attribute { - name: "p" - i: 2 - type: INT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nGiven a matrix, apply Lp-normalization along the provided axis.\n" -----f -input: "X" -output: "Y" -name: "LpPool" -op_type: "LpPool" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "p" - i: 2 - type: INT -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n LpPool consumes an input tensor X and applies Lp pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n Lp pooling consisting of computing the Lp norm on all values of a subset\n of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing." -----f -input: "A" -input: "B" -output: "Y" -name: "MatMul" -op_type: "MatMul" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html\n" -----f -input: "A" -input: "B" -input: "a_zero_point" -input: "b_zero_point" -output: "Y" -name: "MatMulInteger" -op_type: "MatMulInteger" -attribute { - name: "A-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "B-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "a_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "b_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html.\nThe production MUST never overflow. The accumulation may overflow if and only if in 32 bits.\n" -----f -input: "data_0" -output: "max" -name: "Max" -op_type: "Max" -attribute { - name: "data_0-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nElement-wise max of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Y" -output: "Indices" -name: "MaxPool" -op_type: "MaxPool" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "ceil_mode" - i: 0 - type: INT -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "storage_order" - i: 0 - type: INT -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "int8" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "float" - type: STRINGS -} -doc_string: "\n MaxPool consumes an input tensor X and applies max pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n max pooling consisting of computing the max on all values of a\n subset of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing. The output spatial shape will be following:\n ```\n output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1)) / strides_spatial_shape[i] + 1)\n ```\n or\n ```\n output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1)) / strides_spatial_shape[i] + 1)\n ```\n if ceil_mode is enabled\n\n ```\n * pad_shape[i] is sum of pads along axis i\n ```\n\n `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following:\n ```\n VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1) + 1) / strides_spatial_shape[i])\n SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i])\n ```\n And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`:\n ```\n pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + ((kernel_spatial_shape[i] - 1) * dilations[i] + 1) - input_spatial_shape[i]\n ```\n The output of each pooling window is maximum number of elements exclude pad. \n " -----f -input: "X" -input: "rois" -output: "Y" -name: "MaxRoiPool" -op_type: "MaxRoiPool" -attribute { - name: "pooled_shape" - s: "" - type: INTS -} -attribute { - name: "spatial_scale" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "rois-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n ROI max pool consumes an input tensor X and region of interests (RoIs) to\n apply max pooling across each RoI, to produce output 4-D tensor of shape\n (num_rois, channels, pooled_shape[0], pooled_shape[1])." -----f -input: "X" -input: "I" -input: "output_shape" -output: "output" -name: "MaxUnpool" -op_type: "MaxUnpool" -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "I-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "output_shape-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nMaxUnpool essentially computes the partial inverse of the MaxPool op.\n The input information to this op is typically the the output information from a MaxPool op. The first\n input tensor X is the tensor that needs to be unpooled, which is typically the pooled tensor (first output)\n from MaxPool. The second input tensor, I, contains the indices to the (locally maximal) elements corrsponding\n to the elements in the first input tensor X. Input tensor I is typically the second output of the MaxPool op.\n The third (optional) input is a tensor that specifies the output size of the unpooling operation.\n\nMaxUnpool is intended to do \'partial\' inverse of the MaxPool op. \'Partial\' because all the non-maximal\n values from the original input to MaxPool are set to zero in the output of the MaxUnpool op. Pooling\n the result of an unpooling operation should give back the original input to the unpooling op.\n\nMaxUnpool can produce the same output size for several input sizes, which makes unpooling op ambiguous.\n The third input argument, output_size, is meant to disambiguate the op and produce output tensor of\n known/predictable size.\n\nIn addition to the inputs, MaxUnpool takes three attributes, namely kernel_shape, strides, and pads,\n which define the exact unpooling op. The attributes typically have the same values as the corrsponding\n pooling op that the unpooling op is trying to invert.\n" -----f -input: "data_0" -output: "mean" -name: "Mean" -op_type: "Mean" -attribute { - name: "data_0-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nElement-wise mean of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Y" -name: "MeanVarianceNormalization" -op_type: "MeanVarianceNormalization" -attribute { - name: "axes" - ints: 0 - ints: 2 - ints: 3 - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\n A MeanVarianceNormalization Function: Perform mean variance normalization\n on the input tensor X using formula:
``` (X-EX)/sqrt(E(X-EX)^2) ```\n" -----f -input: "data_0" -output: "min" -name: "Min" -op_type: "Min" -attribute { - name: "data_0-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nElement-wise min of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "A" -input: "B" -output: "C" -name: "Mod" -op_type: "Mod" -attribute { - name: "fmod" - i: 0 - type: INT -} -attribute { - name: "A-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\n Performs element-wise binary modulus (with Numpy-style broadcasting support). \n The sign of the remainder is the same as that of the Divisor.\n \n Mod operator can also behave like C fmod() or numpy.fmod. In this case, the sign of the remainder however, will be the same as the Dividend \n (in contrast to integer mod). To force a behavior like numpy.fmod() an \'fmod\' Attribute is provided.\n This attribute is set to 0 by default causing the behavior to be like integer mod. \n Setting this attribute to 1 causes the remainder to be calculated similar to that of numpy.fmod().\n\n If the input type is floating point, then `fmod` attribute must be set to 1.\n \n In case of dividend being zero, the results will be platform dependent.\n\n This operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "R" -input: "T" -input: "inputs" -output: "outputs" -name: "Momentum" -op_type: "Momentum" -attribute { - name: "alpha" - s: "" - type: FLOAT -} -attribute { - name: "beta" - s: "" - type: FLOAT -} -attribute { - name: "mode" - s: "" - type: STRING -} -attribute { - name: "norm_coefficient" - s: "" - type: FLOAT -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - type: STRINGS -} -attribute { - name: "T-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "inputs-types" - strings: "float" - strings: "double" - type: STRINGS -} -doc_string: "\n Compute one iteration of stochastic gradient update with momentum.\n This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. As you can imagine, SG with momentum requires\n several parameters:\n \n - The learning-rate \"R\".\n - The update count \"T\". That is, the number of conducted training iterations. It should\n be zero in the first training iteration.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A decay coefficient of previous accumulated gradient (i.e., momentum) \"alpha\".\n - The scaling coefficient of current gradient \"beta\".\n - An attribute to choose either standard momentum or Nesterov\'s momentum \"mode\" should\n be used.\n\n For the sake of simplicity, assume that there is only one tensor (called \"X\") to be optimized.\n Other necessary inputs are \"X\"\'s gradient (called \"G\") and \"X\"\'s momentum (called \"V\"). This\n Momentum operator maps all these inputs to the new value of \"X\" (called \"X_new\") and its new\n momentum (called \"V_new\").\n \n This operator supports two different momentum algorithms. Set the attribute \"mode\" to\n \"nesterov\" if Nesterov\'s momentum is desired. Otherwise, set the attribute \"model\" to\n \"standard\" to use standard momentum. Computation details are described subsequently.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise operations with numpy-style broadcasting.\n\n Pseudo code for SG with standard momentum:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||^2, where ||X|| is the sum of squared\n // values of all elements in X.\n G_regularized = norm_coefficient * X + G\n\n // In the first training iteration, beta should always be 1.\n beta_adjusted = T > 0 ? beta : 1\n\n // Compute the current momentum based on previous momentum and the current gradient.\n V_new = alpha * V + beta_adjusted * G_regularized\n\n // Update X.\n X_new = X - R * V_new\n\n Pseudo code for SG with Nesterov\'s momentum:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||^2, where ||X|| is the sum of squared\n // values of all elements in X.\n G_regularized = norm_coefficient * X + G;\n\n // In the first training iteration, beta should always be 1.\n beta_adjusted = T > 0 ? beta : 1\n\n // Compute the current momentum based on previous momentum and the current gradient.\n V_new = alpha * V + beta_adjusted * G_regularized;\n\n // Compute final update direction and then update X.\n X_new = X - R * (G_regularized + alpha * V_new)\n\n If one assign this operators to optimize multiple inputs, for example, \"X_1\" and \"X_2\". The same\n pseudo code would be extended to handle all tensors jointly. More specifically, we can view \"X\" as a\n concatenation of \"X_1\" and \"X_2\" (of course, their gradient and accumulate gradient should\n be concatenated too) and then our pseudo code becomes applicable.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Mul" -op_type: "Mul" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPerforms element-wise binary multiplication (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "input" -output: "output" -name: "Multinomial" -op_type: "Multinomial" -attribute { - name: "dtype" - i: 6 - type: INT -} -attribute { - name: "sample_size" - i: 1 - type: INT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nGenerate a tensor of samples from a multinomial distribution according to the probabilities\nof each of the possible outcomes.\n" -----f -input: "X" -output: "Y" -name: "Neg" -op_type: "Neg" -attribute { - name: "X-types" - strings: "int8" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - strings: "int16" - type: STRINGS -} -doc_string: "\nNeg takes one input data (Tensor) and produces one output data\n(Tensor) where each element flipped sign, y = -x, is applied to\nthe tensor elementwise.\n" -----f -input: "input" -input: "target" -input: "weight" -output: "loss" -name: "NegativeLogLikelihoodLoss" -op_type: "NegativeLogLikelihoodLoss" -attribute { - name: "ignore_index" - s: "" - type: INT -} -attribute { - name: "reduction" - s: "mean" - type: STRING -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "target-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "weight-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nA NegativeLogLikelihoodLoss operator computes (weighted) negative log likelihood loss.\nIts \"input\" tensor has the shape of (N, C, d1, d2, ..., dk) where k >= 0.\nThe \"input\" tensor contains log-probabilities for input[n, :, d_1, d_2,..., d_k] being in a class of [0, C).\nThe operator\'s \"target\" input tensor has the shape of (N, d1, d2, ..., dk). It encodes class labels (one of C classes)\nor it may contain a special value (indicated by an attribute ignore_index) for N x d1 x d2 x ... x dk samples.\nThe loss value for input[n, :, d_1, d_2,...d_k] being classified as class c = target[n][d_1][d_2]...[d_k] is computed as:\n\n loss[n][d_1][d_2]...[d_k] = -input[n][c][d_1][d_2]...[d_k].\n\nWhen an optional \"weight\" is provided, the sample loss is calculated as:\n\n loss[n][d_1][d_2]...[d_k] = -input[n][c][d_1][d_2]...[d_k] * weight[c].\n\nloss is zero for the case when target-value equals ignore_index.\n \n loss[n][d_1][d_2]...[d_k] = 0, when target[n][d_1][d_2]...[d_k] = ignore_index\n\nIf \"reduction\" attribute is set to \"none\", the operator\'s output will be the above loss with shape (N, d1, d2, ..., dk).\nIf \"reduction\" attribute is set to \"mean\" (the default attribute value), the output loss is (weight) averaged:\n\n mean(loss), if \"weight\" is not provided,\n\nor if weight is provided,\n\n sum(loss) / sum(weight[target[n][d_1][d_2]...[d_k]]]), for all samples.\n\nIf \"reduction\" attribute is set to \"sum\", the output is a scalar:\n sum(loss).\n\nSee also https://pytorch.org/docs/stable/nn.html#torch.nn.NLLLoss.\n\nExample 1:\n\n // negative log likelihood loss, \"none\" reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n\n loss = np.zeros((N, d1))\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1]\n\n // print(loss)\n // [[-3. -2.]\n // [-0. -2.]]\n\nExample 2:\n\n // weighted negative log likelihood loss, sum reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n weight = [0.2, 0.3, 0.1]\n loss = np.zeros((N, d1))\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1] * weight[c]\n\n loss = np.sum(loss)\n // print(loss)\n // -1.1\n\nExample 3:\n\n // weighted negative log likelihood loss, mean reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n weight = [0.2, 0.3, 0.1]\n loss = np.zeros((N, d1))\n weight_total = 0\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1] * weight[c]\n weight_total = weight_total + weight[c]\n\n loss = np.sum(loss) / weight_total\n // print(loss)\n // -1.57\n" -----f -input: "boxes" -input: "scores" -input: "max_output_boxes_per_class" -input: "iou_threshold" -input: "score_threshold" -output: "selected_indices" -name: "NonMaxSuppression" -op_type: "NonMaxSuppression" -attribute { - name: "center_point_box" - i: 0 - type: INT -} -attribute { - name: "boxes-types" - strings: "float" - type: STRINGS -} -attribute { - name: "scores-types" - strings: "float" - type: STRINGS -} -attribute { - name: "max_output_boxes_per_class-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "iou_threshold-types" - strings: "float" - type: STRINGS -} -attribute { - name: "score_threshold-types" - strings: "float" - type: STRINGS -} -doc_string: "\nFilter out boxes that have high intersection-over-union (IOU) overlap with previously selected boxes.\nBounding boxes with score less than score_threshold are removed. Bounding box format is indicated by attribute center_point_box.\nNote that this algorithm is agnostic to where the origin is in the coordinate system and more generally is invariant to\northogonal transformations and translations of the coordinate system; thus translating or reflections of the coordinate system\nresult in the same boxes being selected by the algorithm.\nThe selected_indices output is a set of integers indexing into the input collection of bounding boxes representing the selected boxes.\nThe bounding box coordinates corresponding to the selected indices can then be obtained using the Gather or GatherND operation.\n" -----f -input: "X" -output: "Y" -name: "NonZero" -op_type: "NonZero" -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\n Returns the indices of the elements that are non-zero\n (in row-major order - by dimension).\n NonZero behaves similar to numpy.nonzero:\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.nonzero.html\n" -----f -input: "X" -output: "Y" -name: "Normalizer" -op_type: "Normalizer" -attribute { - name: "norm" - s: "MAX" - type: STRING -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Normalize the input. There are three normalization modes, which have the corresponding formulas,\n defined using element-wise infix operators \'/\' and \'^\' and tensor-wide functions \'max\' and \'sum\':
\n
\n Max: Y = X / max(X)
\n L1: Y = X / sum(X)
\n L2: Y = sqrt(X^2 / sum(X^2)}
\n In all modes, if the divisor is zero, Y == X.\n
\n For batches, that is, [N,C] tensors, normalization is done along the C axis. In other words, each row\n of the batch is normalized independently.\n" -----f -input: "X" -output: "Y" -name: "Not" -op_type: "Not" -attribute { - name: "X-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nReturns the negation of the input tensor element-wise.\n" -----f -input: "indices" -input: "depth" -input: "values" -output: "output" -name: "OneHot" -op_type: "OneHot" -attribute { - name: "axis" - i: -1 - type: INT -} -attribute { - name: "indices-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "depth-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "values-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\n Produces a one-hot tensor based on inputs.\n The locations represented by the index values in the \'indices\' input tensor will have \'on_value\'\n and the other locations will have \'off_value\' in the output tensor, where \'on_value\' and \'off_value\'\n are specified as part of required input argument \'values\', which is a two-element tensor of format\n [off_value, on_value]. The rank of the output tensor will be one greater than the rank of the\n input tensor. The additional dimension is for one-hot representation. The additional dimension will\n be inserted at the position specified by \'axis\'. If \'axis\' is not specified then then additional\n dimension will be inserted as the innermost dimension, i.e. axis=-1. The size of the additional\n dimension is specified by required scalar input \'depth\'. The type of the output tensor is the same\n as the type of the \'values\' input. Any entries in the \'indices\' input tensor with values outside\n the range [-depth, depth-1] will result in one-hot representation with all \'off_value\' values in the\n output tensor.\n\n when axis = 0:\n output[input[i, j, k], i, j, k] = 1 for all i, j, k and 0 otherwise.\n\n when axis = -1:\n output[i, j, k, input[i, j, k]] = 1 for all i, j, k and 0 otherwise.\n\n" -----f -input: "X" -output: "Y" -name: "OneHotEncoder" -op_type: "OneHotEncoder" -attribute { - name: "cats_int64s" - s: "" - type: INTS -} -attribute { - name: "cats_strings" - s: "" - type: STRINGS -} -attribute { - name: "zeros" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "int32" - strings: "string" - strings: "double" - strings: "int64" - strings: "float" - type: STRINGS -} -doc_string: "\n Replace each input element with an array of ones and zeros, where a single\n one is placed at the index of the category that was passed in. The total category count \n will determine the size of the extra dimension of the output array Y.
\n For example, if we pass a tensor with a single value of 4, and a category count of 8, \n the output will be a tensor with ``[0,0,0,0,1,0,0,0]``.
\n This operator assumes every input feature is from the same set of categories.
\n If the input is a tensor of float, int32, or double, the data will be cast\n to integers and the cats_int64s category list will be used for the lookups.\n" -----f -input: "A" -input: "B" -output: "C" -name: "Or" -op_type: "Or" -attribute { - name: "A-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "B-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `or` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -input: "slope" -output: "Y" -name: "PRelu" -op_type: "PRelu" -attribute { - name: "X-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "slope-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPRelu takes input data (Tensor) and slope tensor as input, and produces one\noutput data (Tensor) where the function `f(x) = slope * x for x < 0`,\n`f(x) = x for x >= 0`., is applied to the data tensor elementwise.\nThis operator supports **unidirectional broadcasting** (tensor slope should be unidirectional broadcastable to input tensor X); for more details please check [the doc](Broadcasting.md)." -----f -input: "data" -input: "pads" -input: "constant_value" -output: "output" -name: "Pad" -op_type: "Pad" -attribute { - name: "mode" - s: "constant" - type: STRING -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "pads-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "constant_value-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGiven a tensor containing the data to be padded (`data`), a tensor containing the number of start and end pad values for axis (`pads`), (optionally) a `mode`, and (optionally) `constant_value`, \na padded tensor (`output`) is generated.\n\nThe three supported `modes` are (similar to corresponding modes supported by `numpy.pad`):\n\n1) `constant`(default) - pads with a given constant value as specified by `constant_value` (which defaults to 0)\n\n2) `reflect` - pads with the reflection of the vector mirrored on the first and last values of the vector along each axis\n\n3) `edge` - pads with the edge values of array\n\n\nExample 1 (`constant` mode):\n Insert 0 pads to the beginning of the second dimension.\n\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'constant\'\n\n constant_value = 0.0\n\n output = \n [\n [\n [0.0, 0.0, 1.0, 1.2],\n [0.0, 0.0, 2.3, 3.4],\n [0.0, 0.0, 4.5, 5.7],\n ],\n ]\n\n\nExample 2 (`reflect` mode):\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'reflect\'\n\n output = \n [\n [\n [1.0, 1.2, 1.0, 1.2],\n [2.3, 3.4, 2.3, 3.4],\n [4.5, 5.7, 4.5, 5.7],\n ],\n ]\n\n\nExample 3 (`edge` mode):\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'edge\'\n\n output = \n [\n [\n [1.0, 1.0, 1.0, 1.2],\n [2.3, 2.3, 2.3, 3.4],\n [4.5, 4.5, 4.5, 5.7],\n ],\n ]\n\n" -----f -input: "X" -input: "Y" -output: "Z" -name: "Pow" -op_type: "Pow" -attribute { - name: "X-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - type: STRINGS -} -attribute { - name: "Y-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPow takes input data (Tensor) and exponent Tensor, and\nproduces one output data (Tensor) where the function `f(x) = x^exponent`,\nis applied to the data tensor elementwise.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md)." -----f -input: "x" -input: "x_scale" -input: "x_zero_point" -input: "w" -input: "w_scale" -input: "w_zero_point" -input: "y_scale" -input: "y_zero_point" -input: "B" -output: "y" -name: "QLinearConv" -op_type: "QLinearConv" -attribute { - name: "auto_pad" - s: "NOTSET" - type: STRING -} -attribute { - name: "dilations" - s: "" - type: INTS -} -attribute { - name: "group" - i: 1 - type: INT -} -attribute { - name: "kernel_shape" - s: "" - type: INTS -} -attribute { - name: "pads" - s: "" - type: INTS -} -attribute { - name: "strides" - s: "" - type: INTS -} -attribute { - name: "x-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "x_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "x_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "w-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "w_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "w_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "y_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "y_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "B-types" - strings: "int32" - type: STRINGS -} -doc_string: "\nThe convolution operator consumes a quantized input tensor, its scale and zero point,\na quantized filter, its scale and zero point, and output\'s scale and zero point,\nand computes the quantized output. Each scale and zero-point pair must have same shape.\nIt means they must be either scalars (per tensor) or 1-D tensors (per output channel).\nEach input or output and its related zero point must have same type.\nWhen bias is present it must be quantized using scale = input scale * weight scale and \nzero point as 0.\n" -----f -input: "a" -input: "a_scale" -input: "a_zero_point" -input: "b" -input: "b_scale" -input: "b_zero_point" -input: "y_scale" -input: "y_zero_point" -output: "y" -name: "QLinearMatMul" -op_type: "QLinearMatMul" -attribute { - name: "a-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "a_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "a_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "b-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "b_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "b_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -attribute { - name: "y_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "y_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html.\nIt consumes two quantized input tensors, their scales and zero points, scale and zero point of output, and computes the quantized output.\nThe quantization formula is y = saturate((x / y_scale) + y_zero_point). For (x / y_scale), it is rounding to nearest ties to even.\nRefer to https://en.wikipedia.org/wiki/Rounding for details. Scale and zero point must have same shape.\nThey must be either scalar (per tensor) or 1-D tensor (per row for \'a\' and per column for \'b\'). If scale and zero point are 1-D tensor,\nthe number of elements of scale and zero point tensor of input \'a\' and output \'y\' should be equal to the number of rows of input \'a\',\nand the number of elements of scale and zero point tensor of input \'b\' should be equal to the number of columns of input \'b\'.\nProduction must never overflow, and accumulation may overflow if and only if in 32 bits.\n" -----f -input: "x" -input: "y_scale" -input: "y_zero_point" -output: "y" -name: "QuantizeLinear" -op_type: "QuantizeLinear" -attribute { - name: "x-types" - strings: "float" - strings: "int32" - type: STRINGS -} -attribute { - name: "y_scale-types" - strings: "float" - type: STRINGS -} -attribute { - name: "y_zero_point-types" - strings: "int8" - strings: "uint8" - type: STRINGS -} -doc_string: "\nThe linear per-tensor/layer quantization operator. It consumes a high precision tensor, a scale, a zero point to compute the low precision / quantized tensor.\nThe quantization formula is y = saturate ((x / y_scale) + y_zero_point). For saturation, it saturates to [0, 255] if it\'s uint8, or [-128, 127] if it\'s int8.\nFor (x / y_scale), it\'s rounding to nearest ties to even. Refer to https://en.wikipedia.org/wiki/Rounding for details. \'y_zero_point\' and \'y\' must have same type.\n" -----f -input: "X" -input: "W" -input: "R" -input: "B" -input: "sequence_lens" -input: "initial_h" -output: "Y" -output: "Y_h" -name: "RNN" -op_type: "RNN" -attribute { - name: "activation_alpha" - s: "" - type: FLOATS -} -attribute { - name: "activation_beta" - s: "" - type: FLOATS -} -attribute { - name: "activations" - strings: "Tanh" - strings: "Tanh" - type: STRINGS -} -attribute { - name: "clip" - s: "" - type: FLOAT -} -attribute { - name: "direction" - s: "forward" - type: STRING -} -attribute { - name: "hidden_size" - s: "" - type: INT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "W-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "R-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "sequence_lens-types" - strings: "int32" - type: STRINGS -} -attribute { - name: "initial_h-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nComputes an one-layer simple RNN. This operator is usually supported\nvia some custom implementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`i` - input gate\n\n`t` - time step (t-1 means previous time step)\n\n`Wi` - W parameter weight matrix for input gate\n\n`Ri` - R recurrence weight matrix for input gate\n\n`Wbi` - W parameter bias vector for input gate\n\n`Rbi` - R parameter bias vector for input gate\n\n`WBi` - W parameter weight matrix for backward input gate\n\n`RBi` - R recurrence weight matrix for backward input gate\n\n`WBbi` - WR bias vectors for backward input gate\n\n`RBbi` - RR bias vectors for backward input gate\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Tanh):\n\n - Ht = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" -----f -output: "output" -name: "RandomNormal" -op_type: "RandomNormal" -attribute { - name: "dtype" - i: 1 - type: INT -} -attribute { - name: "mean" - f: 0.0 - type: FLOAT -} -attribute { - name: "scale" - f: 1.0 - type: FLOAT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "shape" - s: "" - type: INTS -} -doc_string: "\nGenerate a tensor with random values drawn from a normal distribution. The shape\nof the tensor is specified by the `shape` argument and the parameter of the normal distribution\nspecified by `mean` and `scale`.\n\nThe data type is specified by the \'dtype\' argument. The \'dtype\' argument must\nbe one of the data types specified in the \'DataType\' enum field in the\nTensorProto message.\n" -----f -input: "input" -output: "output" -name: "RandomNormalLike" -op_type: "RandomNormalLike" -attribute { - name: "dtype" - s: "" - type: INT -} -attribute { - name: "mean" - f: 0.0 - type: FLOAT -} -attribute { - name: "scale" - f: 1.0 - type: FLOAT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGenerate a tensor with random values drawn from a normal distribution.\nThe shape of the output tensor is copied from the shape of the input tensor,\nand the parameters of the normal distribution are specified by `mean` and `scale`.\n\nThe data type is specified by the \'dtype\' argument, or copied from the input tensor if not provided.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message, and be valid as an output type.\n" -----f -output: "output" -name: "RandomUniform" -op_type: "RandomUniform" -attribute { - name: "dtype" - i: 1 - type: INT -} -attribute { - name: "high" - f: 1.0 - type: FLOAT -} -attribute { - name: "low" - f: 0.0 - type: FLOAT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "shape" - s: "" - type: INTS -} -doc_string: "\nGenerate a tensor with random values drawn from a uniform distribution. The shape\nof the tensor is specified by the `shape` argument and the range by `low` and `high`.\n\nThe data type is specified by the \'dtype\' argument. The \'dtype\' argument must\nbe one of the data types specified in the \'DataType\' enum field in the\nTensorProto message.\n" -----f -input: "input" -output: "output" -name: "RandomUniformLike" -op_type: "RandomUniformLike" -attribute { - name: "dtype" - s: "" - type: INT -} -attribute { - name: "high" - f: 1.0 - type: FLOAT -} -attribute { - name: "low" - f: 0.0 - type: FLOAT -} -attribute { - name: "seed" - s: "" - type: FLOAT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nGenerate a tensor with random values drawn from a uniform distribution.\nThe shape of the output tensor is copied from the shape of the input tensor,\nand the parameters of the uniform distribution are specified by `low` and `high`.\n\nThe data type is specified by the \'dtype\' argument, or copied from the input tensor if not provided.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message and be valid as an output type.\n" -----f -input: "start" -input: "limit" -input: "delta" -output: "output" -name: "Range" -op_type: "Range" -attribute { - name: "start-types" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - strings: "int16" - type: STRINGS -} -attribute { - name: "limit-types" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - strings: "int16" - type: STRINGS -} -attribute { - name: "delta-types" - strings: "int32" - strings: "double" - strings: "int64" - strings: "float" - strings: "int16" - type: STRINGS -} -doc_string: "\nGenerate a tensor containing a sequence of numbers that begin at `start` and extends by increments of `delta`\nup to `limit` (exclusive).\n\nThe number of elements in the output of range is computed as below-\n\n`number_of_elements = max( ceil( (limit - start) / delta ) , 0 )`\n\nThe pseudocode determining the contents of the output is shown below-\n\n`for(int i=0; i) and produces one output data\n(Tensor) where the reciprocal is, y = 1/x, is applied to\nthe tensor elementwise.\n" -----f -input: "data" -output: "reduced" -name: "ReduceL1" -op_type: "ReduceL1" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the L1 norm of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceL2" -op_type: "ReduceL2" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the L2 norm of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceLogSum" -op_type: "ReduceLogSum" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the log sum of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceLogSumExp" -op_type: "ReduceLogSumExp" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the log sum exponent of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceMax" -op_type: "ReduceMax" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "int8" - strings: "float16" - strings: "int32" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the max of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceMean" -op_type: "ReduceMean" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the mean of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceMin" -op_type: "ReduceMin" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "int8" - strings: "float16" - strings: "int32" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the min of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceProd" -op_type: "ReduceProd" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the product of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceSum" -op_type: "ReduceSum" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the sum of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "data" -output: "reduced" -name: "ReduceSumSquare" -op_type: "ReduceSumSquare" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "data-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nComputes the sum square of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." -----f -input: "X" -output: "Y" -name: "Relu" -op_type: "Relu" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nRelu takes one input data (Tensor) and produces one output data\n(Tensor) where the rectified linear function, y = max(0, x), is applied to\nthe tensor elementwise.\n" -----f -input: "data" -input: "shape" -output: "reshaped" -name: "Reshape" -op_type: "Reshape" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "shape-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nReshape the input tensor similar to numpy.reshape.\nFirst input is the data tensor, second input is a shape tensor which specifies the output shape. It outputs the reshaped tensor.\nAt most one dimension of the new shape can be -1. In this case, the value is\ninferred from the size of the tensor and the remaining dimensions. A dimension\ncould also be 0, in which case the actual dimension value is unchanged (i.e. taken\nfrom the input tensor)." -----f -input: "X" -input: "roi" -input: "scales" -input: "sizes" -output: "Y" -name: "Resize" -op_type: "Resize" -attribute { - name: "coordinate_transformation_mode" - s: "half_pixel" - type: STRING -} -attribute { - name: "cubic_coeff_a" - f: -0.75 - type: FLOAT -} -attribute { - name: "exclude_outside" - i: 0 - type: INT -} -attribute { - name: "extrapolation_value" - f: 0.0 - type: FLOAT -} -attribute { - name: "mode" - s: "nearest" - type: STRING -} -attribute { - name: "nearest_mode" - s: "round_prefer_floor" - type: STRING -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "roi-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "scales-types" - strings: "float" - type: STRINGS -} -attribute { - name: "sizes-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nResize the input tensor. In general, it calculates every value in the output tensor as a weighted average of neighborhood (a.k.a. sampling locations) in the input tensor.\nEach dimension value of the output tensor is:\n output_dimension = floor(input_dimension * (roi_end - roi_start) * scale) if input \\\"sizes\\\" is not specified.\n" -----f -input: "input" -input: "sequence_lens" -output: "Y" -name: "ReverseSequence" -op_type: "ReverseSequence" -attribute { - name: "batch_axis" - i: 1 - type: INT -} -attribute { - name: "time_axis" - i: 0 - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "sequence_lens-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nReverse batch of sequences having different lengths specified by `sequence_lens`.\n\nFor each slice i iterating on batch axis, the operator reverses the first sequence_lens[i] elements on time axis,\nand copies elements whose index\'s beyond sequence_lens[i] to the output. So the output slice i contains reversed\nsequences on the first sequence_lens[i] elements, then have original values copied for the other elements.\n\nExample 1:\n input = [[0.0, 4.0, 8.0, 12.0],\n [1.0, 5.0, 9.0, 13.0],\n [2.0, 6.0, 10.0, 14.0],\n [3.0, 7.0, 11.0, 15.0]]\n sequence_lens = [4, 3, 2, 1]\n time_axis = 0\n batch_axis = 1\n\n output = [[3.0, 6.0, 9.0, 12.0],\n [2.0, 5.0, 8.0, 13.0],\n [1.0, 4.0, 10.0, 14.0],\n [0.0, 7.0, 11.0, 15.0]]\n\nExample 2:\n input = [[0.0, 1.0, 2.0, 3.0 ],\n [4.0, 5.0, 6.0, 7.0 ],\n [8.0, 9.0, 10.0, 11.0],\n [12.0, 13.0, 14.0, 15.0]]\n sequence_lens = [1, 2, 3, 4]\n time_axis = 1\n batch_axis = 0\n\n output = [[0.0, 1.0, 2.0, 3.0 ],\n [5.0, 4.0, 6.0, 7.0 ],\n [10.0, 9.0, 8.0, 11.0],\n [15.0, 14.0, 13.0, 12.0]]\n" -----f -input: "X" -input: "rois" -input: "batch_indices" -output: "Y" -name: "RoiAlign" -op_type: "RoiAlign" -attribute { - name: "mode" - s: "avg" - type: STRING -} -attribute { - name: "output_height" - i: 1 - type: INT -} -attribute { - name: "output_width" - i: 1 - type: INT -} -attribute { - name: "sampling_ratio" - i: 0 - type: INT -} -attribute { - name: "spatial_scale" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "rois-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "batch_indices-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nRegion of Interest (RoI) align operation described in the\n[Mask R-CNN paper](https://arxiv.org/abs/1703.06870).\nRoiAlign consumes an input tensor X and region of interests (rois)\nto apply pooling across each RoI; it produces a 4-D tensor of shape\n(num_rois, C, output_height, output_width).\n\nRoiAlign is proposed to avoid the misalignment by removing\nquantizations while converting from original image into feature\nmap and from feature map into RoI feature; in each ROI bin,\nthe value of the sampled locations are computed directly\nthrough bilinear interpolation.\n" -----f -input: "X" -output: "Y" -name: "Round" -op_type: "Round" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nRound takes one input Tensor and rounds the values, element-wise, meaning\nit finds the nearest integer for each value.\nIn case of halfs, the rule is to round them to the nearest even integer.\nThe output tensor has the same shape and type as the input.\n\nExamples:\n```\nround([0.9]) = [1.0]\nround([2.5]) = [2.0]\nround([2.3]) = [2.0]\nround([1.5]) = [2.0]\nround([-4.5]) = [-4.0]\n```\n" -----f -input: "X" -output: "Y" -output: "Z" -name: "SVMClassifier" -op_type: "SVMClassifier" -attribute { - name: "classlabels_ints" - s: "" - type: INTS -} -attribute { - name: "classlabels_strings" - s: "" - type: STRINGS -} -attribute { - name: "coefficients" - s: "" - type: FLOATS -} -attribute { - name: "kernel_params" - s: "" - type: FLOATS -} -attribute { - name: "kernel_type" - s: "LINEAR" - type: STRING -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "prob_a" - s: "" - type: FLOATS -} -attribute { - name: "prob_b" - s: "" - type: FLOATS -} -attribute { - name: "rho" - s: "" - type: FLOATS -} -attribute { - name: "support_vectors" - s: "" - type: FLOATS -} -attribute { - name: "vectors_per_class" - s: "" - type: INTS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Support Vector Machine classifier\n" -----f -input: "X" -output: "Y" -name: "SVMRegressor" -op_type: "SVMRegressor" -attribute { - name: "coefficients" - s: "" - type: FLOATS -} -attribute { - name: "kernel_params" - s: "" - type: FLOATS -} -attribute { - name: "kernel_type" - s: "LINEAR" - type: STRING -} -attribute { - name: "n_supports" - i: 0 - type: INT -} -attribute { - name: "one_class" - i: 0 - type: INT -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "rho" - s: "" - type: FLOATS -} -attribute { - name: "support_vectors" - s: "" - type: FLOATS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Support Vector Machine regression prediction and one-class SVM anomaly detection.\n" -----f -input: "X" -output: "Y" -name: "Scaler" -op_type: "Scaler" -attribute { - name: "offset" - s: "" - type: FLOATS -} -attribute { - name: "scale" - s: "" - type: FLOATS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Rescale input data, for example to standardize features by removing the mean and scaling to unit variance.\n" -----f -input: "initial_state_and_scan_inputs" -output: "final_state_and_scan_outputs" -name: "Scan" -op_type: "Scan" -attribute { - name: "body" - s: "" - type: GRAPH -} -attribute { - name: "num_scan_inputs" - s: "" - type: INT -} -attribute { - name: "scan_input_axes" - s: "" - type: INTS -} -attribute { - name: "scan_input_directions" - s: "" - type: INTS -} -attribute { - name: "scan_output_axes" - s: "" - type: INTS -} -attribute { - name: "scan_output_directions" - s: "" - type: INTS -} -attribute { - name: "initial_state_and_scan_inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nScan can be used to iterate over one or more scan_input tensors,\nconstructing zero or more scan_output tensors. It combines ideas from general recurrences,\nfunctional programming constructs such as scan, fold, map, and zip and is intended to enable\ngeneralizations of RNN-like constructs for sequence-to-sequence processing.\nOther tensors (referred to as state_variables here) can be used to carry a state\nwhen iterating from one element to another (similar to hidden-state in RNNs, also referred\nto as loop-carried dependences in the context of loops).\nMany common usages involve a single scan_input tensor (where functionality\nsimilar to scan, fold and map can be obtained). When more than one scan_input is used,\na behavior similar to zip is obtained.\n\nThe attribute body must be a graph, specifying the computation to be performed in\nevery iteration. It takes as input the current values of the state_variables and\nthe current iterated element of the scan_inputs. It must return the (updated) values\nof the state_variables and zero or more scan_output_element tensors. The values of the\nscan_output_element tensors are concatenated over all the iterations to produce the\nscan_output values of the scan construct (similar to the concatenated intermediate\nhidden-state values of RNN-like constructs). All the output tensors (state_variables as\nwell as scan_output_element tensors) are required to have the same shape in each iteration\nof the loop (a restriction imposed to enable efficient memory allocation).\n\nNote that the iterated element passed to the body subgraph does not have a sequence\naxis. It will have a rank one less than the rank of the corresponding scan_input.\n\nThe scan operation returns the final values of the state_variables as well as the\nscan_outputs.\n\nThe optional attribute scan_input_directions specifies the direction (forward or backward)\nfor each scan input. If this attribute is omitted, all sequences are scanned in the forward\ndirection. A bidirectional scan may be performed by specifying the same tensor input twice\nin the scan_inputs, once with a forward direction, and once with a backward direction.\n\nThe scan_output of the operation is produced by concatenating the scan_output_element\nvalues produced by the body in each iteration. The optional attribute scan_output_directions\nspecifies the direction in which scan_output is constructed (by appending or prepending the\nscan_output_element to scan_output in each iteration) for each scan_output. If this attribute\nis omitted, the scan_output_element is appended to the scan_output in each iteration.\n\nThe optional attribute scan_input_axes specifies the axis to be scanned for each scan_input.\nIf omitted, every scan_input will be scanned in axis 0. For example, if axis 0 is the\nbatch axis and axis 1 is the time axis (to be scanned), specify an axis value of 1.\nNote that scanning a non-zero axis may be less efficient than scanning axis zero.\n\nThe optional attribute scan_output_axes specifies the axis along which the scan_outputs\nare accumulated for each scan_output. For example, if axis 1 is the time axis (to be\nscanned) for both inputs and outputs, specify a scan_input axis and scan_output axis\nvalue of 1.\n\nNote that because of the ONNX restriction that only the last parameter of an operator can\nbe variadic, the initial-states and scan-inputs are listed together as one input parameter.\nSimilarly, the final-states and scan-outputs are listed together as one output parameter.\nThe attribute num_scan_inputs indicates the number M of scan-inputs.\n\nThe behavior of\n\n Scan <\n num_scan_inputs = m,\n body = loop-body,\n scan_input_axes = [axis_1, ..., axis_m]\n > (init_1, ..., init_n, scan_1, ..., scan_m)\n\nis equivalent to the following pseudo-code:\n\n // scan_i.shape[axis_i] denotes the (max) sequence-length of scan_i\n // scan_i.shape[axis_i] is required to be equal to scan_j.shape[axis_j] for all i,j.\n sequence_length = scan_1.shape[axis_1];\n\n // initialize state-variables\n st_1 = init_1; ... st_n = init_n;\n // initialize scan-output variables: [] denotes an empty tensor\n scan_out_1 = []; ...; scan_out_k = [];\n // identify number of iterations:\n\n // execute loop\n for (int t = 0; t < sequence_length; ++t) {\n // generate the scan-input elements: the notation T[t] indicates the sub-tensor\n // of rank one less than T obtained by indexing T at position t along axis k.\n si_1 = scan_1[t];\n ... ;\n si_m = scan_m[t];\n // execute loop-body\n st_1, ..., st_n, so_1, ..., so_k = loop-body(st_1, ..., st_n, si_1, ..., si_m)\n // accumulate the scan-output elements\n scan_out_1 = Concat(scan_out_1, so_1); ... ; scan_out_k = Concat(scan_out_k, so_k);\n }\n\n return st_1, ..., st_n, scan_out_1, ..., scan_out_k;\n\n*Sample usage: Encoding RNN using a Scan*\n\nThe following example shows how a simple RNN over an input tensor %X, with weight tensor %Wi,\nrecurrence weight tensor %Ri, bias tensors %Wbi and %Rbi, and initial hidden-state %H_0 can\nbe encoded as a ScanLoop. Note that the loop-body is a nested graph, and it directly computes\n%Wi, %Ri, %Wbi, and %Rbi (typically constants or initializers in the body graph). If these\nvalues are computed in the outer graph, they need to be passed in as extra state_variables.\n\n graph rnn-encoding {\n %H_0 = ... \n %X = ...\n %Y_h, %Y = Scan[body = , num_scan_inputs=1](%H_0, %X)\n return %Y, %Y_h\n }\n\n graph rnn-cell-1 (\n %H_tminus1[FLOAT, tensor]\n %X_t[FLOAT, tensor]\n ) {\n %Wi = ...\n %Ri = ...\n %Wbi = ...\n %Rbi = ...\n %t1 = X_t * (Wi^T)\n %t2 = H_tminus1*(Ri^T)\n %t3 = Add(%t1, %t2)\n %t4 = Add(%t3, %Wbi)\n %t5 = Add(%t4, %Rbi)\n %Ht = Tanh(%t5)\n %Accumulate = Identity(%Ht)\n return %Ht, %Accumulate\n }\n\n" -----f -input: "data" -input: "indices" -input: "updates" -output: "output" -name: "Scatter" -op_type: "Scatter" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "updates-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nThis operator is deprecated. Please use ScatterElements, which provides the same functionality.\n\nScatter takes three inputs `data`, `updates`, and `indices` of the same\nrank r >= 1 and an optional attribute axis that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value\nto values specified by `updates` at specific index positions specified by\n`indices`. Its output shape is the same as the shape of `data`.\n\nFor each entry in `updates`, the target index in `data` is obtained by combining\nthe corresponding entry in `indices` with the index of the entry itself: the\nindex-value for dimension = axis is obtained from the value of the corresponding\nentry in `indices` and the index-value for dimension != axis is obtained from the\nindex of the entry itself.\n\nFor instance, in a 2-D tensor case, the update corresponding to the [i][j] entry\nis performed as below:\n```\n output[indices[i][j]][j] = updates[i][j] if axis = 0, \n output[i][indices[i][j]] = updates[i][j] if axis = 1,\n```\n\nThis operator is the inverse of GatherElements. It is similar to Torch\'s Scatter operation.\n\nExample 1:\n```\n data = [\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n ]\n indices = [\n [1, 0, 2],\n [0, 2, 1],\n ]\n updates = [\n [1.0, 1.1, 1.2],\n [2.0, 2.1, 2.2],\n ]\n output = [\n [2.0, 1.1, 0.0]\n [1.0, 0.0, 2.2]\n [0.0, 2.1, 1.2]\n ]\n```\nExample 2:\n```\n data = [[1.0, 2.0, 3.0, 4.0, 5.0]]\n indices = [[1, 3]]\n updates = [[1.1, 2.1]]\n axis = 1\n output = [[1.0, 1.1, 3.0, 2.1, 5.0]]\n```\n" -----f -input: "data" -input: "indices" -input: "updates" -output: "output" -name: "ScatterElements" -op_type: "ScatterElements" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "updates-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nScatterElements takes three inputs `data`, `updates`, and `indices` of the same\nrank r >= 1 and an optional attribute axis that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value\nto values specified by `updates` at specific index positions specified by\n`indices`. Its output shape is the same as the shape of `data`.\n\nFor each entry in `updates`, the target index in `data` is obtained by combining\nthe corresponding entry in `indices` with the index of the entry itself: the\nindex-value for dimension = axis is obtained from the value of the corresponding\nentry in `indices` and the index-value for dimension != axis is obtained from the\nindex of the entry itself.\n\nFor instance, in a 2-D tensor case, the update corresponding to the [i][j] entry\nis performed as below:\n```\n output[indices[i][j]][j] = updates[i][j] if axis = 0, \n output[i][indices[i][j]] = updates[i][j] if axis = 1,\n```\n\nThis operator is the inverse of GatherElements. It is similar to Torch\'s Scatter operation.\n\nExample 1:\n```\n data = [\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n ]\n indices = [\n [1, 0, 2],\n [0, 2, 1],\n ]\n updates = [\n [1.0, 1.1, 1.2],\n [2.0, 2.1, 2.2],\n ]\n output = [\n [2.0, 1.1, 0.0]\n [1.0, 0.0, 2.2]\n [0.0, 2.1, 1.2]\n ]\n```\nExample 2:\n```\n data = [[1.0, 2.0, 3.0, 4.0, 5.0]]\n indices = [[1, 3]]\n updates = [[1.1, 2.1]]\n axis = 1\n output = [[1.0, 1.1, 3.0, 2.1, 5.0]]\n```\n" -----f -input: "data" -input: "indices" -input: "updates" -output: "output" -name: "ScatterND" -op_type: "ScatterND" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "indices-types" - strings: "int64" - type: STRINGS -} -attribute { - name: "updates-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nScatterND takes three inputs `data` tensor of rank r >= 1, `indices` tensor of rank q >= 1,\nand `updates` tensor of rank q + r - indices.shape[-1] - 1. The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value to values\nspecified by `updates` at specific index positions specified by `indices`. Its output shape\nis the same as the shape of `data`. Note that `indices` should not have duplicate entries.\nThat is, two or more `updates` for the same index-location is not supported.\n\n`indices` is an integer tensor. Let k denote indices.shape[-1], the last dimension in the shape of `indices`.\n `indices` is treated as a (q-1)-dimensional tensor of k-tuples, where each k-tuple is a partial-index into `data`.\nHence, k can be a value at most the rank of `data`. When k equals rank(data), each update entry specifies an\nupdate to a single element of the tensor. When k is less than rank(data) each update entry specifies an\nupdate to a slice of the tensor.\n\n`updates` is treated as a (q-1)-dimensional tensor of replacement-slice-values. Thus, the\nfirst (q-1) dimensions of updates.shape must match the first (q-1) dimensions of indices.shape.\nThe remaining dimensions of `updates` correspond to the dimensions of the\nreplacement-slice-values. Each replacement-slice-value is a (r-k) dimensional tensor,\ncorresponding to the trailing (r-k) dimensions of `data`. Thus, the shape of `updates`\nmust equal indices.shape[0:q-1] ++ data.shape[k:r-1], where ++ denotes the concatenation\nof shapes.\n\nThe `output` is calculated via the following equation:\n\n output = np.copy(data)\n update_indices = indices.shape[:-1]\n for idx in np.ndindex(update_indices):\n output[indices[idx]] = updates[idx]\n\nThe order of iteration in the above loop is not specified.\nIn particular, indices should not have duplicate entries: that is, if idx1 != idx2, then indices[idx1] != indices[idx2].\nThis ensures that the output value does not depend on the iteration order.\n\nThis operator is the inverse of GatherND.\n\nExample 1:\n```\n data = [1, 2, 3, 4, 5, 6, 7, 8]\n indices = [[4], [3], [1], [7]]\n updates = [9, 10, 11, 12]\n output = [1, 11, 3, 10, 9, 6, 7, 12]\n```\n\nExample 2:\n```\n data = [[[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]]]\n indices = [[0], [2]]\n updates = [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],\n [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]]\n output = [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],\n [[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]]]\n```\n" -----f -input: "X" -output: "Y" -name: "Selu" -op_type: "Selu" -attribute { - name: "alpha" - f: 1.6732632 - type: FLOAT -} -attribute { - name: "gamma" - f: 1.050701 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nSelu takes one input data (Tensor) and produces one output data\n(Tensor) where the scaled exponential linear unit function,\n`y = gamma * (alpha * e^x - alpha) for x <= 0`, `y = gamma * x for x > 0`,\nis applied to the tensor elementwise.\n" -----f -input: "input_sequence" -input: "position" -output: "tensor" -name: "SequenceAt" -op_type: "SequenceAt" -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -attribute { - name: "position-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nOutputs a tensor copy from the tensor at \'position\' in \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n - 1]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n" -----f -input: "inputs" -output: "output_sequence" -name: "SequenceConstruct" -op_type: "SequenceConstruct" -attribute { - name: "inputs-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nConstruct a tensor sequence containing \'inputs\' tensors.\nAll tensors in \'inputs\' must have the same data type.\n" -----f -output: "output" -name: "SequenceEmpty" -op_type: "SequenceEmpty" -attribute { - name: "dtype" - s: "" - type: INT -} -doc_string: "\nConstruct an empty tensor sequence, with given data type.\n" -----f -input: "input_sequence" -input: "position" -output: "output_sequence" -name: "SequenceErase" -op_type: "SequenceErase" -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -attribute { - name: "position-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nOutputs a tensor sequence that removes the tensor at \'position\' from \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n - 1]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n\'position\' is optional, by default it erases the last tensor from \'input_sequence\'.\n" -----f -input: "input_sequence" -input: "tensor" -input: "position" -output: "output_sequence" -name: "SequenceInsert" -op_type: "SequenceInsert" -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -attribute { - name: "tensor-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "position-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nOutputs a tensor sequence that inserts \'tensor\' into \'input_sequence\' at \'position\'.\n\'tensor\' must have the same data type as \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n\'position\' is optional, by default it inserts \'tensor\' to the back of \'input_sequence\'.\n" -----f -input: "input_sequence" -output: "length" -name: "SequenceLength" -op_type: "SequenceLength" -attribute { - name: "input_sequence-types" - strings: "seq(bool" - strings: "seq(complex128" - strings: "seq(string" - strings: "seq(float16" - strings: "seq(int64" - strings: "seq(float" - strings: "seq(int32" - strings: "seq(uint32" - strings: "seq(uint16" - strings: "seq(int8" - strings: "seq(int16" - strings: "seq(complex64" - strings: "seq(uint64" - strings: "seq(double" - strings: "seq(uint8" - type: STRINGS -} -doc_string: "\nProduces a scalar(tensor of empty shape) containing the number of tensors in \'input_sequence\'.\n" -----f -input: "data" -output: "shape" -name: "Shape" -op_type: "Shape" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nTakes a tensor as input and outputs an 1D int64 tensor containing the shape of the input tensor.\n" -----f -input: "input" -output: "output" -name: "Shrink" -op_type: "Shrink" -attribute { - name: "bias" - f: 0.0 - type: FLOAT -} -attribute { - name: "lambd" - f: 0.5 - type: FLOAT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nShrink takes one input data (Tensor) and produces one Tensor output,\nhaving same datatype and shape with input. It has two attributes, lambd and\nbias. The formula of this operator is: If x < -lambd, y = x + bias;\nIf x > lambd, y = x - bias; Otherwise, y = 0.\n" -----f -input: "X" -output: "Y" -name: "Sigmoid" -op_type: "Sigmoid" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nSigmoid takes one input data (Tensor) and produces one output data\n(Tensor) where the sigmoid function, y = 1 / (1 + exp(-x)), is applied to the\ntensor elementwise.\n" -----f -input: "input" -output: "output" -name: "Sign" -op_type: "Sign" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nCalculate the sign of the given input tensor element-wise.\nIf input > 0, output 1. if input < 0, output -1. if input == 0, output 0.\n" -----f -input: "input" -output: "output" -name: "Sin" -op_type: "Sin" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the sine of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Sinh" -op_type: "Sinh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic sine of the given input tensor element-wise.\n" -----f -input: "data" -output: "size" -name: "Size" -op_type: "Size" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nTakes a tensor as input and outputs a int64 scalar that equals to the total number of elements of the input tensor.\n" -----f -input: "data" -input: "starts" -input: "ends" -input: "axes" -input: "steps" -output: "output" -name: "Slice" -op_type: "Slice" -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "starts-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "ends-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "axes-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "steps-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nProduces a slice of the input tensor along multiple axes. Similar to numpy:\nhttps://docs.scipy.org/doc/numpy/reference/arrays.indexing.html\nSlices uses `starts`, `ends`, `axes` and `steps` inputs to specify the start and end\ndimension and step for each axis in the list of axes, it uses this information to\nslice the input `data` tensor. If a negative value is passed for any of the\nstart or end indices, it represents number of elements before the end of that\ndimension. If the value passed to start or end is larger than the `n` (the\nnumber of elements in this dimension), it represents `n`. For slicing to the\nend of a dimension with unknown size, it is recommended to pass in `INT_MAX` \nwhen sclicing forward and \'INT_MIN\' when slicing backward.\nIf a negative value is passed for step, it represents slicing backward. \nHowever step value cannot be 0.\nIf `axes` are omitted, they are set to `[0, ..., ndim-1]`.\nIf `steps` are omitted, they are set to `[1, ..., 1]` of length `len(starts)`\nExample 1:\n data = [\n [1, 2, 3, 4],\n [5, 6, 7, 8],\n ]\n axes = [0, 1]\n starts = [1, 0]\n ends = [2, 3]\n steps = [1, 2]\n result = [\n [5, 7],\n ]\nExample 2:\n data = [\n [1, 2, 3, 4],\n [5, 6, 7, 8],\n ]\n starts = [0, 1]\n ends = [-1, 1000]\n result = [\n [2, 3, 4],\n ]\n" -----f -input: "input" -output: "output" -name: "Softmax" -op_type: "Softmax" -attribute { - name: "axis" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThe operator computes the softmax (normalized exponential) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the softmax values of the corresponding input.\n" -----f -input: "scores" -input: "labels" -input: "weights" -output: "output" -output: "log_prob" -name: "SoftmaxCrossEntropyLoss" -op_type: "SoftmaxCrossEntropyLoss" -attribute { - name: "ignore_index" - s: "" - type: INT -} -attribute { - name: "reduction" - s: "mean" - type: STRING -} -attribute { - name: "scores-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -attribute { - name: "labels-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -attribute { - name: "weights-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "Loss function that measures the softmax cross entropy\nbetween \'scores\' and \'labels\'.\nThis operator first computes a loss tensor whose shape is identical to the labels input.\nIf the input is 2-D with shape (N, C), the loss tensor may be a N-element vector L = (l_1, l_2, ..., l_N).\nIf the input is N-D tensor with shape (N, C, D1, D2, ..., Dk),\nthe loss tensor L may have (N, D1, D2, ..., Dk) as its shape and L[i,][j_1][j_2]...[j_k] denotes a scalar element in L.\nAfter L is available, this operator can optionally do a reduction operator.\n\nshape(scores): (N, C) where C is the number of classes, or (N, C, D1, D2,..., Dk),\n with K >= 1 in case of K-dimensional loss.\nshape(labels): (N) where each value is 0 <= labels[i] <= C-1, or (N, D1, D2,..., Dk),\n with K >= 1 in case of K-dimensional loss.\n\nThe loss for one sample, l_i, can caculated as follows:\n l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk], where i is the index of classes.\nor\n l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk] * weights[c], if \'weights\' is provided.\n\nloss is zero for the case when label-value equals ignore_index.\n l[i][d1][d2]...[dk] = 0, when labels[n][d1][d2]...[dk] = ignore_index\n\nwhere:\n p = Softmax(scores)\n y = Log(p)\n c = labels[i][d1][d2]...[dk]\n\nFinally, L is optionally reduced:\nIf reduction = \'none\', the output is L with shape (N, D1, D2, ..., Dk).\nIf reduction = \'sum\', the output is scalar: Sum(L).\nIf reduction = \'mean\', the output is scalar: ReduceMean(L), or if weight is provided: ReduceSum(L) / ReduceSum(W),\nwhere tensor W is of shape (N, D1, D2, ..., Dk) and W[n][d1][d2]...[dk] = weights[labels[i][d1][d2]...[dk]].\n" -----f -input: "X" -output: "Y" -name: "Softplus" -op_type: "Softplus" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nSoftplus takes one input data (Tensor) and produces one output data\n(Tensor) where the softplus function, y = ln(exp(x) + 1), is applied to\nthe tensor elementwise.\n" -----f -input: "input" -output: "output" -name: "Softsign" -op_type: "Softsign" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the softsign (x/(1+|x|)) of the given input tensor element-wise.\n" -----f -input: "input" -output: "output" -name: "SpaceToDepth" -op_type: "SpaceToDepth" -attribute { - name: "blocksize" - s: "" - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "SpaceToDepth rearranges blocks of spatial data into depth. More specifically,\nthis op outputs a copy of the input tensor where values from the height and width dimensions\nare moved to the depth dimension.\n" -----f -input: "input" -output: "outputs" -name: "Split" -op_type: "Split" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "split" - s: "" - type: INTS -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "Split a tensor into a list of tensors, along the specified\n\'axis\'. Lengths of the parts can be specified using argument \'split\'.\nOtherwise, the tensor is split to equal sized parts.\n" -----f -input: "input" -input: "split" -output: "output_sequence" -name: "SplitToSequence" -op_type: "SplitToSequence" -attribute { - name: "axis" - i: 0 - type: INT -} -attribute { - name: "keepdims" - i: 1 - type: INT -} -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "split-types" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "Split a tensor into a sequence of tensors, along the specified\n\'axis\'. Lengths of the parts can be specified using argument \'split\'.\n\'split\' must contain only positive numbers.\n\'split\' is either a scalar (tensor of empty shape), or a 1-D tensor.\nIf \'split\' is a scalar, then \'input\' will be split into equally sized chunks(if possible).\nLast chunk will be smaller if the \'input\' size along the given axis \'axis\' is not divisible\nby \'split\'.\nOtherwise, the tensor is split into \'size(split)\' chunks, with lengths of the parts on \'axis\'\nspecified in \'split\'. In this scenario, the sum of entries in \'split\' must be equal to the\ndimension size of input tensor on \'axis\'.\n" -----f -input: "X" -output: "Y" -name: "Sqrt" -op_type: "Sqrt" -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nSquare root takes one input data (Tensor) and produces one output data\n(Tensor) where the square root is, y = x^0.5, is applied to\nthe tensor elementwise. If x is negative, then it will return NaN.\n" -----f -input: "data" -output: "squeezed" -name: "Squeeze" -op_type: "Squeeze" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nRemove single-dimensional entries from the shape of a tensor.\nTakes a parameter `axes` with a list of axes to squeeze.\nIf `axes` is not provided, all the single dimensions will be removed from\nthe shape. If an axis is selected with shape entry not equal to one, an error is raised.\n" -----f -input: "X" -output: "Y" -name: "StringNormalizer" -op_type: "StringNormalizer" -attribute { - name: "case_change_action" - s: "NONE" - type: STRING -} -attribute { - name: "is_case_sensitive" - i: 0 - type: INT -} -attribute { - name: "locale" - s: "" - type: STRING -} -attribute { - name: "stopwords" - s: "" - type: STRINGS -} -attribute { - name: "X-types" - strings: "string" - type: STRINGS -} -doc_string: "\nStringNormalization performs string operations for basic cleaning.\nThis operator has only one input (denoted by X) and only one output\n(denoted by Y). This operator first examines the elements in the X,\nand removes elements specified in \"stopwords\" attribute.\nAfter removing stop words, the intermediate result can be further lowercased,\nuppercased, or just returned depending the \"case_change_action\" attribute.\nThis operator only accepts [C]- and [1, C]-tensor.\nIf all elements in X are dropped, the output will be the empty value of string tensor with shape [1]\nif input shape is [C] and shape [1, 1] if input shape is [1, C].\n" -----f -input: "A" -input: "B" -output: "C" -name: "Sub" -op_type: "Sub" -attribute { - name: "A-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -attribute { - name: "B-types" - strings: "float16" - strings: "int32" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "uint64" - type: STRINGS -} -doc_string: "\nPerforms element-wise binary subtraction (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "data_0" -output: "sum" -name: "Sum" -op_type: "Sum" -attribute { - name: "data_0-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nElement-wise sum of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "input" -output: "output" -name: "Tan" -op_type: "Tan" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the tangent of the given input tensor, element-wise.\n" -----f -input: "input" -output: "output" -name: "Tanh" -op_type: "Tanh" -attribute { - name: "input-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nCalculates the hyperbolic tangent of the given input tensor element-wise.\n" -----f -input: "X" -output: "Y" -name: "TfIdfVectorizer" -op_type: "TfIdfVectorizer" -attribute { - name: "max_gram_length" - s: "" - type: INT -} -attribute { - name: "max_skip_count" - s: "" - type: INT -} -attribute { - name: "min_gram_length" - s: "" - type: INT -} -attribute { - name: "mode" - s: "" - type: STRING -} -attribute { - name: "ngram_counts" - s: "" - type: INTS -} -attribute { - name: "ngram_indexes" - s: "" - type: INTS -} -attribute { - name: "pool_int64s" - s: "" - type: INTS -} -attribute { - name: "pool_strings" - s: "" - type: STRINGS -} -attribute { - name: "weights" - s: "" - type: FLOATS -} -attribute { - name: "X-types" - strings: "string" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\nThis transform extracts n-grams from the input sequence and save them as a vector. Input can\nbe either a 1-D or 2-D tensor. For 1-D input, output is the n-gram representation of that input.\nFor 2-D input, the output is also a 2-D tensor whose i-th row is the n-gram representation of the i-th input row.\nMore specifically, if input shape is [C], the corresponding output shape would be [max(ngram_indexes) + 1].\nIf input shape is [N, C], this operator produces a [N, max(ngram_indexes) + 1]-tensor.\n\nIn contrast to standard n-gram extraction, here, the indexes of extracting an n-gram from the original\nsequence are not necessarily consecutive numbers. The discontinuity between indexes are controlled by the number of skips.\nIf the number of skips is 2, we should skip two tokens when scanning through the original sequence.\nLet\'s consider an example. Assume that input sequence is [94, 17, 36, 12, 28] and the number of skips is 2.\nThe associated 2-grams are [94, 12] and [17, 28] respectively indexed by [0, 3] and [1, 4].\nIf the number of skips becomes 0, the 2-grams generated are [94, 17], [17, 36], [36, 12], [12, 28]\nindexed by [0, 1], [1, 2], [2, 3], [3, 4], respectively.\n\nThe output vector (denoted by Y) stores the count of each n-gram;\nY[ngram_indexes[i]] indicates the times that the i-th n-gram is found. The attribute ngram_indexes is used to determine the mapping\nbetween index i and the corresponding n-gram\'s output coordinate. If pool_int64s is [94, 17, 17, 36], ngram_indexes is [1, 0],\nngram_counts=[0, 0], then the Y[0] (first element in Y) and Y[1] (second element in Y) are the counts of [17, 36] and [94, 17],\nrespectively. An n-gram which cannot be found in pool_strings/pool_int64s should be ignored and has no effect on the output.\nNote that we may consider all skips up to S when generating the n-grams.\n\nThe examples used above are true if mode is \"TF\". If mode is \"IDF\", all the counts larger than 1 would be truncated to 1 and\nthe i-th element in weights would be used to scale (by multiplication) the count of the i-th n-gram in pool. If mode is \"TFIDF\",\nthis operator first computes the counts of all n-grams and then scale them by the associated values in the weights attribute.\n\nOnly one of pool_strings and pool_int64s can be set. If pool_int64s is set, the input should be an integer tensor.\nIf pool_strings is set, the input must be a string tensor.\n" -----f -input: "X" -output: "Y" -name: "ThresholdedRelu" -op_type: "ThresholdedRelu" -attribute { - name: "alpha" - f: 1.0 - type: FLOAT -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "float16" - type: STRINGS -} -doc_string: "\nThresholdedRelu takes one input data (Tensor) and produces one output data\n(Tensor) where the rectified linear function, y = x for x > alpha, y = 0 otherwise,\nis applied to the tensor elementwise.\n" -----f -input: "input" -input: "repeats" -output: "output" -name: "Tile" -op_type: "Tile" -attribute { - name: "input-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "repeats-types" - strings: "int64" - type: STRINGS -} -doc_string: "Constructs a tensor by tiling a given tensor.\nThis is the same as function `tile` in Numpy, but no broadcast.\nFor example A = [[1, 2], [3, 4]], B = [1, 2], tile(A, B) = [[1, 2, 1, 2], [3, 4, 3, 4]]\n" -----f -input: "X" -input: "K" -output: "Values" -output: "Indices" -name: "TopK" -op_type: "TopK" -attribute { - name: "axis" - i: -1 - type: INT -} -attribute { - name: "largest" - i: 1 - type: INT -} -attribute { - name: "sorted" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "float" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "K-types" - strings: "int64" - type: STRINGS -} -doc_string: "\nRetrieve the top-K largest or smallest elements along a specified axis. Given an input tensor of\nshape [a_1, a_2, ..., a_n, r] and integer argument k, return two outputs:\n -Value tensor of shape [a_1, a_2, ..., a_{axis-1}, k, a_{axis+1}, ... a_n]\n which contains the values of the top k elements along the specified axis\n -Index tensor of shape [a_1, a_2, ..., a_{axis-1}, k, a_{axis+1}, ... a_n] which\n contains the indices of the top k elements (original indices from the input\n tensor).\n\nIf \"largest\" is 1 (the default value) then the k largest elements are returned.\nIf \"sorted\" is 1 (the default value) then the resulting k elements will be sorted.\nIf \"sorted\" is 0, order of returned \'Values\' and \'Indices\' are undefined.\n\nGiven two equivalent values, this operator uses the indices along the axis as\n a tiebreaker. That is, the element with the lower index will appear first.\n" -----f -input: "data" -output: "transposed" -name: "Transpose" -op_type: "Transpose" -attribute { - name: "perm" - s: "" - type: INTS -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nTranspose the input tensor similar to numpy.transpose. For example, when\nperm=(1, 0, 2), given an input tensor of shape (1, 2, 3), the output shape\nwill be (2, 1, 3).\n" -----f -input: "X" -output: "Y" -output: "Z" -name: "TreeEnsembleClassifier" -op_type: "TreeEnsembleClassifier" -attribute { - name: "base_values" - s: "" - type: FLOATS -} -attribute { - name: "class_ids" - s: "" - type: INTS -} -attribute { - name: "class_nodeids" - s: "" - type: INTS -} -attribute { - name: "class_treeids" - s: "" - type: INTS -} -attribute { - name: "class_weights" - s: "" - type: FLOATS -} -attribute { - name: "classlabels_int64s" - s: "" - type: INTS -} -attribute { - name: "classlabels_strings" - s: "" - type: STRINGS -} -attribute { - name: "nodes_falsenodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_featureids" - s: "" - type: INTS -} -attribute { - name: "nodes_hitrates" - s: "" - type: FLOATS -} -attribute { - name: "nodes_missing_value_tracks_true" - s: "" - type: INTS -} -attribute { - name: "nodes_modes" - s: "" - type: STRINGS -} -attribute { - name: "nodes_nodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_treeids" - s: "" - type: INTS -} -attribute { - name: "nodes_truenodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_values" - s: "" - type: FLOATS -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Tree Ensemble classifier. Returns the top class for each of N inputs.
\n The attributes named \'nodes_X\' form a sequence of tuples, associated by \n index into the sequences, which must all be of equal length. These tuples\n define the nodes.
\n Similarly, all fields prefixed with \'class_\' are tuples of votes at the leaves.\n A leaf may have multiple votes, where each vote is weighted by\n the associated class_weights index.
\n One and only one of classlabels_strings or classlabels_int64s\n will be defined. The class_ids are indices into this list.\n" -----f -input: "X" -output: "Y" -name: "TreeEnsembleRegressor" -op_type: "TreeEnsembleRegressor" -attribute { - name: "aggregate_function" - s: "SUM" - type: STRING -} -attribute { - name: "base_values" - s: "" - type: FLOATS -} -attribute { - name: "n_targets" - s: "" - type: INT -} -attribute { - name: "nodes_falsenodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_featureids" - s: "" - type: INTS -} -attribute { - name: "nodes_hitrates" - s: "" - type: FLOATS -} -attribute { - name: "nodes_missing_value_tracks_true" - s: "" - type: INTS -} -attribute { - name: "nodes_modes" - s: "" - type: STRINGS -} -attribute { - name: "nodes_nodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_treeids" - s: "" - type: INTS -} -attribute { - name: "nodes_truenodeids" - s: "" - type: INTS -} -attribute { - name: "nodes_values" - s: "" - type: FLOATS -} -attribute { - name: "post_transform" - s: "NONE" - type: STRING -} -attribute { - name: "target_ids" - s: "" - type: INTS -} -attribute { - name: "target_nodeids" - s: "" - type: INTS -} -attribute { - name: "target_treeids" - s: "" - type: INTS -} -attribute { - name: "target_weights" - s: "" - type: FLOATS -} -attribute { - name: "X-types" - strings: "float" - strings: "double" - strings: "int32" - strings: "int64" - type: STRINGS -} -doc_string: "\n Tree Ensemble regressor. Returns the regressed values for each input in N.
\n All args with nodes_ are fields of a tuple of tree nodes, and\n it is assumed they are the same length, and an index i will decode the\n tuple across these inputs. Each node id can appear only once\n for each tree id.
\n All fields prefixed with target_ are tuples of votes at the leaves.
\n A leaf may have multiple votes, where each vote is weighted by\n the associated target_weights index.
\n All trees must have their node ids start at 0 and increment by 1.
\n Mode enum is BRANCH_LEQ, BRANCH_LT, BRANCH_GTE, BRANCH_GT, BRANCH_EQ, BRANCH_NEQ, LEAF\n" -----f -input: "X" -output: "Y" -output: "indices" -output: "inverse_indices" -output: "counts" -name: "Unique" -op_type: "Unique" -attribute { - name: "axis" - s: "" - type: INT -} -attribute { - name: "sorted" - i: 1 - type: INT -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nFind the unique elements of a tensor. When an optional attribute \'axis\' is provided, unique subtensors sliced along the \'axis\' are returned. \nOtherwise the input tensor is flattened and unique values of the flattened tensor are returned. \n\nThis operator returns the unique values or sliced unique subtensors of the input tensor and three optional outputs. \nThe first output tensor \'Y\' contains all unique values or subtensors of the input. \nThe second optional output tensor \'indices\' contains indices of \'Y\' elements\' first occurance in \'X\'.. \nThe third optional output tensor \'inverse_indices\' contains, for elements of \'X\', its corresponding indices in \'Y\'. \". \nThe fourth optional output tensor \'counts\' contains the count of each element of \'Y\' in the input. \n\nOutputs are either sorted in ascending order or optionally in the order of the first occurrence of the values in the input. \n\nhttps://docs.scipy.org/doc/numpy/reference/generated/numpy.unique.html\n\nExample 1:\n input_X = [2, 1, 1, 3, 4, 3]\n attribute_sorted = 0\n attribute_axis = None\n output_Y = [2, 1, 3, 4]\n output_indices = [0, 1, 3, 4]\n output_inverse_indices = [0, 1, 1, 2, 3, 2]\n output_counts = [1, 2, 2, 1]\n\nExample 2:\n input_X = [[1, 3], [2, 3]]\n attribute_sorted = 1\n attribute_axis = None\n output_Y = [1, 2, 3]\n output_indices = [0, 2, 1]\n output_inverse_indices = [0, 2, 1, 2]\n output_counts = [1, 1, 2]\n\nExample 3:\n input_X = [[1, 0, 0], [1, 0, 0], [2, 3, 4]]\n attribute_sorted = 1\n attribute_axis = 0\n output_Y = [[1, 0, 0], [2, 3, 4]]\n output_indices = [0, 2]\n output_inverse_indices = [0, 0, 1]\n output_counts = [2, 1]\n\nExample 4:\n input_x = [[[1., 1.], [0., 1.], [2., 1.], [0., 1.]], \n [[1., 1.], [0., 1.], [2., 1.], [0., 1.]]]\n attribute_sorted = 1\n attribute_axis = 1\n\n intermediate data are presented below for better understanding: \n \n there are 4 subtensors sliced along axis 1 of input_x (shape = (2, 4, 2)):\n A: [[1, 1], [1, 1]], \n [[0, 1], [0, 1]], \n [[2, 1], [2, 1]], \n [[0, 1], [0, 1]].\n \n there are 3 unique subtensors: \n [[1, 1], [1, 1]], \n [[0, 1], [0, 1]], \n [[2, 1], [2, 1]].\n \n sorted unique subtensors:\n B: [[0, 1], [0, 1]], \n [[1, 1], [1, 1]], \n [[2, 1], [2, 1]].\n \n output_Y is constructed from B:\n [[[0. 1.], [1. 1.], [2. 1.]], \n [[0. 1.], [1. 1.], [2. 1.]]]\n\n output_indices is to map from B to A:\n [1, 0, 2]\n \n output_inverse_indices is to map from A to B:\n [1, 0, 2, 0]\n\n output_counts = [2 1 1]\n" -----f -input: "data" -output: "expanded" -name: "Unsqueeze" -op_type: "Unsqueeze" -attribute { - name: "axes" - s: "" - type: INTS -} -attribute { - name: "data-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\nInsert single-dimensional entries to the shape of an input tensor (`data`).\nTakes one required argument `axes` - which contains a list of dimension indices and this operator will insert a dimension of value `1` into the corresponding index of the output tensor (`expanded`).\n\nFor example:\n Given an input tensor (`data`) of shape [3, 4, 5], then\n Unsqueeze(data, axes=[0, 4]) outputs a tensor (`expanded`) containing same data as `data` but with shape [1, 3, 4, 5, 1].\n\nThe attribute `axes` should not contain any duplicate entries. It is an error if it contains duplicates.\nThe rank of the output tensor (`output_rank`) is the rank of the input tensor (`data`) plus the number of values in `axes`.\nEach value in `axes` should be within the (inclusive) range [-output_rank , output_rank - 1]. \nThe order of values in `axes` does not matter and can come in any order. \n\n" -----f -input: "X" -input: "scales" -output: "Y" -name: "Upsample" -op_type: "Upsample" -attribute { - name: "mode" - s: "nearest" - type: STRING -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "scales-types" - strings: "float" - type: STRINGS -} -doc_string: "\nUpsample the input tensor.\nEach dimension value of the output tensor is:\n output_dimension = floor(input_dimension * scale).\n" -----f -input: "condition" -input: "X" -input: "Y" -output: "output" -name: "Where" -op_type: "Where" -attribute { - name: "condition-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "X-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -attribute { - name: "Y-types" - strings: "uint16" - strings: "int8" - strings: "bool" - strings: "int32" - strings: "float16" - strings: "uint8" - strings: "string" - strings: "double" - strings: "int64" - strings: "uint32" - strings: "complex64" - strings: "float" - strings: "complex128" - strings: "int16" - strings: "uint64" - type: STRINGS -} -doc_string: "\n Return elements, either from X or Y, depending on condition\n (with Numpy-style broadcasting support).\n Where behaves like numpy.where with three parameters:\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.where.html\n" -----f -input: "A" -input: "B" -output: "C" -name: "Xor" -op_type: "Xor" -attribute { - name: "A-types" - strings: "bool" - type: STRINGS -} -attribute { - name: "B-types" - strings: "bool" - type: STRINGS -} -doc_string: "\nReturns the tensor resulted from performing the `xor` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" -----f -input: "X" -output: "Z" -name: "ZipMap" -op_type: "ZipMap" -attribute { - name: "classlabels_int64s" - s: "" - type: INTS -} -attribute { - name: "classlabels_strings" - s: "" - type: STRINGS -} -attribute { - name: "X-types" - strings: "float" - type: STRINGS -} -doc_string: "\n Creates a map from the input and the attributes.
\n The values are provided by the input tensor, while the keys are specified by the attributes.\n Must provide keys in either classlabels_strings or classlabels_int64s (but not both).
\n The columns of the tensor correspond one-by-one to the keys specified by the attributes. There must be as many columns as keys.
\n" -----f +input: "X" +output: "Y" +name: "Abs" +op_type: "Abs" +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nAbsolute takes one input data (Tensor) and produces one output data\n(Tensor) where the absolute is, y = abs(x), is applied to\nthe tensor elementwise.\n" +----f +input: "input" +output: "output" +name: "Acos" +op_type: "Acos" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the arccosine (inverse of cosine) of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Acosh" +op_type: "Acosh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic arccosine of the given input tensor element-wise.\n" +----f +input: "R" +input: "T" +input: "inputs" +output: "outputs" +name: "Adagrad" +op_type: "Adagrad" +attribute { + name: "decay_factor" + f: 0.0 + type: FLOAT +} +attribute { + name: "epsilon" + f: 1e-06 + type: FLOAT +} +attribute { + name: "norm_coefficient" + f: 0.0 + type: FLOAT +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + type: STRINGS +} +attribute { + name: "T-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "inputs-types" + strings: "float" + strings: "double" + type: STRINGS +} +doc_string: "\n Compute one iteration of ADAGRAD, a stochastic gradient based optimization\n algorithm. This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. As you can imagine, ADAGRAD requires\n some parameters:\n \n - The initial learning-rate \"R\".\n - The update count \"T\". That is, the number of training iterations conducted.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A learning-rate decay factor \"decay_factor\".\n - A small constant \"epsilon\" to avoid dividing-by-zero. \n\n At each ADAGRAD iteration, the optimized tensors are moved along a direction\n computed based on their estimated gradient and accumulated squared gradient. Assume\n that only a single tensor \"X\" is updated by this operator. We need the value of \"X\",\n its gradient \"G\", and its accumulated squared gradient \"H\". Therefore, variables in\n this operator\'s input list are sequentially \"R\", \"T\", \"X\", \"G\", and \"H\". Other\n parameters are given as attributes because they are usually constants. Also, the\n corresponding output tensors are the new value of \"X\" (called \"X_new\"), and then\n the new accumulated squared gradient (called \"H_new\"). Those outputs are computed\n from the given inputs following the pseudo code below.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise arithmetic operations with\n numpy-style broadcasting support. The pseudo code to compute those outputs is:\n\n // Compute a scalar learning-rate factor. At the first update of X, T is generally\n // 0 (0-based update index) or 1 (1-based update index).\n r = R / (1 + T * decay_factor);\n\n // Add gradient of 0.5 * norm_coefficient * ||X||_2^2, where ||X||_2 is the 2-norm.\n G_regularized = norm_coefficient * X + G;\n\n // Compute new accumulated squared gradient.\n H_new = H + G_regularized * G_regularized;\n\n // Compute the adaptive part of per-coordinate learning rate. Note that Sqrt(...)\n // computes element-wise square-root.\n H_adaptive = Sqrt(H_new) + epsilon\n\n // Compute the new value of \"X\".\n X_new = X - r * G_regularized / H_adaptive;\n\n If one assign this operators to optimize multiple inputs, for example, \"X_1\" and \"X_2\", the same\n pseudo code may be extended to handle all tensors jointly. More specifically, we can view \"X\" as a\n concatenation of \"X_1\" and \"X_2\" (of course, their gradient and accumulate gradient should\n be concatenated too) and then just reuse the entire pseudo code.\n\n Note that ADAGRAD was first proposed in http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf.\n In that reference paper, this operator is a special case of the Figure 1\'s composite mirror\n descent update.\n" +----f +input: "R" +input: "T" +input: "inputs" +output: "outputs" +name: "Adam" +op_type: "Adam" +attribute { + name: "alpha" + f: 0.9 + type: FLOAT +} +attribute { + name: "beta" + f: 0.999 + type: FLOAT +} +attribute { + name: "epsilon" + f: 1e-06 + type: FLOAT +} +attribute { + name: "norm_coefficient" + f: 0.0 + type: FLOAT +} +attribute { + name: "norm_coefficient_post" + f: 0.0 + type: FLOAT +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + type: STRINGS +} +attribute { + name: "T-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "inputs-types" + strings: "float" + strings: "double" + type: STRINGS +} +doc_string: "\n Compute one iteration of Adam, a stochastic gradient based optimization\n algorithm. This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. First of all, Adam requires\n some parameters:\n \n - The learning-rate \"R\".\n - The update count \"T\". That is, the number of training iterations conducted.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A small constant \"epsilon\" to avoid dividing-by-zero. \n - Two coefficients, \"alpha\" and \"beta\".\n\n At each Adam iteration, the optimized tensors are moved along a direction\n computed based on their exponentially-averaged historical gradient and\n exponentially-averaged historical squared gradient. Assume that only a tensor\n \"X\" is being optimized. The rest of required information is\n \n - the value of \"X\",\n - \"X\"\'s gradient (denoted by \"G\"),\n - \"X\"\'s exponentially-averaged historical gradient (denoted by \"V\"), and\n - \"X\"\'s exponentially-averaged historical squared gradient (denoted by \"H\").\n\n Some of those parameters are passed into this operator as input tensors and others\n are stored as this operator\'s attributes. Specifically, this operator\'s input tensor\n list is [\"R\", \"T\", \"X\", \"G\", \"V\", \"H\"]. That is, \"R\" is the first input, \"T\" is\n the second input, and so on. Other parameters are given as attributes because they\n are constants. Moreover, the corresponding output tensors are \n \n - the new value of \"X\" (called \"X_new\"),\n - the new exponentially-averaged historical gradient (denoted by \"V_new\"), and\n - the new exponentially-averaged historical squared gradient (denoted by \"H_new\").\n\n Those outputs are computed following the pseudo code below.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise arithmetic operations with\n numpy-style broadcasting support. The pseudo code to compute those outputs is:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||_2^2, where ||X||_2 is the 2-norm.\n G_regularized = norm_coefficient * X + G\n\n // Update exponentially-averaged historical gradient.\n V_new = alpha * V + (1 - alpha) * G_regularized\n\n // Update exponentially-averaged historical squared gradient.\n H_new = beta * H + (1 - beta) * G_regularized * G_regularized\n\n // Compute the element-wise square-root of H_new. V_new will be element-wisely\n // divided by H_sqrt for a better update direction.\n H_sqrt = Sqrt(H_new) + epsilon\n\n // Compute learning-rate. Note that \"alpha**T\"/\"beta**T\" is alpha\'s/beta\'s T-th power.\n R_adjusted = T > 0 ? R * Sqrt(1 - beta**T) / (1 - alpha**T) : R\n\n // Compute new value of \"X\".\n X_new = X - R_adjusted * V_new / H_sqrt\n\n // Post-update regularization.\n X_final = (1 - norm_coefficient_post) * X_new \n\n If there are multiple inputs to be optimized, the pseudo code will be applied\n independently to each of them.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Add" +op_type: "Add" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPerforms element-wise binary addition (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "A" +input: "B" +output: "C" +name: "And" +op_type: "And" +attribute { + name: "A-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "B-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `and` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "data" +output: "reduced" +name: "ArgMax" +op_type: "ArgMax" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "select_last_index" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the indices of the max elements of the input tensor\'s element along the \nprovided axis. The resulting tensor has the same rank as the input if keepdims equal 1. \nIf keepdims equal 0, then the resulting tensor have the reduced dimension pruned. \nIf select_last_index is True (default False), the index of the last occurrence of the max \nis selected if the max appears more than once in the input. Otherwise the index of the \nfirst occurrence is selected.\nThe type of the output tensor is integer." +----f +input: "data" +output: "reduced" +name: "ArgMin" +op_type: "ArgMin" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "select_last_index" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the indices of the min elements of the input tensor\'s element along the \nprovided axis. The resulting tensor has the same rank as the input if keepdims equal 1. \nIf keepdims equal 0, then the resulting tensor have the reduced dimension pruned. \nIf select_last_index is True (default False), the index of the last occurrence of the min \nis selected if the min appears more than once in the input. Otherwise the index of the \nfirst occurrence is selected.\nThe type of the output tensor is integer." +----f +input: "X" +input: "Y" +output: "Z" +name: "ArrayFeatureExtractor" +op_type: "ArrayFeatureExtractor" +attribute { + name: "X-types" + strings: "int32" + strings: "string" + strings: "double" + strings: "int64" + strings: "float" + type: STRINGS +} +attribute { + name: "Y-types" + strings: "int64" + type: STRINGS +} +doc_string: "\n Select elements of the input tensor based on the indices passed.
\n The indices are applied to the last axes of the tensor.\n" +----f +input: "input" +output: "output" +name: "Asin" +op_type: "Asin" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the arcsine (inverse of sine) of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Asinh" +op_type: "Asinh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic arcsine of the given input tensor element-wise.\n" +----f +input: "input" +output: "output" +name: "Atan" +op_type: "Atan" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the arctangent (inverse of tangent) of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Atanh" +op_type: "Atanh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic arctangent of the given input tensor element-wise.\n" +----f +input: "X" +output: "Y" +name: "AveragePool" +op_type: "AveragePool" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "ceil_mode" + i: 0 + type: INT +} +attribute { + name: "count_include_pad" + i: 0 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n AveragePool consumes an input tensor X and applies average pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n average pooling consisting of computing the average on all values of a\n subset of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing. The output spatial shape will be following:\n ```\n output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1)\n ```\n or\n ```\n output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - kernel_spatial_shape[i]) / strides_spatial_shape[i] + 1)\n ```\n if ceil_mode is enabled\n\n ```\n * pad_shape[i] is sum of pads along axis i\n ```\n\n `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following:\n ```\n VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - kernel_spatial_shape[i] + 1) / strides_spatial_shape[i])\n SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i])\n ```\n And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`:\n ```\n pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + kernel_spatial_shape[i] - input_spatial_shape[i]\n ```\n The output of each pooling window is divided by the number of elements (exclude pad when attribute count_include_pad is zero).\n " +----f +input: "X" +input: "scale" +input: "B" +input: "mean" +input: "var" +output: "Y" +output: "mean" +output: "var" +output: "saved_mean" +output: "saved_var" +name: "BatchNormalization" +op_type: "BatchNormalization" +attribute { + name: "epsilon" + f: 1e-05 + type: FLOAT +} +attribute { + name: "momentum" + f: 0.9 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "scale-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "mean-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "var-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCarries out batch normalization as described in the paper\nhttps://arxiv.org/abs/1502.03167. Depending on the mode it is being run,\nthere are multiple cases for the number of outputs, which we list below:\n\nOutput case #1: Y, mean, var, saved_mean, saved_var (training mode)\nOutput case #2: Y (test mode)\n\nFor previous (depreciated) non-spatial cases, implementors are suggested\nto flatten the input shape to (N x C*D1*D2 ..*Dn) before a BatchNormalization Op.\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "X" +output: "Y" +name: "Binarizer" +op_type: "Binarizer" +attribute { + name: "threshold" + f: 0.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Maps the values of the input tensor to either 0 or 1, element-wise, based on the outcome of a comparison against a threshold value.\n" +----f +input: "X" +input: "Y" +output: "Z" +name: "BitShift" +op_type: "BitShift" +attribute { + name: "direction" + s: "" + type: STRING +} +attribute { + name: "X-types" + strings: "uint32" + strings: "uint16" + strings: "uint8" + strings: "uint64" + type: STRINGS +} +attribute { + name: "Y-types" + strings: "uint32" + strings: "uint16" + strings: "uint8" + strings: "uint64" + type: STRINGS +} +doc_string: "\nBitwise shift operator performs element-wise operation. For each input element, if the\n attribute \"direction\" is \"RIGHT\", this operator moves its binary representation toward\n the right side so that the input value is effectively decreased. If the attribute \"direction\"\n is \"LEFT\", bits of binary representation moves toward the left side, which results the\n increase of its actual value. The input X is the tensor to be shifted and another input\n Y specifies the amounts of shifting. For example, if \"direction\" is \"Right\", X is [1, 4],\n and S is [1, 1], the corresponding output Z would be [0, 2]. If \"direction\" is \"LEFT\" with\n X=[1, 2] and S=[1, 2], the corresponding output Y would be [2, 8].\n \n Because this operator supports Numpy-style broadcasting, X\'s and Y\'s shapes are\n not necessarily identical.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md)." +----f +input: "input" +output: "output" +name: "Cast" +op_type: "Cast" +attribute { + name: "to" + s: "" + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "float16" + strings: "int32" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nThe operator casts the elements of a given input tensor to a data type\nspecified by the \'to\' argument and returns an output tensor of the same size in\nthe converted type. The \'to\' argument must be one of the data types specified\nin the \'DataType\' enum field in the TensorProto message.\n\nCasting from string tensor in plain (e.g., \"3.14\" and \"1000\") and scientific numeric representations\n(e.g., \"1e-5\" and \"1E8\") to float types is supported. For example, converting string \"100.5\" to an integer may\nresult 100. There are some string literals reserved for special floating-point values;\n\"+INF\" (and \"INF\"), \"-INF\", and \"NaN\" are positive infinity, negative infinity, and not-a-number, respectively.\nAny string which can exactly match \"+INF\" in a case-insensitive way would be mapped to positive infinite. Similarly,\nthis case-insensitive rule is applied to \"INF\" and \"NaN\". When casting from numeric tensors\nto string tensors, plain floating-point representation (such as \"314.15926\") would be used. \nConverting non-numerical-literal string such as \"Hello World!\" is an undefined behavior. Cases \nof converting string representing floating-point arithmetic value, such as \"2.718\", to INT is an undefined behavior.\n\nConversion from a numerical type to any numerical type is always allowed.\nUser must be aware of precision loss and value change caused by range difference between two types.\nFor example, a 64-bit float 3.1415926459 may be round to a 32-bit float 3.141592. Similarly, converting\nan integer 36 to Boolean may produce 1 because we truncate bits which can\'t be stored in the targeted type.\n" +----f +input: "X" +output: "Y" +name: "CastMap" +op_type: "CastMap" +attribute { + name: "cast_to" + s: "TO_FLOAT" + type: STRING +} +attribute { + name: "map_form" + s: "DENSE" + type: STRING +} +attribute { + name: "max_map" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "map(int64,string" + strings: "map(int64,float" + type: STRINGS +} +doc_string: "\n Converts a map to a tensor.
The map key must be an int64 and the values will be ordered\n in ascending order based on this key.
The operator supports dense packing or sparse packing.\n If using sparse packing, the key cannot exceed the max_map-1 value.\n" +----f +input: "X" +output: "Y" +name: "CategoryMapper" +op_type: "CategoryMapper" +attribute { + name: "cats_int64s" + s: "" + type: INTS +} +attribute { + name: "cats_strings" + s: "" + type: STRINGS +} +attribute { + name: "default_int64" + i: -1 + type: INT +} +attribute { + name: "default_string" + s: "_Unused" + type: STRING +} +attribute { + name: "X-types" + strings: "string" + strings: "int64" + type: STRINGS +} +doc_string: "\n Converts strings to integers and vice versa.
\n Two sequences of equal length are used to map between integers and strings,\n with strings and integers at the same index detailing the mapping.
\n Each operator converts either integers to strings or strings to integers, depending \n on which default value attribute is provided. Only one default value attribute\n should be defined.
\n If the string default value is set, it will convert integers to strings.\n If the int default value is set, it will convert strings to integers.\n" +----f +input: "X" +output: "Y" +name: "Ceil" +op_type: "Ceil" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCeil takes one input data (Tensor) and produces one output data\n(Tensor) where the ceil is, y = ceil(x), is applied to\nthe tensor elementwise.\n" +----f +input: "X" +output: "Y" +name: "Celu" +op_type: "Celu" +attribute { + name: "alpha" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + type: STRINGS +} +doc_string: "\nContinuously Differentiable Exponential Linear Units:\nPerform the linear unit element-wise on the input tensor X\nusing formula: \n\n```\nmax(0,x) + min(0,alpha*(exp(x/alpha)-1))\n```\n" +----f +input: "input" +input: "min" +input: "max" +output: "output" +name: "Clip" +op_type: "Clip" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "min-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "max-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nClip operator limits the given input within an interval. The interval is\nspecified by the inputs \'min\' and \'max\'. They default to\nnumeric_limits::lowest() and numeric_limits::max(), respectively.\n" +----f +input: "input" +input: "condition" +output: "output" +name: "Compress" +op_type: "Compress" +attribute { + name: "axis" + s: "" + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "condition-types" + strings: "bool" + type: STRINGS +} +doc_string: "\n Selects slices from an input tensor along a given axis where condition evaluates to True for each axis index.\n In case axis is not provided, input is flattened before elements are selected.\n Compress behaves like numpy.compress: https://docs.scipy.org/doc/numpy/reference/generated/numpy.compress.html\n " +----f +input: "inputs" +output: "concat_result" +name: "Concat" +op_type: "Concat" +attribute { + name: "axis" + s: "" + type: INT +} +attribute { + name: "inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "Concatenate a list of tensors into a single tensor. All input tensors must have the same shape, except for the dimension size of the axis to concatenate on." +----f +input: "input_sequence" +output: "concat_result" +name: "ConcatFromSequence" +op_type: "ConcatFromSequence" +attribute { + name: "axis" + s: "" + type: INT +} +attribute { + name: "new_axis" + i: 0 + type: INT +} +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +doc_string: "\nConcatenate a sequence of tensors into a single tensor.\nAll input tensors must have the same shape, except for the dimension size of the axis to concatenate on.\nBy default \'new_axis\' is 0, the behavior is similar to numpy.concatenate.\nWhen \'new_axis\' is 1, the behavior is similar to numpy.stack.\n" +----f +output: "output" +name: "Constant" +op_type: "Constant" +attribute { + name: "sparse_value" + s: "" + type: SPARSE_TENSOR +} +attribute { + name: "value" + s: "" + type: TENSOR +} +attribute { + name: "value_float" + s: "" + type: FLOAT +} +attribute { + name: "value_floats" + s: "" + type: FLOATS +} +attribute { + name: "value_int" + s: "" + type: INT +} +attribute { + name: "value_ints" + s: "" + type: INTS +} +attribute { + name: "value_string" + s: "" + type: STRING +} +attribute { + name: "value_strings" + s: "" + type: STRINGS +} +doc_string: "\nThis operator produces a constant tensor. Exactly one of the provided attributes, either value, sparse_value,\nor value_* must be specified.\n" +----f +input: "input" +output: "output" +name: "ConstantOfShape" +op_type: "ConstantOfShape" +attribute { + name: "value" + s: "" + type: TENSOR +} +attribute { + name: "input-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nGenerate a tensor with given value and shape.\n" +----f +input: "X" +input: "W" +input: "B" +output: "Y" +name: "Conv" +op_type: "Conv" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "group" + i: 1 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe convolution operator consumes an input tensor and a filter, and\ncomputes the output." +----f +input: "x" +input: "w" +input: "x_zero_point" +input: "w_zero_point" +output: "y" +name: "ConvInteger" +op_type: "ConvInteger" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "group" + i: 1 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "x-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "w-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "x_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "w_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nThe integer convolution operator consumes an input tensor, its zero-point, a filter, and its zero-point,\nand computes the output. The production MUST never overflow. The accumulation may overflow if and only if in 32 bits.\n" +----f +input: "X" +input: "W" +input: "B" +output: "Y" +name: "ConvTranspose" +op_type: "ConvTranspose" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "group" + i: 1 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "output_padding" + s: "" + type: INTS +} +attribute { + name: "output_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe convolution transpose operator consumes an input tensor and a filter,\nand computes the output.\n\nIf the pads parameter is provided the shape of the output is calculated via the following equation:\n\n output_shape[i] = stride[i] * (input_size[i] - 1) + output_padding[i] + ((kernel_shape[i] - 1) * dilations[i] + 1) - pads[start_i] - pads[end_i]\n\noutput_shape can also be explicitly specified in which case pads values are auto generated using these equations:\n\n total_padding[i] = stride[i] * (input_size[i] - 1) + output_padding[i] + ((kernel_shape[i] - 1) * dilations[i] + 1) - output_shape[i]\n If (auto_pads != SAME_UPPER): pads[start_i] = total_padding[i]/2; pads[end_i] = total_padding[i] - (total_padding[i]/2)\n Else: pads[start_i] = total_padding[i] - (total_padding[i]/2); pads[end_i] = (total_padding[i]/2).\n\n " +----f +input: "input" +output: "output" +name: "Cos" +op_type: "Cos" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the cosine of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Cosh" +op_type: "Cosh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic cosine of the given input tensor element-wise.\n" +----f +input: "x" +input: "axis" +output: "y" +name: "CumSum" +op_type: "CumSum" +attribute { + name: "exclusive" + i: 0 + type: INT +} +attribute { + name: "reverse" + i: 0 + type: INT +} +attribute { + name: "x-types" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "axis-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nPerforms cumulative sum of the input elements along the given axis.\nBy default, it will do the sum inclusively meaning the first element is copied as is.\nThrough an `exclusive` attribute, this behavior can change to exclude the first element.\nIt can also perform summation in the opposite direction of the axis. For that, set `reverse` attribute to 1.\n\nExample:\n```\ninput_x = [1, 2, 3]\naxis=0\noutput = [1, 3, 6]\nexclusive=1\noutput = [0, 1, 3]\nexclusive=0\nreverse=1\noutput = [6, 5, 3]\nexclusive=1\nreverse=1\noutput = [5, 3, 0]\n```\n " +----f +input: "input" +output: "output" +name: "DepthToSpace" +op_type: "DepthToSpace" +attribute { + name: "blocksize" + s: "" + type: INT +} +attribute { + name: "mode" + s: "DCR" + type: STRING +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "DepthToSpace rearranges (permutes) data from depth into blocks of spatial data.\nThis is the reverse transformation of SpaceToDepth. More specifically, this op outputs a copy of\nthe input tensor where values from the depth dimension are moved in spatial blocks to the height\nand width dimensions. By default, `mode` = `DCR`.\nIn the DCR mode, elements along the depth dimension from the input tensor are rearranged in the\nfollowing order: depth, column, and then row. The output y is computed from the input x as below:\n\nb, c, h, w = x.shape\n\ntmp = np.reshape(x, [b, blocksize, blocksize, c // (blocksize**2), h, w])\n\ntmp = np.transpose(tmp, [0, 3, 4, 1, 5, 2])\n\ny = np.reshape(tmp, [b, c // (blocksize**2), h * blocksize, w * blocksize])\n\n\nIn the CRD mode, elements along the depth dimension from the input tensor are rearranged in the\nfollowing order: column, row, and the depth. The output y is computed from the input x as below:\n\nb, c, h, w = x.shape\n\ntmp = np.reshape(x, [b, c // (blocksize ** 2), blocksize, blocksize, h, w])\n\ntmp = np.transpose(tmp, [0, 1, 4, 2, 5, 3])\n\ny = np.reshape(tmp, [b, c // (blocksize ** 2), h * blocksize, w * blocksize])\n\n" +----f +input: "x" +input: "x_scale" +input: "x_zero_point" +output: "y" +name: "DequantizeLinear" +op_type: "DequantizeLinear" +attribute { + name: "x-types" + strings: "int32" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "x_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "x_zero_point-types" + strings: "int32" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nThe linear dequantization operator. It consumes a quantized tensor, a scale, a zero point to compute the full precision tensor.\nThe dequantization formula is y = (x - x_zero_point) * x_scale. \'x_scale\' and \'x_zero_point\' must have same shape.\n\'x_zero_point\' and \'x\' must have same type. \'x\' and \'y\' must have same shape. In the case of dequantizing int32,\nthere\'s no zero point (zero point is supposed to be 0).\n" +----f +input: "X" +output: "Y" +name: "Det" +op_type: "Det" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nDet calculates determinant of a square matrix or batches of square matrices.\nDet takes one input tensor of shape `[*, M, M]`, where `*` is zero or more batch dimensions,\nand the inner-most 2 dimensions form square matrices.\nThe output is a tensor of shape `[*]`, containing the determinants of all input submatrices.\ne.g., When the input is 2-D, the output is a scalar(shape is empty: `[]`).\n" +----f +input: "X" +output: "Y" +name: "DictVectorizer" +op_type: "DictVectorizer" +attribute { + name: "int64_vocabulary" + s: "" + type: INTS +} +attribute { + name: "string_vocabulary" + s: "" + type: STRINGS +} +attribute { + name: "X-types" + strings: "map(int64,float" + strings: "map(int64,string" + strings: "map(string,int64" + strings: "map(string,float" + strings: "map(string,double" + strings: "map(int64,double" + type: STRINGS +} +doc_string: "\n Uses an index mapping to convert a dictionary to an array.
\n Given a dictionary, each key is looked up in the vocabulary attribute corresponding to\n the key type. The index into the vocabulary array at which the key is found is then\n used to index the output 1-D tensor \'Y\' and insert into it the value found in the dictionary \'X\'.
\n The key type of the input map must correspond to the element type of the defined vocabulary attribute.\n Therefore, the output array will be equal in length to the index mapping vector parameter.\n All keys in the input dictionary must be present in the index mapping vector.\n For each item in the input dictionary, insert its value in the output array.\n Any keys not present in the input dictionary, will be zero in the output array.
\n For example: if the ``string_vocabulary`` parameter is set to ``[\"a\", \"c\", \"b\", \"z\"]``,\n then an input of ``{\"a\": 4, \"c\": 8}`` will produce an output of ``[4, 8, 0, 0]``.\n " +----f +input: "A" +input: "B" +output: "C" +name: "Div" +op_type: "Div" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPerforms element-wise binary division (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "data" +input: "ratio" +input: "training_mode" +output: "output" +output: "mask" +name: "Dropout" +op_type: "Dropout" +attribute { + name: "seed" + s: "" + type: INT +} +attribute { + name: "data-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "ratio-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "training_mode-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nDropout takes an input floating-point tensor, an optional input ratio (floating-point scalar) and an optional input training_mode (boolean scalar). It produces two tensor outputs,\noutput (floating-point tensor) and mask (optional `Tensor`). If `training_mode` is true then the output Y will be a random dropout;\nNote that this Dropout scales the masked input data by the following equation, so to convert the trained model into inference mode,\nthe user can simply not pass `training_mode` input or set it to false.\n```\noutput = scale * data * mask,\n```\nwhere\n```\nscale = 1. / (1. - ratio).\n```\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "x" +output: "y" +output: "y_scale" +output: "y_zero_point" +name: "DynamicQuantizeLinear" +op_type: "DynamicQuantizeLinear" +attribute { + name: "x-types" + strings: "float" + type: STRINGS +} +doc_string: "\nA Function to fuse calculation for Scale, Zero Point and FP32->8Bit convertion of FP32 Input data.\nOutputs Scale, ZeroPoint and Quantized Input for a given FP32 Input.\nScale is calculated as:\n```\n y_scale = (max(x) - min(x))/(qmax - qmin)\n * where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8\n * data range is adjusted to include 0.\n```\nZero point is calculated as:\n```\nintermediate_zero_point = qmin - min(x)/y_scale\ny_zero_point = cast(round(saturate(itermediate_zero_point)))\n* where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8\n* for saturation, it saturates to [0, 255] if it\'s uint8, or [-127, 127] if it\'s int8. Right now only uint8 is supported.\n* rounding to nearest ties to even.\n```\nData quantization formula is:\n```\ny = saturate (round (x / y_scale) + y_zero_point)\n* for saturation, it saturates to [0, 255] if it\'s uint8, or [-127, 127] if it\'s int8. Right now only uint8 is supported.\n* rounding to nearest ties to even.\n```\n" +----f +input: "Inputs" +output: "Output" +name: "Einsum" +op_type: "Einsum" +attribute { + name: "equation" + s: "" + type: STRING +} +attribute { + name: "Inputs-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nAn einsum of the form ```term1, term2 -> output-term``` produces an output tensor using the following equation\n\n```output[output-term] = reduce-sum( input1[term1] * input2[term] )```\n\nwhere the reduce-sum performs a summation over all the indices occurring in in the input terms (term1, term2)\nthat do not occur in the output-term.\n\nThe Einsum operator evaluates algebraic tensor operations on a sequence of tensors, using the Einstein summation\nconvention. The equation string contains a comma-separated sequence of lower case letters. Each term corresponds to\nan operand tensor, and the characters within the terms correspond to operands dimensions.\n\nThis sequence may be followed by \"->\" to separate the left and right hand side of the equation.\nIf the equation contains \"->\" followed by the right-hand side, the explicit (not classical) form of the Einstein\nsummation is performed, and the right-hand side indices indicate output tensor dimensions. In other cases,\noutput indices are (implicitly) set to the alphabetically sorted sequence of indices appearing exactly once in the\nequation.\n\nWhen a dimension character is repeated in the left-hand side, it represents summation along the dimension.\n\nThe equation may contain ellipsis (\"...\") to enable broadcasting. Ellipsis must indicate a fixed number of dimensions.\nSpecifically, every occurrence of ellipsis in the equation must represent the same number of dimensions.\nThe right-hand side may contain exactly one ellipsis. In implicit mode, the ellipsis dimensions are set to the\nbeginning of the output. The equation string may contain space (U+0020) character.\n" +----f +input: "X" +output: "Y" +name: "Elu" +op_type: "Elu" +attribute { + name: "alpha" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nElu takes one input data (Tensor) and produces one output data\n(Tensor) where the function `f(x) = alpha * (exp(x) - 1.) for x <\n0`, `f(x) = x for x >= 0`., is applied to the tensor elementwise.\n\n" +----f +input: "A" +input: "B" +output: "C" +name: "Equal" +op_type: "Equal" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "input" +output: "output" +name: "Erf" +op_type: "Erf" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the error function of the given input tensor element-wise.\n" +----f +input: "input" +output: "output" +name: "Exp" +op_type: "Exp" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the exponential of the given input tensor, element-wise.\n" +----f +input: "input" +input: "shape" +output: "output" +name: "Expand" +op_type: "Expand" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "shape-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nBroadcast the input tensor following the given shape and the broadcast rule.\nThe broadcast rule is similar to numpy.array(input) * numpy.ones(shape):\nDimensions are right alignment;\nTwo corresponding dimension must have the same value, or one of them is equal to 1.\nAlso, this operator is similar to numpy.broadcast_to(input, shape),\nbut the major difference is numpy.broadcast_to() does not allow shape to be smaller than input.size().\nIt is possible that the output.shape is not equal to shape, when some dimensions in shape is equal to 1,\nor the shape.ndim < input.shape.ndim.\n" +----f +input: "input" +output: "output" +name: "EyeLike" +op_type: "EyeLike" +attribute { + name: "dtype" + s: "" + type: INT +} +attribute { + name: "k" + i: 0 + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "float16" + strings: "int32" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGenerate a 2D tensor (matrix) with ones on the diagonal and zeros everywhere else. Only 2D\ntensors are supported, i.e. input T1 must be of rank 2. The shape of the output tensor is the\nsame as the input tensor. The data type can be specified by the \'dtype\' argument. If\n\'dtype\' is not specified, then the type of input tensor is used. By default, the main diagonal\nis populated with ones, but attribute \'k\' can be used to populate upper or lower diagonals.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message and be valid as an output type.\n" +----f +input: "X" +output: "Y" +name: "FeatureVectorizer" +op_type: "FeatureVectorizer" +attribute { + name: "inputdimensions" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Concatenates input tensors into one continuous output.
\n All input shapes are 2-D and are concatenated along the second dimention. 1-D tensors are treated as [1,C].\n Inputs are copied to the output maintaining the order of the input arguments.
\n All inputs must be integers or floats, while the output will be all floating point values.\n" +----f +input: "input" +output: "output" +name: "Flatten" +op_type: "Flatten" +attribute { + name: "axis" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nFlattens the input tensor into a 2D matrix. If input tensor has shape\n(d_0, d_1, ... d_n) then the output will have shape\n(d_0 X d_1 ... d_(axis-1), d_axis X d_(axis+1) ... X dn).\n" +----f +input: "X" +output: "Y" +name: "Floor" +op_type: "Floor" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nFloor takes one input data (Tensor) and produces one output data\n(Tensor) where the floor is, y = floor(x), is applied to\nthe tensor elementwise.\n" +----f +input: "X" +input: "W" +input: "R" +input: "B" +input: "sequence_lens" +input: "initial_h" +output: "Y" +output: "Y_h" +name: "GRU" +op_type: "GRU" +attribute { + name: "activation_alpha" + s: "" + type: FLOATS +} +attribute { + name: "activation_beta" + s: "" + type: FLOATS +} +attribute { + name: "activations" + s: "" + type: STRINGS +} +attribute { + name: "clip" + s: "" + type: FLOAT +} +attribute { + name: "direction" + s: "forward" + type: STRING +} +attribute { + name: "hidden_size" + s: "" + type: INT +} +attribute { + name: "linear_before_reset" + i: 0 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "sequence_lens-types" + strings: "int32" + type: STRINGS +} +attribute { + name: "initial_h-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nComputes an one-layer GRU. This operator is usually supported via some custom\nimplementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`z` - update gate\n\n`r` - reset gate\n\n`h` - hidden gate\n\n`t` - time step (t-1 means previous time step)\n\n`W[zrh]` - W parameter weight matrix for update, reset, and hidden gates\n\n`R[zrh]` - R recurrence weight matrix for update, reset, and hidden gates\n\n`Wb[zrh]` - W bias vectors for update, reset, and hidden gates\n\n`Rb[zrh]` - R bias vectors for update, reset, and hidden gates\n\n`WB[zrh]` - W parameter weight matrix for backward update, reset, and hidden gates\n\n`RB[zrh]` - R recurrence weight matrix for backward update, reset, and hidden gates\n\n`WBb[zrh]` - W bias vectors for backward update, reset, and hidden gates\n\n`RBb[zrh]` - R bias vectors for backward update, reset, and hidden gates\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Sigmoid, g=Tanh):\n\n - zt = f(Xt*(Wz^T) + Ht-1*(Rz^T) + Wbz + Rbz)\n\n - rt = f(Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr)\n\n - ht = g(Xt*(Wh^T) + (rt (.) Ht-1)*(Rh^T) + Rbh + Wbh) # default, when linear_before_reset = 0\n\n - ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) + Wbh) # when linear_before_reset != 0\n\n - Ht = (1 - zt) (.) ht + zt (.) Ht-1\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "data" +input: "indices" +output: "output" +name: "Gather" +op_type: "Gather" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nGiven `data` tensor of rank r >= 1, and `indices` tensor of rank q, gather\nentries of the axis dimension of `data` (by default outer-most one as axis=0) indexed by `indices`, and concatenates\nthem in an output tensor of rank q + (r - 1).\n\naxis = 0 :\n\nLet\nk = indices[i_{0}, ..., i_{q-1}]\nThen\noutput[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[k , j_{0}, ..., j_{r-2}]\n\n```\n data = [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ]\n indices = [\n [0, 1],\n [1, 2],\n ]\n output = [\n [\n [1.0, 1.2],\n [2.3, 3.4],\n ],\n [\n [2.3, 3.4],\n [4.5, 5.7],\n ],\n ]\n```\naxis = 1 :\n\nLet\nk = indices[i_{0}, ..., i_{q-1}]\nThen\noutput[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[j_{0}, k, j_{1}, ..., j_{r-2}]\n\n```\n data = [\n [1.0, 1.2, 1.9],\n [2.3, 3.4, 3.9],\n [4.5, 5.7, 5.9],\n ]\n indices = [\n [0, 2],\n ]\n axis = 1,\n output = [\n [\n [1.0, 1.9],\n [2.3, 3.9],\n [4.5, 5.9],\n ],\n ]\n```\n" +----f +input: "data" +input: "indices" +output: "output" +name: "GatherElements" +op_type: "GatherElements" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n\nGatherElements takes two inputs `data` and `indices` of the same rank r >= 1\nand an optional attribute `axis` that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). It is an indexing operation\nthat produces its output by indexing into the input data tensor at index\npositions determined by elements of the `indices` tensor.\nIts output shape is the same as the shape of `indices` and consists of one value\n(gathered from the `data`) for each element in `indices`.\n\nFor instance, in the 3-D case (r = 3), the output produced is determined\nby the following equations: \n```\n out[i][j][k] = input[index[i][j][k]][j][k] if axis = 0,\n out[i][j][k] = input[i][index[i][j][k]][k] if axis = 1,\n out[i][j][k] = input[i][j][index[i][j][k]] if axis = 2,\n```\n\nThis operator is also the inverse of ScatterElements. It is similar to Torch\'s gather operation.\n\nExample 1:\n```\n data = [\n [1, 2],\n [3, 4],\n ]\n indices = [\n [0, 0],\n [1, 0],\n ]\n axis = 1\n output = [\n [\n [1, 1],\n [4, 3],\n ],\n ]\n```\nExample 2:\n```\n data = [\n [1, 2, 3],\n [4, 5, 6],\n [7, 8, 9],\n ]\n indices = [\n [1, 2, 0],\n [2, 0, 0],\n ]\n axis = 0\n output = [\n [\n [4, 8, 3],\n [7, 2, 3],\n ],\n ]\n```\n" +----f +input: "data" +input: "indices" +output: "output" +name: "GatherND" +op_type: "GatherND" +attribute { + name: "batch_dims" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nGiven `data` tensor of rank `r` >= 1, `indices` tensor of rank `q` >= 1, and `batch_dims` integer `b`, this operator gathers \nslices of `data` into an output tensor of rank `q + r - indices_shape[-1] - 1 - b`.\n\n`indices` is an q-dimensional integer tensor, best thought of as a `(q-1)`-dimensional tensor of index-tuples into `data`, \nwhere each element defines a slice of `data`\n\n`batch_dims` (denoted as `b`) is an integer indicating the number of batch dimensions, i.e the leading `b` number of dimensions of \n`data` tensor and `indices` are representing the batches, and the gather starts from the `b+1` dimension. \n\nSome salient points about the inputs\' rank and shape:\n \n1) r >= 1 and q >= 1 are to be honored. There is no dependency condition to be met between ranks `r` and `q`\n\n2) The first `b` dimensions of the shape of `indices` tensor and `data` tensor must be equal.\n\n3) b < min(q, r) is to be honored.\n\n4) The `indices_shape[-1]` should have a value between 1 (inclusive) and rank `r-b` (inclusive) \n\n5) All values in `indices` are expected to be within bounds [-s, s-1] along axis of size `s` (i.e.) `-data_shape[i] <= indices[...,i] <= data_shape[i] - 1`.\n It is an error if any of the index values are out of bounds.\n\nThe output is computed as follows:\n\nThe output tensor is obtained by mapping each index-tuple in the `indices` tensor to the corresponding slice of the input `data`.\n \n1) If `indices_shape[-1] > r-b` => error condition\n\n2) If `indices_shape[-1] == r-b`, since the rank of `indices` is `q`, `indices` can be thought of as `N` `(q-b-1)`-dimensional tensors\n containing 1-D tensors of dimension `r-b`, where `N` is an integer equals to the product of 1 and all the elements in the batch dimensions \n of the indices_shape. Let us think of each such `r-b` ranked tensor as `indices_slice`. Each *scalar value* corresponding to `data[0:b-1,indices_slice]` \n is filled into the corresponding location of the `(q-b-1)`-dimensional tensor to form the `output` tensor (Example 1 below)\n\n3) If `indices_shape[-1] < r-b`, since the rank of `indices` is `q`, `indices` can be thought of as `N` `(q-b-1)`-dimensional tensor\n containing 1-D tensors of dimension `< r-b`. Let us think of each such tensors as `indices_slice`. Each *tensor slice* corresponding \n to `data[0:b-1, indices_slice , :]` is filled into the corresponding location of the `(q-b-1)`-dimensional tensor \n to form the `output` tensor (Examples 2, 3, 4 and 5 below)\n\nThis operator is the inverse of `ScatterND`.\n\n`Example 1`\n\n batch_dims = 0\n\n data = [[0,1],[2,3]] # data_shape = [2, 2]\n\n indices = [[0,0],[1,1]] # indices_shape = [2, 2]\n\n output = [0,3] # output_shape = [2]\n\n`Example 2`\n\n batch_dims = 0\n\n data = [[0,1],[2,3]] # data_shape = [2, 2]\n\n indices = [[1],[0]] # indices_shape = [2, 1]\n\n output = [[2,3],[0,1]] # output_shape = [2, 2]\n\n`Example 3`\n\n batch_dims = 0\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[0,1],[1,0]] # indices_shape = [2, 2]\n\n output = [[2,3],[4,5]] # output_shape = [2, 2] \n\n`Example 4`\n\n batch_dims = 0\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[[0,1]],[[1,0]]] # indices_shape = [2, 1, 2]\n\n output = [[[2,3]],[[4,5]]] # output_shape = [2, 1, 2] \n\n`Example 5`\n\n batch_dims = 1\n\n data = [[[0,1],[2,3]],[[4,5],[6,7]]] # data_shape = [2, 2, 2]\n\n indices = [[1],[0]] # indices_shape = [2, 1]\n\n output = [[2,3],[4,5]] # output_shape = [2, 2] \n\n\n" +----f +input: "A" +input: "B" +input: "C" +output: "Y" +name: "Gemm" +op_type: "Gemm" +attribute { + name: "alpha" + f: 1.0 + type: FLOAT +} +attribute { + name: "beta" + f: 1.0 + type: FLOAT +} +attribute { + name: "transA" + i: 0 + type: INT +} +attribute { + name: "transB" + i: 0 + type: INT +} +attribute { + name: "A-types" + strings: "int32" + strings: "float16" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "int32" + strings: "float16" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "C-types" + strings: "int32" + strings: "float16" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "General Matrix multiplication:\nhttps://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms#Level_3\n\nA\' = transpose(A) if transA else A\n\nB\' = transpose(B) if transB else B\n\nCompute Y = alpha * A\' * B\' + beta * C, where input tensor A has shape (M, K) or (K, M),\ninput tensor B has shape (K, N) or (N, K), input tensor C is broadcastable to shape (M, N),\nand output tensor Y has shape (M, N). A will be transposed before doing the\ncomputation if attribute transA is non-zero, same for B and transB.\nThis operator supports **unidirectional broadcasting** (tensor C should be unidirectional broadcastable to tensor A * B); for more details please check [the doc](Broadcasting.md).\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "X" +output: "Y" +name: "GlobalAveragePool" +op_type: "GlobalAveragePool" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n GlobalAveragePool consumes an input tensor X and applies average pooling across\n the values in the same channel. This is equivalent to AveragePool with kernel size\n equal to the spatial dimension of input tensor." +----f +input: "X" +output: "Y" +name: "GlobalLpPool" +op_type: "GlobalLpPool" +attribute { + name: "p" + i: 2 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n GlobalLpPool consumes an input tensor X and applies lp pool pooling across\n the values in the same channel. This is equivalent to LpPool with kernel size\n equal to the spatial dimension of input tensor." +----f +input: "X" +output: "Y" +name: "GlobalMaxPool" +op_type: "GlobalMaxPool" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n GlobalMaxPool consumes an input tensor X and applies max pooling across\n the values in the same channel. This is equivalent to MaxPool with kernel size\n equal to the spatial dimension of input tensor." +----f +input: "Inputs" +output: "Outputs" +name: "Gradient" +op_type: "Gradient" +attribute { + name: "xs" + s: "" + type: STRINGS +} +attribute { + name: "y" + s: "" + type: STRING +} +attribute { + name: "zs" + s: "" + type: STRINGS +} +attribute { + name: "Inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGradient operator computes the partial derivatives of a specific tensor w.r.t.\nsome other tensors. This operator is widely used in gradient-based training\nalgorithms. To illustrate its use, let\'s consider a computation graph,\n\n```\nX -----.\n |\n v\nW --> Conv --> H --> Gemm --> Y\n ^\n |\n Z\n```\n\n, where W and Z are trainable tensors. Note that operators\' attributes are\nomitted for the sake of simplicity. Let dY/dW (dY/dZ) be the gradient of\nY with respect to W (Z). The user can compute gradient by inserting Gradient\noperator to form another graph shown below.\n\n```\nW --> Conv --> H --> Gemm --> Y\n| ^ ^\n| | |\n| X Z\n| | |\n| | .----------\'\n| | | (W/Z/X is the 1st/2nd/3rd input of Gradient as shown in\n| | | \"xs\" followed by \"zs\")\n| v v\n\'---> Gradient(xs=[\"W\", \"Z\"], zs=[\"X\"], y=\"Y\")\n | |\n | \'-----------------------------------> dY/dW (1st output of Gradient)\n |\n \'---------------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nBy definition, the tensor \"y\" is a function of independent variables in \"xs\"\nand \"zs\". Since we only compute the gradient of \"y\" w.r.t. the differentiable\nvariables in \"xs\", this Gradient only outputs dY/dW and dY/dZ. Note that \"H\"\ncannot appear in \"xs\" and \"zs\". The reason is that \"H\" can be determined by\ntensors \"W\" and \"X\" and therefore \"H\" is not an independent variable.\n\nAll outputs are optional. If needed, for example, user can assign an empty\nstring to the 1st output name of that Gradient to skip the generation of dY/dW.\nNote that the concept of optional outputs can also be found in ONNX\'s RNN, GRU,\nand LSTM.\n\nGradient operator can compute derivative against intermediate tensors. For\nexample, the gradient of Y with respect to H can be done via\n\n```\nW --> Conv --> H --> Gemm --> Y\n ^ | ^\n | | |\n X | Z\n .-------\' |\n | .----------\'\n | | (H/Z is the 1st/2nd input of Gradient as shown in \"xs\")\n v v\n Gradient(xs=[\"H\", \"Z\"], y=\"Y\")\n | |\n | \'-----------------------------------> dY/dH (1st output of Gradient)\n |\n \'---------------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nIt is possible to represent high-order differentiation using Gradient operators.\nFor example, given the following linear model:\n\n```\nW --> Gemm --> Y --> Loss --> O\n ^ ^\n | |\n X L\n```\n\nTo compute the 2nd order derivative of O with respect to W (denoted by\nd^2O/dW^2), one can do\n\n```\nW --> Gemm --> Y --> Loss --> O\n| ^ ^\n| | |\n| X .------------L\n| | | |\n| | | v\n+------+-+> Gradient(xs=[\"X\", \"W\"], zs=[\"L\"], y=\"O\") ---> dO/dX (1st output of Gradient)\n| | | |\n| | | \'---> dO/dW (2nd output of Gradient)\n| v v\n\'---> Gradient(xs=[\"X\", \"W\"], zs=[\"L\"], y=\"dO/dW\") ---> d(dO/dW)dX (1st output of\n | Gradient)\n |\n |\n \'---> d^2O/dW^2 (2nd output of Gradient)\n```\n\nThe tensors named in attributes \"xs\", \"zs\", and \"y\" define the differentiated\ncomputation graph, and the inputs to Gradient node define the values at\nwhich the gradient is computed. We can feed different tensors to the identified\ngraph. For example, one can compute the gradient of Y with respect to H at \na specific value of H, H_1, by providing that value as an input to the Gradient\nnode.\n\n```\nW --> Conv --> H --> Gemm --> Y\n ^ ^\n | |\n X Z\n\n Z_1 (2nd input of Gradient)\n |\n v\nH_1 --> Gradient(xs=[\"H\", \"Z\"], y=\"Y\") ---> dY/dH when H = H_1 and Y = Y_1.\n |\n \'------------------------------> dY/dZ (2nd output of Gradient)\n```\n\nWhen the inputs of Gradient are the tensors named in \"xs\" and \"zs\", the\ncomputation can be optimized. More specifically, intermediate variables in\nforward pass can be reused if the gradient is computed via reverse-mode\nauto-differentiation.\n\n" +----f +input: "Inputs" +output: "Outputs" +name: "GraphCall" +op_type: "GraphCall" +attribute { + name: "graph_name" + s: "" + type: STRING +} +attribute { + name: "Inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nThe GraphCall operator invokes a graph inside TrainingInfoProto\'s\nalgorithm field. The GraphCall inputs and outputs are bound to those of\ninvoked graph by position. If a graph input has an initializer, that input\nis considered optional. All graph outputs are optional.\n\nBelow Python syntax is used for describing dictionary and list.\n\nAssume that ModelProto\'s graph field has\n- name: \"MyInferenceGraph\"\n- input: [\"X\", \"W\", \"Z\"]\n- initializer: [W]\n- output: [\"Y\"]\n\nas visualized below for inference.\n\n```\nX -----.\n |\n v\nW --> Conv --> H --> Gemm --> Y\n ^\n |\n Z\n```\n\nAssume that the training algorithm contains\n\n- inputs: [\"X_1\", \"Z_1\", \"C\"]\n- initializer: [T]\n- outputs: [\"W_new\"]\n\nwith a dictionary\n\n- update_binding: {\"W\": \"W_new\", \"T\": \"T_new\"}\n\nInside the training algorithm graph, one can invoke the inference\ngraph via adding a GraphCall node with\n\n- inputs: [\"X_1\", \"W\", Z_1\"]\n- outputs: [\"Y_1\"]\n- an attribute graph_name=\"MyInferenceGraph\",\n\nThe initializers, \"W\" and \"T\" in this case, in update_binding\nare considered globally-visible and mutable variables, which\ncan be used as inputs of operators in the training graph.\n\nAn example training algorithm graph may look like\n\n```\n.-------- W (a global and mutable variable from\n| | the inference graph)\n| |\n| .-----\'-----------.\n| | |\n| | v\n| | .-- X_1 --> GraphCall(graph_name=\"MyInferenceGraph\")\n| | | | |\n| | | | |\n| | | Z_1 -----\' |\n| | | | V\n| | | | Y_1 ---> Loss ---> O\n| | | | ^\n| | | | |\n| | `--. | C\n| | | | |\n| | | | .----------------\'\n| | | | |\n| | v v v\n| `--> Gradient(xs=[\"W\"], zs=[\"X_1\", \"Z_1\", \"C\"], y=\"O\")\n| |\n| v\n| dO_dW (gradient of W) 1 (a scalar one)\n| | |\n| V v\n| Div <--- T ------------> Add ---> T_new\n| | (T is the number of training iterations.\n| | T is also globally visible and mutable.)\n| v\n`-----> Sub ----> W_new\n```\n\nwhere Loss is a dummy node which computes the minimized objective function.\n\nThe variable \"W\" is an optional input in the called graph.\nIf the user omits it, the input list of GraphCall becomes [\"X_1\", \"\", \"Z_1\"].\nIn this case, from the view of computation graph, the Conv operator invoked by\nGraphCall\'s may be still connected the global \"W\" variable and therefore the\nstructure of the computation graph is unchanged.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Greater" +op_type: "Greater" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `greater` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "A" +input: "B" +output: "C" +name: "GreaterOrEqual" +op_type: "GreaterOrEqual" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `greater_equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Y" +name: "HardSigmoid" +op_type: "HardSigmoid" +attribute { + name: "alpha" + f: 0.2 + type: FLOAT +} +attribute { + name: "beta" + f: 0.5 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nHardSigmoid takes one input data (Tensor) and produces one output data\n(Tensor) where the HardSigmoid function, y = max(0, min(1, alpha * x + beta)),\nis applied to the tensor elementwise.\n" +----f +input: "input" +output: "output" +name: "Hardmax" +op_type: "Hardmax" +attribute { + name: "axis" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe operator computes the hardmax (1 for the first maximum value, and 0 for all others) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the hardmax values of the corresponding input.\n" +----f +input: "input" +output: "output" +name: "Identity" +op_type: "Identity" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "Identity operator" +----f +input: "cond" +output: "outputs" +name: "If" +op_type: "If" +attribute { + name: "else_branch" + s: "" + type: GRAPH +} +attribute { + name: "then_branch" + s: "" + type: GRAPH +} +attribute { + name: "cond-types" + strings: "bool" + type: STRINGS +} +doc_string: "If conditional" +----f +input: "X" +output: "Y" +name: "Imputer" +op_type: "Imputer" +attribute { + name: "imputed_value_floats" + s: "" + type: FLOATS +} +attribute { + name: "imputed_value_int64s" + s: "" + type: INTS +} +attribute { + name: "replaced_value_float" + f: 0.0 + type: FLOAT +} +attribute { + name: "replaced_value_int64" + i: 0 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Replaces inputs that equal one value with another, leaving all other elements alone.
\n This operator is typically used to replace missing values in situations where they have a canonical\n representation, such as -1, 0, NaN, or some extreme value.
\n One and only one of imputed_value_floats or imputed_value_int64s should be defined -- floats if the input tensor\n holds floats, integers if the input tensor holds integers. The imputed values must all fit within the\n width of the tensor element type. One and only one of the replaced_value_float or replaced_value_int64 should be defined,\n which one depends on whether floats or integers are being processed.
\n The imputed_value attribute length can be 1 element, or it can have one element per input feature.
In other words, if the input tensor has the shape [*,F], then the length of the attribute array may be 1 or F. If it is 1, then it is broadcast along the last dimension and applied to each feature.\n" +----f +input: "input" +input: "scale" +input: "B" +output: "output" +name: "InstanceNormalization" +op_type: "InstanceNormalization" +attribute { + name: "epsilon" + f: 1e-05 + type: FLOAT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "scale-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCarries out instance normalization as described in the paper\nhttps://arxiv.org/abs/1607.08022.\n\ny = scale * (x - mean) / sqrt(variance + epsilon) + B,\nwhere mean and variance are computed per instance per channel.\n\n" +----f +input: "X" +output: "Y" +name: "IsInf" +op_type: "IsInf" +attribute { + name: "detect_negative" + i: 1 + type: INT +} +attribute { + name: "detect_positive" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + type: STRINGS +} +doc_string: "Map infinity to true and other values to false." +----f +input: "X" +output: "Y" +name: "IsNaN" +op_type: "IsNaN" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "Returns which elements of the input are NaN." +----f +input: "X" +output: "Y" +name: "LRN" +op_type: "LRN" +attribute { + name: "alpha" + f: 0.0001 + type: FLOAT +} +attribute { + name: "beta" + f: 0.75 + type: FLOAT +} +attribute { + name: "bias" + f: 1.0 + type: FLOAT +} +attribute { + name: "size" + s: "" + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nLocal Response Normalization proposed in the [AlexNet paper](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf).\nIt normalizes over local input regions.\nThe local region is defined across the channels. For an element X[n, c, d1, ..., dk] in a tensor\nof shape (N x C x D1 x D2, ..., Dk), its region is\n{X[n, i, d1, ..., dk] | max(0, c - floor((size - 1) / 2)) <= i <= min(C - 1, c + ceil((size - 1) / 2))}.\n\nsquare_sum[n, c, d1, ..., dk] = sum(X[n, i, d1, ..., dk] ^ 2),\nwhere max(0, c - floor((size - 1) / 2)) <= i <= min(C - 1, c + ceil((size - 1) / 2)).\n\nY[n, c, d1, ..., dk] = X[n, c, d1, ..., dk] / (bias + alpha / size * square_sum[n, c, d1, ..., dk] ) ^ beta\n" +----f +input: "X" +input: "W" +input: "R" +input: "B" +input: "sequence_lens" +input: "initial_h" +input: "initial_c" +input: "P" +output: "Y" +output: "Y_h" +output: "Y_c" +name: "LSTM" +op_type: "LSTM" +attribute { + name: "activation_alpha" + s: "" + type: FLOATS +} +attribute { + name: "activation_beta" + s: "" + type: FLOATS +} +attribute { + name: "activations" + s: "" + type: STRINGS +} +attribute { + name: "clip" + s: "" + type: FLOAT +} +attribute { + name: "direction" + s: "forward" + type: STRING +} +attribute { + name: "hidden_size" + s: "" + type: INT +} +attribute { + name: "input_forget" + i: 0 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "sequence_lens-types" + strings: "int32" + type: STRINGS +} +attribute { + name: "initial_h-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "initial_c-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "P-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nComputes an one-layer LSTM. This operator is usually supported via some\ncustom implementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`i` - input gate\n\n`o` - output gate\n\n`f` - forget gate\n\n`c` - cell gate\n\n`t` - time step (t-1 means previous time step)\n\n`W[iofc]` - W parameter weight matrix for input, output, forget, and cell gates\n\n`R[iofc]` - R recurrence weight matrix for input, output, forget, and cell gates\n\n`Wb[iofc]` - W bias vectors for input, output, forget, and cell gates\n\n`Rb[iofc]` - R bias vectors for input, output, forget, and cell gates\n\n`P[iof]` - P peephole weight vector for input, output, and forget gates\n\n`WB[iofc]` - W parameter weight matrix for backward input, output, forget, and cell gates\n\n`RB[iofc]` - R recurrence weight matrix for backward input, output, forget, and cell gates\n\n`WBb[iofc]` - W bias vectors for backward input, output, forget, and cell gates\n\n`RBb[iofc]` - R bias vectors for backward input, output, forget, and cell gates\n\n`PB[iof]` - P peephole weight vector for backward input, output, and forget gates\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Sigmoid, g=Tanh, h=Tanh):\n\n - it = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Pi (.) Ct-1 + Wbi + Rbi)\n\n - ft = f(Xt*(Wf^T) + Ht-1*(Rf^T) + Pf (.) Ct-1 + Wbf + Rbf)\n\n - ct = g(Xt*(Wc^T) + Ht-1*(Rc^T) + Wbc + Rbc)\n\n - Ct = ft (.) Ct-1 + it (.) ct\n\n - ot = f(Xt*(Wo^T) + Ht-1*(Ro^T) + Po (.) Ct + Wbo + Rbo)\n\n - Ht = ot (.) h(Ct)\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +input: "X" +output: "Y" +name: "LabelEncoder" +op_type: "LabelEncoder" +attribute { + name: "default_float" + f: -0.0 + type: FLOAT +} +attribute { + name: "default_int64" + i: -1 + type: INT +} +attribute { + name: "default_string" + s: "_Unused" + type: STRING +} +attribute { + name: "keys_floats" + s: "" + type: FLOATS +} +attribute { + name: "keys_int64s" + s: "" + type: INTS +} +attribute { + name: "keys_strings" + s: "" + type: STRINGS +} +attribute { + name: "values_floats" + s: "" + type: FLOATS +} +attribute { + name: "values_int64s" + s: "" + type: INTS +} +attribute { + name: "values_strings" + s: "" + type: STRINGS +} +attribute { + name: "X-types" + strings: "string" + strings: "float" + strings: "int64" + type: STRINGS +} +doc_string: "\n Maps each element in the input tensor to another value.
\n The mapping is determined by the two parallel attributes, \'keys_*\' and\n \'values_*\' attribute. The i-th value in the specified \'keys_*\' attribute\n would be mapped to the i-th value in the specified \'values_*\' attribute. It\n implies that input\'s element type and the element type of the specified\n \'keys_*\' should be identical while the output type is identical to the\n specified \'values_*\' attribute. If an input element can not be found in the\n specified \'keys_*\' attribute, the \'default_*\' that matches the specified\n \'values_*\' attribute may be used as its output value.
\n Let\'s consider an example which maps a string tensor to an integer tensor.\n Assume and \'keys_strings\' is [\"Amy\", \"Sally\"], \'values_int64s\' is [5, 6],\n and \'default_int64\' is \'-1\'. The input [\"Dori\", \"Amy\", \"Amy\", \"Sally\",\n \"Sally\"] would be mapped to [-1, 5, 5, 6, 6].
\n Since this operator is an one-to-one mapping, its input and output shapes\n are the same. Notice that only one of \'keys_*\'/\'values_*\' can be set.
\n For key look-up, bit-wise comparison is used so even a float NaN can be\n mapped to a value in \'values_*\' attribute.
\n" +----f +input: "X" +output: "Y" +name: "LeakyRelu" +op_type: "LeakyRelu" +attribute { + name: "alpha" + f: 0.01 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nLeakyRelu takes input data (Tensor) and an argument alpha, and produces one\noutput data (Tensor) where the function `f(x) = alpha * x for x < 0`,\n`f(x) = x for x >= 0`, is applied to the data tensor elementwise.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Less" +op_type: "Less" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `less` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "A" +input: "B" +output: "C" +name: "LessOrEqual" +op_type: "LessOrEqual" +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `less_equal` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Y" +output: "Z" +name: "LinearClassifier" +op_type: "LinearClassifier" +attribute { + name: "classlabels_ints" + s: "" + type: INTS +} +attribute { + name: "classlabels_strings" + s: "" + type: STRINGS +} +attribute { + name: "coefficients" + s: "" + type: FLOATS +} +attribute { + name: "intercepts" + s: "" + type: FLOATS +} +attribute { + name: "multi_class" + i: 0 + type: INT +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Linear classifier\n" +----f +input: "X" +output: "Y" +name: "LinearRegressor" +op_type: "LinearRegressor" +attribute { + name: "coefficients" + s: "" + type: FLOATS +} +attribute { + name: "intercepts" + s: "" + type: FLOATS +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "targets" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Generalized linear regression evaluation.
\n If targets is set to 1 (default) then univariate regression is performed.
\n If targets is set to M then M sets of coefficients must be passed in as a sequence\n and M results will be output for each input n in N.
\n The coefficients array is of length n, and the coefficients for each target are contiguous.\n Intercepts are optional but if provided must match the number of targets.\n" +----f +input: "input" +output: "output" +name: "Log" +op_type: "Log" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the natural log of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "LogSoftmax" +op_type: "LogSoftmax" +attribute { + name: "axis" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe operator computes the logsoftmax (log of softmax) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the logsoftmax values of the corresponding input.\n" +----f +input: "M" +input: "cond" +input: "v_initial" +output: "v_final_and_scan_outputs" +name: "Loop" +op_type: "Loop" +attribute { + name: "body" + s: "" + type: GRAPH +} +attribute { + name: "M-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "cond-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "v_initial-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGeneric Looping construct. This loop has multiple termination conditions:\n\n1) Trip count. Iteration count specified at runtime. Set by\n specifying the input M. Optional. Set to empty string to omit.\n Note that a static trip count (specified at graph construction time) can be\n specified by passing in a constant node for input M.\n2) Loop termination condition. This is an input to the op that determines\n whether to run the first iteration and also a loop-carried dependency for\n the body graph. The body graph must yield a value for the condition variable,\n whether this input is provided or not.\n\nThis table summarizes the operating modes of this operator with equivalent\nC-style code:\n\n Operator inputs defined as (max_trip_count, condition_var).\n\n input (\"\", \"\"):\n for (int i=0; ; ++i) {\n cond = ... // Note this value is ignored, but is required in the body\n }\n\n input (\"\", cond) // Note this is analogous to a while loop\n bool cond = ...;\n for (int i=0; cond; ++i) {\n cond = ...;\n }\n\n input (\"\", 1) // Note this is analogous to a do-while loop\n bool cond = true\n for (int i=0; cond; ++i) {\n cond = ...;\n }\n\n input (trip_count, \"\") // Note this is analogous to a for loop\n int trip_count = ...\n for (int i=0; i < trip_count; ++i) {\n cond = ...; // ignored\n }\n\n input (trip_count, cond)\n int trip_count = ...;\n bool cond = ...;\n for (int i=0; i < trip_count && cond; ++i) {\n cond = ...;\n }\n\n\n*Sample usage - cond as well as trip count*\n\n graph predict-net {\n %a = Constant[value = ]()\n %b = Constant[value = ]()\n %keepgoing = Constant[value = ]()\n %max_trip_count = Constant[value = ]()\n %keepgoing_out, %b_out, %user_defined_vals = Loop[body = ](%max_trip_count, %keepgoing, %b)\n return\n }\n\n graph body-net (\n %i[INT32, scalar] // iteration number\n %keepgoing_in[BOOL, scalar] // incoming loop-termination-condition; not used\n %b_in[INT32, scalar] // incoming value of loop-carried-dependency b\n ) {\n %my_local = Add(%a, %b_in)\n %b_out = Sub(%a, %b_in) // outgoing value of loop-carried-dependency b\n %keepgoing_out = Greater(%my_local, %b_out) // outgoing loop-termination-condition\n %user_defined_val = Add(%b_in, %b_in) // scan-output value to be accumulated\n return %keepgoing_out, %b_out, %user_defined_val\n }\n\n*Sample equivalent C code*\n\n {\n /* User-defined code (enclosing scope) */\n int a = 3, b = 6;\n bool keepgoing = true; // Analogous to input cond\n /* End user-defined code */\n\n /* Implicitly-defined code */\n const int max_trip_count = 10; // Analogous to input M\n int user_defined_vals[]; // Imagine this is resizable\n /* End implicitly-defined code */\n /* initialize loop-carried variables and scan-output variables */\n bool keepgoing_out = keepgoing\n int b_out = b\n\n for (int i=0; i < max_trip_count && keepgoing_out; ++i) {\n /* Implicitly-defined code: bind actual parameter values\n to formal parameter variables of loop-body */\n bool keepgoing_in = keepgoing_out; \n bool b_in = b_out;\n\n /* User-defined code (loop body) */\n int my_local = a + b_in; // Reading value \"a\" from the enclosing scope is fine\n b_out = a - b_in;\n keepgoing_out = my_local > b_out; \n user_defined_val = b_in + b_in; // b_in and b_out are different variables\n /* End user-defined code */\n\n /* Implicitly defined-code */\n user_defined_vals[i] = user_defined_val // accumulate scan-output values\n }\n // int t = my_local; // Can\'t do this. my_local is not accessible here.\n\n // The values below are bound to the output variables of the loop and therefore accessible\n // b_out; user_defined_vals; keepgoing_out;\n }\n\nThere are several things of note in this code snippet:\n\n1) Values from the enclosing scope (i.e. variable \"a\" here) are in scope and can\n be referenced in the inputs of the loop.\n2) Any values computed in the loop body that needs to be used in a subsequent\n iteration or after the loop are modelled using a pair of variables in the loop-body,\n consisting of an input variable (eg., b_in) and an output variable (eg., b_out).\n These are referred to as loop-carried dependences. The loop operation node\n supplies the input value of the input variable for the first iteration, and\n returns the output value of the output variable produced by the final\n iteration.\n3) Scan_output variables are used to implicitly concatenate values computed across\n all the iterations. In the above example, the value of user_defined_val computed\n over all iterations are concatenated and returned as the value of user_defined_vals\n after the loop.\n4) Values created in the body cannot be accessed in the enclosing scope,\n except using the mechanism described above.\n\nNote that the semantics of this op support \"diagonal\" or \"wavefront\" execution.\n(See Step 3 here for an example:\nhttps://devblogs.nvidia.com/optimizing-recurrent-neural-networks-cudnn-5/).\nFrontends should emit multi-layer RNNs as a series of While operators (with\ntime being the inner looping dimension), with each successive layer consuming\nthe scan_outputs from the previous layer, possibly going through several\npoint-wise operators (e.g. dropout, residual connections, linear layer).\n" +----f +input: "input" +output: "output" +name: "LpNormalization" +op_type: "LpNormalization" +attribute { + name: "axis" + i: -1 + type: INT +} +attribute { + name: "p" + i: 2 + type: INT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nGiven a matrix, apply Lp-normalization along the provided axis.\n" +----f +input: "X" +output: "Y" +name: "LpPool" +op_type: "LpPool" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "p" + i: 2 + type: INT +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n LpPool consumes an input tensor X and applies Lp pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n Lp pooling consisting of computing the Lp norm on all values of a subset\n of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing." +----f +input: "A" +input: "B" +output: "Y" +name: "MatMul" +op_type: "MatMul" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html\n" +----f +input: "A" +input: "B" +input: "a_zero_point" +input: "b_zero_point" +output: "Y" +name: "MatMulInteger" +op_type: "MatMulInteger" +attribute { + name: "A-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "B-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "a_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "b_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html.\nThe production MUST never overflow. The accumulation may overflow if and only if in 32 bits.\n" +----f +input: "data_0" +output: "max" +name: "Max" +op_type: "Max" +attribute { + name: "data_0-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nElement-wise max of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Y" +output: "Indices" +name: "MaxPool" +op_type: "MaxPool" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "ceil_mode" + i: 0 + type: INT +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "storage_order" + i: 0 + type: INT +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "int8" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "float" + type: STRINGS +} +doc_string: "\n MaxPool consumes an input tensor X and applies max pooling across\n the tensor according to kernel sizes, stride sizes, and pad lengths.\n max pooling consisting of computing the max on all values of a\n subset of the input tensor according to the kernel size and downsampling the\n data into the output tensor Y for further processing. The output spatial shape will be following:\n ```\n output_spatial_shape[i] = floor((input_spatial_shape[i] + pad_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1)) / strides_spatial_shape[i] + 1)\n ```\n or\n ```\n output_spatial_shape[i] = ceil((input_spatial_shape[i] + pad_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1)) / strides_spatial_shape[i] + 1)\n ```\n if ceil_mode is enabled\n\n ```\n * pad_shape[i] is sum of pads along axis i\n ```\n\n `auto_pad` is a DEPRECATED attribute. If you are using them currently, the output spatial shape will be following:\n ```\n VALID: output_spatial_shape[i] = ceil((input_spatial_shape[i] - ((kernel_spatial_shape[i] - 1) * dilations[i] + 1) + 1) / strides_spatial_shape[i])\n SAME_UPPER or SAME_LOWER: output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides_spatial_shape[i])\n ```\n And pad shape will be following if `SAME_UPPER` or `SAME_LOWER`:\n ```\n pad_shape[i] = (output_spatial_shape[i] - 1) * strides_spatial_shape[i] + ((kernel_spatial_shape[i] - 1) * dilations[i] + 1) - input_spatial_shape[i]\n ```\n The output of each pooling window is maximum number of elements exclude pad. \n " +----f +input: "X" +input: "rois" +output: "Y" +name: "MaxRoiPool" +op_type: "MaxRoiPool" +attribute { + name: "pooled_shape" + s: "" + type: INTS +} +attribute { + name: "spatial_scale" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "rois-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n ROI max pool consumes an input tensor X and region of interests (RoIs) to\n apply max pooling across each RoI, to produce output 4-D tensor of shape\n (num_rois, channels, pooled_shape[0], pooled_shape[1])." +----f +input: "X" +input: "I" +input: "output_shape" +output: "output" +name: "MaxUnpool" +op_type: "MaxUnpool" +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "I-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "output_shape-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nMaxUnpool essentially computes the partial inverse of the MaxPool op.\n The input information to this op is typically the the output information from a MaxPool op. The first\n input tensor X is the tensor that needs to be unpooled, which is typically the pooled tensor (first output)\n from MaxPool. The second input tensor, I, contains the indices to the (locally maximal) elements corrsponding\n to the elements in the first input tensor X. Input tensor I is typically the second output of the MaxPool op.\n The third (optional) input is a tensor that specifies the output size of the unpooling operation.\n\nMaxUnpool is intended to do \'partial\' inverse of the MaxPool op. \'Partial\' because all the non-maximal\n values from the original input to MaxPool are set to zero in the output of the MaxUnpool op. Pooling\n the result of an unpooling operation should give back the original input to the unpooling op.\n\nMaxUnpool can produce the same output size for several input sizes, which makes unpooling op ambiguous.\n The third input argument, output_size, is meant to disambiguate the op and produce output tensor of\n known/predictable size.\n\nIn addition to the inputs, MaxUnpool takes three attributes, namely kernel_shape, strides, and pads,\n which define the exact unpooling op. The attributes typically have the same values as the corrsponding\n pooling op that the unpooling op is trying to invert.\n" +----f +input: "data_0" +output: "mean" +name: "Mean" +op_type: "Mean" +attribute { + name: "data_0-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nElement-wise mean of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Y" +name: "MeanVarianceNormalization" +op_type: "MeanVarianceNormalization" +attribute { + name: "axes" + ints: 0 + ints: 2 + ints: 3 + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\n A MeanVarianceNormalization Function: Perform mean variance normalization\n on the input tensor X using formula:
``` (X-EX)/sqrt(E(X-EX)^2) ```\n" +----f +input: "data_0" +output: "min" +name: "Min" +op_type: "Min" +attribute { + name: "data_0-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nElement-wise min of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "A" +input: "B" +output: "C" +name: "Mod" +op_type: "Mod" +attribute { + name: "fmod" + i: 0 + type: INT +} +attribute { + name: "A-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\n Performs element-wise binary modulus (with Numpy-style broadcasting support). \n The sign of the remainder is the same as that of the Divisor.\n \n Mod operator can also behave like C fmod() or numpy.fmod. In this case, the sign of the remainder however, will be the same as the Dividend \n (in contrast to integer mod). To force a behavior like numpy.fmod() an \'fmod\' Attribute is provided.\n This attribute is set to 0 by default causing the behavior to be like integer mod. \n Setting this attribute to 1 causes the remainder to be calculated similar to that of numpy.fmod().\n\n If the input type is floating point, then `fmod` attribute must be set to 1.\n \n In case of dividend being zero, the results will be platform dependent.\n\n This operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "R" +input: "T" +input: "inputs" +output: "outputs" +name: "Momentum" +op_type: "Momentum" +attribute { + name: "alpha" + s: "" + type: FLOAT +} +attribute { + name: "beta" + s: "" + type: FLOAT +} +attribute { + name: "mode" + s: "" + type: STRING +} +attribute { + name: "norm_coefficient" + s: "" + type: FLOAT +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + type: STRINGS +} +attribute { + name: "T-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "inputs-types" + strings: "float" + strings: "double" + type: STRINGS +} +doc_string: "\n Compute one iteration of stochastic gradient update with momentum.\n This operator can conduct the optimization of multiple tensor variables.\n\n Let\'s define the behavior of this operator. As you can imagine, SG with momentum requires\n several parameters:\n \n - The learning-rate \"R\".\n - The update count \"T\". That is, the number of conducted training iterations. It should\n be zero in the first training iteration.\n - A L2-norm regularization coefficient \"norm_coefficient\".\n - A decay coefficient of previous accumulated gradient (i.e., momentum) \"alpha\".\n - The scaling coefficient of current gradient \"beta\".\n - An attribute to choose either standard momentum or Nesterov\'s momentum \"mode\" should\n be used.\n\n For the sake of simplicity, assume that there is only one tensor (called \"X\") to be optimized.\n Other necessary inputs are \"X\"\'s gradient (called \"G\") and \"X\"\'s momentum (called \"V\"). This\n Momentum operator maps all these inputs to the new value of \"X\" (called \"X_new\") and its new\n momentum (called \"V_new\").\n \n This operator supports two different momentum algorithms. Set the attribute \"mode\" to\n \"nesterov\" if Nesterov\'s momentum is desired. Otherwise, set the attribute \"model\" to\n \"standard\" to use standard momentum. Computation details are described subsequently.\n\n Let \"+\", \"-\", \"*\", and \"/\" are all element-wise operations with numpy-style broadcasting.\n\n Pseudo code for SG with standard momentum:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||^2, where ||X|| is the sum of squared\n // values of all elements in X.\n G_regularized = norm_coefficient * X + G\n\n // In the first training iteration, beta should always be 1.\n beta_adjusted = T > 0 ? beta : 1\n\n // Compute the current momentum based on previous momentum and the current gradient.\n V_new = alpha * V + beta_adjusted * G_regularized\n\n // Update X.\n X_new = X - R * V_new\n\n Pseudo code for SG with Nesterov\'s momentum:\n\n // Add gradient of 0.5 * norm_coefficient * ||X||^2, where ||X|| is the sum of squared\n // values of all elements in X.\n G_regularized = norm_coefficient * X + G;\n\n // In the first training iteration, beta should always be 1.\n beta_adjusted = T > 0 ? beta : 1\n\n // Compute the current momentum based on previous momentum and the current gradient.\n V_new = alpha * V + beta_adjusted * G_regularized;\n\n // Compute final update direction and then update X.\n X_new = X - R * (G_regularized + alpha * V_new)\n\n If one assign this operators to optimize multiple inputs, for example, \"X_1\" and \"X_2\". The same\n pseudo code would be extended to handle all tensors jointly. More specifically, we can view \"X\" as a\n concatenation of \"X_1\" and \"X_2\" (of course, their gradient and accumulate gradient should\n be concatenated too) and then our pseudo code becomes applicable.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Mul" +op_type: "Mul" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPerforms element-wise binary multiplication (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "input" +output: "output" +name: "Multinomial" +op_type: "Multinomial" +attribute { + name: "dtype" + i: 6 + type: INT +} +attribute { + name: "sample_size" + i: 1 + type: INT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nGenerate a tensor of samples from a multinomial distribution according to the probabilities\nof each of the possible outcomes.\n" +----f +input: "X" +output: "Y" +name: "Neg" +op_type: "Neg" +attribute { + name: "X-types" + strings: "int8" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + strings: "int16" + type: STRINGS +} +doc_string: "\nNeg takes one input data (Tensor) and produces one output data\n(Tensor) where each element flipped sign, y = -x, is applied to\nthe tensor elementwise.\n" +----f +input: "input" +input: "target" +input: "weight" +output: "loss" +name: "NegativeLogLikelihoodLoss" +op_type: "NegativeLogLikelihoodLoss" +attribute { + name: "ignore_index" + s: "" + type: INT +} +attribute { + name: "reduction" + s: "mean" + type: STRING +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "target-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "weight-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nA NegativeLogLikelihoodLoss operator computes (weighted) negative log likelihood loss.\nIts \"input\" tensor has the shape of (N, C, d1, d2, ..., dk) where k >= 0.\nThe \"input\" tensor contains log-probabilities for input[n, :, d_1, d_2,..., d_k] being in a class of [0, C).\nThe operator\'s \"target\" input tensor has the shape of (N, d1, d2, ..., dk). It encodes class labels (one of C classes)\nor it may contain a special value (indicated by an attribute ignore_index) for N x d1 x d2 x ... x dk samples.\nThe loss value for input[n, :, d_1, d_2,...d_k] being classified as class c = target[n][d_1][d_2]...[d_k] is computed as:\n\n loss[n][d_1][d_2]...[d_k] = -input[n][c][d_1][d_2]...[d_k].\n\nWhen an optional \"weight\" is provided, the sample loss is calculated as:\n\n loss[n][d_1][d_2]...[d_k] = -input[n][c][d_1][d_2]...[d_k] * weight[c].\n\nloss is zero for the case when target-value equals ignore_index.\n \n loss[n][d_1][d_2]...[d_k] = 0, when target[n][d_1][d_2]...[d_k] = ignore_index\n\nIf \"reduction\" attribute is set to \"none\", the operator\'s output will be the above loss with shape (N, d1, d2, ..., dk).\nIf \"reduction\" attribute is set to \"mean\" (the default attribute value), the output loss is (weight) averaged:\n\n mean(loss), if \"weight\" is not provided,\n\nor if weight is provided,\n\n sum(loss) / sum(weight[target[n][d_1][d_2]...[d_k]]]), for all samples.\n\nIf \"reduction\" attribute is set to \"sum\", the output is a scalar:\n sum(loss).\n\nSee also https://pytorch.org/docs/stable/nn.html#torch.nn.NLLLoss.\n\nExample 1:\n\n // negative log likelihood loss, \"none\" reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n\n loss = np.zeros((N, d1))\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1]\n\n // print(loss)\n // [[-3. -2.]\n // [-0. -2.]]\n\nExample 2:\n\n // weighted negative log likelihood loss, sum reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n weight = [0.2, 0.3, 0.1]\n loss = np.zeros((N, d1))\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1] * weight[c]\n\n loss = np.sum(loss)\n // print(loss)\n // -1.1\n\nExample 3:\n\n // weighted negative log likelihood loss, mean reduction\n N, C, d1 = 2, 3, 2\n input = [[[1.0, 2.0], [2.0, 2.0], [3.0, 2.0]],\n [[0.0, 1.0], [2.0, 2.0], [1.0, 2]]]\n target = [[2, 1], [0, 2]]\n weight = [0.2, 0.3, 0.1]\n loss = np.zeros((N, d1))\n weight_total = 0\n for n in range(N):\n for d_1 in range(d1):\n c = target[n][d_1]\n loss[n][d_1] = -input[n][c][d_1] * weight[c]\n weight_total = weight_total + weight[c]\n\n loss = np.sum(loss) / weight_total\n // print(loss)\n // -1.57\n" +----f +input: "boxes" +input: "scores" +input: "max_output_boxes_per_class" +input: "iou_threshold" +input: "score_threshold" +output: "selected_indices" +name: "NonMaxSuppression" +op_type: "NonMaxSuppression" +attribute { + name: "center_point_box" + i: 0 + type: INT +} +attribute { + name: "boxes-types" + strings: "float" + type: STRINGS +} +attribute { + name: "scores-types" + strings: "float" + type: STRINGS +} +attribute { + name: "max_output_boxes_per_class-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "iou_threshold-types" + strings: "float" + type: STRINGS +} +attribute { + name: "score_threshold-types" + strings: "float" + type: STRINGS +} +doc_string: "\nFilter out boxes that have high intersection-over-union (IOU) overlap with previously selected boxes.\nBounding boxes with score less than score_threshold are removed. Bounding box format is indicated by attribute center_point_box.\nNote that this algorithm is agnostic to where the origin is in the coordinate system and more generally is invariant to\northogonal transformations and translations of the coordinate system; thus translating or reflections of the coordinate system\nresult in the same boxes being selected by the algorithm.\nThe selected_indices output is a set of integers indexing into the input collection of bounding boxes representing the selected boxes.\nThe bounding box coordinates corresponding to the selected indices can then be obtained using the Gather or GatherND operation.\n" +----f +input: "X" +output: "Y" +name: "NonZero" +op_type: "NonZero" +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\n Returns the indices of the elements that are non-zero\n (in row-major order - by dimension).\n NonZero behaves similar to numpy.nonzero:\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.nonzero.html\n" +----f +input: "X" +output: "Y" +name: "Normalizer" +op_type: "Normalizer" +attribute { + name: "norm" + s: "MAX" + type: STRING +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Normalize the input. There are three normalization modes, which have the corresponding formulas,\n defined using element-wise infix operators \'/\' and \'^\' and tensor-wide functions \'max\' and \'sum\':
\n
\n Max: Y = X / max(X)
\n L1: Y = X / sum(X)
\n L2: Y = sqrt(X^2 / sum(X^2)}
\n In all modes, if the divisor is zero, Y == X.\n
\n For batches, that is, [N,C] tensors, normalization is done along the C axis. In other words, each row\n of the batch is normalized independently.\n" +----f +input: "X" +output: "Y" +name: "Not" +op_type: "Not" +attribute { + name: "X-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nReturns the negation of the input tensor element-wise.\n" +----f +input: "indices" +input: "depth" +input: "values" +output: "output" +name: "OneHot" +op_type: "OneHot" +attribute { + name: "axis" + i: -1 + type: INT +} +attribute { + name: "indices-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "depth-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "values-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\n Produces a one-hot tensor based on inputs.\n The locations represented by the index values in the \'indices\' input tensor will have \'on_value\'\n and the other locations will have \'off_value\' in the output tensor, where \'on_value\' and \'off_value\'\n are specified as part of required input argument \'values\', which is a two-element tensor of format\n [off_value, on_value]. The rank of the output tensor will be one greater than the rank of the\n input tensor. The additional dimension is for one-hot representation. The additional dimension will\n be inserted at the position specified by \'axis\'. If \'axis\' is not specified then then additional\n dimension will be inserted as the innermost dimension, i.e. axis=-1. The size of the additional\n dimension is specified by required scalar input \'depth\'. The type of the output tensor is the same\n as the type of the \'values\' input. Any entries in the \'indices\' input tensor with values outside\n the range [-depth, depth-1] will result in one-hot representation with all \'off_value\' values in the\n output tensor.\n\n when axis = 0:\n output[input[i, j, k], i, j, k] = 1 for all i, j, k and 0 otherwise.\n\n when axis = -1:\n output[i, j, k, input[i, j, k]] = 1 for all i, j, k and 0 otherwise.\n\n" +----f +input: "X" +output: "Y" +name: "OneHotEncoder" +op_type: "OneHotEncoder" +attribute { + name: "cats_int64s" + s: "" + type: INTS +} +attribute { + name: "cats_strings" + s: "" + type: STRINGS +} +attribute { + name: "zeros" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "int32" + strings: "string" + strings: "double" + strings: "int64" + strings: "float" + type: STRINGS +} +doc_string: "\n Replace each input element with an array of ones and zeros, where a single\n one is placed at the index of the category that was passed in. The total category count \n will determine the size of the extra dimension of the output array Y.
\n For example, if we pass a tensor with a single value of 4, and a category count of 8, \n the output will be a tensor with ``[0,0,0,0,1,0,0,0]``.
\n This operator assumes every input feature is from the same set of categories.
\n If the input is a tensor of float, int32, or double, the data will be cast\n to integers and the cats_int64s category list will be used for the lookups.\n" +----f +input: "A" +input: "B" +output: "C" +name: "Or" +op_type: "Or" +attribute { + name: "A-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "B-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `or` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +input: "slope" +output: "Y" +name: "PRelu" +op_type: "PRelu" +attribute { + name: "X-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "slope-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPRelu takes input data (Tensor) and slope tensor as input, and produces one\noutput data (Tensor) where the function `f(x) = slope * x for x < 0`,\n`f(x) = x for x >= 0`., is applied to the data tensor elementwise.\nThis operator supports **unidirectional broadcasting** (tensor slope should be unidirectional broadcastable to input tensor X); for more details please check [the doc](Broadcasting.md)." +----f +input: "data" +input: "pads" +input: "constant_value" +output: "output" +name: "Pad" +op_type: "Pad" +attribute { + name: "mode" + s: "constant" + type: STRING +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "pads-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "constant_value-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGiven a tensor containing the data to be padded (`data`), a tensor containing the number of start and end pad values for axis (`pads`), (optionally) a `mode`, and (optionally) `constant_value`, \na padded tensor (`output`) is generated.\n\nThe three supported `modes` are (similar to corresponding modes supported by `numpy.pad`):\n\n1) `constant`(default) - pads with a given constant value as specified by `constant_value` (which defaults to 0)\n\n2) `reflect` - pads with the reflection of the vector mirrored on the first and last values of the vector along each axis\n\n3) `edge` - pads with the edge values of array\n\n\nExample 1 (`constant` mode):\n Insert 0 pads to the beginning of the second dimension.\n\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'constant\'\n\n constant_value = 0.0\n\n output = \n [\n [\n [0.0, 0.0, 1.0, 1.2],\n [0.0, 0.0, 2.3, 3.4],\n [0.0, 0.0, 4.5, 5.7],\n ],\n ]\n\n\nExample 2 (`reflect` mode):\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'reflect\'\n\n output = \n [\n [\n [1.0, 1.2, 1.0, 1.2],\n [2.3, 3.4, 2.3, 3.4],\n [4.5, 5.7, 4.5, 5.7],\n ],\n ]\n\n\nExample 3 (`edge` mode):\n data = \n [\n [1.0, 1.2],\n [2.3, 3.4],\n [4.5, 5.7],\n ] \n\n pads = [0, 2, 0, 0]\n\n mode = \'edge\'\n\n output = \n [\n [\n [1.0, 1.0, 1.0, 1.2],\n [2.3, 2.3, 2.3, 3.4],\n [4.5, 4.5, 4.5, 5.7],\n ],\n ]\n\n" +----f +input: "X" +input: "Y" +output: "Z" +name: "Pow" +op_type: "Pow" +attribute { + name: "X-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + type: STRINGS +} +attribute { + name: "Y-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPow takes input data (Tensor) and exponent Tensor, and\nproduces one output data (Tensor) where the function `f(x) = x^exponent`,\nis applied to the data tensor elementwise.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md)." +----f +input: "x" +input: "x_scale" +input: "x_zero_point" +input: "w" +input: "w_scale" +input: "w_zero_point" +input: "y_scale" +input: "y_zero_point" +input: "B" +output: "y" +name: "QLinearConv" +op_type: "QLinearConv" +attribute { + name: "auto_pad" + s: "NOTSET" + type: STRING +} +attribute { + name: "dilations" + s: "" + type: INTS +} +attribute { + name: "group" + i: 1 + type: INT +} +attribute { + name: "kernel_shape" + s: "" + type: INTS +} +attribute { + name: "pads" + s: "" + type: INTS +} +attribute { + name: "strides" + s: "" + type: INTS +} +attribute { + name: "x-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "x_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "x_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "w-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "w_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "w_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "y_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "y_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "B-types" + strings: "int32" + type: STRINGS +} +doc_string: "\nThe convolution operator consumes a quantized input tensor, its scale and zero point,\na quantized filter, its scale and zero point, and output\'s scale and zero point,\nand computes the quantized output. Each scale and zero-point pair must have same shape.\nIt means they must be either scalars (per tensor) or 1-D tensors (per output channel).\nEach input or output and its related zero point must have same type.\nWhen bias is present it must be quantized using scale = input scale * weight scale and \nzero point as 0.\n" +----f +input: "a" +input: "a_scale" +input: "a_zero_point" +input: "b" +input: "b_scale" +input: "b_zero_point" +input: "y_scale" +input: "y_zero_point" +output: "y" +name: "QLinearMatMul" +op_type: "QLinearMatMul" +attribute { + name: "a-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "a_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "a_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "b-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "b_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "b_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +attribute { + name: "y_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "y_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nMatrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html.\nIt consumes two quantized input tensors, their scales and zero points, scale and zero point of output, and computes the quantized output.\nThe quantization formula is y = saturate((x / y_scale) + y_zero_point). For (x / y_scale), it is rounding to nearest ties to even.\nRefer to https://en.wikipedia.org/wiki/Rounding for details. Scale and zero point must have same shape.\nThey must be either scalar (per tensor) or 1-D tensor (per row for \'a\' and per column for \'b\'). If scale and zero point are 1-D tensor,\nthe number of elements of scale and zero point tensor of input \'a\' and output \'y\' should be equal to the number of rows of input \'a\',\nand the number of elements of scale and zero point tensor of input \'b\' should be equal to the number of columns of input \'b\'.\nProduction must never overflow, and accumulation may overflow if and only if in 32 bits.\n" +----f +input: "x" +input: "y_scale" +input: "y_zero_point" +output: "y" +name: "QuantizeLinear" +op_type: "QuantizeLinear" +attribute { + name: "x-types" + strings: "float" + strings: "int32" + type: STRINGS +} +attribute { + name: "y_scale-types" + strings: "float" + type: STRINGS +} +attribute { + name: "y_zero_point-types" + strings: "int8" + strings: "uint8" + type: STRINGS +} +doc_string: "\nThe linear per-tensor/layer quantization operator. It consumes a high precision tensor, a scale, a zero point to compute the low precision / quantized tensor.\nThe quantization formula is y = saturate ((x / y_scale) + y_zero_point). For saturation, it saturates to [0, 255] if it\'s uint8, or [-128, 127] if it\'s int8.\nFor (x / y_scale), it\'s rounding to nearest ties to even. Refer to https://en.wikipedia.org/wiki/Rounding for details. \'y_zero_point\' and \'y\' must have same type.\n" +----f +input: "X" +input: "W" +input: "R" +input: "B" +input: "sequence_lens" +input: "initial_h" +output: "Y" +output: "Y_h" +name: "RNN" +op_type: "RNN" +attribute { + name: "activation_alpha" + s: "" + type: FLOATS +} +attribute { + name: "activation_beta" + s: "" + type: FLOATS +} +attribute { + name: "activations" + strings: "Tanh" + strings: "Tanh" + type: STRINGS +} +attribute { + name: "clip" + s: "" + type: FLOAT +} +attribute { + name: "direction" + s: "forward" + type: STRING +} +attribute { + name: "hidden_size" + s: "" + type: INT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "W-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "R-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "sequence_lens-types" + strings: "int32" + type: STRINGS +} +attribute { + name: "initial_h-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nComputes an one-layer simple RNN. This operator is usually supported\nvia some custom implementation such as CuDNN.\n\nNotations:\n\n`X` - input tensor\n\n`i` - input gate\n\n`t` - time step (t-1 means previous time step)\n\n`Wi` - W parameter weight matrix for input gate\n\n`Ri` - R recurrence weight matrix for input gate\n\n`Wbi` - W parameter bias vector for input gate\n\n`Rbi` - R parameter bias vector for input gate\n\n`WBi` - W parameter weight matrix for backward input gate\n\n`RBi` - R recurrence weight matrix for backward input gate\n\n`WBbi` - WR bias vectors for backward input gate\n\n`RBbi` - RR bias vectors for backward input gate\n\n`H` - Hidden state\n\n`num_directions` - 2 if direction == bidirectional else 1\n\nActivation functions:\n\n Relu(x) - max(0, x)\n\n Tanh(x) - (1 - e^{-2x})/(1 + e^{-2x})\n\n Sigmoid(x) - 1/(1 + e^{-x})\n\n (NOTE: Below are optional)\n\n Affine(x) - alpha*x + beta\n\n LeakyRelu(x) - x if x >= 0 else alpha * x\n\n ThresholdedRelu(x) - x if x >= alpha else 0\n\n ScaledTanh(x) - alpha*Tanh(beta*x)\n\n HardSigmoid(x) - min(max(alpha*x + beta, 0), 1)\n\n Elu(x) - x if x >= 0 else alpha*(e^x - 1)\n\n Softsign(x) - x/(1 + |x|)\n\n Softplus(x) - log(1 + e^x)\n\nEquations (Default: f=Tanh):\n\n - Ht = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)\nThis operator has **optional** inputs/outputs. See [the doc](IR.md) for more details about the representation of optional arguments. An empty string may be used in the place of an actual argument\'s name to indicate a missing argument. Trailing optional arguments (those not followed by an argument that is present) may also be simply omitted.\n" +----f +output: "output" +name: "RandomNormal" +op_type: "RandomNormal" +attribute { + name: "dtype" + i: 1 + type: INT +} +attribute { + name: "mean" + f: 0.0 + type: FLOAT +} +attribute { + name: "scale" + f: 1.0 + type: FLOAT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "shape" + s: "" + type: INTS +} +doc_string: "\nGenerate a tensor with random values drawn from a normal distribution. The shape\nof the tensor is specified by the `shape` argument and the parameter of the normal distribution\nspecified by `mean` and `scale`.\n\nThe data type is specified by the \'dtype\' argument. The \'dtype\' argument must\nbe one of the data types specified in the \'DataType\' enum field in the\nTensorProto message.\n" +----f +input: "input" +output: "output" +name: "RandomNormalLike" +op_type: "RandomNormalLike" +attribute { + name: "dtype" + s: "" + type: INT +} +attribute { + name: "mean" + f: 0.0 + type: FLOAT +} +attribute { + name: "scale" + f: 1.0 + type: FLOAT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGenerate a tensor with random values drawn from a normal distribution.\nThe shape of the output tensor is copied from the shape of the input tensor,\nand the parameters of the normal distribution are specified by `mean` and `scale`.\n\nThe data type is specified by the \'dtype\' argument, or copied from the input tensor if not provided.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message, and be valid as an output type.\n" +----f +output: "output" +name: "RandomUniform" +op_type: "RandomUniform" +attribute { + name: "dtype" + i: 1 + type: INT +} +attribute { + name: "high" + f: 1.0 + type: FLOAT +} +attribute { + name: "low" + f: 0.0 + type: FLOAT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "shape" + s: "" + type: INTS +} +doc_string: "\nGenerate a tensor with random values drawn from a uniform distribution. The shape\nof the tensor is specified by the `shape` argument and the range by `low` and `high`.\n\nThe data type is specified by the \'dtype\' argument. The \'dtype\' argument must\nbe one of the data types specified in the \'DataType\' enum field in the\nTensorProto message.\n" +----f +input: "input" +output: "output" +name: "RandomUniformLike" +op_type: "RandomUniformLike" +attribute { + name: "dtype" + s: "" + type: INT +} +attribute { + name: "high" + f: 1.0 + type: FLOAT +} +attribute { + name: "low" + f: 0.0 + type: FLOAT +} +attribute { + name: "seed" + s: "" + type: FLOAT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nGenerate a tensor with random values drawn from a uniform distribution.\nThe shape of the output tensor is copied from the shape of the input tensor,\nand the parameters of the uniform distribution are specified by `low` and `high`.\n\nThe data type is specified by the \'dtype\' argument, or copied from the input tensor if not provided.\nThe \'dtype\' argument must be one of the data types specified in the \'DataType\' enum field in the\nTensorProto message and be valid as an output type.\n" +----f +input: "start" +input: "limit" +input: "delta" +output: "output" +name: "Range" +op_type: "Range" +attribute { + name: "start-types" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + strings: "int16" + type: STRINGS +} +attribute { + name: "limit-types" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + strings: "int16" + type: STRINGS +} +attribute { + name: "delta-types" + strings: "int32" + strings: "double" + strings: "int64" + strings: "float" + strings: "int16" + type: STRINGS +} +doc_string: "\nGenerate a tensor containing a sequence of numbers that begin at `start` and extends by increments of `delta`\nup to `limit` (exclusive).\n\nThe number of elements in the output of range is computed as below-\n\n`number_of_elements = max( ceil( (limit - start) / delta ) , 0 )`\n\nThe pseudocode determining the contents of the output is shown below-\n\n`for(int i=0; i) and produces one output data\n(Tensor) where the reciprocal is, y = 1/x, is applied to\nthe tensor elementwise.\n" +----f +input: "data" +output: "reduced" +name: "ReduceL1" +op_type: "ReduceL1" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the L1 norm of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceL2" +op_type: "ReduceL2" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the L2 norm of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceLogSum" +op_type: "ReduceLogSum" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the log sum of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceLogSumExp" +op_type: "ReduceLogSumExp" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the log sum exponent of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceMax" +op_type: "ReduceMax" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "int8" + strings: "float16" + strings: "int32" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the max of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceMean" +op_type: "ReduceMean" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the mean of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceMin" +op_type: "ReduceMin" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "int8" + strings: "float16" + strings: "int32" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the min of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceProd" +op_type: "ReduceProd" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the product of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceSum" +op_type: "ReduceSum" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the sum of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "data" +output: "reduced" +name: "ReduceSumSquare" +op_type: "ReduceSumSquare" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "data-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nComputes the sum square of the input tensor\'s element along the provided axes. The resulted\ntensor has the same rank as the input if keepdims equal 1. If keepdims equal 0, then\nthe resulted tensor have the reduced dimension pruned.\n\nThe above behavior is similar to numpy, with the exception that numpy default keepdims to\nFalse instead of True." +----f +input: "X" +output: "Y" +name: "Relu" +op_type: "Relu" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nRelu takes one input data (Tensor) and produces one output data\n(Tensor) where the rectified linear function, y = max(0, x), is applied to\nthe tensor elementwise.\n" +----f +input: "data" +input: "shape" +output: "reshaped" +name: "Reshape" +op_type: "Reshape" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "shape-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nReshape the input tensor similar to numpy.reshape.\nFirst input is the data tensor, second input is a shape tensor which specifies the output shape. It outputs the reshaped tensor.\nAt most one dimension of the new shape can be -1. In this case, the value is\ninferred from the size of the tensor and the remaining dimensions. A dimension\ncould also be 0, in which case the actual dimension value is unchanged (i.e. taken\nfrom the input tensor)." +----f +input: "X" +input: "roi" +input: "scales" +input: "sizes" +output: "Y" +name: "Resize" +op_type: "Resize" +attribute { + name: "coordinate_transformation_mode" + s: "half_pixel" + type: STRING +} +attribute { + name: "cubic_coeff_a" + f: -0.75 + type: FLOAT +} +attribute { + name: "exclude_outside" + i: 0 + type: INT +} +attribute { + name: "extrapolation_value" + f: 0.0 + type: FLOAT +} +attribute { + name: "mode" + s: "nearest" + type: STRING +} +attribute { + name: "nearest_mode" + s: "round_prefer_floor" + type: STRING +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "roi-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "scales-types" + strings: "float" + type: STRINGS +} +attribute { + name: "sizes-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nResize the input tensor. In general, it calculates every value in the output tensor as a weighted average of neighborhood (a.k.a. sampling locations) in the input tensor.\nEach dimension value of the output tensor is:\n output_dimension = floor(input_dimension * (roi_end - roi_start) * scale) if input \\\"sizes\\\" is not specified.\n" +----f +input: "input" +input: "sequence_lens" +output: "Y" +name: "ReverseSequence" +op_type: "ReverseSequence" +attribute { + name: "batch_axis" + i: 1 + type: INT +} +attribute { + name: "time_axis" + i: 0 + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "sequence_lens-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nReverse batch of sequences having different lengths specified by `sequence_lens`.\n\nFor each slice i iterating on batch axis, the operator reverses the first sequence_lens[i] elements on time axis,\nand copies elements whose index\'s beyond sequence_lens[i] to the output. So the output slice i contains reversed\nsequences on the first sequence_lens[i] elements, then have original values copied for the other elements.\n\nExample 1:\n input = [[0.0, 4.0, 8.0, 12.0],\n [1.0, 5.0, 9.0, 13.0],\n [2.0, 6.0, 10.0, 14.0],\n [3.0, 7.0, 11.0, 15.0]]\n sequence_lens = [4, 3, 2, 1]\n time_axis = 0\n batch_axis = 1\n\n output = [[3.0, 6.0, 9.0, 12.0],\n [2.0, 5.0, 8.0, 13.0],\n [1.0, 4.0, 10.0, 14.0],\n [0.0, 7.0, 11.0, 15.0]]\n\nExample 2:\n input = [[0.0, 1.0, 2.0, 3.0 ],\n [4.0, 5.0, 6.0, 7.0 ],\n [8.0, 9.0, 10.0, 11.0],\n [12.0, 13.0, 14.0, 15.0]]\n sequence_lens = [1, 2, 3, 4]\n time_axis = 1\n batch_axis = 0\n\n output = [[0.0, 1.0, 2.0, 3.0 ],\n [5.0, 4.0, 6.0, 7.0 ],\n [10.0, 9.0, 8.0, 11.0],\n [15.0, 14.0, 13.0, 12.0]]\n" +----f +input: "X" +input: "rois" +input: "batch_indices" +output: "Y" +name: "RoiAlign" +op_type: "RoiAlign" +attribute { + name: "mode" + s: "avg" + type: STRING +} +attribute { + name: "output_height" + i: 1 + type: INT +} +attribute { + name: "output_width" + i: 1 + type: INT +} +attribute { + name: "sampling_ratio" + i: 0 + type: INT +} +attribute { + name: "spatial_scale" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "rois-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "batch_indices-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nRegion of Interest (RoI) align operation described in the\n[Mask R-CNN paper](https://arxiv.org/abs/1703.06870).\nRoiAlign consumes an input tensor X and region of interests (rois)\nto apply pooling across each RoI; it produces a 4-D tensor of shape\n(num_rois, C, output_height, output_width).\n\nRoiAlign is proposed to avoid the misalignment by removing\nquantizations while converting from original image into feature\nmap and from feature map into RoI feature; in each ROI bin,\nthe value of the sampled locations are computed directly\nthrough bilinear interpolation.\n" +----f +input: "X" +output: "Y" +name: "Round" +op_type: "Round" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nRound takes one input Tensor and rounds the values, element-wise, meaning\nit finds the nearest integer for each value.\nIn case of halfs, the rule is to round them to the nearest even integer.\nThe output tensor has the same shape and type as the input.\n\nExamples:\n```\nround([0.9]) = [1.0]\nround([2.5]) = [2.0]\nround([2.3]) = [2.0]\nround([1.5]) = [2.0]\nround([-4.5]) = [-4.0]\n```\n" +----f +input: "X" +output: "Y" +output: "Z" +name: "SVMClassifier" +op_type: "SVMClassifier" +attribute { + name: "classlabels_ints" + s: "" + type: INTS +} +attribute { + name: "classlabels_strings" + s: "" + type: STRINGS +} +attribute { + name: "coefficients" + s: "" + type: FLOATS +} +attribute { + name: "kernel_params" + s: "" + type: FLOATS +} +attribute { + name: "kernel_type" + s: "LINEAR" + type: STRING +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "prob_a" + s: "" + type: FLOATS +} +attribute { + name: "prob_b" + s: "" + type: FLOATS +} +attribute { + name: "rho" + s: "" + type: FLOATS +} +attribute { + name: "support_vectors" + s: "" + type: FLOATS +} +attribute { + name: "vectors_per_class" + s: "" + type: INTS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Support Vector Machine classifier\n" +----f +input: "X" +output: "Y" +name: "SVMRegressor" +op_type: "SVMRegressor" +attribute { + name: "coefficients" + s: "" + type: FLOATS +} +attribute { + name: "kernel_params" + s: "" + type: FLOATS +} +attribute { + name: "kernel_type" + s: "LINEAR" + type: STRING +} +attribute { + name: "n_supports" + i: 0 + type: INT +} +attribute { + name: "one_class" + i: 0 + type: INT +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "rho" + s: "" + type: FLOATS +} +attribute { + name: "support_vectors" + s: "" + type: FLOATS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Support Vector Machine regression prediction and one-class SVM anomaly detection.\n" +----f +input: "X" +output: "Y" +name: "Scaler" +op_type: "Scaler" +attribute { + name: "offset" + s: "" + type: FLOATS +} +attribute { + name: "scale" + s: "" + type: FLOATS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Rescale input data, for example to standardize features by removing the mean and scaling to unit variance.\n" +----f +input: "initial_state_and_scan_inputs" +output: "final_state_and_scan_outputs" +name: "Scan" +op_type: "Scan" +attribute { + name: "body" + s: "" + type: GRAPH +} +attribute { + name: "num_scan_inputs" + s: "" + type: INT +} +attribute { + name: "scan_input_axes" + s: "" + type: INTS +} +attribute { + name: "scan_input_directions" + s: "" + type: INTS +} +attribute { + name: "scan_output_axes" + s: "" + type: INTS +} +attribute { + name: "scan_output_directions" + s: "" + type: INTS +} +attribute { + name: "initial_state_and_scan_inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nScan can be used to iterate over one or more scan_input tensors,\nconstructing zero or more scan_output tensors. It combines ideas from general recurrences,\nfunctional programming constructs such as scan, fold, map, and zip and is intended to enable\ngeneralizations of RNN-like constructs for sequence-to-sequence processing.\nOther tensors (referred to as state_variables here) can be used to carry a state\nwhen iterating from one element to another (similar to hidden-state in RNNs, also referred\nto as loop-carried dependences in the context of loops).\nMany common usages involve a single scan_input tensor (where functionality\nsimilar to scan, fold and map can be obtained). When more than one scan_input is used,\na behavior similar to zip is obtained.\n\nThe attribute body must be a graph, specifying the computation to be performed in\nevery iteration. It takes as input the current values of the state_variables and\nthe current iterated element of the scan_inputs. It must return the (updated) values\nof the state_variables and zero or more scan_output_element tensors. The values of the\nscan_output_element tensors are concatenated over all the iterations to produce the\nscan_output values of the scan construct (similar to the concatenated intermediate\nhidden-state values of RNN-like constructs). All the output tensors (state_variables as\nwell as scan_output_element tensors) are required to have the same shape in each iteration\nof the loop (a restriction imposed to enable efficient memory allocation).\n\nNote that the iterated element passed to the body subgraph does not have a sequence\naxis. It will have a rank one less than the rank of the corresponding scan_input.\n\nThe scan operation returns the final values of the state_variables as well as the\nscan_outputs.\n\nThe optional attribute scan_input_directions specifies the direction (forward or backward)\nfor each scan input. If this attribute is omitted, all sequences are scanned in the forward\ndirection. A bidirectional scan may be performed by specifying the same tensor input twice\nin the scan_inputs, once with a forward direction, and once with a backward direction.\n\nThe scan_output of the operation is produced by concatenating the scan_output_element\nvalues produced by the body in each iteration. The optional attribute scan_output_directions\nspecifies the direction in which scan_output is constructed (by appending or prepending the\nscan_output_element to scan_output in each iteration) for each scan_output. If this attribute\nis omitted, the scan_output_element is appended to the scan_output in each iteration.\n\nThe optional attribute scan_input_axes specifies the axis to be scanned for each scan_input.\nIf omitted, every scan_input will be scanned in axis 0. For example, if axis 0 is the\nbatch axis and axis 1 is the time axis (to be scanned), specify an axis value of 1.\nNote that scanning a non-zero axis may be less efficient than scanning axis zero.\n\nThe optional attribute scan_output_axes specifies the axis along which the scan_outputs\nare accumulated for each scan_output. For example, if axis 1 is the time axis (to be\nscanned) for both inputs and outputs, specify a scan_input axis and scan_output axis\nvalue of 1.\n\nNote that because of the ONNX restriction that only the last parameter of an operator can\nbe variadic, the initial-states and scan-inputs are listed together as one input parameter.\nSimilarly, the final-states and scan-outputs are listed together as one output parameter.\nThe attribute num_scan_inputs indicates the number M of scan-inputs.\n\nThe behavior of\n\n Scan <\n num_scan_inputs = m,\n body = loop-body,\n scan_input_axes = [axis_1, ..., axis_m]\n > (init_1, ..., init_n, scan_1, ..., scan_m)\n\nis equivalent to the following pseudo-code:\n\n // scan_i.shape[axis_i] denotes the (max) sequence-length of scan_i\n // scan_i.shape[axis_i] is required to be equal to scan_j.shape[axis_j] for all i,j.\n sequence_length = scan_1.shape[axis_1];\n\n // initialize state-variables\n st_1 = init_1; ... st_n = init_n;\n // initialize scan-output variables: [] denotes an empty tensor\n scan_out_1 = []; ...; scan_out_k = [];\n // identify number of iterations:\n\n // execute loop\n for (int t = 0; t < sequence_length; ++t) {\n // generate the scan-input elements: the notation T[t] indicates the sub-tensor\n // of rank one less than T obtained by indexing T at position t along axis k.\n si_1 = scan_1[t];\n ... ;\n si_m = scan_m[t];\n // execute loop-body\n st_1, ..., st_n, so_1, ..., so_k = loop-body(st_1, ..., st_n, si_1, ..., si_m)\n // accumulate the scan-output elements\n scan_out_1 = Concat(scan_out_1, so_1); ... ; scan_out_k = Concat(scan_out_k, so_k);\n }\n\n return st_1, ..., st_n, scan_out_1, ..., scan_out_k;\n\n*Sample usage: Encoding RNN using a Scan*\n\nThe following example shows how a simple RNN over an input tensor %X, with weight tensor %Wi,\nrecurrence weight tensor %Ri, bias tensors %Wbi and %Rbi, and initial hidden-state %H_0 can\nbe encoded as a ScanLoop. Note that the loop-body is a nested graph, and it directly computes\n%Wi, %Ri, %Wbi, and %Rbi (typically constants or initializers in the body graph). If these\nvalues are computed in the outer graph, they need to be passed in as extra state_variables.\n\n graph rnn-encoding {\n %H_0 = ... \n %X = ...\n %Y_h, %Y = Scan[body = , num_scan_inputs=1](%H_0, %X)\n return %Y, %Y_h\n }\n\n graph rnn-cell-1 (\n %H_tminus1[FLOAT, tensor]\n %X_t[FLOAT, tensor]\n ) {\n %Wi = ...\n %Ri = ...\n %Wbi = ...\n %Rbi = ...\n %t1 = X_t * (Wi^T)\n %t2 = H_tminus1*(Ri^T)\n %t3 = Add(%t1, %t2)\n %t4 = Add(%t3, %Wbi)\n %t5 = Add(%t4, %Rbi)\n %Ht = Tanh(%t5)\n %Accumulate = Identity(%Ht)\n return %Ht, %Accumulate\n }\n\n" +----f +input: "data" +input: "indices" +input: "updates" +output: "output" +name: "Scatter" +op_type: "Scatter" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "updates-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nThis operator is deprecated. Please use ScatterElements, which provides the same functionality.\n\nScatter takes three inputs `data`, `updates`, and `indices` of the same\nrank r >= 1 and an optional attribute axis that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value\nto values specified by `updates` at specific index positions specified by\n`indices`. Its output shape is the same as the shape of `data`.\n\nFor each entry in `updates`, the target index in `data` is obtained by combining\nthe corresponding entry in `indices` with the index of the entry itself: the\nindex-value for dimension = axis is obtained from the value of the corresponding\nentry in `indices` and the index-value for dimension != axis is obtained from the\nindex of the entry itself.\n\nFor instance, in a 2-D tensor case, the update corresponding to the [i][j] entry\nis performed as below:\n```\n output[indices[i][j]][j] = updates[i][j] if axis = 0, \n output[i][indices[i][j]] = updates[i][j] if axis = 1,\n```\n\nThis operator is the inverse of GatherElements. It is similar to Torch\'s Scatter operation.\n\nExample 1:\n```\n data = [\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n ]\n indices = [\n [1, 0, 2],\n [0, 2, 1],\n ]\n updates = [\n [1.0, 1.1, 1.2],\n [2.0, 2.1, 2.2],\n ]\n output = [\n [2.0, 1.1, 0.0]\n [1.0, 0.0, 2.2]\n [0.0, 2.1, 1.2]\n ]\n```\nExample 2:\n```\n data = [[1.0, 2.0, 3.0, 4.0, 5.0]]\n indices = [[1, 3]]\n updates = [[1.1, 2.1]]\n axis = 1\n output = [[1.0, 1.1, 3.0, 2.1, 5.0]]\n```\n" +----f +input: "data" +input: "indices" +input: "updates" +output: "output" +name: "ScatterElements" +op_type: "ScatterElements" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "updates-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nScatterElements takes three inputs `data`, `updates`, and `indices` of the same\nrank r >= 1 and an optional attribute axis that identifies an axis of `data`\n(by default, the outer-most axis, that is axis 0). The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value\nto values specified by `updates` at specific index positions specified by\n`indices`. Its output shape is the same as the shape of `data`.\n\nFor each entry in `updates`, the target index in `data` is obtained by combining\nthe corresponding entry in `indices` with the index of the entry itself: the\nindex-value for dimension = axis is obtained from the value of the corresponding\nentry in `indices` and the index-value for dimension != axis is obtained from the\nindex of the entry itself.\n\nFor instance, in a 2-D tensor case, the update corresponding to the [i][j] entry\nis performed as below:\n```\n output[indices[i][j]][j] = updates[i][j] if axis = 0, \n output[i][indices[i][j]] = updates[i][j] if axis = 1,\n```\n\nThis operator is the inverse of GatherElements. It is similar to Torch\'s Scatter operation.\n\nExample 1:\n```\n data = [\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n [0.0, 0.0, 0.0],\n ]\n indices = [\n [1, 0, 2],\n [0, 2, 1],\n ]\n updates = [\n [1.0, 1.1, 1.2],\n [2.0, 2.1, 2.2],\n ]\n output = [\n [2.0, 1.1, 0.0]\n [1.0, 0.0, 2.2]\n [0.0, 2.1, 1.2]\n ]\n```\nExample 2:\n```\n data = [[1.0, 2.0, 3.0, 4.0, 5.0]]\n indices = [[1, 3]]\n updates = [[1.1, 2.1]]\n axis = 1\n output = [[1.0, 1.1, 3.0, 2.1, 5.0]]\n```\n" +----f +input: "data" +input: "indices" +input: "updates" +output: "output" +name: "ScatterND" +op_type: "ScatterND" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "indices-types" + strings: "int64" + type: STRINGS +} +attribute { + name: "updates-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nScatterND takes three inputs `data` tensor of rank r >= 1, `indices` tensor of rank q >= 1,\nand `updates` tensor of rank q + r - indices.shape[-1] - 1. The output of the operation\nis produced by creating a copy of the input `data`, and then updating its value to values\nspecified by `updates` at specific index positions specified by `indices`. Its output shape\nis the same as the shape of `data`. Note that `indices` should not have duplicate entries.\nThat is, two or more `updates` for the same index-location is not supported.\n\n`indices` is an integer tensor. Let k denote indices.shape[-1], the last dimension in the shape of `indices`.\n `indices` is treated as a (q-1)-dimensional tensor of k-tuples, where each k-tuple is a partial-index into `data`.\nHence, k can be a value at most the rank of `data`. When k equals rank(data), each update entry specifies an\nupdate to a single element of the tensor. When k is less than rank(data) each update entry specifies an\nupdate to a slice of the tensor.\n\n`updates` is treated as a (q-1)-dimensional tensor of replacement-slice-values. Thus, the\nfirst (q-1) dimensions of updates.shape must match the first (q-1) dimensions of indices.shape.\nThe remaining dimensions of `updates` correspond to the dimensions of the\nreplacement-slice-values. Each replacement-slice-value is a (r-k) dimensional tensor,\ncorresponding to the trailing (r-k) dimensions of `data`. Thus, the shape of `updates`\nmust equal indices.shape[0:q-1] ++ data.shape[k:r-1], where ++ denotes the concatenation\nof shapes.\n\nThe `output` is calculated via the following equation:\n\n output = np.copy(data)\n update_indices = indices.shape[:-1]\n for idx in np.ndindex(update_indices):\n output[indices[idx]] = updates[idx]\n\nThe order of iteration in the above loop is not specified.\nIn particular, indices should not have duplicate entries: that is, if idx1 != idx2, then indices[idx1] != indices[idx2].\nThis ensures that the output value does not depend on the iteration order.\n\nThis operator is the inverse of GatherND.\n\nExample 1:\n```\n data = [1, 2, 3, 4, 5, 6, 7, 8]\n indices = [[4], [3], [1], [7]]\n updates = [9, 10, 11, 12]\n output = [1, 11, 3, 10, 9, 6, 7, 12]\n```\n\nExample 2:\n```\n data = [[[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]]]\n indices = [[0], [2]]\n updates = [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],\n [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]]\n output = [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],\n [[1, 2, 3, 4], [5, 6, 7, 8], [8, 7, 6, 5], [4, 3, 2, 1]],\n [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]],\n [[8, 7, 6, 5], [4, 3, 2, 1], [1, 2, 3, 4], [5, 6, 7, 8]]]\n```\n" +----f +input: "X" +output: "Y" +name: "Selu" +op_type: "Selu" +attribute { + name: "alpha" + f: 1.6732632 + type: FLOAT +} +attribute { + name: "gamma" + f: 1.050701 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nSelu takes one input data (Tensor) and produces one output data\n(Tensor) where the scaled exponential linear unit function,\n`y = gamma * (alpha * e^x - alpha) for x <= 0`, `y = gamma * x for x > 0`,\nis applied to the tensor elementwise.\n" +----f +input: "input_sequence" +input: "position" +output: "tensor" +name: "SequenceAt" +op_type: "SequenceAt" +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +attribute { + name: "position-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nOutputs a tensor copy from the tensor at \'position\' in \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n - 1]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n" +----f +input: "inputs" +output: "output_sequence" +name: "SequenceConstruct" +op_type: "SequenceConstruct" +attribute { + name: "inputs-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nConstruct a tensor sequence containing \'inputs\' tensors.\nAll tensors in \'inputs\' must have the same data type.\n" +----f +output: "output" +name: "SequenceEmpty" +op_type: "SequenceEmpty" +attribute { + name: "dtype" + s: "" + type: INT +} +doc_string: "\nConstruct an empty tensor sequence, with given data type.\n" +----f +input: "input_sequence" +input: "position" +output: "output_sequence" +name: "SequenceErase" +op_type: "SequenceErase" +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +attribute { + name: "position-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nOutputs a tensor sequence that removes the tensor at \'position\' from \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n - 1]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n\'position\' is optional, by default it erases the last tensor from \'input_sequence\'.\n" +----f +input: "input_sequence" +input: "tensor" +input: "position" +output: "output_sequence" +name: "SequenceInsert" +op_type: "SequenceInsert" +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +attribute { + name: "tensor-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "position-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nOutputs a tensor sequence that inserts \'tensor\' into \'input_sequence\' at \'position\'.\n\'tensor\' must have the same data type as \'input_sequence\'.\nAccepted range for \'position\' is in `[-n, n]`, where `n` is the number of tensors in \'input_sequence\'.\nNegative value means counting positions from the back.\n\'position\' is optional, by default it inserts \'tensor\' to the back of \'input_sequence\'.\n" +----f +input: "input_sequence" +output: "length" +name: "SequenceLength" +op_type: "SequenceLength" +attribute { + name: "input_sequence-types" + strings: "seq(bool" + strings: "seq(complex128" + strings: "seq(string" + strings: "seq(float16" + strings: "seq(int64" + strings: "seq(float" + strings: "seq(int32" + strings: "seq(uint32" + strings: "seq(uint16" + strings: "seq(int8" + strings: "seq(int16" + strings: "seq(complex64" + strings: "seq(uint64" + strings: "seq(double" + strings: "seq(uint8" + type: STRINGS +} +doc_string: "\nProduces a scalar(tensor of empty shape) containing the number of tensors in \'input_sequence\'.\n" +----f +input: "data" +output: "shape" +name: "Shape" +op_type: "Shape" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nTakes a tensor as input and outputs an 1D int64 tensor containing the shape of the input tensor.\n" +----f +input: "input" +output: "output" +name: "Shrink" +op_type: "Shrink" +attribute { + name: "bias" + f: 0.0 + type: FLOAT +} +attribute { + name: "lambd" + f: 0.5 + type: FLOAT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nShrink takes one input data (Tensor) and produces one Tensor output,\nhaving same datatype and shape with input. It has two attributes, lambd and\nbias. The formula of this operator is: If x < -lambd, y = x + bias;\nIf x > lambd, y = x - bias; Otherwise, y = 0.\n" +----f +input: "X" +output: "Y" +name: "Sigmoid" +op_type: "Sigmoid" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nSigmoid takes one input data (Tensor) and produces one output data\n(Tensor) where the sigmoid function, y = 1 / (1 + exp(-x)), is applied to the\ntensor elementwise.\n" +----f +input: "input" +output: "output" +name: "Sign" +op_type: "Sign" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nCalculate the sign of the given input tensor element-wise.\nIf input > 0, output 1. if input < 0, output -1. if input == 0, output 0.\n" +----f +input: "input" +output: "output" +name: "Sin" +op_type: "Sin" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the sine of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Sinh" +op_type: "Sinh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic sine of the given input tensor element-wise.\n" +----f +input: "data" +output: "size" +name: "Size" +op_type: "Size" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nTakes a tensor as input and outputs a int64 scalar that equals to the total number of elements of the input tensor.\n" +----f +input: "data" +input: "starts" +input: "ends" +input: "axes" +input: "steps" +output: "output" +name: "Slice" +op_type: "Slice" +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "starts-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "ends-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "axes-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "steps-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nProduces a slice of the input tensor along multiple axes. Similar to numpy:\nhttps://docs.scipy.org/doc/numpy/reference/arrays.indexing.html\nSlices uses `starts`, `ends`, `axes` and `steps` inputs to specify the start and end\ndimension and step for each axis in the list of axes, it uses this information to\nslice the input `data` tensor. If a negative value is passed for any of the\nstart or end indices, it represents number of elements before the end of that\ndimension. If the value passed to start or end is larger than the `n` (the\nnumber of elements in this dimension), it represents `n`. For slicing to the\nend of a dimension with unknown size, it is recommended to pass in `INT_MAX` \nwhen sclicing forward and \'INT_MIN\' when slicing backward.\nIf a negative value is passed for step, it represents slicing backward. \nHowever step value cannot be 0.\nIf `axes` are omitted, they are set to `[0, ..., ndim-1]`.\nIf `steps` are omitted, they are set to `[1, ..., 1]` of length `len(starts)`\nExample 1:\n data = [\n [1, 2, 3, 4],\n [5, 6, 7, 8],\n ]\n axes = [0, 1]\n starts = [1, 0]\n ends = [2, 3]\n steps = [1, 2]\n result = [\n [5, 7],\n ]\nExample 2:\n data = [\n [1, 2, 3, 4],\n [5, 6, 7, 8],\n ]\n starts = [0, 1]\n ends = [-1, 1000]\n result = [\n [2, 3, 4],\n ]\n" +----f +input: "input" +output: "output" +name: "Softmax" +op_type: "Softmax" +attribute { + name: "axis" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThe operator computes the softmax (normalized exponential) values for each layer in the batch\n of the given input.\n\nThe input does not need to explicitly be a 2D vector; rather, it will be\ncoerced into one. For an arbitrary n-dimensional tensor\ninput \\in [a_0, a_1, ..., a_{k-1}, a_k, ..., a_{n-1}] and k is\nthe axis provided, then input will be coerced into a 2-dimensional tensor with\ndimensions [a_0 * ... * a_{k-1}, a_k * ... * a_{n-1}]. For the default\ncase where axis=1, this means the input tensor will be coerced into a 2D tensor\nof dimensions [a_0, a_1 * ... * a_{n-1}], where a_0 is often the batch size.\nIn this situation, we must have a_0 = N and a_1 * ... * a_{n-1} = D.\nEach of these dimensions must be matched correctly, or else the operator\nwill throw errors. The output tensor has the same shape\nand contains the softmax values of the corresponding input.\n" +----f +input: "scores" +input: "labels" +input: "weights" +output: "output" +output: "log_prob" +name: "SoftmaxCrossEntropyLoss" +op_type: "SoftmaxCrossEntropyLoss" +attribute { + name: "ignore_index" + s: "" + type: INT +} +attribute { + name: "reduction" + s: "mean" + type: STRING +} +attribute { + name: "scores-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +attribute { + name: "labels-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +attribute { + name: "weights-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "Loss function that measures the softmax cross entropy\nbetween \'scores\' and \'labels\'.\nThis operator first computes a loss tensor whose shape is identical to the labels input.\nIf the input is 2-D with shape (N, C), the loss tensor may be a N-element vector L = (l_1, l_2, ..., l_N).\nIf the input is N-D tensor with shape (N, C, D1, D2, ..., Dk),\nthe loss tensor L may have (N, D1, D2, ..., Dk) as its shape and L[i,][j_1][j_2]...[j_k] denotes a scalar element in L.\nAfter L is available, this operator can optionally do a reduction operator.\n\nshape(scores): (N, C) where C is the number of classes, or (N, C, D1, D2,..., Dk),\n with K >= 1 in case of K-dimensional loss.\nshape(labels): (N) where each value is 0 <= labels[i] <= C-1, or (N, D1, D2,..., Dk),\n with K >= 1 in case of K-dimensional loss.\n\nThe loss for one sample, l_i, can caculated as follows:\n l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk], where i is the index of classes.\nor\n l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk] * weights[c], if \'weights\' is provided.\n\nloss is zero for the case when label-value equals ignore_index.\n l[i][d1][d2]...[dk] = 0, when labels[n][d1][d2]...[dk] = ignore_index\n\nwhere:\n p = Softmax(scores)\n y = Log(p)\n c = labels[i][d1][d2]...[dk]\n\nFinally, L is optionally reduced:\nIf reduction = \'none\', the output is L with shape (N, D1, D2, ..., Dk).\nIf reduction = \'sum\', the output is scalar: Sum(L).\nIf reduction = \'mean\', the output is scalar: ReduceMean(L), or if weight is provided: ReduceSum(L) / ReduceSum(W),\nwhere tensor W is of shape (N, D1, D2, ..., Dk) and W[n][d1][d2]...[dk] = weights[labels[i][d1][d2]...[dk]].\n" +----f +input: "X" +output: "Y" +name: "Softplus" +op_type: "Softplus" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nSoftplus takes one input data (Tensor) and produces one output data\n(Tensor) where the softplus function, y = ln(exp(x) + 1), is applied to\nthe tensor elementwise.\n" +----f +input: "input" +output: "output" +name: "Softsign" +op_type: "Softsign" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the softsign (x/(1+|x|)) of the given input tensor element-wise.\n" +----f +input: "input" +output: "output" +name: "SpaceToDepth" +op_type: "SpaceToDepth" +attribute { + name: "blocksize" + s: "" + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "SpaceToDepth rearranges blocks of spatial data into depth. More specifically,\nthis op outputs a copy of the input tensor where values from the height and width dimensions\nare moved to the depth dimension.\n" +----f +input: "input" +output: "outputs" +name: "Split" +op_type: "Split" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "split" + s: "" + type: INTS +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "Split a tensor into a list of tensors, along the specified\n\'axis\'. Lengths of the parts can be specified using argument \'split\'.\nOtherwise, the tensor is split to equal sized parts.\n" +----f +input: "input" +input: "split" +output: "output_sequence" +name: "SplitToSequence" +op_type: "SplitToSequence" +attribute { + name: "axis" + i: 0 + type: INT +} +attribute { + name: "keepdims" + i: 1 + type: INT +} +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "split-types" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "Split a tensor into a sequence of tensors, along the specified\n\'axis\'. Lengths of the parts can be specified using argument \'split\'.\n\'split\' must contain only positive numbers.\n\'split\' is either a scalar (tensor of empty shape), or a 1-D tensor.\nIf \'split\' is a scalar, then \'input\' will be split into equally sized chunks(if possible).\nLast chunk will be smaller if the \'input\' size along the given axis \'axis\' is not divisible\nby \'split\'.\nOtherwise, the tensor is split into \'size(split)\' chunks, with lengths of the parts on \'axis\'\nspecified in \'split\'. In this scenario, the sum of entries in \'split\' must be equal to the\ndimension size of input tensor on \'axis\'.\n" +----f +input: "X" +output: "Y" +name: "Sqrt" +op_type: "Sqrt" +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nSquare root takes one input data (Tensor) and produces one output data\n(Tensor) where the square root is, y = x^0.5, is applied to\nthe tensor elementwise. If x is negative, then it will return NaN.\n" +----f +input: "data" +output: "squeezed" +name: "Squeeze" +op_type: "Squeeze" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nRemove single-dimensional entries from the shape of a tensor.\nTakes a parameter `axes` with a list of axes to squeeze.\nIf `axes` is not provided, all the single dimensions will be removed from\nthe shape. If an axis is selected with shape entry not equal to one, an error is raised.\n" +----f +input: "X" +output: "Y" +name: "StringNormalizer" +op_type: "StringNormalizer" +attribute { + name: "case_change_action" + s: "NONE" + type: STRING +} +attribute { + name: "is_case_sensitive" + i: 0 + type: INT +} +attribute { + name: "locale" + s: "" + type: STRING +} +attribute { + name: "stopwords" + s: "" + type: STRINGS +} +attribute { + name: "X-types" + strings: "string" + type: STRINGS +} +doc_string: "\nStringNormalization performs string operations for basic cleaning.\nThis operator has only one input (denoted by X) and only one output\n(denoted by Y). This operator first examines the elements in the X,\nand removes elements specified in \"stopwords\" attribute.\nAfter removing stop words, the intermediate result can be further lowercased,\nuppercased, or just returned depending the \"case_change_action\" attribute.\nThis operator only accepts [C]- and [1, C]-tensor.\nIf all elements in X are dropped, the output will be the empty value of string tensor with shape [1]\nif input shape is [C] and shape [1, 1] if input shape is [1, C].\n" +----f +input: "A" +input: "B" +output: "C" +name: "Sub" +op_type: "Sub" +attribute { + name: "A-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +attribute { + name: "B-types" + strings: "float16" + strings: "int32" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "uint64" + type: STRINGS +} +doc_string: "\nPerforms element-wise binary subtraction (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "data_0" +output: "sum" +name: "Sum" +op_type: "Sum" +attribute { + name: "data_0-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nElement-wise sum of each of the input tensors (with Numpy-style broadcasting support).\nAll inputs and outputs must have the same data type.\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "input" +output: "output" +name: "Tan" +op_type: "Tan" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the tangent of the given input tensor, element-wise.\n" +----f +input: "input" +output: "output" +name: "Tanh" +op_type: "Tanh" +attribute { + name: "input-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nCalculates the hyperbolic tangent of the given input tensor element-wise.\n" +----f +input: "X" +output: "Y" +name: "TfIdfVectorizer" +op_type: "TfIdfVectorizer" +attribute { + name: "max_gram_length" + s: "" + type: INT +} +attribute { + name: "max_skip_count" + s: "" + type: INT +} +attribute { + name: "min_gram_length" + s: "" + type: INT +} +attribute { + name: "mode" + s: "" + type: STRING +} +attribute { + name: "ngram_counts" + s: "" + type: INTS +} +attribute { + name: "ngram_indexes" + s: "" + type: INTS +} +attribute { + name: "pool_int64s" + s: "" + type: INTS +} +attribute { + name: "pool_strings" + s: "" + type: STRINGS +} +attribute { + name: "weights" + s: "" + type: FLOATS +} +attribute { + name: "X-types" + strings: "string" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\nThis transform extracts n-grams from the input sequence and save them as a vector. Input can\nbe either a 1-D or 2-D tensor. For 1-D input, output is the n-gram representation of that input.\nFor 2-D input, the output is also a 2-D tensor whose i-th row is the n-gram representation of the i-th input row.\nMore specifically, if input shape is [C], the corresponding output shape would be [max(ngram_indexes) + 1].\nIf input shape is [N, C], this operator produces a [N, max(ngram_indexes) + 1]-tensor.\n\nIn contrast to standard n-gram extraction, here, the indexes of extracting an n-gram from the original\nsequence are not necessarily consecutive numbers. The discontinuity between indexes are controlled by the number of skips.\nIf the number of skips is 2, we should skip two tokens when scanning through the original sequence.\nLet\'s consider an example. Assume that input sequence is [94, 17, 36, 12, 28] and the number of skips is 2.\nThe associated 2-grams are [94, 12] and [17, 28] respectively indexed by [0, 3] and [1, 4].\nIf the number of skips becomes 0, the 2-grams generated are [94, 17], [17, 36], [36, 12], [12, 28]\nindexed by [0, 1], [1, 2], [2, 3], [3, 4], respectively.\n\nThe output vector (denoted by Y) stores the count of each n-gram;\nY[ngram_indexes[i]] indicates the times that the i-th n-gram is found. The attribute ngram_indexes is used to determine the mapping\nbetween index i and the corresponding n-gram\'s output coordinate. If pool_int64s is [94, 17, 17, 36], ngram_indexes is [1, 0],\nngram_counts=[0, 0], then the Y[0] (first element in Y) and Y[1] (second element in Y) are the counts of [17, 36] and [94, 17],\nrespectively. An n-gram which cannot be found in pool_strings/pool_int64s should be ignored and has no effect on the output.\nNote that we may consider all skips up to S when generating the n-grams.\n\nThe examples used above are true if mode is \"TF\". If mode is \"IDF\", all the counts larger than 1 would be truncated to 1 and\nthe i-th element in weights would be used to scale (by multiplication) the count of the i-th n-gram in pool. If mode is \"TFIDF\",\nthis operator first computes the counts of all n-grams and then scale them by the associated values in the weights attribute.\n\nOnly one of pool_strings and pool_int64s can be set. If pool_int64s is set, the input should be an integer tensor.\nIf pool_strings is set, the input must be a string tensor.\n" +----f +input: "X" +output: "Y" +name: "ThresholdedRelu" +op_type: "ThresholdedRelu" +attribute { + name: "alpha" + f: 1.0 + type: FLOAT +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "float16" + type: STRINGS +} +doc_string: "\nThresholdedRelu takes one input data (Tensor) and produces one output data\n(Tensor) where the rectified linear function, y = x for x > alpha, y = 0 otherwise,\nis applied to the tensor elementwise.\n" +----f +input: "input" +input: "repeats" +output: "output" +name: "Tile" +op_type: "Tile" +attribute { + name: "input-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "repeats-types" + strings: "int64" + type: STRINGS +} +doc_string: "Constructs a tensor by tiling a given tensor.\nThis is the same as function `tile` in Numpy, but no broadcast.\nFor example A = [[1, 2], [3, 4]], B = [1, 2], tile(A, B) = [[1, 2, 1, 2], [3, 4, 3, 4]]\n" +----f +input: "X" +input: "K" +output: "Values" +output: "Indices" +name: "TopK" +op_type: "TopK" +attribute { + name: "axis" + i: -1 + type: INT +} +attribute { + name: "largest" + i: 1 + type: INT +} +attribute { + name: "sorted" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "float" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "K-types" + strings: "int64" + type: STRINGS +} +doc_string: "\nRetrieve the top-K largest or smallest elements along a specified axis. Given an input tensor of\nshape [a_1, a_2, ..., a_n, r] and integer argument k, return two outputs:\n -Value tensor of shape [a_1, a_2, ..., a_{axis-1}, k, a_{axis+1}, ... a_n]\n which contains the values of the top k elements along the specified axis\n -Index tensor of shape [a_1, a_2, ..., a_{axis-1}, k, a_{axis+1}, ... a_n] which\n contains the indices of the top k elements (original indices from the input\n tensor).\n\nIf \"largest\" is 1 (the default value) then the k largest elements are returned.\nIf \"sorted\" is 1 (the default value) then the resulting k elements will be sorted.\nIf \"sorted\" is 0, order of returned \'Values\' and \'Indices\' are undefined.\n\nGiven two equivalent values, this operator uses the indices along the axis as\n a tiebreaker. That is, the element with the lower index will appear first.\n" +----f +input: "data" +output: "transposed" +name: "Transpose" +op_type: "Transpose" +attribute { + name: "perm" + s: "" + type: INTS +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nTranspose the input tensor similar to numpy.transpose. For example, when\nperm=(1, 0, 2), given an input tensor of shape (1, 2, 3), the output shape\nwill be (2, 1, 3).\n" +----f +input: "X" +output: "Y" +output: "Z" +name: "TreeEnsembleClassifier" +op_type: "TreeEnsembleClassifier" +attribute { + name: "base_values" + s: "" + type: FLOATS +} +attribute { + name: "class_ids" + s: "" + type: INTS +} +attribute { + name: "class_nodeids" + s: "" + type: INTS +} +attribute { + name: "class_treeids" + s: "" + type: INTS +} +attribute { + name: "class_weights" + s: "" + type: FLOATS +} +attribute { + name: "classlabels_int64s" + s: "" + type: INTS +} +attribute { + name: "classlabels_strings" + s: "" + type: STRINGS +} +attribute { + name: "nodes_falsenodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_featureids" + s: "" + type: INTS +} +attribute { + name: "nodes_hitrates" + s: "" + type: FLOATS +} +attribute { + name: "nodes_missing_value_tracks_true" + s: "" + type: INTS +} +attribute { + name: "nodes_modes" + s: "" + type: STRINGS +} +attribute { + name: "nodes_nodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_treeids" + s: "" + type: INTS +} +attribute { + name: "nodes_truenodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_values" + s: "" + type: FLOATS +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Tree Ensemble classifier. Returns the top class for each of N inputs.
\n The attributes named \'nodes_X\' form a sequence of tuples, associated by \n index into the sequences, which must all be of equal length. These tuples\n define the nodes.
\n Similarly, all fields prefixed with \'class_\' are tuples of votes at the leaves.\n A leaf may have multiple votes, where each vote is weighted by\n the associated class_weights index.
\n One and only one of classlabels_strings or classlabels_int64s\n will be defined. The class_ids are indices into this list.\n" +----f +input: "X" +output: "Y" +name: "TreeEnsembleRegressor" +op_type: "TreeEnsembleRegressor" +attribute { + name: "aggregate_function" + s: "SUM" + type: STRING +} +attribute { + name: "base_values" + s: "" + type: FLOATS +} +attribute { + name: "n_targets" + s: "" + type: INT +} +attribute { + name: "nodes_falsenodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_featureids" + s: "" + type: INTS +} +attribute { + name: "nodes_hitrates" + s: "" + type: FLOATS +} +attribute { + name: "nodes_missing_value_tracks_true" + s: "" + type: INTS +} +attribute { + name: "nodes_modes" + s: "" + type: STRINGS +} +attribute { + name: "nodes_nodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_treeids" + s: "" + type: INTS +} +attribute { + name: "nodes_truenodeids" + s: "" + type: INTS +} +attribute { + name: "nodes_values" + s: "" + type: FLOATS +} +attribute { + name: "post_transform" + s: "NONE" + type: STRING +} +attribute { + name: "target_ids" + s: "" + type: INTS +} +attribute { + name: "target_nodeids" + s: "" + type: INTS +} +attribute { + name: "target_treeids" + s: "" + type: INTS +} +attribute { + name: "target_weights" + s: "" + type: FLOATS +} +attribute { + name: "X-types" + strings: "float" + strings: "double" + strings: "int32" + strings: "int64" + type: STRINGS +} +doc_string: "\n Tree Ensemble regressor. Returns the regressed values for each input in N.
\n All args with nodes_ are fields of a tuple of tree nodes, and\n it is assumed they are the same length, and an index i will decode the\n tuple across these inputs. Each node id can appear only once\n for each tree id.
\n All fields prefixed with target_ are tuples of votes at the leaves.
\n A leaf may have multiple votes, where each vote is weighted by\n the associated target_weights index.
\n All trees must have their node ids start at 0 and increment by 1.
\n Mode enum is BRANCH_LEQ, BRANCH_LT, BRANCH_GTE, BRANCH_GT, BRANCH_EQ, BRANCH_NEQ, LEAF\n" +----f +input: "X" +output: "Y" +output: "indices" +output: "inverse_indices" +output: "counts" +name: "Unique" +op_type: "Unique" +attribute { + name: "axis" + s: "" + type: INT +} +attribute { + name: "sorted" + i: 1 + type: INT +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nFind the unique elements of a tensor. When an optional attribute \'axis\' is provided, unique subtensors sliced along the \'axis\' are returned. \nOtherwise the input tensor is flattened and unique values of the flattened tensor are returned. \n\nThis operator returns the unique values or sliced unique subtensors of the input tensor and three optional outputs. \nThe first output tensor \'Y\' contains all unique values or subtensors of the input. \nThe second optional output tensor \'indices\' contains indices of \'Y\' elements\' first occurance in \'X\'.. \nThe third optional output tensor \'inverse_indices\' contains, for elements of \'X\', its corresponding indices in \'Y\'. \". \nThe fourth optional output tensor \'counts\' contains the count of each element of \'Y\' in the input. \n\nOutputs are either sorted in ascending order or optionally in the order of the first occurrence of the values in the input. \n\nhttps://docs.scipy.org/doc/numpy/reference/generated/numpy.unique.html\n\nExample 1:\n input_X = [2, 1, 1, 3, 4, 3]\n attribute_sorted = 0\n attribute_axis = None\n output_Y = [2, 1, 3, 4]\n output_indices = [0, 1, 3, 4]\n output_inverse_indices = [0, 1, 1, 2, 3, 2]\n output_counts = [1, 2, 2, 1]\n\nExample 2:\n input_X = [[1, 3], [2, 3]]\n attribute_sorted = 1\n attribute_axis = None\n output_Y = [1, 2, 3]\n output_indices = [0, 2, 1]\n output_inverse_indices = [0, 2, 1, 2]\n output_counts = [1, 1, 2]\n\nExample 3:\n input_X = [[1, 0, 0], [1, 0, 0], [2, 3, 4]]\n attribute_sorted = 1\n attribute_axis = 0\n output_Y = [[1, 0, 0], [2, 3, 4]]\n output_indices = [0, 2]\n output_inverse_indices = [0, 0, 1]\n output_counts = [2, 1]\n\nExample 4:\n input_x = [[[1., 1.], [0., 1.], [2., 1.], [0., 1.]], \n [[1., 1.], [0., 1.], [2., 1.], [0., 1.]]]\n attribute_sorted = 1\n attribute_axis = 1\n\n intermediate data are presented below for better understanding: \n \n there are 4 subtensors sliced along axis 1 of input_x (shape = (2, 4, 2)):\n A: [[1, 1], [1, 1]], \n [[0, 1], [0, 1]], \n [[2, 1], [2, 1]], \n [[0, 1], [0, 1]].\n \n there are 3 unique subtensors: \n [[1, 1], [1, 1]], \n [[0, 1], [0, 1]], \n [[2, 1], [2, 1]].\n \n sorted unique subtensors:\n B: [[0, 1], [0, 1]], \n [[1, 1], [1, 1]], \n [[2, 1], [2, 1]].\n \n output_Y is constructed from B:\n [[[0. 1.], [1. 1.], [2. 1.]], \n [[0. 1.], [1. 1.], [2. 1.]]]\n\n output_indices is to map from B to A:\n [1, 0, 2]\n \n output_inverse_indices is to map from A to B:\n [1, 0, 2, 0]\n\n output_counts = [2 1 1]\n" +----f +input: "data" +output: "expanded" +name: "Unsqueeze" +op_type: "Unsqueeze" +attribute { + name: "axes" + s: "" + type: INTS +} +attribute { + name: "data-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\nInsert single-dimensional entries to the shape of an input tensor (`data`).\nTakes one required argument `axes` - which contains a list of dimension indices and this operator will insert a dimension of value `1` into the corresponding index of the output tensor (`expanded`).\n\nFor example:\n Given an input tensor (`data`) of shape [3, 4, 5], then\n Unsqueeze(data, axes=[0, 4]) outputs a tensor (`expanded`) containing same data as `data` but with shape [1, 3, 4, 5, 1].\n\nThe attribute `axes` should not contain any duplicate entries. It is an error if it contains duplicates.\nThe rank of the output tensor (`output_rank`) is the rank of the input tensor (`data`) plus the number of values in `axes`.\nEach value in `axes` should be within the (inclusive) range [-output_rank , output_rank - 1]. \nThe order of values in `axes` does not matter and can come in any order. \n\n" +----f +input: "X" +input: "scales" +output: "Y" +name: "Upsample" +op_type: "Upsample" +attribute { + name: "mode" + s: "nearest" + type: STRING +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "scales-types" + strings: "float" + type: STRINGS +} +doc_string: "\nUpsample the input tensor.\nEach dimension value of the output tensor is:\n output_dimension = floor(input_dimension * scale).\n" +----f +input: "condition" +input: "X" +input: "Y" +output: "output" +name: "Where" +op_type: "Where" +attribute { + name: "condition-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "X-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +attribute { + name: "Y-types" + strings: "uint16" + strings: "int8" + strings: "bool" + strings: "int32" + strings: "float16" + strings: "uint8" + strings: "string" + strings: "double" + strings: "int64" + strings: "uint32" + strings: "complex64" + strings: "float" + strings: "complex128" + strings: "int16" + strings: "uint64" + type: STRINGS +} +doc_string: "\n Return elements, either from X or Y, depending on condition\n (with Numpy-style broadcasting support).\n Where behaves like numpy.where with three parameters:\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.where.html\n" +----f +input: "A" +input: "B" +output: "C" +name: "Xor" +op_type: "Xor" +attribute { + name: "A-types" + strings: "bool" + type: STRINGS +} +attribute { + name: "B-types" + strings: "bool" + type: STRINGS +} +doc_string: "\nReturns the tensor resulted from performing the `xor` logical operation\nelementwise on the input tensors `A` and `B` (with Numpy-style broadcasting support).\n\nThis operator supports **multidirectional (i.e., Numpy-style) broadcasting**; for more details please check [the doc](Broadcasting.md).\n" +----f +input: "X" +output: "Z" +name: "ZipMap" +op_type: "ZipMap" +attribute { + name: "classlabels_int64s" + s: "" + type: INTS +} +attribute { + name: "classlabels_strings" + s: "" + type: STRINGS +} +attribute { + name: "X-types" + strings: "float" + type: STRINGS +} +doc_string: "\n Creates a map from the input and the attributes.
\n The values are provided by the input tensor, while the keys are specified by the attributes.\n Must provide keys in either classlabels_strings or classlabels_int64s (but not both).
\n The columns of the tensor correspond one-by-one to the keys specified by the attributes. There must be as many columns as keys.
\n" +----f diff --git a/contrib/codegen-tools/onnx-def-gen/onnx_def_gen.py b/contrib/codegen-tools/onnx-def-gen/onnx_def_gen.py index 518924814..dafb9c29c 100644 --- a/contrib/codegen-tools/onnx-def-gen/onnx_def_gen.py +++ b/contrib/codegen-tools/onnx-def-gen/onnx_def_gen.py @@ -1,133 +1,133 @@ -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ - -from onnx.defs import get_all_schemas -from onnx import NodeProto,GraphProto -from google.protobuf import text_format -import onnx.helper - - -nodes = [] -schemas = get_all_schemas() - - -def load_node(input_str): - """ - Return a node - :param input_str: - :return: - """ - node_proto = NodeProto() - text_format.Parse(input_str,node_proto) - return node_proto - -# default values for each type for serialization - - -def convert_attr_type_to_enum(attr_value): - """ - Pass in an attribute from OpDescriptor and - get back out the equivalent enum value - for conversion to an attribute proto. - :param attr_value: the attribute value - :return: - """ - if str(attr_value.type) == 'AttrType.INTS': - return 7 - elif str(attr_value.type) == 'AttrType.UNDEFINED': - return 0 - elif str(attr_value.type) == 'AttrType.FLOATS': - return 6 - elif str(attr_value.type) == 'AttrType.GRAPH': - return 5 - elif str(attr_value.type) == 'AttrType.GRAPHS': - return 10 - elif str(attr_value.type) == 'AttrType.INT': - return 2 - elif str(attr_value.type) == 'AttrType.STRING': - return 3 - elif str(attr_value.type) == 'AttrType.TENSOR': - return 4 - elif str(attr_value.type) == 'AttrType.TENSORS': - return 9 - elif str(attr_value.type) == 'AttrType.SPARSE_TENSOR': - return 11 - elif str(attr_value.type) == 'AttrType.SPARSE_TENSORS': - return 12 - elif str(attr_value.type) == 'AttrType.FLOAT': - return 1 - elif str(attr_value.type) == 'AttrType.STRINGS': - return 8 - else: - raise Exception('Invalid type passed in') - -def create_node_from_schema(schema): - - """ - Convert an OpSchema to a NodeProto - :param schema: the input OpSchema - :return: the equivalent NodeProto - """ - - node_proto = NodeProto() - for attribute in schema.attributes: - attr_value = schema.attributes[attribute] - if attr_value.default_value.name == '': - attr_value_new = onnx.helper.make_attribute(attr_value.name,'') - attr_value_new.type = convert_attr_type_to_enum(attr_value) - node_proto.attribute.append(attr_value_new) - else: - node_proto.attribute.append(attr_value.default_value) - node_proto.op_type = schema.name - node_proto.doc_string = schema.doc - node_proto.name = schema.name - for input_arr in schema.inputs: - input_types = input_arr.types - type_attr = onnx.helper.make_attribute(input_arr.name + '-types', [str(data_type).replace('tensor(', '').replace(')', '') for data_type in input_types]) - node_proto.attribute.append(type_attr) - - if node_proto.input is None: - node_proto.input = [] - node_proto.input.append(input_arr.name) - for output_arr in schema.outputs: - if node_proto.output is None: - node_proto.output = [] - output_types = output_arr.types - type_attr = onnx.helper.make_attribute(output_arr.name + '-types', - [str(data_type).replace('tensor(', '').replace(')', '') for data_type - in output_types]) - node_proto.attribute.append(type_attr) - node_proto.output.append(output_arr.name) - return node_proto - - -nodes = [create_node_from_schema(schema) for schema - in sorted(schemas, key=lambda s: s.name)] - -with open('onnx-op-defs.pb', 'wb') as f: - graph_proto = GraphProto() - graph_proto.node.extend(nodes) - f.write(graph_proto.SerializeToString()) - # for node in nodes: - # message_to_string = text_format.MessageToString(node, as_utf8=True) - # node_2 = load_node(message_to_string) - # f.write(message_to_string + '----f\n') - -# with open('onnx.pbtxt','r') as f: -# nodes = [load_node(node_str) for node_str in f.read().split('----f\n')] -# print(nodes) +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ + +from onnx.defs import get_all_schemas +from onnx import NodeProto,GraphProto +from google.protobuf import text_format +import onnx.helper + + +nodes = [] +schemas = get_all_schemas() + + +def load_node(input_str): + """ + Return a node + :param input_str: + :return: + """ + node_proto = NodeProto() + text_format.Parse(input_str,node_proto) + return node_proto + +# default values for each type for serialization + + +def convert_attr_type_to_enum(attr_value): + """ + Pass in an attribute from OpDescriptor and + get back out the equivalent enum value + for conversion to an attribute proto. + :param attr_value: the attribute value + :return: + """ + if str(attr_value.type) == 'AttrType.INTS': + return 7 + elif str(attr_value.type) == 'AttrType.UNDEFINED': + return 0 + elif str(attr_value.type) == 'AttrType.FLOATS': + return 6 + elif str(attr_value.type) == 'AttrType.GRAPH': + return 5 + elif str(attr_value.type) == 'AttrType.GRAPHS': + return 10 + elif str(attr_value.type) == 'AttrType.INT': + return 2 + elif str(attr_value.type) == 'AttrType.STRING': + return 3 + elif str(attr_value.type) == 'AttrType.TENSOR': + return 4 + elif str(attr_value.type) == 'AttrType.TENSORS': + return 9 + elif str(attr_value.type) == 'AttrType.SPARSE_TENSOR': + return 11 + elif str(attr_value.type) == 'AttrType.SPARSE_TENSORS': + return 12 + elif str(attr_value.type) == 'AttrType.FLOAT': + return 1 + elif str(attr_value.type) == 'AttrType.STRINGS': + return 8 + else: + raise Exception('Invalid type passed in') + +def create_node_from_schema(schema): + + """ + Convert an OpSchema to a NodeProto + :param schema: the input OpSchema + :return: the equivalent NodeProto + """ + + node_proto = NodeProto() + for attribute in schema.attributes: + attr_value = schema.attributes[attribute] + if attr_value.default_value.name == '': + attr_value_new = onnx.helper.make_attribute(attr_value.name,'') + attr_value_new.type = convert_attr_type_to_enum(attr_value) + node_proto.attribute.append(attr_value_new) + else: + node_proto.attribute.append(attr_value.default_value) + node_proto.op_type = schema.name + node_proto.doc_string = schema.doc + node_proto.name = schema.name + for input_arr in schema.inputs: + input_types = input_arr.types + type_attr = onnx.helper.make_attribute(input_arr.name + '-types', [str(data_type).replace('tensor(', '').replace(')', '') for data_type in input_types]) + node_proto.attribute.append(type_attr) + + if node_proto.input is None: + node_proto.input = [] + node_proto.input.append(input_arr.name) + for output_arr in schema.outputs: + if node_proto.output is None: + node_proto.output = [] + output_types = output_arr.types + type_attr = onnx.helper.make_attribute(output_arr.name + '-types', + [str(data_type).replace('tensor(', '').replace(')', '') for data_type + in output_types]) + node_proto.attribute.append(type_attr) + node_proto.output.append(output_arr.name) + return node_proto + + +nodes = [create_node_from_schema(schema) for schema + in sorted(schemas, key=lambda s: s.name)] + +with open('onnx-op-defs.pb', 'wb') as f: + graph_proto = GraphProto() + graph_proto.node.extend(nodes) + f.write(graph_proto.SerializeToString()) + # for node in nodes: + # message_to_string = text_format.MessageToString(node, as_utf8=True) + # node_2 = load_node(message_to_string) + # f.write(message_to_string + '----f\n') + +# with open('onnx.pbtxt','r') as f: +# nodes = [load_node(node_str) for node_str in f.read().split('----f\n')] +# print(nodes) diff --git a/contrib/codegen-tools/onnx-def-gen/save_test.py b/contrib/codegen-tools/onnx-def-gen/save_test.py index 664c60c8e..ef85399a0 100644 --- a/contrib/codegen-tools/onnx-def-gen/save_test.py +++ b/contrib/codegen-tools/onnx-def-gen/save_test.py @@ -1,29 +1,29 @@ -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ - -import tensorflow as tf -from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 - -def graph_as_func(): - S = tf.Variable(tf.constant([1, 2, 3, 4])) - result = tf.scatter_add(S, [0], [10]) - return result -a_function_that_uses_a_graph = tf.function(graph_as_func) -print(a_function_that_uses_a_graph.__attr__) -converted = convert_variables_to_constants_v2(a_function_that_uses_a_graph) +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ + +import tensorflow as tf +from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 + +def graph_as_func(): + S = tf.Variable(tf.constant([1, 2, 3, 4])) + result = tf.scatter_add(S, [0], [10]) + return result +a_function_that_uses_a_graph = tf.function(graph_as_func) +print(a_function_that_uses_a_graph.__attr__) +converted = convert_variables_to_constants_v2(a_function_that_uses_a_graph) print(type(converted)) \ No newline at end of file diff --git a/contrib/codegen-tools/onnx-def-gen/test_onnx_lenet.py b/contrib/codegen-tools/onnx-def-gen/test_onnx_lenet.py index 70d0284b6..993f85aaa 100644 --- a/contrib/codegen-tools/onnx-def-gen/test_onnx_lenet.py +++ b/contrib/codegen-tools/onnx-def-gen/test_onnx_lenet.py @@ -1,20 +1,20 @@ -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ - -import onnx +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ + +import onnx loaded = onnx.load('lenet.onnx') \ No newline at end of file diff --git a/contrib/codegen-tools/onnx-def-gen/test_op_def_gen.py b/contrib/codegen-tools/onnx-def-gen/test_op_def_gen.py index 2b2dacf3a..b0aa0599b 100644 --- a/contrib/codegen-tools/onnx-def-gen/test_op_def_gen.py +++ b/contrib/codegen-tools/onnx-def-gen/test_op_def_gen.py @@ -1,35 +1,35 @@ -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ - -from onnx_tf.common import attr_converter,attr_translator -from onnx_tf.handlers.backend import * -import onnx_tf -import onnx_tf.handlers.handler -import sys,inspect -import tensorflow as tf -from onnx_tf.backend import TensorflowBackend - - -current_module = sys.modules['onnx_tf.handlers.backend'] -modules = inspect.getmembers(current_module) -for name, obj in modules: - obj_modules = inspect.getmembers(obj) - for name2,module2 in obj_modules: - if inspect.isclass(module2): - result = module2 +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ + +from onnx_tf.common import attr_converter,attr_translator +from onnx_tf.handlers.backend import * +import onnx_tf +import onnx_tf.handlers.handler +import sys,inspect +import tensorflow as tf +from onnx_tf.backend import TensorflowBackend + + +current_module = sys.modules['onnx_tf.handlers.backend'] +modules = inspect.getmembers(current_module) +for name, obj in modules: + obj_modules = inspect.getmembers(obj) + for name2,module2 in obj_modules: + if inspect.isclass(module2): + result = module2 print(module2) \ No newline at end of file diff --git a/createTestBackends.gradle b/createTestBackends.gradle new file mode 100644 index 000000000..cbe536802 --- /dev/null +++ b/createTestBackends.gradle @@ -0,0 +1,111 @@ +/* + * + * ****************************************************************************** + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + * + */ + + +ext { + buildTarget = rootProject.ext.buildTarget + apply from: new File("${project.rootProject.projectDir}/chooseBackend.gradle") + + chipList.each { thisChip -> + configurations.register("${thisChip}TestImplementation") { + it.extendsFrom configurations.testImplementation, configurations.implementation + + } + configurations.register("${thisChip}TestRuntime") { + + if(configurations.find {Configuration c -> c.name == "api"}) { + it.extendsFrom configurations.api, configurations.testRuntimeOnly, configurations.implementation,configurations.testImplementation + } else { + it.extendsFrom configurations.testRuntimeOnly, configurations.implementation,configurations.testImplementation + } + + } + + tasks.register("${thisChip}Test", Test) { + it.testClassesDirs = sourceSets.test.output.classesDirs + it.classpath = configurations.getByName("${thisChip}TestRuntime") + it.classpath += sourceSets.test.output.classesDirs + it.classpath += sourceSets.main.output.classesDirs + it.ignoreFailures = true + it.testLogging { + events "PASSED", "SKIPPED", "FAILED", "STANDARD_OUT", "STANDARD_ERROR" + } + it.useJUnitPlatform { + if( project.hasProperty("includeTags") ) { + it.includeTags=project.getProperty("includeTags").split(",") + } + if( project.hasProperty("excludeTags") ) { + it.excludeTags=project.getProperty("excludeTags").split(",") + } + } + ignoreFailures = true + testLogging { + events "PASSED", "SKIPPED", "FAILED", "STANDARD_OUT", "STANDARD_ERROR" + } + + // it.debug = true + it.enabled = true + + it.minHeapSize = "1024m" // initial heap size + it.maxHeapSize = "4096m" // maximum heap size + //it.jvmArgs '-XX:MaxPermSize=256m' // mem argument for the test JVM + + } + tasks.test.dependsOn "${thisChip}Test" + } + + test { + enabled = false + } + + + + dependencies { + if (withCuda()) { + cudaTestRuntime platform(projects.cavisCommonPlatform) + cudaTestRuntime projects.cavisNative.cavisNativeJcublas + cudaTestRuntime group: "org.bytedeco", name: "openblas" + cudaTestRuntime group: "org.bytedeco", name: "openblas", classifier: buildTarget + cudaTestRuntime group: "org.bytedeco", name: "cuda" + cudaTestRuntime group: "org.bytedeco", name: "cuda", classifier: buildTarget + cudaTestRuntime group: "org.bytedeco", name: "cuda", classifier: "${buildTarget}-redist" + cudaTestRuntime(project(":cavis-native:cavis-native-lib")) { + capabilities { + it.requireCapabilities "net.brutex.cavis-native:cavis-native-lib-cuda-support:1.0.0-SNAPSHOT" + } + } + } + + if (withCpu()) { + cpuTestRuntime platform(projects.cavisCommonPlatform) + cpuTestRuntime projects.cavisNative.cavisNativeCpu + cpuTestRuntime group: "org.bytedeco", name: "openblas" + cpuTestRuntime group: "org.bytedeco", name: "openblas", classifier: buildTarget + cpuTestRuntime group: "org.bytedeco", name: "opencv" + cpuTestRuntime group: "org.bytedeco", name: "opencv", classifier: buildTarget + cpuTestRuntime(project(":cavis-native:cavis-native-lib")) { + capabilities { + it.requireCapabilities "net.brutex.cavis-native:cavis-native-lib-cpu-support:1.0.0-SNAPSHOT" + } + } + } + } +} \ No newline at end of file diff --git a/datavec/buildmultiplescalaversions.sh b/datavec/buildmultiplescalaversions.sh deleted file mode 100755 index 148b00885..000000000 --- a/datavec/buildmultiplescalaversions.sh +++ /dev/null @@ -1,71 +0,0 @@ -#! /bin/bash -# -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ -# - -BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -function echoError() { - (>&2 echo "$1") -} - -function sparkError() { - echoError "Changing Spark major version to 2 in the build did not change the state of your working copy, is Spark 1.x still the default ?" - exit 2 -} - -function scalaError() { - echoError "Changing Scala major version to 2.10 in the build did not change the state of your working copy, is Scala 2.11 still the default ?" - exit 2 -} - -function whatchanged() { - cd "$BASEDIR" - for i in $(git status -s --porcelain -- $(find ./ -mindepth 2 -name pom.xml)|awk '{print $2}'); do - echo "$(dirname $i)" - cd "$BASEDIR" - done -} - -set -eu -./change-scala-versions.sh 2.11 -./change-spark-versions.sh 1 # should be idempotent, this is the default -mvn "$@" -./change-spark-versions.sh 2 -if [ -z "$(whatchanged)" ]; then - sparkError; -else - mvn -Dspark.major.version=2 -Dmaven.clean.skip=true -pl $(whatchanged| tr '\n' ',') -amd "$@" -fi -./change-scala-versions.sh 2.10 -./change-spark-versions.sh 1 -if [ -z "$(whatchanged)" ]; then - scalaError; -else - if [[ "${@#-pl}" = "$@" ]]; then - mvn -Dmaven.clean.skip=true -pl $(whatchanged| tr '\n' ',') -amd "$@" - else - # the arguments already tweak the project list ! don't tweak them more - # as this can lead to conflicts (excluding a project that's not part of - # the reactor) - mvn "$@" - fi -fi -./change-scala-versions.sh 2.11 # back to the default -./change-spark-versions.sh 1 diff --git a/datavec/datavec-api/pom.xml b/datavec/datavec-api/pom.xml deleted file mode 100644 index 573c7743a..000000000 --- a/datavec/datavec-api/pom.xml +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - 4.0.0 - - - org.datavec - datavec-parent - 1.0.0-SNAPSHOT - - - datavec-api - - - - org.junit.jupiter - junit-jupiter-api - - - org.junit.vintage - junit-vintage-engine - - - org.apache.commons - commons-lang3 - - - commons-io - commons-io - - - commons-codec - commons-codec - ${commons-codec.version} - - - org.slf4j - slf4j-api - - - joda-time - joda-time - - - - org.nd4j - jackson - ${nd4j.version} - - - org.freemarker - freemarker - ${freemarker.version} - - - - org.nd4j - nd4j-common - - - - org.nd4j - nd4j-api - - - com.clearspring.analytics - stream - ${stream.analytics.version} - - - - net.sf.opencsv - opencsv - ${opencsv.version} - - - com.tdunning - t-digest - ${tdigest.version} - - - it.unimi.dsi - fastutil - ${fastutil.version} - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/conf/Configuration.java b/datavec/datavec-api/src/main/java/org/datavec/api/conf/Configuration.java deleted file mode 100644 index 9aa1cf6d0..000000000 --- a/datavec/datavec-api/src/main/java/org/datavec/api/conf/Configuration.java +++ /dev/null @@ -1,1392 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.conf; - -import org.apache.commons.lang3.StringUtils; -import org.datavec.api.util.ReflectionUtils; -import org.datavec.api.writable.Writable; -import org.datavec.api.writable.WritableType; -import org.nd4j.shade.jackson.core.JsonFactory; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.*; -import org.xml.sax.SAXException; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.*; -import java.net.URL; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -public class Configuration implements Iterable>, Writable, Serializable { - private static final Logger LOG = LoggerFactory.getLogger(Configuration.class); - - private boolean quietmode = true; - - /** - * List of configuration resources. - */ - private ArrayList resources = new ArrayList<>(); - - /** - * List of configuration parameters marked final. - */ - private Set finalParameters = new HashSet<>(); - - private boolean loadDefaults = true; - - /** - * Configuration objects - */ - private static final WeakHashMap REGISTRY = new WeakHashMap<>(); - - /** - * List of default Resources. Resources are loaded in the order of the list - * entries - */ - private static final CopyOnWriteArrayList defaultResources = new CopyOnWriteArrayList<>(); - - private static final ConcurrentMap>> CACHE_CLASSES = new ConcurrentHashMap<>(); - - /** - * Flag to indicate if the storage of resource which updates a key needs - * to be stored for each key - */ - private boolean storeResource; - - /** - * Stores the mapping of key to the resource which modifies or loads - * the key most recently - */ - private HashMap updatingResource; - - static { - //print deprecation warning if hadoop-site.xml is found in classpath - ClassLoader cL = Thread.currentThread().getContextClassLoader(); - if (cL == null) { - cL = Configuration.class.getClassLoader(); - } - if (cL.getResource("hadoop-site.xml") != null) { - LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " - + "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, " - + "mapred-site.xml and hdfs-site.xml to override properties of " - + "core-default.xml, mapred-default.xml and hdfs-default.xml " + "respectively"); - } - addDefaultResource("core-default.xml"); - addDefaultResource("core-site.xml"); - } - - private Properties properties; - private Properties overlay; - private transient ClassLoader classLoader; - { - classLoader = Thread.currentThread().getContextClassLoader(); - if (classLoader == null) { - classLoader = Configuration.class.getClassLoader(); - } - } - - - - /** A new configuration. */ - public Configuration() { - this(true); - } - - /** A new configuration where the behavior of reading from the default - * resources can be turned off. - * - * If the parameter {@code loadDefaults} is false, the new instance - * will not load resources from the default files. - * @param loadDefaults specifies whether to load from the default files - */ - public Configuration(boolean loadDefaults) { - this.loadDefaults = loadDefaults; - synchronized (Configuration.class) { - REGISTRY.put(this, null); - } - this.storeResource = false; - } - - /** - * A new configuration with the same settings and additional facility for - * storage of resource to each key which loads or updates - * the key most recently - * @param other the configuration from which to clone settings - * @param storeResource flag to indicate if the storage of resource to - * each key is to be stored - */ - private Configuration(Configuration other, boolean storeResource) { - this(other); - this.loadDefaults = other.loadDefaults; - this.storeResource = storeResource; - if (storeResource) { - updatingResource = new HashMap<>(); - } - } - - /** - * A new configuration with the same settings cloned from another. - * - * @param other the configuration from which to clone settings. - */ - @SuppressWarnings("unchecked") - public Configuration(Configuration other) { - this.resources = (ArrayList) other.resources.clone(); - synchronized (other) { - if (other.properties != null) { - this.properties = (Properties) other.properties.clone(); - } - - if (other.overlay != null) { - this.overlay = (Properties) other.overlay.clone(); - } - } - - this.finalParameters = new HashSet<>(other.finalParameters); - synchronized (Configuration.class) { - REGISTRY.put(this, null); - } - } - - /** - * Add a default resource. Resources are loaded in the order of the resources - * added. - * @param name file name. File should be present in the classpath. - */ - public static void addDefaultResource(String name) { - // The lock hierarchy is that we must always lock - // instances before locking the class. Since reloadConfiguration - // is synchronized on the instance, we must not call conf.reloadConfiguration - // while holding a lock on Configuration.class. Otherwise we could deadlock - // if that conf is attempting to lock the Class - ArrayList toReload; - synchronized (Configuration.class) { - if (defaultResources.contains(name)) { - return; - } - defaultResources.add(name); - // Make a copy so we don't iterate while not holding the lock - toReload = new ArrayList<>(REGISTRY.size()); - toReload.addAll(REGISTRY.keySet()); - } - for (Configuration conf : toReload) { - if (conf.loadDefaults) { - conf.reloadConfiguration(); - } - } - } - - /** - * Add a configuration resource. - * - * The properties of this resource will override properties of previously - * added resources, unless they were marked final. - * - * @param name resource to be added, the classpath is examined for a file - * with that name. - */ - public void addResource(String name) { - addResourceObject(name); - } - - /** - * Add a configuration resource. - * - * The properties of this resource will override properties of previously - * added resources, unless they were marked final. - * - * @param url url of the resource to be added, the local filesystem is - * examined directly to find the resource, without referring to - * the classpath. - */ - public void addResource(URL url) { - addResourceObject(url); - } - - - /** - * Add a configuration resource. - * - * The properties of this resource will override properties of previously - * added resources, unless they were marked final. - * - * @param in InputStream to deserialize the object from. - */ - public void addResource(InputStream in) { - addResourceObject(in); - } - - - /** - * Reload configuration from previously added resources. - * - * This method will clear all the configuration read from the added - * resources, and final parameters. This will make the resources to - * be read again before accessing the values. Values that are added - * via set methods will overlay values read from the resources. - */ - public synchronized void reloadConfiguration() { - properties = null; // trigger reload - finalParameters.clear(); // clear site-limits - } - - private synchronized void addResourceObject(Object resource) { - resources.add(resource); // add to resources - reloadConfiguration(); - } - - private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); - - private String substituteVars(String expr) { - if (expr == null) { - return null; - } - Matcher match = varPat.matcher(""); - String eval = expr; - int MAX_SUBST = 20; - for (int s = 0; s < MAX_SUBST; s++) { - match.reset(eval); - if (!match.find()) { - return eval; - } - String var = match.group(); - var = var.substring(2, var.length() - 1); // remove ${ .. } - String val = null; - try { - val = System.getProperty(var); - } catch (SecurityException se) { - LOG.warn("Unexpected SecurityException in Configuration", se); - } - if (val == null) { - val = getRaw(var); - } - if (val == null) { - return eval; // return literal ${var}: var is unbound - } - // substitute - eval = eval.substring(0, match.start()) + val + eval.substring(match.end()); - } - throw new IllegalStateException("Variable substitution depth too large: " + MAX_SUBST + " " + expr); - } - - /** - * Get the value of the name property, null if - * no such property exists. - * - * Values are processed for variable expansion - * before being returned. - * - * @param name the property name. - * @return the value of the name property, - * or null if no such property exists. - */ - public String get(String name) { - return substituteVars(getProps().getProperty(name)); - } - - /** - * Get the value of the name property, without doing - * variable expansion. - * - * @param name the property name. - * @return the value of the name property, - * or null if no such property exists. - */ - public String getRaw(String name) { - return getProps().getProperty(name); - } - - /** - * Get the char value of the name property, null if - * no such property exists. - * - * Values are processed for variable expansion - * before being returned. - * - * @param name the property name. - * @return the value of the name property, - * or null if no such property exists. - */ - public char getChar(String name) { - return getProps().getProperty(name).charAt(0); - } - - /** - * Get the char value of the name property, null if - * no such property exists. - * - * Values are processed for variable expansion - * before being returned. - * - * @param name the property name. - * @return the value of the name property, - * or null if no such property exists. - */ - public char getChar(String name, char defaultValue) { - return getProps().getProperty(name, String.valueOf(defaultValue)).charAt(0); - } - - /** - * Set the value of the name property. - * - * @param name property name. - * @param value property value. - */ - public void set(String name, String value) { - getOverlay().setProperty(name, value); - getProps().setProperty(name, value); - } - - /** - * Sets a property if it is currently unset. - * @param name the property name - * @param value the new value - */ - public void setIfUnset(String name, String value) { - if (get(name) == null) { - set(name, value); - } - } - - private synchronized Properties getOverlay() { - if (overlay == null) { - overlay = new Properties(); - } - return overlay; - } - - /** - * Get the value of the name property. If no such property - * exists, then defaultValue is returned. - * - * @param name property name. - * @param defaultValue default value. - * @return property value, or defaultValue if the property - * doesn't exist. - */ - public String get(String name, String defaultValue) { - return substituteVars(getProps().getProperty(name, defaultValue)); - } - - /** - * Get the value of the name property as an int. - * - * If no such property exists, or if the specified value is not a valid - * int, then defaultValue is returned. - * - * @param name property name. - * @param defaultValue default value. - * @return property value as an int, - * or defaultValue. - */ - public int getInt(String name, int defaultValue) { - String valueString = get(name); - if (valueString == null) - return defaultValue; - try { - String hexString = getHexDigits(valueString); - if (hexString != null) { - return Integer.parseInt(hexString, 16); - } - return Integer.parseInt(valueString); - } catch (NumberFormatException e) { - return defaultValue; - } - } - - /** - * Set the value of the name property to an int. - * - * @param name property name. - * @param value int value of the property. - */ - public void setInt(String name, int value) { - set(name, Integer.toString(value)); - } - - - /** - * Get the value of the name property as a long. - * If no such property is specified, or if the specified value is not a valid - * long, then defaultValue is returned. - * - * @param name property name. - * @param defaultValue default value. - * @return property value as a long, - * or defaultValue. - */ - public long getLong(String name, long defaultValue) { - String valueString = get(name); - if (valueString == null) - return defaultValue; - try { - String hexString = getHexDigits(valueString); - if (hexString != null) { - return Long.parseLong(hexString, 16); - } - return Long.parseLong(valueString); - } catch (NumberFormatException e) { - return defaultValue; - } - } - - private String getHexDigits(String value) { - boolean negative = false; - String str = value; - String hexString; - if (value.startsWith("-")) { - negative = true; - str = value.substring(1); - } - if (str.startsWith("0x") || str.startsWith("0X")) { - hexString = str.substring(2); - if (negative) { - hexString = "-" + hexString; - } - return hexString; - } - return null; - } - - /** - * Set the value of the name property to a long. - * - * @param name property name. - * @param value long value of the property. - */ - public void setLong(String name, long value) { - set(name, Long.toString(value)); - } - - /** - * Get the value of the name property as a float. - * If no such property is specified, or if the specified value is not a valid - * float, then defaultValue is returned. - * - * @param name property name. - * @param defaultValue default value. - * @return property value as a float, - * or defaultValue. - */ - public float getFloat(String name, float defaultValue) { - String valueString = get(name); - if (valueString == null) - return defaultValue; - try { - return Float.parseFloat(valueString); - } catch (NumberFormatException e) { - return defaultValue; - } - } - - /** - * Set the value of the name property to a float. - * - * @param name property name. - * @param value property value. - */ - public void setFloat(String name, float value) { - set(name, Float.toString(value)); - } - - /** - * Get the value of the name property as a boolean. - * If no such property is specified, or if the specified value is not a valid - * boolean, then defaultValue is returned. - * - * @param name property name. - * @param defaultValue default value. - * @return property value as a boolean, - * or defaultValue. - */ - public boolean getBoolean(String name, boolean defaultValue) { - String valueString = get(name); - return "true".equals(valueString) || !"false".equals(valueString) && defaultValue; - } - - /** - * Set the value of the name property to a boolean. - * - * @param name property name. - * @param value boolean value of the property. - */ - public void setBoolean(String name, boolean value) { - set(name, Boolean.toString(value)); - } - - /** - * Set the given property, if it is currently unset. - * @param name property name - * @param value new value - */ - public void setBooleanIfUnset(String name, boolean value) { - setIfUnset(name, Boolean.toString(value)); - } - - /** - * Get the value of the name property as a Pattern. - * If no such property is specified, or if the specified value is not a valid - * Pattern, then DefaultValue is returned. - * - * @param name property name - * @param defaultValue default value - * @return property value as a compiled Pattern, or defaultValue - */ - public Pattern getPattern(String name, Pattern defaultValue) { - String valString = get(name); - if (null == valString || "".equals(valString)) { - return defaultValue; - } - try { - return Pattern.compile(valString); - } catch (PatternSyntaxException pse) { - LOG.warn("Regular expression '" + valString + "' for property '" + name + "' not valid. Using default", - pse); - return defaultValue; - } - } - - /** - * Set the given property to Pattern. - * If the pattern is passed as null, sets the empty pattern which results in - * further calls to getPattern(...) returning the default value. - * - * @param name property name - * @param pattern new value - */ - public void setPattern(String name, Pattern pattern) { - if (null == pattern) { - set(name, null); - } else { - set(name, pattern.pattern()); - } - } - - @Override - public void write(DataOutput out) throws IOException { - - } - - @Override - public void readFields(DataInput in) throws IOException { - - } - - /** - * A class that represents a set of positive integer ranges. It parses - * strings of the form: "2-3,5,7-" where ranges are separated by comma and - * the lower/upper bounds are separated by dash. Either the lower or upper - * bound may be omitted meaning all values up to or over. So the string - * above means 2, 3, 5, and 7, 8, 9, ... - */ - public static class IntegerRanges { - private static class Range { - int start; - int end; - } - - List ranges = new ArrayList(); - - public IntegerRanges() {} - - public IntegerRanges(String newValue) { - StringTokenizer itr = new StringTokenizer(newValue, ","); - while (itr.hasMoreTokens()) { - String rng = itr.nextToken().trim(); - String[] parts = rng.split("-", 3); - if (parts.length < 1 || parts.length > 2) { - throw new IllegalArgumentException("integer range badly formed: " + rng); - } - Range r = new Range(); - r.start = convertToInt(parts[0], 0); - if (parts.length == 2) { - r.end = convertToInt(parts[1], Integer.MAX_VALUE); - } else { - r.end = r.start; - } - if (r.start > r.end) { - throw new IllegalArgumentException("IntegerRange from " + r.start + " to " + r.end + " is invalid"); - } - ranges.add(r); - } - } - - /** - * Convert a string to an int treating empty strings as the default value. - * @param value the string value - * @param defaultValue the value for if the string is empty - * @return the desired integer - */ - private static int convertToInt(String value, int defaultValue) { - String trim = value.trim(); - if (trim.length() == 0) { - return defaultValue; - } - return Integer.parseInt(trim); - } - - /** - * Is the given value in the set of ranges - * @param value the value to check - * @return is the value in the ranges? - */ - public boolean isIncluded(int value) { - for (Range r : ranges) { - if (r.start <= value && value <= r.end) { - return true; - } - } - return false; - } - - @Override - public String toString() { - StringBuilder result = new StringBuilder(); - boolean first = true; - for (Range r : ranges) { - if (first) { - first = false; - } else { - result.append(','); - } - result.append(r.start); - result.append('-'); - result.append(r.end); - } - return result.toString(); - } - } - - /** - * Parse the given attribute as a set of integer ranges - * @param name the attribute name - * @param defaultValue the default value if it is not set - * @return a new set of ranges from the configured value - */ - public IntegerRanges getRange(String name, String defaultValue) { - return new IntegerRanges(get(name, defaultValue)); - } - - /** - * Get the comma delimited values of the name property as - * a collection of Strings. - * If no such property is specified then empty collection is returned. - *

- * This is an optimized version of {@link #getStrings(String)} - * - * @param name property name. - * @return property value as a collection of Strings. - */ - public Collection getStringCollection(String name) { - String valueString = get(name); - if(valueString == null) - return null; - return Arrays.asList(StringUtils.split(valueString, ",")); - } - - /** - * Get the comma delimited values of the name property as - * an array of Strings. - * If no such property is specified then null is returned. - * - * @param name property name. - * @return property value as an array of Strings, - * or null. - */ - public String[] getStrings(String name) { - String valueString = get(name); - return StringUtils.split(valueString, ","); - } - - /** - * Get the comma delimited values of the name property as - * an array of Strings. - * If no such property is specified then default value is returned. - * - * @param name property name. - * @param defaultValue The default value - * @return property value as an array of Strings, - * or default value. - */ - public String[] getStrings(String name, String... defaultValue) { - String valueString = get(name); - if (valueString == null) { - return defaultValue; - } else { - return StringUtils.split(valueString, ","); - } - } - - /** - * Get the comma delimited values of the name property as - * a collection of Strings, trimmed of the leading and trailing whitespace. - * If no such property is specified then empty Collection is returned. - * - * @param name property name. - * @return property value as a collection of Strings, or empty Collection - */ - public Collection getTrimmedStringCollection(String name) { - String valueString = get(name); - if (null == valueString) { - return Collections.emptyList(); - } - return Arrays.asList(StringUtils.stripAll(StringUtils.split(valueString, ","))); - } - - /** - * Get the comma delimited values of the name property as - * an array of Strings, trimmed of the leading and trailing whitespace. - * If no such property is specified then an empty array is returned. - * - * @param name property name. - * @return property value as an array of trimmed Strings, - * or empty array. - */ - public String[] getTrimmedStrings(String name) { - String valueString = get(name); - return StringUtils.stripAll(StringUtils.split(valueString, ",")); - } - - /** - * Get the comma delimited values of the name property as - * an array of Strings, trimmed of the leading and trailing whitespace. - * If no such property is specified then default value is returned. - * - * @param name property name. - * @param defaultValue The default value - * @return property value as an array of trimmed Strings, - * or default value. - */ - public String[] getTrimmedStrings(String name, String... defaultValue) { - String valueString = get(name); - if (null == valueString) { - return defaultValue; - } else { - return StringUtils.stripAll(StringUtils.split(valueString, ",")); - } - } - - /** - * Set the array of string values for the name property as - * as comma delimited values. - * - * @param name property name. - * @param values The values - */ - public void setStrings(String name, String... values) { - set(name, StringUtils.join(values, ",")); - } - - /** - * Load a class by name. - * - * @param name the class name. - * @return the class object. - * @throws ClassNotFoundException if the class is not found. - */ - public Class getClassByName(String name) throws ClassNotFoundException { - Map> map = CACHE_CLASSES.get(classLoader); - if (map == null) { - Map> newMap = new ConcurrentHashMap<>(); - map = CACHE_CLASSES.putIfAbsent(classLoader, newMap); - if (map == null) { - map = newMap; - } - } - - Class clazz = map.get(name); - if (clazz == null) { - clazz = Class.forName(name, true, classLoader); - if (clazz != null) { - map.put(name, clazz); - } - } - - return clazz; - } - - /** - * Get the value of the name property - * as an array of Class. - * The value of the property specifies a list of comma separated class names. - * If no such property is specified, then defaultValue is - * returned. - * - * @param name the property name. - * @param defaultValue default value. - * @return property value as a Class[], - * or defaultValue. - */ - public Class[] getClasses(String name, Class... defaultValue) { - String[] classnames = getStrings(name); - if (classnames == null) - return defaultValue; - try { - Class[] classes = new Class[classnames.length]; - for (int i = 0; i < classnames.length; i++) { - classes[i] = getClassByName(classnames[i]); - } - return classes; - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - - /** - * Get the value of the name property as a Class. - * If no such property is specified, then defaultValue is - * returned. - * - * @param name the class name. - * @param defaultValue default value. - * @return property value as a Class, - * or defaultValue. - */ - public Class getClass(String name, Class defaultValue) { - String valueString = get(name); - if (valueString == null) - return defaultValue; - try { - return getClassByName(valueString); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - - /** - * Get the value of the name property as a Class - * implementing the interface specified by xface. - * - * If no such property is specified, then defaultValue is - * returned. - * - * An exception is thrown if the returned class does not implement the named - * interface. - * - * @param name the class name. - * @param defaultValue default value. - * @param xface the interface implemented by the named class. - * @return property value as a Class, - * or defaultValue. - */ - public Class getClass(String name, Class defaultValue, Class xface) { - try { - Class theClass = getClass(name, defaultValue); - if (theClass != null && !xface.isAssignableFrom(theClass)) - throw new RuntimeException(theClass + " not " + xface.getName()); - else if (theClass != null) - return theClass.asSubclass(xface); - else - return null; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Get the value of the name property as a List - * of objects implementing the interface specified by xface. - * - * An exception is thrown if any of the classes does not exist, or if it does - * not implement the named interface. - * - * @param name the property name. - * @param xface the interface implemented by the classes named by - * name. - * @return a List of objects implementing xface. - */ - @SuppressWarnings("unchecked") - public List getInstances(String name, Class xface) { - List ret = new ArrayList<>(); - Class[] classes = getClasses(name); - for (Class cl : classes) { - if (!xface.isAssignableFrom(cl)) { - throw new RuntimeException(cl + " does not implement " + xface); - } - ret.add((U) ReflectionUtils.newInstance(cl, this)); - } - return ret; - } - - /** - * Set the value of the name property to the name of a - * theClass implementing the given interface xface. - * - * An exception is thrown if theClass does not implement the - * interface xface. - * - * @param name property name. - * @param theClass property value. - * @param xface the interface implemented by the named class. - */ - public void setClass(String name, Class theClass, Class xface) { - if (!xface.isAssignableFrom(theClass)) - throw new RuntimeException(theClass + " not " + xface.getName()); - set(name, theClass.getName()); - } - - - - /** - * Get a local file name under a directory named in dirsProp with - * the given path. If dirsProp contains multiple directories, - * then one is chosen based on path's hash code. If the selected - * directory does not exist, an attempt is made to create it. - * - * @param dirsProp directory in which to locate the file. - * @param path file-path. - * @return local file under the directory with the given path. - */ - public File getFile(String dirsProp, String path) throws IOException { - String[] dirs = getStrings(dirsProp); - int hashCode = path.hashCode(); - for (int i = 0; i < dirs.length; i++) { // try each local dir - int index = (hashCode + i & Integer.MAX_VALUE) % dirs.length; - File file = new File(dirs[index], path); - File dir = file.getParentFile(); - if (dir.exists() || dir.mkdirs()) { - return file; - } - } - throw new IOException("No valid local directories in property: " + dirsProp); - } - - /** - * Get the {@link URL} for the named resource. - * - * @param name resource name. - * @return the url for the named resource. - */ - public URL getResource(String name) { - return classLoader.getResource(name); - } - - /** - * Get an input stream attached to the configuration resource with the - * given name. - * - * @param name configuration resource name. - * @return an input stream attached to the resource. - */ - public InputStream getConfResourceAsInputStream(String name) { - try { - URL url = getResource(name); - - if (url == null) { - LOG.info(name + " not found"); - return null; - } else { - LOG.info("found resource " + name + " at " + url); - } - - return url.openStream(); - } catch (Exception e) { - return null; - } - } - - /** - * Get a {@link Reader} attached to the configuration resource with the - * given name. - * - * @param name configuration resource name. - * @return a reader attached to the resource. - */ - public Reader getConfResourceAsReader(String name) { - try { - URL url = getResource(name); - - if (url == null) { - LOG.info(name + " not found"); - return null; - } else { - LOG.info("found resource " + name + " at " + url); - } - - return new InputStreamReader(url.openStream()); - } catch (Exception e) { - return null; - } - } - - private synchronized Properties getProps() { - if (properties == null) { - properties = new Properties(); - loadResources(properties, resources, quietmode); - if (overlay != null) { - properties.putAll(overlay); - if (storeResource) { - for (Map.Entry item : overlay.entrySet()) { - updatingResource.put((String) item.getKey(), "Unknown"); - } - } - } - } - return properties; - } - - /** - * Return the number of keys in the configuration. - * - * @return number of keys in the configuration. - */ - public int size() { - return getProps().size(); - } - - /** - * Clears all keys from the configuration. - */ - public void clear() { - getProps().clear(); - getOverlay().clear(); - } - - /** - * Get an {@link Iterator} to go through the list of String - * key-value pairs in the configuration. - * - * @return an iterator over the entries. - */ - public Iterator> iterator() { - // Get a copy of just the string to string pairs. After the old object - // methods that allow non-strings to be put into configurations are removed, - // we could replace properties with a Map and get rid of this - // code. - Map result = new HashMap<>(); - for (Map.Entry item : getProps().entrySet()) { - if (item.getKey() instanceof String && item.getValue() instanceof String) { - result.put((String) item.getKey(), (String) item.getValue()); - } - } - return result.entrySet().iterator(); - } - - private void loadResources(Properties properties, ArrayList resources, boolean quiet) { - if (loadDefaults) { - // To avoid addResource causing a ConcurrentModificationException - ArrayList toLoad; - synchronized (Configuration.class) { - toLoad = new ArrayList<>(defaultResources); - } - for (String resource : toLoad) { - loadResource(properties, resource, quiet); - } - - //support the hadoop-site.xml as a deprecated case - if (getResource("hadoop-site.xml") != null) { - loadResource(properties, "hadoop-site.xml", quiet); - } - } - - for (Object resource : resources) { - loadResource(properties, resource, quiet); - } - } - - private void loadResource(Properties properties, Object name, boolean quiet) { - try { - DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); - //ignore all comments inside the xml file - docBuilderFactory.setIgnoringComments(true); - - //allow includes in the xml file - docBuilderFactory.setNamespaceAware(true); - try { - docBuilderFactory.setXIncludeAware(true); - } catch (UnsupportedOperationException e) { - LOG.error("Failed to set setXIncludeAware(true) for parser " + docBuilderFactory + ":" + e, e); - } - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = null; - Element root = null; - - if (name instanceof URL) { // an URL resource - URL url = (URL) name; - if (url != null) { - if (!quiet) { - LOG.info("parsing " + url); - } - doc = builder.parse(url.toString()); - } - } else if (name instanceof String) { // a CLASSPATH resource - URL url = getResource((String) name); - if (url != null) { - if (!quiet) { - LOG.info("parsing " + url); - } - doc = builder.parse(url.toString()); - } - } else if (name instanceof InputStream) { - try { - doc = builder.parse((InputStream) name); - } finally { - ((InputStream) name).close(); - } - } else if (name instanceof Element) { - root = (Element) name; - } - - if (doc == null && root == null) { - if (quiet) - return; - throw new RuntimeException(name + " not found"); - } - - if (root == null) { - root = doc.getDocumentElement(); - } - if (!"configuration".equals(root.getTagName())) - LOG.error("bad conf file: top-level element not "); - NodeList props = root.getChildNodes(); - for (int i = 0; i < props.getLength(); i++) { - Node propNode = props.item(i); - if (!(propNode instanceof Element)) - continue; - Element prop = (Element) propNode; - if ("configuration".equals(prop.getTagName())) { - loadResource(properties, prop, quiet); - continue; - } - if (!"property".equals(prop.getTagName())) - LOG.warn("bad conf file: element not "); - NodeList fields = prop.getChildNodes(); - String attr = null; - String value = null; - boolean finalParameter = false; - for (int j = 0; j < fields.getLength(); j++) { - Node fieldNode = fields.item(j); - if (!(fieldNode instanceof Element)) - continue; - Element field = (Element) fieldNode; - if ("name".equals(field.getTagName()) && field.hasChildNodes()) - attr = ((Text) field.getFirstChild()).getData().trim(); - if ("value".equals(field.getTagName()) && field.hasChildNodes()) - value = ((Text) field.getFirstChild()).getData(); - if ("final".equals(field.getTagName()) && field.hasChildNodes()) - finalParameter = "true".equals(((Text) field.getFirstChild()).getData()); - } - - // Ignore this parameter if it has already been marked as 'final' - if (attr != null && value != null) { - if (!finalParameters.contains(attr)) { - properties.setProperty(attr, value); - if (storeResource) { - updatingResource.put(attr, name.toString()); - } - if (finalParameter) - finalParameters.add(attr); - } else { - LOG.warn(name + ":a attempt to override final parameter: " + attr + "; Ignoring."); - } - } - } - - } catch (IOException | ParserConfigurationException | SAXException | DOMException e) { - LOG.error("error parsing conf file: " + e); - throw new RuntimeException(e); - } - } - - /** - * Write out the non-default properties in this configuration to the give - * {@link OutputStream}. - * - * @param out the output stream to write to. - */ - public void writeXml(OutputStream out) throws IOException { - Properties properties = getProps(); - try { - Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); - Element conf = doc.createElement("configuration"); - doc.appendChild(conf); - conf.appendChild(doc.createTextNode("\n")); - for (Enumeration e = properties.keys(); e.hasMoreElements();) { - String name = (String) e.nextElement(); - Object object = properties.get(name); - String value; - if (object instanceof String) { - value = (String) object; - } else { - continue; - } - Element propNode = doc.createElement("property"); - conf.appendChild(propNode); - - Element nameNode = doc.createElement("name"); - nameNode.appendChild(doc.createTextNode(name)); - propNode.appendChild(nameNode); - - Element valueNode = doc.createElement("value"); - valueNode.appendChild(doc.createTextNode(value)); - propNode.appendChild(valueNode); - - conf.appendChild(doc.createTextNode("\n")); - } - - DOMSource source = new DOMSource(doc); - StreamResult result = new StreamResult(out); - TransformerFactory transFactory = TransformerFactory.newInstance(); - Transformer transformer = transFactory.newTransformer(); - transformer.transform(source, result); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Writes out all the parameters and their properties (final and resource) to - * the given {@link Writer} - * The format of the output would be - * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2, - * key2.isFinal,key2.resource}... ] } - * It does not output the parameters of the configuration object which is - * loaded from an input stream. - * @param out the Writer to write to - * @throws IOException - */ - public static void dumpConfiguration(Configuration conf, Writer out) throws IOException { - Configuration config = new Configuration(conf, true); - config.reloadConfiguration(); - JsonFactory dumpFactory = new JsonFactory(); - JsonGenerator dumpGenerator = dumpFactory.createGenerator(out); - dumpGenerator.writeStartObject(); - dumpGenerator.writeFieldName("properties"); - dumpGenerator.writeStartArray(); - dumpGenerator.flush(); - for (Map.Entry item : config.getProps().entrySet()) { - dumpGenerator.writeStartObject(); - dumpGenerator.writeStringField("key", (String) item.getKey()); - dumpGenerator.writeStringField("value", config.get((String) item.getKey())); - dumpGenerator.writeBooleanField("isFinal", config.finalParameters.contains(item.getKey())); - dumpGenerator.writeStringField("resource", config.updatingResource.get(item.getKey())); - dumpGenerator.writeEndObject(); - } - dumpGenerator.writeEndArray(); - dumpGenerator.writeEndObject(); - dumpGenerator.flush(); - } - - /** - * Get the {@link ClassLoader} for this job. - * - * @return the correct class loader. - */ - public ClassLoader getClassLoader() { - return classLoader; - } - - /** - * Set the class loader that will be used to load the various objects. - * - * @param classLoader the new class loader. - */ - public void setClassLoader(ClassLoader classLoader) { - this.classLoader = classLoader; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("Configuration: "); - if (loadDefaults) { - synchronized (Configuration.class) { - toString(defaultResources, sb); - } - if (resources.size() > 0) { - sb.append(", "); - } - } - toString(resources, sb); - return sb.toString(); - } - - private void toString(List resources, StringBuilder sb) { - ListIterator i = resources.listIterator(); - while (i.hasNext()) { - if (i.nextIndex() != 0) { - sb.append(", "); - } - sb.append(i.next()); - } - } - - /** - * Set the quietness-mode. - * - * In the quiet-mode, error and informational messages might not be logged. - * - * @param quietmode true to set quiet-mode on, false - * to turn it off. - */ - public synchronized void setQuietMode(boolean quietmode) { - this.quietmode = quietmode; - } - - /** For debugging. List non-default properties to the terminal and exit. */ - public static void main(String[] args) throws Exception { - new Configuration().writeXml(System.out); - } - - - @Override - public double toDouble() { - throw new UnsupportedOperationException(); - } - - @Override - public float toFloat() { - throw new UnsupportedOperationException(); - } - - @Override - public int toInt() { - throw new UnsupportedOperationException(); - } - - @Override - public long toLong() { - throw new UnsupportedOperationException(); - } - - @Override - public WritableType getType() { - throw new UnsupportedOperationException(); - } - - @Override - public void writeType(DataOutput out) throws IOException { - throw new UnsupportedOperationException(); - } -} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/io/WritableComparator.java b/datavec/datavec-api/src/main/java/org/datavec/api/io/WritableComparator.java deleted file mode 100644 index dca11278e..000000000 --- a/datavec/datavec-api/src/main/java/org/datavec/api/io/WritableComparator.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.io; - - -import org.datavec.api.util.ReflectionUtils; -import org.datavec.api.writable.Writable; - -import java.io.DataInput; -import java.io.IOException; -import java.util.HashMap; - - -public class WritableComparator implements RawComparator { - - private static HashMap comparators = new HashMap<>(); // registry - - /** Get a comparator for a {@link WritableComparable} implementation. */ - public static synchronized WritableComparator get(Class c) { - WritableComparator comparator = comparators.get(c); - if (comparator == null) { - // force the static initializers to run - forceInit(c); - // look to see if it is defined now - comparator = comparators.get(c); - // if not, use the generic one - if (comparator == null) { - comparator = new WritableComparator(c, true); - comparators.put(c, comparator); - } - } - return comparator; - } - - /** - * Force initialization of the static members. - * As of Java 5, referencing a class doesn't force it to initialize. Since - * this class requires that the classes be initialized to declare their - * comparators, we force that initialization to happen. - * @param cls the class to initialize - */ - private static void forceInit(Class cls) { - try { - Class.forName(cls.getName(), true, cls.getClassLoader()); - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("Can't initialize class " + cls, e); - } - } - - /** Register an optimized comparator for a {@link WritableComparable} - * implementation. */ - public static synchronized void define(Class c, WritableComparator comparator) { - comparators.put(c, comparator); - } - - - private final Class keyClass; - private final WritableComparable key1; - private final WritableComparable key2; - private final DataInputBuffer buffer; - - /** Construct for a {@link WritableComparable} implementation. */ - protected WritableComparator(Class keyClass) { - this(keyClass, false); - } - - protected WritableComparator(Class keyClass, boolean createInstances) { - this.keyClass = keyClass; - if (createInstances) { - key1 = newKey(); - key2 = newKey(); - buffer = new DataInputBuffer(); - } else { - key1 = key2 = null; - buffer = null; - } - } - - /** Returns the WritableComparable implementation class. */ - public Class getKeyClass() { - return keyClass; - } - - /** Construct a new {@link WritableComparable} instance. */ - public WritableComparable newKey() { - return ReflectionUtils.newInstance(keyClass, null); - } - - /** Optimization hook. Override this to make SequenceFile.Sorter's scream. - * - *

The default implementation reads the data into two {@link - * WritableComparable}s (using {@link - * Writable#readFields(DataInput)}, then calls {@link - * #compare(WritableComparable,WritableComparable)}. - */ - public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { - try { - buffer.reset(b1, s1, l1); // parse key1 - key1.readFields(buffer); - - buffer.reset(b2, s2, l2); // parse key2 - key2.readFields(buffer); - - } catch (IOException e) { - throw new RuntimeException(e); - } - - return compare(key1, key2); // compare them - } - - /** Compare two WritableComparables. - * - *

The default implementation uses the natural ordering, calling {@link - * Comparable#compareTo(Object)}. */ - @SuppressWarnings("unchecked") - public int compare(WritableComparable a, WritableComparable b) { - return a.compareTo(b); - } - - public int compare(Object a, Object b) { - return compare((WritableComparable) a, (WritableComparable) b); - } - - /** Lexicographic order of binary data. */ - public static int compareBytes(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { - int end1 = s1 + l1; - int end2 = s2 + l2; - for (int i = s1, j = s2; i < end1 && j < end2; i++, j++) { - int a = (b1[i] & 0xff); - int b = (b2[j] & 0xff); - if (a != b) { - return a - b; - } - } - return l1 - l2; - } - - /** Compute hash for binary data. */ - public static int hashBytes(byte[] bytes, int offset, int length) { - int hash = 1; - for (int i = offset; i < offset + length; i++) - hash = (31 * hash) + (int) bytes[i]; - return hash; - } - - /** Compute hash for binary data. */ - public static int hashBytes(byte[] bytes, int length) { - return hashBytes(bytes, 0, length); - } - - /** Parse an unsigned short from a byte array. */ - public static int readUnsignedShort(byte[] bytes, int start) { - return (((bytes[start] & 0xff) << 8) + ((bytes[start + 1] & 0xff))); - } - - /** Parse an integer from a byte array. */ - public static int readInt(byte[] bytes, int start) { - return (((bytes[start] & 0xff) << 24) + ((bytes[start + 1] & 0xff) << 16) + ((bytes[start + 2] & 0xff) << 8) - + ((bytes[start + 3] & 0xff))); - - } - - /** Parse a float from a byte array. */ - public static float readFloat(byte[] bytes, int start) { - return Float.intBitsToFloat(readInt(bytes, start)); - } - - /** Parse a long from a byte array. */ - public static long readLong(byte[] bytes, int start) { - return ((long) (readInt(bytes, start)) << 32) + (readInt(bytes, start + 4) & 0xFFFFFFFFL); - } - - /** Parse a double from a byte array. */ - public static double readDouble(byte[] bytes, int start) { - return Double.longBitsToDouble(readLong(bytes, start)); - } - - /** - * Reads a zero-compressed encoded long from a byte array and returns it. - * @param bytes byte array with decode long - * @param start starting index - * @throws java.io.IOException - * @return deserialized long - */ - public static long readVLong(byte[] bytes, int start) throws IOException { - int len = bytes[start]; - if (len >= -112) { - return len; - } - boolean isNegative = (len < -120); - len = isNegative ? -(len + 120) : -(len + 112); - if (start + 1 + len > bytes.length) - throw new IOException("Not enough number of bytes for a zero-compressed integer"); - long i = 0; - for (int idx = 0; idx < len; idx++) { - i = i << 8; - i = i | (bytes[start + 1 + idx] & 0xFF); - } - return (isNegative ? (~i) : i); - } - - /** - * Reads a zero-compressed encoded integer from a byte array and returns it. - * @param bytes byte array with the encoded integer - * @param start start index - * @throws java.io.IOException - * @return deserialized integer - */ - public static int readVInt(byte[] bytes, int start) throws IOException { - return (int) readVLong(bytes, start); - } -} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/Operation.java b/datavec/datavec-api/src/main/java/org/datavec/api/transform/Operation.java deleted file mode 100644 index b5a4f2ff2..000000000 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/Operation.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.transform; - -public interface Operation { - TOut transform(TIn input); -} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/WindowFunction.java b/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/WindowFunction.java deleted file mode 100644 index d28860241..000000000 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/sequence/window/WindowFunction.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.transform.sequence.window; - -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; - -import java.io.Serializable; -import java.util.List; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") -public interface WindowFunction extends Serializable { - - /** - * Apply the windowing function to the given sequence - * @param sequence the input sequence - * @return the sequence with the window function applied - */ - List>> applyToSequence(List> sequence); - - /** - * - * @param schema - */ - void setInputSchema(Schema schema); - - /** - * - * @return - */ - Schema getInputSchema(); - - /** Get the output schema, given the input schema. Typically the output schema is the same as the input schema, - * but not necessarily (for example, if the window function adds columns for the window start/end times) - * @param inputSchema Schema of the input data - * @return Schema of the output windows - */ - Schema transform(Schema inputSchema); - - -} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/JsonMappers.java b/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/JsonMappers.java deleted file mode 100644 index 3f1ae442a..000000000 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/JsonMappers.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.transform.serde; - -import lombok.extern.slf4j.Slf4j; -import org.datavec.api.transform.serde.legacy.LegacyJsonFormat; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.annotation.PropertyAccessor; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; -import org.nd4j.shade.jackson.datatype.joda.JodaModule; - -@Slf4j -public class JsonMappers { - - private static ObjectMapper jsonMapper; - private static ObjectMapper yamlMapper; - private static ObjectMapper legacyMapper; //For 1.0.0-alpha and earlier TransformProcess etc - - static { - jsonMapper = new ObjectMapper(); - yamlMapper = new ObjectMapper(new YAMLFactory()); - configureMapper(jsonMapper); - configureMapper(yamlMapper); - } - - public static synchronized ObjectMapper getLegacyMapper(){ - if(legacyMapper == null){ - legacyMapper = LegacyJsonFormat.legacyMapper(); - configureMapper(legacyMapper); - } - return legacyMapper; - } - - /** - * @return The default/primary ObjectMapper for deserializing JSON network configurations in DL4J - */ - public static ObjectMapper getMapper(){ - return jsonMapper; - } - - /** - * @return The default/primary ObjectMapper for deserializing network configurations in DL4J (YAML format) - */ - public static ObjectMapper getMapperYaml() { - return yamlMapper; - } - - private static void configureMapper(ObjectMapper ret) { - ret.registerModule(new JodaModule()); - ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); - ret.enable(SerializationFeature.INDENT_OUTPUT); - ret.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); - ret.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); - ret.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY); //Need this otherwise JsonProperty annotations on constructors won't be seen - } - -} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/JsonSerializer.java b/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/JsonSerializer.java deleted file mode 100644 index 2a2475f23..000000000 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/JsonSerializer.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.transform.serde; - -import org.nd4j.shade.jackson.databind.ObjectMapper; - -public class JsonSerializer extends BaseSerializer { - - private ObjectMapper om; - - public JsonSerializer() { - this.om = JsonMappers.getMapper(); - } - - @Override - public ObjectMapper getObjectMapper() { - return om; - } -} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/legacy/LegacyJsonFormat.java b/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/legacy/LegacyJsonFormat.java deleted file mode 100644 index 85f50b631..000000000 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/serde/legacy/LegacyJsonFormat.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.transform.serde.legacy; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import org.datavec.api.transform.Transform; -import org.datavec.api.transform.analysis.columns.*; -import org.datavec.api.transform.condition.BooleanCondition; -import org.datavec.api.transform.condition.Condition; -import org.datavec.api.transform.condition.column.*; -import org.datavec.api.transform.condition.sequence.SequenceLengthCondition; -import org.datavec.api.transform.condition.string.StringRegexColumnCondition; -import org.datavec.api.transform.filter.ConditionFilter; -import org.datavec.api.transform.filter.Filter; -import org.datavec.api.transform.filter.FilterInvalidValues; -import org.datavec.api.transform.filter.InvalidNumColumns; -import org.datavec.api.transform.metadata.*; -import org.datavec.api.transform.ndarray.NDArrayColumnsMathOpTransform; -import org.datavec.api.transform.ndarray.NDArrayDistanceTransform; -import org.datavec.api.transform.ndarray.NDArrayMathFunctionTransform; -import org.datavec.api.transform.ndarray.NDArrayScalarOpTransform; -import org.datavec.api.transform.rank.CalculateSortedRank; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.transform.schema.SequenceSchema; -import org.datavec.api.transform.sequence.ReduceSequenceTransform; -import org.datavec.api.transform.sequence.SequenceComparator; -import org.datavec.api.transform.sequence.SequenceSplit; -import org.datavec.api.transform.sequence.comparator.NumericalColumnComparator; -import org.datavec.api.transform.sequence.comparator.StringComparator; -import org.datavec.api.transform.sequence.split.SequenceSplitTimeSeparation; -import org.datavec.api.transform.sequence.split.SplitMaxLengthSequence; -import org.datavec.api.transform.sequence.trim.SequenceTrimTransform; -import org.datavec.api.transform.sequence.window.OverlappingTimeWindowFunction; -import org.datavec.api.transform.sequence.window.ReduceSequenceByWindowTransform; -import org.datavec.api.transform.sequence.window.TimeWindowFunction; -import org.datavec.api.transform.sequence.window.WindowFunction; -import org.datavec.api.transform.stringreduce.IStringReducer; -import org.datavec.api.transform.stringreduce.StringReducer; -import org.datavec.api.transform.transform.categorical.*; -import org.datavec.api.transform.transform.column.*; -import org.datavec.api.transform.transform.condition.ConditionalCopyValueTransform; -import org.datavec.api.transform.transform.condition.ConditionalReplaceValueTransform; -import org.datavec.api.transform.transform.condition.ConditionalReplaceValueTransformWithDefault; -import org.datavec.api.transform.transform.doubletransform.*; -import org.datavec.api.transform.transform.integer.*; -import org.datavec.api.transform.transform.longtransform.LongColumnsMathOpTransform; -import org.datavec.api.transform.transform.longtransform.LongMathOpTransform; -import org.datavec.api.transform.transform.nlp.TextToCharacterIndexTransform; -import org.datavec.api.transform.transform.parse.ParseDoubleTransform; -import org.datavec.api.transform.transform.sequence.SequenceDifferenceTransform; -import org.datavec.api.transform.transform.sequence.SequenceMovingWindowReduceTransform; -import org.datavec.api.transform.transform.sequence.SequenceOffsetTransform; -import org.datavec.api.transform.transform.string.*; -import org.datavec.api.transform.transform.time.DeriveColumnsFromTimeTransform; -import org.datavec.api.transform.transform.time.StringToTimeTransform; -import org.datavec.api.transform.transform.time.TimeMathOpTransform; -import org.datavec.api.writable.*; -import org.datavec.api.writable.comparator.*; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.databind.ObjectMapper; - -public class LegacyJsonFormat { - - private LegacyJsonFormat(){ } - - /** - * Get a mapper (minus general config) suitable for loading old format JSON - 1.0.0-alpha and before - * @return Object mapper - */ - public static ObjectMapper legacyMapper(){ - ObjectMapper om = new ObjectMapper(); - om.addMixIn(Schema.class, SchemaMixin.class); - om.addMixIn(ColumnMetaData.class, ColumnMetaDataMixin.class); - om.addMixIn(Transform.class, TransformMixin.class); - om.addMixIn(Condition.class, ConditionMixin.class); - om.addMixIn(Writable.class, WritableMixin.class); - om.addMixIn(Filter.class, FilterMixin.class); - om.addMixIn(SequenceComparator.class, SequenceComparatorMixin.class); - om.addMixIn(SequenceSplit.class, SequenceSplitMixin.class); - om.addMixIn(WindowFunction.class, WindowFunctionMixin.class); - om.addMixIn(CalculateSortedRank.class, CalculateSortedRankMixin.class); - om.addMixIn(WritableComparator.class, WritableComparatorMixin.class); - om.addMixIn(ColumnAnalysis.class, ColumnAnalysisMixin.class); - om.addMixIn(IStringReducer.class, IStringReducerMixin.class); - return om; - } - - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes({@JsonSubTypes.Type(value = Schema.class, name = "Schema"), - @JsonSubTypes.Type(value = SequenceSchema.class, name = "SequenceSchema")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class SchemaMixin { } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes({@JsonSubTypes.Type(value = BinaryMetaData.class, name = "Binary"), - @JsonSubTypes.Type(value = BooleanMetaData.class, name = "Boloean"), - @JsonSubTypes.Type(value = CategoricalMetaData.class, name = "Categorical"), - @JsonSubTypes.Type(value = DoubleMetaData.class, name = "Double"), - @JsonSubTypes.Type(value = FloatMetaData.class, name = "Float"), - @JsonSubTypes.Type(value = IntegerMetaData.class, name = "Integer"), - @JsonSubTypes.Type(value = LongMetaData.class, name = "Long"), - @JsonSubTypes.Type(value = NDArrayMetaData.class, name = "NDArray"), - @JsonSubTypes.Type(value = StringMetaData.class, name = "String"), - @JsonSubTypes.Type(value = TimeMetaData.class, name = "Time")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class ColumnMetaDataMixin { } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = CalculateSortedRank.class, name = "CalculateSortedRank"), - @JsonSubTypes.Type(value = CategoricalToIntegerTransform.class, name = "CategoricalToIntegerTransform"), - @JsonSubTypes.Type(value = CategoricalToOneHotTransform.class, name = "CategoricalToOneHotTransform"), - @JsonSubTypes.Type(value = IntegerToCategoricalTransform.class, name = "IntegerToCategoricalTransform"), - @JsonSubTypes.Type(value = StringToCategoricalTransform.class, name = "StringToCategoricalTransform"), - @JsonSubTypes.Type(value = DuplicateColumnsTransform.class, name = "DuplicateColumnsTransform"), - @JsonSubTypes.Type(value = RemoveColumnsTransform.class, name = "RemoveColumnsTransform"), - @JsonSubTypes.Type(value = RenameColumnsTransform.class, name = "RenameColumnsTransform"), - @JsonSubTypes.Type(value = ReorderColumnsTransform.class, name = "ReorderColumnsTransform"), - @JsonSubTypes.Type(value = ConditionalCopyValueTransform.class, name = "ConditionalCopyValueTransform"), - @JsonSubTypes.Type(value = ConditionalReplaceValueTransform.class, name = "ConditionalReplaceValueTransform"), - @JsonSubTypes.Type(value = ConditionalReplaceValueTransformWithDefault.class, name = "ConditionalReplaceValueTransformWithDefault"), - @JsonSubTypes.Type(value = DoubleColumnsMathOpTransform.class, name = "DoubleColumnsMathOpTransform"), - @JsonSubTypes.Type(value = DoubleMathOpTransform.class, name = "DoubleMathOpTransform"), - @JsonSubTypes.Type(value = Log2Normalizer.class, name = "Log2Normalizer"), - @JsonSubTypes.Type(value = MinMaxNormalizer.class, name = "MinMaxNormalizer"), - @JsonSubTypes.Type(value = StandardizeNormalizer.class, name = "StandardizeNormalizer"), - @JsonSubTypes.Type(value = SubtractMeanNormalizer.class, name = "SubtractMeanNormalizer"), - @JsonSubTypes.Type(value = IntegerColumnsMathOpTransform.class, name = "IntegerColumnsMathOpTransform"), - @JsonSubTypes.Type(value = IntegerMathOpTransform.class, name = "IntegerMathOpTransform"), - @JsonSubTypes.Type(value = ReplaceEmptyIntegerWithValueTransform.class, name = "ReplaceEmptyIntegerWithValueTransform"), - @JsonSubTypes.Type(value = ReplaceInvalidWithIntegerTransform.class, name = "ReplaceInvalidWithIntegerTransform"), - @JsonSubTypes.Type(value = LongColumnsMathOpTransform.class, name = "LongColumnsMathOpTransform"), - @JsonSubTypes.Type(value = LongMathOpTransform.class, name = "LongMathOpTransform"), - @JsonSubTypes.Type(value = MapAllStringsExceptListTransform.class, name = "MapAllStringsExceptListTransform"), - @JsonSubTypes.Type(value = RemoveWhiteSpaceTransform.class, name = "RemoveWhiteSpaceTransform"), - @JsonSubTypes.Type(value = ReplaceEmptyStringTransform.class, name = "ReplaceEmptyStringTransform"), - @JsonSubTypes.Type(value = ReplaceStringTransform.class, name = "ReplaceStringTransform"), - @JsonSubTypes.Type(value = StringListToCategoricalSetTransform.class, name = "StringListToCategoricalSetTransform"), - @JsonSubTypes.Type(value = StringMapTransform.class, name = "StringMapTransform"), - @JsonSubTypes.Type(value = DeriveColumnsFromTimeTransform.class, name = "DeriveColumnsFromTimeTransform"), - @JsonSubTypes.Type(value = StringToTimeTransform.class, name = "StringToTimeTransform"), - @JsonSubTypes.Type(value = TimeMathOpTransform.class, name = "TimeMathOpTransform"), - @JsonSubTypes.Type(value = ReduceSequenceByWindowTransform.class, name = "ReduceSequenceByWindowTransform"), - @JsonSubTypes.Type(value = DoubleMathFunctionTransform.class, name = "DoubleMathFunctionTransform"), - @JsonSubTypes.Type(value = AddConstantColumnTransform.class, name = "AddConstantColumnTransform"), - @JsonSubTypes.Type(value = RemoveAllColumnsExceptForTransform.class, name = "RemoveAllColumnsExceptForTransform"), - @JsonSubTypes.Type(value = ParseDoubleTransform.class, name = "ParseDoubleTransform"), - @JsonSubTypes.Type(value = ConvertToString.class, name = "ConvertToStringTransform"), - @JsonSubTypes.Type(value = AppendStringColumnTransform.class, name = "AppendStringColumnTransform"), - @JsonSubTypes.Type(value = SequenceDifferenceTransform.class, name = "SequenceDifferenceTransform"), - @JsonSubTypes.Type(value = ReduceSequenceTransform.class, name = "ReduceSequenceTransform"), - @JsonSubTypes.Type(value = SequenceMovingWindowReduceTransform.class, name = "SequenceMovingWindowReduceTransform"), - @JsonSubTypes.Type(value = IntegerToOneHotTransform.class, name = "IntegerToOneHotTransform"), - @JsonSubTypes.Type(value = SequenceTrimTransform.class, name = "SequenceTrimTransform"), - @JsonSubTypes.Type(value = SequenceOffsetTransform.class, name = "SequenceOffsetTransform"), - @JsonSubTypes.Type(value = NDArrayColumnsMathOpTransform.class, name = "NDArrayColumnsMathOpTransform"), - @JsonSubTypes.Type(value = NDArrayDistanceTransform.class, name = "NDArrayDistanceTransform"), - @JsonSubTypes.Type(value = NDArrayMathFunctionTransform.class, name = "NDArrayMathFunctionTransform"), - @JsonSubTypes.Type(value = NDArrayScalarOpTransform.class, name = "NDArrayScalarOpTransform"), - @JsonSubTypes.Type(value = ChangeCaseStringTransform.class, name = "ChangeCaseStringTransform"), - @JsonSubTypes.Type(value = ConcatenateStringColumns.class, name = "ConcatenateStringColumns"), - @JsonSubTypes.Type(value = StringListToCountsNDArrayTransform.class, name = "StringListToCountsNDArrayTransform"), - @JsonSubTypes.Type(value = StringListToIndicesNDArrayTransform.class, name = "StringListToIndicesNDArrayTransform"), - @JsonSubTypes.Type(value = PivotTransform.class, name = "PivotTransform"), - @JsonSubTypes.Type(value = TextToCharacterIndexTransform.class, name = "TextToCharacterIndexTransform")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class TransformMixin { } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = TrivialColumnCondition.class, name = "TrivialColumnCondition"), - @JsonSubTypes.Type(value = CategoricalColumnCondition.class, name = "CategoricalColumnCondition"), - @JsonSubTypes.Type(value = DoubleColumnCondition.class, name = "DoubleColumnCondition"), - @JsonSubTypes.Type(value = IntegerColumnCondition.class, name = "IntegerColumnCondition"), - @JsonSubTypes.Type(value = LongColumnCondition.class, name = "LongColumnCondition"), - @JsonSubTypes.Type(value = NullWritableColumnCondition.class, name = "NullWritableColumnCondition"), - @JsonSubTypes.Type(value = StringColumnCondition.class, name = "StringColumnCondition"), - @JsonSubTypes.Type(value = TimeColumnCondition.class, name = "TimeColumnCondition"), - @JsonSubTypes.Type(value = StringRegexColumnCondition.class, name = "StringRegexColumnCondition"), - @JsonSubTypes.Type(value = BooleanCondition.class, name = "BooleanCondition"), - @JsonSubTypes.Type(value = NaNColumnCondition.class, name = "NaNColumnCondition"), - @JsonSubTypes.Type(value = InfiniteColumnCondition.class, name = "InfiniteColumnCondition"), - @JsonSubTypes.Type(value = SequenceLengthCondition.class, name = "SequenceLengthCondition")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class ConditionMixin { } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = ArrayWritable.class, name = "ArrayWritable"), - @JsonSubTypes.Type(value = BooleanWritable.class, name = "BooleanWritable"), - @JsonSubTypes.Type(value = ByteWritable.class, name = "ByteWritable"), - @JsonSubTypes.Type(value = DoubleWritable.class, name = "DoubleWritable"), - @JsonSubTypes.Type(value = FloatWritable.class, name = "FloatWritable"), - @JsonSubTypes.Type(value = IntWritable.class, name = "IntWritable"), - @JsonSubTypes.Type(value = LongWritable.class, name = "LongWritable"), - @JsonSubTypes.Type(value = NullWritable.class, name = "NullWritable"), - @JsonSubTypes.Type(value = Text.class, name = "Text"), - @JsonSubTypes.Type(value = BytesWritable.class, name = "BytesWritable")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class WritableMixin { } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = ConditionFilter.class, name = "ConditionFilter"), - @JsonSubTypes.Type(value = FilterInvalidValues.class, name = "FilterInvalidValues"), - @JsonSubTypes.Type(value = InvalidNumColumns.class, name = "InvalidNumCols")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class FilterMixin { } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = NumericalColumnComparator.class, name = "NumericalColumnComparator"), - @JsonSubTypes.Type(value = StringComparator.class, name = "StringComparator")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class SequenceComparatorMixin { } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = SequenceSplitTimeSeparation.class, name = "SequenceSplitTimeSeparation"), - @JsonSubTypes.Type(value = SplitMaxLengthSequence.class, name = "SplitMaxLengthSequence")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class SequenceSplitMixin { } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = TimeWindowFunction.class, name = "TimeWindowFunction"), - @JsonSubTypes.Type(value = OverlappingTimeWindowFunction.class, name = "OverlappingTimeWindowFunction")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class WindowFunctionMixin { } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = CalculateSortedRank.class, name = "CalculateSortedRank")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class CalculateSortedRankMixin { } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = DoubleWritableComparator.class, name = "DoubleWritableComparator"), - @JsonSubTypes.Type(value = FloatWritableComparator.class, name = "FloatWritableComparator"), - @JsonSubTypes.Type(value = IntWritableComparator.class, name = "IntWritableComparator"), - @JsonSubTypes.Type(value = LongWritableComparator.class, name = "LongWritableComparator"), - @JsonSubTypes.Type(value = TextWritableComparator.class, name = "TextWritableComparator")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class WritableComparatorMixin { } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = BytesAnalysis.class, name = "BytesAnalysis"), - @JsonSubTypes.Type(value = CategoricalAnalysis.class, name = "CategoricalAnalysis"), - @JsonSubTypes.Type(value = DoubleAnalysis.class, name = "DoubleAnalysis"), - @JsonSubTypes.Type(value = IntegerAnalysis.class, name = "IntegerAnalysis"), - @JsonSubTypes.Type(value = LongAnalysis.class, name = "LongAnalysis"), - @JsonSubTypes.Type(value = StringAnalysis.class, name = "StringAnalysis"), - @JsonSubTypes.Type(value = TimeAnalysis.class, name = "TimeAnalysis")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class ColumnAnalysisMixin{ } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = StringReducer.class, name = "StringReducer")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class IStringReducerMixin{ } -} diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/WritableComparator.java b/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/WritableComparator.java deleted file mode 100644 index ba625411a..000000000 --- a/datavec/datavec-api/src/main/java/org/datavec/api/writable/comparator/WritableComparator.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.writable.comparator; - -import org.datavec.api.writable.Writable; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; - -import java.io.Serializable; -import java.util.Comparator; - -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") -public interface WritableComparator extends Comparator, Serializable { - -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/AssertTestsExtendBaseClass.java b/datavec/datavec-api/src/test/java/org/datavec/api/AssertTestsExtendBaseClass.java deleted file mode 100644 index 2ce6ea207..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/AssertTestsExtendBaseClass.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api; - -import lombok.extern.slf4j.Slf4j; -import org.datavec.api.transform.serde.testClasses.CustomCondition; -import org.datavec.api.transform.serde.testClasses.CustomFilter; -import org.datavec.api.transform.serde.testClasses.CustomTransform; -import org.nd4j.common.tests.AbstractAssertTestsClass; -import org.nd4j.common.tests.BaseND4JTest; - -import java.util.*; - -@Slf4j -public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { - - @Override - protected Set> getExclusions() { - //Set of classes that are exclusions to the rule (either run manually or have their own logging + timeouts) - Set> res = new HashSet<>(); - res.add(CustomCondition.class); - res.add(CustomFilter.class); - res.add(CustomTransform.class); - return res; - } - - @Override - protected String getPackageName() { - return "org.datavec.api"; - } - - @Override - protected Class getBaseClass() { - return BaseND4JTest.class; - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVLineSequenceRecordReaderTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVLineSequenceRecordReaderTest.java deleted file mode 100644 index cdf15de05..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVLineSequenceRecordReaderTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.records.reader.impl; - -import org.apache.commons.io.FileUtils; -import org.datavec.api.records.reader.SequenceRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVLineSequenceRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.BaseND4JTest; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Csv Line Sequence Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class CSVLineSequenceRecordReaderTest extends BaseND4JTest { - - @TempDir - public Path testDir; - - @Test - @DisplayName("Test") - void test(@TempDir Path testDir) throws Exception { - File f = testDir.toFile(); - File source = new File(f, "temp.csv"); - String str = "a,b,c\n1,2,3,4"; - FileUtils.writeStringToFile(source, str, StandardCharsets.UTF_8); - SequenceRecordReader rr = new CSVLineSequenceRecordReader(); - rr.initialize(new FileSplit(source)); - List> exp0 = Arrays.asList(Collections.singletonList(new Text("a")), Collections.singletonList(new Text("b")), Collections.singletonList(new Text("c"))); - List> exp1 = Arrays.asList(Collections.singletonList(new Text("1")), Collections.singletonList(new Text("2")), Collections.singletonList(new Text("3")), Collections.singletonList(new Text("4"))); - for (int i = 0; i < 3; i++) { - int count = 0; - while (rr.hasNext()) { - List> next = rr.sequenceRecord(); - if (count++ == 0) { - assertEquals(exp0, next); - } else { - assertEquals(exp1, next); - } - } - assertEquals(2, count); - rr.reset(); - } - } - - @Override - public long getTimeoutMilliseconds() { - return Long.MAX_VALUE; - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVRecordReaderTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVRecordReaderTest.java deleted file mode 100644 index 4defe93b6..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/CSVRecordReaderTest.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.records.reader.impl; - -import org.apache.commons.io.FileUtils; -import org.datavec.api.records.Record; -import org.datavec.api.records.metadata.RecordMetaData; -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVRegexRecordReader; -import org.datavec.api.records.writer.impl.FileRecordWriter; -import org.datavec.api.records.writer.impl.csv.CSVRecordWriter; -import org.datavec.api.split.FileSplit; -import org.datavec.api.split.InputStreamInputSplit; -import org.datavec.api.split.StringSplit; -import org.datavec.api.split.partition.NumberOfRecordsPartitioner; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; - -import static org.junit.jupiter.api.Assertions.*; - - -@DisplayName("Csv Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class CSVRecordReaderTest extends BaseND4JTest { - - @Test - @DisplayName("Test Next") - void testNext() throws Exception { - CSVRecordReader reader = new CSVRecordReader(); - reader.initialize(new StringSplit("1,1,8.0,,,,14.0,,,,15.0,,,,,,,,,,,,1")); - while (reader.hasNext()) { - List vals = reader.next(); - List arr = new ArrayList<>(vals); - assertEquals(23, vals.size(), "Entry count"); - Text lastEntry = (Text) arr.get(arr.size() - 1); - assertEquals(1, lastEntry.getLength(), "Last entry garbage"); - } - } - - @Test - @DisplayName("Test Empty Entries") - void testEmptyEntries() throws Exception { - CSVRecordReader reader = new CSVRecordReader(); - reader.initialize(new StringSplit("1,1,8.0,,,,14.0,,,,15.0,,,,,,,,,,,,")); - while (reader.hasNext()) { - List vals = reader.next(); - assertEquals(23, vals.size(), "Entry count"); - } - } - - @Test - @DisplayName("Test Reset") - void testReset() throws Exception { - CSVRecordReader rr = new CSVRecordReader(0, ','); - rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); - int nResets = 5; - for (int i = 0; i < nResets; i++) { - int lineCount = 0; - while (rr.hasNext()) { - List line = rr.next(); - assertEquals(5, line.size()); - lineCount++; - } - assertFalse(rr.hasNext()); - assertEquals(150, lineCount); - rr.reset(); - } - } - - @Test - @DisplayName("Test Reset With Skip Lines") - void testResetWithSkipLines() throws Exception { - CSVRecordReader rr = new CSVRecordReader(10, ','); - rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); - int lineCount = 0; - while (rr.hasNext()) { - rr.next(); - ++lineCount; - } - assertEquals(140, lineCount); - rr.reset(); - lineCount = 0; - while (rr.hasNext()) { - rr.next(); - ++lineCount; - } - assertEquals(140, lineCount); - } - - @Test - @DisplayName("Test Write") - void testWrite() throws Exception { - List> list = new ArrayList<>(); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < 10; i++) { - List temp = new ArrayList<>(); - for (int j = 0; j < 3; j++) { - int v = 100 * i + j; - temp.add(new IntWritable(v)); - sb.append(v); - if (j < 2) - sb.append(","); - else if (i != 9) - sb.append("\n"); - } - list.add(temp); - } - String expected = sb.toString(); - Path p = Files.createTempFile("csvwritetest", "csv"); - p.toFile().deleteOnExit(); - FileRecordWriter writer = new CSVRecordWriter(); - FileSplit fileSplit = new FileSplit(p.toFile()); - writer.initialize(fileSplit, new NumberOfRecordsPartitioner()); - for (List c : list) { - writer.write(c); - } - writer.close(); - // Read file back in; compare - String fileContents = FileUtils.readFileToString(p.toFile(), FileRecordWriter.DEFAULT_CHARSET.name()); - // System.out.println(expected); - // System.out.println("----------"); - // System.out.println(fileContents); - assertEquals(expected, fileContents); - } - - @Test - @DisplayName("Test Tabs As Split 1") - void testTabsAsSplit1() throws Exception { - CSVRecordReader reader = new CSVRecordReader(0, '\t'); - reader.initialize(new FileSplit(new ClassPathResource("datavec-api/tabbed.txt").getFile())); - while (reader.hasNext()) { - List list = new ArrayList<>(reader.next()); - assertEquals(2, list.size()); - } - } - - @Test - @DisplayName("Test Pipes As Split") - void testPipesAsSplit() throws Exception { - CSVRecordReader reader = new CSVRecordReader(0, '|'); - reader.initialize(new FileSplit(new ClassPathResource("datavec-api/issue414.csv").getFile())); - int lineidx = 0; - List sixthColumn = Arrays.asList(13, 95, 15, 25); - while (reader.hasNext()) { - List list = new ArrayList<>(reader.next()); - assertEquals(10, list.size()); - assertEquals((long) sixthColumn.get(lineidx), list.get(5).toInt()); - lineidx++; - } - } - - @Test - @DisplayName("Test With Quotes") - void testWithQuotes() throws Exception { - CSVRecordReader reader = new CSVRecordReader(0, ',', '\"'); - reader.initialize(new StringSplit("1,0,3,\"Braund, Mr. Owen Harris\",male,\"\"\"\"")); - while (reader.hasNext()) { - List vals = reader.next(); - assertEquals(6, vals.size(), "Entry count"); - assertEquals(vals.get(0).toString(), "1"); - assertEquals(vals.get(1).toString(), "0"); - assertEquals(vals.get(2).toString(), "3"); - assertEquals(vals.get(3).toString(), "Braund, Mr. Owen Harris"); - assertEquals(vals.get(4).toString(), "male"); - assertEquals(vals.get(5).toString(), "\""); - } - } - - @Test - @DisplayName("Test Meta") - void testMeta() throws Exception { - CSVRecordReader rr = new CSVRecordReader(0, ','); - rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile())); - int lineCount = 0; - List metaList = new ArrayList<>(); - List> writables = new ArrayList<>(); - while (rr.hasNext()) { - Record r = rr.nextRecord(); - assertEquals(5, r.getRecord().size()); - lineCount++; - RecordMetaData meta = r.getMetaData(); - // System.out.println(r.getRecord() + "\t" + meta.getLocation() + "\t" + meta.getURI()); - metaList.add(meta); - writables.add(r.getRecord()); - } - assertFalse(rr.hasNext()); - assertEquals(150, lineCount); - rr.reset(); - System.out.println("\n\n\n--------------------------------"); - List contents = rr.loadFromMetaData(metaList); - assertEquals(150, contents.size()); - // for(Record r : contents ){ - // System.out.println(r); - // } - List meta2 = new ArrayList<>(); - meta2.add(metaList.get(100)); - meta2.add(metaList.get(90)); - meta2.add(metaList.get(80)); - meta2.add(metaList.get(70)); - meta2.add(metaList.get(60)); - List contents2 = rr.loadFromMetaData(meta2); - assertEquals(writables.get(100), contents2.get(0).getRecord()); - assertEquals(writables.get(90), contents2.get(1).getRecord()); - assertEquals(writables.get(80), contents2.get(2).getRecord()); - assertEquals(writables.get(70), contents2.get(3).getRecord()); - assertEquals(writables.get(60), contents2.get(4).getRecord()); - } - - @Test - @DisplayName("Test Regex") - void testRegex() throws Exception { - CSVRecordReader reader = new CSVRegexRecordReader(0, ",", null, new String[] { null, "(.+) (.+) (.+)" }); - reader.initialize(new StringSplit("normal,1.2.3.4 space separator")); - while (reader.hasNext()) { - List vals = reader.next(); - assertEquals(4, vals.size(), "Entry count"); - assertEquals(vals.get(0).toString(), "normal"); - assertEquals(vals.get(1).toString(), "1.2.3.4"); - assertEquals(vals.get(2).toString(), "space"); - assertEquals(vals.get(3).toString(), "separator"); - } - } - - @Test - @DisplayName("Test Csv Skip All Lines") - void testCsvSkipAllLines() { - assertThrows(NoSuchElementException.class, () -> { - final int numLines = 4; - final List lineList = Arrays.asList((Writable) new IntWritable(numLines - 1), (Writable) new Text("one"), (Writable) new Text("two"), (Writable) new Text("three")); - String header = ",one,two,three"; - List lines = new ArrayList<>(); - for (int i = 0; i < numLines; i++) lines.add(Integer.toString(i) + header); - File tempFile = File.createTempFile("csvSkipLines", ".csv"); - FileUtils.writeLines(tempFile, lines); - CSVRecordReader rr = new CSVRecordReader(numLines, ','); - rr.initialize(new FileSplit(tempFile)); - rr.reset(); - assertTrue(!rr.hasNext()); - rr.next(); - }); - } - - @Test - @DisplayName("Test Csv Skip All But One Line") - void testCsvSkipAllButOneLine() throws IOException, InterruptedException { - final int numLines = 4; - final List lineList = Arrays.asList(new Text(Integer.toString(numLines - 1)), new Text("one"), new Text("two"), new Text("three")); - String header = ",one,two,three"; - List lines = new ArrayList<>(); - for (int i = 0; i < numLines; i++) lines.add(Integer.toString(i) + header); - File tempFile = File.createTempFile("csvSkipLines", ".csv"); - FileUtils.writeLines(tempFile, lines); - CSVRecordReader rr = new CSVRecordReader(numLines - 1, ','); - rr.initialize(new FileSplit(tempFile)); - rr.reset(); - assertTrue(rr.hasNext()); - assertEquals(rr.next(), lineList); - } - - @Test - @DisplayName("Test Stream Reset") - void testStreamReset() throws Exception { - CSVRecordReader rr = new CSVRecordReader(0, ','); - rr.initialize(new InputStreamInputSplit(new ClassPathResource("datavec-api/iris.dat").getInputStream())); - int count = 0; - while (rr.hasNext()) { - assertNotNull(rr.next()); - count++; - } - assertEquals(150, count); - assertFalse(rr.resetSupported()); - try { - rr.reset(); - fail("Expected exception"); - } catch (Exception e) { - String msg = e.getMessage(); - String msg2 = e.getCause().getMessage(); - assertTrue(msg.contains("Error during LineRecordReader reset"),msg); - assertTrue(msg2.contains("Reset not supported from streams"),msg2); - // e.printStackTrace(); - } - } - - @Test - @DisplayName("Test Useful Exception No Init") - void testUsefulExceptionNoInit() { - CSVRecordReader rr = new CSVRecordReader(0, ','); - try { - rr.hasNext(); - fail("Expected exception"); - } catch (Exception e) { - assertTrue( e.getMessage().contains("initialized"),e.getMessage()); - } - try { - rr.next(); - fail("Expected exception"); - } catch (Exception e) { - assertTrue(e.getMessage().contains("initialized"),e.getMessage()); - } - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileBatchRecordReaderTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileBatchRecordReaderTest.java deleted file mode 100644 index f19fa2bec..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/FileBatchRecordReaderTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.records.reader.impl; - -import org.apache.commons.io.FileUtils; -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.SequenceRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; -import org.datavec.api.records.reader.impl.filebatch.FileBatchRecordReader; -import org.datavec.api.records.reader.impl.filebatch.FileBatchSequenceRecordReader; -import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.loader.FileBatch; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.factory.Nd4jBackend; - -@DisplayName("File Batch Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -public class FileBatchRecordReaderTest extends BaseND4JTest { - @TempDir Path testDir; - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Csv") - void testCsv(Nd4jBackend backend) throws Exception { - // This is an unrealistic use case - one line/record per CSV - File baseDir = testDir.toFile(); - List fileList = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - String s = "file_" + i + "," + i + "," + i; - File f = new File(baseDir, "origFile" + i + ".csv"); - FileUtils.writeStringToFile(f, s, StandardCharsets.UTF_8); - fileList.add(f); - } - FileBatch fb = FileBatch.forFiles(fileList); - RecordReader rr = new CSVRecordReader(); - FileBatchRecordReader fbrr = new FileBatchRecordReader(rr, fb); - for (int test = 0; test < 3; test++) { - for (int i = 0; i < 10; i++) { - assertTrue(fbrr.hasNext()); - List next = fbrr.next(); - assertEquals(3, next.size()); - String s1 = "file_" + i; - assertEquals(s1, next.get(0).toString()); - assertEquals(String.valueOf(i), next.get(1).toString()); - assertEquals(String.valueOf(i), next.get(2).toString()); - } - assertFalse(fbrr.hasNext()); - assertTrue(fbrr.resetSupported()); - fbrr.reset(); - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Csv Sequence") - void testCsvSequence(Nd4jBackend backend) throws Exception { - // CSV sequence - 3 lines per file, 10 files - File baseDir = testDir.toFile(); - List fileList = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - StringBuilder sb = new StringBuilder(); - for (int j = 0; j < 3; j++) { - if (j > 0) - sb.append("\n"); - sb.append("file_" + i + "," + i + "," + j); - } - File f = new File(baseDir, "origFile" + i + ".csv"); - FileUtils.writeStringToFile(f, sb.toString(), StandardCharsets.UTF_8); - fileList.add(f); - } - FileBatch fb = FileBatch.forFiles(fileList); - SequenceRecordReader rr = new CSVSequenceRecordReader(); - FileBatchSequenceRecordReader fbrr = new FileBatchSequenceRecordReader(rr, fb); - for (int test = 0; test < 3; test++) { - for (int i = 0; i < 10; i++) { - assertTrue(fbrr.hasNext()); - List> next = fbrr.sequenceRecord(); - assertEquals(3, next.size()); - int count = 0; - for (List step : next) { - String s1 = "file_" + i; - assertEquals(s1, step.get(0).toString()); - assertEquals(String.valueOf(i), step.get(1).toString()); - assertEquals(String.valueOf(count++), step.get(2).toString()); - } - } - assertFalse(fbrr.hasNext()); - assertTrue(fbrr.resetSupported()); - fbrr.reset(); - } - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonLineRecordReaderTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonLineRecordReaderTest.java deleted file mode 100644 index fbeecf312..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonLineRecordReaderTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.records.reader.impl; - -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.impl.jackson.FieldSelection; -import org.datavec.api.records.reader.impl.jackson.JacksonLineRecordReader; -import org.datavec.api.records.reader.impl.jackson.JacksonLineSequenceRecordReader; -import org.datavec.api.split.CollectionInputSplit; -import org.datavec.api.split.FileSplit; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.jackson.core.JsonFactory; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import java.io.File; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Jackson Line Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class JacksonLineRecordReaderTest extends BaseND4JTest { - - @TempDir - public Path testDir; - - public JacksonLineRecordReaderTest() { - } - - private static FieldSelection getFieldSelection() { - return new FieldSelection.Builder().addField("value1").addField("value2").addField("value3").addField("value4").addField("value5").addField("value6").addField("value7").addField("value8").addField("value9").addField("value10").build(); - } - - @Test - @DisplayName("Test Read JSON") - void testReadJSON() throws Exception { - RecordReader rr = new JacksonLineRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory())); - rr.initialize(new FileSplit(new ClassPathResource("datavec-api/json/json_test_3.txt").getFile())); - testJacksonRecordReader(rr); - } - - private static void testJacksonRecordReader(RecordReader rr) { - while (rr.hasNext()) { - List json0 = rr.next(); - // System.out.println(json0); - assert (json0.size() > 0); - } - } - - @Test - @DisplayName("Test Jackson Line Sequence Record Reader") - void testJacksonLineSequenceRecordReader(@TempDir Path testDir) throws Exception { - File dir = testDir.toFile(); - new ClassPathResource("datavec-api/JacksonLineSequenceRecordReaderTest/").copyDirectory(dir); - FieldSelection f = new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b").addField(new Text("MISSING_CX"), "c", "x").build(); - JacksonLineSequenceRecordReader rr = new JacksonLineSequenceRecordReader(f, new ObjectMapper(new JsonFactory())); - File[] files = dir.listFiles(); - Arrays.sort(files); - URI[] u = new URI[files.length]; - for (int i = 0; i < files.length; i++) { - u[i] = files[i].toURI(); - } - rr.initialize(new CollectionInputSplit(u)); - List> expSeq0 = new ArrayList<>(); - expSeq0.add(Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0"))); - expSeq0.add(Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1"))); - expSeq0.add(Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX"))); - List> expSeq1 = new ArrayList<>(); - expSeq1.add(Arrays.asList((Writable) new Text("aValue3"), new Text("bValue3"), new Text("cxValue3"))); - int count = 0; - while (rr.hasNext()) { - List> next = rr.sequenceRecord(); - if (count++ == 0) { - assertEquals(expSeq0, next); - } else { - assertEquals(expSeq1, next); - } - } - assertEquals(2, count); - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonRecordReaderTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonRecordReaderTest.java deleted file mode 100644 index ff5567302..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/JacksonRecordReaderTest.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.records.reader.impl; - -import org.datavec.api.io.labels.PathLabelGenerator; -import org.datavec.api.records.Record; -import org.datavec.api.records.metadata.RecordMetaData; -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.impl.jackson.FieldSelection; -import org.datavec.api.records.reader.impl.jackson.JacksonRecordReader; -import org.datavec.api.split.InputSplit; -import org.datavec.api.split.NumberedFileInputSplit; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.jackson.core.JsonFactory; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.dataformat.xml.XmlFactory; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; -import java.io.File; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Jackson Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class JacksonRecordReaderTest extends BaseND4JTest { - - @TempDir - public Path testDir; - - @Test - @DisplayName("Test Reading Json") - void testReadingJson(@TempDir Path testDir) throws Exception { - // Load 3 values from 3 JSON files - // stricture: a:value, b:value, c:x:value, c:y:value - // And we want to load only a:value, b:value and c:x:value - // For first JSON file: all values are present - // For second JSON file: b:value is missing - // For third JSON file: c:x:value is missing - ClassPathResource cpr = new ClassPathResource("datavec-api/json/"); - File f = testDir.toFile(); - cpr.copyDirectory(f); - String path = new File(f, "json_test_%d.txt").getAbsolutePath(); - InputSplit is = new NumberedFileInputSplit(path, 0, 2); - RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory())); - rr.initialize(is); - testJacksonRecordReader(rr); - } - - @Test - @DisplayName("Test Reading Yaml") - void testReadingYaml(@TempDir Path testDir) throws Exception { - // Exact same information as JSON format, but in YAML format - ClassPathResource cpr = new ClassPathResource("datavec-api/yaml/"); - File f = testDir.toFile(); - cpr.copyDirectory(f); - String path = new File(f, "yaml_test_%d.txt").getAbsolutePath(); - InputSplit is = new NumberedFileInputSplit(path, 0, 2); - RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new YAMLFactory())); - rr.initialize(is); - testJacksonRecordReader(rr); - } - - @Test - @DisplayName("Test Reading Xml") - void testReadingXml(@TempDir Path testDir) throws Exception { - // Exact same information as JSON format, but in XML format - ClassPathResource cpr = new ClassPathResource("datavec-api/xml/"); - File f = testDir.toFile(); - cpr.copyDirectory(f); - String path = new File(f, "xml_test_%d.txt").getAbsolutePath(); - InputSplit is = new NumberedFileInputSplit(path, 0, 2); - RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new XmlFactory())); - rr.initialize(is); - testJacksonRecordReader(rr); - } - - private static FieldSelection getFieldSelection() { - return new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b").addField(new Text("MISSING_CX"), "c", "x").build(); - } - - private static void testJacksonRecordReader(RecordReader rr) { - List json0 = rr.next(); - List exp0 = Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0")); - assertEquals(exp0, json0); - List json1 = rr.next(); - List exp1 = Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1")); - assertEquals(exp1, json1); - List json2 = rr.next(); - List exp2 = Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX")); - assertEquals(exp2, json2); - assertFalse(rr.hasNext()); - // Test reset - rr.reset(); - assertEquals(exp0, rr.next()); - assertEquals(exp1, rr.next()); - assertEquals(exp2, rr.next()); - assertFalse(rr.hasNext()); - } - - @Test - @DisplayName("Test Appending Labels") - void testAppendingLabels(@TempDir Path testDir) throws Exception { - ClassPathResource cpr = new ClassPathResource("datavec-api/json/"); - File f = testDir.toFile(); - cpr.copyDirectory(f); - String path = new File(f, "json_test_%d.txt").getAbsolutePath(); - InputSplit is = new NumberedFileInputSplit(path, 0, 2); - // Insert at the end: - RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1, new LabelGen()); - rr.initialize(is); - List exp0 = Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0"), new IntWritable(0)); - assertEquals(exp0, rr.next()); - List exp1 = Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1"), new IntWritable(1)); - assertEquals(exp1, rr.next()); - List exp2 = Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX"), new IntWritable(2)); - assertEquals(exp2, rr.next()); - // Insert at position 0: - rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1, new LabelGen(), 0); - rr.initialize(is); - exp0 = Arrays.asList((Writable) new IntWritable(0), new Text("aValue0"), new Text("bValue0"), new Text("cxValue0")); - assertEquals(exp0, rr.next()); - exp1 = Arrays.asList((Writable) new IntWritable(1), new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1")); - assertEquals(exp1, rr.next()); - exp2 = Arrays.asList((Writable) new IntWritable(2), new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX")); - assertEquals(exp2, rr.next()); - } - - @Test - @DisplayName("Test Appending Labels Meta Data") - void testAppendingLabelsMetaData(@TempDir Path testDir) throws Exception { - ClassPathResource cpr = new ClassPathResource("datavec-api/json/"); - File f = testDir.toFile(); - cpr.copyDirectory(f); - String path = new File(f, "json_test_%d.txt").getAbsolutePath(); - InputSplit is = new NumberedFileInputSplit(path, 0, 2); - // Insert at the end: - RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1, new LabelGen()); - rr.initialize(is); - List> out = new ArrayList<>(); - while (rr.hasNext()) { - out.add(rr.next()); - } - assertEquals(3, out.size()); - rr.reset(); - List> out2 = new ArrayList<>(); - List outRecord = new ArrayList<>(); - List meta = new ArrayList<>(); - while (rr.hasNext()) { - Record r = rr.nextRecord(); - out2.add(r.getRecord()); - outRecord.add(r); - meta.add(r.getMetaData()); - } - assertEquals(out, out2); - List fromMeta = rr.loadFromMetaData(meta); - assertEquals(outRecord, fromMeta); - } - - @DisplayName("Label Gen") - private static class LabelGen implements PathLabelGenerator { - - @Override - public Writable getLabelForPath(String path) { - if (path.endsWith("0.txt")) - return new IntWritable(0); - else if (path.endsWith("1.txt")) - return new IntWritable(1); - else - return new IntWritable(2); - } - - @Override - public Writable getLabelForPath(URI uri) { - return getLabelForPath(uri.getPath()); - } - - @Override - public boolean inferLabelClasses() { - return true; - } - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/LibSvmRecordReaderTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/LibSvmRecordReaderTest.java deleted file mode 100644 index e421e9ef8..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/LibSvmRecordReaderTest.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.records.reader.impl; - -import org.datavec.api.conf.Configuration; -import org.datavec.api.records.reader.impl.misc.LibSvmRecordReader; -import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.io.ClassPathResource; -import java.io.IOException; -import java.util.*; -import static org.datavec.api.records.reader.impl.misc.LibSvmRecordReader.*; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -@DisplayName("Lib Svm Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class LibSvmRecordReaderTest extends BaseND4JTest { - - @Test - @DisplayName("Test Basic Record") - void testBasicRecord() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 7 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), new IntWritable(7))); - // 2 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, new IntWritable(2))); - // 33 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, new IntWritable(33))); - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); - config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test No Append Label") - void testNoAppendLabel() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 7 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5))); - // 2 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO)); - // 33 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO)); - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - config.setBoolean(SVMLightRecordReader.APPEND_LABEL, false); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test No Label") - void testNoLabel() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5))); - // qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO)); - // 1:1.0 - correct.put(2, Arrays.asList(new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO)); - // - correct.put(3, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO)); - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - config.setBoolean(SVMLightRecordReader.APPEND_LABEL, true); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/noLabels.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test Multioutput Record") - void testMultioutputRecord() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 7 2.45,9 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), new IntWritable(7), new DoubleWritable(2.45), new IntWritable(9))); - // 2,3,4 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, new IntWritable(2), new IntWritable(3), new IntWritable(4))); - // 33,32.0,31.9 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, new IntWritable(33), new DoubleWritable(32.0), new DoubleWritable(31.9))); - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); - config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multioutput.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test Multilabel Record") - void testMultilabelRecord() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 1,3 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), LABEL_ONE, LABEL_ZERO, LABEL_ONE, LABEL_ZERO)); - // 2 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ZERO)); - // 1,2,4 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ONE, LABEL_ONE, LABEL_ZERO, LABEL_ONE)); - // 1:1.0 - correct.put(3, Arrays.asList(new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); - // - correct.put(4, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); - config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); - config.setBoolean(LibSvmRecordReader.MULTILABEL, true); - config.setInt(LibSvmRecordReader.NUM_LABELS, 4); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test Zero Based Indexing") - void testZeroBasedIndexing() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 1,3 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ONE, LABEL_ZERO)); - // 2 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(ZERO, new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ZERO)); - // 1,2,4 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ONE, LABEL_ZERO, LABEL_ONE)); - // 1:1.0 - correct.put(3, Arrays.asList(ZERO, new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); - // - correct.put(4, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - // Zero-based indexing is default - // NOT STANDARD! - config.setBoolean(SVMLightRecordReader.ZERO_BASED_LABEL_INDEXING, true); - config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 11); - config.setBoolean(LibSvmRecordReader.MULTILABEL, true); - config.setInt(LibSvmRecordReader.NUM_LABELS, 5); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test No Such Element Exception") - void testNoSuchElementException() { - assertThrows(NoSuchElementException.class, () -> { - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 11); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - while (rr.hasNext()) rr.next(); - rr.next(); - }); - } - - @Test - @DisplayName("Failed To Set Num Features Exception") - void failedToSetNumFeaturesException() { - assertThrows(UnsupportedOperationException.class, () -> { - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - while (rr.hasNext()) rr.next(); - }); - } - - @Test - @DisplayName("Test Inconsistent Num Labels Exception") - void testInconsistentNumLabelsException() { - assertThrows(UnsupportedOperationException.class, () -> { - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/inconsistentNumLabels.txt").getFile())); - while (rr.hasNext()) rr.next(); - }); - } - - @Test - @DisplayName("Test Inconsistent Num Multiabels Exception") - void testInconsistentNumMultiabelsException() { - assertThrows(UnsupportedOperationException.class, () -> { - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(LibSvmRecordReader.MULTILABEL, false); - config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); - while (rr.hasNext()) rr.next(); - }); - } - - @Test - @DisplayName("Test Feature Index Exceeds Num Features") - void testFeatureIndexExceedsNumFeatures() { - assertThrows(IndexOutOfBoundsException.class, () -> { - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 9); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - rr.next(); - }); - } - - @Test - @DisplayName("Test Label Index Exceeds Num Labels") - void testLabelIndexExceedsNumLabels() { - assertThrows(IndexOutOfBoundsException.class, () -> { - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); - config.setInt(LibSvmRecordReader.NUM_LABELS, 6); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - rr.next(); - }); - } - - @Test - @DisplayName("Test Zero Index Feature Without Using Zero Indexing") - void testZeroIndexFeatureWithoutUsingZeroIndexing() { - assertThrows(IndexOutOfBoundsException.class, () -> { - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); - config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexFeature.txt").getFile())); - rr.next(); - }); - } - - @Test - @DisplayName("Test Zero Index Label Without Using Zero Indexing") - void testZeroIndexLabelWithoutUsingZeroIndexing() { - assertThrows(IndexOutOfBoundsException.class, () -> { - LibSvmRecordReader rr = new LibSvmRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); - config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); - config.setBoolean(LibSvmRecordReader.MULTILABEL, true); - config.setInt(LibSvmRecordReader.NUM_LABELS, 2); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexLabel.txt").getFile())); - rr.next(); - }); - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/SVMLightRecordReaderTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/SVMLightRecordReaderTest.java deleted file mode 100644 index 57ccce4dd..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/SVMLightRecordReaderTest.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.records.reader.impl; - -import org.datavec.api.conf.Configuration; -import org.datavec.api.records.Record; -import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.io.ClassPathResource; -import java.io.IOException; -import java.util.*; -import static org.datavec.api.records.reader.impl.misc.SVMLightRecordReader.*; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -@DisplayName("Svm Light Record Reader Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class SVMLightRecordReaderTest extends BaseND4JTest { - - @Test - @DisplayName("Test Basic Record") - void testBasicRecord() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 7 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), new IntWritable(7))); - // 2 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, new IntWritable(2))); - // 33 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, new IntWritable(33))); - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test No Append Label") - void testNoAppendLabel() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 7 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5))); - // 2 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO)); - // 33 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO)); - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - config.setBoolean(SVMLightRecordReader.APPEND_LABEL, false); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test No Label") - void testNoLabel() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5))); - // qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO)); - // 1:1.0 - correct.put(2, Arrays.asList(new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO)); - // - correct.put(3, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO)); - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - config.setBoolean(SVMLightRecordReader.APPEND_LABEL, true); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/noLabels.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test Multioutput Record") - void testMultioutputRecord() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 7 2.45,9 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), new IntWritable(7), new DoubleWritable(2.45), new IntWritable(9))); - // 2,3,4 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, new IntWritable(2), new IntWritable(3), new IntWritable(4))); - // 33,32.0,31.9 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, new IntWritable(33), new DoubleWritable(32.0), new DoubleWritable(31.9))); - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multioutput.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test Multilabel Record") - void testMultilabelRecord() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 1,3 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), LABEL_ONE, LABEL_ZERO, LABEL_ONE, LABEL_ZERO)); - // 2 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ZERO)); - // 1,2,4 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ONE, LABEL_ONE, LABEL_ZERO, LABEL_ONE)); - // 1:1.0 - correct.put(3, Arrays.asList(new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); - // - correct.put(4, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - config.setBoolean(SVMLightRecordReader.MULTILABEL, true); - config.setInt(SVMLightRecordReader.NUM_LABELS, 4); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test Zero Based Indexing") - void testZeroBasedIndexing() throws IOException, InterruptedException { - Map> correct = new HashMap<>(); - // 1,3 2:1 4:2 6:3 8:4 10:5 - correct.put(0, Arrays.asList(ZERO, ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ONE, LABEL_ZERO)); - // 2 qid:42 1:0.1 2:2 6:6.6 8:80 - correct.put(1, Arrays.asList(ZERO, new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ZERO)); - // 1,2,4 - correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ONE, LABEL_ZERO, LABEL_ONE)); - // 1:1.0 - correct.put(3, Arrays.asList(ZERO, new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); - // - correct.put(4, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - // Zero-based indexing is default - // NOT STANDARD! - config.setBoolean(SVMLightRecordReader.ZERO_BASED_LABEL_INDEXING, true); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 11); - config.setBoolean(SVMLightRecordReader.MULTILABEL, true); - config.setInt(SVMLightRecordReader.NUM_LABELS, 5); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); - int i = 0; - while (rr.hasNext()) { - List record = rr.next(); - assertEquals(correct.get(i), record); - i++; - } - assertEquals(i, correct.size()); - } - - @Test - @DisplayName("Test Next Record") - void testNextRecord() throws IOException, InterruptedException { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - config.setBoolean(SVMLightRecordReader.APPEND_LABEL, false); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - Record record = rr.nextRecord(); - List recordList = record.getRecord(); - assertEquals(new DoubleWritable(1.0), recordList.get(1)); - assertEquals(new DoubleWritable(3.0), recordList.get(5)); - assertEquals(new DoubleWritable(4.0), recordList.get(7)); - record = rr.nextRecord(); - recordList = record.getRecord(); - assertEquals(new DoubleWritable(0.1), recordList.get(0)); - assertEquals(new DoubleWritable(6.6), recordList.get(5)); - assertEquals(new DoubleWritable(80.0), recordList.get(7)); - } - - @Test - @DisplayName("Test No Such Element Exception") - void testNoSuchElementException() { - assertThrows(NoSuchElementException.class, () -> { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 11); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - while (rr.hasNext()) rr.next(); - rr.next(); - }); - } - - @Test - @DisplayName("Failed To Set Num Features Exception") - void failedToSetNumFeaturesException() { - assertThrows(UnsupportedOperationException.class, () -> { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - while (rr.hasNext()) rr.next(); - }); - } - - @Test - @DisplayName("Test Inconsistent Num Labels Exception") - void testInconsistentNumLabelsException() { - assertThrows(UnsupportedOperationException.class, () -> { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/inconsistentNumLabels.txt").getFile())); - while (rr.hasNext()) rr.next(); - }); - } - - @Test - @DisplayName("Failed To Set Num Multiabels Exception") - void failedToSetNumMultiabelsException() { - assertThrows(UnsupportedOperationException.class, () -> { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); - while (rr.hasNext()) rr.next(); - }); - } - - @Test - @DisplayName("Test Feature Index Exceeds Num Features") - void testFeatureIndexExceedsNumFeatures() { - assertThrows(IndexOutOfBoundsException.class, () -> { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 9); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - rr.next(); - }); - } - - @Test - @DisplayName("Test Label Index Exceeds Num Labels") - void testLabelIndexExceedsNumLabels() { - assertThrows(IndexOutOfBoundsException.class, () -> { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - config.setInt(SVMLightRecordReader.NUM_LABELS, 6); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); - rr.next(); - }); - } - - @Test - @DisplayName("Test Zero Index Feature Without Using Zero Indexing") - void testZeroIndexFeatureWithoutUsingZeroIndexing() { - assertThrows(IndexOutOfBoundsException.class, () -> { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexFeature.txt").getFile())); - rr.next(); - }); - } - - @Test - @DisplayName("Test Zero Index Label Without Using Zero Indexing") - void testZeroIndexLabelWithoutUsingZeroIndexing() { - assertThrows(IndexOutOfBoundsException.class, () -> { - SVMLightRecordReader rr = new SVMLightRecordReader(); - Configuration config = new Configuration(); - config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); - config.setBoolean(SVMLightRecordReader.MULTILABEL, true); - config.setInt(SVMLightRecordReader.NUM_LABELS, 2); - rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexLabel.txt").getFile())); - rr.next(); - }); - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestSerialization.java b/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestSerialization.java deleted file mode 100644 index b37d3f2a8..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/records/reader/impl/TestSerialization.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.records.reader.impl; - -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.impl.csv.*; -import org.datavec.api.records.reader.impl.jackson.FieldSelection; -import org.datavec.api.records.reader.impl.jackson.JacksonLineRecordReader; -import org.datavec.api.records.reader.impl.jackson.JacksonRecordReader; -import org.datavec.api.records.reader.impl.misc.LibSvmRecordReader; -import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader; -import org.datavec.api.records.reader.impl.regex.RegexLineRecordReader; -import org.datavec.api.records.reader.impl.regex.RegexSequenceRecordReader; -import org.datavec.api.records.reader.impl.transform.TransformProcessRecordReader; -import org.datavec.api.records.reader.impl.transform.TransformProcessSequenceRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.transform.MathFunction; -import org.datavec.api.transform.TransformProcess; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.jackson.core.JsonFactory; -import org.nd4j.shade.jackson.databind.ObjectMapper; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -public class TestSerialization extends BaseND4JTest { - - @Test - public void testRR() throws Exception { - - List rrs = new ArrayList<>(); - - rrs.add(new CSVNLinesSequenceRecordReader(10)); - rrs.add(new CSVRecordReader(10, ',')); - rrs.add(new CSVSequenceRecordReader(1, ",")); - rrs.add(new CSVVariableSlidingWindowRecordReader(5)); - rrs.add(new CSVRegexRecordReader(0, ",", null, new String[] {null, "(.+) (.+) (.+)"})); - rrs.add(new JacksonRecordReader(new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b") - .addField(new Text("MISSING_CX"), "c", "x").build(), new ObjectMapper(new JsonFactory()))); - rrs.add(new JacksonLineRecordReader(new FieldSelection.Builder().addField("value1") - .addField("value2").build(), new ObjectMapper(new JsonFactory()))); - rrs.add(new LibSvmRecordReader()); - rrs.add(new SVMLightRecordReader()); - rrs.add(new RegexLineRecordReader("(.+) (.+) (.+)", 0)); - rrs.add(new RegexSequenceRecordReader("(.+) (.+) (.+)", 0)); - rrs.add(new TransformProcessRecordReader(new CSVRecordReader(), getTp())); - rrs.add(new TransformProcessSequenceRecordReader(new CSVSequenceRecordReader(), getTp())); - rrs.add(new LineRecordReader()); - - for(RecordReader r : rrs){ - System.out.println(r.getClass().getName()); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream os = new ObjectOutputStream(baos); - os.writeObject(r); - byte[] bytes = baos.toByteArray(); - - ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); - - RecordReader r2 = (RecordReader) ois.readObject(); - } - } - - private static TransformProcess getTp(){ - Schema s = new Schema.Builder().addColumnDouble("d").build(); - TransformProcess tp = new TransformProcess.Builder(s) - .doubleMathFunction("d", MathFunction.ABS) - .build(); - return tp; - } - - @Test - public void testCsvRRSerializationResults() throws Exception { - int skipLines = 3; - RecordReader r1 = new CSVRecordReader(skipLines, '\t'); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream os = new ObjectOutputStream(baos); - os.writeObject(r1); - byte[] bytes = baos.toByteArray(); - ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); - RecordReader r2 = (RecordReader) ois.readObject(); - - File f = new ClassPathResource("datavec-api/iris_tab_delim.txt").getFile(); - - r1.initialize(new FileSplit(f)); - r2.initialize(new FileSplit(f)); - - int count = 0; - while(r1.hasNext()){ - List n1 = r1.next(); - List n2 = r2.next(); - assertEquals(n1, n2); - count++; - } - - assertEquals(150-skipLines, count); - } - -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/split/TransformSplitTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/split/TransformSplitTest.java deleted file mode 100644 index ec7094472..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/split/TransformSplitTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.split; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Collection; -import static java.util.Arrays.asList; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; - -/** - * @author Ede Meijer - */ -@DisplayName("Transform Split Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class TransformSplitTest extends BaseND4JTest { - - @Test - @DisplayName("Test Transform") - void testTransform() throws URISyntaxException { - Collection inputFiles = asList(new URI("file:///foo/bar/../0.csv"), new URI("file:///foo/1.csv")); - InputSplit SUT = new TransformSplit(new CollectionInputSplit(inputFiles), new TransformSplit.URITransform() { - - @Override - public URI apply(URI uri) throws URISyntaxException { - return uri.normalize(); - } - }); - assertArrayEquals(new URI[] { new URI("file:///foo/0.csv"), new URI("file:///foo/1.csv") }, SUT.locations()); - } - - @Test - @DisplayName("Test Search Replace") - void testSearchReplace() throws URISyntaxException { - Collection inputFiles = asList(new URI("file:///foo/1-in.csv"), new URI("file:///foo/2-in.csv")); - InputSplit SUT = TransformSplit.ofSearchReplace(new CollectionInputSplit(inputFiles), "-in.csv", "-out.csv"); - assertArrayEquals(new URI[] { new URI("file:///foo/1-out.csv"), new URI("file:///foo/2-out.csv") }, SUT.locations()); - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/join/TestJoin.java b/datavec/datavec-api/src/test/java/org/datavec/api/transform/join/TestJoin.java deleted file mode 100644 index ea80ea9c9..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/join/TestJoin.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.transform.join; - -import org.datavec.api.transform.ColumnType; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.NullWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -public class TestJoin extends BaseND4JTest { - - @Test - public void testJoin(@TempDir Path testDir) { - - Schema firstSchema = - new Schema.Builder().addColumnString("keyColumn").addColumnsInteger("first0", "first1").build(); - - Schema secondSchema = new Schema.Builder().addColumnString("keyColumn").addColumnsInteger("second0").build(); - - List> first = new ArrayList<>(); - first.add(Arrays.asList(new Text("key0"), new IntWritable(0), new IntWritable(1))); - first.add(Arrays.asList(new Text("key1"), new IntWritable(10), new IntWritable(11))); - - List> second = new ArrayList<>(); - second.add(Arrays.asList(new Text("key0"), new IntWritable(100))); - second.add(Arrays.asList(new Text("key1"), new IntWritable(110))); - - Join join = new Join.Builder(Join.JoinType.Inner).setJoinColumns("keyColumn") - .setSchemas(firstSchema, secondSchema).build(); - - List> expected = new ArrayList<>(); - expected.add(Arrays.asList(new Text("key0"), new IntWritable(0), new IntWritable(1), - new IntWritable(100))); - expected.add(Arrays.asList(new Text("key1"), new IntWritable(10), new IntWritable(11), - new IntWritable(110))); - - - //Check schema: - Schema joinedSchema = join.getOutputSchema(); - assertEquals(4, joinedSchema.numColumns()); - assertEquals(Arrays.asList("keyColumn", "first0", "first1", "second0"), joinedSchema.getColumnNames()); - assertEquals(Arrays.asList(ColumnType.String, ColumnType.Integer, ColumnType.Integer, ColumnType.Integer), - joinedSchema.getColumnTypes()); - - - //Check joining with null values: - expected = new ArrayList<>(); - expected.add(Arrays.asList((Writable) new Text("key0"), new IntWritable(0), new IntWritable(1), - NullWritable.INSTANCE)); - expected.add(Arrays.asList((Writable) new Text("key1"), new IntWritable(10), new IntWritable(11), - NullWritable.INSTANCE)); - for (int i = 0; i < first.size(); i++) { - List out = join.joinExamples(first.get(i), null); - assertEquals(expected.get(i), out); - } - - expected = new ArrayList<>(); - expected.add(Arrays.asList((Writable) new Text("key0"), NullWritable.INSTANCE, NullWritable.INSTANCE, - new IntWritable(100))); - expected.add(Arrays.asList((Writable) new Text("key1"), NullWritable.INSTANCE, NullWritable.INSTANCE, - new IntWritable(110))); - for (int i = 0; i < first.size(); i++) { - List out = join.joinExamples(null, second.get(i)); - assertEquals(expected.get(i), out); - } - } - - - @Test() - public void testJoinValidation() { - assertThrows(IllegalArgumentException.class,() -> { - Schema firstSchema = new Schema.Builder().addColumnString("keyColumn1").addColumnsInteger("first0", "first1") - .build(); - - Schema secondSchema = new Schema.Builder().addColumnString("keyColumn2").addColumnsInteger("second0").build(); - - new Join.Builder(Join.JoinType.Inner).setJoinColumns("keyColumn1", "thisDoesntExist") - .setSchemas(firstSchema, secondSchema).build(); - }); - - } - - @Test() - public void testJoinValidation2() { - assertThrows(IllegalArgumentException.class,() -> { - Schema firstSchema = new Schema.Builder().addColumnString("keyColumn1").addColumnsInteger("first0", "first1") - .build(); - - Schema secondSchema = new Schema.Builder().addColumnString("keyColumn2").addColumnsInteger("second0").build(); - - new Join.Builder(Join.JoinType.Inner).setJoinColumns("keyColumn1").setSchemas(firstSchema, secondSchema) - .build(); - }); - - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpArchTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpArchTest.java deleted file mode 100644 index 42351fd9a..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/ops/AggregableMultiOpArchTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.transform.ops; - -import com.tngtech.archunit.core.importer.ImportOption; -import com.tngtech.archunit.junit.AnalyzeClasses; -import com.tngtech.archunit.junit.ArchTest; -import com.tngtech.archunit.lang.ArchRule; -import com.tngtech.archunit.lang.extension.ArchUnitExtension; -import com.tngtech.archunit.lang.extension.ArchUnitExtensions; -import org.junit.runner.RunWith; -import org.nd4j.common.tests.BaseND4JTest; -import java.io.Serializable; -import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@AnalyzeClasses(packages = "org.datavec.api.transform.ops", importOptions = { ImportOption.DoNotIncludeTests.class }) -@DisplayName("Aggregable Multi Op Arch Test") -class AggregableMultiOpArchTest extends BaseND4JTest { - - @ArchTest - public static final ArchRule ALL_AGGREGATE_OPS_MUST_BE_SERIALIZABLE = classes().that().resideInAPackage("org.datavec.api.transform.ops").and().doNotHaveSimpleName("AggregatorImpls").and().doNotHaveSimpleName("IAggregableReduceOp").and().doNotHaveSimpleName("StringAggregatorImpls").and().doNotHaveFullyQualifiedName("org.datavec.api.transform.ops.StringAggregatorImpls$1").should().implement(Serializable.class).because("All aggregate ops must be serializable."); -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/schema/TestJsonYaml.java b/datavec/datavec-api/src/test/java/org/datavec/api/transform/schema/TestJsonYaml.java deleted file mode 100644 index 45f251924..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/schema/TestJsonYaml.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.transform.schema; - -import org.datavec.api.transform.metadata.ColumnMetaData; -import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JACKSON_SERDE) -public class TestJsonYaml extends BaseND4JTest { - - @Test - public void testToFromJsonYaml() { - - Schema schema = new Schema.Builder() - .addColumnCategorical("Cat", "State1", "State2") - .addColumnDouble("Dbl") - .addColumnDouble("Dbl2", null, 100.0, true, false) - .addColumnInteger("Int") - .addColumnInteger("Int2", 0, 10) - .addColumnLong("Long") - .addColumnLong("Long2", -100L, null) - .addColumnString("Str") - .addColumnString("Str2", "someregexhere", 1, null) - .addColumnTime("TimeCol", DateTimeZone.UTC) - .addColumnTime("TimeCol2", DateTimeZone.UTC, null, 1000L) - .addColumnNDArray("ndarray", new long[]{1, 10}) - .addColumnBoolean("boolean") - .addColumnFloat("float") - .addColumnFloat("float2", -100f, 100f, true, false) - .build(); - - String asJson = schema.toJson(); - // System.out.println(asJson); - - Schema schema2 = Schema.fromJson(asJson); - - int count = schema.numColumns(); - for (int i = 0; i < count; i++) { - ColumnMetaData c1 = schema.getMetaData(i); - ColumnMetaData c2 = schema2.getMetaData(i); - assertEquals(c1, c2); - } - assertEquals(schema, schema2); - - - String asYaml = schema.toYaml(); - // System.out.println(asYaml); - - Schema schema3 = Schema.fromYaml(asYaml); - for (int i = 0; i < schema.numColumns(); i++) { - ColumnMetaData c1 = schema.getMetaData(i); - ColumnMetaData c3 = schema3.getMetaData(i); - assertEquals(c1, c3); - } - assertEquals(schema, schema3); - } - - @Test - public void testMissingPrimitives() { - - Schema schema = new Schema.Builder().addColumnDouble("Dbl2", null, 100.0, false, false).build(); - //Legacy format JSON - String strJson = "{\n" + " \"Schema\" : {\n" - + " \"columns\" : [ {\n" + " \"Double\" : {\n" - + " \"name\" : \"Dbl2\",\n" + " \"maxAllowedValue\" : 100.0\n" + - //" \"allowNaN\" : false,\n" + //Normally included: but exclude here to test - //" \"allowInfinite\" : false\n" + //Normally included: but exclude here to test - " }\n" + " } ]\n" + " }\n" + "}"; - - Schema schema2 = Schema.fromJson(strJson); - assertEquals(schema, schema2); - - - - String strYaml = "--- !\n" + "columns:\n" + "- !\n" + " name: \"Dbl2\"\n" - + " maxAllowedValue: 100.0"; - //" allowNaN: false\n" + //Normally included: but exclude here to test - //" allowInfinite: false"; //Normally included: but exclude here to test - -// Schema schema2a = Schema.fromYaml(strYaml); -// assertEquals(schema, schema2a); - } - - @Test - public void testToFromJsonYamlSequence() { - - Schema schema = new SequenceSchema.Builder().addColumnCategorical("Cat", "State1", "State2") - .addColumnDouble("Dbl").addColumnDouble("Dbl2", null, 100.0, true, false) - .addColumnInteger("Int").addColumnInteger("Int2", 0, 10).addColumnLong("Long") - .addColumnLong("Long2", -100L, null).addColumnString("Str") - .addColumnString("Str2", "someregexhere", 1, null).addColumnTime("TimeCol", DateTimeZone.UTC) - .addColumnTime("TimeCol2", DateTimeZone.UTC, null, 1000L).build(); - - String asJson = schema.toJson(); - // System.out.println(asJson); - - Schema schema2 = Schema.fromJson(asJson); - - int count = schema.numColumns(); - for (int i = 0; i < count; i++) { - ColumnMetaData c1 = schema.getMetaData(i); - ColumnMetaData c2 = schema2.getMetaData(i); - assertEquals(c1, c2); - } - assertEquals(schema, schema2); - - - String asYaml = schema.toYaml(); - // System.out.println(asYaml); - - Schema schema3 = Schema.fromYaml(asYaml); - for (int i = 0; i < schema.numColumns(); i++) { - ColumnMetaData c1 = schema.getMetaData(i); - ColumnMetaData c3 = schema3.getMetaData(i); - assertEquals(c1, c3); - } - assertEquals(schema, schema3); - - } - -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestSequenceSplit.java b/datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestSequenceSplit.java deleted file mode 100644 index 4113ec1d7..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestSequenceSplit.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.transform.sequence; - -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.transform.schema.SequenceSchema; -import org.datavec.api.transform.sequence.split.SequenceSplitTimeSeparation; -import org.datavec.api.writable.LongWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -public class TestSequenceSplit extends BaseND4JTest { - - @Test - public void testSequenceSplitTimeSeparation() { - - Schema schema = new SequenceSchema.Builder().addColumnTime("time", DateTimeZone.UTC).addColumnString("text") - .build(); - - List> inputSequence = new ArrayList<>(); - inputSequence.add(Arrays.asList(new LongWritable(0), new Text("t0"))); - inputSequence.add(Arrays.asList(new LongWritable(1000), new Text("t1"))); - //Second split: 74 seconds later - inputSequence.add(Arrays.asList(new LongWritable(75000), new Text("t2"))); - inputSequence.add(Arrays.asList(new LongWritable(100000), new Text("t3"))); - //Third split: 1 minute and 1 milliseconds later - inputSequence.add(Arrays.asList(new LongWritable(160001), new Text("t4"))); - - SequenceSplit seqSplit = new SequenceSplitTimeSeparation("time", 1, TimeUnit.MINUTES); - seqSplit.setInputSchema(schema); - - List>> splits = seqSplit.split(inputSequence); - assertEquals(3, splits.size()); - - List> exp0 = new ArrayList<>(); - exp0.add(Arrays.asList(new LongWritable(0), new Text("t0"))); - exp0.add(Arrays.asList(new LongWritable(1000), new Text("t1"))); - List> exp1 = new ArrayList<>(); - exp1.add(Arrays.asList(new LongWritable(75000), new Text("t2"))); - exp1.add(Arrays.asList(new LongWritable(100000), new Text("t3"))); - List> exp2 = new ArrayList<>(); - exp2.add(Arrays.asList(new LongWritable(160001), new Text("t4"))); - - assertEquals(exp0, splits.get(0)); - assertEquals(exp1, splits.get(1)); - assertEquals(exp2, splits.get(2)); - } - -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestWindowFunctions.java b/datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestWindowFunctions.java deleted file mode 100644 index a10835299..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/transform/sequence/TestWindowFunctions.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.api.transform.sequence; - -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.transform.schema.SequenceSchema; -import org.datavec.api.transform.sequence.window.OverlappingTimeWindowFunction; -import org.datavec.api.transform.sequence.window.TimeWindowFunction; -import org.datavec.api.transform.sequence.window.WindowFunction; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.LongWritable; -import org.datavec.api.writable.Writable; -import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -public class TestWindowFunctions extends BaseND4JTest { - - @Test - public void testTimeWindowFunction() { - - //Time windowing: 1 second (1000 milliseconds) window - - //Create some data. - List> sequence = new ArrayList<>(); - //First window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2))); - //Second window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 1000L), new IntWritable(3))); - //Third window: empty - //Fourth window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3000L), new IntWritable(4))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3100L), new IntWritable(5))); - - Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) - .addColumnInteger("intcolumn").build(); - - WindowFunction wf = new TimeWindowFunction("timecolumn", 1, TimeUnit.SECONDS); - wf.setInputSchema(schema); - - List>> windows = wf.applyToSequence(sequence); - - assertEquals(4, windows.size()); - assertEquals(3, windows.get(0).size()); - assertEquals(1, windows.get(1).size()); - assertEquals(0, windows.get(2).size()); - assertEquals(2, windows.get(3).size()); - - List> exp0 = new ArrayList<>(); - exp0.add(sequence.get(0)); - exp0.add(sequence.get(1)); - exp0.add(sequence.get(2)); - assertEquals(exp0, windows.get(0)); - - List> exp1 = new ArrayList<>(); - exp1.add(sequence.get(3)); - assertEquals(exp1, windows.get(1)); - - List> exp2 = new ArrayList<>(); - assertEquals(exp2, windows.get(2)); - - List> exp3 = new ArrayList<>(); - exp3.add(sequence.get(4)); - exp3.add(sequence.get(5)); - assertEquals(exp3, windows.get(3)); - } - - @Test - public void testTimeWindowFunctionExcludeEmpty() { - - //Time windowing: 1 second (1000 milliseconds) window - - //Create some data. - List> sequence = new ArrayList<>(); - //First window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2))); - //Second window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 1000L), new IntWritable(3))); - //Third window: empty - //Fourth window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3000L), new IntWritable(4))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3100L), new IntWritable(5))); - - Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) - .addColumnInteger("intcolumn").build(); - - WindowFunction wf = new TimeWindowFunction.Builder().timeColumn("timecolumn").windowSize(1, TimeUnit.SECONDS) - .excludeEmptyWindows(true).build(); - - wf.setInputSchema(schema); - - List>> windows = wf.applyToSequence(sequence); - - assertEquals(3, windows.size()); - assertEquals(3, windows.get(0).size()); - assertEquals(1, windows.get(1).size()); - assertEquals(2, windows.get(2).size()); - - List> exp0 = new ArrayList<>(); - exp0.add(sequence.get(0)); - exp0.add(sequence.get(1)); - exp0.add(sequence.get(2)); - assertEquals(exp0, windows.get(0)); - - List> exp1 = new ArrayList<>(); - exp1.add(sequence.get(3)); - assertEquals(exp1, windows.get(1)); - - List> exp2 = new ArrayList<>(); - exp2.add(sequence.get(4)); - exp2.add(sequence.get(5)); - assertEquals(exp2, windows.get(2)); - } - - @Test - public void testOverlappingTimeWindowFunctionSimple() { - //Compare Overlapping and standard window functions where the window separation is equal to the window size - // In this case, we should get exactly the same results from both. - //Time windowing: 1 second (1000 milliseconds) window - - //Create some data. - List> sequence = new ArrayList<>(); - //First window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 100L), new IntWritable(1))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 200L), new IntWritable(2))); - //Second window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 1000L), new IntWritable(3))); - //Third window: empty - //Fourth window: - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3000L), new IntWritable(4))); - sequence.add(Arrays.asList(new LongWritable(1451606400000L + 3100L), new IntWritable(5))); - - Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) - .addColumnInteger("intcolumn").build(); - - WindowFunction wf = new TimeWindowFunction("timecolumn", 1, TimeUnit.SECONDS); - wf.setInputSchema(schema); - - WindowFunction wf2 = new OverlappingTimeWindowFunction("timecolumn", 1, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); - wf2.setInputSchema(schema); - - List>> windowsExp = wf.applyToSequence(sequence); - List>> windowsAct = wf2.applyToSequence(sequence); - - int[] expSizes = {3, 1, 0, 2}; - assertEquals(4, windowsExp.size()); - assertEquals(4, windowsAct.size()); - for (int i = 0; i < 4; i++) { - assertEquals(expSizes[i], windowsExp.get(i).size()); - assertEquals(expSizes[i], windowsAct.get(i).size()); - - assertEquals(windowsExp.get(i), windowsAct.get(i)); - } - } - - @Test - public void testOverlappingTimeWindowFunction() { - //Create some data. - List> sequence = new ArrayList<>(); - //First window: - sequence.add(Arrays.asList(new LongWritable(0), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(100), new IntWritable(1))); - sequence.add(Arrays.asList(new LongWritable(200), new IntWritable(2))); - sequence.add(Arrays.asList(new LongWritable(1000), new IntWritable(3))); - sequence.add(Arrays.asList(new LongWritable(1500), new IntWritable(4))); - sequence.add(Arrays.asList(new LongWritable(2000), new IntWritable(5))); - sequence.add(Arrays.asList(new LongWritable(5000), new IntWritable(7))); - - - Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) - .addColumnInteger("intcolumn").build(); - //Window size: 2 seconds; calculated every 1 second - WindowFunction wf2 = new OverlappingTimeWindowFunction("timecolumn", 2, TimeUnit.SECONDS, 1, TimeUnit.SECONDS); - wf2.setInputSchema(schema); - - List>> windowsAct = wf2.applyToSequence(sequence); - - //First window: -1000 to 1000 - List> exp0 = new ArrayList<>(); - exp0.add(Arrays.asList(new LongWritable(0), new IntWritable(0))); - exp0.add(Arrays.asList(new LongWritable(100), new IntWritable(1))); - exp0.add(Arrays.asList(new LongWritable(200), new IntWritable(2))); - //Second window: 0 to 2000 - List> exp1 = new ArrayList<>(); - exp1.add(Arrays.asList(new LongWritable(0), new IntWritable(0))); - exp1.add(Arrays.asList(new LongWritable(100), new IntWritable(1))); - exp1.add(Arrays.asList(new LongWritable(200), new IntWritable(2))); - exp1.add(Arrays.asList(new LongWritable(1000), new IntWritable(3))); - exp1.add(Arrays.asList(new LongWritable(1500), new IntWritable(4))); - //Third window: 1000 to 3000 - List> exp2 = new ArrayList<>(); - exp2.add(Arrays.asList(new LongWritable(1000), new IntWritable(3))); - exp2.add(Arrays.asList(new LongWritable(1500), new IntWritable(4))); - exp2.add(Arrays.asList(new LongWritable(2000), new IntWritable(5))); - //Fourth window: 2000 to 4000 - List> exp3 = new ArrayList<>(); - exp3.add(Arrays.asList(new LongWritable(2000), new IntWritable(5))); - //Fifth window: 3000 to 5000 - List> exp4 = new ArrayList<>(); - //Sixth window: 4000 to 6000 - List> exp5 = new ArrayList<>(); - exp5.add(Arrays.asList(new LongWritable(5000), new IntWritable(7))); - //Seventh window: 5000 to 7000 - List> exp6 = new ArrayList<>(); - exp6.add(Arrays.asList(new LongWritable(5000), new IntWritable(7))); - - List>> windowsExp = Arrays.asList(exp0, exp1, exp2, exp3, exp4, exp5, exp6); - - assertEquals(7, windowsAct.size()); - for (int i = 0; i < 7; i++) { - List> exp = windowsExp.get(i); - List> act = windowsAct.get(i); - - assertEquals(exp, act); - } - } - - @Test - public void testOverlappingTimeWindowFunctionExcludeEmpty() { - //Create some data. - List> sequence = new ArrayList<>(); - //First window: - sequence.add(Arrays.asList(new LongWritable(0), new IntWritable(0))); - sequence.add(Arrays.asList(new LongWritable(100), new IntWritable(1))); - sequence.add(Arrays.asList(new LongWritable(200), new IntWritable(2))); - sequence.add(Arrays.asList(new LongWritable(1000), new IntWritable(3))); - sequence.add(Arrays.asList(new LongWritable(1500), new IntWritable(4))); - sequence.add(Arrays.asList(new LongWritable(2000), new IntWritable(5))); - sequence.add(Arrays.asList(new LongWritable(5000), new IntWritable(7))); - - - Schema schema = new SequenceSchema.Builder().addColumnTime("timecolumn", DateTimeZone.UTC) - .addColumnInteger("intcolumn").build(); - //Window size: 2 seconds; calculated every 1 second - // WindowFunction wf2 = new OverlappingTimeWindowFunction("timecolumn",2,TimeUnit.SECONDS,1,TimeUnit.SECONDS); - WindowFunction wf2 = new OverlappingTimeWindowFunction.Builder().timeColumn("timecolumn") - .windowSize(2, TimeUnit.SECONDS).windowSeparation(1, TimeUnit.SECONDS).excludeEmptyWindows(true) - .build(); - wf2.setInputSchema(schema); - - List>> windowsAct = wf2.applyToSequence(sequence); - - //First window: -1000 to 1000 - List> exp0 = new ArrayList<>(); - exp0.add(Arrays.asList(new LongWritable(0), new IntWritable(0))); - exp0.add(Arrays.asList(new LongWritable(100), new IntWritable(1))); - exp0.add(Arrays.asList(new LongWritable(200), new IntWritable(2))); - //Second window: 0 to 2000 - List> exp1 = new ArrayList<>(); - exp1.add(Arrays.asList(new LongWritable(0), new IntWritable(0))); - exp1.add(Arrays.asList(new LongWritable(100), new IntWritable(1))); - exp1.add(Arrays.asList(new LongWritable(200), new IntWritable(2))); - exp1.add(Arrays.asList(new LongWritable(1000), new IntWritable(3))); - exp1.add(Arrays.asList(new LongWritable(1500), new IntWritable(4))); - //Third window: 1000 to 3000 - List> exp2 = new ArrayList<>(); - exp2.add(Arrays.asList(new LongWritable(1000), new IntWritable(3))); - exp2.add(Arrays.asList(new LongWritable(1500), new IntWritable(4))); - exp2.add(Arrays.asList(new LongWritable(2000), new IntWritable(5))); - //Fourth window: 2000 to 4000 - List> exp3 = new ArrayList<>(); - exp3.add(Arrays.asList(new LongWritable(2000), new IntWritable(5))); - //Fifth window: 3000 to 5000 -> Empty: excluded - //Sixth window: 4000 to 6000 - List> exp5 = new ArrayList<>(); - exp5.add(Arrays.asList(new LongWritable(5000), new IntWritable(7))); - //Seventh window: 5000 to 7000 - List> exp6 = new ArrayList<>(); - exp6.add(Arrays.asList(new LongWritable(5000), new IntWritable(7))); - - List>> windowsExp = Arrays.asList(exp0, exp1, exp2, exp3, exp5, exp6); - - assertEquals(6, windowsAct.size()); - for (int i = 0; i < 6; i++) { - List> exp = windowsExp.get(i); - List> act = windowsAct.get(i); - - assertEquals(exp, act); - } - } - -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/util/ClassPathResourceTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/util/ClassPathResourceTest.java deleted file mode 100644 index 10e431ce2..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/util/ClassPathResourceTest.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.util; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStream; -import java.io.InputStreamReader; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.AnyOf.anyOf; -import static org.hamcrest.core.IsEqual.equalTo; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Class Path Resource Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class ClassPathResourceTest extends BaseND4JTest { - - // File sizes are reported slightly different on Linux vs. Windows - private boolean isWindows = false; - - @BeforeEach - void setUp() throws Exception { - String osname = System.getProperty("os.name"); - if (osname != null && osname.toLowerCase().contains("win")) { - isWindows = true; - } - } - - @Test - @DisplayName("Test Get File 1") - void testGetFile1() throws Exception { - File intFile = new ClassPathResource("datavec-api/iris.dat").getFile(); - assertTrue(intFile.exists()); - if (isWindows) { - assertThat(intFile.length(), anyOf(equalTo(2700L), equalTo(2850L))); - } else { - assertEquals(2700, intFile.length()); - } - } - - @Test - @DisplayName("Test Get File Slash 1") - void testGetFileSlash1() throws Exception { - File intFile = new ClassPathResource("datavec-api/iris.dat").getFile(); - assertTrue(intFile.exists()); - if (isWindows) { - assertThat(intFile.length(), anyOf(equalTo(2700L), equalTo(2850L))); - } else { - assertEquals(2700, intFile.length()); - } - } - - @Test - @DisplayName("Test Get File With Space 1") - void testGetFileWithSpace1() throws Exception { - File intFile = new ClassPathResource("datavec-api/csvsequence test.txt").getFile(); - assertTrue(intFile.exists()); - if (isWindows) { - assertThat(intFile.length(), anyOf(equalTo(60L), equalTo(64L))); - } else { - assertEquals(60, intFile.length()); - } - } - - @Test - @DisplayName("Test Input Stream") - void testInputStream() throws Exception { - ClassPathResource resource = new ClassPathResource("datavec-api/csvsequence_1.txt"); - File intFile = resource.getFile(); - if (isWindows) { - assertThat(intFile.length(), anyOf(equalTo(60L), equalTo(64L))); - } else { - assertEquals(60, intFile.length()); - } - InputStream stream = resource.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); - String line = ""; - int cnt = 0; - while ((line = reader.readLine()) != null) { - cnt++; - } - assertEquals(5, cnt); - } - - @Test - @DisplayName("Test Input Stream Slash") - void testInputStreamSlash() throws Exception { - ClassPathResource resource = new ClassPathResource("datavec-api/csvsequence_1.txt"); - File intFile = resource.getFile(); - if (isWindows) { - assertThat(intFile.length(), anyOf(equalTo(60L), equalTo(64L))); - } else { - assertEquals(60, intFile.length()); - } - InputStream stream = resource.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); - String line = ""; - int cnt = 0; - while ((line = reader.readLine()) != null) { - cnt++; - } - assertEquals(5, cnt); - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/util/TimeSeriesUtilsTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/util/TimeSeriesUtilsTest.java deleted file mode 100644 index 4e35d06fe..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/util/TimeSeriesUtilsTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.util; - -import org.datavec.api.timeseries.util.TimeSeriesWritableUtils; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import java.util.ArrayList; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Time Series Utils Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class TimeSeriesUtilsTest extends BaseND4JTest { - - @Test - @DisplayName("Test Time Series Creation") - void testTimeSeriesCreation() { - List>> test = new ArrayList<>(); - List> timeStep = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - timeStep.add(getRecord(5)); - } - test.add(timeStep); - INDArray arr = TimeSeriesWritableUtils.convertWritablesSequence(test).getFirst(); - assertArrayEquals(new long[] { 1, 5, 5 }, arr.shape()); - } - - private List getRecord(int length) { - List ret = new ArrayList<>(); - for (int i = 0; i < length; i++) { - ret.add(new DoubleWritable(1.0)); - } - return ret; - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/writable/RecordConverterTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/writable/RecordConverterTest.java deleted file mode 100644 index a05897a92..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/writable/RecordConverterTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.writable; - -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.guava.collect.Lists; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.util.ndarray.RecordConverter; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import java.util.Arrays; -import java.util.List; -import java.util.TimeZone; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Record Converter Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class RecordConverterTest extends BaseND4JTest { - - @Test - @DisplayName("To Records _ Pass In Classification Data Set _ Expect ND Array And Int Writables") - void toRecords_PassInClassificationDataSet_ExpectNDArrayAndIntWritables() { - INDArray feature1 = Nd4j.create(new double[] { 4, -5.7, 10, -0.1 }, new long[] { 1, 4 }, DataType.FLOAT); - INDArray feature2 = Nd4j.create(new double[] { 11, .7, -1.3, 4 }, new long[] { 1, 4 }, DataType.FLOAT); - INDArray label1 = Nd4j.create(new double[] { 0, 0, 1, 0 }, new long[] { 1, 4 }, DataType.FLOAT); - INDArray label2 = Nd4j.create(new double[] { 0, 1, 0, 0 }, new long[] { 1, 4 }, DataType.FLOAT); - DataSet dataSet = new DataSet(Nd4j.vstack(Lists.newArrayList(feature1, feature2)), Nd4j.vstack(Lists.newArrayList(label1, label2))); - List> writableList = RecordConverter.toRecords(dataSet); - assertEquals(2, writableList.size()); - testClassificationWritables(feature1, 2, writableList.get(0)); - testClassificationWritables(feature2, 1, writableList.get(1)); - } - - @Test - @DisplayName("To Records _ Pass In Regression Data Set _ Expect ND Array And Double Writables") - void toRecords_PassInRegressionDataSet_ExpectNDArrayAndDoubleWritables() { - INDArray feature = Nd4j.create(new double[] { 4, -5.7, 10, -0.1 }, new long[] { 1, 4 }, DataType.FLOAT); - INDArray label = Nd4j.create(new double[] { .5, 2, 3, .5 }, new long[] { 1, 4 }, DataType.FLOAT); - DataSet dataSet = new DataSet(feature, label); - List> writableList = RecordConverter.toRecords(dataSet); - List results = writableList.get(0); - NDArrayWritable ndArrayWritable = (NDArrayWritable) results.get(0); - assertEquals(1, writableList.size()); - assertEquals(5, results.size()); - assertEquals(feature, ndArrayWritable.get()); - for (int i = 0; i < label.shape()[1]; i++) { - DoubleWritable doubleWritable = (DoubleWritable) results.get(i + 1); - assertEquals(label.getDouble(i), doubleWritable.get(), 0); - } - } - - private void testClassificationWritables(INDArray expectedFeatureVector, int expectLabelIndex, List writables) { - NDArrayWritable ndArrayWritable = (NDArrayWritable) writables.get(0); - IntWritable intWritable = (IntWritable) writables.get(1); - assertEquals(2, writables.size()); - assertEquals(expectedFeatureVector, ndArrayWritable.get()); - assertEquals(expectLabelIndex, intWritable.get()); - } - - @Test - @DisplayName("Test ND Array Writable Concat") - void testNDArrayWritableConcat() { - List l = Arrays.asList(new DoubleWritable(1), new NDArrayWritable(Nd4j.create(new double[] { 2, 3, 4 }, new long[] { 1, 3 }, DataType.FLOAT)), new DoubleWritable(5), new NDArrayWritable(Nd4j.create(new double[] { 6, 7, 8 }, new long[] { 1, 3 }, DataType.FLOAT)), new IntWritable(9), new IntWritable(1)); - INDArray exp = Nd4j.create(new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 1 }, new long[] { 1, 10 }, DataType.FLOAT); - INDArray act = RecordConverter.toArray(DataType.FLOAT, l); - assertEquals(exp, act); - } - - @Test - @DisplayName("Test ND Array Writable Concat To Matrix") - void testNDArrayWritableConcatToMatrix() { - List l1 = Arrays.asList(new DoubleWritable(1), new NDArrayWritable(Nd4j.create(new double[] { 2, 3, 4 }, new long[] { 1, 3 }, DataType.FLOAT)), new DoubleWritable(5)); - List l2 = Arrays.asList(new DoubleWritable(6), new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 }, new long[] { 1, 3 }, DataType.FLOAT)), new DoubleWritable(10)); - INDArray exp = Nd4j.create(new double[][] { { 1, 2, 3, 4, 5 }, { 6, 7, 8, 9, 10 } }).castTo(DataType.FLOAT); - INDArray act = RecordConverter.toMatrix(DataType.FLOAT, Arrays.asList(l1, l2)); - assertEquals(exp, act); - } - - @Test - @DisplayName("Test To Record With List Of Object") - void testToRecordWithListOfObject() { - final List list = Arrays.asList((Object) 3, 7.0f, "Foo", "Bar", 1.0, 3f, 3L, 7, 0L); - final Schema schema = new Schema.Builder().addColumnInteger("a").addColumnFloat("b").addColumnString("c").addColumnCategorical("d", "Bar", "Baz").addColumnDouble("e").addColumnFloat("f").addColumnLong("g").addColumnInteger("h").addColumnTime("i", TimeZone.getDefault()).build(); - final List record = RecordConverter.toRecord(schema, list); - assertEquals(record.get(0).toInt(), 3); - assertEquals(record.get(1).toFloat(), 7f, 1e-6); - assertEquals(record.get(2).toString(), "Foo"); - assertEquals(record.get(3).toString(), "Bar"); - assertEquals(record.get(4).toDouble(), 1.0, 1e-6); - assertEquals(record.get(5).toFloat(), 3f, 1e-6); - assertEquals(record.get(6).toLong(), 3L); - assertEquals(record.get(7).toInt(), 7); - assertEquals(record.get(8).toLong(), 0); - } -} diff --git a/datavec/datavec-api/src/test/java/org/datavec/api/writable/WritableTest.java b/datavec/datavec-api/src/test/java/org/datavec/api/writable/WritableTest.java deleted file mode 100644 index 1b97e8aef..000000000 --- a/datavec/datavec-api/src/test/java/org/datavec/api/writable/WritableTest.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.writable; - -import org.datavec.api.writable.batch.NDArrayRecordBatch; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.buffer.DataBuffer; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import org.junit.jupiter.api.DisplayName; - -import static org.junit.jupiter.api.Assertions.*; - -@DisplayName("Writable Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class WritableTest extends BaseND4JTest { - - @Test - @DisplayName("Test Writable Equality Reflexive") - void testWritableEqualityReflexive() { - assertEquals(new IntWritable(1), new IntWritable(1)); - assertEquals(new LongWritable(1), new LongWritable(1)); - assertEquals(new DoubleWritable(1), new DoubleWritable(1)); - assertEquals(new FloatWritable(1), new FloatWritable(1)); - assertEquals(new Text("Hello"), new Text("Hello")); - assertEquals(new BytesWritable("Hello".getBytes()), new BytesWritable("Hello".getBytes())); - INDArray ndArray = Nd4j.rand(new int[] { 1, 100 }); - assertEquals(new NDArrayWritable(ndArray), new NDArrayWritable(ndArray)); - assertEquals(new NullWritable(), new NullWritable()); - assertEquals(new BooleanWritable(true), new BooleanWritable(true)); - byte b = 0; - assertEquals(new ByteWritable(b), new ByteWritable(b)); - } - - @Test - @DisplayName("Test Bytes Writable Indexing") - void testBytesWritableIndexing() { - byte[] doubleWrite = new byte[16]; - ByteBuffer wrapped = ByteBuffer.wrap(doubleWrite); - Buffer buffer = (Buffer) wrapped; - wrapped.putDouble(1.0); - wrapped.putDouble(2.0); - buffer.rewind(); - BytesWritable byteWritable = new BytesWritable(doubleWrite); - assertEquals(2, byteWritable.getDouble(1), 1e-1); - DataBuffer dataBuffer = Nd4j.createBuffer(new double[] { 1, 2 }); - double[] d1 = dataBuffer.asDouble(); - double[] d2 = byteWritable.asNd4jBuffer(DataType.DOUBLE, 8).asDouble(); - assertArrayEquals(d1, d2, 0.0); - } - - @Test - @DisplayName("Test Byte Writable") - void testByteWritable() { - byte b = 0xfffffffe; - assertEquals(new IntWritable(-2), new ByteWritable(b)); - assertEquals(new LongWritable(-2), new ByteWritable(b)); - assertEquals(new ByteWritable(b), new IntWritable(-2)); - assertEquals(new ByteWritable(b), new LongWritable(-2)); - // those would cast to the same Int - byte minus126 = 0xffffff82; - assertNotEquals(new ByteWritable(minus126), new IntWritable(130)); - } - - @Test - @DisplayName("Test Int Long Writable") - void testIntLongWritable() { - assertEquals(new IntWritable(1), new LongWritable(1l)); - assertEquals(new LongWritable(2l), new IntWritable(2)); - long l = 1L << 34; - // those would cast to the same Int - assertNotEquals(new LongWritable(l), new IntWritable(4)); - } - - @Test - @DisplayName("Test Double Float Writable") - void testDoubleFloatWritable() { - assertEquals(new DoubleWritable(1d), new FloatWritable(1f)); - assertEquals(new FloatWritable(2f), new DoubleWritable(2d)); - // we defer to Java equality for Floats - assertNotEquals(new DoubleWritable(1.1d), new FloatWritable(1.1f)); - // same idea as above - assertNotEquals(new DoubleWritable(1.1d), new FloatWritable((float) 1.1d)); - assertNotEquals(new DoubleWritable((double) Float.MAX_VALUE + 1), new FloatWritable(Float.POSITIVE_INFINITY)); - } - - @Test - @DisplayName("Test Fuzzies") - void testFuzzies() { - assertTrue(new DoubleWritable(1.1d).fuzzyEquals(new FloatWritable(1.1f), 1e-6d)); - assertTrue(new FloatWritable(1.1f).fuzzyEquals(new DoubleWritable(1.1d), 1e-6d)); - byte b = 0xfffffffe; - assertTrue(new ByteWritable(b).fuzzyEquals(new DoubleWritable(-2.0), 1e-6d)); - assertFalse(new IntWritable(1).fuzzyEquals(new FloatWritable(1.1f), 1e-2d)); - assertTrue(new IntWritable(1).fuzzyEquals(new FloatWritable(1.05f), 1e-1d)); - assertTrue(new LongWritable(1).fuzzyEquals(new DoubleWritable(1.05f), 1e-1d)); - } - - @Test - @DisplayName("Test ND Array Record Batch") - void testNDArrayRecordBatch() { - Nd4j.getRandom().setSeed(12345); - // Outer list over writables/columns, inner list over examples - List> orig = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - orig.add(new ArrayList()); - } - for (int i = 0; i < 5; i++) { - orig.get(0).add(Nd4j.rand(1, 10)); - orig.get(1).add(Nd4j.rand(new int[] { 1, 5, 6 })); - orig.get(2).add(Nd4j.rand(new int[] { 1, 3, 4, 5 })); - } - // Outer list over examples, inner list over writables - List> origByExample = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - origByExample.add(Arrays.asList(orig.get(0).get(i), orig.get(1).get(i), orig.get(2).get(i))); - } - List batched = new ArrayList<>(); - for (List l : orig) { - batched.add(Nd4j.concat(0, l.toArray(new INDArray[5]))); - } - NDArrayRecordBatch batch = new NDArrayRecordBatch(batched); - assertEquals(5, batch.size()); - for (int i = 0; i < 5; i++) { - List act = batch.get(i); - List unboxed = new ArrayList<>(); - for (Writable w : act) { - unboxed.add(((NDArrayWritable) w).get()); - } - List exp = origByExample.get(i); - assertEquals(exp.size(), unboxed.size()); - for (int j = 0; j < exp.size(); j++) { - assertEquals(exp.get(j), unboxed.get(j)); - } - } - Iterator> iter = batch.iterator(); - int count = 0; - while (iter.hasNext()) { - List next = iter.next(); - List unboxed = new ArrayList<>(); - for (Writable w : next) { - unboxed.add(((NDArrayWritable) w).get()); - } - List exp = origByExample.get(count++); - assertEquals(exp.size(), unboxed.size()); - for (int j = 0; j < exp.size(); j++) { - assertEquals(exp.get(j), unboxed.get(j)); - } - } - assertEquals(5, count); - } -} diff --git a/datavec/datavec-arrow/pom.xml b/datavec/datavec-arrow/pom.xml deleted file mode 100644 index 626975817..000000000 --- a/datavec/datavec-arrow/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - - - 4.0.0 - - - org.datavec - datavec-parent - 1.0.0-SNAPSHOT - - - datavec-arrow - - datavec-arrow - - - - org.datavec - datavec-api - ${project.version} - - - org.apache.arrow - arrow-vector - ${arrow.version} - - - org.apache.arrow - arrow-memory - ${arrow.version} - - - org.apache.arrow - arrow-format - ${arrow.version} - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/ArrowConverterTest.java b/datavec/datavec-arrow/src/test/java/org/datavec/arrow/ArrowConverterTest.java deleted file mode 100644 index c21bd3e32..000000000 --- a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/ArrowConverterTest.java +++ /dev/null @@ -1,441 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.arrow; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.TimeStampMilliVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.VectorUnloader; -import org.apache.arrow.vector.ipc.ArrowFileWriter; -import org.apache.arrow.vector.types.FloatingPointPrecision; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.datavec.api.records.Record; -import org.datavec.api.records.metadata.RecordMetaData; -import org.datavec.api.records.metadata.RecordMetaDataIndex; -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.transform.ColumnType; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.*; -import org.datavec.arrow.recordreader.ArrowRecordReader; -import org.datavec.arrow.recordreader.ArrowWritableRecordBatch; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.primitives.Pair; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.*; -import static java.nio.channels.Channels.newChannel; -import static org.junit.jupiter.api.Assertions.*; - -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Arrow Converter Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class ArrowConverterTest extends BaseND4JTest { - - private static BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE); - - @TempDir - public Path testDir; - - @Test - @DisplayName("Test To Array From IND Array") - void testToArrayFromINDArray() { - Schema.Builder schemaBuilder = new Schema.Builder(); - schemaBuilder.addColumnNDArray("outputArray", new long[] { 1, 4 }); - Schema schema = schemaBuilder.build(); - int numRows = 4; - List> ret = new ArrayList<>(numRows); - for (int i = 0; i < numRows; i++) { - ret.add(Arrays.asList(new NDArrayWritable(Nd4j.linspace(1, 4, 4).reshape(1, 4)))); - } - List fieldVectors = ArrowConverter.toArrowColumns(bufferAllocator, schema, ret); - ArrowWritableRecordBatch arrowWritableRecordBatch = new ArrowWritableRecordBatch(fieldVectors, schema); - INDArray array = ArrowConverter.toArray(arrowWritableRecordBatch); - assertArrayEquals(new long[] { 4, 4 }, array.shape()); - INDArray assertion = Nd4j.repeat(Nd4j.linspace(1, 4, 4), 4).reshape(4, 4); - assertEquals(assertion, array); - } - - @Test - @DisplayName("Test Arrow Column IND Array") - void testArrowColumnINDArray() { - Schema.Builder schema = new Schema.Builder(); - List single = new ArrayList<>(); - int numCols = 2; - INDArray arr = Nd4j.linspace(1, 4, 4); - for (int i = 0; i < numCols; i++) { - schema.addColumnNDArray(String.valueOf(i), new long[] { 1, 4 }); - single.add(String.valueOf(i)); - } - Schema buildSchema = schema.build(); - List> list = new ArrayList<>(); - List firstRow = new ArrayList<>(); - for (int i = 0; i < numCols; i++) { - firstRow.add(new NDArrayWritable(arr)); - } - list.add(firstRow); - List fieldVectors = ArrowConverter.toArrowColumns(bufferAllocator, buildSchema, list); - assertEquals(numCols, fieldVectors.size()); - assertEquals(1, fieldVectors.get(0).getValueCount()); - assertFalse(fieldVectors.get(0).isNull(0)); - ArrowWritableRecordBatch arrowWritableRecordBatch = ArrowConverter.toArrowWritables(fieldVectors, buildSchema); - assertEquals(1, arrowWritableRecordBatch.size()); - Writable writable = arrowWritableRecordBatch.get(0).get(0); - assertTrue(writable instanceof NDArrayWritable); - NDArrayWritable ndArrayWritable = (NDArrayWritable) writable; - assertEquals(arr, ndArrayWritable.get()); - Writable writable1 = ArrowConverter.fromEntry(0, fieldVectors.get(0), ColumnType.NDArray); - NDArrayWritable ndArrayWritablewritable1 = (NDArrayWritable) writable1; - System.out.println(ndArrayWritablewritable1.get()); - } - - @Test - @DisplayName("Test Arrow Column String") - void testArrowColumnString() { - Schema.Builder schema = new Schema.Builder(); - List single = new ArrayList<>(); - for (int i = 0; i < 2; i++) { - schema.addColumnInteger(String.valueOf(i)); - single.add(String.valueOf(i)); - } - List fieldVectors = ArrowConverter.toArrowColumnsStringSingle(bufferAllocator, schema.build(), single); - List> records = ArrowConverter.toArrowWritables(fieldVectors, schema.build()); - List> assertion = new ArrayList<>(); - assertion.add(Arrays.asList(new IntWritable(0), new IntWritable(1))); - assertEquals(assertion, records); - List> batch = new ArrayList<>(); - for (int i = 0; i < 2; i++) { - batch.add(Arrays.asList(String.valueOf(i), String.valueOf(i))); - } - List fieldVectorsBatch = ArrowConverter.toArrowColumnsString(bufferAllocator, schema.build(), batch); - List> batchRecords = ArrowConverter.toArrowWritables(fieldVectorsBatch, schema.build()); - List> assertionBatch = new ArrayList<>(); - assertionBatch.add(Arrays.asList(new IntWritable(0), new IntWritable(0))); - assertionBatch.add(Arrays.asList(new IntWritable(1), new IntWritable(1))); - assertEquals(assertionBatch, batchRecords); - } - - @Test - @DisplayName("Test Arrow Batch Set Time") - void testArrowBatchSetTime() { - Schema.Builder schema = new Schema.Builder(); - List single = new ArrayList<>(); - for (int i = 0; i < 2; i++) { - schema.addColumnTime(String.valueOf(i), TimeZone.getDefault()); - single.add(String.valueOf(i)); - } - List> input = Arrays.asList(Arrays.asList(new LongWritable(0), new LongWritable(1)), Arrays.asList(new LongWritable(2), new LongWritable(3))); - List fieldVector = ArrowConverter.toArrowColumns(bufferAllocator, schema.build(), input); - ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector, schema.build()); - List assertion = Arrays.asList(new LongWritable(4), new LongWritable(5)); - writableRecordBatch.set(1, Arrays.asList(new LongWritable(4), new LongWritable(5))); - List recordTest = writableRecordBatch.get(1); - assertEquals(assertion, recordTest); - } - - @Test - @DisplayName("Test Arrow Batch Set") - void testArrowBatchSet() { - Schema.Builder schema = new Schema.Builder(); - List single = new ArrayList<>(); - for (int i = 0; i < 2; i++) { - schema.addColumnInteger(String.valueOf(i)); - single.add(String.valueOf(i)); - } - List> input = Arrays.asList(Arrays.asList(new IntWritable(0), new IntWritable(1)), Arrays.asList(new IntWritable(2), new IntWritable(3))); - List fieldVector = ArrowConverter.toArrowColumns(bufferAllocator, schema.build(), input); - ArrowWritableRecordBatch writableRecordBatch = new ArrowWritableRecordBatch(fieldVector, schema.build()); - List assertion = Arrays.asList(new IntWritable(4), new IntWritable(5)); - writableRecordBatch.set(1, Arrays.asList(new IntWritable(4), new IntWritable(5))); - List recordTest = writableRecordBatch.get(1); - assertEquals(assertion, recordTest); - } - - @Test - @DisplayName("Test Arrow Columns String Time Series") - void testArrowColumnsStringTimeSeries() { - Schema.Builder schema = new Schema.Builder(); - List>> entries = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - schema.addColumnInteger(String.valueOf(i)); - } - for (int i = 0; i < 5; i++) { - List> arr = Arrays.asList(Arrays.asList(String.valueOf(i), String.valueOf(i), String.valueOf(i))); - entries.add(arr); - } - List fieldVectors = ArrowConverter.toArrowColumnsStringTimeSeries(bufferAllocator, schema.build(), entries); - assertEquals(3, fieldVectors.size()); - assertEquals(5, fieldVectors.get(0).getValueCount()); - INDArray exp = Nd4j.create(5, 3); - for (int i = 0; i < 5; i++) { - exp.getRow(i).assign(i); - } - // Convert to ArrowWritableRecordBatch - note we can't do this in general with time series... - ArrowWritableRecordBatch wri = ArrowConverter.toArrowWritables(fieldVectors, schema.build()); - INDArray arr = ArrowConverter.toArray(wri); - assertArrayEquals(new long[] { 5, 3 }, arr.shape()); - assertEquals(exp, arr); - } - - @Test - @DisplayName("Test Convert Vector") - void testConvertVector() { - Schema.Builder schema = new Schema.Builder(); - List>> entries = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - schema.addColumnInteger(String.valueOf(i)); - } - for (int i = 0; i < 5; i++) { - List> arr = Arrays.asList(Arrays.asList(String.valueOf(i), String.valueOf(i), String.valueOf(i))); - entries.add(arr); - } - List fieldVectors = ArrowConverter.toArrowColumnsStringTimeSeries(bufferAllocator, schema.build(), entries); - INDArray arr = ArrowConverter.convertArrowVector(fieldVectors.get(0), schema.build().getType(0)); - assertEquals(5, arr.length()); - } - - @Test - @DisplayName("Test Create ND Array") - void testCreateNDArray() throws Exception { - val recordsToWrite = recordToWrite(); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(), recordsToWrite.getFirst(), byteArrayOutputStream); - File f = testDir.toFile(); - File tmpFile = new File(f, "tmp-arrow-file-" + UUID.randomUUID().toString() + ".arrorw"); - FileOutputStream outputStream = new FileOutputStream(tmpFile); - tmpFile.deleteOnExit(); - ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(), recordsToWrite.getFirst(), outputStream); - outputStream.flush(); - outputStream.close(); - Pair schemaArrowWritableRecordBatchPair = ArrowConverter.readFromFile(tmpFile); - assertEquals(recordsToWrite.getFirst(), schemaArrowWritableRecordBatchPair.getFirst()); - assertEquals(recordsToWrite.getRight(), schemaArrowWritableRecordBatchPair.getRight().toArrayList()); - byte[] arr = byteArrayOutputStream.toByteArray(); - val read = ArrowConverter.readFromBytes(arr); - assertEquals(recordsToWrite, read); - // send file - File tmp = tmpDataFile(recordsToWrite); - ArrowRecordReader recordReader = new ArrowRecordReader(); - recordReader.initialize(new FileSplit(tmp)); - recordReader.next(); - ArrowWritableRecordBatch currentBatch = recordReader.getCurrentBatch(); - INDArray arr2 = ArrowConverter.toArray(currentBatch); - assertEquals(2, arr2.rows()); - assertEquals(2, arr2.columns()); - } - - @Test - @DisplayName("Test Convert To Arrow Vectors") - void testConvertToArrowVectors() { - INDArray matrix = Nd4j.linspace(1, 4, 4).reshape(2, 2); - val vectors = ArrowConverter.convertToArrowVector(matrix, Arrays.asList("test", "test2"), ColumnType.Double, bufferAllocator); - assertEquals(matrix.rows(), vectors.size()); - INDArray vector = Nd4j.linspace(1, 4, 4); - val vectors2 = ArrowConverter.convertToArrowVector(vector, Arrays.asList("test"), ColumnType.Double, bufferAllocator); - assertEquals(1, vectors2.size()); - assertEquals(matrix.length(), vectors2.get(0).getValueCount()); - } - - @Test - @DisplayName("Test Schema Conversion Basic") - void testSchemaConversionBasic() { - Schema.Builder schemaBuilder = new Schema.Builder(); - for (int i = 0; i < 2; i++) { - schemaBuilder.addColumnDouble("test-" + i); - schemaBuilder.addColumnInteger("testi-" + i); - schemaBuilder.addColumnLong("testl-" + i); - schemaBuilder.addColumnFloat("testf-" + i); - } - Schema schema = schemaBuilder.build(); - val schema2 = ArrowConverter.toArrowSchema(schema); - assertEquals(8, schema2.getFields().size()); - val convertedSchema = ArrowConverter.toDatavecSchema(schema2); - assertEquals(schema, convertedSchema); - } - - @Test - @DisplayName("Test Read Schema And Records From Byte Array") - void testReadSchemaAndRecordsFromByteArray() throws Exception { - BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); - int valueCount = 3; - List fields = new ArrayList<>(); - fields.add(ArrowConverter.field("field1", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE))); - fields.add(ArrowConverter.intField("field2")); - List fieldVectors = new ArrayList<>(); - fieldVectors.add(ArrowConverter.vectorFor(allocator, "field1", new float[] { 1, 2, 3 })); - fieldVectors.add(ArrowConverter.vectorFor(allocator, "field2", new int[] { 1, 2, 3 })); - org.apache.arrow.vector.types.pojo.Schema schema = new org.apache.arrow.vector.types.pojo.Schema(fields); - VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, valueCount); - VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1); - vectorUnloader.getRecordBatch(); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - try (ArrowFileWriter arrowFileWriter = new ArrowFileWriter(schemaRoot1, null, newChannel(byteArrayOutputStream))) { - arrowFileWriter.writeBatch(); - } catch (IOException e) { - log.error("", e); - } - byte[] arr = byteArrayOutputStream.toByteArray(); - val arr2 = ArrowConverter.readFromBytes(arr); - assertEquals(2, arr2.getFirst().numColumns()); - assertEquals(3, arr2.getRight().size()); - val arrowCols = ArrowConverter.toArrowColumns(allocator, arr2.getFirst(), arr2.getRight()); - assertEquals(2, arrowCols.size()); - assertEquals(valueCount, arrowCols.get(0).getValueCount()); - } - - @Test - @DisplayName("Test Vector For Edge Cases") - void testVectorForEdgeCases() { - BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); - val vector = ArrowConverter.vectorFor(allocator, "field1", new float[] { Float.MIN_VALUE, Float.MAX_VALUE }); - assertEquals(Float.MIN_VALUE, vector.get(0), 1e-2); - assertEquals(Float.MAX_VALUE, vector.get(1), 1e-2); - val vectorInt = ArrowConverter.vectorFor(allocator, "field1", new int[] { Integer.MIN_VALUE, Integer.MAX_VALUE }); - assertEquals(Integer.MIN_VALUE, vectorInt.get(0), 1e-2); - assertEquals(Integer.MAX_VALUE, vectorInt.get(1), 1e-2); - } - - @Test - @DisplayName("Test Vector For") - void testVectorFor() { - BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); - val vector = ArrowConverter.vectorFor(allocator, "field1", new float[] { 1, 2, 3 }); - assertEquals(3, vector.getValueCount()); - assertEquals(1, vector.get(0), 1e-2); - assertEquals(2, vector.get(1), 1e-2); - assertEquals(3, vector.get(2), 1e-2); - val vectorLong = ArrowConverter.vectorFor(allocator, "field1", new long[] { 1, 2, 3 }); - assertEquals(3, vectorLong.getValueCount()); - assertEquals(1, vectorLong.get(0), 1e-2); - assertEquals(2, vectorLong.get(1), 1e-2); - assertEquals(3, vectorLong.get(2), 1e-2); - val vectorInt = ArrowConverter.vectorFor(allocator, "field1", new int[] { 1, 2, 3 }); - assertEquals(3, vectorInt.getValueCount()); - assertEquals(1, vectorInt.get(0), 1e-2); - assertEquals(2, vectorInt.get(1), 1e-2); - assertEquals(3, vectorInt.get(2), 1e-2); - val vectorDouble = ArrowConverter.vectorFor(allocator, "field1", new double[] { 1, 2, 3 }); - assertEquals(3, vectorDouble.getValueCount()); - assertEquals(1, vectorDouble.get(0), 1e-2); - assertEquals(2, vectorDouble.get(1), 1e-2); - assertEquals(3, vectorDouble.get(2), 1e-2); - val vectorBool = ArrowConverter.vectorFor(allocator, "field1", new boolean[] { true, true, false }); - assertEquals(3, vectorBool.getValueCount()); - assertEquals(1, vectorBool.get(0), 1e-2); - assertEquals(1, vectorBool.get(1), 1e-2); - assertEquals(0, vectorBool.get(2), 1e-2); - } - - @Test - @DisplayName("Test Record Reader And Write File") - void testRecordReaderAndWriteFile() throws Exception { - val recordsToWrite = recordToWrite(); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(), recordsToWrite.getFirst(), byteArrayOutputStream); - byte[] arr = byteArrayOutputStream.toByteArray(); - val read = ArrowConverter.readFromBytes(arr); - assertEquals(recordsToWrite, read); - // send file - File tmp = tmpDataFile(recordsToWrite); - RecordReader recordReader = new ArrowRecordReader(); - recordReader.initialize(new FileSplit(tmp)); - List record = recordReader.next(); - assertEquals(2, record.size()); - } - - @Test - @DisplayName("Test Record Reader Meta Data List") - void testRecordReaderMetaDataList() throws Exception { - val recordsToWrite = recordToWrite(); - // send file - File tmp = tmpDataFile(recordsToWrite); - RecordReader recordReader = new ArrowRecordReader(); - RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0, tmp.toURI(), ArrowRecordReader.class); - recordReader.loadFromMetaData(Arrays.asList(recordMetaDataIndex)); - Record record = recordReader.nextRecord(); - assertEquals(2, record.getRecord().size()); - } - - @Test - @DisplayName("Test Dates") - void testDates() { - Date now = new Date(); - BufferAllocator bufferAllocator = new RootAllocator(Long.MAX_VALUE); - TimeStampMilliVector timeStampMilliVector = ArrowConverter.vectorFor(bufferAllocator, "col1", new Date[] { now }); - assertEquals(now.getTime(), timeStampMilliVector.get(0)); - } - - @Test - @DisplayName("Test Record Reader Meta Data") - void testRecordReaderMetaData() throws Exception { - val recordsToWrite = recordToWrite(); - // send file - File tmp = tmpDataFile(recordsToWrite); - RecordReader recordReader = new ArrowRecordReader(); - RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0, tmp.toURI(), ArrowRecordReader.class); - recordReader.loadFromMetaData(recordMetaDataIndex); - Record record = recordReader.nextRecord(); - assertEquals(2, record.getRecord().size()); - } - - private File tmpDataFile(Pair>> recordsToWrite) throws IOException { - File f = testDir.toFile(); - // send file - File tmp = new File(f, "tmp-file-" + UUID.randomUUID().toString()); - tmp.mkdirs(); - File tmpFile = new File(tmp, "data.arrow"); - tmpFile.deleteOnExit(); - FileOutputStream bufferedOutputStream = new FileOutputStream(tmpFile); - ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(), recordsToWrite.getFirst(), bufferedOutputStream); - bufferedOutputStream.flush(); - bufferedOutputStream.close(); - return tmp; - } - - private Pair>> recordToWrite() { - List> records = new ArrayList<>(); - records.add(Arrays.asList(new DoubleWritable(0.0), new DoubleWritable(0.0))); - records.add(Arrays.asList(new DoubleWritable(0.0), new DoubleWritable(0.0))); - Schema.Builder schemaBuilder = new Schema.Builder(); - for (int i = 0; i < 2; i++) { - schemaBuilder.addColumnFloat("col-" + i); - } - return Pair.of(schemaBuilder.build(), records); - } -} diff --git a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/RecordMapperTest.java b/datavec/datavec-arrow/src/test/java/org/datavec/arrow/RecordMapperTest.java deleted file mode 100644 index fbbc71f9f..000000000 --- a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/RecordMapperTest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.arrow; - -import lombok.val; -import org.apache.commons.io.FileUtils; -import org.datavec.api.records.mapper.RecordMapper; -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.records.writer.impl.csv.CSVRecordWriter; -import org.datavec.api.split.FileSplit; -import org.datavec.api.split.InputSplit; -import org.datavec.api.split.partition.NumberOfRecordsPartitioner; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.Writable; -import org.datavec.arrow.recordreader.ArrowRecordReader; -import org.datavec.arrow.recordreader.ArrowRecordWriter; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.BaseND4JTest; -import org.nd4j.common.primitives.Triple; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Record Mapper Test") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class RecordMapperTest extends BaseND4JTest { - - @Test - @DisplayName("Test Multi Write") - void testMultiWrite() throws Exception { - val recordsPair = records(); - Path p = Files.createTempFile("arrowwritetest", ".arrow"); - FileUtils.write(p.toFile(), recordsPair.getFirst()); - p.toFile().deleteOnExit(); - int numReaders = 2; - RecordReader[] readers = new RecordReader[numReaders]; - InputSplit[] splits = new InputSplit[numReaders]; - for (int i = 0; i < readers.length; i++) { - FileSplit split = new FileSplit(p.toFile()); - ArrowRecordReader arrowRecordReader = new ArrowRecordReader(); - readers[i] = arrowRecordReader; - splits[i] = split; - } - ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(recordsPair.getMiddle()); - FileSplit split = new FileSplit(p.toFile()); - arrowRecordWriter.initialize(split, new NumberOfRecordsPartitioner()); - arrowRecordWriter.writeBatch(recordsPair.getRight()); - CSVRecordWriter csvRecordWriter = new CSVRecordWriter(); - Path p2 = Files.createTempFile("arrowwritetest", ".csv"); - FileUtils.write(p2.toFile(), recordsPair.getFirst()); - p.toFile().deleteOnExit(); - FileSplit outputCsv = new FileSplit(p2.toFile()); - RecordMapper mapper = RecordMapper.builder().batchSize(10).inputUrl(split).outputUrl(outputCsv).partitioner(new NumberOfRecordsPartitioner()).readersToConcat(readers).splitPerReader(splits).recordWriter(csvRecordWriter).build(); - mapper.copy(); - } - - @Test - @DisplayName("Test Copy From Arrow To Csv") - void testCopyFromArrowToCsv() throws Exception { - val recordsPair = records(); - Path p = Files.createTempFile("arrowwritetest", ".arrow"); - FileUtils.write(p.toFile(), recordsPair.getFirst()); - p.toFile().deleteOnExit(); - ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(recordsPair.getMiddle()); - FileSplit split = new FileSplit(p.toFile()); - arrowRecordWriter.initialize(split, new NumberOfRecordsPartitioner()); - arrowRecordWriter.writeBatch(recordsPair.getRight()); - ArrowRecordReader arrowRecordReader = new ArrowRecordReader(); - arrowRecordReader.initialize(split); - CSVRecordWriter csvRecordWriter = new CSVRecordWriter(); - Path p2 = Files.createTempFile("arrowwritetest", ".csv"); - FileUtils.write(p2.toFile(), recordsPair.getFirst()); - p.toFile().deleteOnExit(); - FileSplit outputCsv = new FileSplit(p2.toFile()); - RecordMapper mapper = RecordMapper.builder().batchSize(10).inputUrl(split).outputUrl(outputCsv).partitioner(new NumberOfRecordsPartitioner()).recordReader(arrowRecordReader).recordWriter(csvRecordWriter).build(); - mapper.copy(); - CSVRecordReader recordReader = new CSVRecordReader(); - recordReader.initialize(outputCsv); - List> loadedCSvRecords = recordReader.next(10); - assertEquals(10, loadedCSvRecords.size()); - } - - @Test - @DisplayName("Test Copy From Csv To Arrow") - void testCopyFromCsvToArrow() throws Exception { - val recordsPair = records(); - Path p = Files.createTempFile("csvwritetest", ".csv"); - FileUtils.write(p.toFile(), recordsPair.getFirst()); - p.toFile().deleteOnExit(); - CSVRecordReader recordReader = new CSVRecordReader(); - FileSplit fileSplit = new FileSplit(p.toFile()); - ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(recordsPair.getMiddle()); - File outputFile = Files.createTempFile("outputarrow", "arrow").toFile(); - FileSplit outputFileSplit = new FileSplit(outputFile); - RecordMapper mapper = RecordMapper.builder().batchSize(10).inputUrl(fileSplit).outputUrl(outputFileSplit).partitioner(new NumberOfRecordsPartitioner()).recordReader(recordReader).recordWriter(arrowRecordWriter).build(); - mapper.copy(); - ArrowRecordReader arrowRecordReader = new ArrowRecordReader(); - arrowRecordReader.initialize(outputFileSplit); - List> next = arrowRecordReader.next(10); - System.out.println(next); - assertEquals(10, next.size()); - } - - private Triple>> records() { - List> list = new ArrayList<>(); - StringBuilder sb = new StringBuilder(); - int numColumns = 3; - for (int i = 0; i < 10; i++) { - List temp = new ArrayList<>(); - for (int j = 0; j < numColumns; j++) { - int v = 100 * i + j; - temp.add(new IntWritable(v)); - sb.append(v); - if (j < 2) - sb.append(","); - else if (i != 9) - sb.append("\n"); - } - list.add(temp); - } - Schema.Builder schemaBuilder = new Schema.Builder(); - for (int i = 0; i < numColumns; i++) { - schemaBuilder.addColumnInteger(String.valueOf(i)); - } - return Triple.of(sb.toString(), schemaBuilder.build(), list); - } -} diff --git a/datavec/datavec-data/datavec-data-image/pom.xml b/datavec/datavec-data/datavec-data-image/pom.xml deleted file mode 100644 index 0bc71b7d2..000000000 --- a/datavec/datavec-data/datavec-data-image/pom.xml +++ /dev/null @@ -1,140 +0,0 @@ - - - - - - 4.0.0 - - - org.datavec - datavec-data - 1.0.0-SNAPSHOT - - - datavec-data-image - - - - org.datavec - datavec-api - - - com.github.jai-imageio - jai-imageio-core - 1.3.0 - - - com.twelvemonkeys.imageio - imageio-jpeg - 3.1.1 - - - com.twelvemonkeys.imageio - imageio-tiff - 3.1.1 - - - com.twelvemonkeys.imageio - imageio-psd - 3.1.1 - - - com.twelvemonkeys.imageio - imageio-bmp - 3.1.1 - - - com.google.android - android - 4.1.1.4 - - - * - * - - - true - - - org.bytedeco - javacpp - ${javacpp.version} - - - org.bytedeco - javacv - ${javacv.version} - - - org.bytedeco - opencv-platform - ${opencv.version}-${javacpp-presets.version} - - - org.bytedeco - leptonica-platform - ${leptonica.version}-${javacpp-presets.version} - - - org.bytedeco - hdf5-platform - ${hdf5.version}-${javacpp-presets.version} - - - org.bytedeco - ffmpeg-platform - ${ffmpeg.version}-${javacpp.version} - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - com.google.android:android - - - - - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/FileBatchRecordReaderTest.java b/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/FileBatchRecordReaderTest.java deleted file mode 100644 index 8956cbaf9..000000000 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/recordreader/FileBatchRecordReaderTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.image.recordreader; - -import org.apache.commons.io.FileUtils; -import org.datavec.api.io.labels.ParentPathLabelGenerator; -import org.datavec.api.io.labels.PathLabelGenerator; -import org.datavec.api.records.reader.impl.filebatch.FileBatchRecordReader; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.NDArrayWritable; -import org.datavec.api.writable.Writable; -import org.datavec.image.loader.NativeImageLoader; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.loader.FileBatch; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.common.io.ClassPathResource; -import java.io.File; -import java.util.*; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("File Batch Record Reader Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class FileBatchRecordReaderTest { - - @TempDir - public Path testDir; - - @Test - @DisplayName("Test Csv") - void testCsv(@TempDir Path testDir,@TempDir Path baseDirPath) throws Exception { - File extractedSourceDir = testDir.toFile(); - new ClassPathResource("datavec-data-image/testimages").copyDirectory(extractedSourceDir); - File baseDir = baseDirPath.toFile(); - List c = new ArrayList<>(FileUtils.listFiles(extractedSourceDir, null, true)); - assertEquals(6, c.size()); - Collections.sort(c, new Comparator() { - - @Override - public int compare(File o1, File o2) { - return o1.getPath().compareTo(o2.getPath()); - } - }); - FileBatch fb = FileBatch.forFiles(c); - File saveFile = new File(baseDir, "saved.zip"); - fb.writeAsZip(saveFile); - fb = FileBatch.readFromZip(saveFile); - PathLabelGenerator labelMaker = new ParentPathLabelGenerator(); - ImageRecordReader rr = new ImageRecordReader(32, 32, 1, labelMaker); - rr.setLabels(Arrays.asList("class0", "class1")); - FileBatchRecordReader fbrr = new FileBatchRecordReader(rr, fb); - NativeImageLoader il = new NativeImageLoader(32, 32, 1); - for (int test = 0; test < 3; test++) { - for (int i = 0; i < 6; i++) { - assertTrue(fbrr.hasNext()); - List next = fbrr.next(); - assertEquals(2, next.size()); - INDArray exp; - switch(i) { - case 0: - exp = il.asMatrix(new File(extractedSourceDir, "class0/0.jpg")); - break; - case 1: - exp = il.asMatrix(new File(extractedSourceDir, "class0/1.png")); - break; - case 2: - exp = il.asMatrix(new File(extractedSourceDir, "class0/2.jpg")); - break; - case 3: - exp = il.asMatrix(new File(extractedSourceDir, "class1/A.jpg")); - break; - case 4: - exp = il.asMatrix(new File(extractedSourceDir, "class1/B.png")); - break; - case 5: - exp = il.asMatrix(new File(extractedSourceDir, "class1/C.jpg")); - break; - default: - throw new RuntimeException(); - } - Writable expLabel = (i < 3 ? new IntWritable(0) : new IntWritable(1)); - assertEquals(((NDArrayWritable) next.get(0)).get(), exp); - assertEquals(expLabel, next.get(1)); - } - assertFalse(fbrr.hasNext()); - assertTrue(fbrr.resetSupported()); - fbrr.reset(); - } - } -} diff --git a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/JsonYamlTest.java b/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/JsonYamlTest.java deleted file mode 100644 index cbf4797a8..000000000 --- a/datavec/datavec-data/datavec-data-image/src/test/java/org/datavec/image/transform/JsonYamlTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.image.transform; - -import org.datavec.image.data.ImageWritable; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import java.io.IOException; -import java.util.List; -import java.util.Random; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Json Yaml Test") -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JACKSON_SERDE) -class JsonYamlTest { - - @Test - @DisplayName("Test Json Yaml Image Transform Process") - void testJsonYamlImageTransformProcess() throws IOException { - int seed = 12345; - Random random = new Random(seed); - // from org.bytedeco.javacpp.opencv_imgproc - int COLOR_BGR2Luv = 50; - int CV_BGR2GRAY = 6; - ImageTransformProcess itp = new ImageTransformProcess.Builder().colorConversionTransform(COLOR_BGR2Luv).cropImageTransform(10).equalizeHistTransform(CV_BGR2GRAY).flipImageTransform(0).resizeImageTransform(300, 300).rotateImageTransform(30).scaleImageTransform(3).warpImageTransform((float) 0.5).build(); - String asJson = itp.toJson(); - String asYaml = itp.toYaml(); - // System.out.println(asJson); - // System.out.println("\n\n\n"); - // System.out.println(asYaml); - ImageWritable img = TestImageTransform.makeRandomImage(0, 0, 3); - ImageWritable imgJson = new ImageWritable(img.getFrame().clone()); - ImageWritable imgYaml = new ImageWritable(img.getFrame().clone()); - ImageWritable imgAll = new ImageWritable(img.getFrame().clone()); - ImageTransformProcess itpFromJson = ImageTransformProcess.fromJson(asJson); - ImageTransformProcess itpFromYaml = ImageTransformProcess.fromYaml(asYaml); - List transformList = itp.getTransformList(); - List transformListJson = itpFromJson.getTransformList(); - List transformListYaml = itpFromYaml.getTransformList(); - for (int i = 0; i < transformList.size(); i++) { - ImageTransform it = transformList.get(i); - ImageTransform itJson = transformListJson.get(i); - ImageTransform itYaml = transformListYaml.get(i); - System.out.println(i + "\t" + it); - img = it.transform(img); - imgJson = itJson.transform(imgJson); - imgYaml = itYaml.transform(imgYaml); - if (it instanceof RandomCropTransform) { - assertTrue(img.getFrame().imageHeight == imgJson.getFrame().imageHeight); - assertTrue(img.getFrame().imageWidth == imgJson.getFrame().imageWidth); - assertTrue(img.getFrame().imageHeight == imgYaml.getFrame().imageHeight); - assertTrue(img.getFrame().imageWidth == imgYaml.getFrame().imageWidth); - } else if (it instanceof FilterImageTransform) { - assertEquals(img.getFrame().imageHeight, imgJson.getFrame().imageHeight); - assertEquals(img.getFrame().imageWidth, imgJson.getFrame().imageWidth); - assertEquals(img.getFrame().imageChannels, imgJson.getFrame().imageChannels); - assertEquals(img.getFrame().imageHeight, imgYaml.getFrame().imageHeight); - assertEquals(img.getFrame().imageWidth, imgYaml.getFrame().imageWidth); - assertEquals(img.getFrame().imageChannels, imgYaml.getFrame().imageChannels); - } else { - assertEquals(img, imgJson); - assertEquals(img, imgYaml); - } - } - imgAll = itp.execute(imgAll); - assertEquals(imgAll, img); - } -} diff --git a/datavec/datavec-data/pom.xml b/datavec/datavec-data/pom.xml deleted file mode 100644 index 8ed687669..000000000 --- a/datavec/datavec-data/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - 4.0.0 - - - org.datavec - datavec-parent - 1.0.0-SNAPSHOT - - - datavec-data - pom - - datavec-data - - - datavec-data-image - - - - - - org.datavec - datavec-api - ${project.version} - - - - - - - org.nd4j - nd4j-api - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/datavec/datavec-excel/pom.xml b/datavec/datavec-excel/pom.xml deleted file mode 100644 index cabad4fe8..000000000 --- a/datavec/datavec-excel/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - - - - 4.0.0 - - - org.datavec - datavec-parent - 1.0.0-SNAPSHOT - - - datavec-excel - - datavec-excel - - - - org.datavec - datavec-api - ${project.version} - - - - org.apache.poi - poi - ${poi.version} - - - - org.apache.poi - poi-ooxml - ${poi.version} - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/datavec/datavec-jdbc/datavecTests/README_DO_NOT_TOUCH_FILES.txt b/datavec/datavec-jdbc/datavecTests/README_DO_NOT_TOUCH_FILES.txt deleted file mode 100644 index a4bc14529..000000000 --- a/datavec/datavec-jdbc/datavecTests/README_DO_NOT_TOUCH_FILES.txt +++ /dev/null @@ -1,9 +0,0 @@ - -# ************************************************************************* -# *** DO NOT TOUCH FILES IN THIS DIRECTORY! *** -# *** FILES IN THIS DIRECTORY AND SUBDIRECTORIES CONSTITUTE A DERBY *** -# *** DATABASE, WHICH INCLUDES THE DATA (USER AND SYSTEM) AND THE *** -# *** FILES NECESSARY FOR DATABASE RECOVERY. *** -# *** EDITING, ADDING, OR DELETING ANY OF THESE FILES MAY CAUSE DATA *** -# *** CORRUPTION AND LEAVE THE DATABASE IN A NON-RECOVERABLE STATE. *** -# ************************************************************************* \ No newline at end of file diff --git a/datavec/datavec-jdbc/datavecTests/db.lck b/datavec/datavec-jdbc/datavecTests/db.lck deleted file mode 100644 index 38f385e2c..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/db.lck and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/log/README_DO_NOT_TOUCH_FILES.txt b/datavec/datavec-jdbc/datavecTests/log/README_DO_NOT_TOUCH_FILES.txt deleted file mode 100644 index 56df292f6..000000000 --- a/datavec/datavec-jdbc/datavecTests/log/README_DO_NOT_TOUCH_FILES.txt +++ /dev/null @@ -1,8 +0,0 @@ - -# ************************************************************************* -# *** DO NOT TOUCH FILES IN THIS DIRECTORY! *** -# *** FILES IN THIS DIRECTORY ARE USED BY THE DERBY DATABASE RECOVERY *** -# *** SYSTEM. EDITING, ADDING, OR DELETING FILES IN THIS DIRECTORY *** -# *** WILL CAUSE THE DERBY RECOVERY SYSTEM TO FAIL, LEADING TO *** -# *** NON-RECOVERABLE CORRUPT DATABASES. *** -# ************************************************************************* \ No newline at end of file diff --git a/datavec/datavec-jdbc/datavecTests/log/log.ctrl b/datavec/datavec-jdbc/datavecTests/log/log.ctrl deleted file mode 100644 index 1041ea725..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/log/log.ctrl and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/log/log1.dat b/datavec/datavec-jdbc/datavecTests/log/log1.dat deleted file mode 100644 index 847a9ecd5..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/log/log1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/log/logmirror.ctrl b/datavec/datavec-jdbc/datavecTests/log/logmirror.ctrl deleted file mode 100644 index 090e4db1c..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/log/logmirror.ctrl and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/README_DO_NOT_TOUCH_FILES.txt b/datavec/datavec-jdbc/datavecTests/seg0/README_DO_NOT_TOUCH_FILES.txt deleted file mode 100644 index 2bdad0612..000000000 --- a/datavec/datavec-jdbc/datavecTests/seg0/README_DO_NOT_TOUCH_FILES.txt +++ /dev/null @@ -1,8 +0,0 @@ - -# ************************************************************************* -# *** DO NOT TOUCH FILES IN THIS DIRECTORY! *** -# *** FILES IN THIS DIRECTORY ARE USED BY THE DERBY DATABASE TO STORE *** -# *** USER AND SYSTEM DATA. EDITING, ADDING, OR DELETING FILES IN THIS *** -# *** DIRECTORY WILL CORRUPT THE ASSOCIATED DERBY DATABASE AND MAKE *** -# *** IT NON-RECOVERABLE. *** -# ************************************************************************* \ No newline at end of file diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c10.dat b/datavec/datavec-jdbc/datavecTests/seg0/c10.dat deleted file mode 100644 index 55df5fd59..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c10.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c101.dat b/datavec/datavec-jdbc/datavecTests/seg0/c101.dat deleted file mode 100644 index 14c6b0f73..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c101.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c111.dat b/datavec/datavec-jdbc/datavecTests/seg0/c111.dat deleted file mode 100644 index 5b8e5cfad..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c111.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c121.dat b/datavec/datavec-jdbc/datavecTests/seg0/c121.dat deleted file mode 100644 index 92ed00eeb..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c121.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c130.dat b/datavec/datavec-jdbc/datavecTests/seg0/c130.dat deleted file mode 100644 index d775b95a0..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c130.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c141.dat b/datavec/datavec-jdbc/datavecTests/seg0/c141.dat deleted file mode 100644 index bf08ff674..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c141.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c150.dat b/datavec/datavec-jdbc/datavecTests/seg0/c150.dat deleted file mode 100644 index e2ea5767c..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c150.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c161.dat b/datavec/datavec-jdbc/datavecTests/seg0/c161.dat deleted file mode 100644 index 90960e6d4..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c161.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c171.dat b/datavec/datavec-jdbc/datavecTests/seg0/c171.dat deleted file mode 100644 index 55d40959a..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c171.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c180.dat b/datavec/datavec-jdbc/datavecTests/seg0/c180.dat deleted file mode 100644 index 8b6950618..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c180.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c191.dat b/datavec/datavec-jdbc/datavecTests/seg0/c191.dat deleted file mode 100644 index 5e31e3bed..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c191.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c1a1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c1a1.dat deleted file mode 100644 index e7013d6c6..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c1a1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c1b1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c1b1.dat deleted file mode 100644 index 25fee6b49..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c1b1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c1c0.dat b/datavec/datavec-jdbc/datavecTests/seg0/c1c0.dat deleted file mode 100644 index c5b91e2c3..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c1c0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c1d1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c1d1.dat deleted file mode 100644 index 451f02f45..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c1d1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c1e0.dat b/datavec/datavec-jdbc/datavecTests/seg0/c1e0.dat deleted file mode 100644 index 761408d3b..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c1e0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c1f1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c1f1.dat deleted file mode 100644 index 78d701f45..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c1f1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c20.dat b/datavec/datavec-jdbc/datavecTests/seg0/c20.dat deleted file mode 100644 index 3b91d36d4..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c20.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c200.dat b/datavec/datavec-jdbc/datavecTests/seg0/c200.dat deleted file mode 100644 index c3a7808dd..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c200.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c211.dat b/datavec/datavec-jdbc/datavecTests/seg0/c211.dat deleted file mode 100644 index 54e158695..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c211.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c221.dat b/datavec/datavec-jdbc/datavecTests/seg0/c221.dat deleted file mode 100644 index 59900bc01..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c221.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c230.dat b/datavec/datavec-jdbc/datavecTests/seg0/c230.dat deleted file mode 100644 index 97788f003..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c230.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c241.dat b/datavec/datavec-jdbc/datavecTests/seg0/c241.dat deleted file mode 100644 index be1d28d0a..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c241.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c251.dat b/datavec/datavec-jdbc/datavecTests/seg0/c251.dat deleted file mode 100644 index c6fab1e70..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c251.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c260.dat b/datavec/datavec-jdbc/datavecTests/seg0/c260.dat deleted file mode 100644 index 25f81fde7..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c260.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c271.dat b/datavec/datavec-jdbc/datavecTests/seg0/c271.dat deleted file mode 100644 index 51cde573e..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c271.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c281.dat b/datavec/datavec-jdbc/datavecTests/seg0/c281.dat deleted file mode 100644 index cfed875df..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c281.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c290.dat b/datavec/datavec-jdbc/datavecTests/seg0/c290.dat deleted file mode 100644 index a85589e54..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c290.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c2a1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c2a1.dat deleted file mode 100644 index 8e2ed6afe..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c2a1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c2b1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c2b1.dat deleted file mode 100644 index 2a2969247..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c2b1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c2c1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c2c1.dat deleted file mode 100644 index 5511575f6..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c2c1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c2d0.dat b/datavec/datavec-jdbc/datavecTests/seg0/c2d0.dat deleted file mode 100644 index 4adc6e447..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c2d0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c2e1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c2e1.dat deleted file mode 100644 index b37b9b254..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c2e1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c2f0.dat b/datavec/datavec-jdbc/datavecTests/seg0/c2f0.dat deleted file mode 100644 index d854b4b48..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c2f0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c300.dat b/datavec/datavec-jdbc/datavecTests/seg0/c300.dat deleted file mode 100644 index 2053e0105..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c300.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c31.dat b/datavec/datavec-jdbc/datavecTests/seg0/c31.dat deleted file mode 100644 index 36b2b77ae..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c31.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c311.dat b/datavec/datavec-jdbc/datavecTests/seg0/c311.dat deleted file mode 100644 index f60c260f8..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c311.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c321.dat b/datavec/datavec-jdbc/datavecTests/seg0/c321.dat deleted file mode 100644 index a9d745366..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c321.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c331.dat b/datavec/datavec-jdbc/datavecTests/seg0/c331.dat deleted file mode 100644 index 85ee72b31..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c331.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c340.dat b/datavec/datavec-jdbc/datavecTests/seg0/c340.dat deleted file mode 100644 index d99b11a3f..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c340.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c351.dat b/datavec/datavec-jdbc/datavecTests/seg0/c351.dat deleted file mode 100644 index f822f4cb0..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c351.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c361.dat b/datavec/datavec-jdbc/datavecTests/seg0/c361.dat deleted file mode 100644 index b5c8f259d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c361.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c371.dat b/datavec/datavec-jdbc/datavecTests/seg0/c371.dat deleted file mode 100644 index ad11f01b2..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c371.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c380.dat b/datavec/datavec-jdbc/datavecTests/seg0/c380.dat deleted file mode 100644 index e0969191f..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c380.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c391.dat b/datavec/datavec-jdbc/datavecTests/seg0/c391.dat deleted file mode 100644 index 8ae566785..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c391.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c3a1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c3a1.dat deleted file mode 100644 index 44d86e55e..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c3a1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c3b1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c3b1.dat deleted file mode 100644 index 1bdf6bc7f..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c3b1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c3c0.dat b/datavec/datavec-jdbc/datavecTests/seg0/c3c0.dat deleted file mode 100644 index 4d061cf06..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c3c0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c3d1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c3d1.dat deleted file mode 100644 index 45c9fa244..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c3d1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c3e1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c3e1.dat deleted file mode 100644 index 48f53e682..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c3e1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c3f1.dat b/datavec/datavec-jdbc/datavecTests/seg0/c3f1.dat deleted file mode 100644 index 08acdcee3..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c3f1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c400.dat b/datavec/datavec-jdbc/datavecTests/seg0/c400.dat deleted file mode 100644 index 1e8976f89..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c400.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c41.dat b/datavec/datavec-jdbc/datavecTests/seg0/c41.dat deleted file mode 100644 index ba33dd45e..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c41.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c411.dat b/datavec/datavec-jdbc/datavecTests/seg0/c411.dat deleted file mode 100644 index 8aba2fb6b..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c411.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c421.dat b/datavec/datavec-jdbc/datavecTests/seg0/c421.dat deleted file mode 100644 index 65775ee7b..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c421.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c430.dat b/datavec/datavec-jdbc/datavecTests/seg0/c430.dat deleted file mode 100644 index 55c948db3..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c430.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c441.dat b/datavec/datavec-jdbc/datavecTests/seg0/c441.dat deleted file mode 100644 index 3948b2a3c..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c441.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c451.dat b/datavec/datavec-jdbc/datavecTests/seg0/c451.dat deleted file mode 100644 index fe1ab73e1..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c451.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c461.dat b/datavec/datavec-jdbc/datavecTests/seg0/c461.dat deleted file mode 100644 index e6d98541f..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c461.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c470.dat b/datavec/datavec-jdbc/datavecTests/seg0/c470.dat deleted file mode 100644 index c9f2eb1ca..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c470.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c481.dat b/datavec/datavec-jdbc/datavecTests/seg0/c481.dat deleted file mode 100644 index 397b29172..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c481.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c51.dat b/datavec/datavec-jdbc/datavecTests/seg0/c51.dat deleted file mode 100644 index 275dc512b..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c51.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c60.dat b/datavec/datavec-jdbc/datavecTests/seg0/c60.dat deleted file mode 100644 index 6e07040ff..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c60.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c71.dat b/datavec/datavec-jdbc/datavecTests/seg0/c71.dat deleted file mode 100644 index f19c0b854..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c71.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c81.dat b/datavec/datavec-jdbc/datavecTests/seg0/c81.dat deleted file mode 100644 index 8a1494e5b..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c81.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c830.dat b/datavec/datavec-jdbc/datavecTests/seg0/c830.dat deleted file mode 100644 index de30561b1..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c830.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c841.dat b/datavec/datavec-jdbc/datavecTests/seg0/c841.dat deleted file mode 100644 index 9667929b3..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c841.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/c90.dat b/datavec/datavec-jdbc/datavecTests/seg0/c90.dat deleted file mode 100644 index 6a084f5fc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/c90.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/ca1.dat b/datavec/datavec-jdbc/datavecTests/seg0/ca1.dat deleted file mode 100644 index 66b00bbfd..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/ca1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/cb1.dat b/datavec/datavec-jdbc/datavecTests/seg0/cb1.dat deleted file mode 100644 index c9c91f290..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/cb1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/cc0.dat b/datavec/datavec-jdbc/datavecTests/seg0/cc0.dat deleted file mode 100644 index 226872006..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/cc0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/cd1.dat b/datavec/datavec-jdbc/datavecTests/seg0/cd1.dat deleted file mode 100644 index d919a1b0e..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/cd1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/ce1.dat b/datavec/datavec-jdbc/datavecTests/seg0/ce1.dat deleted file mode 100644 index 299e0c409..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/ce1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/cf0.dat b/datavec/datavec-jdbc/datavecTests/seg0/cf0.dat deleted file mode 100644 index 1754d17cb..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/cf0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d660.dat b/datavec/datavec-jdbc/datavecTests/seg0/d660.dat deleted file mode 100644 index 898f6a132..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d660.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d671.dat b/datavec/datavec-jdbc/datavecTests/seg0/d671.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d671.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d680.dat b/datavec/datavec-jdbc/datavecTests/seg0/d680.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d680.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d691.dat b/datavec/datavec-jdbc/datavecTests/seg0/d691.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d691.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d6a0.dat b/datavec/datavec-jdbc/datavecTests/seg0/d6a0.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d6a0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d6b1.dat b/datavec/datavec-jdbc/datavecTests/seg0/d6b1.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d6b1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d6c0.dat b/datavec/datavec-jdbc/datavecTests/seg0/d6c0.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d6c0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d6d1.dat b/datavec/datavec-jdbc/datavecTests/seg0/d6d1.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d6d1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d6e0.dat b/datavec/datavec-jdbc/datavecTests/seg0/d6e0.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d6e0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d6f1.dat b/datavec/datavec-jdbc/datavecTests/seg0/d6f1.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d6f1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d700.dat b/datavec/datavec-jdbc/datavecTests/seg0/d700.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d700.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d711.dat b/datavec/datavec-jdbc/datavecTests/seg0/d711.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d711.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d720.dat b/datavec/datavec-jdbc/datavecTests/seg0/d720.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d720.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d731.dat b/datavec/datavec-jdbc/datavecTests/seg0/d731.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d731.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d740.dat b/datavec/datavec-jdbc/datavecTests/seg0/d740.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d740.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d751.dat b/datavec/datavec-jdbc/datavecTests/seg0/d751.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d751.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d760.dat b/datavec/datavec-jdbc/datavecTests/seg0/d760.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d760.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d771.dat b/datavec/datavec-jdbc/datavecTests/seg0/d771.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d771.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d780.dat b/datavec/datavec-jdbc/datavecTests/seg0/d780.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d780.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d791.dat b/datavec/datavec-jdbc/datavecTests/seg0/d791.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d791.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d7a0.dat b/datavec/datavec-jdbc/datavecTests/seg0/d7a0.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d7a0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d7b1.dat b/datavec/datavec-jdbc/datavecTests/seg0/d7b1.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d7b1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d7c0.dat b/datavec/datavec-jdbc/datavecTests/seg0/d7c0.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d7c0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d7d1.dat b/datavec/datavec-jdbc/datavecTests/seg0/d7d1.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d7d1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d7e0.dat b/datavec/datavec-jdbc/datavecTests/seg0/d7e0.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d7e0.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d7f1.dat b/datavec/datavec-jdbc/datavecTests/seg0/d7f1.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d7f1.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d800.dat b/datavec/datavec-jdbc/datavecTests/seg0/d800.dat deleted file mode 100644 index cb8a93ab3..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d800.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d810.dat b/datavec/datavec-jdbc/datavecTests/seg0/d810.dat deleted file mode 100644 index bcd2bd90d..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d810.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/seg0/d821.dat b/datavec/datavec-jdbc/datavecTests/seg0/d821.dat deleted file mode 100644 index 48501facc..000000000 Binary files a/datavec/datavec-jdbc/datavecTests/seg0/d821.dat and /dev/null differ diff --git a/datavec/datavec-jdbc/datavecTests/service.properties b/datavec/datavec-jdbc/datavecTests/service.properties deleted file mode 100644 index 9589d9ad7..000000000 --- a/datavec/datavec-jdbc/datavecTests/service.properties +++ /dev/null @@ -1,23 +0,0 @@ -#C:\Users\agibs\Documents\GitHub\eclipse-deeplearning4j\datavec\datavec-jdbc\datavecTests -# ******************************************************************** -# *** Please do NOT edit this file. *** -# *** CHANGING THE CONTENT OF THIS FILE MAY CAUSE DATA CORRUPTION. *** -# ******************************************************************** -#Mon Mar 22 08:49:04 JST 2021 -SysschemasIndex2Identifier=225 -SyscolumnsIdentifier=144 -SysconglomeratesIndex1Identifier=49 -SysconglomeratesIdentifier=32 -SyscolumnsIndex2Identifier=177 -SysschemasIndex1Identifier=209 -SysconglomeratesIndex3Identifier=81 -SystablesIndex2Identifier=129 -SyscolumnsIndex1Identifier=161 -derby.serviceProtocol=org.apache.derby.database.Database -SysschemasIdentifier=192 -derby.storage.propertiesId=16 -SysconglomeratesIndex2Identifier=65 -derby.serviceLocale=en_US -SystablesIdentifier=96 -SystablesIndex1Identifier=113 -#--- last line, don't put anything after this line --- diff --git a/datavec/datavec-jdbc/pom.xml b/datavec/datavec-jdbc/pom.xml deleted file mode 100644 index 25e8899d0..000000000 --- a/datavec/datavec-jdbc/pom.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - - - 4.0.0 - - - org.datavec - datavec-parent - 1.0.0-SNAPSHOT - - - datavec-jdbc - - - 1.7 - 2.4.12 - 10.13.1.1 - - - - - org.datavec - datavec-api - ${project.version} - - - commons-dbutils - commons-dbutils - ${dbutils.version} - - - com.zaxxer - HikariCP-java7 - ${hikaricp.version} - - - org.apache.derby - derby - ${derby.version} - test - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/datavec/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/JDBCRecordReaderTest.java b/datavec/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/JDBCRecordReaderTest.java deleted file mode 100644 index 65c7bf2b2..000000000 --- a/datavec/datavec-jdbc/src/test/java/org/datavec/api/records/reader/impl/JDBCRecordReaderTest.java +++ /dev/null @@ -1,314 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.api.records.reader.impl; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.File; -import java.net.URI; -import java.sql.Connection; -import java.sql.ResultSet; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import org.apache.commons.dbutils.DbUtils; -import org.apache.derby.jdbc.EmbeddedDataSource; -import org.datavec.api.conf.Configuration; -import org.datavec.api.records.Record; -import org.datavec.api.records.listener.RecordListener; -import org.datavec.api.records.listener.impl.LogRecordListener; -import org.datavec.api.records.metadata.RecordMetaData; -import org.datavec.jdbc.records.metadata.RecordMetaDataJdbc; -import org.datavec.api.records.metadata.RecordMetaDataLine; -import org.datavec.jdbc.records.reader.impl.jdbc.JDBCRecordReader; -import org.datavec.api.writable.BooleanWritable; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.FloatWritable; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.LongWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.junit.jupiter.api.*; - -import org.junit.jupiter.api.io.TempDir; - -import java.nio.file.Path; -import java.util.UUID; - -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.TagNames; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -@DisplayName("Jdbc Record Reader Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -public class JDBCRecordReaderTest { - - - Connection conn; - - EmbeddedDataSource dataSource; - - private final String dbName = "datavecTests"; - - private final String driverClassName = "org.apache.derby.jdbc.EmbeddedDriver"; - - @BeforeEach - void setUp() throws Exception { - dataSource = new EmbeddedDataSource(); - dataSource.setDatabaseName(dbName); - dataSource.setCreateDatabase("create"); - conn = dataSource.getConnection(); - TestDb.dropTables(conn); - TestDb.buildCoffeeTable(conn); - } - - @AfterEach - void tearDown() throws Exception { - DbUtils.closeQuietly(conn); - } - - @Test - @DisplayName("Test Simple Iter") - void testSimpleIter( @TempDir Path testDir) throws Exception { - File f = testDir.resolve("new-folder").toFile(); - assertTrue(f.mkdirs()); - System.setProperty("derby.system.home", f.getAbsolutePath()); - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - List> records = new ArrayList<>(); - while (reader.hasNext()) { - List values = reader.next(); - records.add(values); - } - assertFalse(records.isEmpty()); - List first = records.get(0); - assertEquals(new Text("Bolivian Dark"), first.get(0)); - assertEquals(new Text("14-001"), first.get(1)); - assertEquals(new DoubleWritable(8.95), first.get(2)); - } - } - - @Test - @DisplayName("Test Simple With Listener") - void testSimpleWithListener() throws Exception { - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - RecordListener recordListener = new LogRecordListener(); - reader.setListeners(recordListener); - reader.next(); - assertTrue(recordListener.invoked()); - } - } - - @Test - @DisplayName("Test Reset") - void testReset() throws Exception { - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - List> records = new ArrayList<>(); - records.add(reader.next()); - reader.reset(); - records.add(reader.next()); - assertEquals(2, records.size()); - assertEquals(new Text("Bolivian Dark"), records.get(0).get(0)); - assertEquals(new Text("Bolivian Dark"), records.get(1).get(0)); - } - } - - @Test - @DisplayName("Test Lacking Data Source Should Fail") - void testLackingDataSourceShouldFail() { - assertThrows(IllegalStateException.class, () -> { - try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) { - reader.initialize(null); - } - }); - } - - @Test - @DisplayName("Test Configuration Data Source Initialization") - void testConfigurationDataSourceInitialization() throws Exception { - try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) { - Configuration conf = new Configuration(); - conf.set(JDBCRecordReader.JDBC_URL, "jdbc:derby:" + dbName + ";create=true"); - conf.set(JDBCRecordReader.JDBC_DRIVER_CLASS_NAME, driverClassName); - reader.initialize(conf, null); - assertTrue(reader.hasNext()); - } - } - - @Test - @DisplayName("Test Init Configuration Missing Parameters Should Fail") - void testInitConfigurationMissingParametersShouldFail() { - assertThrows(IllegalArgumentException.class, () -> { - try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) { - Configuration conf = new Configuration(); - conf.set(JDBCRecordReader.JDBC_URL, "should fail anyway"); - reader.initialize(conf, null); - } - }); - } - - @Test - @DisplayName("Test Record Data Input Stream Should Fail") - void testRecordDataInputStreamShouldFail() { - assertThrows(UnsupportedOperationException.class, () -> { - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - reader.record(null, null); - } - }); - } - - @Test - @DisplayName("Test Load From Meta Data") - void testLoadFromMetaData() throws Exception { - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - RecordMetaDataJdbc rmd = new RecordMetaDataJdbc(new URI(conn.getMetaData().getURL()), "SELECT * FROM Coffee WHERE ProdNum = ?", Collections.singletonList("14-001"), reader.getClass()); - Record res = reader.loadFromMetaData(rmd); - assertNotNull(res); - assertEquals(new Text("Bolivian Dark"), res.getRecord().get(0)); - assertEquals(new Text("14-001"), res.getRecord().get(1)); - assertEquals(new DoubleWritable(8.95), res.getRecord().get(2)); - } - } - - @Test - @DisplayName("Test Next Record") - void testNextRecord() throws Exception { - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - Record r = reader.nextRecord(); - List fields = r.getRecord(); - RecordMetaData meta = r.getMetaData(); - assertNotNull(r); - assertNotNull(fields); - assertNotNull(meta); - assertEquals(new Text("Bolivian Dark"), fields.get(0)); - assertEquals(new Text("14-001"), fields.get(1)); - assertEquals(new DoubleWritable(8.95), fields.get(2)); - assertEquals(RecordMetaDataJdbc.class, meta.getClass()); - } - } - - @Test - @DisplayName("Test Next Record And Recover") - void testNextRecordAndRecover() throws Exception { - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - Record r = reader.nextRecord(); - List fields = r.getRecord(); - RecordMetaData meta = r.getMetaData(); - Record recovered = reader.loadFromMetaData(meta); - List fieldsRecovered = recovered.getRecord(); - assertEquals(fields.size(), fieldsRecovered.size()); - for (int i = 0; i < fields.size(); i++) { - assertEquals(fields.get(i), fieldsRecovered.get(i)); - } - } - } - - // Resetting the record reader when initialized as forward only should fail - @Test - @DisplayName("Test Reset Forward Only Should Fail") - void testResetForwardOnlyShouldFail() { - assertThrows(RuntimeException.class, () -> { - try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee", dataSource)) { - Configuration conf = new Configuration(); - conf.setInt(JDBCRecordReader.JDBC_RESULTSET_TYPE, ResultSet.TYPE_FORWARD_ONLY); - reader.initialize(conf, null); - reader.next(); - reader.reset(); - } - }); - } - - @Test - @DisplayName("Test Read All Types") - void testReadAllTypes() throws Exception { - TestDb.buildAllTypesTable(conn); - try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM AllTypes", dataSource)) { - reader.initialize(null); - List item = reader.next(); - assertEquals(item.size(), 15); - // boolean to boolean - assertEquals(BooleanWritable.class, item.get(0).getClass()); - // date to text - assertEquals(Text.class, item.get(1).getClass()); - // time to text - assertEquals(Text.class, item.get(2).getClass()); - // timestamp to text - assertEquals(Text.class, item.get(3).getClass()); - // char to text - assertEquals(Text.class, item.get(4).getClass()); - // long varchar to text - assertEquals(Text.class, item.get(5).getClass()); - // varchar to text - assertEquals(Text.class, item.get(6).getClass()); - assertEquals(DoubleWritable.class, // float to double (derby's float is an alias of double by default) - item.get(7).getClass()); - // real to float - assertEquals(FloatWritable.class, item.get(8).getClass()); - // decimal to double - assertEquals(DoubleWritable.class, item.get(9).getClass()); - // numeric to double - assertEquals(DoubleWritable.class, item.get(10).getClass()); - // double to double - assertEquals(DoubleWritable.class, item.get(11).getClass()); - // integer to integer - assertEquals(IntWritable.class, item.get(12).getClass()); - // small int to integer - assertEquals(IntWritable.class, item.get(13).getClass()); - // bigint to long - assertEquals(LongWritable.class, item.get(14).getClass()); - } - } - - @Test - @DisplayName("Test Next No More Should Fail") - void testNextNoMoreShouldFail() { - assertThrows(RuntimeException.class, () -> { - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - while (reader.hasNext()) { - reader.next(); - } - reader.next(); - } - }); - } - - @Test - @DisplayName("Test Invalid Metadata Should Fail") - void testInvalidMetadataShouldFail() { - assertThrows(IllegalArgumentException.class, () -> { - try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) { - RecordMetaDataLine md = new RecordMetaDataLine(1, new URI("file://test"), JDBCRecordReader.class); - reader.loadFromMetaData(md); - } - }); - } - - private JDBCRecordReader getInitializedReader(String query) throws Exception { - // ProdNum column - int[] indices = { 1 }; - JDBCRecordReader reader = new JDBCRecordReader(query, dataSource, "SELECT * FROM Coffee WHERE ProdNum = ?", indices); - reader.setTrimStrings(true); - reader.initialize(null); - return reader; - } -} diff --git a/datavec/datavec-local/pom.xml b/datavec/datavec-local/pom.xml deleted file mode 100644 index 8348b82b8..000000000 --- a/datavec/datavec-local/pom.xml +++ /dev/null @@ -1,84 +0,0 @@ - - - - - - 4.0.0 - - - org.datavec - datavec-parent - 1.0.0-SNAPSHOT - - - datavec-local - - datavec-local - - - 1.8 - 1.8 - - - - - com.codepoetics - protonpack - ${protonpack.version} - - - org.datavec - datavec-api - ${project.version} - - - org.datavec - datavec-arrow - ${project.version} - - - org.nd4j - nd4j-common - - - org.nd4j - python4j-numpy - ${project.version} - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestLineRecordReaderFunction.java b/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestLineRecordReaderFunction.java deleted file mode 100644 index 93779f598..000000000 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestLineRecordReaderFunction.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.local.transforms.functions; - -import org.apache.commons.io.FileUtils; - - -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.writable.Writable; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; - -import java.io.File; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -public class TestLineRecordReaderFunction { - - @Test - public void testLineRecordReader() throws Exception { - - File dataFile = new ClassPathResource("iris.dat").getFile(); - List lines = FileUtils.readLines(dataFile); - - List linesRdd = (lines); - - CSVRecordReader rr = new CSVRecordReader(0, ','); - - List> out = linesRdd.stream().map(input -> new LineRecordReaderFunction(rr).apply(input)).collect(Collectors.toList()); - List> outList = out; - - - CSVRecordReader rr2 = new CSVRecordReader(0, ','); - rr2.initialize(new FileSplit(dataFile)); - Set> expectedSet = new HashSet<>(); - int totalCount = 0; - while (rr2.hasNext()) { - expectedSet.add(rr2.next()); - totalCount++; - } - - assertEquals(totalCount, outList.size()); - - for (List line : outList) { - assertTrue(expectedSet.contains(line)); - } - } -} diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestNDArrayToWritablesFunction.java b/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestNDArrayToWritablesFunction.java deleted file mode 100644 index cbf21bd9e..000000000 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestNDArrayToWritablesFunction.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.local.transforms.functions; - -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.NDArrayWritable; -import org.datavec.api.writable.Writable; - -import org.datavec.local.transforms.misc.NDArrayToWritablesFunction; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag -public class TestNDArrayToWritablesFunction { - - @Test - public void testNDArrayToWritablesScalars() throws Exception { - INDArray arr = Nd4j.arange(5); - List expected = new ArrayList<>(); - for (int i = 0; i < 5; i++) - expected.add(new DoubleWritable(i)); - List actual = new NDArrayToWritablesFunction().apply(arr); - assertEquals(expected, actual); - } - - @Test - public void testNDArrayToWritablesArray() throws Exception { - INDArray arr = Nd4j.arange(5); - List expected = Arrays.asList(new NDArrayWritable(arr)); - List actual = new NDArrayToWritablesFunction(true).apply(arr); - assertEquals(expected, actual); - } -} diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToNDArrayFunction.java b/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToNDArrayFunction.java deleted file mode 100644 index e233c6804..000000000 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToNDArrayFunction.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.local.transforms.functions; - -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.NDArrayWritable; -import org.datavec.api.writable.Writable; - -import org.datavec.local.transforms.misc.WritablesToNDArrayFunction; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag -public class TestWritablesToNDArrayFunction { - - @Test - public void testWritablesToNDArrayAllScalars() throws Exception { - Nd4j.setDataType(DataType.FLOAT); - List l = new ArrayList<>(); - for (int i = 0; i < 5; i++) - l.add(new IntWritable(i)); - INDArray expected = Nd4j.arange(5f).castTo(DataType.FLOAT).reshape(1, 5); - assertEquals(expected, new WritablesToNDArrayFunction().apply(l)); - } - - @Test - public void testWritablesToNDArrayMixed() throws Exception { - Nd4j.setDataType(DataType.FLOAT); - List l = new ArrayList<>(); - l.add(new IntWritable(0)); - l.add(new IntWritable(1)); - INDArray arr = Nd4j.arange(2, 5).reshape(1, 3); - l.add(new NDArrayWritable(arr)); - l.add(new IntWritable(5)); - arr = Nd4j.arange(6, 9).reshape(1, 3); - l.add(new NDArrayWritable(arr)); - l.add(new IntWritable(9)); - - INDArray expected = Nd4j.arange(10).castTo(DataType.FLOAT).reshape(1, 10); - assertEquals(expected, new WritablesToNDArrayFunction().apply(l)); - } -} diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToStringFunctions.java b/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToStringFunctions.java deleted file mode 100644 index 0bb465152..000000000 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/functions/TestWritablesToStringFunctions.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.local.transforms.functions; - - - - -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; - - -import org.datavec.local.transforms.misc.SequenceWritablesToStringFunction; -import org.datavec.local.transforms.misc.WritablesToStringFunction; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -public class TestWritablesToStringFunctions { - - - - @Test - public void testWritablesToString() throws Exception { - - List l = Arrays.asList(new DoubleWritable(1.5), new Text("someValue")); - String expected = l.get(0).toString() + "," + l.get(1).toString(); - - assertEquals(expected, new WritablesToStringFunction(",").apply(l)); - } - - @Test - public void testSequenceWritablesToString() throws Exception { - - List> l = Arrays.asList(Arrays.asList(new DoubleWritable(1.5), new Text("someValue")), - Arrays.asList(new DoubleWritable(2.5), new Text("otherValue"))); - - String expected = l.get(0).get(0).toString() + "," + l.get(0).get(1).toString() + "\n" - + l.get(1).get(0).toString() + "," + l.get(1).get(1).toString(); - - assertEquals(expected, new SequenceWritablesToStringFunction(",").apply(l)); - } -} diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/ExecutionTest.java b/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/ExecutionTest.java deleted file mode 100644 index 534b8f44a..000000000 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/ExecutionTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.local.transforms.transform; - -import org.datavec.api.transform.MathFunction; -import org.datavec.api.transform.MathOp; -import org.datavec.api.transform.ReduceOp; -import org.datavec.api.transform.TransformProcess; -import org.datavec.api.transform.condition.ConditionOp; -import org.datavec.api.transform.condition.column.DoubleColumnCondition; -import org.datavec.api.transform.reduce.Reducer; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.transform.schema.SequenceSchema; -import org.datavec.api.writable.*; -import org.datavec.local.transforms.LocalTransformExecutor; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.ops.transforms.Transforms; -import java.util.*; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import static java.time.Duration.ofMillis; -import static org.junit.jupiter.api.Assertions.assertTimeout; - -@DisplayName("Execution Test") -@Tag(TagNames.FILE_IO) -@NativeTag -class ExecutionTest { - - @Test - @DisplayName("Test Execution Ndarray") - void testExecutionNdarray() { - Schema schema = new Schema.Builder().addColumnNDArray("first", new long[] { 1, 32577 }).addColumnNDArray("second", new long[] { 1, 32577 }).build(); - TransformProcess transformProcess = new TransformProcess.Builder(schema).ndArrayMathFunctionTransform("first", MathFunction.SIN).ndArrayMathFunctionTransform("second", MathFunction.COS).build(); - List> functions = new ArrayList<>(); - List firstRow = new ArrayList<>(); - INDArray firstArr = Nd4j.linspace(1, 4, 4); - INDArray secondArr = Nd4j.linspace(1, 4, 4); - firstRow.add(new NDArrayWritable(firstArr)); - firstRow.add(new NDArrayWritable(secondArr)); - functions.add(firstRow); - List> execute = LocalTransformExecutor.execute(functions, transformProcess); - INDArray firstResult = ((NDArrayWritable) execute.get(0).get(0)).get(); - INDArray secondResult = ((NDArrayWritable) execute.get(0).get(1)).get(); - INDArray expected = Transforms.sin(firstArr); - INDArray secondExpected = Transforms.cos(secondArr); - assertEquals(expected, firstResult); - assertEquals(secondExpected, secondResult); - } - - @Test - @DisplayName("Test Execution Simple") - void testExecutionSimple() { - Schema schema = new Schema.Builder().addColumnInteger("col0").addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").addColumnFloat("col3").build(); - TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1").doubleMathOp("col2", MathOp.Add, 10.0).floatMathOp("col3", MathOp.Add, 5f).build(); - List> inputData = new ArrayList<>(); - inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1), new FloatWritable(0.3f))); - inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1), new FloatWritable(1.7f))); - inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1), new FloatWritable(3.6f))); - List> rdd = (inputData); - List> out = new ArrayList<>(LocalTransformExecutor.execute(rdd, tp)); - Collections.sort(out, (o1, o2) -> Integer.compare(o1.get(0).toInt(), o2.get(0).toInt())); - List> expected = new ArrayList<>(); - expected.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1), new FloatWritable(5.3f))); - expected.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1), new FloatWritable(6.7f))); - expected.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1), new FloatWritable(8.6f))); - assertEquals(expected, out); - } - - @Test - @DisplayName("Test Filter") - void testFilter() { - Schema filterSchema = new Schema.Builder().addColumnDouble("col1").addColumnDouble("col2").addColumnDouble("col3").build(); - List> inputData = new ArrayList<>(); - inputData.add(Arrays.asList(new IntWritable(0), new DoubleWritable(1), new DoubleWritable(0.1))); - inputData.add(Arrays.asList(new IntWritable(1), new DoubleWritable(3), new DoubleWritable(1.1))); - inputData.add(Arrays.asList(new IntWritable(2), new DoubleWritable(3), new DoubleWritable(2.1))); - TransformProcess transformProcess = new TransformProcess.Builder(filterSchema).filter(new DoubleColumnCondition("col1", ConditionOp.LessThan, 1)).build(); - List> execute = LocalTransformExecutor.execute(inputData, transformProcess); - assertEquals(2, execute.size()); - } - - @Test - @DisplayName("Test Execution Sequence") - void testExecutionSequence() { - Schema schema = new SequenceSchema.Builder().addColumnInteger("col0").addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build(); - TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1").doubleMathOp("col2", MathOp.Add, 10.0).build(); - List>> inputSequences = new ArrayList<>(); - List> seq1 = new ArrayList<>(); - seq1.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); - seq1.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); - seq1.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); - List> seq2 = new ArrayList<>(); - seq2.add(Arrays.asList(new IntWritable(3), new Text("state0"), new DoubleWritable(3.1))); - seq2.add(Arrays.asList(new IntWritable(4), new Text("state1"), new DoubleWritable(4.1))); - inputSequences.add(seq1); - inputSequences.add(seq2); - List>> rdd = (inputSequences); - List>> out = LocalTransformExecutor.executeSequenceToSequence(rdd, tp); - Collections.sort(out, (o1, o2) -> -Integer.compare(o1.size(), o2.size())); - List>> expectedSequence = new ArrayList<>(); - List> seq1e = new ArrayList<>(); - seq1e.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1))); - seq1e.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1))); - seq1e.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1))); - List> seq2e = new ArrayList<>(); - seq2e.add(Arrays.asList(new IntWritable(3), new IntWritable(0), new DoubleWritable(13.1))); - seq2e.add(Arrays.asList(new IntWritable(4), new IntWritable(1), new DoubleWritable(14.1))); - expectedSequence.add(seq1e); - expectedSequence.add(seq2e); - assertEquals(expectedSequence, out); - } - - @Test - @DisplayName("Test Reduction Global") - void testReductionGlobal() { - List> in = Arrays.asList(Arrays.asList(new Text("first"), new DoubleWritable(3.0)), Arrays.asList(new Text("second"), new DoubleWritable(5.0))); - List> inData = in; - Schema s = new Schema.Builder().addColumnString("textCol").addColumnDouble("doubleCol").build(); - TransformProcess tp = new TransformProcess.Builder(s).reduce(new Reducer.Builder(ReduceOp.TakeFirst).takeFirstColumns("textCol").meanColumns("doubleCol").build()).build(); - List> outRdd = LocalTransformExecutor.execute(inData, tp); - List> out = outRdd; - List> expOut = Collections.singletonList(Arrays.asList(new Text("first"), new DoubleWritable(4.0))); - assertEquals(expOut, out); - } - - @Test - @DisplayName("Test Reduction By Key") - void testReductionByKey() { - List> in = Arrays.asList(Arrays.asList(new IntWritable(0), new Text("first"), new DoubleWritable(3.0)), Arrays.asList(new IntWritable(0), new Text("second"), new DoubleWritable(5.0)), Arrays.asList(new IntWritable(1), new Text("f"), new DoubleWritable(30.0)), Arrays.asList(new IntWritable(1), new Text("s"), new DoubleWritable(50.0))); - List> inData = in; - Schema s = new Schema.Builder().addColumnInteger("intCol").addColumnString("textCol").addColumnDouble("doubleCol").build(); - TransformProcess tp = new TransformProcess.Builder(s).reduce(new Reducer.Builder(ReduceOp.TakeFirst).keyColumns("intCol").takeFirstColumns("textCol").meanColumns("doubleCol").build()).build(); - List> outRdd = LocalTransformExecutor.execute(inData, tp); - List> out = outRdd; - List> expOut = Arrays.asList(Arrays.asList(new IntWritable(0), new Text("first"), new DoubleWritable(4.0)), Arrays.asList(new IntWritable(1), new Text("f"), new DoubleWritable(40.0))); - out = new ArrayList<>(out); - Collections.sort(out, Comparator.comparingInt(o -> o.get(0).toInt())); - assertEquals(expOut, out); - } - -} diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/join/TestJoin.java b/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/join/TestJoin.java deleted file mode 100644 index 3cfa9b0c1..000000000 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/join/TestJoin.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.local.transforms.transform.join; - - -import org.datavec.api.transform.ColumnType; -import org.datavec.api.transform.join.Join; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.*; - - -import org.datavec.local.transforms.LocalTransformExecutor; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.*; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -public class TestJoin { - - @Test - public void testJoinOneToMany_ManyToOne() { - - Schema customerInfoSchema = - new Schema.Builder().addColumnLong("customerID").addColumnString("customerName").build(); - - Schema purchasesSchema = new Schema.Builder().addColumnLong("purchaseID").addColumnLong("customerID") - .addColumnDouble("amount").build(); - - List> infoList = new ArrayList<>(); - infoList.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"))); - infoList.add(Arrays.asList(new LongWritable(98765), new Text("Customer98765"))); - infoList.add(Arrays.asList(new LongWritable(50000), new Text("Customer50000"))); - - List> purchaseList = new ArrayList<>(); - purchaseList.add(Arrays.asList(new LongWritable(1000000), new LongWritable(12345), - new DoubleWritable(10.00))); - purchaseList.add(Arrays.asList(new LongWritable(1000001), new LongWritable(12345), - new DoubleWritable(20.00))); - purchaseList.add(Arrays.asList(new LongWritable(1000002), new LongWritable(98765), - new DoubleWritable(30.00))); - - Join join = new Join.Builder(Join.JoinType.RightOuter).setJoinColumns("customerID") - .setSchemas(customerInfoSchema, purchasesSchema).build(); - - List> expected = new ArrayList<>(); - expected.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"), - new LongWritable(1000000), new DoubleWritable(10.00))); - expected.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"), - new LongWritable(1000001), new DoubleWritable(20.00))); - expected.add(Arrays.asList(new LongWritable(98765), new Text("Customer98765"), - new LongWritable(1000002), new DoubleWritable(30.00))); - - - - List> info = (infoList); - List> purchases = (purchaseList); - - List> joined = LocalTransformExecutor.executeJoin(join, info, purchases); - List> joinedList = new ArrayList<>(joined); - //Sort by order ID (column 3, index 2) - Collections.sort(joinedList, (o1, o2) -> Long.compare(o1.get(2).toLong(), o2.get(2).toLong())); - assertEquals(expected, joinedList); - - assertEquals(3, joinedList.size()); - - List expectedColNames = Arrays.asList("customerID", "customerName", "purchaseID", "amount"); - assertEquals(expectedColNames, join.getOutputSchema().getColumnNames()); - - List expectedColTypes = - Arrays.asList(ColumnType.Long, ColumnType.String, ColumnType.Long, ColumnType.Double); - assertEquals(expectedColTypes, join.getOutputSchema().getColumnTypes()); - - - //Test Many to one: same thing, but swap the order... - Join join2 = new Join.Builder(Join.JoinType.LeftOuter).setJoinColumns("customerID") - .setSchemas(purchasesSchema, customerInfoSchema).build(); - - List> expectedManyToOne = new ArrayList<>(); - expectedManyToOne.add(Arrays.asList(new LongWritable(1000000), new LongWritable(12345), - new DoubleWritable(10.00), new Text("Customer12345"))); - expectedManyToOne.add(Arrays.asList(new LongWritable(1000001), new LongWritable(12345), - new DoubleWritable(20.00), new Text("Customer12345"))); - expectedManyToOne.add(Arrays.asList(new LongWritable(1000002), new LongWritable(98765), - new DoubleWritable(30.00), new Text("Customer98765"))); - - List> joined2 = LocalTransformExecutor.executeJoin(join2, purchases, info); - List> joinedList2 = new ArrayList<>(joined2); - //Sort by order ID (column 0) - Collections.sort(joinedList2, (o1, o2) -> Long.compare(o1.get(0).toLong(), o2.get(0).toLong())); - assertEquals(3, joinedList2.size()); - - assertEquals(expectedManyToOne, joinedList2); - - List expectedColNames2 = Arrays.asList("purchaseID", "customerID", "amount", "customerName"); - assertEquals(expectedColNames2, join2.getOutputSchema().getColumnNames()); - - List expectedColTypes2 = - Arrays.asList(ColumnType.Long, ColumnType.Long, ColumnType.Double, ColumnType.String); - assertEquals(expectedColTypes2, join2.getOutputSchema().getColumnTypes()); - } - - - @Test - public void testJoinManyToMany() { - Schema schema1 = new Schema.Builder().addColumnLong("id") - .addColumnCategorical("category", Arrays.asList("cat0", "cat1", "cat2")).build(); - - Schema schema2 = new Schema.Builder().addColumnLong("otherId") - .addColumnCategorical("otherCategory", Arrays.asList("cat0", "cat1", "cat2")).build(); - - List> first = new ArrayList<>(); - first.add(Arrays.asList(new LongWritable(0), new Text("cat0"))); - first.add(Arrays.asList(new LongWritable(1), new Text("cat0"))); - first.add(Arrays.asList(new LongWritable(2), new Text("cat1"))); - - List> second = new ArrayList<>(); - second.add(Arrays.asList(new LongWritable(100), new Text("cat0"))); - second.add(Arrays.asList(new LongWritable(101), new Text("cat0"))); - second.add(Arrays.asList(new LongWritable(102), new Text("cat2"))); - - - - List> expOuterJoin = new ArrayList<>(); - expOuterJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); - expOuterJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); - expOuterJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); - expOuterJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); - expOuterJoin.add(Arrays.asList(new LongWritable(2), new Text("cat1"), new NullWritable())); - expOuterJoin.add(Arrays.asList(new NullWritable(), new Text("cat2"), new LongWritable(102))); - - List> expLeftJoin = new ArrayList<>(); - expLeftJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); - expLeftJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); - expLeftJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); - expLeftJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); - expLeftJoin.add(Arrays.asList(new LongWritable(2), new Text("cat1"), new NullWritable())); - - - List> expRightJoin = new ArrayList<>(); - expRightJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); - expRightJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); - expRightJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); - expRightJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); - expRightJoin.add(Arrays.asList(new NullWritable(), new Text("cat2"), new LongWritable(102))); - - List> expInnerJoin = new ArrayList<>(); - expInnerJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); - expInnerJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); - expInnerJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); - expInnerJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); - - List> firstRDD = (first); - List> secondRDD = (second); - - int count = 0; - for (Join.JoinType jt : Join.JoinType.values()) { - Join join = new Join.Builder(jt).setJoinColumnsLeft("category").setJoinColumnsRight("otherCategory") - .setSchemas(schema1, schema2).build(); - List> out = - new ArrayList<>(LocalTransformExecutor.executeJoin(join, firstRDD, secondRDD)); - - //Sort output by column 0, then column 1, then column 2 for comparison to expected... - Collections.sort(out, (o1, o2) -> { - Writable w1 = o1.get(0); - Writable w2 = o2.get(0); - if (w1 instanceof NullWritable) - return 1; - else if (w2 instanceof NullWritable) - return -1; - int c = Long.compare(w1.toLong(), w2.toLong()); - if (c != 0) - return c; - c = o1.get(1).toString().compareTo(o2.get(1).toString()); - if (c != 0) - return c; - w1 = o1.get(2); - w2 = o2.get(2); - if (w1 instanceof NullWritable) - return 1; - else if (w2 instanceof NullWritable) - return -1; - return Long.compare(w1.toLong(), w2.toLong()); - }); - - switch (jt) { - case Inner: - assertEquals(expInnerJoin, out); - break; - case LeftOuter: - assertEquals(expLeftJoin, out); - break; - case RightOuter: - assertEquals(expRightJoin, out); - break; - case FullOuter: - assertEquals(expOuterJoin, out); - break; - } - count++; - } - - assertEquals(4, count); - } - -} diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/rank/TestCalculateSortedRank.java b/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/rank/TestCalculateSortedRank.java deleted file mode 100644 index 1be235d47..000000000 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/rank/TestCalculateSortedRank.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.local.transforms.transform.rank; - - -import org.datavec.api.transform.ColumnType; -import org.datavec.api.transform.TransformProcess; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.datavec.api.writable.comparator.DoubleWritableComparator; - - -import org.datavec.local.transforms.LocalTransformExecutor; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -public class TestCalculateSortedRank { - - @Test - public void testCalculateSortedRank() { - - List> data = new ArrayList<>(); - data.add(Arrays.asList((Writable) new Text("0"), new DoubleWritable(0.0))); - data.add(Arrays.asList((Writable) new Text("3"), new DoubleWritable(0.3))); - data.add(Arrays.asList((Writable) new Text("2"), new DoubleWritable(0.2))); - data.add(Arrays.asList((Writable) new Text("1"), new DoubleWritable(0.1))); - - List> rdd = (data); - - Schema schema = new Schema.Builder().addColumnsString("TextCol").addColumnDouble("DoubleCol").build(); - - TransformProcess tp = new TransformProcess.Builder(schema) - .calculateSortedRank("rank", "DoubleCol", new DoubleWritableComparator()).build(); - - Schema outSchema = tp.getFinalSchema(); - assertEquals(3, outSchema.numColumns()); - assertEquals(Arrays.asList("TextCol", "DoubleCol", "rank"), outSchema.getColumnNames()); - assertEquals(Arrays.asList(ColumnType.String, ColumnType.Double, ColumnType.Long), outSchema.getColumnTypes()); - - List> out = LocalTransformExecutor.execute(rdd, tp); - - List> collected = out; - assertEquals(4, collected.size()); - for (int i = 0; i < 4; i++) - assertEquals(3, collected.get(i).size()); - - for (List example : collected) { - int exampleNum = example.get(0).toInt(); - int rank = example.get(2).toInt(); - assertEquals(exampleNum, rank); - } - } - -} diff --git a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/sequence/TestConvertToSequence.java b/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/sequence/TestConvertToSequence.java deleted file mode 100644 index 3987f2930..000000000 --- a/datavec/datavec-local/src/test/java/org/datavec/local/transforms/transform/sequence/TestConvertToSequence.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.local.transforms.transform.sequence; - - -import org.datavec.api.transform.TransformProcess; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.transform.sequence.comparator.NumericalColumnComparator; -import org.datavec.api.writable.LongWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; - - -import org.datavec.arrow.recordreader.ArrowWritableRecordTimeSeriesBatch; -import org.datavec.local.transforms.LocalTransformExecutor; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -public class TestConvertToSequence { - - @Test - public void testConvertToSequenceCompoundKey() { - - Schema s = new Schema.Builder().addColumnsString("key1", "key2").addColumnLong("time").build(); - - List> allExamples = - Arrays.asList(Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(10)), - Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(10)), - Arrays.asList(new Text("k1a"), new Text("k2a"), - new LongWritable(-10)), - Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(5)), - Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(0))); - - TransformProcess tp = new TransformProcess.Builder(s) - .convertToSequence(Arrays.asList("key1", "key2"), new NumericalColumnComparator("time")) - .build(); - - List> rdd = (allExamples); - - List>> out = LocalTransformExecutor.executeToSequence(rdd, tp); - - assertEquals(2, out.size()); - List> seq0; - List> seq1; - if (out.get(0).size() == 3) { - seq0 = out.get(0); - seq1 = out.get(1); - } else { - seq0 = out.get(1); - seq1 = out.get(0); - } - - List> expSeq0 = Arrays.asList( - Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(-10)), - Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(0)), - Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(10))); - - List> expSeq1 = Arrays.asList( - Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(5)), - Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(10))); - - assertEquals(expSeq0, seq0); - assertEquals(expSeq1, seq1); - } - - @Test - public void testConvertToSequenceLength1() { - - Schema s = new Schema.Builder() - .addColumnsString("string") - .addColumnLong("long") - .build(); - - List> allExamples = Arrays.asList( - Arrays.asList(new Text("a"), new LongWritable(0)), - Arrays.asList(new Text("b"), new LongWritable(1)), - Arrays.asList(new Text("c"), new LongWritable(2))); - - TransformProcess tp = new TransformProcess.Builder(s) - .convertToSequence() - .build(); - - List> rdd = (allExamples); - - ArrowWritableRecordTimeSeriesBatch out = (ArrowWritableRecordTimeSeriesBatch) LocalTransformExecutor.executeToSequence(rdd, tp); - - List>> out2 = out.toArrayList(); - - assertEquals(3, out2.size()); - - for( int i = 0; i < 3; i++) { - assertTrue(out2.contains(Collections.singletonList(allExamples.get(i)))); - } - } -} diff --git a/datavec/datavec-spark/pom.xml b/datavec/datavec-spark/pom.xml deleted file mode 100644 index 81211e93a..000000000 --- a/datavec/datavec-spark/pom.xml +++ /dev/null @@ -1,157 +0,0 @@ - - - - - - 4.0.0 - - - org.datavec - datavec-parent - 1.0.0-SNAPSHOT - - - 1.0.0-SNAPSHOT - datavec-spark_2.11 - - - - 2.11.12 - 2.11 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.8 - 1.8 - - - - - - - - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - org.nd4j - guava - ${project.version} - - - org.nd4j - nd4j-common - - - com.tdunning - t-digest - 3.2 - test - - - org.scala-lang - scala-library - ${scala.version} - - - org.apache.spark - spark-sql_2.11 - ${spark.version} - provided - - - commons-collections - commons-collections - ${commons-collections.version} - - - commons-io - commons-io - - - org.apache.commons - commons-math3 - ${commons-math3.version} - - - org.slf4j - slf4j-api - - - org.apache.spark - spark-core_2.11 - ${spark.version} - provided - - - com.google.code.findbugs - jsr305 - - - - - org.datavec - datavec-api - ${project.parent.version} - - - - org.datavec - datavec-data-image - ${project.parent.version} - test - - - org.datavec - datavec-local - ${datavec.version} - test - - - org.nd4j - nd4j-common-tests - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/BaseSparkTest.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/BaseSparkTest.java deleted file mode 100644 index 605438d25..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/BaseSparkTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.spark; - -import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -import java.io.File; -import java.io.Serializable; -import java.net.URI; - -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.resources.Downloader; - -@Slf4j -@DisplayName("Base Spark Test") -public abstract class BaseSparkTest implements Serializable { - - protected static JavaSparkContext sc; - - @SneakyThrows - @BeforeEach - void before() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - sc = getContext(); - } - - @AfterEach - synchronized void after() { - sc.close(); - // Wait until it's stopped, to avoid race conditions during tests - for (int i = 0; i < 100; i++) { - if (!sc.sc().stopped().get()) { - try { - Thread.sleep(100L); - } catch (InterruptedException e) { - log.error("", e); - } - } else { - break; - } - } - if (!sc.sc().stopped().get()) { - throw new RuntimeException("Spark context is not stopped after 10s"); - } - sc = null; - } - - public synchronized JavaSparkContext getContext() { - if (sc != null) - return sc; - SparkConf sparkConf = new SparkConf().setMaster("local[*]").set("spark.driver.host", "localhost").set("spark.driverEnv.SPARK_LOCAL_IP", "127.0.0.1").set("spark.executorEnv.SPARK_LOCAL_IP", "127.0.0.1").setAppName("sparktest"); - if (useKryo()) { - sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); - } - sc = new JavaSparkContext(sparkConf); - return sc; - } - - public boolean useKryo() { - return false; - } -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestLineRecordReaderFunction.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestLineRecordReaderFunction.java deleted file mode 100644 index 3a9c36dc7..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestLineRecordReaderFunction.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.spark.functions; - -import org.apache.commons.io.FileUtils; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.writable.Writable; -import org.datavec.spark.BaseSparkTest; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; - -import java.io.File; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -public class TestLineRecordReaderFunction extends BaseSparkTest { - - @Test - public void testLineRecordReader() throws Exception { - - File dataFile = new ClassPathResource("iris.dat").getFile(); - List lines = FileUtils.readLines(dataFile); - - JavaSparkContext sc = getContext(); - JavaRDD linesRdd = sc.parallelize(lines); - - CSVRecordReader rr = new CSVRecordReader(0, ','); - - JavaRDD> out = linesRdd.map(new LineRecordReaderFunction(rr)); - List> outList = out.collect(); - - - CSVRecordReader rr2 = new CSVRecordReader(0, ','); - rr2.initialize(new FileSplit(dataFile)); - Set> expectedSet = new HashSet<>(); - int totalCount = 0; - while (rr2.hasNext()) { - expectedSet.add(rr2.next()); - totalCount++; - } - - assertEquals(totalCount, outList.size()); - - for (List line : outList) { - assertTrue(expectedSet.contains(line)); - } - } -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestNDArrayToWritablesFunction.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestNDArrayToWritablesFunction.java deleted file mode 100644 index e9e1668ae..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestNDArrayToWritablesFunction.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.spark.functions; - -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.NDArrayWritable; -import org.datavec.api.writable.Writable; -import org.datavec.spark.transform.misc.NDArrayToWritablesFunction; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -public class TestNDArrayToWritablesFunction { - - @Test - public void testNDArrayToWritablesScalars() throws Exception { - INDArray arr = Nd4j.arange(5); - List expected = new ArrayList<>(); - for (int i = 0; i < 5; i++) - expected.add(new DoubleWritable(i)); - List actual = new NDArrayToWritablesFunction().call(arr); - assertEquals(expected, actual); - } - - @Test - public void testNDArrayToWritablesArray() throws Exception { - INDArray arr = Nd4j.arange(5); - List expected = Arrays.asList(new NDArrayWritable(arr)); - List actual = new NDArrayToWritablesFunction(true).call(arr); - assertEquals(expected, actual); - } -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderFunction.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderFunction.java deleted file mode 100644 index d3790c150..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestSequenceRecordReaderFunction.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.spark.functions; - -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.input.PortableDataStream; -import org.datavec.api.conf.Configuration; -import org.datavec.api.records.reader.SequenceRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.split.InputSplit; -import org.datavec.api.writable.ArrayWritable; -import org.datavec.api.writable.Writable; -import org.datavec.spark.BaseSparkTest; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; - -import java.io.File; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -public class TestSequenceRecordReaderFunction extends BaseSparkTest { - - - - @Test - public void testSequenceRecordReaderFunctionCSV(@TempDir Path testDir) throws Exception { - JavaSparkContext sc = getContext(); - - File f = testDir.toFile(); - new ClassPathResource("datavec-spark/csvsequence/").copyDirectory(f); - - String path = f.getAbsolutePath() + "/*"; - - JavaPairRDD origData = sc.binaryFiles(path); - assertEquals(3, origData.count()); //3 CSV files - - SequenceRecordReaderFunction srrf = new SequenceRecordReaderFunction(new CSVSequenceRecordReader(1, ",")); //CSV, skip 1 line - JavaRDD>> rdd = origData.map(srrf); - List>> listSpark = rdd.collect(); - - assertEquals(3, listSpark.size()); - for (int i = 0; i < 3; i++) { - List> thisSequence = listSpark.get(i); - assertEquals(4, thisSequence.size()); //Expect exactly 4 time steps in sequence - for (List c : thisSequence) { - assertEquals(3, c.size()); //3 values per time step - } - } - - //Load normally, and check that we get the same results (order not withstanding) - InputSplit is = new FileSplit(f, new String[] {"txt"}, true); - // System.out.println("Locations:"); - // System.out.println(Arrays.toString(is.locations())); - - SequenceRecordReader srr = new CSVSequenceRecordReader(1, ","); - srr.initialize(is); - - List>> list = new ArrayList<>(3); - while (srr.hasNext()) { - list.add(srr.sequenceRecord()); - } - assertEquals(3, list.size()); - - // System.out.println("Spark list:"); - // for(List> c : listSpark ) System.out.println(c); - // System.out.println("Local list:"); - // for(List> c : list ) System.out.println(c); - - //Check that each of the values from Spark equals exactly one of the values doing it normally - boolean[] found = new boolean[3]; - for (int i = 0; i < 3; i++) { - int foundIndex = -1; - List> collection = listSpark.get(i); - for (int j = 0; j < 3; j++) { - if (collection.equals(list.get(j))) { - if (foundIndex != -1) - fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen) - foundIndex = j; - if (found[foundIndex]) - fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list - found[foundIndex] = true; //mark this one as seen before - } - } - } - int count = 0; - for (boolean b : found) - if (b) - count++; - assertEquals(3, count); //Expect all 3 and exactly 3 pairwise matches between spark and local versions - } - - - - -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestWritablesToNDArrayFunction.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestWritablesToNDArrayFunction.java deleted file mode 100644 index 3a6a61b93..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestWritablesToNDArrayFunction.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.spark.functions; - -import org.datavec.api.writable.*; -import org.datavec.spark.transform.misc.WritablesToNDArrayFunction; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -public class TestWritablesToNDArrayFunction { - - @Test - public void testWritablesToNDArrayAllScalars() throws Exception { - List l = new ArrayList<>(); - for (int i = 0; i < 5; i++) - l.add(new IntWritable(i)); - INDArray expected = Nd4j.arange(5).castTo(DataType.FLOAT).reshape(1,5); - assertEquals(expected, new WritablesToNDArrayFunction().call(l)); - } - - @Test - public void testWritablesToNDArrayMixed() throws Exception { - List l = new ArrayList<>(); - l.add(new IntWritable(0)); - l.add(new IntWritable(1)); - INDArray arr = Nd4j.arange(2, 5); - l.add(new NDArrayWritable(arr)); - l.add(new IntWritable(5)); - arr = Nd4j.arange(6, 9); - l.add(new NDArrayWritable(arr)); - l.add(new IntWritable(9)); - - INDArray expected = Nd4j.arange(10).castTo(DataType.FLOAT).reshape(1,10); - assertEquals(expected, new WritablesToNDArrayFunction().call(l)); - } -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestWritablesToStringFunctions.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestWritablesToStringFunctions.java deleted file mode 100644 index 011f673db..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/functions/TestWritablesToStringFunctions.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.spark.functions; - -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.PairFunction; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.datavec.spark.BaseSparkTest; -import org.datavec.spark.transform.misc.SequenceWritablesToStringFunction; -import org.datavec.spark.transform.misc.WritablesToStringFunction; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; -import scala.Tuple2; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -public class TestWritablesToStringFunctions extends BaseSparkTest { - - @Test - public void testCGroup() { - List> leftMap = new ArrayList<>(); - List> rightMap = new ArrayList<>(); - - leftMap.add(new Tuple2<>("cat","adam")); - leftMap.add(new Tuple2<>("dog","adam")); - - rightMap.add(new Tuple2<>("fish","alex")); - rightMap.add(new Tuple2<>("cat","alice")); - rightMap.add(new Tuple2<>("dog","steve")); - - List pets = Arrays.asList("cat","dog"); - - - - JavaSparkContext sc = getContext(); - JavaPairRDD left = sc.parallelize(leftMap).mapToPair((PairFunction, String, String>) stringStringTuple2 -> stringStringTuple2); - - JavaPairRDD right = sc.parallelize(rightMap).mapToPair((PairFunction, String, String>) stringStringTuple2 -> stringStringTuple2); - - System.out.println(left.cogroup(right).collect()); - } - - @Test - public void testWritablesToString() throws Exception { - - List l = Arrays.asList(new DoubleWritable(1.5), new Text("someValue")); - String expected = l.get(0).toString() + "," + l.get(1).toString(); - - assertEquals(expected, new WritablesToStringFunction(",").call(l)); - } - - @Test - public void testSequenceWritablesToString() throws Exception { - - List> l = Arrays.asList(Arrays.asList(new DoubleWritable(1.5), new Text("someValue")), - Arrays.asList(new DoubleWritable(2.5), new Text("otherValue"))); - - String expected = l.get(0).get(0).toString() + "," + l.get(0).get(1).toString() + "\n" - + l.get(1).get(0).toString() + "," + l.get(1).get(1).toString(); - - assertEquals(expected, new SequenceWritablesToStringFunction(",").call(l)); - } -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/ExecutionTest.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/ExecutionTest.java deleted file mode 100644 index fff0d201f..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/ExecutionTest.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.datavec.spark.transform; - -import org.apache.spark.api.java.JavaRDD; -import org.datavec.api.transform.MathOp; -import org.datavec.api.transform.ReduceOp; -import org.datavec.api.transform.TransformProcess; -import org.datavec.api.transform.reduce.Reducer; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.transform.schema.SequenceSchema; -import org.datavec.api.transform.transform.categorical.FirstDigitTransform; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.datavec.spark.BaseSparkTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.*; - -import static java.time.Duration.ofMillis; -import static org.junit.jupiter.api.Assertions.*; - -@DisplayName("Execution Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -class ExecutionTest extends BaseSparkTest { - - @Test - @DisplayName("Test Execution Simple") - void testExecutionSimple() { - Schema schema = new Schema.Builder().addColumnInteger("col0").addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build(); - TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1").doubleMathOp("col2", MathOp.Add, 10.0).build(); - List> inputData = new ArrayList<>(); - inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); - inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); - inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); - JavaRDD> rdd = sc.parallelize(inputData); - List> out = new ArrayList<>(SparkTransformExecutor.execute(rdd, tp).collect()); - Collections.sort(out, Comparator.comparingInt(o -> o.get(0).toInt())); - List> expected = new ArrayList<>(); - expected.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1))); - expected.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1))); - expected.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1))); - assertEquals(expected, out); - } - - @Test - @DisplayName("Test Execution Sequence") - void testExecutionSequence() { - Schema schema = new SequenceSchema.Builder().addColumnInteger("col0").addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build(); - TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1").doubleMathOp("col2", MathOp.Add, 10.0).build(); - List>> inputSequences = new ArrayList<>(); - List> seq1 = new ArrayList<>(); - seq1.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); - seq1.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); - seq1.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); - List> seq2 = new ArrayList<>(); - seq2.add(Arrays.asList(new IntWritable(4), new Text("state1"), new DoubleWritable(4.1))); - seq2.add(Arrays.asList(new IntWritable(3), new Text("state0"), new DoubleWritable(3.1))); - inputSequences.add(seq1); - inputSequences.add(seq2); - JavaRDD>> rdd = sc.parallelize(inputSequences); - List>> out = new ArrayList<>(SparkTransformExecutor.executeSequenceToSequence(rdd, tp).collect()); - Collections.sort(out, (o1, o2) -> -Integer.compare(o1.size(), o2.size())); - List>> expectedSequence = new ArrayList<>(); - List> seq1e = new ArrayList<>(); - seq1e.add(Arrays.asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1))); - seq1e.add(Arrays.asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1))); - seq1e.add(Arrays.asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1))); - List> seq2e = new ArrayList<>(); - seq2e.add(Arrays.asList(new IntWritable(4), new IntWritable(1), new DoubleWritable(14.1))); - seq2e.add(Arrays.asList(new IntWritable(3), new IntWritable(0), new DoubleWritable(13.1))); - expectedSequence.add(seq1e); - expectedSequence.add(seq2e); - assertEquals(expectedSequence, out); - } - - @Test - @DisplayName("Test Reduction Global") - void testReductionGlobal() { - List> in = Arrays.asList(Arrays.asList(new Text("first"), new DoubleWritable(3.0)), Arrays.asList(new Text("second"), new DoubleWritable(5.0))); - JavaRDD> inData = sc.parallelize(in); - Schema s = new Schema.Builder().addColumnString("textCol").addColumnDouble("doubleCol").build(); - TransformProcess tp = new TransformProcess.Builder(s).reduce(new Reducer.Builder(ReduceOp.TakeFirst).takeFirstColumns("textCol").meanColumns("doubleCol").build()).build(); - JavaRDD> outRdd = SparkTransformExecutor.execute(inData, tp); - List> out = outRdd.collect(); - List> expOut = Collections.singletonList(Arrays.asList(new Text("first"), new DoubleWritable(4.0))); - assertEquals(expOut, out); - } - - @Test - @DisplayName("Test Reduction By Key") - void testReductionByKey() { - List> in = Arrays.asList(Arrays.asList(new IntWritable(0), new Text("first"), new DoubleWritable(3.0)), Arrays.asList(new IntWritable(0), new Text("second"), new DoubleWritable(5.0)), Arrays.asList(new IntWritable(1), new Text("f"), new DoubleWritable(30.0)), Arrays.asList(new IntWritable(1), new Text("s"), new DoubleWritable(50.0))); - JavaRDD> inData = sc.parallelize(in); - Schema s = new Schema.Builder().addColumnInteger("intCol").addColumnString("textCol").addColumnDouble("doubleCol").build(); - TransformProcess tp = new TransformProcess.Builder(s).reduce(new Reducer.Builder(ReduceOp.TakeFirst).keyColumns("intCol").takeFirstColumns("textCol").meanColumns("doubleCol").build()).build(); - JavaRDD> outRdd = SparkTransformExecutor.execute(inData, tp); - List> out = outRdd.collect(); - List> expOut = Arrays.asList(Arrays.asList(new IntWritable(0), new Text("first"), new DoubleWritable(4.0)), Arrays.asList(new IntWritable(1), new Text("f"), new DoubleWritable(40.0))); - out = new ArrayList<>(out); - Collections.sort(out, Comparator.comparingInt(o -> o.get(0).toInt())); - assertEquals(expOut, out); - } - - @Test - @DisplayName("Test Unique Multi Col") - void testUniqueMultiCol() { - Schema schema = new Schema.Builder().addColumnInteger("col0").addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build(); - List> inputData = new ArrayList<>(); - inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); - inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); - inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); - inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); - inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); - inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); - inputData.add(Arrays.asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1))); - inputData.add(Arrays.asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1))); - inputData.add(Arrays.asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1))); - JavaRDD> rdd = sc.parallelize(inputData); - Map> l = AnalyzeSpark.getUnique(Arrays.asList("col0", "col1"), schema, rdd); - assertEquals(2, l.size()); - List c0 = l.get("col0"); - assertEquals(3, c0.size()); - assertTrue(c0.contains(new IntWritable(0)) && c0.contains(new IntWritable(1)) && c0.contains(new IntWritable(2))); - List c1 = l.get("col1"); - assertEquals(3, c1.size()); - assertTrue(c1.contains(new Text("state0")) && c1.contains(new Text("state1")) && c1.contains(new Text("state2"))); - } - - - - @Test - @DisplayName("Test First Digit Transform Benfords Law") - void testFirstDigitTransformBenfordsLaw() { - Schema s = new Schema.Builder().addColumnString("data").addColumnDouble("double").addColumnString("stringNumber").build(); - List> in = Arrays.asList(Arrays.asList(new Text("a"), new DoubleWritable(3.14159), new Text("8e-4")), Arrays.asList(new Text("a2"), new DoubleWritable(3.14159), new Text("7e-4")), Arrays.asList(new Text("b"), new DoubleWritable(2.71828), new Text("7e2")), Arrays.asList(new Text("c"), new DoubleWritable(1.61803), new Text("6e8")), Arrays.asList(new Text("c"), new DoubleWritable(1.61803), new Text("2.0")), Arrays.asList(new Text("c"), new DoubleWritable(1.61803), new Text("2.1")), Arrays.asList(new Text("c"), new DoubleWritable(1.61803), new Text("2.2")), Arrays.asList(new Text("c"), new DoubleWritable(-2), new Text("non numerical"))); - // Test Benfords law use case: - TransformProcess tp = new TransformProcess.Builder(s).firstDigitTransform("double", "fdDouble", FirstDigitTransform.Mode.EXCEPTION_ON_INVALID).firstDigitTransform("stringNumber", "stringNumber", FirstDigitTransform.Mode.INCLUDE_OTHER_CATEGORY).removeAllColumnsExceptFor("stringNumber").categoricalToOneHot("stringNumber").reduce(new Reducer.Builder(ReduceOp.Sum).build()).build(); - JavaRDD> rdd = sc.parallelize(in); - List> out = SparkTransformExecutor.execute(rdd, tp).collect(); - assertEquals(1, out.size()); - List l = out.get(0); - List exp = Arrays.asList(// 0 - new IntWritable(0), // 1 - new IntWritable(0), // 2 - new IntWritable(3), // 3 - new IntWritable(0), // 4 - new IntWritable(0), // 5 - new IntWritable(0), // 6 - new IntWritable(1), // 7 - new IntWritable(2), // 8 - new IntWritable(1), // 9 - new IntWritable(0), // Other - new IntWritable(1)); - assertEquals(exp, l); - } -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/join/TestJoin.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/join/TestJoin.java deleted file mode 100644 index d6a32e194..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/join/TestJoin.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.spark.transform.join; - -import org.apache.spark.api.java.JavaRDD; -import org.datavec.api.transform.ColumnType; -import org.datavec.api.transform.join.Join; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.*; -import org.datavec.spark.BaseSparkTest; -import org.datavec.spark.transform.SparkTransformExecutor; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.*; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -public class TestJoin extends BaseSparkTest { - - @Test - public void testJoinOneToMany_ManyToOne() { - - Schema customerInfoSchema = - new Schema.Builder().addColumnLong("customerID").addColumnString("customerName").build(); - - Schema purchasesSchema = new Schema.Builder().addColumnLong("purchaseID").addColumnLong("customerID") - .addColumnDouble("amount").build(); - - List> infoList = new ArrayList<>(); - infoList.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"))); - infoList.add(Arrays.asList(new LongWritable(98765), new Text("Customer98765"))); - infoList.add(Arrays.asList(new LongWritable(50000), new Text("Customer50000"))); - - List> purchaseList = new ArrayList<>(); - purchaseList.add(Arrays.asList(new LongWritable(1000000), new LongWritable(12345), - new DoubleWritable(10.00))); - purchaseList.add(Arrays.asList(new LongWritable(1000001), new LongWritable(12345), - new DoubleWritable(20.00))); - purchaseList.add(Arrays.asList(new LongWritable(1000002), new LongWritable(98765), - new DoubleWritable(30.00))); - - Join join = new Join.Builder(Join.JoinType.RightOuter).setJoinColumns("customerID") - .setSchemas(customerInfoSchema, purchasesSchema).build(); - - List> expected = new ArrayList<>(); - expected.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"), - new LongWritable(1000000), new DoubleWritable(10.00))); - expected.add(Arrays.asList(new LongWritable(12345), new Text("Customer12345"), - new LongWritable(1000001), new DoubleWritable(20.00))); - expected.add(Arrays.asList(new LongWritable(98765), new Text("Customer98765"), - new LongWritable(1000002), new DoubleWritable(30.00))); - - - - JavaRDD> info = sc.parallelize(infoList); - JavaRDD> purchases = sc.parallelize(purchaseList); - - JavaRDD> joined = SparkTransformExecutor.executeJoin(join, info, purchases); - List> joinedList = new ArrayList<>(joined.collect()); - //Sort by order ID (column 3, index 2) - Collections.sort(joinedList, new Comparator>() { - @Override - public int compare(List o1, List o2) { - return Long.compare(o1.get(2).toLong(), o2.get(2).toLong()); - } - }); - assertEquals(expected, joinedList); - - assertEquals(3, joinedList.size()); - - List expectedColNames = Arrays.asList("customerID", "customerName", "purchaseID", "amount"); - assertEquals(expectedColNames, join.getOutputSchema().getColumnNames()); - - List expectedColTypes = - Arrays.asList(ColumnType.Long, ColumnType.String, ColumnType.Long, ColumnType.Double); - assertEquals(expectedColTypes, join.getOutputSchema().getColumnTypes()); - - - //Test Many to one: same thing, but swap the order... - Join join2 = new Join.Builder(Join.JoinType.LeftOuter).setJoinColumns("customerID") - .setSchemas(purchasesSchema, customerInfoSchema).build(); - - List> expectedManyToOne = new ArrayList<>(); - expectedManyToOne.add(Arrays.asList(new LongWritable(1000000), new LongWritable(12345), - new DoubleWritable(10.00), new Text("Customer12345"))); - expectedManyToOne.add(Arrays.asList(new LongWritable(1000001), new LongWritable(12345), - new DoubleWritable(20.00), new Text("Customer12345"))); - expectedManyToOne.add(Arrays.asList(new LongWritable(1000002), new LongWritable(98765), - new DoubleWritable(30.00), new Text("Customer98765"))); - - JavaRDD> joined2 = SparkTransformExecutor.executeJoin(join2, purchases, info); - List> joinedList2 = new ArrayList<>(joined2.collect()); - //Sort by order ID (column 0) - Collections.sort(joinedList2, new Comparator>() { - @Override - public int compare(List o1, List o2) { - return Long.compare(o1.get(0).toLong(), o2.get(0).toLong()); - } - }); - assertEquals(3, joinedList2.size()); - - assertEquals(expectedManyToOne, joinedList2); - - List expectedColNames2 = Arrays.asList("purchaseID", "customerID", "amount", "customerName"); - assertEquals(expectedColNames2, join2.getOutputSchema().getColumnNames()); - - List expectedColTypes2 = - Arrays.asList(ColumnType.Long, ColumnType.Long, ColumnType.Double, ColumnType.String); - assertEquals(expectedColTypes2, join2.getOutputSchema().getColumnTypes()); - } - - - @Test - public void testJoinManyToMany() { - Schema schema1 = new Schema.Builder().addColumnLong("id") - .addColumnCategorical("category", Arrays.asList("cat0", "cat1", "cat2")).build(); - - Schema schema2 = new Schema.Builder().addColumnLong("otherId") - .addColumnCategorical("otherCategory", Arrays.asList("cat0", "cat1", "cat2")).build(); - - List> first = new ArrayList<>(); - first.add(Arrays.asList(new LongWritable(0), new Text("cat0"))); - first.add(Arrays.asList(new LongWritable(1), new Text("cat0"))); - first.add(Arrays.asList(new LongWritable(2), new Text("cat1"))); - - List> second = new ArrayList<>(); - second.add(Arrays.asList(new LongWritable(100), new Text("cat0"))); - second.add(Arrays.asList(new LongWritable(101), new Text("cat0"))); - second.add(Arrays.asList(new LongWritable(102), new Text("cat2"))); - - - - List> expOuterJoin = new ArrayList<>(); - expOuterJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); - expOuterJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); - expOuterJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); - expOuterJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); - expOuterJoin.add(Arrays.asList(new LongWritable(2), new Text("cat1"), new NullWritable())); - expOuterJoin.add(Arrays.asList(new NullWritable(), new Text("cat2"), new LongWritable(102))); - - List> expLeftJoin = new ArrayList<>(); - expLeftJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); - expLeftJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); - expLeftJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); - expLeftJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); - expLeftJoin.add(Arrays.asList(new LongWritable(2), new Text("cat1"), new NullWritable())); - - - List> expRightJoin = new ArrayList<>(); - expRightJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); - expRightJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); - expRightJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); - expRightJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); - expRightJoin.add(Arrays.asList(new NullWritable(), new Text("cat2"), new LongWritable(102))); - - List> expInnerJoin = new ArrayList<>(); - expInnerJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(100))); - expInnerJoin.add(Arrays.asList(new LongWritable(0), new Text("cat0"), new LongWritable(101))); - expInnerJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(100))); - expInnerJoin.add(Arrays.asList(new LongWritable(1), new Text("cat0"), new LongWritable(101))); - - JavaRDD> firstRDD = sc.parallelize(first); - JavaRDD> secondRDD = sc.parallelize(second); - - int count = 0; - for (Join.JoinType jt : Join.JoinType.values()) { - Join join = new Join.Builder(jt).setJoinColumnsLeft("category").setJoinColumnsRight("otherCategory") - .setSchemas(schema1, schema2).build(); - List> out = - new ArrayList<>(SparkTransformExecutor.executeJoin(join, firstRDD, secondRDD).collect()); - - //Sort output by column 0, then column 1, then column 2 for comparison to expected... - Collections.sort(out, new Comparator>() { - @Override - public int compare(List o1, List o2) { - Writable w1 = o1.get(0); - Writable w2 = o2.get(0); - if (w1 instanceof NullWritable) - return 1; - else if (w2 instanceof NullWritable) - return -1; - int c = Long.compare(w1.toLong(), w2.toLong()); - if (c != 0) - return c; - c = o1.get(1).toString().compareTo(o2.get(1).toString()); - if (c != 0) - return c; - w1 = o1.get(2); - w2 = o2.get(2); - if (w1 instanceof NullWritable) - return 1; - else if (w2 instanceof NullWritable) - return -1; - return Long.compare(w1.toLong(), w2.toLong()); - } - }); - - switch (jt) { - case Inner: - assertEquals(expInnerJoin, out); - break; - case LeftOuter: - assertEquals(expLeftJoin, out); - break; - case RightOuter: - assertEquals(expRightJoin, out); - break; - case FullOuter: - assertEquals(expOuterJoin, out); - break; - } - count++; - } - - assertEquals(4, count); - } - -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/rank/TestCalculateSortedRank.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/rank/TestCalculateSortedRank.java deleted file mode 100644 index de6818e06..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/rank/TestCalculateSortedRank.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.spark.transform.rank; - -import org.apache.spark.api.java.JavaRDD; -import org.datavec.api.transform.ColumnType; -import org.datavec.api.transform.TransformProcess; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.datavec.api.writable.comparator.DoubleWritableComparator; -import org.datavec.spark.BaseSparkTest; -import org.datavec.spark.transform.SparkTransformExecutor; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -public class TestCalculateSortedRank extends BaseSparkTest { - - @Test - public void testCalculateSortedRank() { - - List> data = new ArrayList<>(); - data.add(Arrays.asList(new Text("0"), new DoubleWritable(0.0))); - data.add(Arrays.asList(new Text("3"), new DoubleWritable(0.3))); - data.add(Arrays.asList(new Text("2"), new DoubleWritable(0.2))); - data.add(Arrays.asList(new Text("1"), new DoubleWritable(0.1))); - - JavaRDD> rdd = sc.parallelize(data); - - Schema schema = new Schema.Builder().addColumnsString("TextCol").addColumnDouble("DoubleCol").build(); - - TransformProcess tp = new TransformProcess.Builder(schema) - .calculateSortedRank("rank", "DoubleCol", new DoubleWritableComparator()).build(); - - Schema outSchema = tp.getFinalSchema(); - assertEquals(3, outSchema.numColumns()); - assertEquals(Arrays.asList("TextCol", "DoubleCol", "rank"), outSchema.getColumnNames()); - assertEquals(Arrays.asList(ColumnType.String, ColumnType.Double, ColumnType.Long), outSchema.getColumnTypes()); - - JavaRDD> out = SparkTransformExecutor.execute(rdd, tp); - - List> collected = out.collect(); - assertEquals(4, collected.size()); - for (int i = 0; i < 4; i++) - assertEquals(3, collected.get(i).size()); - - for (List example : collected) { - int exampleNum = example.get(0).toInt(); - int rank = example.get(2).toInt(); - assertEquals(exampleNum, rank); - } - } - -} diff --git a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/sequence/TestConvertToSequence.java b/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/sequence/TestConvertToSequence.java deleted file mode 100644 index 3874cbaa3..000000000 --- a/datavec/datavec-spark/src/test/java/org/datavec/spark/transform/sequence/TestConvertToSequence.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.datavec.spark.transform.sequence; - -import org.apache.spark.api.java.JavaRDD; -import org.datavec.api.transform.TransformProcess; -import org.datavec.api.transform.schema.Schema; -import org.datavec.api.transform.sequence.comparator.NumericalColumnComparator; -import org.datavec.api.writable.LongWritable; -import org.datavec.api.writable.Text; -import org.datavec.api.writable.Writable; -import org.datavec.spark.BaseSparkTest; -import org.datavec.spark.transform.SparkTransformExecutor; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -public class TestConvertToSequence extends BaseSparkTest { - - @Test - public void testConvertToSequenceCompoundKey() { - - Schema s = new Schema.Builder().addColumnsString("key1", "key2").addColumnLong("time").build(); - - List> allExamples; - allExamples = Arrays.asList(Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(10)), - Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(10)), - Arrays.asList(new Text("k1a"), new Text("k2a"), - new LongWritable(-10)), - Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(5)), - Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(0))); - - TransformProcess tp = new TransformProcess.Builder(s) - .convertToSequence(Arrays.asList("key1", "key2"), new NumericalColumnComparator("time")) - .build(); - - JavaRDD> rdd = sc.parallelize(allExamples); - - List>> out = SparkTransformExecutor.executeToSequence(rdd, tp).collect(); - - assertEquals(2, out.size()); - List> seq0; - List> seq1; - if (out.get(0).size() == 3) { - seq0 = out.get(0); - seq1 = out.get(1); - } else { - seq0 = out.get(1); - seq1 = out.get(0); - } - - List> expSeq0 = Arrays.asList( - Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(-10)), - Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(0)), - Arrays.asList(new Text("k1a"), new Text("k2a"), new LongWritable(10))); - - List> expSeq1 = Arrays.asList( - Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(5)), - Arrays.asList(new Text("k1b"), new Text("k2b"), new LongWritable(10))); - - assertEquals(expSeq0, seq0); - assertEquals(expSeq1, seq1); - } - - @Test - public void testConvertToSequenceLength1(){ - - Schema s = new Schema.Builder() - .addColumnsString("string") - .addColumnLong("long") - .build(); - - List> allExamples = Arrays.asList( - Arrays.asList(new Text("a"), new LongWritable(0)), - Arrays.asList(new Text("b"), new LongWritable(1)), - Arrays.asList(new Text("c"), new LongWritable(2))); - - TransformProcess tp = new TransformProcess.Builder(s) - .convertToSequence() - .build(); - - JavaRDD> rdd = sc.parallelize(allExamples); - - JavaRDD>> out = SparkTransformExecutor.executeToSequence(rdd, tp); - - List>> out2 = out.collect(); - - assertEquals(3, out2.size()); - - for( int i=0; i<3; i++ ){ - assertTrue(out2.contains(Collections.singletonList(allExamples.get(i)))); - } - } -} diff --git a/datavec/pom.xml b/datavec/pom.xml deleted file mode 100644 index dd8e923b2..000000000 --- a/datavec/pom.xml +++ /dev/null @@ -1,276 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j - 1.0.0-SNAPSHOT - - - org.datavec - datavec-parent - pom - - DataVec - Vectorization Rosetta Stone for the JVM - - - - - datavec-api - datavec-data - datavec-spark - datavec-local - datavec-jdbc - datavec-excel - datavec-arrow - - - - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - commons-io - commons-io - ${commons-io.version} - - - org.slf4j - slf4j-api - ${slf4j.version} - - - joda-time - joda-time - ${jodatime.version} - - - - org.nd4j - nd4j-common - ${nd4j.version} - - - - org.nd4j - nd4j-api - ${nd4j.version} - - - - - - - org.junit.jupiter - junit-jupiter-api - - - org.junit.jupiter - junit-jupiter-params - - - org.junit.vintage - junit-vintage-engine - - - com.tngtech.archunit - archunit-junit5-engine - ${archunit.version} - test - - - com.tngtech.archunit - archunit-junit5-api - ${archunit.version} - test - - - org.projectlombok - lombok - ${lombok.version} - provided - - - ch.qos.logback - logback-classic - ${logback.version} - test - - - - - - - - org.kuali.maven.wagons - maven-s3-wagon - 1.2.1 - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - ${maven-enforcer-plugin.version} - - - test - enforce-test-resources - - enforce - - - ${skipTestResourceEnforcement} - - - nd4j-tests-cpu,nd4j-tests-cuda - false - - - true - - - - - - org.eclipse.m2e - lifecycle-mapping - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - com.lewisd - lint-maven-plugin - ${maven-lint-plugin.version} - - true - - DuplicateDep - RedundantDepVersion - RedundantPluginVersion - - - ${project.build.directory}/maven-lint-result.xml - - - - pom-lint - validate - - check - - - - - - net.revelc.code.formatter - formatter-maven-plugin - - - datavec-api - datavec-arrow - datavec-data - datavec-excel - datavec-jdbc - datavec-local - datavec-python - datavec-spark - datavec-spark-inference-parent - - - - - - pl.project13.maven - git-commit-id-plugin - - - - org.codehaus.mojo - build-helper-maven-plugin - - - - - - - nd4j-tests-cpu - - - org.nd4j - nd4j-native - ${nd4j.version} - test - - - org.deeplearning4j - dl4j-test-resources - ${nd4j.version} - test - - - - - nd4j-tests-cuda - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - org.deeplearning4j - dl4j-test-resources - ${nd4j.version} - test - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - - - diff --git a/deeplearning4j.ipr b/deeplearning4j.ipr new file mode 100644 index 000000000..654bca4c6 --- /dev/null +++ b/deeplearning4j.ipr @@ -0,0 +1,213 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deeplearning4j.iws b/deeplearning4j.iws new file mode 100644 index 000000000..57de9a0c5 --- /dev/null +++ b/deeplearning4j.iws @@ -0,0 +1,418 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deeplearning4j/CONTRIBUTORS.md b/deeplearning4j/CONTRIBUTORS.md old mode 100755 new mode 100644 diff --git a/deeplearning4j/README.md b/deeplearning4j/README.md old mode 100755 new mode 100644 diff --git a/deeplearning4j/buildmultiplescalaversions.sh b/deeplearning4j/buildmultiplescalaversions.sh old mode 100755 new mode 100644 diff --git a/deeplearning4j/deeplearning4j-common-tests/pom.xml b/deeplearning4j/deeplearning4j-common-tests/pom.xml deleted file mode 100644 index 7e1f27e15..000000000 --- a/deeplearning4j/deeplearning4j-common-tests/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-common-tests - - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - provided - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - provided - - - - org.nd4j - nd4j-api - ${project.version} - - - org.nd4j - nd4j-common-tests - ${nd4j.version} - - - ch.qos.logback - logback-classic - - - - - - nd4j-tests-cpu - - - org.nd4j - nd4j-native - ${project.version} - test - - - - - nd4j-tests-cuda - - - org.nd4j - nd4j-cuda-11.0 - ${project.version} - test - - - - - diff --git a/deeplearning4j/deeplearning4j-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java b/deeplearning4j/deeplearning4j-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java deleted file mode 100644 index 8baa8cc6c..000000000 --- a/deeplearning4j/deeplearning4j-common-tests/src/main/java/org/deeplearning4j/BaseDL4JTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j; - -import ch.qos.logback.classic.LoggerContext; -import org.bytedeco.javacpp.Pointer; -import org.junit.jupiter.api.*; - -import org.nd4j.common.base.Preconditions; -import org.nd4j.common.config.ND4JSystemProperties; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.linalg.api.ops.executioner.OpExecutioner; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.profiler.ProfilerConfig; -import org.slf4j.ILoggerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.lang.management.ManagementFactory; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - - -@DisplayName("Base DL 4 J Test") -public abstract class BaseDL4JTest { - - private static Logger log = LoggerFactory.getLogger(BaseDL4JTest.class.getName()); - - protected long startTime; - - protected int threadCountBefore; - - private final int DEFAULT_THREADS = Runtime.getRuntime().availableProcessors(); - - /** - * Override this to specify the number of threads for C++ execution, via - * {@link org.nd4j.linalg.factory.Environment#setMaxMasterThreads(int)} - * @return Number of threads to use for C++ op execution - */ - public int numThreads() { - return DEFAULT_THREADS; - } - - /** - * Override this method to set the default timeout for methods in the test class - */ - public long getTimeoutMilliseconds() { - return 90_000; - } - - /** - * Override this to set the profiling mode for the tests defined in the child class - */ - public OpExecutioner.ProfilingMode getProfilingMode() { - return OpExecutioner.ProfilingMode.SCOPE_PANIC; - } - - /** - * Override this to set the datatype of the tests defined in the child class - */ - public DataType getDataType() { - return DataType.DOUBLE; - } - - public DataType getDefaultFPDataType() { - return getDataType(); - } - - protected static Boolean integrationTest; - - /** - * @return True if integration tests maven profile is enabled, false otherwise. - */ - public static boolean isIntegrationTests() { - if (integrationTest == null) { - String prop = System.getenv("DL4J_INTEGRATION_TESTS"); - integrationTest = Boolean.parseBoolean(prop); - } - return integrationTest; - } - - /** - * Call this as the first line of a test in order to skip that test, only when the integration tests maven profile is not enabled. - * This can be used to dynamically skip integration tests when the integration test profile is not enabled. - * Note that the integration test profile is not enabled by default - "integration-tests" profile - */ - public static void skipUnlessIntegrationTests() { - assumeTrue(isIntegrationTests(), "Skipping integration test - integration profile is not enabled"); - } - - @BeforeEach - @Timeout(90000L) - void beforeTest(TestInfo testInfo) { - log.info("{}.{}", getClass().getSimpleName(), testInfo.getTestMethod().get().getName()); - // Suppress ND4J initialization - don't need this logged for every test... - System.setProperty(ND4JSystemProperties.LOG_INITIALIZATION, "false"); - System.setProperty(ND4JSystemProperties.ND4J_IGNORE_AVX, "true"); - Nd4j.getExecutioner().setProfilingMode(getProfilingMode()); - Nd4j.getExecutioner().setProfilingConfig(ProfilerConfig.builder().build()); - Nd4j.setDefaultDataTypes(getDataType(), getDefaultFPDataType()); - Nd4j.getExecutioner().setProfilingConfig(ProfilerConfig.builder().build()); - Nd4j.getExecutioner().enableDebugMode(false); - Nd4j.getExecutioner().enableVerboseMode(false); - int numThreads = numThreads(); - Preconditions.checkState(numThreads > 0, "Number of threads must be > 0"); - if (numThreads != Nd4j.getEnvironment().maxMasterThreads()) { - Nd4j.getEnvironment().setMaxMasterThreads(numThreads); - } - startTime = System.currentTimeMillis(); - threadCountBefore = ManagementFactory.getThreadMXBean().getThreadCount(); - } - - @AfterEach - void afterTest(TestInfo testInfo) { - // Attempt to keep workspaces isolated between tests - Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); - MemoryWorkspace currWS = Nd4j.getMemoryManager().getCurrentWorkspace(); - Nd4j.getMemoryManager().setCurrentWorkspace(null); - if (currWS != null) { - // Not really safe to continue testing under this situation... other tests will likely fail with obscure - // errors that are hard to track back to this - log.error("Open workspace leaked from test! Exiting - {}, isOpen = {} - {}", currWS.getId(), currWS.isScopeActive(), currWS); - System.out.println("Open workspace leaked from test! Exiting - " + currWS.getId() + ", isOpen = " + currWS.isScopeActive() + " - " + currWS); - System.out.flush(); - // Try to flush logs also: - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - ILoggerFactory lf = LoggerFactory.getILoggerFactory(); - if (lf instanceof LoggerContext) { - ((LoggerContext) lf).stop(); - } - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - System.exit(1); - } - StringBuilder sb = new StringBuilder(); - long maxPhys = Pointer.maxPhysicalBytes(); - long maxBytes = Pointer.maxBytes(); - long currPhys = Pointer.physicalBytes(); - long currBytes = Pointer.totalBytes(); - long jvmTotal = Runtime.getRuntime().totalMemory(); - long jvmMax = Runtime.getRuntime().maxMemory(); - int threadsAfter = ManagementFactory.getThreadMXBean().getThreadCount(); - long duration = System.currentTimeMillis() - startTime; - sb.append(getClass().getSimpleName()).append(".").append(testInfo.getTestMethod().get().getName()).append(": ").append(duration).append(" ms").append(", threadCount: (").append(threadCountBefore).append("->").append(threadsAfter).append(")").append(", jvmTotal=").append(jvmTotal).append(", jvmMax=").append(jvmMax).append(", totalBytes=").append(currBytes).append(", maxBytes=").append(maxBytes).append(", currPhys=").append(currPhys).append(", maxPhys=").append(maxPhys); - List ws = Nd4j.getWorkspaceManager().getAllWorkspacesForCurrentThread(); - if (ws != null && ws.size() > 0) { - long currSize = 0; - for (MemoryWorkspace w : ws) { - currSize += w.getCurrentSize(); - } - if (currSize > 0) { - sb.append(", threadWSSize=").append(currSize).append(" (").append(ws.size()).append(" WSs)"); - } - } - Properties p = Nd4j.getExecutioner().getEnvironmentInformation(); - Object o = p.get("cuda.devicesInformation"); - if (o instanceof List) { - List> l = (List>) o; - if (l.size() > 0) { - sb.append(" [").append(l.size()).append(" GPUs: "); - for (int i = 0; i < l.size(); i++) { - Map m = l.get(i); - if (i > 0) - sb.append(","); - sb.append("(").append(m.get("cuda.freeMemory")).append(" free, ").append(m.get("cuda.totalMemory")).append(" total)"); - } - sb.append("]"); - } - } - log.info(sb.toString()); - } -} diff --git a/deeplearning4j/deeplearning4j-common/pom.xml b/deeplearning4j/deeplearning4j-common/pom.xml deleted file mode 100644 index e2be6465f..000000000 --- a/deeplearning4j/deeplearning4j-common/pom.xml +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-common - - - - org.nd4j - nd4j-common - ${nd4j.version} - - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/DL4JClassLoadingTest.java b/deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/DL4JClassLoadingTest.java deleted file mode 100644 index 73757e214..000000000 --- a/deeplearning4j/deeplearning4j-common/src/test/java/org/deeplearning4j/common/config/DL4JClassLoadingTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.common.config; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import org.deeplearning4j.common.config.dummies.TestAbstract; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Dl 4 J Class Loading Test") -class DL4JClassLoadingTest { - - private static final String PACKAGE_PREFIX = "org.deeplearning4j.common.config.dummies."; - - @Test - @DisplayName("Test Create New Instance _ constructor Without Arguments") - void testCreateNewInstance_constructorWithoutArguments() { - /* Given */ - String className = PACKAGE_PREFIX + "TestDummy"; - /* When */ - Object instance = DL4JClassLoading.createNewInstance(className); - /* Then */ - assertNotNull(instance); - assertEquals(className, instance.getClass().getName()); - } - - @Test - @DisplayName("Test Create New Instance _ constructor With Argument _ implicit Argument Types") - void testCreateNewInstance_constructorWithArgument_implicitArgumentTypes() { - /* Given */ - String className = PACKAGE_PREFIX + "TestColor"; - /* When */ - TestAbstract instance = DL4JClassLoading.createNewInstance(className, TestAbstract.class, "white"); - /* Then */ - assertNotNull(instance); - assertEquals(className, instance.getClass().getName()); - } - - @Test - @DisplayName("Test Create New Instance _ constructor With Argument _ explicit Argument Types") - void testCreateNewInstance_constructorWithArgument_explicitArgumentTypes() { - /* Given */ - String colorClassName = PACKAGE_PREFIX + "TestColor"; - String rectangleClassName = PACKAGE_PREFIX + "TestRectangle"; - /* When */ - TestAbstract color = DL4JClassLoading.createNewInstance(colorClassName, Object.class, new Class[] { int.class, int.class, int.class }, 45, 175, 200); - TestAbstract rectangle = DL4JClassLoading.createNewInstance(rectangleClassName, Object.class, new Class[] { int.class, int.class, TestAbstract.class }, 10, 15, color); - /* Then */ - assertNotNull(color); - assertEquals(colorClassName, color.getClass().getName()); - assertNotNull(rectangle); - assertEquals(rectangleClassName, rectangle.getClass().getName()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/pom.xml b/deeplearning4j/deeplearning4j-core/pom.xml deleted file mode 100644 index 3af6638aa..000000000 --- a/deeplearning4j/deeplearning4j-core/pom.xml +++ /dev/null @@ -1,198 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-core - - - 1g - 1g - - - - - - org.nd4j - nd4j-api - ${nd4j.version} - - - org.nd4j - nd4j-common - ${nd4j.version} - - - - - - - org.deeplearning4j - deeplearning4j-datasets - ${project.version} - - - org.deeplearning4j - deeplearning4j-datavec-iterators - ${project.version} - - - org.deeplearning4j - deeplearning4j-modelimport - ${project.version} - - - - org.slf4j - slf4j-api - - - ch.qos.logback - logback-classic - test - - - org.deeplearning4j - deeplearning4j-nn - ${project.version} - - - org.apache.commons - commons-math3 - - - commons-io - commons-io - ${commonsio.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - org.junit.jupiter - junit-jupiter-api - - - org.junit.vintage - junit-vintage-engine - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - org.nd4j - nd4j-api - - - org.apache.commons - commons-lang3 - ${commonslang.version} - - - - org.nd4j - jackson - ${nd4j.version} - - - org.projectlombok - lombok - ${lombok.version} - provided - - - org.datavec - datavec-api - ${datavec.version} - - - org.datavec - datavec-data-image - ${datavec.version} - - - - org.deeplearning4j - deeplearning4j-ui-components - ${project.version} - - - - javax.xml.bind - jaxb-api - ${jaxb.version} - provided - - - - com.github.oshi - oshi-json - ${oshi.version} - - - com.github.oshi - oshi-core - ${oshi.version} - - - - - - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - - - diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/RandomTests.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/RandomTests.java deleted file mode 100644 index 01df1b517..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/RandomTests.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j; - -import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.RmsProp; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.nio.file.Files; -import java.util.concurrent.CountDownLatch; - -@NativeTag -@Tag(TagNames.RNG) -public class RandomTests extends BaseDL4JTest { - - @Test - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.LONG_TEST) - public void testReproduce() throws Exception { - - final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new RmsProp()) - .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list() - .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(28 * 28).nOut(10) - .activation(Activation.TANH).build()) - .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( - LossFunctions.LossFunction.MCXENT).nIn(10).nOut(10) - .activation(Activation.SOFTMAX).build()) - .build(); - - for (int e = 0; e < 3; e++) { - - int nThreads = 10; - final CountDownLatch l = new CountDownLatch(nThreads); - for (int i = 0; i < nThreads; i++) { - final int j = i; - Thread t = new Thread(new Runnable() { - @Override - public void run() { - try { - MultiLayerNetwork net = new MultiLayerNetwork(conf.clone()); - net.init(); - DataSetIterator iter = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(10, false, 12345), 100); - net.fit(iter); - } catch (Throwable t) { - System.out.println("Thread failed: " + j); - t.printStackTrace(); - } finally { - l.countDown(); - } - } - }); - t.start(); - } - - l.await(); - System.out.println("DONE " + e + "\n"); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/MnistFetcherTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/MnistFetcherTest.java deleted file mode 100644 index 9b0d7c050..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/MnistFetcherTest.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.common.resources.DL4JResources; -import org.deeplearning4j.datasets.base.MnistFetcher; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.junit.jupiter.api.*; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.conditions.Conditions; - -import java.io.File; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.*; - -@DisplayName("Mnist Fetcher Test") -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.NDARRAY_ETL) -class MnistFetcherTest extends BaseDL4JTest { - - @TempDir public static Path tempPath; - - @BeforeAll - static void setup() throws Exception { - DL4JResources.setBaseDirectory(tempPath.toFile()); - } - - @AfterAll - static void after() throws Exception { - DL4JResources.resetBaseDirectoryLocation(); - } - - @Test - @DisplayName("Test Mnist") - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.FILE_IO) - void testMnist() throws Exception { - MnistDataSetIterator iter = new MnistDataSetIterator(32, 60000, false, true, false, -1); - int count = 0; - while (iter.hasNext()) { - DataSet ds = iter.next(); - INDArray arr = ds.getFeatures().sum(1); - int countMatch = Nd4j.getExecutioner().execAndReturn(new MatchCondition(arr, Conditions.equals(0))).z().getInt(0); - assertEquals(0, countMatch); - count++; - } - assertEquals(60000 / 32, count); - count = 0; - iter = new MnistDataSetIterator(32, false, 12345); - while (iter.hasNext()) { - DataSet ds = iter.next(); - INDArray arr = ds.getFeatures().sum(1); - int countMatch = Nd4j.getExecutioner().execAndReturn(new MatchCondition(arr, Conditions.equals(0))).z().getInt(0); - assertEquals(0, countMatch); - count++; - } - assertEquals((int) Math.ceil(10000 / 32.0), count); - iter.close(); - } - - @Test - @DisplayName("Test Mnist Data Fetcher") - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.FILE_IO) - @Disabled("Temp directory not being set properly on CI") - @Tag(TagNames.NEEDS_VERIFY) - void testMnistDataFetcher() throws Exception { - MnistFetcher mnistFetcher = new MnistFetcher(); - File mnistDir = mnistFetcher.downloadAndUntar(); - assertTrue(mnistDir.isDirectory()); - - } - - @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.FILE_IO) - @Disabled("Temp directory not being set properly on CI") - @Tag(TagNames.NEEDS_VERIFY) - public void testMnistSubset() throws Exception { - final int numExamples = 100; - MnistDataSetIterator iter1 = new MnistDataSetIterator(10, numExamples, false, true, true, 123); - int examples1 = 0; - int itCount1 = 0; - while (iter1.hasNext()) { - itCount1++; - examples1 += iter1.next().numExamples(); - } - assertEquals(10, itCount1); - assertEquals(100, examples1); - iter1.close(); - MnistDataSetIterator iter2 = new MnistDataSetIterator(10, numExamples, false, true, true, 123); - iter2.close(); - int examples2 = 0; - int itCount2 = 0; - for (int i = 0; i < 10; i++) { - itCount2++; - examples2 += iter2.next().numExamples(); - } - assertFalse(iter2.hasNext()); - assertEquals(10, itCount2); - assertEquals(100, examples2); - MnistDataSetIterator iter3 = new MnistDataSetIterator(19, numExamples, false, true, true, 123); - iter3.close(); - int examples3 = 0; - int itCount3 = 0; - while (iter3.hasNext()) { - itCount3++; - examples3 += iter3.next().numExamples(); - } - assertEquals(100, examples3); - assertEquals((int) Math.ceil(100 / 19.0), itCount3); - MnistDataSetIterator iter4 = new MnistDataSetIterator(32, true, 12345); - int count4 = 0; - while (iter4.hasNext()) { - count4 += iter4.next().numExamples(); - } - assertEquals(60000, count4); - iter4.close(); - iter1.close(); - } - - @Test - @DisplayName("Test Subset Repeatability") - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.FILE_IO) - @Disabled("Temp directory not being set properly on CI") - @Tag(TagNames.NEEDS_VERIFY) - void testSubsetRepeatability() throws Exception { - MnistDataSetIterator it = new MnistDataSetIterator(1, 1, false, false, true, 0); - DataSet d1 = it.next(); - for (int i = 0; i < 10; i++) { - it.reset(); - DataSet d2 = it.next(); - assertEquals(d1.get(0).getFeatures(), d2.get(0).getFeatures()); - } - it.close(); - // Check larger number: - it = new MnistDataSetIterator(8, 32, false, false, true, 12345); - Set featureLabelSet = new HashSet<>(); - while (it.hasNext()) { - DataSet ds = it.next(); - INDArray f = ds.getFeatures(); - INDArray l = ds.getLabels(); - for (int i = 0; i < f.size(0); i++) { - featureLabelSet.add(f.getRow(i).toString() + "\t" + l.getRow(i).toString()); - } - } - assertEquals(32, featureLabelSet.size()); - it.close(); - for (int i = 0; i < 3; i++) { - it.reset(); - Set flSet2 = new HashSet<>(); - while (it.hasNext()) { - DataSet ds = it.next(); - INDArray f = ds.getFeatures(); - INDArray l = ds.getLabels(); - for (int j = 0; j < f.size(0); j++) { - flSet2.add(f.getRow(j).toString() + "\t" + l.getRow(j).toString()); - } - } - assertEquals(featureLabelSet, flSet2); - } - - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetiteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetiteratorTest.java deleted file mode 100644 index 4153d52cc..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderDataSetiteratorTest.java +++ /dev/null @@ -1,1220 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.datavec; - - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.factory.Nd4jBackend; -import org.nd4j.shade.guava.io.Files; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.datavec.api.io.labels.ParentPathLabelGenerator; -import org.datavec.api.records.Record; -import org.datavec.api.records.metadata.RecordMetaData; -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.SequenceRecordReader; -import org.datavec.api.records.reader.impl.collection.CollectionRecordReader; -import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.split.InputStreamInputSplit; -import org.datavec.api.split.NumberedFileInputSplit; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.NDArrayWritable; -import org.datavec.api.writable.Writable; -import org.datavec.image.recordreader.ImageRecordReader; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.datasets.datavec.exception.ZeroLengthSequenceException; -import org.deeplearning4j.datasets.datavec.tools.SpecialImageRecordReader; -import org.nd4j.linalg.dataset.AsyncDataSetIterator; -import org.junit.jupiter.api.Disabled; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization; -import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.primitives.Pair; -import org.nd4j.common.resources.Resources; -import java.io.*; -import java.util.*; -import static org.junit.jupiter.api.Assertions.*; -import static org.nd4j.linalg.indexing.NDArrayIndex.all; -import static org.nd4j.linalg.indexing.NDArrayIndex.point; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.jupiter.api.Assertions.assertThrows; - -@Slf4j -@DisplayName("Record Reader Data Setiterator Test") -@Disabled -@NativeTag -class RecordReaderDataSetiteratorTest extends BaseDL4JTest { - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @TempDir - public Path temporaryFolder; - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Record Reader") - void testRecordReader(Nd4jBackend nd4jBackend) throws Exception { - RecordReader recordReader = new CSVRecordReader(); - FileSplit csv = new FileSplit(Resources.asFile("csv-example.csv")); - recordReader.initialize(csv); - DataSetIterator iter = new RecordReaderDataSetIterator(recordReader, 34); - DataSet next = iter.next(); - assertEquals(34, next.numExamples()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Record Reader Max Batch Limit") - void testRecordReaderMaxBatchLimit(Nd4jBackend backend) throws Exception { - RecordReader recordReader = new CSVRecordReader(); - FileSplit csv = new FileSplit(Resources.asFile("csv-example.csv")); - recordReader.initialize(csv); - DataSetIterator iter = new RecordReaderDataSetIterator(recordReader, 10, -1, -1, 2); - DataSet ds = iter.next(); - assertFalse(ds == null); - assertEquals(10, ds.numExamples()); - iter.hasNext(); - iter.next(); - assertEquals(false, iter.hasNext()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Record Reader Multi Regression") - void testRecordReaderMultiRegression(Nd4jBackend backend) throws Exception { - for (boolean builder : new boolean[] { false, true }) { - RecordReader csv = new CSVRecordReader(); - csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); - int batchSize = 3; - int labelIdxFrom = 3; - int labelIdxTo = 4; - DataSetIterator iter; - if (builder) { - iter = new RecordReaderDataSetIterator.Builder(csv, batchSize).regression(labelIdxFrom, labelIdxTo).build(); - } else { - iter = new RecordReaderDataSetIterator(csv, batchSize, labelIdxFrom, labelIdxTo, true); - } - DataSet ds = iter.next(); - INDArray f = ds.getFeatures(); - INDArray l = ds.getLabels(); - assertArrayEquals(new long[] { 3, 3 }, f.shape()); - assertArrayEquals(new long[] { 3, 2 }, l.shape()); - // Check values: - double[][] fExpD = new double[][] { { 5.1, 3.5, 1.4 }, { 4.9, 3.0, 1.4 }, { 4.7, 3.2, 1.3 } }; - double[][] lExpD = new double[][] { { 0.2, 0 }, { 0.2, 0 }, { 0.2, 0 } }; - INDArray fExp = Nd4j.create(fExpD).castTo(DataType.FLOAT); - INDArray lExp = Nd4j.create(lExpD).castTo(DataType.FLOAT); - assertEquals(fExp, f); - assertEquals(lExp, l); - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader") - @Tag(TagNames.NDARRAY_INDEXING) - void testSequenceRecordReader(Nd4jBackend backend) throws Exception { - File rootDir = temporaryFolder.toFile(); - // need to manually extract - for (int i = 0; i < 3; i++) { - FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); - FileUtils.copyFile(Resources.asFile(String.format("csvsequencelabels_%d.txt", i)), new File(rootDir, String.format("csvsequencelabels_%d.txt", i))); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); - assertEquals(3, iter.inputColumns()); - assertEquals(4, iter.totalOutcomes()); - List dsList = new ArrayList<>(); - while (iter.hasNext()) { - dsList.add(iter.next()); - } - // 3 files - assertEquals(3, dsList.size()); - for (int i = 0; i < 3; i++) { - DataSet ds = dsList.get(i); - INDArray features = ds.getFeatures(); - INDArray labels = ds.getLabels(); - // 1 example in mini-batch - assertEquals(1, features.size(0)); - assertEquals(1, labels.size(0)); - // 3 values per line/time step - assertEquals(3, features.size(1)); - // 1 value per line, but 4 possible values -> one-hot vector - assertEquals(4, labels.size(1)); - // sequence length = 4 - assertEquals(4, features.size(2)); - assertEquals(4, labels.size(2)); - } - // Check features vs. expected: - INDArray expF0 = Nd4j.create(1, 3, 4); - expF0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0, 1, 2 })); - expF0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 10, 11, 12 })); - expF0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 20, 21, 22 })); - expF0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 30, 31, 32 })); - assertEquals(dsList.get(0).getFeatures(), expF0); - INDArray expF1 = Nd4j.create(1, 3, 4); - expF1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 100, 101, 102 })); - expF1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 110, 111, 112 })); - expF1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 120, 121, 122 })); - expF1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 130, 131, 132 })); - assertEquals(dsList.get(1).getFeatures(), expF1); - INDArray expF2 = Nd4j.create(1, 3, 4); - expF2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 200, 201, 202 })); - expF2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 210, 211, 212 })); - expF2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 220, 221, 222 })); - expF2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 230, 231, 232 })); - assertEquals(dsList.get(2).getFeatures(), expF2); - // Check labels vs. expected: - INDArray expL0 = Nd4j.create(1, 4, 4); - expL0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 1, 0, 0, 0 })); - expL0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - expL0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 0, 0, 1, 0 })); - expL0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 0, 0, 0, 1 })); - assertEquals(dsList.get(0).getLabels(), expL0); - INDArray expL1 = Nd4j.create(1, 4, 4); - expL1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0, 0, 0, 1 })); - expL1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 0, 0, 1, 0 })); - expL1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - expL1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 1, 0, 0, 0 })); - assertEquals(dsList.get(1).getLabels(), expL1); - INDArray expL2 = Nd4j.create(1, 4, 4); - expL2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - expL2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 1, 0, 0, 0 })); - expL2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 0, 0, 0, 1 })); - expL2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 0, 0, 1, 0 })); - assertEquals(dsList.get(2).getLabels(), expL2); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Meta") - void testSequenceRecordReaderMeta(Nd4jBackend backend) throws Exception { - File rootDir = temporaryFolder.toFile(); - // need to manually extract - for (int i = 0; i < 3; i++) { - FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); - FileUtils.copyFile(Resources.asFile(String.format("csvsequencelabels_%d.txt", i)), new File(rootDir, String.format("csvsequencelabels_%d.txt", i))); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); - iter.setCollectMetaData(true); - assertEquals(3, iter.inputColumns()); - assertEquals(4, iter.totalOutcomes()); - while (iter.hasNext()) { - DataSet ds = iter.next(); - List meta = ds.getExampleMetaData(RecordMetaData.class); - DataSet fromMeta = iter.loadFromMetaData(meta); - assertEquals(ds, fromMeta); - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Regression") - void testSequenceRecordReaderRegression(Nd4jBackend backend) throws Exception { - // need to manually extract - File rootDir = temporaryFolder.toFile(); - for (int i = 0; i < 3; i++) { - FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 0, true); - assertEquals(3, iter.inputColumns()); - assertEquals(3, iter.totalOutcomes()); - List dsList = new ArrayList<>(); - while (iter.hasNext()) { - dsList.add(iter.next()); - } - // 3 files - assertEquals(3, dsList.size()); - for (int i = 0; i < 3; i++) { - DataSet ds = dsList.get(i); - INDArray features = ds.getFeatures(); - INDArray labels = ds.getLabels(); - // 1 examples, 3 values, 4 time steps - assertArrayEquals(new long[] { 1, 3, 4 }, features.shape()); - assertArrayEquals(new long[] { 1, 3, 4 }, labels.shape()); - assertEquals(features, labels); - } - // Also test regression + reset from a single reader: - featureReader.reset(); - iter = new SequenceRecordReaderDataSetIterator(featureReader, 1, 0, 2, true); - int count = 0; - while (iter.hasNext()) { - DataSet ds = iter.next(); - assertEquals(2, ds.getFeatures().size(1)); - assertEquals(1, ds.getLabels().size(1)); - count++; - } - assertEquals(3, count); - iter.reset(); - count = 0; - while (iter.hasNext()) { - iter.next(); - count++; - } - assertEquals(3, count); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Multi Regression") - void testSequenceRecordReaderMultiRegression(Nd4jBackend backend) throws Exception { - File rootDir = temporaryFolder.toFile(); - // need to manually extract - for (int i = 0; i < 3; i++) { - FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - SequenceRecordReader reader = new CSVSequenceRecordReader(1, ","); - reader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(reader, 1, 2, 1, true); - assertEquals(1, iter.inputColumns()); - assertEquals(2, iter.totalOutcomes()); - List dsList = new ArrayList<>(); - while (iter.hasNext()) { - dsList.add(iter.next()); - } - // 3 files - assertEquals(3, dsList.size()); - for (int i = 0; i < 3; i++) { - DataSet ds = dsList.get(i); - INDArray features = ds.getFeatures(); - INDArray labels = ds.getLabels(); - // 1 examples, 1 values, 4 time steps - assertArrayEquals(new long[] { 1, 1, 4 }, features.shape()); - assertArrayEquals(new long[] { 1, 2, 4 }, labels.shape()); - INDArray f2d = features.get(point(0), all(), all()).transpose(); - INDArray l2d = labels.get(point(0), all(), all()).transpose(); - switch(i) { - case 0: - assertEquals(Nd4j.create(new double[] { 0, 10, 20, 30 }, new int[] { 4, 1 }).castTo(DataType.FLOAT), f2d); - assertEquals(Nd4j.create(new double[][] { { 1, 2 }, { 11, 12 }, { 21, 22 }, { 31, 32 } }).castTo(DataType.FLOAT), l2d); - break; - case 1: - assertEquals(Nd4j.create(new double[] { 100, 110, 120, 130 }, new int[] { 4, 1 }).castTo(DataType.FLOAT), f2d); - assertEquals(Nd4j.create(new double[][] { { 101, 102 }, { 111, 112 }, { 121, 122 }, { 131, 132 } }).castTo(DataType.FLOAT), l2d); - break; - case 2: - assertEquals(Nd4j.create(new double[] { 200, 210, 220, 230 }, new int[] { 4, 1 }).castTo(DataType.FLOAT), f2d); - assertEquals(Nd4j.create(new double[][] { { 201, 202 }, { 211, 212 }, { 221, 222 }, { 231, 232 } }).castTo(DataType.FLOAT), l2d); - break; - default: - throw new RuntimeException(); - } - } - iter.reset(); - int count = 0; - while (iter.hasNext()) { - iter.next(); - count++; - } - assertEquals(3, count); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Reset") - void testSequenceRecordReaderReset(Nd4jBackend backend) throws Exception { - File rootDir = temporaryFolder.toFile(); - // need to manually extract - for (int i = 0; i < 3; i++) { - FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); - FileUtils.copyFile(Resources.asFile(String.format("csvsequencelabels_%d.txt", i)), new File(rootDir, String.format("csvsequencelabels_%d.txt", i))); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); - assertEquals(3, iter.inputColumns()); - assertEquals(4, iter.totalOutcomes()); - int nResets = 5; - for (int i = 0; i < nResets; i++) { - iter.reset(); - int count = 0; - while (iter.hasNext()) { - DataSet ds = iter.next(); - INDArray features = ds.getFeatures(); - INDArray labels = ds.getLabels(); - assertArrayEquals(new long[] { 1, 3, 4 }, features.shape()); - assertArrayEquals(new long[] { 1, 4, 4 }, labels.shape()); - count++; - } - assertEquals(3, count); - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test CSV Loading Regression") - void testCSVLoadingRegression(Nd4jBackend backend) throws Exception { - int nLines = 30; - int nFeatures = 5; - int miniBatchSize = 10; - int labelIdx = 0; - String path = "rr_csv_test_rand.csv"; - Pair p = makeRandomCSV(path, nLines, nFeatures); - double[][] data = p.getFirst(); - RecordReader testReader = new CSVRecordReader(); - testReader.initialize(new FileSplit(p.getSecond())); - DataSetIterator iter = new RecordReaderDataSetIterator(testReader, miniBatchSize, labelIdx, labelIdx, true); - int miniBatch = 0; - while (iter.hasNext()) { - DataSet test = iter.next(); - INDArray features = test.getFeatures(); - INDArray labels = test.getLabels(); - assertArrayEquals(new long[] { miniBatchSize, nFeatures }, features.shape()); - assertArrayEquals(new long[] { miniBatchSize, 1 }, labels.shape()); - int startRow = miniBatch * miniBatchSize; - for (int i = 0; i < miniBatchSize; i++) { - double labelExp = data[startRow + i][labelIdx]; - double labelAct = labels.getDouble(i); - assertEquals(labelExp, labelAct, 1e-5f); - int featureCount = 0; - for (int j = 0; j < nFeatures + 1; j++) { - if (j == labelIdx) - continue; - double featureExp = data[startRow + i][j]; - double featureAct = features.getDouble(i, featureCount++); - assertEquals(featureExp, featureAct, 1e-5f); - } - } - miniBatch++; - } - assertEquals(nLines / miniBatchSize, miniBatch); - } - - public Pair makeRandomCSV(String tempFile, int nLines, int nFeatures) throws IOException { - File temp = temporaryFolder.resolve(tempFile).toFile(); - temp.mkdirs(); - temp.deleteOnExit(); - Random rand = new Random(12345); - double[][] dArr = new double[nLines][nFeatures + 1]; - try (PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(temp)))) { - for (int i = 0; i < nLines; i++) { - // First column: label - dArr[i][0] = rand.nextDouble(); - out.print(dArr[i][0]); - for (int j = 0; j < nFeatures; j++) { - dArr[i][j + 1] = rand.nextDouble(); - out.print("," + dArr[i][j + 1]); - } - out.println(); - } - } catch (IOException e) { - log.error("", e); - } - return new Pair<>(dArr, temp); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Variable Length Sequence") - void testVariableLengthSequence(Nd4jBackend backend) throws Exception { - File rootDir = temporaryFolder.toFile(); - // need to manually extract - for (int i = 0; i < 3; i++) { - FileUtils.copyFile(Resources.asFile(String.format("csvsequence_%d.txt", i)), new File(rootDir, String.format("csvsequence_%d.txt", i))); - FileUtils.copyFile(Resources.asFile(String.format("csvsequencelabelsShort_%d.txt", i)), new File(rootDir, String.format("csvsequencelabelsShort_%d.txt", i))); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabelsShort_%d.txt"); - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); - featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReaderDataSetIterator iterAlignStart = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_START); - SequenceRecordReaderDataSetIterator iterAlignEnd = new SequenceRecordReaderDataSetIterator(featureReader2, labelReader2, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); - assertEquals(3, iterAlignStart.inputColumns()); - assertEquals(4, iterAlignStart.totalOutcomes()); - assertEquals(3, iterAlignEnd.inputColumns()); - assertEquals(4, iterAlignEnd.totalOutcomes()); - List dsListAlignStart = new ArrayList<>(); - while (iterAlignStart.hasNext()) { - dsListAlignStart.add(iterAlignStart.next()); - } - List dsListAlignEnd = new ArrayList<>(); - while (iterAlignEnd.hasNext()) { - dsListAlignEnd.add(iterAlignEnd.next()); - } - // 3 files - assertEquals(3, dsListAlignStart.size()); - // 3 files - assertEquals(3, dsListAlignEnd.size()); - for (int i = 0; i < 3; i++) { - DataSet ds = dsListAlignStart.get(i); - INDArray features = ds.getFeatures(); - INDArray labels = ds.getLabels(); - // 1 example in mini-batch - assertEquals(1, features.size(0)); - assertEquals(1, labels.size(0)); - // 3 values per line/time step - assertEquals(3, features.size(1)); - // 1 value per line, but 4 possible values -> one-hot vector - assertEquals(4, labels.size(1)); - // sequence length = 4 - assertEquals(4, features.size(2)); - assertEquals(4, labels.size(2)); - DataSet ds2 = dsListAlignEnd.get(i); - features = ds2.getFeatures(); - labels = ds2.getLabels(); - // 1 example in mini-batch - assertEquals(1, features.size(0)); - assertEquals(1, labels.size(0)); - // 3 values per line/time step - assertEquals(3, features.size(1)); - // 1 value per line, but 4 possible values -> one-hot vector - assertEquals(4, labels.size(1)); - // sequence length = 4 - assertEquals(4, features.size(2)); - assertEquals(4, labels.size(2)); - } - // Check features vs. expected: - // Here: labels always longer than features -> same features for align start and align end - INDArray expF0 = Nd4j.create(1, 3, 4); - expF0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0, 1, 2 })); - expF0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 10, 11, 12 })); - expF0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 20, 21, 22 })); - expF0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 30, 31, 32 })); - assertEquals(expF0, dsListAlignStart.get(0).getFeatures()); - assertEquals(expF0, dsListAlignEnd.get(0).getFeatures()); - INDArray expF1 = Nd4j.create(1, 3, 4); - expF1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 100, 101, 102 })); - expF1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 110, 111, 112 })); - expF1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 120, 121, 122 })); - expF1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 130, 131, 132 })); - assertEquals(expF1, dsListAlignStart.get(1).getFeatures()); - assertEquals(expF1, dsListAlignEnd.get(1).getFeatures()); - INDArray expF2 = Nd4j.create(1, 3, 4); - expF2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 200, 201, 202 })); - expF2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 210, 211, 212 })); - expF2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 220, 221, 222 })); - expF2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 230, 231, 232 })); - assertEquals(expF2, dsListAlignStart.get(2).getFeatures()); - assertEquals(expF2, dsListAlignEnd.get(2).getFeatures()); - // Check features mask array: - // null: equivalent to all 1s (i.e., present for all time steps) - INDArray featuresMaskExpected = null; - for (int i = 0; i < 3; i++) { - INDArray featuresMaskStart = dsListAlignStart.get(i).getFeaturesMaskArray(); - INDArray featuresMaskEnd = dsListAlignEnd.get(i).getFeaturesMaskArray(); - assertEquals(featuresMaskExpected, featuresMaskStart); - assertEquals(featuresMaskExpected, featuresMaskEnd); - } - // Check labels vs. expected: - // First: aligning start - INDArray expL0 = Nd4j.create(1, 4, 4); - expL0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 1, 0, 0, 0 })); - expL0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - assertEquals(expL0, dsListAlignStart.get(0).getLabels()); - INDArray expL1 = Nd4j.create(1, 4, 4); - expL1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - assertEquals(expL1, dsListAlignStart.get(1).getLabels()); - INDArray expL2 = Nd4j.create(1, 4, 4); - expL2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0, 0, 0, 1 })); - expL2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 0, 0, 1, 0 })); - expL2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - assertEquals(expL2, dsListAlignStart.get(2).getLabels()); - // Second: align end - INDArray expL0end = Nd4j.create(1, 4, 4); - expL0end.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 1, 0, 0, 0 })); - expL0end.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - assertEquals(expL0end, dsListAlignEnd.get(0).getLabels()); - INDArray expL1end = Nd4j.create(1, 4, 4); - expL1end.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - assertEquals(expL1end, dsListAlignEnd.get(1).getLabels()); - INDArray expL2end = Nd4j.create(1, 4, 4); - expL2end.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 0, 0, 0, 1 })); - expL2end.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 0, 0, 1, 0 })); - expL2end.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 0, 1, 0, 0 })); - assertEquals(expL2end, dsListAlignEnd.get(2).getLabels()); - // Check labels mask array - INDArray[] labelsMaskExpectedStart = new INDArray[] { Nd4j.create(new float[] { 1, 1, 0, 0 }, new int[] { 1, 4 }), Nd4j.create(new float[] { 1, 0, 0, 0 }, new int[] { 1, 4 }), Nd4j.create(new float[] { 1, 1, 1, 0 }, new int[] { 1, 4 }) }; - INDArray[] labelsMaskExpectedEnd = new INDArray[] { Nd4j.create(new float[] { 0, 0, 1, 1 }, new int[] { 1, 4 }), Nd4j.create(new float[] { 0, 0, 0, 1 }, new int[] { 1, 4 }), Nd4j.create(new float[] { 0, 1, 1, 1 }, new int[] { 1, 4 }) }; - for (int i = 0; i < 3; i++) { - INDArray labelsMaskStart = dsListAlignStart.get(i).getLabelsMaskArray(); - INDArray labelsMaskEnd = dsListAlignEnd.get(i).getLabelsMaskArray(); - assertEquals(labelsMaskExpectedStart[i], labelsMaskStart); - assertEquals(labelsMaskExpectedEnd[i], labelsMaskEnd); - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Single Reader") - void testSequenceRecordReaderSingleReader(Nd4jBackend backend) throws Exception { - File rootDir = temporaryFolder.toFile(); - // need to manually extract - for (int i = 0; i < 3; i++) { - FileUtils.copyFile(Resources.asFile(String.format("csvsequenceSingle_%d.txt", i)), new File(rootDir, String.format("csvsequenceSingle_%d.txt", i))); - } - String path = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequenceSingle_%d.txt"); - SequenceRecordReader reader = new CSVSequenceRecordReader(1, ","); - reader.initialize(new NumberedFileInputSplit(path, 0, 2)); - SequenceRecordReaderDataSetIterator iteratorClassification = new SequenceRecordReaderDataSetIterator(reader, 1, 3, 0, false); - assertTrue(iteratorClassification.hasNext()); - SequenceRecordReader reader2 = new CSVSequenceRecordReader(1, ","); - reader2.initialize(new NumberedFileInputSplit(path, 0, 2)); - SequenceRecordReaderDataSetIterator iteratorRegression = new SequenceRecordReaderDataSetIterator(reader2, 1, 1, 0, true); - INDArray expF0 = Nd4j.create(1, 2, 4); - expF0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 1, 2 })); - expF0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 11, 12 })); - expF0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 21, 22 })); - expF0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 31, 32 })); - INDArray expF1 = Nd4j.create(1, 2, 4); - expF1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 101, 102 })); - expF1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 111, 112 })); - expF1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 121, 122 })); - expF1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 131, 132 })); - INDArray expF2 = Nd4j.create(1, 2, 4); - expF2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 201, 202 })); - expF2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 211, 212 })); - expF2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 221, 222 })); - expF2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 231, 232 })); - INDArray[] expF = new INDArray[] { expF0, expF1, expF2 }; - // Expected out for classification: - INDArray expOut0 = Nd4j.create(1, 3, 4); - expOut0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 1, 0, 0 })); - expOut0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 0, 1, 0 })); - expOut0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 0, 0, 1 })); - expOut0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 1, 0, 0 })); - INDArray expOut1 = Nd4j.create(1, 3, 4); - expOut1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0, 1, 0 })); - expOut1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 0, 0, 1 })); - expOut1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 1, 0, 0 })); - expOut1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 0, 0, 1 })); - INDArray expOut2 = Nd4j.create(1, 3, 4); - expOut2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0, 1, 0 })); - expOut2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 1, 0, 0 })); - expOut2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 0, 1, 0 })); - expOut2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 0, 0, 1 })); - INDArray[] expOutClassification = new INDArray[] { expOut0, expOut1, expOut2 }; - // Expected out for regression: - INDArray expOutR0 = Nd4j.create(1, 1, 4); - expOutR0.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 0 })); - expOutR0.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 1 })); - expOutR0.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 2 })); - expOutR0.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 0 })); - INDArray expOutR1 = Nd4j.create(1, 1, 4); - expOutR1.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 1 })); - expOutR1.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 2 })); - expOutR1.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 0 })); - expOutR1.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 2 })); - INDArray expOutR2 = Nd4j.create(1, 1, 4); - expOutR2.tensorAlongDimension(0, 1).assign(Nd4j.create(new double[] { 1 })); - expOutR2.tensorAlongDimension(1, 1).assign(Nd4j.create(new double[] { 0 })); - expOutR2.tensorAlongDimension(2, 1).assign(Nd4j.create(new double[] { 1 })); - expOutR2.tensorAlongDimension(3, 1).assign(Nd4j.create(new double[] { 2 })); - INDArray[] expOutRegression = new INDArray[] { expOutR0, expOutR1, expOutR2 }; - int countC = 0; - while (iteratorClassification.hasNext()) { - DataSet ds = iteratorClassification.next(); - INDArray f = ds.getFeatures(); - INDArray l = ds.getLabels(); - assertNull(ds.getFeaturesMaskArray()); - assertNull(ds.getLabelsMaskArray()); - assertArrayEquals(new long[] { 1, 2, 4 }, f.shape()); - // One-hot representation - assertArrayEquals(new long[] { 1, 3, 4 }, l.shape()); - assertEquals(expF[countC], f); - assertEquals(expOutClassification[countC++], l); - } - assertEquals(3, countC); - assertEquals(3, iteratorClassification.totalOutcomes()); - int countF = 0; - while (iteratorRegression.hasNext()) { - DataSet ds = iteratorRegression.next(); - INDArray f = ds.getFeatures(); - INDArray l = ds.getLabels(); - assertNull(ds.getFeaturesMaskArray()); - assertNull(ds.getLabelsMaskArray()); - assertArrayEquals(new long[] { 1, 2, 4 }, f.shape()); - // Regression (single output) - assertArrayEquals(new long[] { 1, 1, 4 }, l.shape()); - assertEquals(expF[countF], f); - assertEquals(expOutRegression[countF++], l); - } - assertEquals(3, countF); - assertEquals(1, iteratorRegression.totalOutcomes()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Single Reader With Empty Sequence Throws") - void testSequenceRecordReaderSingleReaderWithEmptySequenceThrows(Nd4jBackend backend) { - assertThrows(ZeroLengthSequenceException.class, () -> { - SequenceRecordReader reader = new CSVSequenceRecordReader(1, ","); - reader.initialize(new FileSplit(Resources.asFile("empty.txt"))); - new SequenceRecordReaderDataSetIterator(reader, 1, -1, 1, true).next(); - }); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Two Readers With Empty Feature Sequence Throws") - void testSequenceRecordReaderTwoReadersWithEmptyFeatureSequenceThrows(Nd4jBackend backend) { - assertThrows(ZeroLengthSequenceException.class, () -> { - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new FileSplit(Resources.asFile("empty.txt"))); - labelReader.initialize(new FileSplit(Resources.asFile("csvsequencelabels_0.txt"))); - new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, -1, true).next(); - }); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Two Readers With Empty Label Sequence Throws") - void testSequenceRecordReaderTwoReadersWithEmptyLabelSequenceThrows(Nd4jBackend backend) { - assertThrows(ZeroLengthSequenceException.class, () -> { - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - File f = Resources.asFile("csvsequence_0.txt"); - featureReader.initialize(new FileSplit(f)); - labelReader.initialize(new FileSplit(Resources.asFile("empty.txt"))); - new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, -1, true).next(); - }); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Sequence Record Reader Single Reader Meta Data") - void testSequenceRecordReaderSingleReaderMetaData(Nd4jBackend backend) throws Exception { - File rootDir = temporaryFolder.toFile(); - // need to manually extract - for (int i = 0; i < 3; i++) { - FileUtils.copyFile(Resources.asFile(String.format("csvsequenceSingle_%d.txt", i)), new File(rootDir, String.format("csvsequenceSingle_%d.txt", i))); - } - String path = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequenceSingle_%d.txt"); - SequenceRecordReader reader = new CSVSequenceRecordReader(1, ","); - reader.initialize(new NumberedFileInputSplit(path, 0, 2)); - SequenceRecordReaderDataSetIterator iteratorClassification = new SequenceRecordReaderDataSetIterator(reader, 1, 3, 0, false); - SequenceRecordReader reader2 = new CSVSequenceRecordReader(1, ","); - reader2.initialize(new NumberedFileInputSplit(path, 0, 2)); - SequenceRecordReaderDataSetIterator iteratorRegression = new SequenceRecordReaderDataSetIterator(reader2, 1, 1, 0, true); - iteratorClassification.setCollectMetaData(true); - iteratorRegression.setCollectMetaData(true); - while (iteratorClassification.hasNext()) { - DataSet ds = iteratorClassification.next(); - DataSet fromMeta = iteratorClassification.loadFromMetaData(ds.getExampleMetaData(RecordMetaData.class)); - assertEquals(ds, fromMeta); - } - while (iteratorRegression.hasNext()) { - DataSet ds = iteratorRegression.next(); - DataSet fromMeta = iteratorRegression.loadFromMetaData(ds.getExampleMetaData(RecordMetaData.class)); - assertEquals(ds, fromMeta); - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Seq RRDSI Array Writable One Reader") - void testSeqRRDSIArrayWritableOneReader(Nd4jBackend backend) { - List> sequence1 = new ArrayList<>(); - sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 }, new long[] { 1, 3 })), new IntWritable(0))); - sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 }, new long[] { 1, 3 })), new IntWritable(1))); - List> sequence2 = new ArrayList<>(); - sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 }, new long[] { 1, 3 })), new IntWritable(2))); - sequence2.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 }, new long[] { 1, 3 })), new IntWritable(3))); - SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, 4, 1, false); - DataSet ds = iter.next(); - // 2 examples, 3 values per time step, 2 time steps - INDArray expFeatures = Nd4j.create(2, 3, 2); - expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 } })); - expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 } })); - INDArray expLabels = Nd4j.create(2, 4, 2); - expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 0 }, { 0, 1 }, { 0, 0 }, { 0, 0 } })); - expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 0, 0 }, { 0, 0 }, { 1, 0 }, { 0, 1 } })); - assertEquals(expFeatures, ds.getFeatures()); - assertEquals(expLabels, ds.getLabels()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Seq RRDSI Array Writable One Reader Regression") - void testSeqRRDSIArrayWritableOneReaderRegression(Nd4jBackend backend) { - // Regression, where the output is an array writable - List> sequence1 = new ArrayList<>(); - sequence1.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 }, new long[] { 1, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 100, 200, 300 }, new long[] { 1, 3 })))); - sequence1.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 }, new long[] { 1, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 400, 500, 600 }, new long[] { 1, 3 })))); - List> sequence2 = new ArrayList<>(); - sequence2.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 }, new long[] { 1, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 700, 800, 900 }, new long[] { 1, 3 })))); - sequence2.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 }, new long[] { 1, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 1000, 1100, 1200 }, new long[] { 1, 3 })))); - SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, -1, 1, true); - DataSet ds = iter.next(); - // 2 examples, 3 values per time step, 2 time steps - INDArray expFeatures = Nd4j.create(2, 3, 2); - expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 } })); - expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 } })); - INDArray expLabels = Nd4j.create(2, 3, 2); - expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 100, 400 }, { 200, 500 }, { 300, 600 } })); - expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 700, 1000 }, { 800, 1100 }, { 900, 1200 } })); - assertEquals(expFeatures, ds.getFeatures()); - assertEquals(expLabels, ds.getLabels()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Seq RRDSI Multiple Array Writables One Reader") - void testSeqRRDSIMultipleArrayWritablesOneReader(Nd4jBackend backend) { - // Input with multiple array writables: - List> sequence1 = new ArrayList<>(); - sequence1.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 }, new long[] { 1, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 100, 200, 300 }, new long[] { 1, 3 })), new IntWritable(0))); - sequence1.add(Arrays.asList((Writable) new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 }, new long[] { 1, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 400, 500, 600 }, new long[] { 1, 3 })), new IntWritable(1))); - List> sequence2 = new ArrayList<>(); - sequence2.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 }, new long[] { 1, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 700, 800, 900 }, new long[] { 1, 3 })), new IntWritable(2))); - sequence2.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 }, new long[] { 1, 3 })), new NDArrayWritable(Nd4j.create(new double[] { 1000, 1100, 1200 }, new long[] { 1, 3 })), new IntWritable(3))); - SequenceRecordReader rr = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rr, 2, 4, 2, false); - DataSet ds = iter.next(); - // 2 examples, 6 values per time step, 2 time steps - INDArray expFeatures = Nd4j.create(2, 6, 2); - expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 }, { 100, 400 }, { 200, 500 }, { 300, 600 } })); - expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 }, { 700, 1000 }, { 800, 1100 }, { 900, 1200 } })); - INDArray expLabels = Nd4j.create(2, 4, 2); - expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 0 }, { 0, 1 }, { 0, 0 }, { 0, 0 } })); - expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 0, 0 }, { 0, 0 }, { 1, 0 }, { 0, 1 } })); - assertEquals(expFeatures, ds.getFeatures()); - assertEquals(expLabels, ds.getLabels()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Seq RRDSI Array Writable Two Readers") - void testSeqRRDSIArrayWritableTwoReaders(Nd4jBackend backend) { - List> sequence1 = new ArrayList<>(); - sequence1.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 1, 2, 3 }, new long[] { 1, 3 })), new IntWritable(100))); - sequence1.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 4, 5, 6 }, new long[] { 1, 3 })), new IntWritable(200))); - List> sequence2 = new ArrayList<>(); - sequence2.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 7, 8, 9 }, new long[] { 1, 3 })), new IntWritable(300))); - sequence2.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 10, 11, 12 }, new long[] { 1, 3 })), new IntWritable(400))); - SequenceRecordReader rrFeatures = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); - List> sequence1L = new ArrayList<>(); - sequence1L.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 100, 200, 300 }, new long[] { 1, 3 })), new IntWritable(101))); - sequence1L.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 400, 500, 600 }, new long[] { 1, 3 })), new IntWritable(201))); - List> sequence2L = new ArrayList<>(); - sequence2L.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 700, 800, 900 }, new long[] { 1, 3 })), new IntWritable(301))); - sequence2L.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 1000, 1100, 1200 }, new long[] { 1, 3 })), new IntWritable(401))); - SequenceRecordReader rrLabels = new CollectionSequenceRecordReader(Arrays.asList(sequence1L, sequence2L)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rrFeatures, rrLabels, 2, -1, true); - // 2 examples, 4 values per time step, 2 time steps - INDArray expFeatures = Nd4j.create(2, 4, 2); - expFeatures.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 1, 4 }, { 2, 5 }, { 3, 6 }, { 100, 200 } })); - expFeatures.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 7, 10 }, { 8, 11 }, { 9, 12 }, { 300, 400 } })); - INDArray expLabels = Nd4j.create(2, 4, 2); - expLabels.tensorAlongDimension(0, 1, 2).assign(Nd4j.create(new double[][] { { 100, 400 }, { 200, 500 }, { 300, 600 }, { 101, 201 } })); - expLabels.tensorAlongDimension(1, 1, 2).assign(Nd4j.create(new double[][] { { 700, 1000 }, { 800, 1100 }, { 900, 1200 }, { 301, 401 } })); - DataSet ds = iter.next(); - assertEquals(expFeatures, ds.getFeatures()); - assertEquals(expLabels, ds.getLabels()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Record Reader Meta Data") - void testRecordReaderMetaData() throws Exception { - RecordReader csv = new CSVRecordReader(); - csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); - int batchSize = 10; - int labelIdx = 4; - int numClasses = 3; - RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(csv, batchSize, labelIdx, numClasses); - rrdsi.setCollectMetaData(true); - while (rrdsi.hasNext()) { - DataSet ds = rrdsi.next(); - List meta = ds.getExampleMetaData(RecordMetaData.class); - int i = 0; - for (RecordMetaData m : meta) { - Record r = csv.loadFromMetaData(m); - INDArray row = ds.getFeatures().getRow(i); - // if(i <= 3) { - // System.out.println(m.getLocation() + "\t" + r.getRecord() + "\t" + row); - // } - for (int j = 0; j < 4; j++) { - double exp = r.getRecord().get(j).toDouble(); - double act = row.getDouble(j); - assertEquals( exp, act, 1e-6,"Failed on idx: " + j); - } - i++; - } - // System.out.println(); - DataSet fromMeta = rrdsi.loadFromMetaData(meta); - assertEquals(ds, fromMeta); - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test RRDS Iwith Async") - void testRRDSIwithAsync(Nd4jBackend backend) throws Exception { - RecordReader csv = new CSVRecordReader(); - csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); - int batchSize = 10; - int labelIdx = 4; - int numClasses = 3; - RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(csv, batchSize, labelIdx, numClasses); - AsyncDataSetIterator adsi = new AsyncDataSetIterator(rrdsi, 8, true); - while (adsi.hasNext()) { - DataSet ds = adsi.next(); - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Record Reader Data Set Iterator ND Array Writable Labels") - void testRecordReaderDataSetIteratorNDArrayWritableLabels(Nd4jBackend backend) { - Collection> data = new ArrayList<>(); - data.add(Arrays.asList(new DoubleWritable(0), new DoubleWritable(1), new NDArrayWritable(Nd4j.create(new double[] { 1.1, 2.1, 3.1 }, new long[] { 1, 3 })))); - data.add(Arrays.asList(new DoubleWritable(2), new DoubleWritable(3), new NDArrayWritable(Nd4j.create(new double[] { 4.1, 5.1, 6.1 }, new long[] { 1, 3 })))); - data.add(Arrays.asList(new DoubleWritable(4), new DoubleWritable(5), new NDArrayWritable(Nd4j.create(new double[] { 7.1, 8.1, 9.1 }, new long[] { 1, 3 })))); - RecordReader rr = new CollectionRecordReader(data); - int batchSize = 3; - int labelIndexFrom = 2; - int labelIndexTo = 2; - boolean regression = true; - DataSetIterator rrdsi = new RecordReaderDataSetIterator(rr, batchSize, labelIndexFrom, labelIndexTo, regression); - DataSet ds = rrdsi.next(); - INDArray expFeatures = Nd4j.create(new float[][] { { 0, 1 }, { 2, 3 }, { 4, 5 } }); - INDArray expLabels = Nd4j.create(new float[][] { { 1.1f, 2.1f, 3.1f }, { 4.1f, 5.1f, 6.1f }, { 7.1f, 8.1f, 9.1f } }); - assertEquals(expFeatures, ds.getFeatures()); - assertEquals(expLabels, ds.getLabels()); - // ALSO: test if we have NDArrayWritables for BOTH the features and the labels - data = new ArrayList<>(); - data.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 0, 1 }, new long[] { 1, 2 })), new NDArrayWritable(Nd4j.create(new double[] { 1.1, 2.1, 3.1 }, new long[] { 1, 3 })))); - data.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 2, 3 }, new long[] { 1, 2 })), new NDArrayWritable(Nd4j.create(new double[] { 4.1, 5.1, 6.1 }, new long[] { 1, 3 })))); - data.add(Arrays.asList(new NDArrayWritable(Nd4j.create(new double[] { 4, 5 }, new long[] { 1, 2 })), new NDArrayWritable(Nd4j.create(new double[] { 7.1, 8.1, 9.1 }, new long[] { 1, 3 })))); - labelIndexFrom = 1; - labelIndexTo = 1; - rr = new CollectionRecordReader(data); - rrdsi = new RecordReaderDataSetIterator(rr, batchSize, labelIndexFrom, labelIndexTo, regression); - DataSet ds2 = rrdsi.next(); - assertEquals(expFeatures, ds2.getFeatures()); - assertEquals(expLabels, ds2.getLabels()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @Disabled - @DisplayName("Special RR Test 4") - void specialRRTest4(Nd4jBackend backend) throws Exception { - RecordReader rr = new SpecialImageRecordReader(25000, 10, 3, 224, 224); - RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr, 128); - int cnt = 0; - int examples = 0; - while (rrdsi.hasNext()) { - DataSet ds = rrdsi.next(); - assertEquals(128, ds.numExamples()); - for (int i = 0; i < ds.numExamples(); i++) { - INDArray example = ds.getFeatures().tensorAlongDimension(i, 1, 2, 3).dup(); - // assertEquals("Failed on DataSet [" + cnt + "], example [" + i + "]", (double) examples, example.meanNumber().doubleValue(), 0.01); - // assertEquals("Failed on DataSet [" + cnt + "], example [" + i + "]", (double) examples, ds.getLabels().getRow(i).meanNumber().doubleValue(), 0.01); - examples++; - } - cnt++; - } - } - - /* - @Test - public void specialRRTest1() throws Exception { - RecordReader rr = new SpecialImageRecordReader(250, 10,3, 224, 224); - DataSetIterator rrdsi = new ParallelRecordReaderDataSetIterator.Builder(rr) - .setBatchSize(10) - .numberOfWorkers(1) - .build(); - - int cnt = 0; - int examples = 0; - while (rrdsi.hasNext()) { - DataSet ds = rrdsi.next(); - for (int i = 0; i < ds.numExamples(); i++) { - INDArray example = ds.getFeatures().tensorAlongDimension(i, 1, 2, 3).dup(); - assertEquals("Failed on DataSet ["+ cnt + "], example ["+ i +"]",(double) examples, example.meanNumber().doubleValue(), 0.01); - examples++; - } - cnt++; - log.info("DataSet {} passed...", cnt); - } - - assertEquals(25, cnt); - } - - - @Test - public void specialRRTest2() throws Exception { - RecordReader rr = new SpecialImageRecordReader(250, 10,3, 224, 224); - DataSetIterator rrdsi = new ParallelRecordReaderDataSetIterator.Builder(rr) - .setBatchSize(10) - .numberOfWorkers(1) - .prefetchBufferSize(4) - .build(); - - rrdsi = new AsyncDataSetIterator(rrdsi); - - int cnt = 0; - int examples = 0; - while (rrdsi.hasNext()) { - DataSet ds = rrdsi.next(); - for (int i = 0; i < ds.numExamples(); i++) { - INDArray example = ds.getFeatures().tensorAlongDimension(i, 1, 2, 3).dup(); - assertEquals("Failed on DataSet ["+ cnt + "], example ["+ i +"]",(double) examples, example.meanNumber().doubleValue(), 0.01); - examples++; - } - cnt++; - } - - assertEquals(25, cnt); - } - - - @Test - public void specialRRTest3() throws Exception { - RecordReader rr = new SpecialImageRecordReader(400, 10,3, 224, 224); - DataSetIterator rrdsi = new ParallelRecordReaderDataSetIterator.Builder(rr) - .setBatchSize(128) - .numberOfWorkers(2) - .prefetchBufferSize(2) - .build(); - - log.info("DataType: {}", Nd4j.dataType() ); - - // rrdsi = new AsyncDataSetIterator(rrdsi); - - int cnt = 0; - int examples = 0; - while (rrdsi.hasNext()) { - DataSet ds = rrdsi.next(); - for (int i = 0; i < ds.numExamples(); i++) { - INDArray example = ds.getFeatures().tensorAlongDimension(i, 1, 2, 3).dup(); - assertEquals("Failed on DataSet ["+ cnt + "], example ["+ i +"]",(double) examples, example.meanNumber().doubleValue(), 0.01); - examples++; - } - cnt++; - } - - } - */ - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Record Reader Data Set Iterator Concat") - void testRecordReaderDataSetIteratorConcat(Nd4jBackend backend) { - // [DoubleWritable, DoubleWritable, NDArrayWritable([1,10]), IntWritable] -> concatenate to a [1,13] feature vector automatically. - List l = Arrays.asList(new DoubleWritable(1), new NDArrayWritable(Nd4j.create(new double[] { 2, 3, 4 })), new DoubleWritable(5), new NDArrayWritable(Nd4j.create(new double[] { 6, 7, 8 })), new IntWritable(9), new IntWritable(1)); - RecordReader rr = new CollectionRecordReader(Collections.singletonList(l)); - DataSetIterator iter = new RecordReaderDataSetIterator(rr, 1, 5, 3); - DataSet ds = iter.next(); - INDArray expF = Nd4j.create(new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }, new int[] { 1, 9 }); - INDArray expL = Nd4j.create(new float[] { 0, 1, 0 }, new int[] { 1, 3 }); - assertEquals(expF, ds.getFeatures()); - assertEquals(expL, ds.getLabels()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Record Reader Data Set Iterator Concat 2") - void testRecordReaderDataSetIteratorConcat2(Nd4jBackend backend) { - List l = new ArrayList<>(); - l.add(new IntWritable(0)); - l.add(new NDArrayWritable(Nd4j.arange(1, 9))); - l.add(new IntWritable(9)); - RecordReader rr = new CollectionRecordReader(Collections.singletonList(l)); - DataSetIterator iter = new RecordReaderDataSetIterator(rr, 1); - DataSet ds = iter.next(); - INDArray expF = Nd4j.create(new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, new int[] { 1, 10 }); - assertEquals(expF, ds.getFeatures()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Record Reader Data Set Iterator Disjoint Features") - void testRecordReaderDataSetIteratorDisjointFeatures(Nd4jBackend backend) { - // Idea: input vector is like [f,f,f,f,l,l,f,f] or similar - i.e., label writables aren't start/end - List l = Arrays.asList(new DoubleWritable(1), new NDArrayWritable(Nd4j.create(new float[] { 2, 3, 4 }, new long[] { 1, 3 })), new DoubleWritable(5), new NDArrayWritable(Nd4j.create(new float[] { 6, 7, 8 }, new long[] { 1, 3 }))); - INDArray expF = Nd4j.create(new float[] { 1, 6, 7, 8 }, new long[] { 1, 4 }); - INDArray expL = Nd4j.create(new float[] { 2, 3, 4, 5 }, new long[] { 1, 4 }); - RecordReader rr = new CollectionRecordReader(Collections.singletonList(l)); - DataSetIterator iter = new RecordReaderDataSetIterator(rr, 1, 1, 2, true); - DataSet ds = iter.next(); - assertEquals(expF, ds.getFeatures()); - assertEquals(expL, ds.getLabels()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Normalizer Prefetch Reset") - void testNormalizerPrefetchReset(Nd4jBackend backend) throws Exception { - // Check NPE fix for: https://github.com/eclipse/deeplearning4j/issues/4214 - RecordReader csv = new CSVRecordReader(); - csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); - int batchSize = 3; - DataSetIterator iter = new RecordReaderDataSetIterator(csv, batchSize, 4, 4, true); - DataNormalization normalizer = new NormalizerMinMaxScaler(0, 1); - normalizer.fit(iter); - iter.setPreProcessor(normalizer); - // Prefetch - iter.inputColumns(); - iter.totalOutcomes(); - iter.hasNext(); - iter.reset(); - iter.next(); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Reading From Stream") - void testReadingFromStream(Nd4jBackend backend) throws Exception { - for (boolean b : new boolean[] { false, true }) { - int batchSize = 1; - int labelIndex = 4; - int numClasses = 3; - InputStream dataFile = Resources.asStream("iris.txt"); - RecordReader recordReader = new CSVRecordReader(0, ','); - recordReader.initialize(new InputStreamInputSplit(dataFile)); - assertTrue(recordReader.hasNext()); - assertFalse(recordReader.resetSupported()); - DataSetIterator iterator; - if (b) { - iterator = new RecordReaderDataSetIterator.Builder(recordReader, batchSize).classification(labelIndex, numClasses).build(); - } else { - iterator = new RecordReaderDataSetIterator(recordReader, batchSize, labelIndex, numClasses); - } - assertFalse(iterator.resetSupported()); - int count = 0; - while (iterator.hasNext()) { - assertNotNull(iterator.next()); - count++; - } - assertEquals(150, count); - try { - iterator.reset(); - fail("Expected exception"); - } catch (Exception e) { - // expected - } - } - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Images RRDSI") - void testImagesRRDSI(Nd4jBackend backend) throws Exception { - File parentDir = temporaryFolder.toFile(); - parentDir.deleteOnExit(); - String str1 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Zico/"); - String str2 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Ziwang_Xu/"); - File f2 = new File(str2); - File f1 = new File(str1); - f1.mkdirs(); - f2.mkdirs(); - TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f1.getPath(), "Zico_0001.jpg")), new ClassPathResource("lfwtest/Zico/Zico_0001.jpg").getInputStream()); - TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f2.getPath(), "Ziwang_Xu_0001.jpg")), new ClassPathResource("lfwtest/Ziwang_Xu/Ziwang_Xu_0001.jpg").getInputStream()); - Random r = new Random(12345); - ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator(); - ImageRecordReader rr1 = new ImageRecordReader(28, 28, 3, labelMaker); - rr1.initialize(new FileSplit(parentDir)); - RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr1, 2); - DataSet ds = rrdsi.next(); - assertArrayEquals(new long[] { 2, 3, 28, 28 }, ds.getFeatures().shape()); - assertArrayEquals(new long[] { 2, 2 }, ds.getLabels().shape()); - // Check the same thing via the builder: - rr1.reset(); - rrdsi = new RecordReaderDataSetIterator.Builder(rr1, 2).classification(1, 2).build(); - ds = rrdsi.next(); - assertArrayEquals(new long[] { 2, 3, 28, 28 }, ds.getFeatures().shape()); - assertArrayEquals(new long[] { 2, 2 }, ds.getLabels().shape()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Seq RRDSI No Labels") - void testSeqRRDSINoLabels(Nd4jBackend backend) { - List> sequence1 = new ArrayList<>(); - sequence1.add(Arrays.asList(new DoubleWritable(1), new DoubleWritable(2))); - sequence1.add(Arrays.asList(new DoubleWritable(3), new DoubleWritable(4))); - sequence1.add(Arrays.asList(new DoubleWritable(5), new DoubleWritable(6))); - List> sequence2 = new ArrayList<>(); - sequence2.add(Arrays.asList(new DoubleWritable(10), new DoubleWritable(20))); - sequence2.add(Arrays.asList(new DoubleWritable(30), new DoubleWritable(40))); - SequenceRecordReader rrFeatures = new CollectionSequenceRecordReader(Arrays.asList(sequence1, sequence2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(rrFeatures, 2, -1, -1); - DataSet ds = iter.next(); - assertNotNull(ds.getFeatures()); - assertNull(ds.getLabels()); - } - - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @DisplayName("Test Collect Meta Data") - void testCollectMetaData(Nd4jBackend backend) { - RecordReaderDataSetIterator trainIter = new RecordReaderDataSetIterator.Builder(new CollectionRecordReader(Collections.>emptyList()), 1).collectMetaData(true).build(); - assertTrue(trainIter.isCollectMetaData()); - trainIter.setCollectMetaData(false); - assertFalse(trainIter.isCollectMetaData()); - trainIter.setCollectMetaData(true); - assertTrue(trainIter.isCollectMetaData()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIteratorTest.java deleted file mode 100644 index db010c2ea..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/datavec/RecordReaderMultiDataSetIteratorTest.java +++ /dev/null @@ -1,740 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.datavec; - - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.shade.guava.io.Files; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.datavec.api.conf.Configuration; -import org.datavec.api.io.labels.ParentPathLabelGenerator; -import org.datavec.api.records.Record; -import org.datavec.api.records.metadata.RecordMetaData; -import org.datavec.api.records.reader.BaseRecordReader; -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.SequenceRecordReader; -import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; -import org.datavec.api.split.CollectionInputSplit; -import org.datavec.api.split.FileSplit; -import org.datavec.api.split.InputSplit; -import org.datavec.api.split.NumberedFileInputSplit; -import org.datavec.api.util.ndarray.RecordConverter; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.Writable; -import org.datavec.image.recordreader.ImageRecordReader; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.MultiDataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.NDArrayIndex; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.resources.Resources; -import java.io.*; -import java.net.URI; -import java.util.*; -import static org.junit.jupiter.api.Assertions.*; -import static org.nd4j.linalg.indexing.NDArrayIndex.all; -import static org.nd4j.linalg.indexing.NDArrayIndex.interval; -import static org.nd4j.linalg.indexing.NDArrayIndex.point; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Record Reader Multi Data Set Iterator Test") -@Disabled -@Tag(TagNames.FILE_IO) -class RecordReaderMultiDataSetIteratorTest extends BaseDL4JTest { - - @TempDir - public Path temporaryFolder; - - - - @Test - @DisplayName("Tests Basic") - void testsBasic() throws Exception { - // Load details from CSV files; single input/output -> compare to RecordReaderDataSetIterator - RecordReader rr = new CSVRecordReader(0, ','); - rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); - RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr, 10, 4, 3); - RecordReader rr2 = new CSVRecordReader(0, ','); - rr2.initialize(new FileSplit(Resources.asFile("iris.txt"))); - MultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10).addReader("reader", rr2).addInput("reader", 0, 3).addOutputOneHot("reader", 4, 3).build(); - while (rrdsi.hasNext()) { - DataSet ds = rrdsi.next(); - INDArray fds = ds.getFeatures(); - INDArray lds = ds.getLabels(); - MultiDataSet mds = rrmdsi.next(); - assertEquals(1, mds.getFeatures().length); - assertEquals(1, mds.getLabels().length); - assertNull(mds.getFeaturesMaskArrays()); - assertNull(mds.getLabelsMaskArrays()); - INDArray fmds = mds.getFeatures(0); - INDArray lmds = mds.getLabels(0); - assertNotNull(fmds); - assertNotNull(lmds); - assertEquals(fds, fmds); - assertEquals(lds, lmds); - } - assertFalse(rrmdsi.hasNext()); - // need to manually extract - File rootDir = temporaryFolder.toFile(); - for (int i = 0; i < 3; i++) { - new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); - } - // Load time series from CSV sequence files; compare to SequenceRecordReaderDataSetIterator - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); - SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); - featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - MultiDataSetIterator srrmdsi = new RecordReaderMultiDataSetIterator.Builder(1).addSequenceReader("in", featureReader2).addSequenceReader("out", labelReader2).addInput("in").addOutputOneHot("out", 0, 4).build(); - while (iter.hasNext()) { - DataSet ds = iter.next(); - INDArray fds = ds.getFeatures(); - INDArray lds = ds.getLabels(); - MultiDataSet mds = srrmdsi.next(); - assertEquals(1, mds.getFeatures().length); - assertEquals(1, mds.getLabels().length); - assertNull(mds.getFeaturesMaskArrays()); - assertNull(mds.getLabelsMaskArrays()); - INDArray fmds = mds.getFeatures(0); - INDArray lmds = mds.getLabels(0); - assertNotNull(fmds); - assertNotNull(lmds); - assertEquals(fds, fmds); - assertEquals(lds, lmds); - } - assertFalse(srrmdsi.hasNext()); - } - - @Test - @DisplayName("Tests Basic Meta") - void testsBasicMeta() throws Exception { - // As per testBasic - but also loading metadata - RecordReader rr2 = new CSVRecordReader(0, ','); - rr2.initialize(new FileSplit(Resources.asFile("iris.txt"))); - RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10).addReader("reader", rr2).addInput("reader", 0, 3).addOutputOneHot("reader", 4, 3).build(); - rrmdsi.setCollectMetaData(true); - int count = 0; - while (rrmdsi.hasNext()) { - MultiDataSet mds = rrmdsi.next(); - MultiDataSet fromMeta = rrmdsi.loadFromMetaData(mds.getExampleMetaData(RecordMetaData.class)); - assertEquals(mds, fromMeta); - count++; - } - assertEquals(150 / 10, count); - } - - @Test - @DisplayName("Test Splitting CSV") - void testSplittingCSV() throws Exception { - // Here's the idea: take Iris, and split it up into 2 inputs and 2 output arrays - // Inputs: columns 0 and 1-2 - // Outputs: columns 3, and 4->OneHot - // need to manually extract - RecordReader rr = new CSVRecordReader(0, ','); - rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); - RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(rr, 10, 4, 3); - RecordReader rr2 = new CSVRecordReader(0, ','); - rr2.initialize(new FileSplit(Resources.asFile("iris.txt"))); - MultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10).addReader("reader", rr2).addInput("reader", 0, 0).addInput("reader", 1, 2).addOutput("reader", 3, 3).addOutputOneHot("reader", 4, 3).build(); - while (rrdsi.hasNext()) { - DataSet ds = rrdsi.next(); - INDArray fds = ds.getFeatures(); - INDArray lds = ds.getLabels(); - MultiDataSet mds = rrmdsi.next(); - assertEquals(2, mds.getFeatures().length); - assertEquals(2, mds.getLabels().length); - assertNull(mds.getFeaturesMaskArrays()); - assertNull(mds.getLabelsMaskArrays()); - INDArray[] fmds = mds.getFeatures(); - INDArray[] lmds = mds.getLabels(); - assertNotNull(fmds); - assertNotNull(lmds); - for (int i = 0; i < fmds.length; i++) assertNotNull(fmds[i]); - for (int i = 0; i < lmds.length; i++) assertNotNull(lmds[i]); - // Get the subsets of the original iris data - INDArray expIn1 = fds.get(all(), interval(0, 0, true)); - INDArray expIn2 = fds.get(all(), interval(1, 2, true)); - INDArray expOut1 = fds.get(all(), interval(3, 3, true)); - INDArray expOut2 = lds; - assertEquals(expIn1, fmds[0]); - assertEquals(expIn2, fmds[1]); - assertEquals(expOut1, lmds[0]); - assertEquals(expOut2, lmds[1]); - } - assertFalse(rrmdsi.hasNext()); - } - - @Test - @DisplayName("Test Splitting CSV Meta") - void testSplittingCSVMeta() throws Exception { - // Here's the idea: take Iris, and split it up into 2 inputs and 2 output arrays - // Inputs: columns 0 and 1-2 - // Outputs: columns 3, and 4->OneHot - RecordReader rr2 = new CSVRecordReader(0, ','); - rr2.initialize(new FileSplit(Resources.asFile("iris.txt"))); - RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10).addReader("reader", rr2).addInput("reader", 0, 0).addInput("reader", 1, 2).addOutput("reader", 3, 3).addOutputOneHot("reader", 4, 3).build(); - rrmdsi.setCollectMetaData(true); - int count = 0; - while (rrmdsi.hasNext()) { - MultiDataSet mds = rrmdsi.next(); - MultiDataSet fromMeta = rrmdsi.loadFromMetaData(mds.getExampleMetaData(RecordMetaData.class)); - assertEquals(mds, fromMeta); - count++; - } - assertEquals(150 / 10, count); - } - - @Test - @DisplayName("Test Splitting CSV Sequence") - void testSplittingCSVSequence() throws Exception { - // Idea: take CSV sequences, and split "csvsequence_i.txt" into two separate inputs; keep "csvSequencelables_i.txt" - // as standard one-hot output - // need to manually extract - File rootDir = temporaryFolder.toFile(); - for (int i = 0; i < 3; i++) { - new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReaderDataSetIterator iter = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false); - SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); - featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - MultiDataSetIterator srrmdsi = new RecordReaderMultiDataSetIterator.Builder(1).addSequenceReader("seq1", featureReader2).addSequenceReader("seq2", labelReader2).addInput("seq1", 0, 1).addInput("seq1", 2, 2).addOutputOneHot("seq2", 0, 4).build(); - while (iter.hasNext()) { - DataSet ds = iter.next(); - INDArray fds = ds.getFeatures(); - INDArray lds = ds.getLabels(); - MultiDataSet mds = srrmdsi.next(); - assertEquals(2, mds.getFeatures().length); - assertEquals(1, mds.getLabels().length); - assertNull(mds.getFeaturesMaskArrays()); - assertNull(mds.getLabelsMaskArrays()); - INDArray[] fmds = mds.getFeatures(); - INDArray[] lmds = mds.getLabels(); - assertNotNull(fmds); - assertNotNull(lmds); - for (int i = 0; i < fmds.length; i++) assertNotNull(fmds[i]); - for (int i = 0; i < lmds.length; i++) assertNotNull(lmds[i]); - INDArray expIn1 = fds.get(all(), NDArrayIndex.interval(0, 1, true), all()); - INDArray expIn2 = fds.get(all(), NDArrayIndex.interval(2, 2, true), all()); - assertEquals(expIn1, fmds[0]); - assertEquals(expIn2, fmds[1]); - assertEquals(lds, lmds[0]); - } - assertFalse(srrmdsi.hasNext()); - } - - @Test - @DisplayName("Test Splitting CSV Sequence Meta") - void testSplittingCSVSequenceMeta() throws Exception { - // Idea: take CSV sequences, and split "csvsequence_i.txt" into two separate inputs; keep "csvSequencelables_i.txt" - // as standard one-hot output - // need to manually extract - File rootDir = temporaryFolder.toFile(); - for (int i = 0; i < 3; i++) { - new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabels_%d.txt"); - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); - featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - RecordReaderMultiDataSetIterator srrmdsi = new RecordReaderMultiDataSetIterator.Builder(1).addSequenceReader("seq1", featureReader2).addSequenceReader("seq2", labelReader2).addInput("seq1", 0, 1).addInput("seq1", 2, 2).addOutputOneHot("seq2", 0, 4).build(); - srrmdsi.setCollectMetaData(true); - int count = 0; - while (srrmdsi.hasNext()) { - MultiDataSet mds = srrmdsi.next(); - MultiDataSet fromMeta = srrmdsi.loadFromMetaData(mds.getExampleMetaData(RecordMetaData.class)); - assertEquals(mds, fromMeta); - count++; - } - assertEquals(3, count); - } - - @Test - @DisplayName("Test Input Validation") - void testInputValidation() { - // Test: no readers - try { - MultiDataSetIterator r = new RecordReaderMultiDataSetIterator.Builder(1).addInput("something").addOutput("something").build(); - fail("Should have thrown exception"); - } catch (Exception e) { - } - // Test: reference to reader that doesn't exist - try { - RecordReader rr = new CSVRecordReader(0, ','); - rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); - MultiDataSetIterator r = new RecordReaderMultiDataSetIterator.Builder(1).addReader("iris", rr).addInput("thisDoesntExist", 0, 3).addOutputOneHot("iris", 4, 3).build(); - fail("Should have thrown exception"); - } catch (Exception e) { - } - // Test: no inputs or outputs - try { - RecordReader rr = new CSVRecordReader(0, ','); - rr.initialize(new FileSplit(Resources.asFile("iris.txt"))); - MultiDataSetIterator r = new RecordReaderMultiDataSetIterator.Builder(1).addReader("iris", rr).build(); - fail("Should have thrown exception"); - } catch (Exception e) { - } - } - - @Test - @DisplayName("Test Variable Length TS") - void testVariableLengthTS() throws Exception { - // need to manually extract - File rootDir = temporaryFolder.toFile(); - for (int i = 0; i < 3; i++) { - new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); - } - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabelsShort_%d.txt"); - // Set up SequenceRecordReaderDataSetIterators for comparison - SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader = new CSVSequenceRecordReader(1, ","); - featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReader featureReader2 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader2 = new CSVSequenceRecordReader(1, ","); - featureReader2.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader2.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReaderDataSetIterator iterAlignStart = new SequenceRecordReaderDataSetIterator(featureReader, labelReader, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_START); - SequenceRecordReaderDataSetIterator iterAlignEnd = new SequenceRecordReaderDataSetIterator(featureReader2, labelReader2, 1, 4, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); - // Set up - SequenceRecordReader featureReader3 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader3 = new CSVSequenceRecordReader(1, ","); - featureReader3.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader3.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReader featureReader4 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader4 = new CSVSequenceRecordReader(1, ","); - featureReader4.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader4.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - RecordReaderMultiDataSetIterator rrmdsiStart = new RecordReaderMultiDataSetIterator.Builder(1).addSequenceReader("in", featureReader3).addSequenceReader("out", labelReader3).addInput("in").addOutputOneHot("out", 0, 4).sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_START).build(); - RecordReaderMultiDataSetIterator rrmdsiEnd = new RecordReaderMultiDataSetIterator.Builder(1).addSequenceReader("in", featureReader4).addSequenceReader("out", labelReader4).addInput("in").addOutputOneHot("out", 0, 4).sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_END).build(); - while (iterAlignStart.hasNext()) { - DataSet dsStart = iterAlignStart.next(); - DataSet dsEnd = iterAlignEnd.next(); - MultiDataSet mdsStart = rrmdsiStart.next(); - MultiDataSet mdsEnd = rrmdsiEnd.next(); - assertEquals(1, mdsStart.getFeatures().length); - assertEquals(1, mdsStart.getLabels().length); - // assertEquals(1, mdsStart.getFeaturesMaskArrays().length); //Features data is always longer -> don't need mask arrays for it - assertEquals(1, mdsStart.getLabelsMaskArrays().length); - assertEquals(1, mdsEnd.getFeatures().length); - assertEquals(1, mdsEnd.getLabels().length); - // assertEquals(1, mdsEnd.getFeaturesMaskArrays().length); - assertEquals(1, mdsEnd.getLabelsMaskArrays().length); - assertEquals(dsStart.getFeatures(), mdsStart.getFeatures(0)); - assertEquals(dsStart.getLabels(), mdsStart.getLabels(0)); - assertEquals(dsStart.getLabelsMaskArray(), mdsStart.getLabelsMaskArray(0)); - assertEquals(dsEnd.getFeatures(), mdsEnd.getFeatures(0)); - assertEquals(dsEnd.getLabels(), mdsEnd.getLabels(0)); - assertEquals(dsEnd.getLabelsMaskArray(), mdsEnd.getLabelsMaskArray(0)); - } - assertFalse(rrmdsiStart.hasNext()); - assertFalse(rrmdsiEnd.hasNext()); - } - - @Test - @DisplayName("Test Variable Length TS Meta") - void testVariableLengthTSMeta() throws Exception { - // need to manually extract - File rootDir = temporaryFolder.toFile(); - for (int i = 0; i < 3; i++) { - new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabels_%d.txt", i)).getTempFileFromArchive(rootDir); - new ClassPathResource(String.format("csvsequencelabelsShort_%d.txt", i)).getTempFileFromArchive(rootDir); - } - // Set up SequenceRecordReaderDataSetIterators for comparison - String featuresPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequence_%d.txt"); - String labelsPath = FilenameUtils.concat(rootDir.getAbsolutePath(), "csvsequencelabelsShort_%d.txt"); - // Set up - SequenceRecordReader featureReader3 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader3 = new CSVSequenceRecordReader(1, ","); - featureReader3.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader3.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - SequenceRecordReader featureReader4 = new CSVSequenceRecordReader(1, ","); - SequenceRecordReader labelReader4 = new CSVSequenceRecordReader(1, ","); - featureReader4.initialize(new NumberedFileInputSplit(featuresPath, 0, 2)); - labelReader4.initialize(new NumberedFileInputSplit(labelsPath, 0, 2)); - RecordReaderMultiDataSetIterator rrmdsiStart = new RecordReaderMultiDataSetIterator.Builder(1).addSequenceReader("in", featureReader3).addSequenceReader("out", labelReader3).addInput("in").addOutputOneHot("out", 0, 4).sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_START).build(); - RecordReaderMultiDataSetIterator rrmdsiEnd = new RecordReaderMultiDataSetIterator.Builder(1).addSequenceReader("in", featureReader4).addSequenceReader("out", labelReader4).addInput("in").addOutputOneHot("out", 0, 4).sequenceAlignmentMode(RecordReaderMultiDataSetIterator.AlignmentMode.ALIGN_END).build(); - rrmdsiStart.setCollectMetaData(true); - rrmdsiEnd.setCollectMetaData(true); - int count = 0; - while (rrmdsiStart.hasNext()) { - MultiDataSet mdsStart = rrmdsiStart.next(); - MultiDataSet mdsEnd = rrmdsiEnd.next(); - MultiDataSet mdsStartFromMeta = rrmdsiStart.loadFromMetaData(mdsStart.getExampleMetaData(RecordMetaData.class)); - MultiDataSet mdsEndFromMeta = rrmdsiEnd.loadFromMetaData(mdsEnd.getExampleMetaData(RecordMetaData.class)); - assertEquals(mdsStart, mdsStartFromMeta); - assertEquals(mdsEnd, mdsEndFromMeta); - count++; - } - assertFalse(rrmdsiStart.hasNext()); - assertFalse(rrmdsiEnd.hasNext()); - assertEquals(3, count); - } - - @Test - @DisplayName("Test Images RRDMSI") - void testImagesRRDMSI() throws Exception { - File parentDir = temporaryFolder.toFile(); - parentDir.deleteOnExit(); - String str1 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Zico/"); - String str2 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Ziwang_Xu/"); - File f1 = new File(str1); - File f2 = new File(str2); - f1.mkdirs(); - f2.mkdirs(); - TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f1.getPath(), "Zico_0001.jpg")), new ClassPathResource("lfwtest/Zico/Zico_0001.jpg").getInputStream()); - TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f2.getPath(), "Ziwang_Xu_0001.jpg")), new ClassPathResource("lfwtest/Ziwang_Xu/Ziwang_Xu_0001.jpg").getInputStream()); - int outputNum = 2; - Random r = new Random(12345); - ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator(); - ImageRecordReader rr1 = new ImageRecordReader(10, 10, 1, labelMaker); - ImageRecordReader rr1s = new ImageRecordReader(5, 5, 1, labelMaker); - rr1.initialize(new FileSplit(parentDir)); - rr1s.initialize(new FileSplit(parentDir)); - MultiDataSetIterator trainDataIterator = new RecordReaderMultiDataSetIterator.Builder(1).addReader("rr1", rr1).addReader("rr1s", rr1s).addInput("rr1", 0, 0).addInput("rr1s", 0, 0).addOutputOneHot("rr1s", 1, outputNum).build(); - // Now, do the same thing with ImageRecordReader, and check we get the same results: - ImageRecordReader rr1_b = new ImageRecordReader(10, 10, 1, labelMaker); - ImageRecordReader rr1s_b = new ImageRecordReader(5, 5, 1, labelMaker); - rr1_b.initialize(new FileSplit(parentDir)); - rr1s_b.initialize(new FileSplit(parentDir)); - DataSetIterator dsi1 = new RecordReaderDataSetIterator(rr1_b, 1, 1, 2); - DataSetIterator dsi2 = new RecordReaderDataSetIterator(rr1s_b, 1, 1, 2); - for (int i = 0; i < 2; i++) { - MultiDataSet mds = trainDataIterator.next(); - DataSet d1 = dsi1.next(); - DataSet d2 = dsi2.next(); - assertEquals(d1.getFeatures(), mds.getFeatures(0)); - assertEquals(d2.getFeatures(), mds.getFeatures(1)); - assertEquals(d1.getLabels(), mds.getLabels(0)); - } - } - - @Test - @DisplayName("Test Images RRDMSI _ Batched") - void testImagesRRDMSI_Batched() throws Exception { - File parentDir = temporaryFolder.toFile(); - parentDir.deleteOnExit(); - String str1 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Zico/"); - String str2 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Ziwang_Xu/"); - File f1 = new File(str1); - File f2 = new File(str2); - f1.mkdirs(); - f2.mkdirs(); - TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f1.getPath(), "Zico_0001.jpg")), new ClassPathResource("lfwtest/Zico/Zico_0001.jpg").getInputStream()); - TestUtils.writeStreamToFile(new File(FilenameUtils.concat(f2.getPath(), "Ziwang_Xu_0001.jpg")), new ClassPathResource("lfwtest/Ziwang_Xu/Ziwang_Xu_0001.jpg").getInputStream()); - int outputNum = 2; - ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator(); - ImageRecordReader rr1 = new ImageRecordReader(10, 10, 1, labelMaker); - ImageRecordReader rr1s = new ImageRecordReader(5, 5, 1, labelMaker); - URI[] uris = new FileSplit(parentDir).locations(); - rr1.initialize(new CollectionInputSplit(uris)); - rr1s.initialize(new CollectionInputSplit(uris)); - MultiDataSetIterator trainDataIterator = new RecordReaderMultiDataSetIterator.Builder(2).addReader("rr1", rr1).addReader("rr1s", rr1s).addInput("rr1", 0, 0).addInput("rr1s", 0, 0).addOutputOneHot("rr1s", 1, outputNum).build(); - // Now, do the same thing with ImageRecordReader, and check we get the same results: - ImageRecordReader rr1_b = new ImageRecordReader(10, 10, 1, labelMaker); - ImageRecordReader rr1s_b = new ImageRecordReader(5, 5, 1, labelMaker); - rr1_b.initialize(new FileSplit(parentDir)); - rr1s_b.initialize(new FileSplit(parentDir)); - DataSetIterator dsi1 = new RecordReaderDataSetIterator(rr1_b, 2, 1, 2); - DataSetIterator dsi2 = new RecordReaderDataSetIterator(rr1s_b, 2, 1, 2); - MultiDataSet mds = trainDataIterator.next(); - DataSet d1 = dsi1.next(); - DataSet d2 = dsi2.next(); - assertEquals(d1.getFeatures(), mds.getFeatures(0)); - assertEquals(d2.getFeatures(), mds.getFeatures(1)); - assertEquals(d1.getLabels(), mds.getLabels(0)); - // Check label assignment: - File currentFile = rr1_b.getCurrentFile(); - INDArray expLabels; - if (currentFile.getAbsolutePath().contains("Zico")) { - expLabels = Nd4j.create(new double[][] { { 0, 1 }, { 1, 0 } }); - } else { - expLabels = Nd4j.create(new double[][] { { 1, 0 }, { 0, 1 } }); - } - assertEquals(expLabels, d1.getLabels()); - assertEquals(expLabels, d2.getLabels()); - } - - @Test - @DisplayName("Test Time Series Random Offset") - void testTimeSeriesRandomOffset() { - // 2 in, 2 out, 3 total sequences of length [1,3,5] - List> seq1 = Arrays.asList(Arrays.asList(new DoubleWritable(1.0), new DoubleWritable(2.0))); - List> seq2 = Arrays.asList(Arrays.asList(new DoubleWritable(10.0), new DoubleWritable(11.0)), Arrays.asList(new DoubleWritable(20.0), new DoubleWritable(21.0)), Arrays.asList(new DoubleWritable(30.0), new DoubleWritable(31.0))); - List> seq3 = Arrays.asList(Arrays.asList(new DoubleWritable(100.0), new DoubleWritable(101.0)), Arrays.asList(new DoubleWritable(200.0), new DoubleWritable(201.0)), Arrays.asList(new DoubleWritable(300.0), new DoubleWritable(301.0)), Arrays.asList(new DoubleWritable(400.0), new DoubleWritable(401.0)), Arrays.asList(new DoubleWritable(500.0), new DoubleWritable(501.0))); - Collection>> seqs = Arrays.asList(seq1, seq2, seq3); - SequenceRecordReader rr = new CollectionSequenceRecordReader(seqs); - RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(3).addSequenceReader("rr", rr).addInput("rr", 0, 0).addOutput("rr", 1, 1).timeSeriesRandomOffset(true, 1234L).build(); - // Provides seed for each minibatch - Random r = new Random(1234); - long seed = r.nextLong(); - // Use same RNG seed in new RNG for each minibatch - Random r2 = new Random(seed); - // 0 to 4 inclusive - int expOffsetSeq1 = r2.nextInt(5 - 1 + 1); - int expOffsetSeq2 = r2.nextInt(5 - 3 + 1); - // Longest TS, always 0 - int expOffsetSeq3 = 0; - // With current seed: 3, 1, 0 - // System.out.println(expOffsetSeq1 + "\t" + expOffsetSeq2 + "\t" + expOffsetSeq3); - MultiDataSet mds = rrmdsi.next(); - INDArray expMask = Nd4j.create(new double[][] { { 0, 0, 0, 1, 0 }, { 0, 1, 1, 1, 0 }, { 1, 1, 1, 1, 1 } }); - assertEquals(expMask, mds.getFeaturesMaskArray(0)); - assertEquals(expMask, mds.getLabelsMaskArray(0)); - INDArray f = mds.getFeatures(0); - INDArray l = mds.getLabels(0); - INDArray expF1 = Nd4j.create(new double[] { 1.0 }, new int[] { 1, 1 }); - INDArray expL1 = Nd4j.create(new double[] { 2.0 }, new int[] { 1, 1 }); - INDArray expF2 = Nd4j.create(new double[] { 10, 20, 30 }, new int[] { 1, 3 }); - INDArray expL2 = Nd4j.create(new double[] { 11, 21, 31 }, new int[] { 1, 3 }); - INDArray expF3 = Nd4j.create(new double[] { 100, 200, 300, 400, 500 }, new int[] { 1, 5 }); - INDArray expL3 = Nd4j.create(new double[] { 101, 201, 301, 401, 501 }, new int[] { 1, 5 }); - assertEquals(expF1, f.get(point(0), all(), NDArrayIndex.interval(expOffsetSeq1, expOffsetSeq1 + 1))); - assertEquals(expL1, l.get(point(0), all(), NDArrayIndex.interval(expOffsetSeq1, expOffsetSeq1 + 1))); - assertEquals(expF2, f.get(point(1), all(), NDArrayIndex.interval(expOffsetSeq2, expOffsetSeq2 + 3))); - assertEquals(expL2, l.get(point(1), all(), NDArrayIndex.interval(expOffsetSeq2, expOffsetSeq2 + 3))); - assertEquals(expF3, f.get(point(2), all(), NDArrayIndex.interval(expOffsetSeq3, expOffsetSeq3 + 5))); - assertEquals(expL3, l.get(point(2), all(), NDArrayIndex.interval(expOffsetSeq3, expOffsetSeq3 + 5))); - } - - @Test - @DisplayName("Test Seq RRDSI Masking") - void testSeqRRDSIMasking() { - // This also tests RecordReaderMultiDataSetIterator, by virtue of - List>> features = new ArrayList<>(); - List>> labels = new ArrayList<>(); - features.add(Arrays.asList(l(new DoubleWritable(1)), l(new DoubleWritable(2)), l(new DoubleWritable(3)))); - features.add(Arrays.asList(l(new DoubleWritable(4)), l(new DoubleWritable(5)))); - labels.add(Arrays.asList(l(new IntWritable(0)))); - labels.add(Arrays.asList(l(new IntWritable(1)))); - CollectionSequenceRecordReader fR = new CollectionSequenceRecordReader(features); - CollectionSequenceRecordReader lR = new CollectionSequenceRecordReader(labels); - SequenceRecordReaderDataSetIterator seqRRDSI = new SequenceRecordReaderDataSetIterator(fR, lR, 2, 2, false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); - DataSet ds = seqRRDSI.next(); - INDArray fMask = Nd4j.create(new double[][] { { 1, 1, 1 }, { 1, 1, 0 } }); - INDArray lMask = Nd4j.create(new double[][] { { 0, 0, 1 }, { 0, 1, 0 } }); - assertEquals(fMask, ds.getFeaturesMaskArray()); - assertEquals(lMask, ds.getLabelsMaskArray()); - INDArray f = Nd4j.create(new double[][] { { 1, 2, 3 }, { 4, 5, 0 } }); - INDArray l = Nd4j.create(2, 2, 3); - l.putScalar(0, 0, 2, 1.0); - l.putScalar(1, 1, 1, 1.0); - assertEquals(f, ds.getFeatures().get(all(), point(0), all())); - assertEquals(l, ds.getLabels()); - } - - private static List l(Writable... in) { - return Arrays.asList(in); - } - - @Test - @DisplayName("Test Exclude String Col CSV") - void testExcludeStringColCSV() throws Exception { - File csvFile = temporaryFolder.toFile(); - StringBuilder sb = new StringBuilder(); - for (int i = 1; i <= 10; i++) { - if (i > 1) { - sb.append("\n"); - } - sb.append("skip_").append(i).append(",").append(i).append(",").append(i + 0.5); - } - FileUtils.writeStringToFile(csvFile, sb.toString()); - RecordReader rr = new CSVRecordReader(); - rr.initialize(new FileSplit(csvFile)); - RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10).addReader("rr", rr).addInput("rr", 1, 1).addOutput("rr", 2, 2).build(); - INDArray expFeatures = Nd4j.linspace(1, 10, 10).reshape(1, 10).transpose(); - INDArray expLabels = Nd4j.linspace(1, 10, 10).addi(0.5).reshape(1, 10).transpose(); - MultiDataSet mds = rrmdsi.next(); - assertFalse(rrmdsi.hasNext()); - assertEquals(expFeatures, mds.getFeatures(0).castTo(expFeatures.dataType())); - assertEquals(expLabels, mds.getLabels(0).castTo(expLabels.dataType())); - } - - private static final int nX = 32; - - private static final int nY = 32; - - private static final int nZ = 28; - - @Test - @DisplayName("Test RRMDSI 5 D") - void testRRMDSI5D() { - int batchSize = 5; - CustomRecordReader recordReader = new CustomRecordReader(); - DataSetIterator dataIter = new RecordReaderDataSetIterator(recordReader, batchSize, 1, /* Index of label in records */ - 2); - int count = 0; - while (dataIter.hasNext()) { - DataSet ds = dataIter.next(); - int offset = 5 * count; - for (int i = 0; i < 5; i++) { - INDArray act = ds.getFeatures().get(interval(i, i, true), all(), all(), all(), all()); - INDArray exp = Nd4j.valueArrayOf(new int[] { 1, 1, nZ, nX, nY }, i + offset); - assertEquals(exp, act); - } - count++; - } - assertEquals(2, count); - } - - @DisplayName("Custom Record Reader") - static class CustomRecordReader extends BaseRecordReader { - - int n = 0; - - CustomRecordReader() { - } - - @Override - public boolean batchesSupported() { - return false; - } - - @Override - public List> next(int num) { - throw new RuntimeException("Not implemented"); - } - - @Override - public List next() { - INDArray nd = Nd4j.create(new float[nZ * nY * nX], new int[] { 1, 1, nZ, nY, nX }, 'c').assign(n); - final List res = RecordConverter.toRecord(nd); - res.add(new IntWritable(0)); - n++; - return res; - } - - @Override - public boolean hasNext() { - return n < 10; - } - - final static ArrayList labels = new ArrayList<>(2); - - static { - labels.add("lbl0"); - labels.add("lbl1"); - } - - @Override - public List getLabels() { - return labels; - } - - @Override - public void reset() { - n = 0; - } - - @Override - public boolean resetSupported() { - return true; - } - - @Override - public List record(URI uri, DataInputStream dataInputStream) { - return next(); - } - - @Override - public Record nextRecord() { - List r = next(); - return new org.datavec.api.records.impl.Record(r, null); - } - - @Override - public Record loadFromMetaData(RecordMetaData recordMetaData) throws IOException { - throw new RuntimeException("Not implemented"); - } - - @Override - public List loadFromMetaData(List recordMetaDatas) { - throw new RuntimeException("Not implemented"); - } - - @Override - public void close() { - } - - @Override - public void setConf(Configuration conf) { - } - - @Override - public Configuration getConf() { - return null; - } - - @Override - public void initialize(InputSplit split) { - n = 0; - } - - @Override - public void initialize(Configuration conf, InputSplit split) { - n = 0; - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java deleted file mode 100644 index 56759fc07..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/fetchers/SvhnDataFetcherTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.fetchers; - -import org.deeplearning4j.BaseDL4JTest; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import java.io.File; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -/** - * @author saudet - */ -@DisplayName("Svhn Data Fetcher Test") -@Tag(TagNames.FILE_IO) -@NativeTag -class SvhnDataFetcherTest extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - // Shouldn't take this long but slow download or drive access on CI machines may need extra time. - return 480_000_000L; - } - - @Test - @DisplayName("Test Svhn Data Fetcher") - void testSvhnDataFetcher() throws Exception { - // Ignore unless integration tests - CI can get caught up on slow disk access - assumeTrue(isIntegrationTests()); - SvhnDataFetcher fetch = new SvhnDataFetcher(); - File path = fetch.getDataSetPath(DataSetType.TRAIN); - File path2 = fetch.getDataSetPath(DataSetType.TEST); - File path3 = fetch.getDataSetPath(DataSetType.VALIDATION); - assertTrue(path.isDirectory()); - assertTrue(path2.isDirectory()); - assertTrue(path3.isDirectory()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIteratorTest.java deleted file mode 100644 index aa9e6a825..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncDataSetIteratorTest.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.iterator; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.callbacks.InterleavedDataSetCallback; -import org.deeplearning4j.datasets.iterator.tools.VariableTimeseriesGenerator; -import org.deeplearning4j.nn.util.TestDataSetConsumer; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.jupiter.api.Assertions.assertThrows; - -@Slf4j -@DisplayName("Async Data Set Iterator Test") -@NativeTag -class AsyncDataSetIteratorTest extends BaseDL4JTest { - - private ExistingDataSetIterator backIterator; - - private static final int TEST_SIZE = 100; - - private static final int ITERATIONS = 10; - - // time spent in consumer thread, milliseconds - private static final long EXECUTION_TIME = 5; - - private static final long EXECUTION_SMALL = 1; - - @BeforeEach - void setUp() throws Exception { - List iterable = new ArrayList<>(); - for (int i = 0; i < TEST_SIZE; i++) { - iterable.add(new DataSet(Nd4j.create(new float[100]), Nd4j.create(new float[10]))); - } - backIterator = new ExistingDataSetIterator(iterable); - } - - @Test - @DisplayName("Has Next 1") - void hasNext1() throws Exception { - for (int iter = 0; iter < ITERATIONS; iter++) { - for (int prefetchSize = 2; prefetchSize <= 8; prefetchSize++) { - AsyncDataSetIterator iterator = new AsyncDataSetIterator(backIterator, prefetchSize); - int cnt = 0; - while (iterator.hasNext()) { - DataSet ds = iterator.next(); - assertNotEquals(null, ds); - cnt++; - } - assertEquals( TEST_SIZE, cnt,"Failed on iteration: " + iter + ", prefetchSize: " + prefetchSize); - iterator.shutdown(); - } - } - } - - @Test - @DisplayName("Has Next With Reset And Load") - void hasNextWithResetAndLoad() throws Exception { - int[] prefetchSizes; - if (isIntegrationTests()) { - prefetchSizes = new int[] { 2, 3, 4, 5, 6, 7, 8 }; - } else { - prefetchSizes = new int[] { 2, 3, 8 }; - } - for (int iter = 0; iter < ITERATIONS; iter++) { - for (int prefetchSize : prefetchSizes) { - AsyncDataSetIterator iterator = new AsyncDataSetIterator(backIterator, prefetchSize); - TestDataSetConsumer consumer = new TestDataSetConsumer(EXECUTION_SMALL); - int cnt = 0; - while (iterator.hasNext()) { - DataSet ds = iterator.next(); - consumer.consumeOnce(ds, false); - cnt++; - if (cnt == TEST_SIZE / 2) - iterator.reset(); - } - assertEquals(TEST_SIZE + (TEST_SIZE / 2), cnt); - iterator.shutdown(); - } - } - } - - @Test - @DisplayName("Test With Load") - void testWithLoad() { - for (int iter = 0; iter < ITERATIONS; iter++) { - AsyncDataSetIterator iterator = new AsyncDataSetIterator(backIterator, 8); - TestDataSetConsumer consumer = new TestDataSetConsumer(iterator, EXECUTION_TIME); - consumer.consumeWhileHasNext(true); - assertEquals(TEST_SIZE, consumer.getCount()); - iterator.shutdown(); - } - } - - @Test - @DisplayName("Test With Exception") - void testWithException() { - assertThrows(ArrayIndexOutOfBoundsException.class, () -> { - ExistingDataSetIterator crashingIterator = new ExistingDataSetIterator(new IterableWithException(100)); - AsyncDataSetIterator iterator = new AsyncDataSetIterator(crashingIterator, 8); - TestDataSetConsumer consumer = new TestDataSetConsumer(iterator, EXECUTION_SMALL); - consumer.consumeWhileHasNext(true); - iterator.shutdown(); - }); - } - - @DisplayName("Iterable With Exception") - private class IterableWithException implements Iterable { - - private final AtomicLong counter = new AtomicLong(0); - - private final int crashIteration; - - public IterableWithException(int iteration) { - crashIteration = iteration; - } - - @Override - public Iterator iterator() { - counter.set(0); - return new Iterator() { - - @Override - public boolean hasNext() { - return true; - } - - @Override - public DataSet next() { - if (counter.incrementAndGet() >= crashIteration) - throw new ArrayIndexOutOfBoundsException("Thrown as expected"); - return new DataSet(Nd4j.create(10), Nd4j.create(10)); - } - - @Override - public void remove() { - } - }; - } - } - - @Test - @DisplayName("Test Variable Time Series 1") - void testVariableTimeSeries1() throws Exception { - int numBatches = isIntegrationTests() ? 1000 : 100; - int batchSize = isIntegrationTests() ? 32 : 8; - int timeStepsMin = 10; - int timeStepsMax = isIntegrationTests() ? 500 : 100; - int valuesPerTimestep = isIntegrationTests() ? 128 : 16; - AsyncDataSetIterator adsi = new AsyncDataSetIterator(new VariableTimeseriesGenerator(1192, numBatches, batchSize, valuesPerTimestep, timeStepsMin, timeStepsMax, 10), 2, true); - for (int e = 0; e < 10; e++) { - int cnt = 0; - while (adsi.hasNext()) { - DataSet ds = adsi.next(); - // log.info("Features ptr: {}", AtomicAllocator.getInstance().getPointer(mds.getFeatures()[0].data()).address()); - assertEquals( (double) cnt, ds.getFeatures().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals( (double) cnt + 0.25, ds.getLabels().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals( (double) cnt + 0.5, ds.getFeaturesMaskArray().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals( (double) cnt + 0.75, ds.getLabelsMaskArray().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - cnt++; - } - adsi.reset(); - // log.info("Epoch {} finished...", e); - } - } - - @Test - @DisplayName("Test Variable Time Series 2") - void testVariableTimeSeries2() throws Exception { - AsyncDataSetIterator adsi = new AsyncDataSetIterator(new VariableTimeseriesGenerator(1192, 100, 32, 128, 100, 100, 100), 2, true, new InterleavedDataSetCallback(2 * 2)); - for (int e = 0; e < 5; e++) { - int cnt = 0; - while (adsi.hasNext()) { - DataSet ds = adsi.next(); - ds.detach(); - // log.info("Features ptr: {}", AtomicAllocator.getInstance().getPointer(mds.getFeatures()[0].data()).address()); - assertEquals((double) cnt, ds.getFeatures().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals((double) cnt + 0.25, ds.getLabels().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals( (double) cnt + 0.5, ds.getFeaturesMaskArray().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals((double) cnt + 0.75, ds.getLabelsMaskArray().meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - cnt++; - } - adsi.reset(); - // log.info("Epoch {} finished...", e); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIteratorTest.java deleted file mode 100644 index 70a1307a4..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/AsyncMultiDataSetIteratorTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.iterator; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.tools.VariableMultiTimeseriesGenerator; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.linalg.dataset.api.MultiDataSet; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -/* - @Test - public void testResetBug() throws Exception { - // /home/raver119/develop/dl4j-examples/src/main/resources/uci/train/features - - SequenceRecordReader trainFeatures = new CSVSequenceRecordReader(); - trainFeatures.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/train/features" + "/%d.csv", 0, 449)); - RecordReader trainLabels = new CSVRecordReader(); - trainLabels.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/train/labels" + "/%d.csv", 0, 449)); - - int miniBatchSize = 10; - int numLabelClasses = 6; - MultiDataSetIterator trainData = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize) - .addSequenceReader("features", trainFeatures) - .addReader("labels", trainLabels) - .addInput("features") - .addOutputOneHot("labels", 0, numLabelClasses) - .build(); - - //Normalize the training data - MultiDataNormalization normalizer = new MultiNormalizerStandardize(); - normalizer.fit(trainData); //Collect training data statistics - trainData.reset(); - - - SequenceRecordReader testFeatures = new CSVSequenceRecordReader(); - testFeatures.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/test/features" + "/%d.csv", 0, 149)); - RecordReader testLabels = new CSVRecordReader(); - testLabels.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/test/labels" + "/%d.csv", 0, 149)); - - MultiDataSetIterator testData = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize) - .addSequenceReader("features", testFeatures) - .addReader("labels", testLabels) - .addInput("features") - .addOutputOneHot("labels", 0, numLabelClasses) - .build(); - - System.out.println("-------------- HASH 1----------------"); - testData.reset(); - while(testData.hasNext()){ - System.out.println(Arrays.hashCode(testData.next().getFeatures(0).data().asFloat())); - } - - System.out.println("-------------- HASH 2 ----------------"); - testData.reset(); - testData.hasNext(); //***** Remove this (or move to after async creation), and we get expected results ***** - val adsi = new AsyncMultiDataSetIterator(testData, 4, true); //OR remove this (keeping hasNext) and we get expected results - //val adsi = new AsyncShieldMultiDataSetIterator(testData); - while(adsi.hasNext()){ - System.out.println(Arrays.hashCode(adsi.next().getFeatures(0).data().asFloat())); - } - } - */ -@DisplayName("Async Multi Data Set Iterator Test") -@NativeTag -class AsyncMultiDataSetIteratorTest extends BaseDL4JTest { - - /** - * THIS TEST SHOULD BE ALWAYS RUN WITH DOUBLE PRECISION, WITHOUT ANY EXCLUSIONS - * - * @throws Exception - */ - @Test - @DisplayName("Test Variable Time Series 1") - void testVariableTimeSeries1() throws Exception { - int numBatches = isIntegrationTests() ? 1000 : 100; - int batchSize = isIntegrationTests() ? 32 : 8; - int timeStepsMin = 10; - int timeStepsMax = isIntegrationTests() ? 500 : 100; - int valuesPerTimestep = isIntegrationTests() ? 128 : 16; - val iterator = new VariableMultiTimeseriesGenerator(1192, numBatches, batchSize, valuesPerTimestep, timeStepsMin, timeStepsMax, 10); - iterator.reset(); - iterator.hasNext(); - val amdsi = new AsyncMultiDataSetIterator(iterator, 2, true); - for (int e = 0; e < 10; e++) { - int cnt = 0; - while (amdsi.hasNext()) { - MultiDataSet mds = amdsi.next(); - // log.info("Features ptr: {}", AtomicAllocator.getInstance().getPointer(mds.getFeatures()[0].data()).address()); - assertEquals( (double) cnt, mds.getFeatures()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals( (double) cnt + 0.25, mds.getLabels()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals((double) cnt + 0.5, mds.getFeaturesMaskArrays()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals((double) cnt + 0.75, mds.getLabelsMaskArrays()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - cnt++; - } - amdsi.reset(); - log.info("Epoch {} finished...", e); - } - } - - @Test - @DisplayName("Test Variable Time Series 2") - void testVariableTimeSeries2() throws Exception { - int numBatches = isIntegrationTests() ? 1000 : 100; - int batchSize = isIntegrationTests() ? 32 : 8; - int timeStepsMin = 10; - int timeStepsMax = isIntegrationTests() ? 500 : 100; - int valuesPerTimestep = isIntegrationTests() ? 128 : 16; - val iterator = new VariableMultiTimeseriesGenerator(1192, numBatches, batchSize, valuesPerTimestep, timeStepsMin, timeStepsMax, 10); - for (int e = 0; e < 10; e++) { - iterator.reset(); - iterator.hasNext(); - val amdsi = new AsyncMultiDataSetIterator(iterator, 2, true); - int cnt = 0; - while (amdsi.hasNext()) { - MultiDataSet mds = amdsi.next(); - // log.info("Features ptr: {}", AtomicAllocator.getInstance().getPointer(mds.getFeatures()[0].data()).address()); - assertEquals( (double) cnt, mds.getFeatures()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals((double) cnt + 0.25, mds.getLabels()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals( (double) cnt + 0.5, mds.getFeaturesMaskArrays()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - assertEquals( (double) cnt + 0.75, mds.getLabelsMaskArrays()[0].meanNumber().doubleValue(), 1e-10,"Failed on epoch " + e + "; iteration: " + cnt + ";"); - cnt++; - } - } - } - /* - @Test - public void testResetBug() throws Exception { - // /home/raver119/develop/dl4j-examples/src/main/resources/uci/train/features - - SequenceRecordReader trainFeatures = new CSVSequenceRecordReader(); - trainFeatures.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/train/features" + "/%d.csv", 0, 449)); - RecordReader trainLabels = new CSVRecordReader(); - trainLabels.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/train/labels" + "/%d.csv", 0, 449)); - - int miniBatchSize = 10; - int numLabelClasses = 6; - MultiDataSetIterator trainData = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize) - .addSequenceReader("features", trainFeatures) - .addReader("labels", trainLabels) - .addInput("features") - .addOutputOneHot("labels", 0, numLabelClasses) - .build(); - - //Normalize the training data - MultiDataNormalization normalizer = new MultiNormalizerStandardize(); - normalizer.fit(trainData); //Collect training data statistics - trainData.reset(); - - - SequenceRecordReader testFeatures = new CSVSequenceRecordReader(); - testFeatures.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/test/features" + "/%d.csv", 0, 149)); - RecordReader testLabels = new CSVRecordReader(); - testLabels.initialize(new NumberedFileInputSplit("/home/raver119/develop/dl4j-examples/src/main/resources/uci/test/labels" + "/%d.csv", 0, 149)); - - MultiDataSetIterator testData = new RecordReaderMultiDataSetIterator.Builder(miniBatchSize) - .addSequenceReader("features", testFeatures) - .addReader("labels", testLabels) - .addInput("features") - .addOutputOneHot("labels", 0, numLabelClasses) - .build(); - - System.out.println("-------------- HASH 1----------------"); - testData.reset(); - while(testData.hasNext()){ - System.out.println(Arrays.hashCode(testData.next().getFeatures(0).data().asFloat())); - } - - System.out.println("-------------- HASH 2 ----------------"); - testData.reset(); - testData.hasNext(); //***** Remove this (or move to after async creation), and we get expected results ***** - val adsi = new AsyncMultiDataSetIterator(testData, 4, true); //OR remove this (keeping hasNext) and we get expected results - //val adsi = new AsyncShieldMultiDataSetIterator(testData); - while(adsi.hasNext()){ - System.out.println(Arrays.hashCode(adsi.next().getFeatures(0).data().asFloat())); - } - } - */ -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java deleted file mode 100644 index 25246f848..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.iterator; - -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.image.loader.CifarLoader; -import org.datavec.image.loader.LFWLoader; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; -import org.deeplearning4j.datasets.fetchers.DataSetType; -import org.deeplearning4j.datasets.iterator.impl.*; -import org.deeplearning4j.eval.Evaluation; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.listeners.CollectScoresIterationListener; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Data Set Iterator Test") -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -class DataSetIteratorTest extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - // Should run quickly; increased to large timeout due to occasonal slow CI downloads - return 360000; - } - - @Test - @DisplayName("Test Batch Size Of One Iris") - void testBatchSizeOfOneIris() throws Exception { - // Test for (a) iterators returning correct number of examples, and - // (b) Labels are a proper one-hot vector (i.e., sum is 1.0) - // Iris: - DataSetIterator iris = new IrisDataSetIterator(1, 5); - int irisC = 0; - while (iris.hasNext()) { - irisC++; - DataSet ds = iris.next(); - assertTrue(ds.getLabels().sum(Integer.MAX_VALUE).getDouble(0) == 1.0); - } - assertEquals(5, irisC); - } - - @Test - @DisplayName("Test Batch Size Of One Mnist") - void testBatchSizeOfOneMnist() throws Exception { - // MNIST: - DataSetIterator mnist = new MnistDataSetIterator(1, 5); - int mnistC = 0; - while (mnist.hasNext()) { - mnistC++; - DataSet ds = mnist.next(); - assertTrue(ds.getLabels().sum(Integer.MAX_VALUE).getDouble(0) == 1.0); - } - assertEquals(5, mnistC); - } - - @Test - @DisplayName("Test Mnist") - void testMnist() throws Exception { - ClassPathResource cpr = new ClassPathResource("mnist_first_200.txt"); - CSVRecordReader rr = new CSVRecordReader(0, ','); - rr.initialize(new FileSplit(cpr.getTempFileFromArchive())); - RecordReaderDataSetIterator dsi = new RecordReaderDataSetIterator(rr, 10, 0, 10); - MnistDataSetIterator iter = new MnistDataSetIterator(10, 200, false, true, false, 0); - while (dsi.hasNext()) { - DataSet dsExp = dsi.next(); - DataSet dsAct = iter.next(); - INDArray fExp = dsExp.getFeatures(); - fExp.divi(255); - INDArray lExp = dsExp.getLabels(); - INDArray fAct = dsAct.getFeatures(); - INDArray lAct = dsAct.getLabels(); - assertEquals(fExp, fAct.castTo(fExp.dataType())); - assertEquals(lExp, lAct.castTo(lExp.dataType())); - } - assertFalse(iter.hasNext()); - } - - @Test - @DisplayName("Test Lfw Iterator") - void testLfwIterator() throws Exception { - int numExamples = 1; - int row = 28; - int col = 28; - int channels = 1; - LFWDataSetIterator iter = new LFWDataSetIterator(numExamples, new int[] { row, col, channels }, true); - assertTrue(iter.hasNext()); - DataSet data = iter.next(); - assertEquals(numExamples, data.getLabels().size(0)); - assertEquals(row, data.getFeatures().size(2)); - } - - @Test - @DisplayName("Test Tiny Image Net Iterator") - void testTinyImageNetIterator() throws Exception { - int numClasses = 200; - int row = 64; - int col = 64; - int channels = 3; - TinyImageNetDataSetIterator iter = new TinyImageNetDataSetIterator(1, DataSetType.TEST); - assertTrue(iter.hasNext()); - DataSet data = iter.next(); - assertEquals(numClasses, data.getLabels().size(1)); - assertArrayEquals(new long[] { 1, channels, row, col }, data.getFeatures().shape()); - } - - @Test - @DisplayName("Test Tiny Image Net Iterator 2") - void testTinyImageNetIterator2() throws Exception { - int numClasses = 200; - int row = 224; - int col = 224; - int channels = 3; - TinyImageNetDataSetIterator iter = new TinyImageNetDataSetIterator(1, new int[] { row, col }, DataSetType.TEST); - assertTrue(iter.hasNext()); - DataSet data = iter.next(); - assertEquals(numClasses, data.getLabels().size(1)); - assertArrayEquals(new long[] { 1, channels, row, col }, data.getFeatures().shape()); - } - - @Test - @DisplayName("Test Lfw Model") - void testLfwModel() throws Exception { - final int numRows = 28; - final int numColumns = 28; - int numChannels = 3; - int outputNum = LFWLoader.NUM_LABELS; - int numSamples = LFWLoader.NUM_IMAGES; - int batchSize = 2; - int seed = 123; - int listenerFreq = 1; - LFWDataSetIterator lfw = new LFWDataSetIterator(batchSize, numSamples, new int[] { numRows, numColumns, numChannels }, outputNum, false, true, 1.0, new Random(seed)); - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list().layer(0, new ConvolutionLayer.Builder(5, 5).nIn(numChannels).nOut(6).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).stride(1, 1).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(numRows, numColumns, numChannels)); - MultiLayerNetwork model = new MultiLayerNetwork(builder.build()); - model.init(); - model.setListeners(new ScoreIterationListener(listenerFreq)); - model.fit(lfw.next()); - DataSet dataTest = lfw.next(); - INDArray output = model.output(dataTest.getFeatures()); - Evaluation eval = new Evaluation(outputNum); - eval.eval(dataTest.getLabels(), output); - // System.out.println(eval.stats()); - } - - @Test - @DisplayName("Test Cifar 10 Iterator") - void testCifar10Iterator() throws Exception { - int numExamples = 1; - int row = 32; - int col = 32; - int channels = 3; - Cifar10DataSetIterator iter = new Cifar10DataSetIterator(numExamples); - assertTrue(iter.hasNext()); - DataSet data = iter.next(); - assertEquals(numExamples, data.getLabels().size(0)); - assertEquals(channels * row * col, data.getFeatures().ravel().length()); - } - - // Ignored for now - CIFAR iterator needs work - https://github.com/eclipse/deeplearning4j/issues/4673 - @Test - @Disabled - @DisplayName("Test Cifar Model") - void testCifarModel() throws Exception { - // Streaming - runCifar(false); - // Preprocess - runCifar(true); - } - - public void runCifar(boolean preProcessCifar) throws Exception { - final int height = 32; - final int width = 32; - int channels = 3; - int outputNum = CifarLoader.NUM_LABELS; - int batchSize = 5; - int seed = 123; - int listenerFreq = 1; - Cifar10DataSetIterator cifar = new Cifar10DataSetIterator(batchSize); - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list().layer(0, new ConvolutionLayer.Builder(5, 5).nIn(channels).nOut(6).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(height, width, channels)); - MultiLayerNetwork model = new MultiLayerNetwork(builder.build()); - model.init(); - // model.setListeners(Arrays.asList((TrainingListener) new ScoreIterationListener(listenerFreq))); - CollectScoresIterationListener listener = new CollectScoresIterationListener(listenerFreq); - model.setListeners(listener); - model.fit(cifar); - cifar = new Cifar10DataSetIterator(batchSize); - Evaluation eval = new Evaluation(cifar.getLabels()); - while (cifar.hasNext()) { - DataSet testDS = cifar.next(batchSize); - INDArray output = model.output(testDS.getFeatures()); - eval.eval(testDS.getLabels(), output); - } - // System.out.println(eval.stats(true)); - listener.exportScores(System.out); - } - - @Test - @DisplayName("Test Iterator Data Set Iterator Combining") - void testIteratorDataSetIteratorCombining() { - // Test combining of a bunch of small (size 1) data sets together - int batchSize = 3; - int numBatches = 4; - int featureSize = 5; - int labelSize = 6; - Nd4j.getRandom().setSeed(12345); - List orig = new ArrayList<>(); - for (int i = 0; i < batchSize * numBatches; i++) { - INDArray features = Nd4j.rand(1, featureSize); - INDArray labels = Nd4j.rand(1, labelSize); - orig.add(new DataSet(features, labels)); - } - DataSetIterator iter = new IteratorDataSetIterator(orig.iterator(), batchSize); - int count = 0; - while (iter.hasNext()) { - DataSet ds = iter.next(); - assertArrayEquals(new long[] { batchSize, featureSize }, ds.getFeatures().shape()); - assertArrayEquals(new long[] { batchSize, labelSize }, ds.getLabels().shape()); - List fList = new ArrayList<>(); - List lList = new ArrayList<>(); - for (int i = 0; i < batchSize; i++) { - DataSet dsOrig = orig.get(count * batchSize + i); - fList.add(dsOrig.getFeatures()); - lList.add(dsOrig.getLabels()); - } - INDArray fExp = Nd4j.vstack(fList); - INDArray lExp = Nd4j.vstack(lList); - assertEquals(fExp, ds.getFeatures()); - assertEquals(lExp, ds.getLabels()); - count++; - } - assertEquals(count, numBatches); - } - - @Test - @DisplayName("Test Iterator Data Set Iterator Splitting") - void testIteratorDataSetIteratorSplitting() { - // Test splitting large data sets into smaller ones - int origBatchSize = 4; - int origNumDSs = 3; - int batchSize = 3; - int numBatches = 4; - int featureSize = 5; - int labelSize = 6; - Nd4j.getRandom().setSeed(12345); - List orig = new ArrayList<>(); - for (int i = 0; i < origNumDSs; i++) { - INDArray features = Nd4j.rand(origBatchSize, featureSize); - INDArray labels = Nd4j.rand(origBatchSize, labelSize); - orig.add(new DataSet(features, labels)); - } - List expected = new ArrayList<>(); - expected.add(new DataSet(orig.get(0).getFeatures().getRows(0, 1, 2), orig.get(0).getLabels().getRows(0, 1, 2))); - expected.add(new DataSet(Nd4j.vstack(orig.get(0).getFeatures().getRows(3), orig.get(1).getFeatures().getRows(0, 1)), Nd4j.vstack(orig.get(0).getLabels().getRows(3), orig.get(1).getLabels().getRows(0, 1)))); - expected.add(new DataSet(Nd4j.vstack(orig.get(1).getFeatures().getRows(2, 3), orig.get(2).getFeatures().getRows(0)), Nd4j.vstack(orig.get(1).getLabels().getRows(2, 3), orig.get(2).getLabels().getRows(0)))); - expected.add(new DataSet(orig.get(2).getFeatures().getRows(1, 2, 3), orig.get(2).getLabels().getRows(1, 2, 3))); - DataSetIterator iter = new IteratorDataSetIterator(orig.iterator(), batchSize); - int count = 0; - while (iter.hasNext()) { - DataSet ds = iter.next(); - assertEquals(expected.get(count), ds); - count++; - } - assertEquals(count, numBatches); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIteratorTest.java deleted file mode 100644 index c0bd12c5b..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationDataSetIteratorTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.iterator; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -import org.junit.jupiter.api.DisplayName; - -@DisplayName("Early Termination Data Set Iterator Test") -@NativeTag -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -class EarlyTerminationDataSetIteratorTest extends BaseDL4JTest { - - int minibatchSize = 10; - - int numExamples = 105; - - - - @Test - @DisplayName("Test Next And Reset") - void testNextAndReset() throws Exception { - int terminateAfter = 2; - DataSetIterator iter = new MnistDataSetIterator(minibatchSize, numExamples); - EarlyTerminationDataSetIterator earlyEndIter = new EarlyTerminationDataSetIterator(iter, terminateAfter); - assertTrue(earlyEndIter.hasNext()); - int batchesSeen = 0; - List seenData = new ArrayList<>(); - while (earlyEndIter.hasNext()) { - DataSet path = earlyEndIter.next(); - assertFalse(path == null); - seenData.add(path); - batchesSeen++; - } - assertEquals(batchesSeen, terminateAfter); - // check data is repeated after reset - earlyEndIter.reset(); - batchesSeen = 0; - while (earlyEndIter.hasNext()) { - DataSet path = earlyEndIter.next(); - assertEquals(seenData.get(batchesSeen).getFeatures(), path.getFeatures()); - assertEquals(seenData.get(batchesSeen).getLabels(), path.getLabels()); - batchesSeen++; - } - } - - @Test - @DisplayName("Test Next Num") - void testNextNum() throws IOException { - int terminateAfter = 1; - DataSetIterator iter = new MnistDataSetIterator(minibatchSize, numExamples); - EarlyTerminationDataSetIterator earlyEndIter = new EarlyTerminationDataSetIterator(iter, terminateAfter); - earlyEndIter.next(10); - assertEquals(false, earlyEndIter.hasNext()); - earlyEndIter.reset(); - assertEquals(true, earlyEndIter.hasNext()); - } - - @Test - @DisplayName("Test calls to Next Not Allowed") - void testCallstoNextNotAllowed() throws IOException { - assertThrows(RuntimeException.class,() -> { - int terminateAfter = 1; - DataSetIterator iter = new MnistDataSetIterator(minibatchSize, numExamples); - EarlyTerminationDataSetIterator earlyEndIter = new EarlyTerminationDataSetIterator(iter, terminateAfter); - earlyEndIter.next(10); - iter.reset(); - earlyEndIter.next(10); - }); - - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIteratorTest.java deleted file mode 100644 index a729d0d04..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/EarlyTerminationMultiDataSetIteratorTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.iterator; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Tags; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.dataset.adapter.MultiDataSetIteratorAdapter; -import org.nd4j.linalg.dataset.api.MultiDataSet; -import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -@DisplayName("Early Termination Multi Data Set Iterator Test") -@NativeTag -@Tag(TagNames.FILE_IO) -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -class EarlyTerminationMultiDataSetIteratorTest extends BaseDL4JTest { - - int minibatchSize = 5; - - int numExamples = 105; - - - - @Test - @DisplayName("Test Next And Reset") - void testNextAndReset() throws Exception { - int terminateAfter = 2; - MultiDataSetIterator iter = new MultiDataSetIteratorAdapter(new MnistDataSetIterator(minibatchSize, numExamples)); - int count = 0; - List seenMDS = new ArrayList<>(); - while (count < terminateAfter) { - seenMDS.add(iter.next()); - count++; - } - iter.reset(); - EarlyTerminationMultiDataSetIterator earlyEndIter = new EarlyTerminationMultiDataSetIterator(iter, terminateAfter); - assertTrue(earlyEndIter.hasNext()); - count = 0; - while (earlyEndIter.hasNext()) { - MultiDataSet path = earlyEndIter.next(); - assertEquals(path.getFeatures()[0], seenMDS.get(count).getFeatures()[0]); - assertEquals(path.getLabels()[0], seenMDS.get(count).getLabels()[0]); - count++; - } - assertEquals(count, terminateAfter); - // check data is repeated - earlyEndIter.reset(); - count = 0; - while (earlyEndIter.hasNext()) { - MultiDataSet path = earlyEndIter.next(); - assertEquals(path.getFeatures()[0], seenMDS.get(count).getFeatures()[0]); - assertEquals(path.getLabels()[0], seenMDS.get(count).getLabels()[0]); - count++; - } - } - - @Test - @DisplayName("Test Next Num") - void testNextNum() throws IOException { - int terminateAfter = 1; - MultiDataSetIterator iter = new MultiDataSetIteratorAdapter(new MnistDataSetIterator(minibatchSize, numExamples)); - EarlyTerminationMultiDataSetIterator earlyEndIter = new EarlyTerminationMultiDataSetIterator(iter, terminateAfter); - earlyEndIter.next(10); - assertEquals(false, earlyEndIter.hasNext()); - earlyEndIter.reset(); - assertEquals(true, earlyEndIter.hasNext()); - } - - @Test - @DisplayName("Test calls to Next Not Allowed") - void testCallstoNextNotAllowed() throws IOException { - assertThrows(RuntimeException.class,() -> { - int terminateAfter = 1; - MultiDataSetIterator iter = new MultiDataSetIteratorAdapter(new MnistDataSetIterator(minibatchSize, numExamples)); - EarlyTerminationMultiDataSetIterator earlyEndIter = new EarlyTerminationMultiDataSetIterator(iter, terminateAfter); - earlyEndIter.next(10); - iter.reset(); - earlyEndIter.next(10); - }); - - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/JointParallelDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/JointParallelDataSetIteratorTest.java deleted file mode 100644 index 0846a0112..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/JointParallelDataSetIteratorTest.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.iterator; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.parallel.JointParallelDataSetIterator; -import org.deeplearning4j.datasets.iterator.tools.SimpleVariableGenerator; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.dataset.api.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.iterator.enums.InequalityHandling; -import org.nd4j.linalg.factory.Nd4j; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Joint Parallel Data Set Iterator Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class JointParallelDataSetIteratorTest extends BaseDL4JTest { - - /** - * Simple test, checking datasets alignment. They all should have the same data for the same cycle - * - * @throws Exception - */ - @Test - @DisplayName("Test Joint Iterator 1") - void testJointIterator1() throws Exception { - DataSetIterator iteratorA = new SimpleVariableGenerator(119, 100, 32, 100, 10); - DataSetIterator iteratorB = new SimpleVariableGenerator(119, 100, 32, 100, 10); - JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator.Builder(InequalityHandling.STOP_EVERYONE).addSourceIterator(iteratorA).addSourceIterator(iteratorB).build(); - int cnt = 0; - int example = 0; - while (jpdsi.hasNext()) { - DataSet ds = jpdsi.next(); - assertNotNull(ds,"Failed on iteration " + cnt); - // ds.detach(); - // ds.migrate(); - assertEquals( (double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - assertEquals( (double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - cnt++; - if (cnt % 2 == 0) - example++; - } - assertEquals(100, example); - assertEquals(200, cnt); - } - - /** - * This test checks for pass_null scenario, so in total we should have 300 real datasets + 100 nulls - * @throws Exception - */ - @Test - @DisplayName("Test Joint Iterator 2") - void testJointIterator2() throws Exception { - DataSetIterator iteratorA = new SimpleVariableGenerator(119, 200, 32, 100, 10); - DataSetIterator iteratorB = new SimpleVariableGenerator(119, 100, 32, 100, 10); - JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator.Builder(InequalityHandling.PASS_NULL).addSourceIterator(iteratorA).addSourceIterator(iteratorB).build(); - int cnt = 0; - int example = 0; - int nulls = 0; - while (jpdsi.hasNext()) { - DataSet ds = jpdsi.next(); - if (cnt < 200) - assertNotNull(ds,"Failed on iteration " + cnt); - if (ds == null) - nulls++; - if (cnt % 2 == 2) { - assertEquals((double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - assertEquals((double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - } - cnt++; - if (cnt % 2 == 0) - example++; - } - assertEquals(100, nulls); - assertEquals(200, example); - assertEquals(400, cnt); - } - - /** - * Testing relocate - * - * @throws Exception - */ - @Test - @DisplayName("Test Joint Iterator 3") - void testJointIterator3() throws Exception { - DataSetIterator iteratorA = new SimpleVariableGenerator(119, 200, 32, 100, 10); - DataSetIterator iteratorB = new SimpleVariableGenerator(119, 100, 32, 100, 10); - JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator.Builder(InequalityHandling.RELOCATE).addSourceIterator(iteratorA).addSourceIterator(iteratorB).build(); - int cnt = 0; - int example = 0; - while (jpdsi.hasNext()) { - DataSet ds = jpdsi.next(); - assertNotNull(ds,"Failed on iteration " + cnt); - assertEquals((double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - assertEquals( (double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - cnt++; - if (cnt < 200) { - if (cnt % 2 == 0) - example++; - } else - example++; - } - assertEquals(300, cnt); - assertEquals(200, example); - } - - /** - * Testing relocate - * - * @throws Exception - */ - @Test - @DisplayName("Test Joint Iterator 4") - void testJointIterator4() throws Exception { - DataSetIterator iteratorA = new SimpleVariableGenerator(119, 200, 32, 100, 10); - DataSetIterator iteratorB = new SimpleVariableGenerator(119, 100, 32, 100, 10); - JointParallelDataSetIterator jpdsi = new JointParallelDataSetIterator.Builder(InequalityHandling.RESET).addSourceIterator(iteratorA).addSourceIterator(iteratorB).build(); - int cnt = 0; - int cnt_sec = 0; - int example_sec = 0; - int example = 0; - while (jpdsi.hasNext()) { - DataSet ds = jpdsi.next(); - assertNotNull(ds,"Failed on iteration " + cnt); - if (cnt % 2 == 0) { - assertEquals( (double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - assertEquals((double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - } else { - if (cnt <= 200) { - assertEquals((double) example, ds.getFeatures().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - assertEquals( (double) example + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt); - } else { - assertEquals((double) example_sec, ds.getFeatures().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt + ", second iteration " + cnt_sec); - assertEquals((double) example_sec + 0.5, ds.getLabels().meanNumber().doubleValue(), 0.001,"Failed on iteration " + cnt + ", second iteration " + cnt_sec); - } - } - cnt++; - if (cnt % 2 == 0) - example++; - if (cnt > 201 && cnt % 2 == 1) { - cnt_sec++; - example_sec++; - } - } - assertEquals(400, cnt); - assertEquals(200, example); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/RandomDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/RandomDataSetIteratorTest.java deleted file mode 100644 index e744d3cb4..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/RandomDataSetIteratorTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.datasets.iterator; - -import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.MultiDataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Random Data Set Iterator Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class RandomDataSetIteratorTest extends BaseDL4JTest { - - @Test - @DisplayName("Test DSI") - void testDSI() { - DataSetIterator iter = new RandomDataSetIterator(5, new long[] { 3, 4 }, new long[] { 3, 5 }, RandomDataSetIterator.Values.RANDOM_UNIFORM, RandomDataSetIterator.Values.ONE_HOT); - int count = 0; - while (iter.hasNext()) { - count++; - DataSet ds = iter.next(); - assertArrayEquals(new long[] { 3, 4 }, ds.getFeatures().shape()); - assertArrayEquals(new long[] { 3, 5 }, ds.getLabels().shape()); - assertTrue(ds.getFeatures().minNumber().doubleValue() >= 0.0 && ds.getFeatures().maxNumber().doubleValue() <= 1.0); - assertEquals(Nd4j.ones(3), ds.getLabels().sum(1)); - } - assertEquals(5, count); - } - - @Test - @DisplayName("Test MDSI") - void testMDSI() { - Nd4j.getRandom().setSeed(12345); - MultiDataSetIterator iter = new RandomMultiDataSetIterator.Builder(5).addFeatures(new long[] { 3, 4 }, RandomMultiDataSetIterator.Values.INTEGER_0_100).addFeatures(new long[] { 3, 5 }, RandomMultiDataSetIterator.Values.BINARY).addLabels(new long[] { 3, 6 }, RandomMultiDataSetIterator.Values.ZEROS).build(); - int count = 0; - while (iter.hasNext()) { - count++; - MultiDataSet mds = iter.next(); - assertEquals(2, mds.numFeatureArrays()); - assertEquals(1, mds.numLabelsArrays()); - assertArrayEquals(new long[] { 3, 4 }, mds.getFeatures(0).shape()); - assertArrayEquals(new long[] { 3, 5 }, mds.getFeatures(1).shape()); - assertArrayEquals(new long[] { 3, 6 }, mds.getLabels(0).shape()); - assertTrue(mds.getFeatures(0).minNumber().doubleValue() >= 0 && mds.getFeatures(0).maxNumber().doubleValue() <= 100.0 && mds.getFeatures(0).maxNumber().doubleValue() > 2.0); - assertTrue(mds.getFeatures(1).minNumber().doubleValue() == 0.0 && mds.getFeatures(1).maxNumber().doubleValue() == 1.0); - assertEquals(0.0, mds.getLabels(0).sumNumber().doubleValue(), 0.0); - } - assertEquals(5, count); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvalJsonTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvalJsonTest.java deleted file mode 100644 index f8bcfcb7b..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvalJsonTest.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.eval; - -import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.evaluation.curves.Histogram; -import org.nd4j.evaluation.curves.PrecisionRecallCurve; -import org.nd4j.evaluation.curves.RocCurve; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution; -import org.nd4j.linalg.factory.Nd4j; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -import static org.junit.jupiter.api.Assertions.*; - -@DisplayName("Eval Json Test") -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.JACKSON_SERDE) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -class EvalJsonTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Serde Empty") - void testSerdeEmpty() { - boolean print = false; - org.nd4j.evaluation.IEvaluation[] arr = new org.nd4j.evaluation.IEvaluation[] { new Evaluation(), new EvaluationBinary(), new ROCBinary(10), new ROCMultiClass(10), new RegressionEvaluation(3), new RegressionEvaluation(), new EvaluationCalibration() }; - for (org.nd4j.evaluation.IEvaluation e : arr) { - String json = e.toJson(); - String stats = e.stats(); - if (print) { - System.out.println(e.getClass() + "\n" + json + "\n\n"); - } - IEvaluation fromJson = (IEvaluation) org.nd4j.evaluation.BaseEvaluation.fromJson(json, org.nd4j.evaluation.BaseEvaluation.class); - assertEquals(e.toJson(), fromJson.toJson()); - } - } - - @Test - @DisplayName("Test Serde") - void testSerde() { - boolean print = false; - Nd4j.getRandom().setSeed(12345); - Evaluation evaluation = new Evaluation(); - EvaluationBinary evaluationBinary = new EvaluationBinary(); - ROC roc = new ROC(2); - ROCBinary roc2 = new ROCBinary(2); - ROCMultiClass roc3 = new ROCMultiClass(2); - RegressionEvaluation regressionEvaluation = new RegressionEvaluation(); - EvaluationCalibration ec = new EvaluationCalibration(); - org.nd4j.evaluation.IEvaluation[] arr = new org.nd4j.evaluation.IEvaluation[] { evaluation, evaluationBinary, roc, roc2, roc3, regressionEvaluation, ec }; - INDArray evalLabel = Nd4j.create(10, 3); - for (int i = 0; i < 10; i++) { - evalLabel.putScalar(i, i % 3, 1.0); - } - INDArray evalProb = Nd4j.rand(10, 3); - evalProb.diviColumnVector(evalProb.sum(true, 1)); - evaluation.eval(evalLabel, evalProb); - roc3.eval(evalLabel, evalProb); - ec.eval(evalLabel, evalProb); - evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 3), 0.5)); - evalProb = Nd4j.rand(10, 3); - evaluationBinary.eval(evalLabel, evalProb); - roc2.eval(evalLabel, evalProb); - evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 1), 0.5)); - evalProb = Nd4j.rand(10, 1); - roc.eval(evalLabel, evalProb); - regressionEvaluation.eval(Nd4j.rand(10, 3), Nd4j.rand(10, 3)); - for (org.nd4j.evaluation.IEvaluation e : arr) { - String json = e.toJson(); - if (print) { - System.out.println(e.getClass() + "\n" + json + "\n\n"); - } - IEvaluation fromJson = (IEvaluation) BaseEvaluation.fromJson(json, org.nd4j.evaluation.BaseEvaluation.class); - assertEquals(e.toJson(), fromJson.toJson()); - } - } - - @Test - @DisplayName("Test Serde Exact Roc") - void testSerdeExactRoc() { - Nd4j.getRandom().setSeed(12345); - boolean print = false; - ROC roc = new ROC(0); - ROCBinary roc2 = new ROCBinary(0); - ROCMultiClass roc3 = new ROCMultiClass(0); - org.nd4j.evaluation.IEvaluation[] arr = new org.nd4j.evaluation.IEvaluation[] { roc, roc2, roc3 }; - INDArray evalLabel = Nd4j.create(100, 3); - for (int i = 0; i < 100; i++) { - evalLabel.putScalar(i, i % 3, 1.0); - } - INDArray evalProb = Nd4j.rand(100, 3); - evalProb.diviColumnVector(evalProb.sum(1)); - roc3.eval(evalLabel, evalProb); - evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(100, 3), 0.5)); - evalProb = Nd4j.rand(100, 3); - roc2.eval(evalLabel, evalProb); - evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(100, 1), 0.5)); - evalProb = Nd4j.rand(100, 1); - roc.eval(evalLabel, evalProb); - for (org.nd4j.evaluation.IEvaluation e : arr) { - System.out.println(e.getClass()); - String json = e.toJson(); - String stats = e.stats(); - if (print) { - System.out.println(json + "\n\n"); - } - org.nd4j.evaluation.IEvaluation fromJson = BaseEvaluation.fromJson(json, org.nd4j.evaluation.BaseEvaluation.class); - assertEquals(e, fromJson); - if (fromJson instanceof ROC) { - // Shouldn't have probAndLabel, but should have stored AUC and AUPRC - assertNull(((ROC) fromJson).getProbAndLabel()); - assertTrue(((ROC) fromJson).calculateAUC() > 0.0); - assertTrue(((ROC) fromJson).calculateAUCPR() > 0.0); - assertEquals(((ROC) e).getRocCurve(), ((ROC) fromJson).getRocCurve()); - assertEquals(((ROC) e).getPrecisionRecallCurve(), ((ROC) fromJson).getPrecisionRecallCurve()); - } else if (e instanceof ROCBinary) { - org.nd4j.evaluation.classification.ROC[] rocs = ((ROCBinary) fromJson).getUnderlying(); - org.nd4j.evaluation.classification.ROC[] origRocs = ((ROCBinary) e).getUnderlying(); - // for(ROC r : rocs ){ - for (int i = 0; i < origRocs.length; i++) { - org.nd4j.evaluation.classification.ROC r = rocs[i]; - org.nd4j.evaluation.classification.ROC origR = origRocs[i]; - // Shouldn't have probAndLabel, but should have stored AUC and AUPRC, AND stored curves - assertNull(r.getProbAndLabel()); - assertEquals(origR.calculateAUC(), origR.calculateAUC(), 1e-6); - assertEquals(origR.calculateAUCPR(), origR.calculateAUCPR(), 1e-6); - assertEquals(origR.getRocCurve(), origR.getRocCurve()); - assertEquals(origR.getPrecisionRecallCurve(), origR.getPrecisionRecallCurve()); - } - } else if (e instanceof ROCMultiClass) { - org.nd4j.evaluation.classification.ROC[] rocs = ((ROCMultiClass) fromJson).getUnderlying(); - org.nd4j.evaluation.classification.ROC[] origRocs = ((ROCMultiClass) e).getUnderlying(); - for (int i = 0; i < origRocs.length; i++) { - org.nd4j.evaluation.classification.ROC r = rocs[i]; - org.nd4j.evaluation.classification.ROC origR = origRocs[i]; - // Shouldn't have probAndLabel, but should have stored AUC and AUPRC, AND stored curves - assertNull(r.getProbAndLabel()); - assertEquals(origR.calculateAUC(), origR.calculateAUC(), 1e-6); - assertEquals(origR.calculateAUCPR(), origR.calculateAUCPR(), 1e-6); - assertEquals(origR.getRocCurve(), origR.getRocCurve()); - assertEquals(origR.getPrecisionRecallCurve(), origR.getPrecisionRecallCurve()); - } - } - } - } - - @Test - @DisplayName("Test Json Yaml Curves") - void testJsonYamlCurves() { - ROC roc = new ROC(0); - INDArray evalLabel = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(100, 1), 0.5)); - INDArray evalProb = Nd4j.rand(100, 1); - roc.eval(evalLabel, evalProb); - RocCurve c = roc.getRocCurve(); - PrecisionRecallCurve prc = roc.getPrecisionRecallCurve(); - String json1 = c.toJson(); - String json2 = prc.toJson(); - RocCurve c2 = RocCurve.fromJson(json1); - PrecisionRecallCurve prc2 = PrecisionRecallCurve.fromJson(json2); - assertEquals(c, c2); - assertEquals(prc, prc2); - // System.out.println(json1); - // Also test: histograms - EvaluationCalibration ec = new EvaluationCalibration(); - evalLabel = Nd4j.create(10, 3); - for (int i = 0; i < 10; i++) { - evalLabel.putScalar(i, i % 3, 1.0); - } - evalProb = Nd4j.rand(10, 3); - evalProb.diviColumnVector(evalProb.sum(1)); - ec.eval(evalLabel, evalProb); - Histogram[] histograms = new Histogram[] { ec.getResidualPlotAllClasses(), ec.getResidualPlot(0), ec.getResidualPlot(1), ec.getProbabilityHistogramAllClasses(), ec.getProbabilityHistogram(0), ec.getProbabilityHistogram(1) }; - for (Histogram h : histograms) { - String json = h.toJson(); - String yaml = h.toYaml(); - Histogram h2 = Histogram.fromJson(json); - Histogram h3 = Histogram.fromYaml(yaml); - assertEquals(h, h2); - assertEquals(h2, h3); - } - } - - @Test - @DisplayName("Test Json With Custom Threshold") - void testJsonWithCustomThreshold() { - // Evaluation - binary threshold - Evaluation e = new Evaluation(0.25); - String json = e.toJson(); - String yaml = e.toYaml(); - Evaluation eFromJson = Evaluation.fromJson(json); - Evaluation eFromYaml = Evaluation.fromYaml(yaml); - assertEquals(0.25, eFromJson.getBinaryDecisionThreshold(), 1e-6); - assertEquals(0.25, eFromYaml.getBinaryDecisionThreshold(), 1e-6); - // Evaluation: custom cost array - INDArray costArray = Nd4j.create(new double[] { 1.0, 2.0, 3.0 }); - Evaluation e2 = new Evaluation(costArray); - json = e2.toJson(); - yaml = e2.toYaml(); - eFromJson = Evaluation.fromJson(json); - eFromYaml = Evaluation.fromYaml(yaml); - assertEquals(e2.getCostArray(), eFromJson.getCostArray()); - assertEquals(e2.getCostArray(), eFromYaml.getCostArray()); - // EvaluationBinary - per-output binary threshold - INDArray threshold = Nd4j.create(new double[] { 1.0, 0.5, 0.25 }); - EvaluationBinary eb = new EvaluationBinary(threshold); - json = eb.toJson(); - yaml = eb.toYaml(); - EvaluationBinary ebFromJson = EvaluationBinary.fromJson(json); - EvaluationBinary ebFromYaml = EvaluationBinary.fromYaml(yaml); - assertEquals(threshold, ebFromJson.getDecisionThreshold()); - assertEquals(threshold, ebFromYaml.getDecisionThreshold()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvalTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvalTest.java deleted file mode 100644 index 723fbc18b..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/EvalTest.java +++ /dev/null @@ -1,465 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.eval; - -import org.datavec.api.records.metadata.RecordMetaData; -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.SequenceRecordReader; -import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.api.writable.FloatWritable; -import org.datavec.api.writable.Writable; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; -import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; -import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; -import org.deeplearning4j.datasets.iterator.IteratorMultiDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.*; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.listeners.EvaluativeListener; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.SplitTestAndTrain; -import org.nd4j.linalg.dataset.api.MultiDataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.resources.Resources; -import java.util.*; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Eval Test") -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.JACKSON_SERDE) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -class EvalTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Iris") - void testIris() { - // Network config - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(42).updater(new Sgd(1e-6)).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(2).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).build(); - // Instantiate model - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - model.addListeners(new ScoreIterationListener(1)); - // Train-test split - DataSetIterator iter = new IrisDataSetIterator(150, 150); - DataSet next = iter.next(); - next.shuffle(); - SplitTestAndTrain trainTest = next.splitTestAndTrain(5, new Random(42)); - // Train - DataSet train = trainTest.getTrain(); - train.normalizeZeroMeanZeroUnitVariance(); - // Test - DataSet test = trainTest.getTest(); - test.normalizeZeroMeanZeroUnitVariance(); - INDArray testFeature = test.getFeatures(); - INDArray testLabel = test.getLabels(); - // Fitting model - model.fit(train); - // Get predictions from test feature - INDArray testPredictedLabel = model.output(testFeature); - // Eval with class number - // // Specify class num here - org.nd4j.evaluation.classification.Evaluation eval = new org.nd4j.evaluation.classification.Evaluation(3); - eval.eval(testLabel, testPredictedLabel); - double eval1F1 = eval.f1(); - double eval1Acc = eval.accuracy(); - // Eval without class number - // // No class num - org.nd4j.evaluation.classification.Evaluation eval2 = new org.nd4j.evaluation.classification.Evaluation(); - eval2.eval(testLabel, testPredictedLabel); - double eval2F1 = eval2.f1(); - double eval2Acc = eval2.accuracy(); - // Assert the two implementations give same f1 and accuracy (since one batch) - assertTrue(eval1F1 == eval2F1 && eval1Acc == eval2Acc); - org.nd4j.evaluation.classification.Evaluation evalViaMethod = model.evaluate(new ListDataSetIterator<>(Collections.singletonList(test))); - checkEvaluationEquality(eval, evalViaMethod); - // System.out.println(eval.getConfusionMatrix().toString()); - // System.out.println(eval.getConfusionMatrix().toCSV()); - // System.out.println(eval.getConfusionMatrix().toHTML()); - // System.out.println(eval.confusionToString()); - eval.getConfusionMatrix().toString(); - eval.getConfusionMatrix().toCSV(); - eval.getConfusionMatrix().toHTML(); - eval.confusionToString(); - } - - private static void assertMapEquals(Map first, Map second) { - assertEquals(first.keySet(), second.keySet()); - for (Integer i : first.keySet()) { - assertEquals(first.get(i), second.get(i)); - } - } - - private static void checkEvaluationEquality(org.nd4j.evaluation.classification.Evaluation evalExpected, org.nd4j.evaluation.classification.Evaluation evalActual) { - assertEquals(evalExpected.accuracy(), evalActual.accuracy(), 1e-3); - assertEquals(evalExpected.f1(), evalActual.f1(), 1e-3); - assertEquals(evalExpected.getNumRowCounter(), evalActual.getNumRowCounter(), 1e-3); - assertMapEquals(evalExpected.falseNegatives(), evalActual.falseNegatives()); - assertMapEquals(evalExpected.falsePositives(), evalActual.falsePositives()); - assertMapEquals(evalExpected.trueNegatives(), evalActual.trueNegatives()); - assertMapEquals(evalExpected.truePositives(), evalActual.truePositives()); - assertEquals(evalExpected.precision(), evalActual.precision(), 1e-3); - assertEquals(evalExpected.recall(), evalActual.recall(), 1e-3); - assertEquals(evalExpected.falsePositiveRate(), evalActual.falsePositiveRate(), 1e-3); - assertEquals(evalExpected.falseNegativeRate(), evalActual.falseNegativeRate(), 1e-3); - assertEquals(evalExpected.falseAlarmRate(), evalActual.falseAlarmRate(), 1e-3); - assertEquals(evalExpected.getConfusionMatrix(), evalActual.getConfusionMatrix()); - } - - @Test - @DisplayName("Test Evaluation With Meta Data") - void testEvaluationWithMetaData() throws Exception { - RecordReader csv = new CSVRecordReader(); - csv.initialize(new FileSplit(Resources.asFile("iris.txt"))); - int batchSize = 10; - int labelIdx = 4; - int numClasses = 3; - RecordReaderDataSetIterator rrdsi = new RecordReaderDataSetIterator(csv, batchSize, labelIdx, numClasses); - NormalizerStandardize ns = new NormalizerStandardize(); - ns.fit(rrdsi); - rrdsi.setPreProcessor(ns); - rrdsi.reset(); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).list().layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(4).nOut(3).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int i = 0; i < 4; i++) { - net.fit(rrdsi); - rrdsi.reset(); - } - org.nd4j.evaluation.classification.Evaluation e = new org.nd4j.evaluation.classification.Evaluation(); - // *** New: Enable collection of metadata (stored in the DataSets) *** - rrdsi.setCollectMetaData(true); - while (rrdsi.hasNext()) { - DataSet ds = rrdsi.next(); - // *** New - cross dependencies here make types difficult, usid Object internally in DataSet for this*** - List meta = ds.getExampleMetaData(RecordMetaData.class); - INDArray out = net.output(ds.getFeatures()); - // *** New - evaluate and also store metadata *** - e.eval(ds.getLabels(), out, meta); - } - // System.out.println(e.stats()); - e.stats(); - // System.out.println("\n\n*** Prediction Errors: ***"); - // *** New - get list of prediction errors from evaluation *** - List errors = e.getPredictionErrors(); - List metaForErrors = new ArrayList<>(); - for (org.nd4j.evaluation.meta.Prediction p : errors) { - metaForErrors.add((RecordMetaData) p.getRecordMetaData()); - } - // *** New - dynamically load a subset of the data, just for prediction errors *** - DataSet ds = rrdsi.loadFromMetaData(metaForErrors); - INDArray output = net.output(ds.getFeatures()); - int count = 0; - for (org.nd4j.evaluation.meta.Prediction t : errors) { - String s = t + "\t\tRaw Data: " + // *** New - load subset of data from MetaData object (usually batched for efficiency) *** - csv.loadFromMetaData((RecordMetaData) t.getRecordMetaData()).getRecord() + "\tNormalized: " + ds.getFeatures().getRow(count) + "\tLabels: " + ds.getLabels().getRow(count) + "\tNetwork predictions: " + output.getRow(count); - // System.out.println(s); - count++; - } - int errorCount = errors.size(); - double expAcc = 1.0 - errorCount / 150.0; - assertEquals(expAcc, e.accuracy(), 1e-5); - org.nd4j.evaluation.classification.ConfusionMatrix confusion = e.getConfusionMatrix(); - int[] actualCounts = new int[3]; - int[] predictedCounts = new int[3]; - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - // (actual,predicted) - int entry = confusion.getCount(i, j); - List list = e.getPredictions(i, j); - assertEquals(entry, list.size()); - actualCounts[i] += entry; - predictedCounts[j] += entry; - } - } - for (int i = 0; i < 3; i++) { - List actualClassI = e.getPredictionsByActualClass(i); - List predictedClassI = e.getPredictionByPredictedClass(i); - assertEquals(actualCounts[i], actualClassI.size()); - assertEquals(predictedCounts[i], predictedClassI.size()); - } - // Finally: test doEvaluation methods - rrdsi.reset(); - org.nd4j.evaluation.classification.Evaluation e2 = new org.nd4j.evaluation.classification.Evaluation(); - net.doEvaluation(rrdsi, e2); - for (int i = 0; i < 3; i++) { - List actualClassI = e2.getPredictionsByActualClass(i); - List predictedClassI = e2.getPredictionByPredictedClass(i); - assertEquals(actualCounts[i], actualClassI.size()); - assertEquals(predictedCounts[i], predictedClassI.size()); - } - ComputationGraph cg = net.toComputationGraph(); - rrdsi.reset(); - e2 = new org.nd4j.evaluation.classification.Evaluation(); - cg.doEvaluation(rrdsi, e2); - for (int i = 0; i < 3; i++) { - List actualClassI = e2.getPredictionsByActualClass(i); - List predictedClassI = e2.getPredictionByPredictedClass(i); - assertEquals(actualCounts[i], actualClassI.size()); - assertEquals(predictedCounts[i], predictedClassI.size()); - } - } - - private static void apply(org.nd4j.evaluation.classification.Evaluation e, int nTimes, INDArray predicted, INDArray actual) { - for (int i = 0; i < nTimes; i++) { - e.eval(actual, predicted); - } - } - - @Test - @DisplayName("Test Eval Splitting") - void testEvalSplitting() { - // Test for "tbptt-like" functionality - for (WorkspaceMode ws : WorkspaceMode.values()) { - System.out.println("Starting test for workspace mode: " + ws); - int nIn = 4; - int layerSize = 5; - int nOut = 6; - int tbpttLength = 10; - int tsLength = 5 * tbpttLength + tbpttLength / 2; - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).trainingWorkspaceMode(ws).inferenceWorkspaceMode(ws).list().layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).build()).layer(new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut).activation(Activation.SOFTMAX).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).trainingWorkspaceMode(ws).inferenceWorkspaceMode(ws).list().layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).build()).layer(new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut).activation(Activation.SOFTMAX).build()).tBPTTLength(10).backpropType(BackpropType.TruncatedBPTT).build(); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - net2.setParams(net1.params()); - for (boolean useMask : new boolean[] { false, true }) { - INDArray in1 = Nd4j.rand(new int[] { 3, nIn, tsLength }); - INDArray out1 = TestUtils.randomOneHotTimeSeries(3, nOut, tsLength); - INDArray in2 = Nd4j.rand(new int[] { 5, nIn, tsLength }); - INDArray out2 = TestUtils.randomOneHotTimeSeries(5, nOut, tsLength); - INDArray lMask1 = null; - INDArray lMask2 = null; - if (useMask) { - lMask1 = Nd4j.create(3, tsLength); - lMask2 = Nd4j.create(5, tsLength); - Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask1, 0.5)); - Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask2, 0.5)); - } - List l = Arrays.asList(new DataSet(in1, out1, null, lMask1), new DataSet(in2, out2, null, lMask2)); - DataSetIterator iter = new ExistingDataSetIterator(l); - // System.out.println("Net 1 eval"); - org.nd4j.evaluation.IEvaluation[] e1 = net1.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation()); - // System.out.println("Net 2 eval"); - org.nd4j.evaluation.IEvaluation[] e2 = net2.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation()); - assertEquals(e1[0], e2[0]); - assertEquals(e1[1], e2[1]); - assertEquals(e1[2], e2[2]); - } - } - } - - @Test - @DisplayName("Test Eval Splitting Comp Graph") - void testEvalSplittingCompGraph() { - // Test for "tbptt-like" functionality - for (WorkspaceMode ws : WorkspaceMode.values()) { - System.out.println("Starting test for workspace mode: " + ws); - int nIn = 4; - int layerSize = 5; - int nOut = 6; - int tbpttLength = 10; - int tsLength = 5 * tbpttLength + tbpttLength / 2; - ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).trainingWorkspaceMode(ws).inferenceWorkspaceMode(ws).graphBuilder().addInputs("in").addLayer("0", new LSTM.Builder().nIn(nIn).nOut(layerSize).build(), "in").addLayer("1", new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut).activation(Activation.SOFTMAX).build(), "0").setOutputs("1").build(); - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).trainingWorkspaceMode(ws).inferenceWorkspaceMode(ws).graphBuilder().addInputs("in").addLayer("0", new LSTM.Builder().nIn(nIn).nOut(layerSize).build(), "in").addLayer("1", new RnnOutputLayer.Builder().nIn(layerSize).nOut(nOut).activation(Activation.SOFTMAX).build(), "0").setOutputs("1").tBPTTLength(10).backpropType(BackpropType.TruncatedBPTT).build(); - ComputationGraph net1 = new ComputationGraph(conf1); - net1.init(); - ComputationGraph net2 = new ComputationGraph(conf2); - net2.init(); - net2.setParams(net1.params()); - for (boolean useMask : new boolean[] { false, true }) { - INDArray in1 = Nd4j.rand(new int[] { 3, nIn, tsLength }); - INDArray out1 = TestUtils.randomOneHotTimeSeries(3, nOut, tsLength); - INDArray in2 = Nd4j.rand(new int[] { 5, nIn, tsLength }); - INDArray out2 = TestUtils.randomOneHotTimeSeries(5, nOut, tsLength); - INDArray lMask1 = null; - INDArray lMask2 = null; - if (useMask) { - lMask1 = Nd4j.create(3, tsLength); - lMask2 = Nd4j.create(5, tsLength); - Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask1, 0.5)); - Nd4j.getExecutioner().exec(new BernoulliDistribution(lMask2, 0.5)); - } - List l = Arrays.asList(new DataSet(in1, out1), new DataSet(in2, out2)); - DataSetIterator iter = new ExistingDataSetIterator(l); - // System.out.println("Eval net 1"); - org.nd4j.evaluation.IEvaluation[] e1 = net1.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation()); - // System.out.println("Eval net 2"); - org.nd4j.evaluation.IEvaluation[] e2 = net2.doEvaluation(iter, new org.nd4j.evaluation.classification.Evaluation(), new org.nd4j.evaluation.classification.ROCMultiClass(), new org.nd4j.evaluation.regression.RegressionEvaluation()); - assertEquals(e1[0], e2[0]); - assertEquals(e1[1], e2[1]); - assertEquals(e1[2], e2[2]); - } - } - } - - @Test - @DisplayName("Test Eval Splitting 2") - void testEvalSplitting2() { - List> seqFeatures = new ArrayList<>(); - List step = Arrays.asList(new FloatWritable(0), new FloatWritable(0), new FloatWritable(0)); - for (int i = 0; i < 30; i++) { - seqFeatures.add(step); - } - List> seqLabels = Collections.singletonList(Collections.singletonList(new FloatWritable(0))); - SequenceRecordReader fsr = new CollectionSequenceRecordReader(Collections.singletonList(seqFeatures)); - SequenceRecordReader lsr = new CollectionSequenceRecordReader(Collections.singletonList(seqLabels)); - DataSetIterator testData = new SequenceRecordReaderDataSetIterator(fsr, lsr, 1, -1, true, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, new LSTM.Builder().activation(Activation.TANH).nIn(3).nOut(3).build()).layer(1, new RnnOutputLayer.Builder().activation(Activation.SIGMOID).lossFunction(LossFunctions.LossFunction.XENT).nIn(3).nOut(1).build()).backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(10).tBPTTBackwardLength(10).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - net.evaluate(testData); - } - - @Test - @DisplayName("Test Evaluative Listener Simple") - void testEvaluativeListenerSimple() { - // Sanity check: https://github.com/eclipse/deeplearning4j/issues/5351 - // Network config - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(42).updater(new Sgd(1e-6)).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(2).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).build(); - // Instantiate model - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // Train-test split - DataSetIterator iter = new IrisDataSetIterator(30, 150); - DataSetIterator iterTest = new IrisDataSetIterator(30, 150); - net.setListeners(new EvaluativeListener(iterTest, 3)); - for (int i = 0; i < 3; i++) { - net.fit(iter); - } - } - - @Test - @DisplayName("Test Multi Output Eval Simple") - void testMultiOutputEvalSimple() { - Nd4j.getRandom().setSeed(12345); - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder().addInputs("in").addLayer("out1", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build(), "in").addLayer("out2", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build(), "in").setOutputs("out1", "out2").build(); - ComputationGraph cg = new ComputationGraph(conf); - cg.init(); - List list = new ArrayList<>(); - DataSetIterator iter = new IrisDataSetIterator(30, 150); - while (iter.hasNext()) { - DataSet ds = iter.next(); - list.add(new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[] { ds.getFeatures() }, new INDArray[] { ds.getLabels(), ds.getLabels() })); - } - org.nd4j.evaluation.classification.Evaluation e = new org.nd4j.evaluation.classification.Evaluation(); - org.nd4j.evaluation.regression.RegressionEvaluation e2 = new org.nd4j.evaluation.regression.RegressionEvaluation(); - Map evals = new HashMap<>(); - evals.put(0, new org.nd4j.evaluation.IEvaluation[] { e }); - evals.put(1, new org.nd4j.evaluation.IEvaluation[] { e2 }); - cg.evaluate(new IteratorMultiDataSetIterator(list.iterator(), 30), evals); - assertEquals(150, e.getNumRowCounter()); - assertEquals(150, e2.getExampleCountPerColumn().getInt(0)); - } - - @Test - @DisplayName("Test Multi Output Eval CG") - void testMultiOutputEvalCG() { - // Simple sanity check on evaluation - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").layer("0", new EmbeddingSequenceLayer.Builder().nIn(10).nOut(10).build(), "in").layer("1", new LSTM.Builder().nIn(10).nOut(10).build(), "0").layer("2", new LSTM.Builder().nIn(10).nOut(10).build(), "0").layer("out1", new RnnOutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build(), "1").layer("out2", new RnnOutputLayer.Builder().nIn(10).nOut(20).activation(Activation.SOFTMAX).build(), "2").setOutputs("out1", "out2").build(); - ComputationGraph cg = new ComputationGraph(conf); - cg.init(); - org.nd4j.linalg.dataset.MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[] { Nd4j.create(10, 1, 10) }, new INDArray[] { Nd4j.create(10, 10, 10), Nd4j.create(10, 20, 10) }); - Map m = new HashMap<>(); - m.put(0, new org.nd4j.evaluation.IEvaluation[] { new org.nd4j.evaluation.classification.Evaluation() }); - m.put(1, new org.nd4j.evaluation.IEvaluation[] { new org.nd4j.evaluation.classification.Evaluation() }); - cg.evaluate(new SingletonMultiDataSetIterator(mds), m); - } - - @Test - @DisplayName("Test Invalid Evaluation") - void testInvalidEvaluation() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new DenseLayer.Builder().nIn(4).nOut(10).build()).layer(new OutputLayer.Builder().nIn(10).nOut(3).lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.RELU).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSetIterator iter = new IrisDataSetIterator(150, 150); - try { - net.evaluate(iter); - fail("Expected exception"); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("Evaluation")); - } - try { - net.evaluateROC(iter, 0); - fail("Expected exception"); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROC")); - } - try { - net.evaluateROCMultiClass(iter, 0); - fail("Expected exception"); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROCMultiClass")); - } - ComputationGraph cg = net.toComputationGraph(); - try { - cg.evaluate(iter); - fail("Expected exception"); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("Evaluation")); - } - try { - cg.evaluateROC(iter, 0); - fail("Expected exception"); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROC")); - } - try { - cg.evaluateROCMultiClass(iter, 0); - fail("Expected exception"); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("Classifier") && e.getMessage().contains("ROCMultiClass")); - } - // Disable validation, and check same thing: - net.getLayerWiseConfigurations().setValidateOutputLayerConfig(false); - net.evaluate(iter); - net.evaluateROCMultiClass(iter, 0); - cg.getConfiguration().setValidateOutputLayerConfig(false); - cg.evaluate(iter); - cg.evaluateROCMultiClass(iter, 0); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/ROCTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/ROCTest.java deleted file mode 100644 index 5fa6da135..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/ROCTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.eval; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.evaluation.curves.RocCurve; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.api.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@DisplayName("Roc Test") -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.JACKSON_SERDE) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -class ROCTest extends BaseDL4JTest { - - private static Map expTPR; - - private static Map expFPR; - - static { - expTPR = new HashMap<>(); - double totalPositives = 5.0; - // All 10 predicted as class 1, of which 5 of 5 are correct - expTPR.put(0 / 10.0, 5.0 / totalPositives); - expTPR.put(1 / 10.0, 5.0 / totalPositives); - expTPR.put(2 / 10.0, 5.0 / totalPositives); - expTPR.put(3 / 10.0, 5.0 / totalPositives); - expTPR.put(4 / 10.0, 5.0 / totalPositives); - expTPR.put(5 / 10.0, 5.0 / totalPositives); - // Threshold: 0.4 -> last 4 predicted; last 5 actual - expTPR.put(6 / 10.0, 4.0 / totalPositives); - expTPR.put(7 / 10.0, 3.0 / totalPositives); - expTPR.put(8 / 10.0, 2.0 / totalPositives); - expTPR.put(9 / 10.0, 1.0 / totalPositives); - expTPR.put(10 / 10.0, 0.0 / totalPositives); - expFPR = new HashMap<>(); - double totalNegatives = 5.0; - // All 10 predicted as class 1, but all 5 true negatives are predicted positive - expFPR.put(0 / 10.0, 5.0 / totalNegatives); - // 1 true negative is predicted as negative; 4 false positives - expFPR.put(1 / 10.0, 4.0 / totalNegatives); - // 2 true negatives are predicted as negative; 3 false positives - expFPR.put(2 / 10.0, 3.0 / totalNegatives); - expFPR.put(3 / 10.0, 2.0 / totalNegatives); - expFPR.put(4 / 10.0, 1.0 / totalNegatives); - expFPR.put(5 / 10.0, 0.0 / totalNegatives); - expFPR.put(6 / 10.0, 0.0 / totalNegatives); - expFPR.put(7 / 10.0, 0.0 / totalNegatives); - expFPR.put(8 / 10.0, 0.0 / totalNegatives); - expFPR.put(9 / 10.0, 0.0 / totalNegatives); - expFPR.put(10 / 10.0, 0.0 / totalNegatives); - } - - @Test - @DisplayName("Roc Eval Sanity Check") - void RocEvalSanityCheck() { - DataSetIterator iter = new IrisDataSetIterator(150, 150); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).seed(12345).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()).layer(1, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - NormalizerStandardize ns = new NormalizerStandardize(); - DataSet ds = iter.next(); - ns.fit(ds); - ns.transform(ds); - iter.setPreProcessor(ns); - for (int i = 0; i < 10; i++) { - net.fit(ds); - } - for (int steps : new int[] { 32, 0 }) { - // Steps = 0: exact - System.out.println("steps: " + steps); - iter.reset(); - ds = iter.next(); - INDArray f = ds.getFeatures(); - INDArray l = ds.getLabels(); - INDArray out = net.output(f); - // System.out.println(f); - // System.out.println(out); - ROCMultiClass manual = new ROCMultiClass(steps); - manual.eval(l, out); - iter.reset(); - ROCMultiClass roc = net.evaluateROCMultiClass(iter, steps); - for (int i = 0; i < 3; i++) { - double rocExp = manual.calculateAUC(i); - double rocAct = roc.calculateAUC(i); - assertEquals(rocExp, rocAct, 1e-6); - RocCurve rc = roc.getRocCurve(i); - RocCurve rm = manual.getRocCurve(i); - assertEquals(rc, rm); - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/RegressionEvalTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/RegressionEvalTest.java deleted file mode 100644 index c858ad0b0..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/eval/RegressionEvalTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.eval; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.Collections; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.nd4j.linalg.indexing.NDArrayIndex.all; -import static org.nd4j.linalg.indexing.NDArrayIndex.interval; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Regression Eval Test") -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.JACKSON_SERDE) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -class RegressionEvalTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Regression Eval Methods") - void testRegressionEvalMethods() { - // Basic sanity check - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.ZERO).list().layer(0, new OutputLayer.Builder().activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(5).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray f = Nd4j.zeros(4, 10); - INDArray l = Nd4j.ones(4, 5); - DataSet ds = new DataSet(f, l); - DataSetIterator iter = new ExistingDataSetIterator(Collections.singletonList(ds)); - org.nd4j.evaluation.regression.RegressionEvaluation re = net.evaluateRegression(iter); - for (int i = 0; i < 5; i++) { - assertEquals(1.0, re.meanSquaredError(i), 1e-6); - assertEquals(1.0, re.meanAbsoluteError(i), 1e-6); - } - ComputationGraphConfiguration graphConf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.ZERO).graphBuilder().addInputs("in").addLayer("0", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(10).nOut(5).build(), "in").setOutputs("0").build(); - ComputationGraph cg = new ComputationGraph(graphConf); - cg.init(); - RegressionEvaluation re2 = cg.evaluateRegression(iter); - for (int i = 0; i < 5; i++) { - assertEquals(1.0, re2.meanSquaredError(i), 1e-6); - assertEquals(1.0, re2.meanAbsoluteError(i), 1e-6); - } - } - - @Test - @DisplayName("Test Regression Eval Per Output Masking") - void testRegressionEvalPerOutputMasking() { - INDArray l = Nd4j.create(new double[][] { { 1, 2, 3 }, { 10, 20, 30 }, { -5, -10, -20 } }); - INDArray predictions = Nd4j.zeros(l.shape()); - INDArray mask = Nd4j.create(new double[][] { { 0, 1, 1 }, { 1, 1, 0 }, { 0, 1, 0 } }); - RegressionEvaluation re = new RegressionEvaluation(); - re.eval(l, predictions, mask); - double[] mse = new double[] { (10 * 10) / 1.0, (2 * 2 + 20 * 20 + 10 * 10) / 3, (3 * 3) / 1.0 }; - double[] mae = new double[] { 10.0, (2 + 20 + 10) / 3.0, 3.0 }; - double[] rmse = new double[] { 10.0, Math.sqrt((2 * 2 + 20 * 20 + 10 * 10) / 3.0), 3.0 }; - for (int i = 0; i < 3; i++) { - assertEquals(mse[i], re.meanSquaredError(i), 1e-6); - assertEquals(mae[i], re.meanAbsoluteError(i), 1e-6); - assertEquals(rmse[i], re.rootMeanSquaredError(i), 1e-6); - } - } - - @Test - @DisplayName("Test Regression Eval Time Series Split") - void testRegressionEvalTimeSeriesSplit() { - INDArray out1 = Nd4j.rand(new int[] { 3, 5, 20 }); - INDArray outSub1 = out1.get(all(), all(), interval(0, 10)); - INDArray outSub2 = out1.get(all(), all(), interval(10, 20)); - INDArray label1 = Nd4j.rand(new int[] { 3, 5, 20 }); - INDArray labelSub1 = label1.get(all(), all(), interval(0, 10)); - INDArray labelSub2 = label1.get(all(), all(), interval(10, 20)); - RegressionEvaluation e1 = new RegressionEvaluation(); - RegressionEvaluation e2 = new RegressionEvaluation(); - e1.eval(label1, out1); - e2.eval(labelSub1, outSub1); - e2.eval(labelSub2, outSub2); - assertEquals(e1, e2); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestRecordReaders.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestRecordReaders.java deleted file mode 100644 index b9521d7d5..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/exceptions/TestRecordReaders.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.exceptions; - -import org.datavec.api.records.reader.impl.collection.CollectionRecordReader; -import org.datavec.api.records.reader.impl.collection.CollectionSequenceRecordReader; -import org.datavec.api.writable.DoubleWritable; -import org.datavec.api.writable.IntWritable; -import org.datavec.api.writable.Writable; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; -import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; -import org.deeplearning4j.exception.DL4JException; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.dataset.api.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; - -import static junit.framework.TestCase.fail; -import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -public class TestRecordReaders extends BaseDL4JTest { - - @Test - public void testClassIndexOutsideOfRangeRRDSI() { - Collection> c = new ArrayList<>(); - c.add(Arrays.asList(new DoubleWritable(0.5), new IntWritable(0))); - c.add(Arrays.asList(new DoubleWritable(1.0), new IntWritable(2))); - - CollectionRecordReader crr = new CollectionRecordReader(c); - - RecordReaderDataSetIterator iter = new RecordReaderDataSetIterator(crr, 2, 1, 2); - - try { - DataSet ds = iter.next(); - fail("Expected exception"); - } catch (Exception e) { - assertTrue( e.getMessage().contains("to one-hot"),e.getMessage()); - } - } - - @Test - public void testClassIndexOutsideOfRangeRRMDSI() { - - Collection>> c = new ArrayList<>(); - Collection> seq1 = new ArrayList<>(); - seq1.add(Arrays.asList(new DoubleWritable(0.0), new IntWritable(0))); - seq1.add(Arrays.asList(new DoubleWritable(0.0), new IntWritable(1))); - c.add(seq1); - - Collection> seq2 = new ArrayList<>(); - seq2.add(Arrays.asList(new DoubleWritable(0.0), new IntWritable(0))); - seq2.add(Arrays.asList(new DoubleWritable(0.0), new IntWritable(2))); - c.add(seq2); - - CollectionSequenceRecordReader csrr = new CollectionSequenceRecordReader(c); - DataSetIterator dsi = new SequenceRecordReaderDataSetIterator(csrr, 2, 2, 1); - - try { - DataSet ds = dsi.next(); - fail("Expected exception"); - } catch (Exception e) { - assertTrue(e.getMessage().contains("to one-hot"),e.getMessage()); - } - } - - @Test - public void testClassIndexOutsideOfRangeRRMDSI_MultipleReaders() { - - Collection>> c1 = new ArrayList<>(); - Collection> seq1 = new ArrayList<>(); - seq1.add(Arrays.asList(new DoubleWritable(0.0))); - seq1.add(Arrays.asList(new DoubleWritable(0.0))); - c1.add(seq1); - - Collection> seq2 = new ArrayList<>(); - seq2.add(Arrays.asList(new DoubleWritable(0.0))); - seq2.add(Arrays.asList(new DoubleWritable(0.0))); - c1.add(seq2); - - Collection>> c2 = new ArrayList<>(); - Collection> seq1a = new ArrayList<>(); - seq1a.add(Arrays.asList(new IntWritable(0))); - seq1a.add(Arrays.asList(new IntWritable(1))); - c2.add(seq1a); - - Collection> seq2a = new ArrayList<>(); - seq2a.add(Arrays.asList(new IntWritable(0))); - seq2a.add(Arrays.asList(new IntWritable(2))); - c2.add(seq2a); - - CollectionSequenceRecordReader csrr = new CollectionSequenceRecordReader(c1); - CollectionSequenceRecordReader csrrLabels = new CollectionSequenceRecordReader(c2); - DataSetIterator dsi = new SequenceRecordReaderDataSetIterator(csrr, csrrLabels, 2, 2); - - try { - DataSet ds = dsi.next(); - fail("Expected exception"); - } catch (Exception e) { - assertTrue(e.getMessage().contains("to one-hot"),e.getMessage()); - } - } - -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java deleted file mode 100644 index f6559040b..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/AttentionLayerTest.java +++ /dev/null @@ -1,319 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.gradientcheck; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.graph.AttentionVertex; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Disabled; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.Random; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; - -@Disabled -@DisplayName("Attention Layer Test") -@NativeTag -@Tag(TagNames.EVAL_METRICS) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -class AttentionLayerTest extends BaseDL4JTest { - - - - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - - @Test - @DisplayName("Test Self Attention Layer") - void testSelfAttentionLayer() { - int nIn = 3; - int nOut = 2; - int tsLength = 4; - int layerSize = 4; - for (int mb : new int[] { 1, 3 }) { - for (boolean inputMask : new boolean[] { false, true }) { - for (boolean projectInput : new boolean[] { false, true }) { - INDArray in = Nd4j.rand(DataType.DOUBLE, new int[] { mb, nIn, tsLength }); - INDArray labels = TestUtils.randomOneHot(mb, nOut); - String maskType = (inputMask ? "inputMask" : "none"); - INDArray inMask = null; - if (inputMask) { - inMask = Nd4j.ones(mb, tsLength); - for (int i = 0; i < mb; i++) { - int firstMaskedStep = tsLength - 1 - i; - if (firstMaskedStep == 0) { - firstMaskedStep = tsLength; - } - for (int j = firstMaskedStep; j < tsLength; j++) { - inMask.putScalar(i, j, 0.0); - } - } - } - String name = "testSelfAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; - System.out.println("Starting test: " + name); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).updater(new NoOp()).weightInit(WeightInit.XAVIER).list().layer(new LSTM.Builder().nOut(layerSize).build()).layer(projectInput ? new SelfAttentionLayer.Builder().nOut(4).nHeads(2).projectInput(true).build() : new SelfAttentionLayer.Builder().nHeads(1).projectInput(false).build()).layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build()).layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).setInputType(InputType.recurrent(nIn)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in).labels(labels).inputMask(inMask).subset(true).maxPerParam(100)); - assertTrue(gradOK,name); - } - } - } - } - - @Test - @DisplayName("Test Learned Self Attention Layer") - void testLearnedSelfAttentionLayer() { - int nIn = 3; - int nOut = 2; - int tsLength = 4; - int layerSize = 4; - int numQueries = 3; - for (boolean inputMask : new boolean[] { false, true }) { - for (int mb : new int[] { 3, 1 }) { - for (boolean projectInput : new boolean[] { false, true }) { - INDArray in = Nd4j.rand(DataType.DOUBLE, new int[] { mb, nIn, tsLength }); - INDArray labels = TestUtils.randomOneHot(mb, nOut); - String maskType = (inputMask ? "inputMask" : "none"); - INDArray inMask = null; - if (inputMask) { - inMask = Nd4j.ones(mb, tsLength); - for (int i = 0; i < mb; i++) { - int firstMaskedStep = tsLength - 1 - i; - if (firstMaskedStep == 0) { - firstMaskedStep = tsLength; - } - for (int j = firstMaskedStep; j < tsLength; j++) { - inMask.putScalar(i, j, 0.0); - } - } - } - String name = "testLearnedSelfAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; - System.out.println("Starting test: " + name); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).updater(new NoOp()).weightInit(WeightInit.XAVIER).list().layer(new LSTM.Builder().nOut(layerSize).build()).layer(projectInput ? new LearnedSelfAttentionLayer.Builder().nOut(4).nHeads(2).nQueries(numQueries).projectInput(true).build() : new LearnedSelfAttentionLayer.Builder().nHeads(1).nQueries(numQueries).projectInput(false).build()).layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build()).layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).setInputType(InputType.recurrent(nIn)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in).labels(labels).inputMask(inMask).subset(true).maxPerParam(100)); - assertTrue(gradOK,name); - } - } - } - } - - @Test - @DisplayName("Test Learned Self Attention Layer _ different Mini Batch Sizes") - void testLearnedSelfAttentionLayer_differentMiniBatchSizes() { - int nIn = 3; - int nOut = 2; - int tsLength = 4; - int layerSize = 4; - int numQueries = 3; - Random r = new Random(12345); - for (boolean inputMask : new boolean[] { false, true }) { - for (boolean projectInput : new boolean[] { false, true }) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).updater(new NoOp()).weightInit(WeightInit.XAVIER).list().layer(new LSTM.Builder().nOut(layerSize).build()).layer(projectInput ? new LearnedSelfAttentionLayer.Builder().nOut(4).nHeads(2).nQueries(numQueries).projectInput(true).build() : new LearnedSelfAttentionLayer.Builder().nHeads(1).nQueries(numQueries).projectInput(false).build()).layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build()).layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).setInputType(InputType.recurrent(nIn)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int mb : new int[] { 3, 1 }) { - INDArray in = Nd4j.rand(DataType.DOUBLE, new int[] { mb, nIn, tsLength }); - INDArray labels = TestUtils.randomOneHot(mb, nOut); - String maskType = (inputMask ? "inputMask" : "none"); - INDArray inMask = null; - if (inputMask) { - inMask = Nd4j.ones(DataType.INT, mb, tsLength); - for (int i = 0; i < mb; i++) { - int firstMaskedStep = tsLength - 1 - i; - if (firstMaskedStep == 0) { - firstMaskedStep = tsLength; - } - for (int j = firstMaskedStep; j < tsLength; j++) { - inMask.putScalar(i, j, 0.0); - } - } - } - String name = "testLearnedSelfAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; - System.out.println("Starting test: " + name); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in).labels(labels).inputMask(inMask).subset(true).maxPerParam(100)); - assertTrue(gradOK,name); - } - } - } - } - - @Test - @DisplayName("Test Recurrent Attention Layer _ differing Time Steps") - void testRecurrentAttentionLayer_differingTimeSteps() { - assertThrows(IllegalArgumentException.class, () -> { - int nIn = 9; - int nOut = 5; - int layerSize = 8; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.IDENTITY).updater(new NoOp()).weightInit(WeightInit.XAVIER).list().layer(new LSTM.Builder().nOut(layerSize).build()).layer(new RecurrentAttentionLayer.Builder().nIn(layerSize).nOut(layerSize).nHeads(1).projectInput(false).hasBias(false).build()).layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build()).layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).setInputType(InputType.recurrent(nIn)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - final INDArray initialInput = Nd4j.rand(new int[] { 8, nIn, 7 }); - final INDArray goodNextInput = Nd4j.rand(new int[] { 8, nIn, 7 }); - final INDArray badNextInput = Nd4j.rand(new int[] { 8, nIn, 12 }); - final INDArray labels = Nd4j.rand(new int[] { 8, nOut }); - net.fit(initialInput, labels); - net.fit(goodNextInput, labels); - net.fit(badNextInput, labels); - }); - - } - - @Test - @DisplayName("Test Recurrent Attention Layer") - void testRecurrentAttentionLayer() { - int nIn = 4; - int nOut = 2; - int tsLength = 3; - int layerSize = 3; - for (int mb : new int[] { 3, 1 }) { - for (boolean inputMask : new boolean[] { true, false }) { - INDArray in = Nd4j.rand(DataType.DOUBLE, new int[] { mb, nIn, tsLength }); - INDArray labels = TestUtils.randomOneHot(mb, nOut); - String maskType = (inputMask ? "inputMask" : "none"); - INDArray inMask = null; - if (inputMask) { - inMask = Nd4j.ones(mb, tsLength); - for (int i = 0; i < mb; i++) { - int firstMaskedStep = tsLength - 1 - i; - if (firstMaskedStep == 0) { - firstMaskedStep = tsLength; - } - for (int j = firstMaskedStep; j < tsLength; j++) { - inMask.putScalar(i, j, 0.0); - } - } - } - String name = "testRecurrentAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType; - System.out.println("Starting test: " + name); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.IDENTITY).updater(new NoOp()).weightInit(WeightInit.XAVIER).list().layer(new LSTM.Builder().nOut(layerSize).build()).layer(new RecurrentAttentionLayer.Builder().nIn(layerSize).nOut(layerSize).nHeads(1).projectInput(false).hasBias(false).build()).layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build()).layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).setInputType(InputType.recurrent(nIn)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // System.out.println("Original"); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in).labels(labels).inputMask(inMask).subset(true).maxPerParam(100)); - assertTrue(gradOK,name); - } - } - } - - @Test - @DisplayName("Test Attention Vertex") - void testAttentionVertex() { - int nIn = 3; - int nOut = 2; - int tsLength = 3; - int layerSize = 3; - Random r = new Random(12345); - for (boolean inputMask : new boolean[] { false, true }) { - for (int mb : new int[] { 3, 1 }) { - for (boolean projectInput : new boolean[] { false, true }) { - INDArray in = Nd4j.rand(DataType.DOUBLE, new int[] { mb, nIn, tsLength }); - INDArray labels = TestUtils.randomOneHot(mb, nOut); - String maskType = (inputMask ? "inputMask" : "none"); - INDArray inMask = null; - if (inputMask) { - inMask = Nd4j.ones(mb, tsLength); - for (int i = 0; i < mb; i++) { - int firstMaskedStep = tsLength - 1 - i; - if (firstMaskedStep == 0) { - firstMaskedStep = tsLength; - } - for (int j = firstMaskedStep; j < tsLength; j++) { - inMask.putScalar(i, j, 0.0); - } - } - } - String name = "testAttentionVertex() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; - System.out.println("Starting test: " + name); - ComputationGraphConfiguration graph = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).updater(new NoOp()).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("input").addLayer("rnnKeys", new SimpleRnn.Builder().nOut(layerSize).build(), "input").addLayer("rnnQueries", new SimpleRnn.Builder().nOut(layerSize).build(), "input").addLayer("rnnValues", new SimpleRnn.Builder().nOut(layerSize).build(), "input").addVertex("attention", projectInput ? new AttentionVertex.Builder().nOut(4).nHeads(2).projectInput(true).nInQueries(layerSize).nInKeys(layerSize).nInValues(layerSize).build() : new AttentionVertex.Builder().nOut(3).nHeads(1).projectInput(false).nInQueries(layerSize).nInKeys(layerSize).nInValues(layerSize).build(), "rnnQueries", "rnnKeys", "rnnValues").addLayer("pooling", new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build(), "attention").addLayer("output", new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "pooling").setOutputs("output").setInputTypes(InputType.recurrent(nIn)).build(); - ComputationGraph net = new ComputationGraph(graph); - net.init(); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[] { in }).labels(new INDArray[] { labels }).inputMask(inMask != null ? new INDArray[] { inMask } : null).subset(true).maxPerParam(100)); - assertTrue(gradOK,name); - } - } - } - } - - @Test - @DisplayName("Test Attention Vertex Same Input") - void testAttentionVertexSameInput() { - int nIn = 3; - int nOut = 2; - int tsLength = 4; - int layerSize = 4; - Random r = new Random(12345); - for (boolean inputMask : new boolean[] { false, true }) { - for (int mb : new int[] { 3, 1 }) { - for (boolean projectInput : new boolean[] { false, true }) { - INDArray in = Nd4j.rand(new int[] { mb, nIn, tsLength }); - INDArray labels = TestUtils.randomOneHot(mb, nOut); - String maskType = (inputMask ? "inputMask" : "none"); - INDArray inMask = null; - if (inputMask) { - inMask = Nd4j.ones(mb, tsLength); - for (int i = 0; i < mb; i++) { - int firstMaskedStep = tsLength - 1 - i; - if (firstMaskedStep == 0) { - firstMaskedStep = tsLength; - } - for (int j = firstMaskedStep; j < tsLength; j++) { - inMask.putScalar(i, j, 0.0); - } - } - } - String name = "testAttentionVertex() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput; - System.out.println("Starting test: " + name); - ComputationGraphConfiguration graph = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).updater(new NoOp()).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("input").addLayer("rnn", new SimpleRnn.Builder().activation(Activation.TANH).nOut(layerSize).build(), "input").addVertex("attention", projectInput ? new AttentionVertex.Builder().nOut(4).nHeads(2).projectInput(true).nInQueries(layerSize).nInKeys(layerSize).nInValues(layerSize).build() : new AttentionVertex.Builder().nOut(4).nHeads(1).projectInput(false).nInQueries(layerSize).nInKeys(layerSize).nInValues(layerSize).build(), "rnn", "rnn", "rnn").addLayer("pooling", new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build(), "attention").addLayer("output", new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "pooling").setOutputs("output").setInputTypes(InputType.recurrent(nIn)).build(); - ComputationGraph net = new ComputationGraph(graph); - net.init(); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[] { in }).labels(new INDArray[] { labels }).inputMask(inMask != null ? new INDArray[] { inMask } : null)); - assertTrue(gradOK,name); - } - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java deleted file mode 100644 index 1eba038b3..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/BNGradientCheckTest.java +++ /dev/null @@ -1,433 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.gradientcheck; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.distribution.UniformDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.executioner.OpExecutioner; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization; -import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.linalg.profiler.OpProfiler; -import org.nd4j.linalg.profiler.ProfilerConfig; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Random; -import java.util.Set; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - */ -@DisplayName("Bn Gradient Check Test") -@NativeTag -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -class BNGradientCheckTest extends BaseDL4JTest { - - static { - Nd4j.setDataType(DataType.DOUBLE); - } - - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - - @Test - @DisplayName("Test Gradient 2 d Simple") - void testGradient2dSimple() { - DataNormalization scaler = new NormalizerMinMaxScaler(); - DataSetIterator iter = new IrisDataSetIterator(150, 150); - scaler.fit(iter); - iter.setPreProcessor(scaler); - DataSet ds = iter.next(); - INDArray input = ds.getFeatures(); - INDArray labels = ds.getLabels(); - for (boolean useLogStd : new boolean[] { true, false }) { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).seed(12345L).dist(new NormalDistribution(0, 1)).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY).build()).layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).nOut(3).build()).layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()); - MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); - mln.init(); - // for (int j = 0; j < mln.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - // Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - // i.e., runningMean = decay * runningMean + (1-decay) * batchMean - // However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input).labels(labels).excludeParams(excludeParams)); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - - @Test - @DisplayName("Test Gradient Cnn Simple") - void testGradientCnnSimple() { - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int depth = 1; - int hw = 4; - int nOut = 4; - INDArray input = Nd4j.rand(new int[] { minibatch, depth, hw, hw }); - INDArray labels = Nd4j.zeros(minibatch, nOut); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - labels.putScalar(i, r.nextInt(nOut), 1.0); - } - for (boolean useLogStd : new boolean[] { true, false }) { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).seed(12345L).dist(new NormalDistribution(0, 2)).list().layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nIn(depth).nOut(2).activation(Activation.IDENTITY).build()).layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).build()).layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(hw, hw, depth)); - MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); - mln.init(); - // for (int j = 0; j < mln.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - // Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - // i.e., runningMean = decay * runningMean + (1-decay) * batchMean - // However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input).labels(labels).excludeParams(excludeParams)); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - - @Test - @DisplayName("Test Gradient BN With CN Nand Subsampling") - void testGradientBNWithCNNandSubsampling() { - // Parameterized test, testing combinations of: - // (a) activation function - // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') - // (c) Loss function (with specified output activations) - // (d) l1 and l2 values - Activation[] activFns = { Activation.SIGMOID, Activation.TANH, Activation.IDENTITY }; - // If true: run some backprop steps first - boolean[] characteristic = { true }; - LossFunctions.LossFunction[] lossFunctions = { LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE }; - // i.e., lossFunctions[i] used with outputActivations[i] here - Activation[] outputActivations = { Activation.SOFTMAX, Activation.TANH }; - double[] l2vals = { 0.0, 0.1, 0.1 }; - // i.e., use l2vals[j] with l1vals[j] - double[] l1vals = { 0.0, 0.0, 0.2 }; - Nd4j.getRandom().setSeed(12345); - int minibatch = 4; - int depth = 2; - int hw = 5; - int nOut = 2; - INDArray input = Nd4j.rand(new int[] { minibatch, depth, hw, hw }).muli(5).subi(2.5); - INDArray labels = TestUtils.randomOneHot(minibatch, nOut); - DataSet ds = new DataSet(input, labels); - Random rng = new Random(12345); - for (boolean useLogStd : new boolean[] { true, false }) { - for (Activation afn : activFns) { - for (boolean doLearningFirst : characteristic) { - for (int i = 0; i < lossFunctions.length; i++) { - for (int j = 0; j < l2vals.length; j++) { - // Skip 2 of every 3 tests: from 24 cases to 8, still with decent coverage - if (rng.nextInt(3) != 0) - continue; - LossFunctions.LossFunction lf = lossFunctions[i]; - Activation outputActivation = outputActivations[i]; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).l2(l2vals[j]).optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).updater(new NoOp()).dist(new UniformDistribution(-2, 2)).seed(12345L).list().layer(0, new ConvolutionLayer.Builder(2, 2).stride(1, 1).nOut(3).activation(afn).build()).layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).build()).layer(2, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(1, 1).build()).layer(3, new BatchNormalization()).layer(4, new ActivationLayer.Builder().activation(afn).build()).layer(5, new OutputLayer.Builder(lf).activation(outputActivation).nOut(nOut).build()).setInputType(InputType.convolutional(hw, hw, depth)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - String name = new Object() { - }.getClass().getEnclosingMethod().getName(); - // System.out.println("Num params: " + mln.numParams()); - if (doLearningFirst) { - // Run a number of iterations of learning - mln.setInput(ds.getFeatures()); - mln.setLabels(ds.getLabels()); - mln.computeGradientAndScore(); - double scoreBefore = mln.score(); - for (int k = 0; k < 20; k++) mln.fit(ds); - mln.computeGradientAndScore(); - double scoreAfter = mln.score(); - // Can't test in 'characteristic mode of operation' if not learning - String msg = name + " - score did not (sufficiently) decrease during learning - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")"; - assertTrue(scoreAfter < 0.9 * scoreBefore,msg); - } - System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l1=" + l1vals[j] + ", l2=" + l2vals[j]); - // for (int k = 0; k < mln.getnLayers(); k++) - // System.out.println("Layer " + k + " # params: " + mln.getLayer(k).numParams()); - // Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - // i.e., runningMean = decay * runningMean + (1-decay) * batchMean - // However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "3_mean", "3_var", "1_log10stdev", "3_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input).labels(labels).excludeParams(excludeParams).subset(true).maxPerParam(// Most params are in output layer, only these should be skipped with this threshold - 25)); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - } - } - } - } - - @Test - @DisplayName("Test Gradient Dense") - void testGradientDense() { - // Parameterized test, testing combinations of: - // (a) activation function - // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') - // (c) Loss function (with specified output activations) - // (d) l1 and l2 values - Activation[] activFns = { Activation.TANH, Activation.IDENTITY }; - // If true: run some backprop steps first - boolean[] characteristic = { true }; - LossFunctions.LossFunction[] lossFunctions = { LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE }; - // i.e., lossFunctions[i] used with outputActivations[i] here - Activation[] outputActivations = { Activation.SOFTMAX, Activation.TANH }; - double[] l2vals = { 0.0, 0.1 }; - // i.e., use l2vals[j] with l1vals[j] - double[] l1vals = { 0.0, 0.2 }; - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int nIn = 5; - int nOut = 3; - INDArray input = Nd4j.rand(new int[] { minibatch, nIn }); - INDArray labels = Nd4j.zeros(minibatch, nOut); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - labels.putScalar(i, r.nextInt(nOut), 1.0); - } - DataSet ds = new DataSet(input, labels); - for (boolean useLogStd : new boolean[] { true, false }) { - for (Activation afn : activFns) { - for (boolean doLearningFirst : characteristic) { - for (int i = 0; i < lossFunctions.length; i++) { - for (int j = 0; j < l2vals.length; j++) { - LossFunctions.LossFunction lf = lossFunctions[i]; - Activation outputActivation = outputActivations[i]; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).l2(l2vals[j]).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp()).dist(new UniformDistribution(-2, 2)).seed(12345L).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(4).activation(afn).build()).layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).build()).layer(2, new DenseLayer.Builder().nIn(4).nOut(4).build()).layer(3, new BatchNormalization.Builder().useLogStd(useLogStd).build()).layer(4, new OutputLayer.Builder(lf).activation(outputActivation).nOut(nOut).build()); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - String name = new Object() { - }.getClass().getEnclosingMethod().getName(); - if (doLearningFirst) { - // Run a number of iterations of learning - mln.setInput(ds.getFeatures()); - mln.setLabels(ds.getLabels()); - mln.computeGradientAndScore(); - double scoreBefore = mln.score(); - for (int k = 0; k < 10; k++) mln.fit(ds); - mln.computeGradientAndScore(); - double scoreAfter = mln.score(); - // Can't test in 'characteristic mode of operation' if not learning - String msg = name + " - score did not (sufficiently) decrease during learning - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")"; - assertTrue(scoreAfter < 0.8 * scoreBefore,msg); - } - System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l1=" + l1vals[j] + ", l2=" + l2vals[j]); - // for (int k = 0; k < mln.getnLayers(); k++) - // System.out.println("Layer " + k + " # params: " + mln.getLayer(k).numParams()); - // Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - // i.e., runningMean = decay * runningMean + (1-decay) * batchMean - // However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "3_mean", "3_var", "1_log10stdev", "3_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input).labels(labels).excludeParams(excludeParams)); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - } - } - } - } - - @Test - @DisplayName("Test Gradient 2 d Fixed Gamma Beta") - void testGradient2dFixedGammaBeta() { - DataNormalization scaler = new NormalizerMinMaxScaler(); - DataSetIterator iter = new IrisDataSetIterator(150, 150); - scaler.fit(iter); - iter.setPreProcessor(scaler); - DataSet ds = iter.next(); - INDArray input = ds.getFeatures(); - INDArray labels = ds.getLabels(); - for (boolean useLogStd : new boolean[] { true, false }) { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).seed(12345L).dist(new NormalDistribution(0, 1)).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY).build()).layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).lockGammaBeta(true).gamma(2.0).beta(0.5).nOut(3).build()).layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()); - MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); - mln.init(); - // for (int j = 0; j < mln.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - // Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - // i.e., runningMean = decay * runningMean + (1-decay) * batchMean - // However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input).labels(labels).excludeParams(excludeParams)); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - - @Test - @DisplayName("Test Gradient Cnn Fixed Gamma Beta") - void testGradientCnnFixedGammaBeta() { - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int depth = 1; - int hw = 4; - int nOut = 4; - INDArray input = Nd4j.rand(new int[] { minibatch, depth, hw, hw }); - INDArray labels = Nd4j.zeros(minibatch, nOut); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - labels.putScalar(i, r.nextInt(nOut), 1.0); - } - for (boolean useLogStd : new boolean[] { true, false }) { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).seed(12345L).dist(new NormalDistribution(0, 2)).list().layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nIn(depth).nOut(2).activation(Activation.IDENTITY).build()).layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).lockGammaBeta(true).gamma(2.0).beta(0.5).build()).layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(hw, hw, depth)); - MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); - mln.init(); - // for (int j = 0; j < mln.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - // Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - // i.e., runningMean = decay * runningMean + (1-decay) * batchMean - // However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input).labels(labels).excludeParams(excludeParams)); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - - @Test - @DisplayName("Test Batch Norm Comp Graph Simple") - void testBatchNormCompGraphSimple() { - int numClasses = 2; - int height = 3; - int width = 3; - int channels = 1; - long seed = 123; - int minibatchSize = 3; - for (boolean useLogStd : new boolean[] { true, false }) { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed).updater(new NoOp()).dataType(DataType.DOUBLE).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").setInputTypes(InputType.convolutional(height, width, channels)).addLayer("bn", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "in").addLayer("out", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(numClasses).build(), "bn").setOutputs("out").build(); - ComputationGraph net = new ComputationGraph(conf); - net.init(); - Random r = new Random(12345); - // Order: examples, channels, height, width - INDArray input = Nd4j.rand(new int[] { minibatchSize, channels, height, width }); - INDArray labels = Nd4j.zeros(minibatchSize, numClasses); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, r.nextInt(numClasses) }, 1.0); - } - // Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - // i.e., runningMean = decay * runningMean + (1-decay) * batchMean - // However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("bn_mean", "bn_var")); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[] { input }).labels(new INDArray[] { labels }).excludeParams(excludeParams)); - assertTrue(gradOK); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Gradient BN With CN Nand Subsampling Comp Graph") - void testGradientBNWithCNNandSubsamplingCompGraph() { - // Parameterized test, testing combinations of: - // (a) activation function - // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') - // (c) Loss function (with specified output activations) - // (d) l1 and l2 values - Activation[] activFns = { Activation.TANH, Activation.IDENTITY }; - boolean doLearningFirst = true; - LossFunctions.LossFunction[] lossFunctions = { LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD }; - // i.e., lossFunctions[i] used with outputActivations[i] here - Activation[] outputActivations = { Activation.SOFTMAX }; - double[] l2vals = { 0.0, 0.1 }; - // i.e., use l2vals[j] with l1vals[j] - double[] l1vals = { 0.0, 0.2 }; - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int depth = 2; - int hw = 5; - int nOut = 3; - INDArray input = Nd4j.rand(new int[] { minibatch, depth, hw, hw }); - INDArray labels = Nd4j.zeros(minibatch, nOut); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - labels.putScalar(i, r.nextInt(nOut), 1.0); - } - DataSet ds = new DataSet(input, labels); - for (boolean useLogStd : new boolean[] { true, false }) { - for (Activation afn : activFns) { - for (int i = 0; i < lossFunctions.length; i++) { - for (int j = 0; j < l2vals.length; j++) { - LossFunctions.LossFunction lf = lossFunctions[i]; - Activation outputActivation = outputActivations[i]; - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).updater(new NoOp()).dist(new UniformDistribution(-2, 2)).seed(12345L).graphBuilder().addInputs("in").addLayer("0", new ConvolutionLayer.Builder(2, 2).stride(1, 1).nOut(3).activation(afn).build(), "in").addLayer("1", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "0").addLayer("2", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(1, 1).build(), "1").addLayer("3", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "2").addLayer("4", new ActivationLayer.Builder().activation(afn).build(), "3").addLayer("5", new OutputLayer.Builder(lf).activation(outputActivation).nOut(nOut).build(), "4").setOutputs("5").setInputTypes(InputType.convolutional(hw, hw, depth)).build(); - ComputationGraph net = new ComputationGraph(conf); - net.init(); - String name = new Object() { - }.getClass().getEnclosingMethod().getName(); - if (doLearningFirst) { - // Run a number of iterations of learning - net.setInput(0, ds.getFeatures()); - net.setLabels(ds.getLabels()); - net.computeGradientAndScore(); - double scoreBefore = net.score(); - for (int k = 0; k < 20; k++) net.fit(ds); - net.computeGradientAndScore(); - double scoreAfter = net.score(); - // Can't test in 'characteristic mode of operation' if not learning - String msg = name + " - score did not (sufficiently) decrease during learning - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")"; - assertTrue(scoreAfter < 0.9 * scoreBefore,msg); - } - System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + ", l1=" + l1vals[j] + ", l2=" + l2vals[j]); - // for (int k = 0; k < net.getNumLayers(); k++) - // System.out.println("Layer " + k + " # params: " + net.getLayer(k).numParams()); - // Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - // i.e., runningMean = decay * runningMean + (1-decay) * batchMean - // However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "3_mean", "3_var", "1_log10stdev", "3_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[] { input }).labels(new INDArray[] { labels }).excludeParams(excludeParams)); - assertTrue(gradOK); - TestUtils.testModelSerialization(net); - } - } - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java deleted file mode 100644 index 917f3aa99..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN1DGradientCheckTest.java +++ /dev/null @@ -1,368 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.gradientcheck; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.RNNFormat; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.convolutional.Cropping1D; -import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.util.Convolution1DUtils; -import org.deeplearning4j.util.ConvolutionUtils; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.NDArrayIndex; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.io.File; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Cnn 1 D Gradient Check Test") -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -class CNN1DGradientCheckTest extends BaseDL4JTest { - - private static final boolean PRINT_RESULTS = true; - - private static final boolean RETURN_ON_FIRST_FAILURE = false; - - private static final double DEFAULT_EPS = 1e-6; - - private static final double DEFAULT_MAX_REL_ERROR = 1e-3; - - private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; - - static { - Nd4j.setDataType(DataType.DOUBLE); - } - - @Override - public long getTimeoutMilliseconds() { - return 180000; - } - - @Test - @DisplayName("Test Cnn 1 D With Locally Connected 1 D") - void testCnn1DWithLocallyConnected1D() { - Nd4j.getRandom().setSeed(1337); - int[] minibatchSizes = { 2, 3 }; - int length = 7; - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int finalNOut = 4; - int[] kernels = { 1 }; - int stride = 1; - int padding = 0; - Activation[] activations = { Activation.SIGMOID }; - for (Activation afn : activations) { - for (int minibatchSize : minibatchSizes) { - for (int kernel : kernels) { - INDArray input = Nd4j.rand(new int[] { minibatchSize, convNIn, length }); - INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length); - for (int i = 0; i < minibatchSize; i++) { - for (int j = 0; j < length; j++) { - labels.putScalar(new int[] { i, i % finalNOut, j }, 1.0); - } - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list().layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel).stride(stride).padding(padding).nIn(convNIn).nOut(convNOut1).rnnDataFormat(RNNFormat.NCW).build()).layer(new LocallyConnected1D.Builder().activation(afn).kernelSize(kernel).stride(stride).padding(padding).nIn(convNOut1).nOut(convNOut2).hasBias(false).build()).layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).setInputType(InputType.recurrent(convNIn, length)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "Minibatch=" + minibatchSize + ", activationFn=" + afn + ", kernel = " + kernel; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - - @Test - @DisplayName("Test Cnn 1 D With Cropping 1 D") - void testCnn1DWithCropping1D() { - Nd4j.getRandom().setSeed(1337); - int[] minibatchSizes = { 1, 3 }; - int length = 7; - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int finalNOut = 4; - int[] kernels = { 1, 2, 4 }; - int stride = 1; - int padding = 0; - int cropping = 1; - int croppedLength = length - 2 * cropping; - Activation[] activations = { Activation.SIGMOID }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - for (int kernel : kernels) { - INDArray input = Nd4j.rand(new int[] { minibatchSize, convNIn, length }); - INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, croppedLength); - for (int i = 0; i < minibatchSize; i++) { - for (int j = 0; j < croppedLength; j++) { - labels.putScalar(new int[] { i, i % finalNOut, j }, 1.0); - } - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list().layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel).stride(stride).padding(padding).nOut(convNOut1).build()).layer(new Cropping1D.Builder(cropping).build()).layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel).stride(stride).padding(padding).nOut(convNOut2).build()).layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).setInputType(InputType.recurrent(convNIn, length, RNNFormat.NCW)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn + ", kernel = " + kernel; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - - TestUtils.testModelSerialization(net); - } - } - } - } - } - - @Test - @DisplayName("Test Cnn 1 D With Zero Padding 1 D") - void testCnn1DWithZeroPadding1D() { - Nd4j.getRandom().setSeed(1337); - int[] minibatchSizes = { 1, 3 }; - int length = 7; - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int finalNOut = 4; - int[] kernels = { 1, 2, 4 }; - int stride = 1; - int pnorm = 2; - int padding = 0; - int zeroPadding = 2; - int paddedLength = length + 2 * zeroPadding; - Activation[] activations = { Activation.SIGMOID }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - for (int kernel : kernels) { - INDArray input = Nd4j.rand(new int[] { minibatchSize, convNIn, length }); - INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, paddedLength); - for (int i = 0; i < minibatchSize; i++) { - for (int j = 0; j < paddedLength; j++) { - labels.putScalar(new int[] { i, i % finalNOut, j }, 1.0); - } - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list().layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel).stride(stride).padding(padding).nOut(convNOut1).build()).layer(new ZeroPadding1DLayer.Builder(zeroPadding).build()).layer(new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel).stride(stride).padding(padding).nOut(convNOut2).build()).layer(new ZeroPadding1DLayer.Builder(0).build()).layer(new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel).stride(stride).padding(padding).pnorm(pnorm).build()).layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).setInputType(InputType.recurrent(convNIn, length, RNNFormat.NCW)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn + ", kernel = " + kernel; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - - TestUtils.testModelSerialization(net); - } - } - } - } - } - - @Test - @DisplayName("Test Cnn 1 D With Subsampling 1 D") - void testCnn1DWithSubsampling1D() { - Nd4j.getRandom().setSeed(12345); - int[] minibatchSizes = { 1, 3 }; - int length = 7; - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int finalNOut = 4; - int[] kernels = { 1, 2, 4 }; - int stride = 1; - int padding = 0; - int pnorm = 2; - Activation[] activations = { Activation.SIGMOID, Activation.TANH }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - for (int kernel : kernels) { - INDArray input = Nd4j.rand(new int[] { minibatchSize, convNIn, length }); - INDArray labels = Nd4j.zeros(minibatchSize, finalNOut, length); - for (int i = 0; i < minibatchSize; i++) { - for (int j = 0; j < length; j++) { - labels.putScalar(new int[] { i, i % finalNOut, j }, 1.0); - } - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).convolutionMode(ConvolutionMode.Same).list().layer(0, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel).stride(stride).padding(padding).nOut(convNOut1).build()).layer(1, new Convolution1DLayer.Builder().activation(afn).kernelSize(kernel).stride(stride).padding(padding).nOut(convNOut2).build()).layer(2, new Subsampling1DLayer.Builder(poolingType).kernelSize(kernel).stride(stride).padding(padding).pnorm(pnorm).build()).layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).setInputType(InputType.recurrent(convNIn, length, RNNFormat.NCW)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn + ", kernel = " + kernel; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - - TestUtils.testModelSerialization(net); - } - } - } - } - } - - @Test - @DisplayName("Test Cnn 1 d With Masking") - void testCnn1dWithMasking() { - int length = 12; - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int finalNOut = 3; - int pnorm = 2; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG }; - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (ConvolutionMode cm : new ConvolutionMode[] { ConvolutionMode.Same, ConvolutionMode.Truncate }) { - for (int stride : new int[] { 1, 2 }) { - String s = cm + ", stride=" + stride + ", pooling=" + poolingType; - log.info("Starting test: " + s); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).dist(new NormalDistribution(0, 1)).convolutionMode(cm).seed(12345).list().layer(new Convolution1DLayer.Builder().kernelSize(2).rnnDataFormat(RNNFormat.NCW).stride(stride).nIn(convNIn).nOut(convNOut1).build()).layer(new Subsampling1DLayer.Builder(poolingType).kernelSize(2).stride(stride).pnorm(pnorm).build()).layer(new Convolution1DLayer.Builder().kernelSize(2).rnnDataFormat(RNNFormat.NCW).stride(stride).nIn(convNOut1).nOut(convNOut2).build()).layer(new GlobalPoolingLayer(PoolingType.AVG)).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).setInputType(InputType.recurrent(convNIn, length)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray f = Nd4j.rand(new int[] { 2, convNIn, length }); - INDArray fm = Nd4j.create(2, length); - fm.get(NDArrayIndex.point(0), NDArrayIndex.all()).assign(1); - fm.get(NDArrayIndex.point(1), NDArrayIndex.interval(0, 6)).assign(1); - INDArray label = TestUtils.randomOneHot(2, finalNOut); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(f).labels(label).inputMask(fm)); - assertTrue(gradOK,s); - TestUtils.testModelSerialization(net); - // TODO also check that masked step values don't impact forward pass, score or gradients - DataSet ds = new DataSet(f, label, fm, null); - double scoreBefore = net.score(ds); - net.setInput(f); - net.setLabels(label); - net.setLayerMaskArrays(fm, null); - net.computeGradientAndScore(); - INDArray gradBefore = net.getFlattenedGradients().dup(); - f.putScalar(1, 0, 10, 10.0); - f.putScalar(1, 1, 11, 20.0); - double scoreAfter = net.score(ds); - net.setInput(f); - net.setLabels(label); - net.setLayerMaskArrays(fm, null); - net.computeGradientAndScore(); - INDArray gradAfter = net.getFlattenedGradients().dup(); - assertEquals(scoreBefore, scoreAfter, 1e-6); - assertEquals(gradBefore, gradAfter); - } - } - } - } - - @Test - @DisplayName("Test Cnn 1 Causal") - void testCnn1Causal() throws Exception { - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int finalNOut = 3; - int[] lengths = { 11, 12, 13, 9, 10, 11 }; - int[] kernels = { 2, 3, 2, 4, 2, 3 }; - int[] dilations = { 1, 1, 2, 1, 2, 1 }; - int[] strides = { 1, 2, 1, 2, 1, 1 }; - boolean[] masks = { false, true, false, true, false, true }; - boolean[] hasB = { true, false, true, false, true, true }; - for (int i = 0; i < lengths.length; i++) { - int length = lengths[i]; - int k = kernels[i]; - int d = dilations[i]; - int st = strides[i]; - boolean mask = masks[i]; - boolean hasBias = hasB[i]; - // TODO has bias - String s = "k=" + k + ", s=" + st + " d=" + d + ", seqLen=" + length; - log.info("Starting test: " + s); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).weightInit(new NormalDistribution(0, 1)).seed(12345).list().layer(new Convolution1DLayer.Builder().kernelSize(k).dilation(d).hasBias(hasBias).convolutionMode(ConvolutionMode.Causal).stride(st).nOut(convNOut1).build()).layer(new Convolution1DLayer.Builder().kernelSize(k).dilation(d).convolutionMode(ConvolutionMode.Causal).stride(st).nOut(convNOut2).build()).layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).setInputType(InputType.recurrent(convNIn, length, RNNFormat.NCW)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray f = Nd4j.rand(DataType.DOUBLE, 2, convNIn, length); - INDArray fm = null; - if (mask) { - fm = Nd4j.create(2, length); - fm.get(NDArrayIndex.point(0), NDArrayIndex.all()).assign(1); - fm.get(NDArrayIndex.point(1), NDArrayIndex.interval(0, length - 2)).assign(1); - } - long outSize1 = Convolution1DUtils.getOutputSize(length, k, st, 0, ConvolutionMode.Causal, d); - long outSize2 = Convolution1DUtils.getOutputSize(outSize1, k, st, 0, ConvolutionMode.Causal, d); - INDArray label = TestUtils.randomOneHotTimeSeries(2, finalNOut, (int) outSize2); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(f).labels(label).inputMask(fm)); - assertTrue(gradOK,s); - TestUtils.testModelSerialization(net); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java deleted file mode 100644 index b6c1cd933..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNN3DGradientCheckTest.java +++ /dev/null @@ -1,401 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.gradientcheck; - -import lombok.extern.java.Log; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.convolutional.Cropping3D; -import org.deeplearning4j.nn.conf.preprocessor.Cnn3DToFeedForwardPreProcessor; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataBuffer; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.Arrays; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Log -@DisplayName("Cnn 3 D Gradient Check Test") -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -class CNN3DGradientCheckTest extends BaseDL4JTest { - - private static final boolean PRINT_RESULTS = true; - - private static final boolean RETURN_ON_FIRST_FAILURE = false; - - private static final double DEFAULT_EPS = 1e-6; - - private static final double DEFAULT_MAX_REL_ERROR = 1e-3; - - private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; - - static { - Nd4j.setDataType(DataType.DOUBLE); - } - - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - - @Test - @DisplayName("Test Cnn 3 D Plain") - void testCnn3DPlain() { - Nd4j.getRandom().setSeed(1337); - // Note: we checked this with a variety of parameters, but it takes a lot of time. - int[] depths = { 6 }; - int[] heights = { 6 }; - int[] widths = { 6 }; - int[] minibatchSizes = { 3 }; - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int denseNOut = 5; - int finalNOut = 42; - int[][] kernels = { { 2, 2, 2 } }; - int[][] strides = { { 1, 1, 1 } }; - Activation[] activations = { Activation.SIGMOID }; - ConvolutionMode[] modes = { ConvolutionMode.Truncate, ConvolutionMode.Same }; - for (Activation afn : activations) { - for (int miniBatchSize : minibatchSizes) { - for (int depth : depths) { - for (int height : heights) { - for (int width : widths) { - for (ConvolutionMode mode : modes) { - for (int[] kernel : kernels) { - for (int[] stride : strides) { - for (Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) { - int outDepth = mode == ConvolutionMode.Same ? depth / stride[0] : (depth - kernel[0]) / stride[0] + 1; - int outHeight = mode == ConvolutionMode.Same ? height / stride[1] : (height - kernel[1]) / stride[1] + 1; - int outWidth = mode == ConvolutionMode.Same ? width / stride[2] : (width - kernel[2]) / stride[2] + 1; - INDArray input; - if (df == Convolution3D.DataFormat.NDHWC) { - input = Nd4j.rand(new int[] { miniBatchSize, depth, height, width, convNIn }); - } else { - input = Nd4j.rand(new int[] { miniBatchSize, convNIn, depth, height, width }); - } - INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); - for (int i = 0; i < miniBatchSize; i++) { - labels.putScalar(new int[] { i, i % finalNOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL).dist(new NormalDistribution(0, 1)).list().layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel).stride(stride).nIn(convNIn).nOut(convNOut1).hasBias(false).convolutionMode(mode).dataFormat(df).build()).layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1).nIn(convNOut1).nOut(convNOut2).hasBias(false).convolutionMode(mode).dataFormat(df).build()).layer(2, new DenseLayer.Builder().nOut(denseNOut).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).inputPreProcessor(2, new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, convNOut2, df == Convolution3D.DataFormat.NCDHW)).setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "DataFormat = " + df + ", minibatch size = " + miniBatchSize + ", activationFn=" + afn + ", kernel = " + Arrays.toString(kernel) + ", stride = " + Arrays.toString(stride) + ", mode = " + mode.toString() + ", input depth " + depth + ", input height " + height + ", input width " + width; - if (PRINT_RESULTS) { - log.info(msg); - // for (int j = 0; j < net.getnLayers(); j++) { - // log.info("Layer " + j + " # params: " + net.getLayer(j).numParams()); - // } - } - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(128)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - } - } - } - } - } - } - - @Test - @DisplayName("Test Cnn 3 D Zero Padding") - void testCnn3DZeroPadding() { - Nd4j.getRandom().setSeed(42); - int depth = 4; - int height = 4; - int width = 4; - int[] minibatchSizes = { 3 }; - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int denseNOut = 5; - int finalNOut = 42; - int[] kernel = { 2, 2, 2 }; - int[] zeroPadding = { 1, 1, 2, 2, 3, 3 }; - Activation[] activations = { Activation.SIGMOID }; - ConvolutionMode[] modes = { ConvolutionMode.Truncate, ConvolutionMode.Same }; - for (Activation afn : activations) { - for (int miniBatchSize : minibatchSizes) { - for (ConvolutionMode mode : modes) { - int outDepth = mode == ConvolutionMode.Same ? depth : (depth - kernel[0]) + 1; - int outHeight = mode == ConvolutionMode.Same ? height : (height - kernel[1]) + 1; - int outWidth = mode == ConvolutionMode.Same ? width : (width - kernel[2]) + 1; - outDepth += zeroPadding[0] + zeroPadding[1]; - outHeight += zeroPadding[2] + zeroPadding[3]; - outWidth += zeroPadding[4] + zeroPadding[5]; - INDArray input = Nd4j.rand(new int[] { miniBatchSize, convNIn, depth, height, width }); - INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); - for (int i = 0; i < miniBatchSize; i++) { - labels.putScalar(new int[] { i, i % finalNOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL).dist(new NormalDistribution(0, 1)).list().layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel).nIn(convNIn).nOut(convNOut1).hasBias(false).convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW).build()).layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1).nIn(convNOut1).nOut(convNOut2).hasBias(false).convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW).build()).layer(2, new ZeroPadding3DLayer.Builder(zeroPadding).build()).layer(3, new DenseLayer.Builder().nOut(denseNOut).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).inputPreProcessor(3, new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, convNOut2, true)).setInputType(InputType.convolutional3D(depth, height, width, convNIn)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString() + ", input depth " + depth + ", input height " + height + ", input width " + width; - if (PRINT_RESULTS) { - log.info(msg); - // for (int j = 0; j < net.getnLayers(); j++) { - // log.info("Layer " + j + " # params: " + net.getLayer(j).numParams()); - // } - } - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(512)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - - @Test - @DisplayName("Test Cnn 3 D Pooling") - void testCnn3DPooling() { - Nd4j.getRandom().setSeed(42); - int depth = 4; - int height = 4; - int width = 4; - int[] minibatchSizes = { 3 }; - int convNIn = 2; - int convNOut = 4; - int denseNOut = 5; - int finalNOut = 42; - int[] kernel = { 2, 2, 2 }; - Activation[] activations = { Activation.SIGMOID }; - Subsampling3DLayer.PoolingType[] poolModes = { Subsampling3DLayer.PoolingType.AVG }; - ConvolutionMode[] modes = { ConvolutionMode.Truncate }; - for (Activation afn : activations) { - for (int miniBatchSize : minibatchSizes) { - for (Subsampling3DLayer.PoolingType pool : poolModes) { - for (ConvolutionMode mode : modes) { - for (Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) { - int outDepth = depth / kernel[0]; - int outHeight = height / kernel[1]; - int outWidth = width / kernel[2]; - INDArray input = Nd4j.rand(df == Convolution3D.DataFormat.NCDHW ? new int[] { miniBatchSize, convNIn, depth, height, width } : new int[] { miniBatchSize, depth, height, width, convNIn }); - INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); - for (int i = 0; i < miniBatchSize; i++) { - labels.putScalar(new int[] { i, i % finalNOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).weightInit(WeightInit.XAVIER).dist(new NormalDistribution(0, 1)).list().layer(0, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1).nIn(convNIn).nOut(convNOut).hasBias(false).convolutionMode(mode).dataFormat(df).build()).layer(1, new Subsampling3DLayer.Builder(kernel).poolingType(pool).convolutionMode(mode).dataFormat(df).build()).layer(2, new DenseLayer.Builder().nOut(denseNOut).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).inputPreProcessor(2, new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, convNOut, df)).setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString() + ", input depth " + depth + ", input height " + height + ", input width " + width + ", dataFormat=" + df; - if (PRINT_RESULTS) { - log.info(msg); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - } - } - - @Test - @DisplayName("Test Cnn 3 D Upsampling") - void testCnn3DUpsampling() { - Nd4j.getRandom().setSeed(42); - int depth = 2; - int height = 2; - int width = 2; - int[] minibatchSizes = { 3 }; - int convNIn = 2; - int convNOut = 4; - int denseNOut = 5; - int finalNOut = 42; - int[] upsamplingSize = { 2, 2, 2 }; - Activation[] activations = { Activation.SIGMOID }; - ConvolutionMode[] modes = { ConvolutionMode.Truncate }; - for (Activation afn : activations) { - for (int miniBatchSize : minibatchSizes) { - for (ConvolutionMode mode : modes) { - for (Convolution3D.DataFormat df : Convolution3D.DataFormat.values()) { - int outDepth = depth * upsamplingSize[0]; - int outHeight = height * upsamplingSize[1]; - int outWidth = width * upsamplingSize[2]; - INDArray input = df == Convolution3D.DataFormat.NCDHW ? Nd4j.rand(miniBatchSize, convNIn, depth, height, width) : Nd4j.rand(miniBatchSize, depth, height, width, convNIn); - INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); - for (int i = 0; i < miniBatchSize; i++) { - labels.putScalar(new int[] { i, i % finalNOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL).dist(new NormalDistribution(0, 1)).seed(12345).list().layer(0, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1).nIn(convNIn).nOut(convNOut).hasBias(false).convolutionMode(mode).dataFormat(df).build()).layer(1, new Upsampling3D.Builder(upsamplingSize[0]).dataFormat(df).build()).layer(2, new DenseLayer.Builder().nOut(denseNOut).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).inputPreProcessor(2, new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, convNOut, true)).setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn + ", kernel = " + Arrays.toString(upsamplingSize) + ", mode = " + mode.toString() + ", input depth " + depth + ", input height " + height + ", input width " + width; - if (PRINT_RESULTS) { - log.info(msg); - // for (int j = 0; j < net.getnLayers(); j++) { - // log.info("Layer " + j + " # params: " + net.getLayer(j).numParams()); - // } - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - } - - @Test - @DisplayName("Test Cnn 3 D Cropping") - void testCnn3DCropping() { - Nd4j.getRandom().setSeed(42); - int depth = 6; - int height = 6; - int width = 6; - int[] minibatchSizes = { 3 }; - int convNIn = 2; - int convNOut1 = 3; - int convNOut2 = 4; - int denseNOut = 5; - int finalNOut = 8; - int[] kernel = { 1, 1, 1 }; - int[] cropping = { 0, 0, 1, 1, 2, 2 }; - Activation[] activations = { Activation.SIGMOID }; - ConvolutionMode[] modes = { ConvolutionMode.Same }; - for (Activation afn : activations) { - for (int miniBatchSize : minibatchSizes) { - for (ConvolutionMode mode : modes) { - int outDepth = mode == ConvolutionMode.Same ? depth : (depth - kernel[0]) + 1; - int outHeight = mode == ConvolutionMode.Same ? height : (height - kernel[1]) + 1; - int outWidth = mode == ConvolutionMode.Same ? width : (width - kernel[2]) + 1; - outDepth -= cropping[0] + cropping[1]; - outHeight -= cropping[2] + cropping[3]; - outWidth -= cropping[4] + cropping[5]; - INDArray input = Nd4j.rand(new int[] { miniBatchSize, convNIn, depth, height, width }); - INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); - for (int i = 0; i < miniBatchSize; i++) { - labels.putScalar(new int[] { i, i % finalNOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).weightInit(WeightInit.LECUN_NORMAL).dist(new NormalDistribution(0, 1)).list().layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel).nIn(convNIn).nOut(convNOut1).hasBias(false).convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW).build()).layer(1, new Convolution3D.Builder().activation(afn).kernelSize(1, 1, 1).nIn(convNOut1).nOut(convNOut2).hasBias(false).convolutionMode(mode).dataFormat(Convolution3D.DataFormat.NCDHW).build()).layer(2, new Cropping3D.Builder(cropping).build()).layer(3, new DenseLayer.Builder().nOut(denseNOut).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).inputPreProcessor(3, new Cnn3DToFeedForwardPreProcessor(outDepth, outHeight, outWidth, convNOut2, true)).setInputType(InputType.convolutional3D(depth, height, width, convNIn)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "Minibatch size = " + miniBatchSize + ", activationFn=" + afn + ", kernel = " + Arrays.toString(kernel) + ", mode = " + mode.toString() + ", input depth " + depth + ", input height " + height + ", input width " + width; - if (PRINT_RESULTS) { - log.info(msg); - // for (int j = 0; j < net.getnLayers(); j++) { - // log.info("Layer " + j + " # params: " + net.getLayer(j).numParams()); - // } - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - - @Test - @DisplayName("Test Deconv 3 d") - void testDeconv3d() { - Nd4j.getRandom().setSeed(12345); - // Note: we checked this with a variety of parameters, but it takes a lot of time. - int[] depths = { 8, 8, 9 }; - int[] heights = { 8, 9, 9 }; - int[] widths = { 8, 8, 9 }; - int[][] kernels = { { 2, 2, 2 }, { 3, 3, 3 }, { 2, 3, 2 } }; - int[][] strides = { { 1, 1, 1 }, { 1, 1, 1 }, { 2, 2, 2 } }; - Activation[] activations = { Activation.SIGMOID, Activation.TANH, Activation.IDENTITY }; - ConvolutionMode[] modes = { ConvolutionMode.Truncate, ConvolutionMode.Same, ConvolutionMode.Same }; - int[] mbs = { 1, 3, 2 }; - Convolution3D.DataFormat[] dataFormats = new Convolution3D.DataFormat[] { Convolution3D.DataFormat.NCDHW, Convolution3D.DataFormat.NDHWC, Convolution3D.DataFormat.NCDHW }; - int convNIn = 2; - int finalNOut = 2; - int[] deconvOut = { 2, 3, 4 }; - for (int i = 0; i < activations.length; i++) { - Activation afn = activations[i]; - int miniBatchSize = mbs[i]; - int depth = depths[i]; - int height = heights[i]; - int width = widths[i]; - ConvolutionMode mode = modes[i]; - int[] kernel = kernels[i]; - int[] stride = strides[i]; - Convolution3D.DataFormat df = dataFormats[i]; - int dOut = deconvOut[i]; - INDArray input; - if (df == Convolution3D.DataFormat.NDHWC) { - input = Nd4j.rand(new int[] { miniBatchSize, depth, height, width, convNIn }); - } else { - input = Nd4j.rand(new int[] { miniBatchSize, convNIn, depth, height, width }); - } - INDArray labels = Nd4j.zeros(miniBatchSize, finalNOut); - for (int j = 0; j < miniBatchSize; j++) { - labels.putScalar(new int[] { j, j % finalNOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).weightInit(new NormalDistribution(0, 0.1)).list().layer(0, new Convolution3D.Builder().activation(afn).kernelSize(kernel).stride(stride).nIn(convNIn).nOut(dOut).hasBias(false).convolutionMode(mode).dataFormat(df).build()).layer(1, new Deconvolution3D.Builder().activation(afn).kernelSize(kernel).stride(stride).nOut(dOut).hasBias(false).convolutionMode(mode).dataFormat(df).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(finalNOut).build()).setInputType(InputType.convolutional3D(df, depth, height, width, convNIn)).build(); - String json = conf.toJson(); - MultiLayerConfiguration c2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, c2); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "DataFormat = " + df + ", minibatch size = " + miniBatchSize + ", activationFn=" + afn + ", kernel = " + Arrays.toString(kernel) + ", stride = " + Arrays.toString(stride) + ", mode = " + mode.toString() + ", input depth " + depth + ", input height " + height + ", input width " + width; - if (PRINT_RESULTS) { - log.info(msg); - } - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(64)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java deleted file mode 100644 index 16c36f592..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java +++ /dev/null @@ -1,856 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.gradientcheck; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.CNN2DFormat; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.Nd4jBackend; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - -import static org.deeplearning4j.nn.conf.ConvolutionMode.Same; -import static org.deeplearning4j.nn.conf.ConvolutionMode.Truncate; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.shade.guava.collect.Lists; - -@DisplayName("Cnn Gradient Check Test") -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -class CNNGradientCheckTest extends BaseDL4JTest { - - private static final boolean PRINT_RESULTS = true; - - private static final boolean RETURN_ON_FIRST_FAILURE = false; - - private static final double DEFAULT_EPS = 1e-6; - - private static final double DEFAULT_MAX_REL_ERROR = 1e-3; - - private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; - - static { - Nd4j.setDataType(DataType.DOUBLE); - } - - - - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(CNN2DFormat format : CNN2DFormat.values()) { - args.add(Arguments.of(format,nd4jBackend)); - } - } - return args.stream(); - } - - @Override - public long getTimeoutMilliseconds() { - return 999990000L; - } - - @DisplayName("Test Gradient CNNMLN") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - public void testGradientCNNMLN(CNN2DFormat format,Nd4jBackend backend) { - if (// Only test NCHW due to flat input format... - format != CNN2DFormat.NCHW) - return; - // Parameterized test, testing combinations of: - // (a) activation function - // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') - // (c) Loss function (with specified output activations) - Activation[] activFns = { Activation.SIGMOID, Activation.TANH }; - // If true: run some backprop steps first - boolean[] characteristic = { false, true }; - LossFunctions.LossFunction[] lossFunctions = { LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE }; - // i.e., lossFunctions[i] used with outputActivations[i] here - Activation[] outputActivations = { Activation.SOFTMAX, Activation.TANH }; - DataSet ds = new IrisDataSetIterator(150, 150).next(); - ds.normalizeZeroMeanZeroUnitVariance(); - INDArray input = ds.getFeatures(); - INDArray labels = ds.getLabels(); - for (Activation afn : activFns) { - for (boolean doLearningFirst : characteristic) { - for (int i = 0; i < lossFunctions.length; i++) { - LossFunctions.LossFunction lf = lossFunctions[i]; - Activation outputActivation = outputActivations[i]; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp()).weightInit(WeightInit.XAVIER).seed(12345L).list().layer(0, new ConvolutionLayer.Builder(1, 1).nOut(6).activation(afn).build()).layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).build()).setInputType(InputType.convolutionalFlat(1, 4, 1)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - String name = new Object() { - }.getClass().getEnclosingMethod().getName(); - if (doLearningFirst) { - // Run a number of iterations of learning - mln.setInput(ds.getFeatures()); - mln.setLabels(ds.getLabels()); - mln.computeGradientAndScore(); - double scoreBefore = mln.score(); - for (int j = 0; j < 10; j++) mln.fit(ds); - mln.computeGradientAndScore(); - double scoreAfter = mln.score(); - // Can't test in 'characteristic mode of operation' if not learning - String msg = name + " - score did not (sufficiently) decrease during learning - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")"; - assertTrue(scoreAfter < 0.9 * scoreBefore,msg); - } - if (PRINT_RESULTS) { - System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst); - // for (int j = 0; j < mln.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - } - } - - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - @DisplayName("Test Gradient CNNL 1 L 2 MLN") - void testGradientCNNL1L2MLN(CNN2DFormat format,Nd4jBackend backend) { - if (// Only test NCHW due to flat input format... - format != CNN2DFormat.NCHW) - return; - // Parameterized test, testing combinations of: - // (a) activation function - // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') - // (c) Loss function (with specified output activations) - DataSet ds = new IrisDataSetIterator(150, 150).next(); - ds.normalizeZeroMeanZeroUnitVariance(); - INDArray input = ds.getFeatures(); - INDArray labels = ds.getLabels(); - // use l2vals[i] with l1vals[i] - double[] l2vals = { 0.4, 0.0, 0.4, 0.4 }; - double[] l1vals = { 0.0, 0.0, 0.5, 0.0 }; - double[] biasL2 = { 0.0, 0.0, 0.0, 0.2 }; - double[] biasL1 = { 0.0, 0.0, 0.6, 0.0 }; - Activation[] activFns = { Activation.SIGMOID, Activation.TANH, Activation.ELU, Activation.SOFTPLUS }; - // If true: run some backprop steps first - boolean[] characteristic = { false, true, false, true }; - LossFunctions.LossFunction[] lossFunctions = { LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE, LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE }; - // i.e., lossFunctions[i] used with outputActivations[i] here - Activation[] outputActivations = { Activation.SOFTMAX, Activation.TANH, Activation.SOFTMAX, Activation.IDENTITY }; - for (int i = 0; i < l2vals.length; i++) { - Activation afn = activFns[i]; - boolean doLearningFirst = characteristic[i]; - LossFunctions.LossFunction lf = lossFunctions[i]; - Activation outputActivation = outputActivations[i]; - double l2 = l2vals[i]; - double l1 = l1vals[i]; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).l2(l2).l1(l1).l2Bias(biasL2[i]).l1Bias(biasL1[i]).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).seed(12345L).list().layer(0, new ConvolutionLayer.Builder(new int[] { 1, 1 }).nIn(1).nOut(6).weightInit(WeightInit.XAVIER).activation(afn).updater(new NoOp()).build()).layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).weightInit(WeightInit.XAVIER).updater(new NoOp()).build()).setInputType(InputType.convolutionalFlat(1, 4, 1)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - String testName = new Object() { - }.getClass().getEnclosingMethod().getName(); - if (doLearningFirst) { - // Run a number of iterations of learning - mln.setInput(ds.getFeatures()); - mln.setLabels(ds.getLabels()); - mln.computeGradientAndScore(); - double scoreBefore = mln.score(); - for (int j = 0; j < 10; j++) mln.fit(ds); - mln.computeGradientAndScore(); - double scoreAfter = mln.score(); - // Can't test in 'characteristic mode of operation' if not learning - String msg = testName + "- score did not (sufficiently) decrease during learning - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")"; - assertTrue(scoreAfter < 0.8 * scoreBefore,msg); - } - if (PRINT_RESULTS) { - System.out.println(testName + "- activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst); - // for (int j = 0; j < mln.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - - @Disabled - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - @DisplayName("Test Cnn With Space To Depth") - void testCnnWithSpaceToDepth(CNN2DFormat format,Nd4jBackend backend) { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int minibatchSize = 2; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int blocks = 2; - String[] activations = { "sigmoid" }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - for (String afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).list().layer(new ConvolutionLayer.Builder(kernel).nIn(inputDepth).hasBias(false).nOut(1).build()).layer(new SpaceToDepthLayer.Builder(blocks, SpaceToDepthLayer.DataFormat.NCHW).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2 * 2 * 4).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - - @DisplayName("Test Cnn With Space To Batch") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - public void testCnnWithSpaceToBatch(CNN2DFormat format,Nd4jBackend backend) { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int[] minibatchSizes = { 2, 4 }; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int[] blocks = { 2, 2 }; - String[] activations = { "sigmoid", "tanh" }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - boolean nchw = format == CNN2DFormat.NCHW; - for (String afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(4 * minibatchSize, nOut); - for (int i = 0; i < 4 * minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).weightInit(new NormalDistribution(0, 1)).list().layer(new ConvolutionLayer.Builder(kernel).nIn(inputDepth).nOut(3).dataFormat(format).build()).layer(new SpaceToBatchLayer.Builder(blocks).dataFormat(format).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = format + " - poolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - // Also check compgraph: - ComputationGraph cg = net.toComputationGraph(); - gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(cg).inputs(new INDArray[] { input }).labels(new INDArray[] { labels })); - assertTrue(gradOK,msg + " - compgraph"); - TestUtils.testModelSerialization(net); - } - } - } - } - - @DisplayName("Test Cnn With Upsampling") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnWithUpsampling(CNN2DFormat format,Nd4jBackend backend) { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int[] minibatchSizes = { 1, 3 }; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int size = 2; - boolean nchw = format == CNN2DFormat.NCHW; - for (int minibatchSize : minibatchSizes) { - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).list().layer(new ConvolutionLayer.Builder(kernel, stride, padding).nIn(inputDepth).dataFormat(format).nOut(3).build()).layer(// output: 4*2 =8 -> 8x8x3 - new Upsampling2D.Builder().size(size).dataFormat(format).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(8 * 8 * 3).nOut(4).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "Upsampling - minibatch=" + minibatchSize; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - - @DisplayName("Test Cnn With Subsampling") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnWithSubsampling(CNN2DFormat format,Nd4jBackend backend) { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int[] minibatchSizes = { 1, 3 }; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int pnorm = 2; - Activation[] activations = { Activation.SIGMOID, Activation.TANH }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - boolean nchw = format == CNN2DFormat.NCHW; - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).dist(new NormalDistribution(0, 1)).list().layer(0, new ConvolutionLayer.Builder(kernel, stride, padding).nIn(inputDepth).dataFormat(format).nOut(3).build()).layer(1, new SubsamplingLayer.Builder(poolingType).dataFormat(format).kernelSize(kernel).stride(stride).padding(padding).pnorm(pnorm).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3 * 3 * 3).nOut(4).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = format + " - poolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - - @DisplayName("Test Cnn With Subsampling V 2") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnWithSubsamplingV2(CNN2DFormat format,Nd4jBackend backend) { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int[] minibatchSizes = { 1, 3 }; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int pNorm = 3; - Activation[] activations = { Activation.SIGMOID, Activation.TANH }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - boolean nchw = format == CNN2DFormat.NCHW; - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).dist(new NormalDistribution(0, 1)).list().layer(0, new ConvolutionLayer.Builder(kernel, stride, padding).nIn(inputDepth).dataFormat(format).nOut(3).build()).layer(1, new SubsamplingLayer.Builder(poolingType).dataFormat(format).kernelSize(kernel).stride(stride).padding(padding).pnorm(pNorm).build()).layer(2, new ConvolutionLayer.Builder(kernel, stride, padding).dataFormat(format).nIn(3).nOut(2).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2 * 2 * 2).nOut(4).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - - @DisplayName("Test Cnn Locally Connected 2 D") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnLocallyConnected2D(CNN2DFormat format,Nd4jBackend backend) { - int nOut = 3; - int width = 5; - int height = 5; - Nd4j.getRandom().setSeed(12345); - int[] inputDepths = new int[] { 1, 2, 4 }; - Activation[] activations = { Activation.SIGMOID, Activation.TANH, Activation.SOFTPLUS }; - int[] minibatch = { 2, 1, 3 }; - boolean nchw = format == CNN2DFormat.NCHW; - for (int i = 0; i < inputDepths.length; i++) { - int inputDepth = inputDepths[i]; - Activation afn = activations[i]; - int minibatchSize = minibatch[i]; - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new NoOp()).dataType(DataType.DOUBLE).activation(afn).list().layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).dataFormat(format).padding(0, 0).nIn(inputDepth).nOut(2).build()).layer(1, new LocallyConnected2D.Builder().nIn(2).nOut(7).kernelSize(2, 2).dataFormat(format).setInputSize(4, 4).convolutionMode(ConvolutionMode.Strict).hasBias(false).stride(1, 1).padding(0, 0).build()).layer(2, new ConvolutionLayer.Builder().nIn(7).nOut(2).kernelSize(2, 2).dataFormat(format).stride(1, 1).padding(0, 0).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2 * 2 * 2).nOut(nOut).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - assertEquals(ConvolutionMode.Truncate, ((ConvolutionLayer) conf.getConf(0).getLayer()).getConvolutionMode()); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "Minibatch=" + minibatchSize + ", activationFn=" + afn; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - - @DisplayName("Test Cnn Multi Layer") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnMultiLayer(CNN2DFormat format,Nd4jBackend backend) { - int nOut = 2; - int[] minibatchSizes = { 1, 2, 5 }; - int width = 5; - int height = 5; - int[] inputDepths = { 1, 2, 4 }; - Activation[] activations = { Activation.SIGMOID, Activation.TANH }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG }; - Nd4j.getRandom().setSeed(12345); - boolean nchw = format == CNN2DFormat.NCHW; - for (int inputDepth : inputDepths) { - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new NoOp()).dataType(DataType.DOUBLE).activation(afn).list().layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).dataFormat(format).padding(0, 0).nIn(inputDepth).nOut(2).build()).layer(1, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(2, 2).dataFormat(format).stride(1, 1).padding(0, 0).build()).layer(2, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(2, 2).dataFormat(format).stride(1, 1).padding(0, 0).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2 * 2 * 2).nOut(nOut).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - assertEquals(ConvolutionMode.Truncate, ((ConvolutionLayer) conf.getConf(0).getLayer()).getConvolutionMode()); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int i = 0; i < 4; i++) { - System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - } - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - } - - @DisplayName("Test Cnn Same Padding Mode") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnSamePaddingMode(CNN2DFormat format,Nd4jBackend backend) { - int nOut = 2; - int[] minibatchSizes = { 1, 3, 3, 2, 1, 2 }; - // Same padding mode: insensitive to exact input size... - int[] heights = new int[] { 4, 5, 6, 5, 4, 4 }; - int[] kernelSizes = new int[] { 2, 3, 2, 3, 2, 3 }; - int[] inputDepths = { 1, 2, 4, 3, 2, 3 }; - int width = 5; - Nd4j.getRandom().setSeed(12345); - boolean nchw = format == CNN2DFormat.NCHW; - for (int i = 0; i < minibatchSizes.length; i++) { - int inputDepth = inputDepths[i]; - int minibatchSize = minibatchSizes[i]; - int height = heights[i]; - int k = kernelSizes[i]; - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.SIGMOID).convolutionMode(Same).list().layer(0, new ConvolutionLayer.Builder().name("layer 0").kernelSize(k, k).dataFormat(format).stride(1, 1).padding(0, 0).nIn(inputDepth).nOut(2).build()).layer(1, new SubsamplingLayer.Builder().dataFormat(format).poolingType(SubsamplingLayer.PoolingType.MAX).kernelSize(k, k).stride(1, 1).padding(0, 0).build()).layer(2, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(k, k).dataFormat(format).stride(1, 1).padding(0, 0).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int j = 0; j < net.getLayers().length; j++) { - System.out.println("nParams, layer " + j + ": " + net.getLayer(j).numParams()); - } - String msg = "Minibatch=" + minibatchSize + ", inDepth=" + inputDepth + ", height=" + height + ", kernelSize=" + k; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - - @DisplayName("Test Cnn Same Padding Mode Strided") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnSamePaddingModeStrided(CNN2DFormat format,Nd4jBackend backend) { - int nOut = 2; - int[] minibatchSizes = { 1, 3 }; - int width = 16; - int height = 16; - int[] kernelSizes = new int[] { 2, 3 }; - int[] strides = { 1, 2, 3 }; - int[] inputDepths = { 1, 3 }; - Nd4j.getRandom().setSeed(12345); - boolean nchw = format == CNN2DFormat.NCHW; - for (int inputDepth : inputDepths) { - for (int minibatchSize : minibatchSizes) { - for (int stride : strides) { - for (int k : kernelSizes) { - for (boolean convFirst : new boolean[] { true, false }) { - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - Layer convLayer = new ConvolutionLayer.Builder().name("layer 0").kernelSize(k, k).dataFormat(format).stride(stride, stride).padding(0, 0).nIn(inputDepth).nOut(2).build(); - Layer poolLayer = new SubsamplingLayer.Builder().poolingType(SubsamplingLayer.PoolingType.MAX).kernelSize(k, k).dataFormat(format).stride(stride, stride).padding(0, 0).build(); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(Same).list().layer(0, convFirst ? convLayer : poolLayer).layer(1, convFirst ? poolLayer : convLayer).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int i = 0; i < net.getLayers().length; i++) { - System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - } - String msg = "Minibatch=" + minibatchSize + ", inDepth=" + inputDepth + ", height=" + height + ", kernelSize=" + k + ", stride = " + stride + ", convLayer first = " + convFirst; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(128)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - } - } - - @DisplayName("Test Cnn Zero Padding Layer") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnZeroPaddingLayer(CNN2DFormat format,Nd4jBackend backend) { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int width = 6; - int height = 6; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int[] minibatchSizes = { 1, 3, 2 }; - int[] inputDepths = { 1, 3, 2 }; - int[][] zeroPadLayer = new int[][] { { 0, 0, 0, 0 }, { 1, 1, 0, 0 }, { 2, 2, 2, 2 } }; - boolean nchw = format == CNN2DFormat.NCHW; - for (int i = 0; i < minibatchSizes.length; i++) { - int minibatchSize = minibatchSizes[i]; - int inputDepth = inputDepths[i]; - int[] zeroPad = zeroPadLayer[i]; - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).dist(new NormalDistribution(0, 1)).list().layer(0, new ConvolutionLayer.Builder(kernel, stride, padding).dataFormat(format).nIn(inputDepth).nOut(3).build()).layer(1, new ZeroPaddingLayer.Builder(zeroPad).dataFormat(format).build()).layer(2, new ConvolutionLayer.Builder(kernel, stride, padding).nIn(3).nOut(3).dataFormat(format).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(4).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // Check zero padding activation shape - org.deeplearning4j.nn.layers.convolution.ZeroPaddingLayer zpl = (org.deeplearning4j.nn.layers.convolution.ZeroPaddingLayer) net.getLayer(1); - long[] expShape; - if (nchw) { - expShape = new long[] { minibatchSize, inputDepth, height + zeroPad[0] + zeroPad[1], width + zeroPad[2] + zeroPad[3] }; - } else { - expShape = new long[] { minibatchSize, height + zeroPad[0] + zeroPad[1], width + zeroPad[2] + zeroPad[3], inputDepth }; - } - INDArray out = zpl.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(expShape, out.shape()); - String msg = "minibatch=" + minibatchSize + ", channels=" + inputDepth + ", zeroPad = " + Arrays.toString(zeroPad); - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - - @DisplayName("Test Deconvolution 2 D") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testDeconvolution2D(CNN2DFormat format,Nd4jBackend backend) { - int nOut = 2; - int[] minibatchSizes = new int[] { 1, 3, 3, 1, 3 }; - int[] kernelSizes = new int[] { 1, 1, 1, 3, 3 }; - int[] strides = { 1, 1, 2, 2, 2 }; - int[] dilation = { 1, 2, 1, 2, 2 }; - Activation[] activations = new Activation[] { Activation.SIGMOID, Activation.TANH, Activation.SIGMOID, Activation.SIGMOID, Activation.SIGMOID }; - ConvolutionMode[] cModes = new ConvolutionMode[] { Same, Same, Truncate, Truncate, Truncate }; - int width = 7; - int height = 7; - int inputDepth = 3; - Nd4j.getRandom().setSeed(12345); - boolean nchw = format == CNN2DFormat.NCHW; - for (int i = 0; i < minibatchSizes.length; i++) { - int minibatchSize = minibatchSizes[i]; - int k = kernelSizes[i]; - int s = strides[i]; - int d = dilation[i]; - ConvolutionMode cm = cModes[i]; - Activation act = activations[i]; - int w = d * width; - int h = d * height; - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, h, w } : new long[] { minibatchSize, h, w, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int j = 0; j < minibatchSize; j++) { - labels.putScalar(new int[] { j, j % nOut }, 1.0); - } - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(act).list().layer(new Deconvolution2D.Builder().name("deconvolution_2D_layer").kernelSize(k, k).stride(s, s).dataFormat(format).dilation(d, d).convolutionMode(cm).nIn(inputDepth).nOut(nOut).build()); - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(h, w, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int j = 0; j < net.getLayers().length; j++) { - System.out.println("nParams, layer " + j + ": " + net.getLayer(j).numParams()); - } - String msg = " - mb=" + minibatchSize + ", k=" + k + ", s=" + s + ", d=" + d + ", cm=" + cm; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(100)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - - @DisplayName("Test Separable Conv 2 D") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testSeparableConv2D(CNN2DFormat format,Nd4jBackend backend) { - int nOut = 2; - int width = 6; - int height = 6; - int inputDepth = 3; - Nd4j.getRandom().setSeed(12345); - int[] ks = new int[] { 1, 3, 3, 1, 3 }; - int[] ss = new int[] { 1, 1, 1, 2, 2 }; - int[] ds = new int[] { 1, 1, 2, 2, 2 }; - ConvolutionMode[] cms = new ConvolutionMode[] { Truncate, Truncate, Truncate, Truncate, Truncate }; - int[] mb = new int[] { 1, 1, 1, 3, 3 }; - boolean nchw = format == CNN2DFormat.NCHW; - for (int t = 0; t < ks.length; t++) { - int k = ks[t]; - int s = ss[t]; - int d = ds[t]; - ConvolutionMode cm = cms[t]; - int minibatchSize = mb[t]; - // Use larger input with larger dilation values (to avoid invalid config) - int w = d * width; - int h = d * height; - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, h, w } : new long[] { minibatchSize, h, w, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(cm).list().layer(new SeparableConvolution2D.Builder().name("Separable conv 2D layer").kernelSize(k, k).stride(s, s).dilation(d, d).depthMultiplier(3).dataFormat(format).nIn(inputDepth).nOut(2).build()); - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(h, w, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int i = 0; i < net.getLayers().length; i++) { - System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - } - String msg = " - mb=" + minibatchSize + ", k=" + k + ", s=" + s + ", d=" + d + ", cm=" + cm; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(// Most params are in output layer - 50)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - - @DisplayName("Test Cnn Dilated") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCnnDilated(CNN2DFormat format,Nd4jBackend backend) { - int nOut = 2; - int minibatchSize = 2; - int width = 8; - int height = 8; - int inputDepth = 2; - Nd4j.getRandom().setSeed(12345); - boolean[] sub = new boolean[] { true, true, false, true, false }; - int[] stride = new int[] { 1, 1, 1, 2, 2 }; - int[] kernel = new int[] { 2, 3, 3, 3, 3 }; - int[] ds = new int[] { 2, 2, 3, 3, 2 }; - ConvolutionMode[] cms = new ConvolutionMode[] { Same, Truncate, Truncate, Same, Truncate }; - boolean nchw = format == CNN2DFormat.NCHW; - for (int t = 0; t < sub.length; t++) { - boolean subsampling = sub[t]; - int s = stride[t]; - int k = kernel[t]; - int d = ds[t]; - ConvolutionMode cm = cms[t]; - // Use larger input with larger dilation values (to avoid invalid config) - int w = d * width; - int h = d * height; - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, h, w } : new long[] { minibatchSize, h, w, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(cm).list().layer(new ConvolutionLayer.Builder().name("layer 0").kernelSize(k, k).stride(s, s).dilation(d, d).dataFormat(format).nIn(inputDepth).nOut(2).build()); - if (subsampling) { - b.layer(new SubsamplingLayer.Builder().poolingType(SubsamplingLayer.PoolingType.MAX).kernelSize(k, k).stride(s, s).dilation(d, d).dataFormat(format).build()); - } else { - b.layer(new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(k, k).stride(s, s).dilation(d, d).dataFormat(format).build()); - } - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(h, w, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int i = 0; i < net.getLayers().length; i++) { - System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - } - String msg = (subsampling ? "subsampling" : "conv") + " - mb=" + minibatchSize + ", k=" + k + ", s=" + s + ", d=" + d + ", cm=" + cm; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - - @DisplayName("Test Cropping 2 D Layer") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testCropping2DLayer(CNN2DFormat format,Nd4jBackend backend) { - Nd4j.getRandom().setSeed(12345); - int nOut = 2; - int width = 12; - int height = 11; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int[][] cropTestCases = new int[][] { { 0, 0, 0, 0 }, { 1, 1, 0, 0 }, { 2, 2, 2, 2 }, { 1, 2, 3, 4 } }; - int[] inputDepths = { 1, 2, 3, 2 }; - int[] minibatchSizes = { 2, 1, 3, 2 }; - boolean nchw = format == CNN2DFormat.NCHW; - for (int i = 0; i < cropTestCases.length; i++) { - int inputDepth = inputDepths[i]; - int minibatchSize = minibatchSizes[i]; - int[] crop = cropTestCases[i]; - long[] inShape = nchw ? new long[] { minibatchSize, inputDepth, height, width } : new long[] { minibatchSize, height, width, inputDepth }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).convolutionMode(ConvolutionMode.Same).weightInit(new NormalDistribution(0, 1)).list().layer(new ConvolutionLayer.Builder(kernel, stride, padding).dataFormat(format).nIn(inputDepth).nOut(2).build()).layer(new Cropping2D.Builder(crop).dataFormat(format).build()).layer(new ConvolutionLayer.Builder(kernel, stride, padding).dataFormat(format).nIn(2).nOut(2).build()).layer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.AVG).kernelSize(3, 3).stride(3, 3).dataFormat(format).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(height, width, inputDepth, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // Check cropping activation shape - org.deeplearning4j.nn.layers.convolution.Cropping2DLayer cl = (org.deeplearning4j.nn.layers.convolution.Cropping2DLayer) net.getLayer(1); - long[] expShape; - if (nchw) { - expShape = new long[] { minibatchSize, inputDepth, height - crop[0] - crop[1], width - crop[2] - crop[3] }; - } else { - expShape = new long[] { minibatchSize, height - crop[0] - crop[1], width - crop[2] - crop[3], inputDepth }; - } - INDArray out = cl.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(expShape, out.shape()); - String msg = format + " - minibatch=" + minibatchSize + ", channels=" + inputDepth + ", zeroPad = " + Arrays.toString(crop); - if (PRINT_RESULTS) { - System.out.println(msg); - } - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(160)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - - @DisplayName("Test Depthwise Conv 2 D") - @ParameterizedTest - @MethodSource("org.deeplearning4j.gradientcheck.CNNGradientCheckTest#params") - void testDepthwiseConv2D(CNN2DFormat format,Nd4jBackend backendt) { - int nIn = 3; - int depthMultiplier = 2; - int nOut = nIn * depthMultiplier; - int width = 5; - int height = 5; - Nd4j.getRandom().setSeed(12345); - int[] ks = new int[] { 1, 3, 3, 1, 3 }; - int[] ss = new int[] { 1, 1, 1, 2, 2 }; - ConvolutionMode[] cms = new ConvolutionMode[] { Truncate, Truncate, Truncate, Truncate, Truncate }; - int[] mb = new int[] { 1, 1, 1, 3, 3 }; - boolean nchw = format == CNN2DFormat.NCHW; - for (int t = 0; t < ks.length; t++) { - int k = ks[t]; - int s = ss[t]; - ConvolutionMode cm = cms[t]; - int minibatchSize = mb[t]; - long[] inShape = nchw ? new long[] { minibatchSize, nIn, height, width } : new long[] { minibatchSize, height, width, nIn }; - INDArray input = Nd4j.rand(DataType.DOUBLE, inShape); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(cm).list().layer(new Convolution2D.Builder().kernelSize(1, 1).stride(1, 1).nIn(nIn).nOut(nIn).dataFormat(format).build()).layer(new DepthwiseConvolution2D.Builder().name("depth-wise conv 2D layer").cudnnAllowFallback(false).kernelSize(k, k).stride(s, s).depthMultiplier(depthMultiplier).nIn(nIn).build()); - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(height, width, nIn, format)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int i = 0; i < net.getLayers().length; i++) { - System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - } - String msg = " - mb=" + minibatchSize + ", k=" + k + ", nIn=" + nIn + ", depthMul=" + depthMultiplier + ", s=" + s + ", cm=" + cm; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(256)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CapsnetGradientCheckTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CapsnetGradientCheckTest.java deleted file mode 100644 index 2f3e0b05b..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CapsnetGradientCheckTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.gradientcheck; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.UniformDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ActivationLayer; -import org.deeplearning4j.nn.conf.layers.CapsuleLayer; -import org.deeplearning4j.nn.conf.layers.CapsuleStrengthLayer; -import org.deeplearning4j.nn.conf.layers.LossLayer; -import org.deeplearning4j.nn.conf.layers.PrimaryCapsules; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInitDistribution; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.impl.ActivationSoftmax; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.impl.LossNegativeLogLikelihood; -import java.util.Random; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Disabled -@DisplayName("Capsnet Gradient Check Test") -@Tag(TagNames.NDARRAY_ETL) -@Tag(TagNames.TRAINING) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -class CapsnetGradientCheckTest extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - - @Test - @DisplayName("Test Caps Net") - void testCapsNet() { - int[] minibatchSizes = { 8, 16 }; - int width = 6; - int height = 6; - int inputDepth = 4; - int[] primaryCapsDims = { 2, 4 }; - int[] primaryCapsChannels = { 8 }; - int[] capsules = { 5 }; - int[] capsuleDims = { 4, 8 }; - int[] routings = { 1 }; - Nd4j.getRandom().setSeed(12345); - for (int routing : routings) { - for (int primaryCapsDim : primaryCapsDims) { - for (int primarpCapsChannel : primaryCapsChannels) { - for (int capsule : capsules) { - for (int capsuleDim : capsuleDims) { - for (int minibatchSize : minibatchSizes) { - INDArray input = Nd4j.rand(minibatchSize, inputDepth * height * width).mul(10).reshape(-1, inputDepth, height, width); - INDArray labels = Nd4j.zeros(minibatchSize, capsule); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % capsule }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).seed(123).updater(new NoOp()).weightInit(new WeightInitDistribution(new UniformDistribution(-6, 6))).list().layer(new PrimaryCapsules.Builder(primaryCapsDim, primarpCapsChannel).kernelSize(3, 3).stride(2, 2).build()).layer(new CapsuleLayer.Builder(capsule, capsuleDim, routing).build()).layer(new CapsuleStrengthLayer.Builder().build()).layer(new ActivationLayer.Builder(new ActivationSoftmax()).build()).layer(new LossLayer.Builder(new LossNegativeLogLikelihood()).build()).setInputType(InputType.convolutional(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - for (int i = 0; i < 4; i++) { - System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - } - String msg = "minibatch=" + minibatchSize + ", PrimaryCaps: " + primarpCapsChannel + " channels, " + primaryCapsDim + " dimensions, Capsules: " + capsule + " capsules with " + capsuleDim + " dimensions and " + routing + " routings"; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(100)); - assertTrue(gradOK,msg); - TestUtils.testModelSerialization(net); - } - } - } - } - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/adapters/ArgmaxAdapterTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/adapters/ArgmaxAdapterTest.java deleted file mode 100644 index f0c0d819c..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/adapters/ArgmaxAdapterTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.adapters; - -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.factory.Nd4j; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Argmax Adapter Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class ArgmaxAdapterTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Softmax _ 2 D _ 1") - void testSoftmax_2D_1() { - val in = new double[][] { { 1, 3, 2 }, { 4, 5, 6 } }; - val adapter = new ArgmaxAdapter(); - val result = adapter.apply(Nd4j.create(in)); - assertArrayEquals(new int[] { 1, 2 }, result); - } - - @Test - @DisplayName("Test Softmax _ 1 D _ 1") - void testSoftmax_1D_1() { - val in = new double[] { 1, 3, 2 }; - val adapter = new ArgmaxAdapter(); - val result = adapter.apply(Nd4j.create(in)); - assertArrayEquals(new int[] { 1 }, result); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/adapters/Regression2dAdapterTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/adapters/Regression2dAdapterTest.java deleted file mode 100644 index 06b1fcae6..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/adapters/Regression2dAdapterTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.adapters; - -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.util.ArrayUtil; -import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; - -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Regression 2 d Adapter Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class Regression2dAdapterTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Regression Adapter _ 2 D _ 1") - void testRegressionAdapter_2D_1() throws Exception { - val in = new double[][] { { 1, 2, 3 }, { 4, 5, 6 } }; - val adapter = new Regression2dAdapter(); - val result = adapter.apply(Nd4j.create(in)); - assertArrayEquals(ArrayUtil.flatten(in), ArrayUtil.flatten(result), 1e-5); - } - - @Test - @DisplayName("Test Regression Adapter _ 2 D _ 2") - void testRegressionAdapter_2D_2() throws Exception { - val in = new double[] { 1, 2, 3 }; - val adapter = new Regression2dAdapter(); - val result = adapter.apply(Nd4j.create(in)); - assertArrayEquals(in, ArrayUtil.flatten(result), 1e-5); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/ComputationGraphConfigurationTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/ComputationGraphConfigurationTest.java deleted file mode 100644 index 12de3949e..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/ComputationGraphConfigurationTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.exception.DL4JInvalidConfigException; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.graph.ElementWiseVertex; -import org.deeplearning4j.nn.conf.graph.GraphVertex; -import org.deeplearning4j.nn.conf.graph.MergeVertex; -import org.deeplearning4j.nn.conf.graph.SubsetVertex; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.inputs.InvalidInputTypeException; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.recurrent.Bidirectional; -import org.deeplearning4j.nn.conf.memory.MemoryReport; -import org.deeplearning4j.nn.conf.misc.TestGraphVertex; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToCnnPreProcessor; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Computation Graph Configuration Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class ComputationGraphConfigurationTest extends BaseDL4JTest { - - @Test - @DisplayName("Test JSON Basic") - void testJSONBasic() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).dist(new NormalDistribution(0, 1)).updater(new NoOp()).graphBuilder().addInputs("input").appendLayer("firstLayer", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.TANH).build()).addLayer("outputLayer", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(5).nOut(3).build(), "firstLayer").setOutputs("outputLayer").build(); - String json = conf.toJson(); - ComputationGraphConfiguration conf2 = ComputationGraphConfiguration.fromJson(json); - assertEquals(json, conf2.toJson()); - assertEquals(conf, conf2); - } - - @Test - @DisplayName("Test JSON Basic 2") - void testJSONBasic2() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder().addInputs("input").addLayer("cnn1", new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(5).build(), "input").addLayer("cnn2", new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(5).build(), "input").addLayer("max1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).build(), "cnn1", "cnn2").addLayer("dnn1", new DenseLayer.Builder().nOut(7).build(), "max1").addLayer("max2", new SubsamplingLayer.Builder().build(), "max1").addLayer("output", new OutputLayer.Builder().nIn(7).nOut(10).activation(Activation.SOFTMAX).build(), "dnn1", "max2").setOutputs("output").inputPreProcessor("cnn1", new FeedForwardToCnnPreProcessor(32, 32, 3)).inputPreProcessor("cnn2", new FeedForwardToCnnPreProcessor(32, 32, 3)).inputPreProcessor("dnn1", new CnnToFeedForwardPreProcessor(8, 8, 5)).build(); - String json = conf.toJson(); - ComputationGraphConfiguration conf2 = ComputationGraphConfiguration.fromJson(json); - assertEquals(json, conf2.toJson()); - assertEquals(conf, conf2); - } - - @Test - @DisplayName("Test JSON With Graph Nodes") - void testJSONWithGraphNodes() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder().addInputs("input1", "input2").addLayer("cnn1", new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(5).build(), "input1").addLayer("cnn2", new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(5).build(), "input2").addVertex("merge1", new MergeVertex(), "cnn1", "cnn2").addVertex("subset1", new SubsetVertex(0, 1), "merge1").addLayer("dense1", new DenseLayer.Builder().nIn(20).nOut(5).build(), "subset1").addLayer("dense2", new DenseLayer.Builder().nIn(20).nOut(5).build(), "subset1").addVertex("add", new ElementWiseVertex(ElementWiseVertex.Op.Add), "dense1", "dense2").addLayer("out", new OutputLayer.Builder().nIn(1).nOut(1).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build(), "add").setOutputs("out").build(); - String json = conf.toJson(); - // System.out.println(json); - ComputationGraphConfiguration conf2 = ComputationGraphConfiguration.fromJson(json); - assertEquals(json, conf2.toJson()); - assertEquals(conf, conf2); - } - - @Test - @DisplayName("Test Invalid Configurations") - void testInvalidConfigurations() { - // Test no inputs for a layer: - try { - new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input1").addLayer("dense1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "input1").addLayer("out", new OutputLayer.Builder().nIn(2).nOut(2).build()).setOutputs("out").build(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - exception is good - log.info(e.toString()); - } - // Use appendLayer on first layer - try { - new NeuralNetConfiguration.Builder().graphBuilder().appendLayer("dense1", new DenseLayer.Builder().nIn(2).nOut(2).build()).addLayer("out", new OutputLayer.Builder().nIn(2).nOut(2).build()).setOutputs("out").build(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - exception is good - log.info(e.toString()); - } - // Test no network inputs - try { - new NeuralNetConfiguration.Builder().graphBuilder().addLayer("dense1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "input1").addLayer("out", new OutputLayer.Builder().nIn(2).nOut(2).build(), "dense1").setOutputs("out").build(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - exception is good - log.info(e.toString()); - } - // Test no network outputs - try { - new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input1").addLayer("dense1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "input1").addLayer("out", new OutputLayer.Builder().nIn(2).nOut(2).build(), "dense1").build(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - exception is good - log.info(e.toString()); - } - // Test: invalid input - try { - new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input1").addLayer("dense1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "input1").addLayer("out", new OutputLayer.Builder().nIn(2).nOut(2).build(), "thisDoesntExist").setOutputs("out").build(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - exception is good - log.info(e.toString()); - } - // Test: graph with cycles - try { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input1").addLayer("dense1", new DenseLayer.Builder().nIn(2).nOut(2).build(), "input1", "dense3").addLayer("dense2", new DenseLayer.Builder().nIn(2).nOut(2).build(), "dense1").addLayer("dense3", new DenseLayer.Builder().nIn(2).nOut(2).build(), "dense2").addLayer("out", new OutputLayer.Builder().nIn(2).nOut(2).lossFunction(LossFunctions.LossFunction.MSE).build(), "dense1").setOutputs("out").build(); - // Cycle detection happens in ComputationGraph.init() - ComputationGraph graph = new ComputationGraph(conf); - graph.init(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - exception is good - log.info(e.toString()); - } - // Test: input != inputType count mismatch - try { - new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input1", "input2").setInputTypes(new InputType.InputTypeRecurrent(10, 12)).addLayer("cnn1", new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(5).build(), "input1").addLayer("cnn2", new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(5).build(), "input2").addVertex("merge1", new MergeVertex(), "cnn1", "cnn2").addVertex("subset1", new SubsetVertex(0, 1), "merge1").addLayer("dense1", new DenseLayer.Builder().nIn(20).nOut(5).build(), "subset1").addLayer("dense2", new DenseLayer.Builder().nIn(20).nOut(5).build(), "subset1").addVertex("add", new ElementWiseVertex(ElementWiseVertex.Op.Add), "dense1", "dense2").addLayer("out", new OutputLayer.Builder().nIn(1).nOut(1).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build(), "add").setOutputs("out").build(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalArgumentException e) { - // OK - exception is good - log.info(e.toString()); - } - } - - @Test - @DisplayName("Test Configuration With Runtime JSON Subtypes") - void testConfigurationWithRuntimeJSONSubtypes() { - // Idea: suppose someone wants to use a ComputationGraph with a custom GraphVertex - // (i.e., one not built into DL4J). Check that this works for JSON serialization - // using runtime/reflection subtype mechanism in ComputationGraphConfiguration.fromJson() - // Check a standard GraphVertex implementation, plus a static inner graph vertex - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").addVertex("test", new TestGraphVertex(3, 7), "in").addVertex("test2", new StaticInnerGraphVertex(4, 5), "in").setOutputs("test", "test2").build(); - String json = conf.toJson(); - // System.out.println(json); - ComputationGraphConfiguration conf2 = ComputationGraphConfiguration.fromJson(json); - assertEquals(conf, conf2); - assertEquals(json, conf2.toJson()); - TestGraphVertex tgv = (TestGraphVertex) conf2.getVertices().get("test"); - assertEquals(3, tgv.getFirstVal()); - assertEquals(7, tgv.getSecondVal()); - StaticInnerGraphVertex sigv = (StaticInnerGraphVertex) conf.getVertices().get("test2"); - assertEquals(4, sigv.getFirstVal()); - assertEquals(5, sigv.getSecondVal()); - } - - @Test - @DisplayName("Test Output Order Doesnt Change When Cloning") - void testOutputOrderDoesntChangeWhenCloning() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").addLayer("out1", new OutputLayer.Builder().nIn(1).nOut(1).build(), "in").addLayer("out2", new OutputLayer.Builder().nIn(1).nOut(1).build(), "in").addLayer("out3", new OutputLayer.Builder().nIn(1).nOut(1).build(), "in").validateOutputLayerConfig(false).setOutputs("out1", "out2", "out3").build(); - ComputationGraphConfiguration cloned = conf.clone(); - String json = conf.toJson(); - String jsonCloned = cloned.toJson(); - assertEquals(json, jsonCloned); - } - - @Test - @DisplayName("Test Allow Disconnected Layers") - void testAllowDisconnectedLayers() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").addLayer("bidirectional", new Bidirectional(new LSTM.Builder().activation(Activation.TANH).nOut(10).build()), "in").addLayer("out", new RnnOutputLayer.Builder().nOut(6).lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).build(), "bidirectional").addLayer("disconnected_layer", new Bidirectional(new LSTM.Builder().activation(Activation.TANH).nOut(10).build()), "in").setOutputs("out").setInputTypes(new InputType.InputTypeRecurrent(10, 12)).allowDisconnected(true).build(); - ComputationGraph graph = new ComputationGraph(conf); - graph.init(); - } - - @Test - @DisplayName("Test Bidirectional Graph Summary") - void testBidirectionalGraphSummary() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").addLayer("bidirectional", new Bidirectional(new LSTM.Builder().activation(Activation.TANH).nOut(10).build()), "in").addLayer("out", new RnnOutputLayer.Builder().nOut(6).lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).build(), "bidirectional").setOutputs("out").setInputTypes(new InputType.InputTypeRecurrent(10, 12)).build(); - ComputationGraph graph = new ComputationGraph(conf); - graph.init(); - graph.summary(); - } - - @AllArgsConstructor - @NoArgsConstructor - @Data - @EqualsAndHashCode(callSuper = false) - @DisplayName("Static Inner Graph Vertex") - static class StaticInnerGraphVertex extends GraphVertex { - - private int firstVal; - - private int secondVal; - - @Override - public GraphVertex clone() { - return new TestGraphVertex(firstVal, secondVal); - } - - @Override - public long numParams(boolean backprop) { - return 0; - } - - @Override - public int minVertexInputs() { - return 1; - } - - @Override - public int maxVertexInputs() { - return 1; - } - - @Override - public org.deeplearning4j.nn.graph.vertex.GraphVertex instantiate(ComputationGraph graph, String name, int idx, INDArray paramsView, boolean initializeParams, DataType networkDatatype) { - throw new UnsupportedOperationException("Not supported"); - } - - @Override - public InputType getOutputType(int layerIndex, InputType... vertexInputs) throws InvalidInputTypeException { - throw new UnsupportedOperationException(); - } - - @Override - public MemoryReport getMemoryReport(InputType... inputTypes) { - throw new UnsupportedOperationException(); - } - } - - @Test - @DisplayName("Test Invalid Output Layer") - void testInvalidOutputLayer() { - /* - Test case (invalid configs) - 1. nOut=1 + softmax - 2. mcxent + tanh - 3. xent + softmax - 4. xent + relu - 5. mcxent + sigmoid - */ - LossFunctions.LossFunction[] lf = new LossFunctions.LossFunction[] { LossFunctions.LossFunction.MCXENT, LossFunctions.LossFunction.MCXENT, LossFunctions.LossFunction.XENT, LossFunctions.LossFunction.XENT, LossFunctions.LossFunction.MCXENT }; - int[] nOut = new int[] { 1, 3, 3, 3, 3 }; - Activation[] activations = new Activation[] { Activation.SOFTMAX, Activation.TANH, Activation.SOFTMAX, Activation.RELU, Activation.SIGMOID }; - for (int i = 0; i < lf.length; i++) { - for (boolean lossLayer : new boolean[] { false, true }) { - for (boolean validate : new boolean[] { true, false }) { - String s = "nOut=" + nOut[i] + ",lossFn=" + lf[i] + ",lossLayer=" + lossLayer + ",validate=" + validate; - if (nOut[i] == 1 && lossLayer) - // nOuts are not availabel in loss layer, can't expect it to detect this case - continue; - try { - new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").layer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in").layer("1", !lossLayer ? new OutputLayer.Builder().nIn(10).nOut(nOut[i]).activation(activations[i]).lossFunction(lf[i]).build() : new LossLayer.Builder().activation(activations[i]).lossFunction(lf[i]).build(), "0").setOutputs("1").validateOutputLayerConfig(validate).build(); - if (validate) { - fail("Expected exception: " + s); - } - } catch (DL4JInvalidConfigException e) { - if (validate) { - assertTrue(e.getMessage().toLowerCase().contains("invalid output"),s); - } else { - fail("Validation should not be enabled"); - } - } - } - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/JsonTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/JsonTest.java deleted file mode 100644 index 991a16d19..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/JsonTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.LossLayer; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.lossfunctions.ILossFunction; -import org.nd4j.linalg.lossfunctions.impl.*; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Json Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class JsonTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Json Loss Functions") - void testJsonLossFunctions() { - ILossFunction[] lossFunctions = new ILossFunction[] { new LossBinaryXENT(), new LossBinaryXENT(), new LossCosineProximity(), new LossHinge(), new LossKLD(), new LossKLD(), new LossL1(), new LossL1(), new LossL2(), new LossL2(), new LossMAE(), new LossMAE(), new LossMAPE(), new LossMAPE(), new LossMCXENT(), new LossMSE(), new LossMSE(), new LossMSLE(), new LossMSLE(), new LossNegativeLogLikelihood(), new LossNegativeLogLikelihood(), new LossPoisson(), new LossSquaredHinge(), new LossFMeasure(), new LossFMeasure(2.0) }; - Activation[] outputActivationFn = new Activation[] { // xent - Activation.SIGMOID, // xent - Activation.SIGMOID, // cosine - Activation.TANH, // hinge -> trying to predict 1 or -1 - Activation.TANH, // kld -> probab so should be between 0 and 1 - Activation.SIGMOID, // kld + softmax - Activation.SOFTMAX, // l1 - Activation.TANH, // l1 + softmax - Activation.SOFTMAX, // l2 - Activation.TANH, // l2 + softmax - Activation.SOFTMAX, // mae - Activation.IDENTITY, // mae + softmax - Activation.SOFTMAX, // mape - Activation.IDENTITY, // mape + softmax - Activation.SOFTMAX, // mcxent - Activation.SOFTMAX, // mse - Activation.IDENTITY, // mse + softmax - Activation.SOFTMAX, // msle - requires positive labels/activations due to log - Activation.SIGMOID, // msle + softmax - Activation.SOFTMAX, // nll - Activation.SIGMOID, // nll + softmax - Activation.SOFTMAX, // poisson - requires positive predictions due to log... not sure if this is the best option - Activation.SIGMOID, // squared hinge - Activation.TANH, // f-measure (binary, single sigmoid output) - Activation.SIGMOID, // f-measure (binary, 2-label softmax output) - Activation.SOFTMAX }; - int[] nOut = new int[] { // xent - 1, // xent - 3, // cosine - 5, // hinge - 3, // kld - 3, // kld + softmax - 3, // l1 - 3, // l1 + softmax - 3, // l2 - 3, // l2 + softmax - 3, // mae - 3, // mae + softmax - 3, // mape - 3, // mape + softmax - 3, // mcxent - 3, // mse - 3, // mse + softmax - 3, // msle - 3, // msle + softmax - 3, // nll - 3, // nll + softmax - 3, // poisson - 3, // squared hinge - 3, // f-measure (binary, single sigmoid output) - 1, // f-measure (binary, 2-label softmax output) - 2 }; - for (int i = 0; i < lossFunctions.length; i++) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(Updater.ADAM).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(nOut[i]).activation(Activation.TANH).build()).layer(1, new LossLayer.Builder().lossFunction(lossFunctions[i]).activation(outputActivationFn[i]).build()).validateOutputLayerConfig(false).build(); - String json = conf.toJson(); - String yaml = conf.toYaml(); - MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json); - MultiLayerConfiguration fromYaml = MultiLayerConfiguration.fromYaml(yaml); - assertEquals(conf, fromJson); - assertEquals(conf, fromYaml); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/MultiLayerNeuralNetConfigurationTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/MultiLayerNeuralNetConfigurationTest.java deleted file mode 100644 index c3f285f4d..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/MultiLayerNeuralNetConfigurationTest.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.exception.DL4JInvalidConfigException; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.weightnoise.DropConnect; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.io.*; -import java.util.Arrays; -import java.util.Properties; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Multi Layer Neural Net Configuration Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class MultiLayerNeuralNetConfigurationTest extends BaseDL4JTest { - - @TempDir - public Path testDir; - - @Test - @DisplayName("Test Json") - void testJson() throws Exception { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().dist(new NormalDistribution(1, 1e-1)).build()).inputPreProcessor(0, new CnnToFeedForwardPreProcessor()).build(); - String json = conf.toJson(); - MultiLayerConfiguration from = MultiLayerConfiguration.fromJson(json); - assertEquals(conf.getConf(0), from.getConf(0)); - Properties props = new Properties(); - props.put("json", json); - String key = props.getProperty("json"); - assertEquals(json, key); - File f = testDir.resolve("props").toFile(); - f.deleteOnExit(); - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f)); - props.store(bos, ""); - bos.flush(); - bos.close(); - BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f)); - Properties props2 = new Properties(); - props2.load(bis); - bis.close(); - assertEquals(props2.getProperty("json"), props.getProperty("json")); - String json2 = props2.getProperty("json"); - MultiLayerConfiguration conf3 = MultiLayerConfiguration.fromJson(json2); - assertEquals(conf.getConf(0), conf3.getConf(0)); - } - - @Test - @DisplayName("Test Convnet Json") - void testConvnetJson() { - final int numRows = 76; - final int numColumns = 76; - int nChannels = 3; - int outputNum = 6; - int seed = 123; - // setup the network - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).l1(1e-1).l2(2e-4).weightNoise(new DropConnect(0.5)).miniBatch(true).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list().layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).build()).layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).build()).layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()).layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(numRows, numColumns, nChannels)); - MultiLayerConfiguration conf = builder.build(); - String json = conf.toJson(); - MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, conf2); - } - - @Test - @DisplayName("Test Upsampling Convnet Json") - void testUpsamplingConvnetJson() { - final int numRows = 76; - final int numColumns = 76; - int nChannels = 3; - int outputNum = 6; - int seed = 123; - // setup the network - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).l1(1e-1).l2(2e-4).dropOut(0.5).miniBatch(true).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list().layer(new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(new Upsampling2D.Builder().size(2).build()).layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(new Upsampling2D.Builder().size(2).build()).layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()).layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(numRows, numColumns, nChannels)); - MultiLayerConfiguration conf = builder.build(); - String json = conf.toJson(); - MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, conf2); - } - - @Test - @DisplayName("Test Global Pooling Json") - void testGlobalPoolingJson() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()).dist(new NormalDistribution(0, 1.0)).seed(12345L).list().layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nOut(5).build()).layer(1, new GlobalPoolingLayer.Builder().poolingType(PoolingType.PNORM).pnorm(3).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(3).build()).setInputType(InputType.convolutional(32, 32, 1)).build(); - String str = conf.toJson(); - MultiLayerConfiguration fromJson = conf.fromJson(str); - assertEquals(conf, fromJson); - } - - @Test - @DisplayName("Test Yaml") - void testYaml() throws Exception { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().dist(new NormalDistribution(1, 1e-1)).build()).inputPreProcessor(0, new CnnToFeedForwardPreProcessor()).build(); - String json = conf.toYaml(); - MultiLayerConfiguration from = MultiLayerConfiguration.fromYaml(json); - assertEquals(conf.getConf(0), from.getConf(0)); - Properties props = new Properties(); - props.put("json", json); - String key = props.getProperty("json"); - assertEquals(json, key); - File f = testDir.resolve("props").toFile(); - f.deleteOnExit(); - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f)); - props.store(bos, ""); - bos.flush(); - bos.close(); - BufferedInputStream bis = new BufferedInputStream(new FileInputStream(f)); - Properties props2 = new Properties(); - props2.load(bis); - bis.close(); - assertEquals(props2.getProperty("json"), props.getProperty("json")); - String yaml = props2.getProperty("json"); - MultiLayerConfiguration conf3 = MultiLayerConfiguration.fromYaml(yaml); - assertEquals(conf.getConf(0), conf3.getConf(0)); - } - - @Test - @DisplayName("Test Clone") - void testClone() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().build()).layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).build()).inputPreProcessor(1, new CnnToFeedForwardPreProcessor()).build(); - MultiLayerConfiguration conf2 = conf.clone(); - assertEquals(conf, conf2); - assertNotSame(conf, conf2); - assertNotSame(conf.getConfs(), conf2.getConfs()); - for (int i = 0; i < conf.getConfs().size(); i++) { - assertNotSame(conf.getConf(i), conf2.getConf(i)); - } - assertNotSame(conf.getInputPreProcessors(), conf2.getInputPreProcessors()); - for (Integer layer : conf.getInputPreProcessors().keySet()) { - assertNotSame(conf.getInputPreProcess(layer), conf2.getInputPreProcess(layer)); - } - } - - @Test - @DisplayName("Test Random Weight Init") - void testRandomWeightInit() { - MultiLayerNetwork model1 = new MultiLayerNetwork(getConf()); - model1.init(); - Nd4j.getRandom().setSeed(12345L); - MultiLayerNetwork model2 = new MultiLayerNetwork(getConf()); - model2.init(); - float[] p1 = model1.params().data().asFloat(); - float[] p2 = model2.params().data().asFloat(); - System.out.println(Arrays.toString(p1)); - System.out.println(Arrays.toString(p2)); - assertArrayEquals(p1, p2, 0.0f); - } - - @Test - @DisplayName("Test Training Listener") - void testTrainingListener() { - MultiLayerNetwork model1 = new MultiLayerNetwork(getConf()); - model1.init(); - model1.addListeners(new ScoreIterationListener(1)); - MultiLayerNetwork model2 = new MultiLayerNetwork(getConf()); - model2.addListeners(new ScoreIterationListener(1)); - model2.init(); - Layer[] l1 = model1.getLayers(); - for (int i = 0; i < l1.length; i++) assertTrue(l1[i].getListeners() != null && l1[i].getListeners().size() == 1); - Layer[] l2 = model2.getLayers(); - for (int i = 0; i < l2.length; i++) assertTrue(l2[i].getListeners() != null && l2[i].getListeners().size() == 1); - } - - private static MultiLayerConfiguration getConf() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345l).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).dist(new NormalDistribution(0, 1)).build()).layer(1, new OutputLayer.Builder().nIn(2).nOut(1).activation(Activation.TANH).dist(new NormalDistribution(0, 1)).lossFunction(LossFunctions.LossFunction.MSE).build()).build(); - return conf; - } - - @Test - @DisplayName("Test Invalid Config") - void testInvalidConfig() { - try { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - log.error("", e); - } catch (Throwable e) { - log.error("", e); - fail("Unexpected exception thrown for invalid config"); - } - try { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(1, new DenseLayer.Builder().nIn(3).nOut(4).build()).layer(2, new OutputLayer.Builder().nIn(4).nOut(5).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - log.info(e.toString()); - } catch (Throwable e) { - log.error("", e); - fail("Unexpected exception thrown for invalid config"); - } - try { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()).layer(2, new OutputLayer.Builder().nIn(4).nOut(5).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - fail("No exception thrown for invalid configuration"); - } catch (IllegalStateException e) { - // OK - log.info(e.toString()); - } catch (Throwable e) { - log.error("", e); - fail("Unexpected exception thrown for invalid config"); - } - } - - @Test - @DisplayName("Test List Overloads") - void testListOverloads() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()).layer(1, new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DenseLayer dl = (DenseLayer) conf.getConf(0).getLayer(); - assertEquals(3, dl.getNIn()); - assertEquals(4, dl.getNOut()); - OutputLayer ol = (OutputLayer) conf.getConf(1).getLayer(); - assertEquals(4, ol.getNIn()); - assertEquals(5, ol.getNOut()); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build()).layer(1, new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder().seed(12345).list(new DenseLayer.Builder().nIn(3).nOut(4).build(), new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net3 = new MultiLayerNetwork(conf3); - net3.init(); - assertEquals(conf, conf2); - assertEquals(conf, conf3); - } - - @Test - @DisplayName("Test Bias Lr") - void testBiasLr() { - // setup the network - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new Adam(1e-2)).biasUpdater(new Adam(0.5)).list().layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()).layer(2, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)).build(); - org.deeplearning4j.nn.conf.layers.BaseLayer l0 = (BaseLayer) conf.getConf(0).getLayer(); - org.deeplearning4j.nn.conf.layers.BaseLayer l1 = (BaseLayer) conf.getConf(1).getLayer(); - org.deeplearning4j.nn.conf.layers.BaseLayer l2 = (BaseLayer) conf.getConf(2).getLayer(); - org.deeplearning4j.nn.conf.layers.BaseLayer l3 = (BaseLayer) conf.getConf(3).getLayer(); - assertEquals(0.5, ((Adam) l0.getUpdaterByParam("b")).getLearningRate(), 1e-6); - assertEquals(1e-2, ((Adam) l0.getUpdaterByParam("W")).getLearningRate(), 1e-6); - assertEquals(0.5, ((Adam) l1.getUpdaterByParam("b")).getLearningRate(), 1e-6); - assertEquals(1e-2, ((Adam) l1.getUpdaterByParam("W")).getLearningRate(), 1e-6); - assertEquals(0.5, ((Adam) l2.getUpdaterByParam("b")).getLearningRate(), 1e-6); - assertEquals(1e-2, ((Adam) l2.getUpdaterByParam("W")).getLearningRate(), 1e-6); - assertEquals(0.5, ((Adam) l3.getUpdaterByParam("b")).getLearningRate(), 1e-6); - assertEquals(1e-2, ((Adam) l3.getUpdaterByParam("W")).getLearningRate(), 1e-6); - } - - @Test - @DisplayName("Test Invalid Output Layer") - void testInvalidOutputLayer() { - /* - Test case (invalid configs) - 1. nOut=1 + softmax - 2. mcxent + tanh - 3. xent + softmax - 4. xent + relu - 5. mcxent + sigmoid - */ - LossFunctions.LossFunction[] lf = new LossFunctions.LossFunction[] { LossFunctions.LossFunction.MCXENT, LossFunctions.LossFunction.MCXENT, LossFunctions.LossFunction.XENT, LossFunctions.LossFunction.XENT, LossFunctions.LossFunction.MCXENT }; - int[] nOut = new int[] { 1, 3, 3, 3, 3 }; - Activation[] activations = new Activation[] { Activation.SOFTMAX, Activation.TANH, Activation.SOFTMAX, Activation.RELU, Activation.SIGMOID }; - for (int i = 0; i < lf.length; i++) { - for (boolean lossLayer : new boolean[] { false, true }) { - for (boolean validate : new boolean[] { true, false }) { - String s = "nOut=" + nOut[i] + ",lossFn=" + lf[i] + ",lossLayer=" + lossLayer + ",validate=" + validate; - if (nOut[i] == 1 && lossLayer) - // nOuts are not availabel in loss layer, can't expect it to detect this case - continue; - try { - new NeuralNetConfiguration.Builder().list().layer(new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(!lossLayer ? new OutputLayer.Builder().nIn(10).nOut(nOut[i]).activation(activations[i]).lossFunction(lf[i]).build() : new LossLayer.Builder().activation(activations[i]).lossFunction(lf[i]).build()).validateOutputLayerConfig(validate).build(); - if (validate) { - fail("Expected exception: " + s); - } - } catch (DL4JInvalidConfigException e) { - if (validate) { - assertTrue(e.getMessage().toLowerCase().contains("invalid output"),s); - } else { - fail("Validation should not be enabled"); - } - } - } - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java deleted file mode 100644 index 2f799679b..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.layers.BaseLayer; -import org.deeplearning4j.nn.conf.layers.BatchNormalization; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; -import org.deeplearning4j.nn.conf.stepfunctions.DefaultStepFunction; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.deeplearning4j.nn.weights.*; -import org.deeplearning4j.optimize.api.ConvexOptimizer; -import org.deeplearning4j.optimize.solvers.StochasticGradientDescent; -import org.deeplearning4j.optimize.stepfunctions.NegativeDefaultStepFunction; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.scalar.LeakyReLU; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.learning.regularization.Regularization; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Neural Net Configuration Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class NeuralNetConfigurationTest extends BaseDL4JTest { - - final DataSet trainingSet = createData(); - - public DataSet createData() { - int numFeatures = 40; - // have to be at least two or else output layer gradient is a scalar and cause exception - INDArray input = Nd4j.create(2, numFeatures); - INDArray labels = Nd4j.create(2, 2); - INDArray row0 = Nd4j.create(1, numFeatures); - row0.assign(0.1); - input.putRow(0, row0); - // set the 4th column - labels.put(0, 1, 1); - INDArray row1 = Nd4j.create(1, numFeatures); - row1.assign(0.2); - input.putRow(1, row1); - // set the 2nd column - labels.put(1, 0, 1); - return new DataSet(input, labels); - } - - @Test - @DisplayName("Test Json") - void testJson() { - NeuralNetConfiguration conf = getConfig(1, 1, new WeightInitXavier(), true); - String json = conf.toJson(); - NeuralNetConfiguration read = NeuralNetConfiguration.fromJson(json); - assertEquals(conf, read); - } - - @Test - @DisplayName("Test Yaml") - void testYaml() { - NeuralNetConfiguration conf = getConfig(1, 1, new WeightInitXavier(), true); - String json = conf.toYaml(); - NeuralNetConfiguration read = NeuralNetConfiguration.fromYaml(json); - assertEquals(conf, read); - } - - @Test - @DisplayName("Test Clone") - void testClone() { - NeuralNetConfiguration conf = getConfig(1, 1, new WeightInitUniform(), true); - BaseLayer bl = (BaseLayer) conf.getLayer(); - conf.setStepFunction(new DefaultStepFunction()); - NeuralNetConfiguration conf2 = conf.clone(); - assertEquals(conf, conf2); - assertNotSame(conf, conf2); - assertNotSame(conf.getLayer(), conf2.getLayer()); - assertNotSame(conf.getStepFunction(), conf2.getStepFunction()); - } - - @Test - @DisplayName("Test RNG") - void testRNG() { - DenseLayer layer = new DenseLayer.Builder().nIn(trainingSet.numInputs()).nOut(trainingSet.numOutcomes()).weightInit(WeightInit.UNIFORM).activation(Activation.TANH).build(); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).layer(layer).build(); - long numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - Layer model = conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY); - DenseLayer layer2 = new DenseLayer.Builder().nIn(trainingSet.numInputs()).nOut(trainingSet.numOutcomes()).weightInit(WeightInit.UNIFORM).activation(Activation.TANH).build(); - NeuralNetConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(123).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).layer(layer2).build(); - long numParams2 = conf2.getLayer().initializer().numParams(conf); - INDArray params2 = Nd4j.create(1, numParams); - Layer model2 = conf2.getLayer().instantiate(conf2, null, 0, params2, true, params.dataType()); - INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY); - assertEquals(modelWeights, modelWeights2); - } - - @Test - @DisplayName("Test Set Seed Size") - void testSetSeedSize() { - Nd4j.getRandom().setSeed(123); - Layer model = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitXavier(), true); - INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY); - Nd4j.getRandom().setSeed(123); - Layer model2 = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitXavier(), true); - INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY); - assertEquals(modelWeights, modelWeights2); - } - - @Test - @DisplayName("Test Set Seed Normalized") - void testSetSeedNormalized() { - Nd4j.getRandom().setSeed(123); - Layer model = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitXavier(), true); - INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY); - Nd4j.getRandom().setSeed(123); - Layer model2 = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitXavier(), true); - INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY); - assertEquals(modelWeights, modelWeights2); - } - - @Test - @DisplayName("Test Set Seed Xavier") - void testSetSeedXavier() { - Nd4j.getRandom().setSeed(123); - Layer model = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitUniform(), true); - INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY); - Nd4j.getRandom().setSeed(123); - Layer model2 = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitUniform(), true); - INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY); - assertEquals(modelWeights, modelWeights2); - } - - @Test - @DisplayName("Test Set Seed Distribution") - void testSetSeedDistribution() { - Nd4j.getRandom().setSeed(123); - Layer model = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitDistribution(new NormalDistribution(1, 1)), true); - INDArray modelWeights = model.getParam(DefaultParamInitializer.WEIGHT_KEY); - Nd4j.getRandom().setSeed(123); - Layer model2 = getLayer(trainingSet.numInputs(), trainingSet.numOutcomes(), new WeightInitDistribution(new NormalDistribution(1, 1)), true); - INDArray modelWeights2 = model2.getParam(DefaultParamInitializer.WEIGHT_KEY); - assertEquals(modelWeights, modelWeights2); - } - - private static NeuralNetConfiguration getConfig(int nIn, int nOut, IWeightInit weightInit, boolean pretrain) { - DenseLayer layer = new DenseLayer.Builder().nIn(nIn).nOut(nOut).weightInit(weightInit).activation(Activation.TANH).build(); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).layer(layer).build(); - return conf; - } - - private static Layer getLayer(int nIn, int nOut, IWeightInit weightInit, boolean preTrain) { - NeuralNetConfiguration conf = getConfig(nIn, nOut, weightInit, preTrain); - long numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - return conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - } - - @Test - @DisplayName("Test Learning Rate By Param") - void testLearningRateByParam() { - double lr = 0.01; - double biasLr = 0.02; - int[] nIns = { 4, 3, 3 }; - int[] nOuts = { 3, 3, 3 }; - int oldScore = 1; - int newScore = 1; - int iteration = 3; - INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).list().layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(new Sgd(lr)).biasUpdater(new Sgd(biasLr)).build()).layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).updater(new Sgd(0.7)).build()).layer(2, new OutputLayer.Builder().nIn(nIns[2]).nOut(nOuts[2]).lossFunction(LossFunctions.LossFunction.MSE).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - ConvexOptimizer opt = new StochasticGradientDescent(net.getDefaultConfiguration(), new NegativeDefaultStepFunction(), null, net); - assertEquals(lr, ((Sgd) net.getLayer(0).conf().getLayer().getUpdaterByParam("W")).getLearningRate(), 1e-4); - assertEquals(biasLr, ((Sgd) net.getLayer(0).conf().getLayer().getUpdaterByParam("b")).getLearningRate(), 1e-4); - assertEquals(0.7, ((Sgd) net.getLayer(1).conf().getLayer().getUpdaterByParam("gamma")).getLearningRate(), 1e-4); - // From global LR - assertEquals(0.3, ((Sgd) net.getLayer(2).conf().getLayer().getUpdaterByParam("W")).getLearningRate(), 1e-4); - // From global LR - assertEquals(0.3, ((Sgd) net.getLayer(2).conf().getLayer().getUpdaterByParam("W")).getLearningRate(), 1e-4); - } - - @Test - @DisplayName("Test Leakyrelu Alpha") - void testLeakyreluAlpha() { - // FIXME: Make more generic to use neuralnetconfs - int sizeX = 4; - int scaleX = 10; - System.out.println("Here is a leaky vector.."); - INDArray leakyVector = Nd4j.linspace(-1, 1, sizeX, Nd4j.dataType()); - leakyVector = leakyVector.mul(scaleX); - System.out.println(leakyVector); - double myAlpha = 0.5; - System.out.println("======================"); - System.out.println("Exec and Return: Leaky Relu transformation with alpha = 0.5 .."); - System.out.println("======================"); - INDArray outDef = Nd4j.getExecutioner().exec(new LeakyReLU(leakyVector.dup(), myAlpha)); - System.out.println(outDef); - String confActivation = "leakyrelu"; - Object[] confExtra = { myAlpha }; - INDArray outMine = Nd4j.getExecutioner().exec(new LeakyReLU(leakyVector.dup(), myAlpha)); - System.out.println("======================"); - System.out.println("Exec and Return: Leaky Relu transformation with a value via getOpFactory"); - System.out.println("======================"); - System.out.println(outMine); - // Test equality for ndarray elementwise - // assertArrayEquals(..) - } - - @Test - @DisplayName("Test L 1 L 2 By Param") - void testL1L2ByParam() { - double l1 = 0.01; - double l2 = 0.07; - int[] nIns = { 4, 3, 3 }; - int[] nOuts = { 3, 3, 3 }; - int oldScore = 1; - int newScore = 1; - int iteration = 3; - INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l1(l1).l2(l2).list().layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).build()).layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).l2(0.5).build()).layer(2, new OutputLayer.Builder().nIn(nIns[2]).nOut(nOuts[2]).lossFunction(LossFunctions.LossFunction.MSE).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - ConvexOptimizer opt = new StochasticGradientDescent(net.getDefaultConfiguration(), new NegativeDefaultStepFunction(), null, net); - assertEquals(l1, TestUtils.getL1(net.getLayer(0).conf().getLayer().getRegularizationByParam("W")), 1e-4); - List r = net.getLayer(0).conf().getLayer().getRegularizationByParam("b"); - assertEquals(0, r.size()); - r = net.getLayer(1).conf().getLayer().getRegularizationByParam("beta"); - assertTrue(r == null || r.isEmpty()); - r = net.getLayer(1).conf().getLayer().getRegularizationByParam("gamma"); - assertTrue(r == null || r.isEmpty()); - r = net.getLayer(1).conf().getLayer().getRegularizationByParam("mean"); - assertTrue(r == null || r.isEmpty()); - r = net.getLayer(1).conf().getLayer().getRegularizationByParam("var"); - assertTrue(r == null || r.isEmpty()); - assertEquals(l2, TestUtils.getL2(net.getLayer(2).conf().getLayer().getRegularizationByParam("W")), 1e-4); - r = net.getLayer(2).conf().getLayer().getRegularizationByParam("b"); - assertTrue(r == null || r.isEmpty()); - } - - @Test - @DisplayName("Test Layer Pretrain Config") - void testLayerPretrainConfig() { - boolean pretrain = true; - VariationalAutoencoder layer = new VariationalAutoencoder.Builder().nIn(10).nOut(5).updater(new Sgd(1e-1)).lossFunction(LossFunctions.LossFunction.KL_DIVERGENCE).build(); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().seed(42).layer(layer).build(); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/dropout/TestDropout.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/dropout/TestDropout.java deleted file mode 100644 index ef3ff355f..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/dropout/TestDropout.java +++ /dev/null @@ -1,620 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.dropout; - -import lombok.Data; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.graph.LayerVertex; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.DropoutLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.conditions.Conditions; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.primitives.Pair; -import org.nd4j.linalg.schedule.MapSchedule; -import org.nd4j.linalg.schedule.ScheduleType; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.nd4j.linalg.indexing.NDArrayIndex.all; -import static org.nd4j.linalg.indexing.NDArrayIndex.point; - -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -public class TestDropout extends BaseDL4JTest { - - @Test - public void testBasicConfig(){ - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .dropOut(0.6) - .list() - .layer(new DenseLayer.Builder().nIn(10).nOut(10).build()) - .layer(new DenseLayer.Builder().nIn(10).nOut(10).dropOut(0.7).build()) - .layer(new DenseLayer.Builder().nIn(10).nOut(10).dropOut(new AlphaDropout(0.5)).build()) - .build(); - - assertEquals(new Dropout(0.6), conf.getConf(0).getLayer().getIDropout()); - assertEquals(new Dropout(0.7), conf.getConf(1).getLayer().getIDropout()); - assertEquals(new AlphaDropout(0.5), conf.getConf(2).getLayer().getIDropout()); - - - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() - .dropOut(0.6) - .graphBuilder() - .addInputs("in") - .addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in") - .addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).dropOut(0.7).build(), "0") - .addLayer("2", new DenseLayer.Builder().nIn(10).nOut(10).dropOut(new AlphaDropout(0.5)).build(), "1") - .setOutputs("2") - .build(); - - assertEquals(new Dropout(0.6), ((LayerVertex)conf2.getVertices().get("0")).getLayerConf().getLayer().getIDropout()); - assertEquals(new Dropout(0.7), ((LayerVertex)conf2.getVertices().get("1")).getLayerConf().getLayer().getIDropout()); - assertEquals(new AlphaDropout(0.5), ((LayerVertex)conf2.getVertices().get("2")).getLayerConf().getLayer().getIDropout()); - } - - @Test - public void testCalls(){ - - CustomDropout d1 = new CustomDropout(); - CustomDropout d2 = new CustomDropout(); - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .list() - .layer(new DenseLayer.Builder().nIn(4).nOut(3).dropOut(d1).build()) - .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).dropOut(d2).nIn(3).nOut(3).build()) - .build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - List l = new ArrayList<>(); - l.add(new DataSet(Nd4j.rand(5,4), Nd4j.rand(5,3))); - l.add(new DataSet(Nd4j.rand(5,4), Nd4j.rand(5,3))); - l.add(new DataSet(Nd4j.rand(5,4), Nd4j.rand(5,3))); - - DataSetIterator iter = new ExistingDataSetIterator(l); - - net.fit(iter); - net.fit(iter); - - List> expList = Arrays.asList( - new Pair<>(0, 0), - new Pair<>(1, 0), - new Pair<>(2, 0), - new Pair<>(3, 1), - new Pair<>(4, 1), - new Pair<>(5, 1)); - - assertEquals(expList, d1.getAllCalls()); - assertEquals(expList, d2.getAllCalls()); - - assertEquals(expList, d1.getAllReverseCalls()); - assertEquals(expList, d2.getAllReverseCalls()); - - - d1 = new CustomDropout(); - d2 = new CustomDropout(); - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() - .graphBuilder() - .addInputs("in") - .addLayer("0", new DenseLayer.Builder().nIn(4).nOut(3).dropOut(d1).build(), "in") - .addLayer("1", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).dropOut(d2).nIn(3).nOut(3).build(), "0") - .setOutputs("1") - .build(); - - ComputationGraph net2 = new ComputationGraph(conf2); - net2.init(); - - net2.fit(iter); - net2.fit(iter); - - assertEquals(expList, d1.getAllCalls()); - assertEquals(expList, d2.getAllCalls()); - } - - @Data - public static class CustomDropout implements IDropout{ - private List> allCalls = new ArrayList<>(); - private List> allReverseCalls = new ArrayList<>(); - - @Override - public INDArray applyDropout(INDArray inputActivations, INDArray result, int iteration, int epoch, LayerWorkspaceMgr workspaceMgr) { - allCalls.add(new Pair<>(iteration, epoch)); - return inputActivations; - } - - @Override - public INDArray backprop(INDArray gradAtOutput, INDArray gradAtInput, int iteration, int epoch) { - allReverseCalls.add(new Pair<>(iteration, epoch)); - return gradAtInput; - } - - @Override - public void clear() { - - } - - @Override - public IDropout clone() { - return this; - } - } - - @Test - public void testSerialization(){ - - IDropout[] dropouts = new IDropout[]{ - new Dropout(0.5), - new AlphaDropout(0.5), - new GaussianDropout(0.1), - new GaussianNoise(0.1)}; - - for(IDropout id : dropouts) { - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .dropOut(id) - .list() - .layer(new DenseLayer.Builder().nIn(4).nOut(3).build()) - .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(3).nOut(3).build()) - .build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - TestUtils.testModelSerialization(net); - - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder() - .dropOut(id) - .graphBuilder() - .addInputs("in") - .addLayer("0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "in") - .addLayer("1", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(3).nOut(3).build(), "0") - .setOutputs("1") - .build(); - - ComputationGraph net2 = new ComputationGraph(conf2); - net2.init(); - - TestUtils.testModelSerialization(net2); - } - } - - @Test - public void testDropoutValues(){ - Nd4j.getRandom().setSeed(12345); - - Dropout d = new Dropout(0.5); - - INDArray in = Nd4j.ones(10, 10); - INDArray out = d.applyDropout(in, Nd4j.create(10,10), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - - assertEquals(in, Nd4j.ones(10, 10)); - - int countZeros = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(0))).getInt(0); - int countTwos = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(2))).getInt(0); - - assertEquals(100, countZeros + countTwos); //Should only be 0 or 2 - //Stochastic, but this should hold for most cases - assertTrue(countZeros >= 25 && countZeros <= 75); - assertTrue(countTwos >= 25 && countTwos <= 75); - - //Test schedule: - d = new Dropout(new MapSchedule.Builder(ScheduleType.ITERATION).add(0, 0.5).add(5, 0.1).build()); - for( int i=0; i<10; i++ ) { - out = d.applyDropout(in, Nd4j.create(in.shape()), i, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - assertEquals(in, Nd4j.ones(10, 10)); - countZeros = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(0))).getInt(0); - - if(i < 5){ - countTwos = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(2))).getInt(0); - assertEquals( 100, countZeros + countTwos,String.valueOf(i)); //Should only be 0 or 2 - //Stochastic, but this should hold for most cases - assertTrue(countZeros >= 25 && countZeros <= 75); - assertTrue(countTwos >= 25 && countTwos <= 75); - } else { - int countInverse = Nd4j.getExecutioner().exec(new MatchCondition(out, Conditions.equals(1.0/0.1))).getInt(0); - assertEquals(100, countZeros + countInverse); //Should only be 0 or 10 - //Stochastic, but this should hold for most cases - assertTrue(countZeros >= 80); - assertTrue(countInverse <= 20); - } - } - } - - @Test - public void testGaussianDropoutValues(){ - Nd4j.getRandom().setSeed(12345); - - GaussianDropout d = new GaussianDropout(0.1); //sqrt(0.1/(1-0.1)) = 0.3333 stdev - - INDArray in = Nd4j.ones(50, 50); - INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - - assertEquals(in, Nd4j.ones(50, 50)); - - double mean = out.meanNumber().doubleValue(); - double stdev = out.stdNumber().doubleValue(); - - assertEquals(1.0, mean, 0.05); - assertEquals(0.333, stdev, 0.02); - } - - @Test - public void testGaussianNoiseValues(){ - Nd4j.getRandom().setSeed(12345); - - GaussianNoise d = new GaussianNoise(0.1); //sqrt(0.1/(1-0.1)) = 0.3333 stdev - - INDArray in = Nd4j.ones(50, 50); - INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - - assertEquals(in, Nd4j.ones(50, 50)); - - double mean = out.meanNumber().doubleValue(); - double stdev = out.stdNumber().doubleValue(); - - assertEquals(1.0, mean, 0.05); - assertEquals(0.1, stdev, 0.01); - } - - @Test - public void testAlphaDropoutValues(){ - Nd4j.getRandom().setSeed(12345); - - double p = 0.4; - AlphaDropout d = new AlphaDropout(p); - - double SELU_ALPHA = 1.6732632423543772; - double SELU_LAMBDA = 1.0507009873554804; - double alphaPrime = - SELU_LAMBDA * SELU_ALPHA; - double a = 1.0 / Math.sqrt((p + alphaPrime * alphaPrime * p * (1-p))); - double b = -1.0 / Math.sqrt(p + alphaPrime * alphaPrime * p * (1-p)) * (1-p) * alphaPrime; - - double actA = d.a(p); - double actB = d.b(p); - - assertEquals(a, actA, 1e-6); - assertEquals(b, actB, 1e-6); - - INDArray in = Nd4j.ones(10, 10); - INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - - int countValueDropped = 0; - int countEqn = 0; - double eqn = a * 1 + b; - double valueDropped = a * alphaPrime + b; - for(int i=0; i<100; i++ ){ - double v = out.getDouble(i); - if(v >= valueDropped - 1e-6 && v <= valueDropped + 1e-6){ - countValueDropped++; - } else if(v >= eqn - 1e-6 && v <= eqn + 1e-6){ - countEqn++; - } - - } - - assertEquals(100, countValueDropped + countEqn); - assertTrue(countValueDropped >= 25 && countValueDropped <= 75); - assertTrue(countEqn >= 25 && countEqn <= 75); - } - - - @Test - public void testSpatialDropout5DValues(){ - Nd4j.getRandom().setSeed(12345); - - SpatialDropout d = new SpatialDropout(0.5); - - INDArray in = Nd4j.ones(10, 10, 5, 5, 5); - INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - - assertEquals(in, Nd4j.ones(10, 10, 5, 5, 5)); - - //Now, we expect all values for a given depth to be the same... 0 or 2 - int countZero = 0; - int countTwo = 0; - for( int i=0; i<10; i++ ){ - for( int j=0; j<10; j++ ){ - double value = out.getDouble(i,j,0,0,0); - assertTrue( value == 0 || value == 2.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,5,}, value); - INDArray act = out.get(point(i), point(j), all(), all(),all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countTwo++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 25 && countZero <= 75); - assertTrue(countTwo >= 25 && countTwo <= 75); - - //Test schedule: - d = new SpatialDropout(new MapSchedule.Builder(ScheduleType.ITERATION).add(0, 0.5).add(5, 0.1).build()); - for( int i=0; i<10; i++ ) { - out = d.applyDropout(in, Nd4j.create(in.shape()), i, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - assertEquals(in, Nd4j.ones(10, 10, 5, 5, 5)); - - if(i < 5){ - countZero = 0; - countTwo = 0; - for( int m=0; m<10; m++ ){ - for( int j=0; j<10; j++ ){ - double value = out.getDouble(m,j,0,0,0); - assertTrue( value == 0 || value == 2.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,5,}, value); - INDArray act = out.get(point(m), point(j), all(), all(), all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countTwo++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 25 && countZero <= 75); - assertTrue(countTwo >= 25 && countTwo <= 75); - } else { - countZero = 0; - int countInverse = 0; - for( int m=0; m<10; m++ ){ - for( int j=0; j<10; j++ ){ - double value = out.getDouble(m,j,0,0,0); - assertTrue( value == 0 || value == 10.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,5,}, value); - INDArray act = out.get(point(m), point(j), all(), all(),all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countInverse++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 80); - assertTrue(countInverse <= 20); - } - } - } - - - @Test - public void testSpatialDropoutValues(){ - Nd4j.getRandom().setSeed(12345); - - SpatialDropout d = new SpatialDropout(0.5); - - INDArray in = Nd4j.ones(10, 10, 5, 5); - INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - - assertEquals(in, Nd4j.ones(10, 10, 5, 5)); - - //Now, we expect all values for a given depth to be the same... 0 or 2 - int countZero = 0; - int countTwo = 0; - for( int i=0; i<10; i++ ){ - for( int j=0; j<10; j++ ){ - double value = out.getDouble(i,j,0,0); - assertTrue( value == 0 || value == 2.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,}, value); - INDArray act = out.get(point(i), point(j), all(), all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countTwo++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 25 && countZero <= 75); - assertTrue(countTwo >= 25 && countTwo <= 75); - - //Test schedule: - d = new SpatialDropout(new MapSchedule.Builder(ScheduleType.ITERATION).add(0, 0.5).add(5, 0.1).build()); - for( int i=0; i<10; i++ ) { - out = d.applyDropout(in, Nd4j.create(in.shape()), i, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - assertEquals(in, Nd4j.ones(10, 10, 5, 5)); - - if(i < 5){ - countZero = 0; - countTwo = 0; - for( int m=0; m<10; m++ ){ - for( int j=0; j<10; j++ ){ - double value = out.getDouble(m,j,0,0); - assertTrue( value == 0 || value == 2.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,}, value); - INDArray act = out.get(point(m), point(j), all(), all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countTwo++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 25 && countZero <= 75); - assertTrue(countTwo >= 25 && countTwo <= 75); - } else { - countZero = 0; - int countInverse = 0; - for( int m=0; m<10; m++ ){ - for( int j=0; j<10; j++ ){ - double value = out.getDouble(m,j,0,0); - assertTrue( value == 0 || value == 10.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{5,5,}, value); - INDArray act = out.get(point(m), point(j), all(), all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countInverse++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 80); - assertTrue(countInverse <= 20); - } - } - } - - @Test - public void testSpatialDropoutValues3D(){ - Nd4j.getRandom().setSeed(12345); - - SpatialDropout d = new SpatialDropout(0.5); - - INDArray in = Nd4j.ones(10, 8, 12); - INDArray out = d.applyDropout(in, Nd4j.create(in.shape()), 0, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - - assertEquals(in, Nd4j.ones(10, 8, 12)); - - //Now, we expect all values for a given depth to be the same... 0 or 2 - int countZero = 0; - int countTwo = 0; - for( int i=0; i<10; i++ ){ - for( int j=0; j<8; j++ ){ - double value = out.getDouble(i,j,0); - assertTrue( value == 0 || value == 2.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{12}, value); - INDArray act = out.get(point(i), point(j), all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countTwo++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 20 && countZero <= 60); - assertTrue(countTwo >= 20 && countTwo <= 60); - - //Test schedule: - d = new SpatialDropout(new MapSchedule.Builder(ScheduleType.ITERATION).add(0, 0.5).add(5, 0.1).build()); - for( int i=0; i<10; i++ ) { - out = d.applyDropout(in, Nd4j.create(in.shape()), i, 0, LayerWorkspaceMgr.noWorkspacesImmutable()); - assertEquals(in, Nd4j.ones(10, 8, 12)); - - if(i < 5){ - countZero = 0; - countTwo = 0; - for( int m=0; m<10; m++ ){ - for( int j=0; j<8; j++ ){ - double value = out.getDouble(m,j,0); - assertTrue( value == 0 || value == 2.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{12}, value); - INDArray act = out.get(point(m), point(j), all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countTwo++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 20 && countZero <= 60); - assertTrue(countTwo >= 20 && countTwo <= 60); - } else { - countZero = 0; - int countInverse = 0; - for( int m=0; m<10; m++ ){ - for( int j=0; j<8; j++ ){ - double value = out.getDouble(m,j,0); - assertTrue( value == 0 || value == 10.0); - INDArray exp = Nd4j.valueArrayOf(new int[]{12}, value); - INDArray act = out.get(point(m), point(j), all()); - assertEquals(exp, act); - - if(value == 0.0){ - countZero++; - } else { - countInverse++; - } - } - } - - //Stochastic, but this should hold for most cases - assertTrue(countZero >= 60); - assertTrue(countInverse <= 15); - } - } - } - - @Test - public void testSpatialDropoutJSON(){ - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .list() - .layer(new DropoutLayer.Builder(new SpatialDropout(0.5)).build()) - .build(); - - String asJson = conf.toJson(); - MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(asJson); - - assertEquals(conf, fromJson); - } - -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertexTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertexTest.java deleted file mode 100644 index 550fb0d15..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/graph/ElementWiseVertexTest.java +++ /dev/null @@ -1,509 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf.graph; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.ActivationLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.activations.impl.ActivationSigmoid; -import org.nd4j.linalg.activations.impl.ActivationTanH; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.rng.distribution.impl.UniformDistribution; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; -import org.nd4j.linalg.ops.transforms.Transforms; -import org.nd4j.common.primitives.Pair; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Element Wise Vertex Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class ElementWiseVertexTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Element Wise Vertex Num Params") - void testElementWiseVertexNumParams() { - /* - * https://github.com/eclipse/deeplearning4j/pull/3514#issuecomment-307754386 - * from @agibsonccc: check for the basics: like 0 numParams - */ - ElementWiseVertex.Op[] ops = new ElementWiseVertex.Op[] { ElementWiseVertex.Op.Add, ElementWiseVertex.Op.Subtract, ElementWiseVertex.Op.Product }; - for (ElementWiseVertex.Op op : ops) { - ElementWiseVertex ewv = new ElementWiseVertex(op); - Assertions.assertEquals(0, ewv.numParams(true)); - Assertions.assertEquals(0, ewv.numParams(false)); - } - } - - @Test - @DisplayName("Test Element Wise Vertex Forward Add") - void testElementWiseVertexForwardAdd() { - int batchsz = 24; - int featuresz = 17; - ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input1", "input2", "input3").addLayer("denselayer", new DenseLayer.Builder().nIn(featuresz).nOut(1).activation(Activation.IDENTITY).build(), "input1").addVertex("elementwiseAdd", new ElementWiseVertex(ElementWiseVertex.Op.Add), "input1", "input2", "input3").addLayer("Add", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), "elementwiseAdd").setOutputs("Add", "denselayer").build(); - ComputationGraph cg = new ComputationGraph(cgc); - cg.init(); - INDArray input1 = Nd4j.rand(batchsz, featuresz); - INDArray input2 = Nd4j.rand(batchsz, featuresz); - INDArray input3 = Nd4j.rand(batchsz, featuresz); - INDArray target = input1.dup().addi(input2).addi(input3); - INDArray output = cg.output(input1, input2, input3)[0]; - INDArray squared = output.sub(target.castTo(output.dataType())); - double rms = squared.mul(squared).sumNumber().doubleValue(); - Assertions.assertEquals(0.0, rms, this.epsilon); - } - - @Test - @DisplayName("Test Element Wise Vertex Forward Product") - void testElementWiseVertexForwardProduct() { - int batchsz = 24; - int featuresz = 17; - ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input1", "input2", "input3").addLayer("denselayer", new DenseLayer.Builder().nIn(featuresz).nOut(1).activation(Activation.IDENTITY).build(), "input1").addVertex("elementwiseProduct", new ElementWiseVertex(ElementWiseVertex.Op.Product), "input1", "input2", "input3").addLayer("Product", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), "elementwiseProduct").setOutputs("Product", "denselayer").build(); - ComputationGraph cg = new ComputationGraph(cgc); - cg.init(); - INDArray input1 = Nd4j.rand(batchsz, featuresz); - INDArray input2 = Nd4j.rand(batchsz, featuresz); - INDArray input3 = Nd4j.rand(batchsz, featuresz); - INDArray target = input1.dup().muli(input2).muli(input3); - INDArray output = cg.output(input1, input2, input3)[0]; - INDArray squared = output.sub(target.castTo(output.dataType())); - double rms = squared.mul(squared).sumNumber().doubleValue(); - Assertions.assertEquals(0.0, rms, this.epsilon); - } - - @Test - @DisplayName("Test Element Wise Vertex Forward Subtract") - void testElementWiseVertexForwardSubtract() { - int batchsz = 24; - int featuresz = 17; - ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input1", "input2").addLayer("denselayer", new DenseLayer.Builder().nIn(featuresz).nOut(1).activation(Activation.IDENTITY).build(), "input1").addVertex("elementwiseSubtract", new ElementWiseVertex(ElementWiseVertex.Op.Subtract), "input1", "input2").addLayer("Subtract", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), "elementwiseSubtract").setOutputs("Subtract", "denselayer").build(); - ComputationGraph cg = new ComputationGraph(cgc); - cg.init(); - INDArray input1 = Nd4j.rand(batchsz, featuresz); - INDArray input2 = Nd4j.rand(batchsz, featuresz); - INDArray target = input1.dup().subi(input2); - INDArray output = cg.output(input1, input2)[0]; - INDArray squared = output.sub(target); - double rms = Math.sqrt(squared.mul(squared).sumNumber().doubleValue()); - Assertions.assertEquals(0.0, rms, this.epsilon); - } - - @Test - @DisplayName("Test Element Wise Vertex Full Add") - void testElementWiseVertexFullAdd() { - int batchsz = 24; - int featuresz = 17; - int midsz = 13; - int outputsz = 11; - ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).dataType(DataType.DOUBLE).biasInit(0.0).updater(new Sgd()).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder().addInputs("input1", "input2", "input3").addLayer("dense1", new DenseLayer.Builder().nIn(featuresz).nOut(midsz).activation(new ActivationTanH()).build(), "input1").addLayer("dense2", new DenseLayer.Builder().nIn(featuresz).nOut(midsz).activation(new ActivationTanH()).build(), "input2").addLayer("dense3", new DenseLayer.Builder().nIn(featuresz).nOut(midsz).activation(new ActivationTanH()).build(), "input3").addVertex("elementwiseAdd", new ElementWiseVertex(ElementWiseVertex.Op.Add), "dense1", "dense2", "dense3").addLayer("output", new OutputLayer.Builder().nIn(midsz).nOut(outputsz).activation(new ActivationSigmoid()).lossFunction(LossFunction.MSE).build(), "elementwiseAdd").setOutputs("output").build(); - ComputationGraph cg = new ComputationGraph(cgc); - cg.init(); - INDArray input1 = Nd4j.rand(new int[] { batchsz, featuresz }, new UniformDistribution(-1, 1)); - INDArray input2 = Nd4j.rand(new int[] { batchsz, featuresz }, new UniformDistribution(-1, 1)); - INDArray input3 = Nd4j.rand(new int[] { batchsz, featuresz }, new UniformDistribution(-1, 1)); - INDArray target = nullsafe(Nd4j.rand(new int[] { batchsz, outputsz }, new UniformDistribution(0, 1))); - cg.setInputs(input1, input2, input3); - cg.setLabels(target); - cg.computeGradientAndScore(); - // Let's figure out what our params are now. - Map params = cg.paramTable(); - INDArray dense1_W = nullsafe(params.get("dense1_W")); - INDArray dense1_b = nullsafe(params.get("dense1_b")); - INDArray dense2_W = nullsafe(params.get("dense2_W")); - INDArray dense2_b = nullsafe(params.get("dense2_b")); - INDArray dense3_W = nullsafe(params.get("dense3_W")); - INDArray dense3_b = nullsafe(params.get("dense3_b")); - INDArray output_W = nullsafe(params.get("output_W")); - INDArray output_b = nullsafe(params.get("output_b")); - // Now, let's calculate what we expect the output to be. - INDArray mh = input1.mmul(dense1_W).addi(dense1_b.repmat(batchsz, 1)); - INDArray m = (Transforms.tanh(mh)); - INDArray nh = input2.mmul(dense2_W).addi(dense2_b.repmat(batchsz, 1)); - INDArray n = (Transforms.tanh(nh)); - INDArray oh = input3.mmul(dense3_W).addi(dense3_b.repmat(batchsz, 1)); - INDArray o = (Transforms.tanh(oh)); - INDArray middle = Nd4j.zeros(batchsz, midsz); - middle.addi(m).addi(n).addi(o); - INDArray expect = Nd4j.zeros(batchsz, outputsz); - expect.addi(Transforms.sigmoid(middle.mmul(output_W).addi(output_b.repmat(batchsz, 1)))); - INDArray output = nullsafe(cg.output(input1, input2, input3)[0]); - Assertions.assertEquals(0.0, mse(output, expect), this.epsilon); - Pair pgd = cg.gradientAndScore(); - double score = pgd.getSecond(); - Assertions.assertEquals(score, mse(output, target), this.epsilon); - Map gradients = pgd.getFirst().gradientForVariable(); - /* - * So. Let's say we have inputs a, b, c - * mh = a W1 + b1 - * m = tanh(mh) - * - * nh = b W2 + b2 - * n = tanh(nh) - * - * oh = c W3 + b3 - * o = tanh(oh) - * - * s = m+n+o - * - * yh = s W4 + b4 - * y = sigmoid(yh) - * - * E = (y-t)^2 - * dE/dy = 2 (y-t) - * - * dy/dyh = y * (1-y) - * dE/dyh = 2 * y * (1-y) * (y-t) - * - * dyh/dW4 = s.transpose() - * dyh/db4 = Nd4j.ones(1, batchsz) - * dyh/ds = W4.tranpose() - * - * ds/dm = Nd4j.ones(1, midsz) - * - * dm/dmh = 1-(m^2) - * - * dmh/dW1 = a.transpose() - * dmh/db1 = Nd4j.ones(1, batchsz) - * - */ - INDArray y = output; - INDArray s = middle; - INDArray W4 = output_W; - INDArray dEdy = Nd4j.zeros(target.shape()); - // This should be of size batchsz x outputsz - dEdy.addi(y).subi(target).muli(2); - // Why? Because the LossFunction divides by the _element size_ of the output. - dEdy.divi(target.shape()[1]); - // This is of size batchsz x outputsz - INDArray dydyh = y.mul(y.mul(-1).add(1)); - INDArray dEdyh = dydyh.mul(dEdy); - INDArray dyhdW4 = s.transpose(); - INDArray dEdW4 = nullsafe(dyhdW4.mmul(dEdyh)); - INDArray dyhdb4 = Nd4j.ones(1, batchsz); - INDArray dEdb4 = nullsafe(dyhdb4.mmul(dEdyh)); - INDArray dyhds = W4.transpose(); - INDArray dEds = dEdyh.mmul(dyhds); - INDArray dsdm = Nd4j.ones(batchsz, midsz); - INDArray dEdm = dsdm.mul(dEds); - INDArray dmdmh = (m.mul(m)).mul(-1).add(1); - INDArray dEdmh = dmdmh.mul(dEdm); - INDArray dmhdW1 = input1.transpose(); - INDArray dEdW1 = nullsafe(dmhdW1.mmul(dEdmh)); - INDArray dmhdb1 = Nd4j.ones(1, batchsz); - INDArray dEdb1 = nullsafe(dmhdb1.mmul(dEdmh)); - INDArray dsdn = Nd4j.ones(batchsz, midsz); - INDArray dEdn = dsdn.mul(dEds); - INDArray dndnh = (n.mul(n)).mul(-1).add(1); - INDArray dEdnh = dndnh.mul(dEdn); - INDArray dnhdW2 = input2.transpose(); - INDArray dEdW2 = nullsafe(dnhdW2.mmul(dEdnh)); - INDArray dnhdb2 = Nd4j.ones(1, batchsz); - INDArray dEdb2 = nullsafe(dnhdb2.mmul(dEdnh)); - INDArray dsdo = Nd4j.ones(batchsz, midsz); - INDArray dEdo = dsdo.mul(dEds); - INDArray dodoh = (o.mul(o)).mul(-1).add(1); - INDArray dEdoh = dodoh.mul(dEdo); - INDArray dohdW3 = input3.transpose(); - INDArray dEdW3 = nullsafe(dohdW3.mmul(dEdoh)); - INDArray dohdb3 = Nd4j.ones(1, batchsz); - INDArray dEdb3 = nullsafe(dohdb3.mmul(dEdoh)); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_W")), dEdW4), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_b")), dEdb4), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_W")), dEdW1), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_b")), dEdb1), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_W")), dEdW2), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_b")), dEdb2), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense3_W")), dEdW3), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense3_b")), dEdb3), this.epsilon); - } - - @Test - @DisplayName("Test Element Wise Vertex Full Product") - void testElementWiseVertexFullProduct() { - int batchsz = 24; - int featuresz = 17; - int midsz = 13; - int outputsz = 11; - ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).dataType(DataType.DOUBLE).biasInit(0.0).updater(new Sgd()).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder().addInputs("input1", "input2", "input3").addLayer("dense1", new DenseLayer.Builder().nIn(featuresz).nOut(midsz).activation(new ActivationTanH()).build(), "input1").addLayer("dense2", new DenseLayer.Builder().nIn(featuresz).nOut(midsz).activation(new ActivationTanH()).build(), "input2").addLayer("dense3", new DenseLayer.Builder().nIn(featuresz).nOut(midsz).activation(new ActivationTanH()).build(), "input3").addVertex("elementwiseProduct", new ElementWiseVertex(ElementWiseVertex.Op.Product), "dense1", "dense2", "dense3").addLayer("output", new OutputLayer.Builder().nIn(midsz).nOut(outputsz).activation(new ActivationSigmoid()).lossFunction(LossFunction.MSE).build(), "elementwiseProduct").setOutputs("output").build(); - ComputationGraph cg = new ComputationGraph(cgc); - cg.init(); - INDArray input1 = Nd4j.rand(new int[] { batchsz, featuresz }, new UniformDistribution(-1, 1)); - INDArray input2 = Nd4j.rand(new int[] { batchsz, featuresz }, new UniformDistribution(-1, 1)); - INDArray input3 = Nd4j.rand(new int[] { batchsz, featuresz }, new UniformDistribution(-1, 1)); - INDArray target = nullsafe(Nd4j.rand(new int[] { batchsz, outputsz }, new UniformDistribution(0, 1))); - cg.setInputs(input1, input2, input3); - cg.setLabels(target); - cg.computeGradientAndScore(); - // Let's figure out what our params are now. - Map params = cg.paramTable(); - INDArray dense1_W = nullsafe(params.get("dense1_W")); - INDArray dense1_b = nullsafe(params.get("dense1_b")); - INDArray dense2_W = nullsafe(params.get("dense2_W")); - INDArray dense2_b = nullsafe(params.get("dense2_b")); - INDArray dense3_W = nullsafe(params.get("dense3_W")); - INDArray dense3_b = nullsafe(params.get("dense3_b")); - INDArray output_W = nullsafe(params.get("output_W")); - INDArray output_b = nullsafe(params.get("output_b")); - // Now, let's calculate what we expect the output to be. - INDArray mh = input1.mmul(dense1_W).addi(dense1_b.repmat(batchsz, 1)); - INDArray m = (Transforms.tanh(mh)); - INDArray nh = input2.mmul(dense2_W).addi(dense2_b.repmat(batchsz, 1)); - INDArray n = (Transforms.tanh(nh)); - INDArray oh = input3.mmul(dense3_W).addi(dense3_b.repmat(batchsz, 1)); - INDArray o = (Transforms.tanh(oh)); - INDArray middle = Nd4j.ones(batchsz, midsz); - middle.muli(m).muli(n).muli(o); - INDArray expect = Nd4j.zeros(batchsz, outputsz); - expect.addi(Transforms.sigmoid(middle.mmul(output_W).addi(output_b.repmat(batchsz, 1)))); - INDArray output = nullsafe(cg.output(input1, input2, input3)[0]); - Assertions.assertEquals(0.0, mse(output, expect), this.epsilon); - Pair pgd = cg.gradientAndScore(); - double score = pgd.getSecond(); - Assertions.assertEquals(score, mse(output, target), this.epsilon); - Map gradients = pgd.getFirst().gradientForVariable(); - /* - * So. Let's say we have inputs a, b, c - * mh = a W1 + b1 - * m = tanh(mh) - * - * nh = b W2 + b2 - * n = tanh(nh) - * - * oh = c W3 + b3 - * o = tanh(oh) - * - * s = m*n*o - * - * yh = s W4 + b4 - * y = sigmoid(yh) - * - * E = (y-t)^2 - * dE/dy = 2 (y-t) - * - * dy/dyh = y * (1-y) - * dE/dyh = 2 * y * (1-y) * (y-t) - * - * dyh/dW4 = s.transpose() - * dyh/db4 = Nd4j.ones(1, batchsz) - * dyh/ds = W4.tranpose() - * - * ds/dm = Nd4j.ones(1, midsz).mul(o).mul(n) // Basically the _rest_ of the middle layers - * - * dm/dmh = 1-(m^2) - * - * dmh/dW1 = a.transpose() - * dmh/db1 = Nd4j.ones(1, batchsz) - * - */ - INDArray y = output; - INDArray s = middle; - INDArray W4 = output_W; - INDArray dEdy = Nd4j.zeros(target.shape()); - // This should be of size batchsz x outputsz - dEdy.addi(y).subi(target).muli(2); - // Why? Because the LossFunction divides by the _element size_ of the output. - dEdy.divi(target.shape()[1]); - // This is of size batchsz x outputsz - INDArray dydyh = y.mul(y.mul(-1).add(1)); - INDArray dEdyh = dydyh.mul(dEdy); - INDArray dyhdW4 = s.transpose(); - INDArray dEdW4 = nullsafe(dyhdW4.mmul(dEdyh)); - INDArray dyhdb4 = Nd4j.ones(1, batchsz); - INDArray dEdb4 = nullsafe(dyhdb4.mmul(dEdyh)); - INDArray dyhds = W4.transpose(); - INDArray dEds = dEdyh.mmul(dyhds); - INDArray dsdm = Nd4j.ones(batchsz, midsz).muli(n).muli(o); - INDArray dEdm = dsdm.mul(dEds); - INDArray dmdmh = (m.mul(m)).mul(-1).add(1); - INDArray dEdmh = dmdmh.mul(dEdm); - INDArray dmhdW1 = input1.transpose(); - INDArray dEdW1 = nullsafe(dmhdW1.mmul(dEdmh)); - INDArray dmhdb1 = Nd4j.ones(1, batchsz); - INDArray dEdb1 = nullsafe(dmhdb1.mmul(dEdmh)); - INDArray dsdn = Nd4j.ones(batchsz, midsz).muli(m).muli(o); - INDArray dEdn = dsdn.mul(dEds); - INDArray dndnh = (n.mul(n)).mul(-1).add(1); - INDArray dEdnh = dndnh.mul(dEdn); - INDArray dnhdW2 = input2.transpose(); - INDArray dEdW2 = nullsafe(dnhdW2.mmul(dEdnh)); - INDArray dnhdb2 = Nd4j.ones(1, batchsz); - INDArray dEdb2 = nullsafe(dnhdb2.mmul(dEdnh)); - INDArray dsdo = Nd4j.ones(batchsz, midsz).muli(m).muli(n); - INDArray dEdo = dsdo.mul(dEds); - INDArray dodoh = (o.mul(o)).mul(-1).add(1); - INDArray dEdoh = dodoh.mul(dEdo); - INDArray dohdW3 = input3.transpose(); - INDArray dEdW3 = nullsafe(dohdW3.mmul(dEdoh)); - INDArray dohdb3 = Nd4j.ones(1, batchsz); - INDArray dEdb3 = nullsafe(dohdb3.mmul(dEdoh)); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_W")), dEdW4), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_b")), dEdb4), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_W")), dEdW1), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_b")), dEdb1), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_W")), dEdW2), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_b")), dEdb2), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense3_W")), dEdW3), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense3_b")), dEdb3), this.epsilon); - } - - @Test - @DisplayName("Test Element Wise Vertex Full Subtract") - void testElementWiseVertexFullSubtract() { - int batchsz = 24; - int featuresz = 17; - int midsz = 13; - int outputsz = 11; - ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).dataType(DataType.DOUBLE).biasInit(0.0).updater(new Sgd()).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder().addInputs("input1", "input2").addLayer("dense1", new DenseLayer.Builder().nIn(featuresz).nOut(midsz).activation(new ActivationTanH()).build(), "input1").addLayer("dense2", new DenseLayer.Builder().nIn(featuresz).nOut(midsz).activation(new ActivationTanH()).build(), "input2").addVertex("elementwiseSubtract", new ElementWiseVertex(ElementWiseVertex.Op.Subtract), "dense1", "dense2").addLayer("output", new OutputLayer.Builder().nIn(midsz).nOut(outputsz).activation(new ActivationSigmoid()).lossFunction(LossFunction.MSE).build(), "elementwiseSubtract").setOutputs("output").build(); - ComputationGraph cg = new ComputationGraph(cgc); - cg.init(); - INDArray input1 = Nd4j.rand(new int[] { batchsz, featuresz }, new UniformDistribution(-1, 1)); - INDArray input2 = Nd4j.rand(new int[] { batchsz, featuresz }, new UniformDistribution(-1, 1)); - INDArray target = nullsafe(Nd4j.rand(new int[] { batchsz, outputsz }, new UniformDistribution(0, 1))); - cg.setInputs(input1, input2); - cg.setLabels(target); - cg.computeGradientAndScore(); - // Let's figure out what our params are now. - Map params = cg.paramTable(); - INDArray dense1_W = nullsafe(params.get("dense1_W")); - INDArray dense1_b = nullsafe(params.get("dense1_b")); - INDArray dense2_W = nullsafe(params.get("dense2_W")); - INDArray dense2_b = nullsafe(params.get("dense2_b")); - INDArray output_W = nullsafe(params.get("output_W")); - INDArray output_b = nullsafe(params.get("output_b")); - // Now, let's calculate what we expect the output to be. - INDArray mh = input1.mmul(dense1_W).addi(dense1_b.repmat(batchsz, 1)); - INDArray m = (Transforms.tanh(mh)); - INDArray nh = input2.mmul(dense2_W).addi(dense2_b.repmat(batchsz, 1)); - INDArray n = (Transforms.tanh(nh)); - INDArray middle = Nd4j.zeros(batchsz, midsz); - middle.addi(m).subi(n); - INDArray expect = Nd4j.zeros(batchsz, outputsz); - expect.addi(Transforms.sigmoid(middle.mmul(output_W).addi(output_b.repmat(batchsz, 1)))); - INDArray output = nullsafe(cg.output(input1, input2)[0]); - Assertions.assertEquals(0.0, mse(output, expect), this.epsilon); - Pair pgd = cg.gradientAndScore(); - double score = pgd.getSecond(); - Assertions.assertEquals(score, mse(output, target), this.epsilon); - Map gradients = pgd.getFirst().gradientForVariable(); - /* - * So. Let's say we have inputs a, b, c - * mh = a W1 + b1 - * m = tanh(mh) - * - * nh = b W2 + b2 - * n = tanh(nh) - * - * s = m-n - * - * yh = s W4 + b4 - * y = sigmoid(yh) - * - * E = (y-t)^2 - * dE/dy = 2 (y-t) - * - * dy/dyh = y * (1-y) - * dE/dyh = 2 * y * (1-y) * (y-t) - * - * dyh/dW4 = s.transpose() - * dyh/db4 = Nd4j.ones(1, batchsz) - * dyh/ds = W4.tranpose() - * - * ds/dm = Nd4j.ones(1, midsz) - * ds/dn = Nd4j.ones(1, midsz).muli(-1) - * - * dm/dmh = 1-(m^2) - * - * dmh/dW1 = a.transpose() - * dmh/db1 = Nd4j.ones(1, batchsz) - * - */ - INDArray y = output; - INDArray s = middle; - INDArray W4 = output_W; - INDArray dEdy = Nd4j.zeros(target.shape()); - // This should be of size batchsz x outputsz - dEdy.addi(y).subi(target).muli(2); - // Why? Because the LossFunction divides by the _element size_ of the output. - dEdy.divi(target.shape()[1]); - // This is of size batchsz x outputsz - INDArray dydyh = y.mul(y.mul(-1).add(1)); - INDArray dEdyh = dydyh.mul(dEdy); - INDArray dyhdW4 = s.transpose(); - INDArray dEdW4 = nullsafe(dyhdW4.mmul(dEdyh)); - INDArray dyhdb4 = Nd4j.ones(1, batchsz); - INDArray dEdb4 = nullsafe(dyhdb4.mmul(dEdyh)); - INDArray dyhds = W4.transpose(); - INDArray dEds = dEdyh.mmul(dyhds); - INDArray dsdm = Nd4j.ones(batchsz, midsz); - INDArray dEdm = dsdm.mul(dEds); - INDArray dmdmh = (m.mul(m)).mul(-1).add(1); - INDArray dEdmh = dmdmh.mul(dEdm); - INDArray dmhdW1 = input1.transpose(); - INDArray dEdW1 = nullsafe(dmhdW1.mmul(dEdmh)); - INDArray dmhdb1 = Nd4j.ones(1, batchsz); - INDArray dEdb1 = nullsafe(dmhdb1.mmul(dEdmh)); - INDArray dsdn = Nd4j.ones(batchsz, midsz).muli(-1); - INDArray dEdn = dsdn.mul(dEds); - INDArray dndnh = (n.mul(n)).mul(-1).add(1); - INDArray dEdnh = dndnh.mul(dEdn); - INDArray dnhdW2 = input2.transpose(); - INDArray dEdW2 = nullsafe(dnhdW2.mmul(dEdnh)); - INDArray dnhdb2 = Nd4j.ones(1, batchsz); - INDArray dEdb2 = nullsafe(dnhdb2.mmul(dEdnh)); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_W")), dEdW4), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("output_b")), dEdb4), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_W")), dEdW1), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense1_b")), dEdb1), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_W")), dEdW2), this.epsilon); - Assertions.assertEquals(0, mse(nullsafe(gradients.get("dense2_b")), dEdb2), this.epsilon); - } - - private static double mse(INDArray output, INDArray target) { - double mse_expect = Transforms.pow(output.sub(target), 2.0).sumNumber().doubleValue() / (output.columns() * output.rows()); - return mse_expect; - } - - private static T nullsafe(T obj) { - if (obj == null) - throw new NullPointerException(); - T clean = obj; - return clean; - } - - private double epsilon = 1e-10; -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/graph/ShiftVertexTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/graph/ShiftVertexTest.java deleted file mode 100644 index 6631c426d..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/graph/ShiftVertexTest.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf.graph; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.ActivationLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.activations.BaseActivationFunction; -import org.nd4j.linalg.activations.impl.ActivationSigmoid; -import org.nd4j.linalg.activations.impl.ActivationTanH; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; -import org.nd4j.common.primitives.Pair; -import java.util.Map; -import java.util.TreeMap; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Shift Vertex Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class ShiftVertexTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Shift Vertex Num Params True") - void testShiftVertexNumParamsTrue() { - /* - * https://github.com/eclipse/deeplearning4j/pull/3514#issuecomment-307754386 - * from @agibsonccc: check for the basics: like 0 numParams - */ - // The 0.7 doesn't really matter. - ShiftVertex sv = new ShiftVertex(0.7); - Assertions.assertEquals(0, sv.numParams(true)); - } - - @Test - @DisplayName("Test Shift Vertex Num Params False") - void testShiftVertexNumParamsFalse() { - /* - * https://github.com/eclipse/deeplearning4j/pull/3514#issuecomment-307754386 - * from @agibsonccc: check for the basics: like 0 numParams - */ - // The 0.7 doesn't really matter. - ShiftVertex sv = new ShiftVertex(0.7); - Assertions.assertEquals(0, sv.numParams(false)); - } - - @Test - @DisplayName("Test Get") - void testGet() { - ShiftVertex sv = new ShiftVertex(0.7); - Assertions.assertEquals(0.7, sv.getShiftFactor(), this.epsilon); - } - - @Test - @DisplayName("Test Simple") - void testSimple() { - /* - * This function _simply_ tests whether ShiftVertex is _in fact_ adding the shift value to it's inputs. - */ - // Just first n primes / 10. - INDArray input = Nd4j.create(new double[][] { { 0.2, 0.3, 0.5 }, { 0.7, 1.1, 1.3 }, { 1.7, 1.9, 2.3 }, { 2.9, 3.1, 3.7 } }); - double sf = 4.1; - ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("input").addLayer("denselayer", new DenseLayer.Builder().nIn(input.columns()).nOut(1).activation(Activation.IDENTITY).build(), "input").addLayer("identityinputactivation", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), "input").addVertex("shiftvertex", new ShiftVertex(sf), "identityinputactivation").addLayer("identityshiftvertex", new ActivationLayer.Builder().activation(Activation.IDENTITY).build(), "shiftvertex").setOutputs("identityshiftvertex", "denselayer").build(); - ComputationGraph cg = new ComputationGraph(cgc); - cg.init(); - // We can call outputSingle, because we only have a single output layer. It has nothing to do with minibatches. - INDArray output = cg.output(true, input)[0]; - INDArray target = Nd4j.zeros(input.shape()); - target.addi(input); - target.addi(sf); - INDArray squared = output.sub(target); - double rms = squared.mul(squared).sumNumber().doubleValue(); - Assertions.assertEquals(0.0, rms, this.epsilon); - } - - @Test - @DisplayName("Test Comprehensive") - void testComprehensive() { - /* - * This function tests ShiftVertex more comprehensively. Specifically, it verifies that the lossfunction works as - * expected on a ComputationGraph _with_ a ShiftVertex and it verifies that the derivatives produced by - * back propagating work as expected. - */ - BaseActivationFunction a1 = new ActivationTanH(); - BaseActivationFunction a2 = new ActivationSigmoid(); - // Just first n primes / 10. - INDArray input = Nd4j.create(new double[][] { { 0.2, 0.3, 0.5 }, { 0.7, 1.1, 1.3 }, { 1.7, 1.9, 2.3 }, { 2.9, 3.1, 3.7 } }); - double sf = 4.1; - // Actually, given that I'm using a sigmoid on the output, - // these should really be between 0 and 1 - INDArray target = Nd4j.create(new double[][] { { 0.05, 0.10, 0.15, 0.20, 0.25 }, { 0.30, 0.35, 0.40, 0.45, 0.50 }, { 0.55, 0.60, 0.65, 0.70, 0.75 }, { 0.80, 0.85, 0.90, 0.95, 0.99 } }); - ComputationGraphConfiguration cgc = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).dataType(DataType.DOUBLE).updater(new Sgd(0.01)).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder().addInputs("input").addLayer("denselayer", new DenseLayer.Builder().nIn(input.columns()).nOut(input.columns()).activation(a1).build(), "input").addVertex("shiftvertex", new ShiftVertex(sf), "denselayer").addLayer("output", new OutputLayer.Builder().nIn(input.columns()).nOut(target.columns()).activation(a2).lossFunction(LossFunction.MSE).build(), "shiftvertex").setOutputs("output").build(); - ComputationGraph cg = new ComputationGraph(cgc); - cg.init(); - cg.setInput(0, input); - cg.setLabel(0, target); - cg.computeGradientAndScore(); - double score_dl4j = cg.score(); - Map weights = cg.paramTable(); - Gradient g = cg.gradient(); - Map gradients = g.gradientForVariable(); - Map manual_gradients = new TreeMap(); - INDArray W = nullsafe(weights.get("denselayer_W")); - INDArray b = nullsafe(weights.get("denselayer_b")); - INDArray V = nullsafe(weights.get("output_W")); - INDArray c = nullsafe(weights.get("output_b")); - Map manual_weights = new TreeMap(); - manual_weights.put("denselayer_W", W); - manual_weights.put("denselayer_b", b); - manual_weights.put("output_W", V); - manual_weights.put("output_b", c); - // First things first, let's calculate the score. - long batchsz = input.shape()[0]; - INDArray z = input.castTo(W.dataType()).mmul(W).add(b.repmat(batchsz, 1)); - // activation modifies it's input!! - INDArray a = a1.getActivation(z.dup(), true).add(sf); - INDArray q = a.mmul(V).add(c.repmat(batchsz, 1)); - INDArray o = nullsafe(a2.getActivation(q.dup(), true)); - double score_manual = sum_errors(o, target) / (o.columns() * o.rows()); - /* - * So. We have - * z5 = input1 * W15 + input2 * W25 + input3 * W35 + b5 - * a5 = activation(z5) + sr - * q9 = a1 * V19 + a2 * V29 + a3 * V39 + c9 - * o9 = activation(q9) - * - * dE/do = 2(o-t) - * doj/dqj = activation'(qj) - * dqj/dVij = ai dqj/dai = Vij dqj/dbj = 1 - * - * dq1/dv11 = a1 dq2/dV12 = a1 dq3/dV13 = a1 ... - * dq1/dv21 = a2 dq2... - */ - // Nd4j.zeros(target.shape()); - INDArray dEdo = target.like(); - // This should be of size batchsz x outputsz - dEdo.addi(o.castTo(dEdo.dataType())).subi(target).muli(2); - // Why? Because the LossFunction divides by the _element size_ of the output. - dEdo.divi(target.shape()[1]); - Pair derivs2 = a2.backprop(q, dEdo); - // This should be of size batchsz x outputsz (dE/do * do/dq) this _should_ be o * (1-o) * dE/do for Sigmoid. - INDArray dEdq = derivs2.getFirst(); - // Should be o = q^3 do/dq = 3 q^2 for Cube. - /* - INDArray dodq = q.mul(q).mul(3); - INDArray tbv = dodq.mul(dEdo); - System.err.println("----"); - System.err.println(q); - System.err.println(o); - System.err.println(tbv); - System.err.println(dEdq); - */ - INDArray dqdc = Nd4j.ones(1, batchsz); - // This should be of size 1 x outputsz - INDArray dEdc = dqdc.mmul(dEdq); - INDArray dEdV = a.transpose().mmul(dEdq); - // This should be dEdo * dodq * dqda - INDArray dEda = dEdq.mmul(V.transpose()); - Pair derivs1 = a1.backprop(z, dEda); - INDArray dEdz = derivs1.getFirst(); - INDArray dzdb = Nd4j.ones(1, batchsz); - INDArray dEdb = dzdb.mmul(dEdz); - INDArray dEdW = input.transpose().mmul(dEdz); - manual_gradients.put("output_b", dEdc); - manual_gradients.put("output_W", dEdV); - manual_gradients.put("denselayer_b", dEdb); - manual_gradients.put("denselayer_W", dEdW); - double summse = Math.pow((score_manual - score_dl4j), 2); - int denominator = 1; - for (Map.Entry mesi : gradients.entrySet()) { - String name = mesi.getKey(); - INDArray dl4j_gradient = nullsafe(mesi.getValue()); - INDArray manual_gradient = nullsafe(manual_gradients.get(name)); - double se = sum_errors(dl4j_gradient, manual_gradient); - summse += se; - denominator += dl4j_gradient.columns() * dl4j_gradient.rows(); - } - Assertions.assertEquals(0.0, summse / denominator, this.epsilon); - } - - private static double sum_errors(INDArray a, INDArray b) { - INDArray o = a.sub(b.castTo(a.dataType())); - return o.mul(o).sumNumber().doubleValue(); - } - - private static T nullsafe(T obj) { - if (obj == null) - throw new NullPointerException(); - T clean = obj; - return clean; - } - - private double epsilon = 1e-10; -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java deleted file mode 100644 index f9568b9d7..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerBuilderTest.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.dropout.Dropout; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.activations.IActivation; -import org.nd4j.linalg.activations.impl.ActivationSoftmax; -import org.nd4j.linalg.activations.impl.ActivationTanH; -import org.nd4j.linalg.convolution.Convolution; -import org.nd4j.linalg.learning.config.AdaGrad; -import org.nd4j.linalg.learning.config.IUpdater; -import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; -import java.io.*; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * @author Jeffrey Tang. - */ -@DisplayName("Layer Builder Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class LayerBuilderTest extends BaseDL4JTest { - - final double DELTA = 1e-15; - - int numIn = 10; - - int numOut = 5; - - double drop = 0.3; - - IActivation act = new ActivationSoftmax(); - - PoolingType poolType = PoolingType.MAX; - - int[] kernelSize = new int[] { 2, 2 }; - - int[] stride = new int[] { 2, 2 }; - - int[] padding = new int[] { 1, 1 }; - - int k = 1; - - Convolution.Type convType = Convolution.Type.VALID; - - LossFunction loss = LossFunction.MCXENT; - - WeightInit weight = WeightInit.XAVIER; - - double corrupt = 0.4; - - double sparsity = 0.3; - - double corruptionLevel = 0.5; - - double dropOut = 0.1; - - IUpdater updater = new AdaGrad(); - - GradientNormalization gradNorm = GradientNormalization.ClipL2PerParamType; - - double gradNormThreshold = 8; - - @Test - @DisplayName("Test Layer") - void testLayer() throws Exception { - DenseLayer layer = new DenseLayer.Builder().activation(act).weightInit(weight).dropOut(dropOut).updater(updater).gradientNormalization(gradNorm).gradientNormalizationThreshold(gradNormThreshold).build(); - checkSerialization(layer); - assertEquals(act, layer.getActivationFn()); - assertEquals(weight.getWeightInitFunction(), layer.getWeightInitFn()); - assertEquals(new Dropout(dropOut), layer.getIDropout()); - assertEquals(updater, layer.getIUpdater()); - assertEquals(gradNorm, layer.getGradientNormalization()); - assertEquals(gradNormThreshold, layer.getGradientNormalizationThreshold(), 0.0); - } - - @Test - @DisplayName("Test Feed Forward Layer") - void testFeedForwardLayer() throws Exception { - DenseLayer ff = new DenseLayer.Builder().nIn(numIn).nOut(numOut).build(); - checkSerialization(ff); - assertEquals(numIn, ff.getNIn()); - assertEquals(numOut, ff.getNOut()); - } - - @Test - @DisplayName("Test Convolution Layer") - void testConvolutionLayer() throws Exception { - ConvolutionLayer conv = new ConvolutionLayer.Builder(kernelSize, stride, padding).build(); - checkSerialization(conv); - // assertEquals(convType, conv.getConvolutionType()); - assertArrayEquals(kernelSize, conv.getKernelSize()); - assertArrayEquals(stride, conv.getStride()); - assertArrayEquals(padding, conv.getPadding()); - } - - @Test - @DisplayName("Test Subsampling Layer") - void testSubsamplingLayer() throws Exception { - SubsamplingLayer sample = new SubsamplingLayer.Builder(poolType, stride).kernelSize(kernelSize).padding(padding).build(); - checkSerialization(sample); - assertArrayEquals(padding, sample.getPadding()); - assertArrayEquals(kernelSize, sample.getKernelSize()); - assertEquals(poolType, sample.getPoolingType()); - assertArrayEquals(stride, sample.getStride()); - } - - @Test - @DisplayName("Test Output Layer") - void testOutputLayer() throws Exception { - OutputLayer out = new OutputLayer.Builder(loss).build(); - checkSerialization(out); - } - - @Test - @DisplayName("Test Rnn Output Layer") - void testRnnOutputLayer() throws Exception { - RnnOutputLayer out = new RnnOutputLayer.Builder(loss).build(); - checkSerialization(out); - } - - @Test - @DisplayName("Test Auto Encoder") - void testAutoEncoder() throws Exception { - AutoEncoder enc = new AutoEncoder.Builder().corruptionLevel(corruptionLevel).sparsity(sparsity).build(); - checkSerialization(enc); - assertEquals(corruptionLevel, enc.getCorruptionLevel(), DELTA); - assertEquals(sparsity, enc.getSparsity(), DELTA); - } - - @Test - @DisplayName("Test Graves LSTM") - void testGravesLSTM() throws Exception { - GravesLSTM glstm = new GravesLSTM.Builder().forgetGateBiasInit(1.5).activation(Activation.TANH).nIn(numIn).nOut(numOut).build(); - checkSerialization(glstm); - assertEquals(glstm.getForgetGateBiasInit(), 1.5, 0.0); - assertEquals(glstm.nIn, numIn); - assertEquals(glstm.nOut, numOut); - assertTrue(glstm.getActivationFn() instanceof ActivationTanH); - } - - @Test - @DisplayName("Test Graves Bidirectional LSTM") - void testGravesBidirectionalLSTM() throws Exception { - final GravesBidirectionalLSTM glstm = new GravesBidirectionalLSTM.Builder().forgetGateBiasInit(1.5).activation(Activation.TANH).nIn(numIn).nOut(numOut).build(); - checkSerialization(glstm); - assertEquals(1.5, glstm.getForgetGateBiasInit(), 0.0); - assertEquals(glstm.nIn, numIn); - assertEquals(glstm.nOut, numOut); - assertTrue(glstm.getActivationFn() instanceof ActivationTanH); - } - - @Test - @DisplayName("Test Embedding Layer") - void testEmbeddingLayer() throws Exception { - EmbeddingLayer el = new EmbeddingLayer.Builder().nIn(10).nOut(5).build(); - checkSerialization(el); - assertEquals(10, el.getNIn()); - assertEquals(5, el.getNOut()); - } - - @Test - @DisplayName("Test Batch Norm Layer") - void testBatchNormLayer() throws Exception { - BatchNormalization bN = new BatchNormalization.Builder().nIn(numIn).nOut(numOut).gamma(2).beta(1).decay(0.5).lockGammaBeta(true).build(); - checkSerialization(bN); - assertEquals(numIn, bN.nIn); - assertEquals(numOut, bN.nOut); - assertEquals(true, bN.isLockGammaBeta()); - assertEquals(0.5, bN.decay, 1e-4); - assertEquals(2, bN.gamma, 1e-4); - assertEquals(1, bN.beta, 1e-4); - } - - @Test - @DisplayName("Test Activation Layer") - void testActivationLayer() throws Exception { - ActivationLayer activationLayer = new ActivationLayer.Builder().activation(act).build(); - checkSerialization(activationLayer); - assertEquals(act, activationLayer.activationFn); - } - - private void checkSerialization(Layer layer) throws Exception { - NeuralNetConfiguration confExpected = new NeuralNetConfiguration.Builder().layer(layer).build(); - NeuralNetConfiguration confActual; - // check Java serialization - byte[] data; - try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutput out = new ObjectOutputStream(bos)) { - out.writeObject(confExpected); - data = bos.toByteArray(); - } - try (ByteArrayInputStream bis = new ByteArrayInputStream(data); - ObjectInput in = new ObjectInputStream(bis)) { - confActual = (NeuralNetConfiguration) in.readObject(); - } - assertEquals(confExpected.getLayer(), confActual.getLayer(), "unequal Java serialization"); - // check JSON - String json = confExpected.toJson(); - confActual = NeuralNetConfiguration.fromJson(json); - assertEquals(confExpected.getLayer(), confActual.getLayer(), "unequal JSON serialization"); - // check YAML - String yaml = confExpected.toYaml(); - confActual = NeuralNetConfiguration.fromYaml(yaml); - assertEquals(confExpected.getLayer(), confActual.getLayer(), "unequal YAML serialization"); - // check the layer's use of callSuper on equals method - confActual.getLayer().setIDropout(new Dropout(new java.util.Random().nextDouble())); - assertNotEquals(confExpected.getLayer(), confActual.getLayer(), "broken equals method (missing callSuper?)"); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java deleted file mode 100644 index 5da010a68..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigTest.java +++ /dev/null @@ -1,455 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.Distribution; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.distribution.UniformDistribution; -import org.deeplearning4j.nn.conf.dropout.Dropout; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInitDistribution; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.learning.config.AdaDelta; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.learning.config.RmsProp; -import org.nd4j.linalg.schedule.MapSchedule; -import org.nd4j.linalg.schedule.ScheduleType; -import java.util.HashMap; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/* - @Test - public void testLearningRatePolicyExponential() { - double lr = 2; - double lrDecayRate = 5; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr) - .updater(Updater.SGD) - .learningRateDecayPolicy(LearningRatePolicy.Exponential).lrPolicyDecayRate(lrDecayRate).list() - .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Exponential, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Exponential, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - } - - @Test - public void testLearningRatePolicyInverse() { - double lr = 2; - double lrDecayRate = 5; - double power = 3; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(lrDecayRate) - .lrPolicyPower(power).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Inverse, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Inverse, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - assertEquals(power, conf.getConf(0).getLrPolicyPower(), 0.0); - assertEquals(power, conf.getConf(1).getLrPolicyPower(), 0.0); - } - - - @Test - public void testLearningRatePolicySteps() { - double lr = 2; - double lrDecayRate = 5; - double steps = 4; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Step).lrPolicyDecayRate(lrDecayRate) - .lrPolicySteps(steps).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Step, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Step, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - assertEquals(steps, conf.getConf(0).getLrPolicySteps(), 0.0); - assertEquals(steps, conf.getConf(1).getLrPolicySteps(), 0.0); - } - - @Test - public void testLearningRatePolicyPoly() { - double lr = 2; - double lrDecayRate = 5; - double power = 3; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Poly).lrPolicyDecayRate(lrDecayRate) - .lrPolicyPower(power).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Poly, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Poly, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - assertEquals(power, conf.getConf(0).getLrPolicyPower(), 0.0); - assertEquals(power, conf.getConf(1).getLrPolicyPower(), 0.0); - } - - @Test - public void testLearningRatePolicySigmoid() { - double lr = 2; - double lrDecayRate = 5; - double steps = 4; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Sigmoid).lrPolicyDecayRate(lrDecayRate) - .lrPolicySteps(steps).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Sigmoid, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Sigmoid, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - assertEquals(steps, conf.getConf(0).getLrPolicySteps(), 0.0); - assertEquals(steps, conf.getConf(1).getLrPolicySteps(), 0.0); - } - -*/ -@DisplayName("Layer Config Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class LayerConfigTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Layer Name") - void testLayerName() { - String name1 = "genisys"; - String name2 = "bill"; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).name(name1).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).name(name2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(name1, conf.getConf(0).getLayer().getLayerName()); - assertEquals(name2, conf.getConf(1).getLayer().getLayerName()); - } - - @Test - @DisplayName("Test Activation Layerwise Override") - void testActivationLayerwiseOverride() { - // Without layerwise override: - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.RELU).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(((BaseLayer) conf.getConf(0).getLayer()).getActivationFn().toString(), "relu"); - assertEquals(((BaseLayer) conf.getConf(1).getLayer()).getActivationFn().toString(), "relu"); - // With - conf = new NeuralNetConfiguration.Builder().activation(Activation.RELU).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).activation(Activation.TANH).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(((BaseLayer) conf.getConf(0).getLayer()).getActivationFn().toString(), "relu"); - assertEquals(((BaseLayer) conf.getConf(1).getLayer()).getActivationFn().toString(), "tanh"); - } - - @Test - @DisplayName("Test Weight Bias Init Layerwise Override") - void testWeightBiasInitLayerwiseOverride() { - // Without layerwise override: - final Distribution defaultDistribution = new NormalDistribution(0, 1.0); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dist(defaultDistribution).biasInit(1).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(new WeightInitDistribution(defaultDistribution), ((BaseLayer) conf.getConf(0).getLayer()).getWeightInitFn()); - assertEquals(new WeightInitDistribution(defaultDistribution), ((BaseLayer) conf.getConf(1).getLayer()).getWeightInitFn()); - assertEquals(1, ((BaseLayer) conf.getConf(0).getLayer()).getBiasInit(), 0.0); - assertEquals(1, ((BaseLayer) conf.getConf(1).getLayer()).getBiasInit(), 0.0); - // With: - final Distribution overriddenDistribution = new UniformDistribution(0, 1); - conf = new NeuralNetConfiguration.Builder().dist(defaultDistribution).biasInit(1).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).dist(overriddenDistribution).biasInit(0).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(new WeightInitDistribution(defaultDistribution), ((BaseLayer) conf.getConf(0).getLayer()).getWeightInitFn()); - assertEquals(new WeightInitDistribution(overriddenDistribution), ((BaseLayer) conf.getConf(1).getLayer()).getWeightInitFn()); - assertEquals(1, ((BaseLayer) conf.getConf(0).getLayer()).getBiasInit(), 0.0); - assertEquals(0, ((BaseLayer) conf.getConf(1).getLayer()).getBiasInit(), 0.0); - } - - /* - @Test - public void testLrL1L2LayerwiseOverride() { - //Idea: Set some common values for all layers. Then selectively override - // the global config, and check they actually work. - - //Learning rate without layerwise override: - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(0.3).list() - .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(0.3, ((BaseLayer) conf.getConf(0).getLayer()).getLearningRate(), 0.0); - assertEquals(0.3, ((BaseLayer) conf.getConf(1).getLayer()).getLearningRate(), 0.0); - - //With: - conf = new NeuralNetConfiguration.Builder().learningRate(0.3).list() - .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).learningRate(0.2).build()).build(); - - net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(0.3, ((BaseLayer) conf.getConf(0).getLayer()).getLearningRate(), 0.0); - assertEquals(0.2, ((BaseLayer) conf.getConf(1).getLayer()).getLearningRate(), 0.0); - - //L1 and L2 without layerwise override: - conf = new NeuralNetConfiguration.Builder().l1(0.1).l2(0.2).list() - .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(0.1, ((BaseLayer) conf.getConf(0).getLayer()).getL1(), 0.0); - assertEquals(0.1, ((BaseLayer) conf.getConf(1).getLayer()).getL1(), 0.0); - assertEquals(0.2, ((BaseLayer) conf.getConf(0).getLayer()).getL2(), 0.0); - assertEquals(0.2, ((BaseLayer) conf.getConf(1).getLayer()).getL2(), 0.0); - - //L1 and L2 with layerwise override: - conf = new NeuralNetConfiguration.Builder().l1(0.1).l2(0.2).list() - .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l1(0.9).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.8).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(0.9, ((BaseLayer) conf.getConf(0).getLayer()).getL1(), 0.0); - assertEquals(0.1, ((BaseLayer) conf.getConf(1).getLayer()).getL1(), 0.0); - assertEquals(0.2, ((BaseLayer) conf.getConf(0).getLayer()).getL2(), 0.0); - assertEquals(0.8, ((BaseLayer) conf.getConf(1).getLayer()).getL2(), 0.0); - }*/ - @Test - @DisplayName("Test Dropout Layerwise Override") - void testDropoutLayerwiseOverride() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dropOut(1.0).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(new Dropout(1.0), conf.getConf(0).getLayer().getIDropout()); - assertEquals(new Dropout(1.0), conf.getConf(1).getLayer().getIDropout()); - conf = new NeuralNetConfiguration.Builder().dropOut(1.0).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).dropOut(2.0).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(new Dropout(1.0), conf.getConf(0).getLayer().getIDropout()); - assertEquals(new Dropout(2.0), conf.getConf(1).getLayer().getIDropout()); - } - - @Test - @DisplayName("Test Momentum Layerwise Override") - void testMomentumLayerwiseOverride() { - Map testMomentumAfter = new HashMap<>(); - testMomentumAfter.put(0, 0.1); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(1.0, new MapSchedule(ScheduleType.ITERATION, testMomentumAfter))).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(0.1, ((Nesterovs) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0, 0), 0.0); - assertEquals(0.1, ((Nesterovs) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0, 0), 0.0); - Map testMomentumAfter2 = new HashMap<>(); - testMomentumAfter2.put(0, 0.2); - conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(1.0, new MapSchedule(ScheduleType.ITERATION, testMomentumAfter))).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new Nesterovs(1.0, new MapSchedule(ScheduleType.ITERATION, testMomentumAfter2))).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(0.1, ((Nesterovs) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0, 0), 0.0); - assertEquals(0.2, ((Nesterovs) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getMomentumISchedule().valueAt(0, 0), 0.0); - } - - @Test - @DisplayName("Test Updater Rho Rms Decay Layerwise Override") - void testUpdaterRhoRmsDecayLayerwiseOverride() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new AdaDelta(0.5, 0.9)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new AdaDelta(0.01, 0.9)).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertTrue(((BaseLayer) conf.getConf(0).getLayer()).getIUpdater() instanceof AdaDelta); - assertTrue(((BaseLayer) conf.getConf(1).getLayer()).getIUpdater() instanceof AdaDelta); - assertEquals(0.5, ((AdaDelta) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getRho(), 0.0); - assertEquals(0.01, ((AdaDelta) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getRho(), 0.0); - conf = new NeuralNetConfiguration.Builder().updater(new RmsProp(1.0, 2.0, RmsProp.DEFAULT_RMSPROP_EPSILON)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).updater(new RmsProp(1.0, 1.0, RmsProp.DEFAULT_RMSPROP_EPSILON)).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new AdaDelta(0.5, AdaDelta.DEFAULT_ADADELTA_EPSILON)).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - assertTrue(((BaseLayer) conf.getConf(0).getLayer()).getIUpdater() instanceof RmsProp); - assertTrue(((BaseLayer) conf.getConf(1).getLayer()).getIUpdater() instanceof AdaDelta); - assertEquals(1.0, ((RmsProp) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getRmsDecay(), 0.0); - assertEquals(0.5, ((AdaDelta) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getRho(), 0.0); - } - - @Test - @DisplayName("Test Updater Adam Params Layerwise Override") - void testUpdaterAdamParamsLayerwiseOverride() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Adam(1.0, 0.5, 0.5, 1e-8)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new Adam(1.0, 0.6, 0.7, 1e-8)).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta1(), 0.0); - assertEquals(0.6, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta1(), 0.0); - assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta2(), 0.0); - assertEquals(0.7, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta2(), 0.0); - } - - @Test - @DisplayName("Test Gradient Normalization Layerwise Override") - void testGradientNormalizationLayerwiseOverride() { - // Learning rate without layerwise override: - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, ((BaseLayer) conf.getConf(0).getLayer()).getGradientNormalization()); - assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, ((BaseLayer) conf.getConf(1).getLayer()).getGradientNormalization()); - assertEquals(10, ((BaseLayer) conf.getConf(0).getLayer()).getGradientNormalizationThreshold(), 0.0); - assertEquals(10, ((BaseLayer) conf.getConf(1).getLayer()).getGradientNormalizationThreshold(), 0.0); - // With: - conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).gradientNormalization(GradientNormalization.None).gradientNormalizationThreshold(2.5).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(GradientNormalization.ClipElementWiseAbsoluteValue, ((BaseLayer) conf.getConf(0).getLayer()).getGradientNormalization()); - assertEquals(GradientNormalization.None, ((BaseLayer) conf.getConf(1).getLayer()).getGradientNormalization()); - assertEquals(10, ((BaseLayer) conf.getConf(0).getLayer()).getGradientNormalizationThreshold(), 0.0); - assertEquals(2.5, ((BaseLayer) conf.getConf(1).getLayer()).getGradientNormalizationThreshold(), 0.0); - } - /* - @Test - public void testLearningRatePolicyExponential() { - double lr = 2; - double lrDecayRate = 5; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().learningRate(lr) - .updater(Updater.SGD) - .learningRateDecayPolicy(LearningRatePolicy.Exponential).lrPolicyDecayRate(lrDecayRate).list() - .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Exponential, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Exponential, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - } - - @Test - public void testLearningRatePolicyInverse() { - double lr = 2; - double lrDecayRate = 5; - double power = 3; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(lrDecayRate) - .lrPolicyPower(power).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Inverse, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Inverse, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - assertEquals(power, conf.getConf(0).getLrPolicyPower(), 0.0); - assertEquals(power, conf.getConf(1).getLrPolicyPower(), 0.0); - } - - - @Test - public void testLearningRatePolicySteps() { - double lr = 2; - double lrDecayRate = 5; - double steps = 4; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Step).lrPolicyDecayRate(lrDecayRate) - .lrPolicySteps(steps).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Step, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Step, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - assertEquals(steps, conf.getConf(0).getLrPolicySteps(), 0.0); - assertEquals(steps, conf.getConf(1).getLrPolicySteps(), 0.0); - } - - @Test - public void testLearningRatePolicyPoly() { - double lr = 2; - double lrDecayRate = 5; - double power = 3; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Poly).lrPolicyDecayRate(lrDecayRate) - .lrPolicyPower(power).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Poly, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Poly, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - assertEquals(power, conf.getConf(0).getLrPolicyPower(), 0.0); - assertEquals(power, conf.getConf(1).getLrPolicyPower(), 0.0); - } - - @Test - public void testLearningRatePolicySigmoid() { - double lr = 2; - double lrDecayRate = 5; - double steps = 4; - int iterations = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().iterations(iterations).learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Sigmoid).lrPolicyDecayRate(lrDecayRate) - .lrPolicySteps(steps).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()) - .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - assertEquals(LearningRatePolicy.Sigmoid, conf.getConf(0).getLearningRatePolicy()); - assertEquals(LearningRatePolicy.Sigmoid, conf.getConf(1).getLearningRatePolicy()); - assertEquals(lrDecayRate, conf.getConf(0).getLrPolicyDecayRate(), 0.0); - assertEquals(lrDecayRate, conf.getConf(1).getLrPolicyDecayRate(), 0.0); - assertEquals(steps, conf.getConf(0).getLrPolicySteps(), 0.0); - assertEquals(steps, conf.getConf(1).getLrPolicySteps(), 0.0); - } - -*/ -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java deleted file mode 100644 index 68ff2e7fa..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.Updater; -import org.deeplearning4j.nn.conf.distribution.Distribution; -import org.deeplearning4j.nn.conf.distribution.GaussianDistribution; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.weightnoise.DropConnect; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.weights.WeightInitDistribution; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.learning.config.RmsProp; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.schedule.MapSchedule; -import org.nd4j.linalg.schedule.ScheduleType; -import java.util.HashMap; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.jupiter.api.Assertions.assertThrows; - -@DisplayName("Layer Config Validation Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class LayerConfigValidationTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Drop Connect") - void testDropConnect() { - // Warning thrown only since some layers may not have l1 or l2 - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).weightNoise(new DropConnect(0.5)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - } - - @Test - @DisplayName("Test L 1 L 2 Not Set") - void testL1L2NotSet() { - // Warning thrown only since some layers may not have l1 or l2 - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - } - - @Test - @Disabled - @DisplayName("Test Reg Not Set L 1 Global") - void testRegNotSetL1Global() { - assertThrows(IllegalStateException.class, () -> { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).l1(0.5).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - }); - } - - - - @Test - @DisplayName("Test Weight Init Dist Not Set") - void testWeightInitDistNotSet() { - // Warning thrown only since global dist can be set with a different weight init locally - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).dist(new GaussianDistribution(1e-3, 2)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - } - - @Test - @DisplayName("Test Nesterovs Not Set Global") - void testNesterovsNotSetGlobal() { - // Warnings only thrown - Map testMomentumAfter = new HashMap<>(); - testMomentumAfter.put(0, 0.1); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(1.0, new MapSchedule(ScheduleType.ITERATION, testMomentumAfter))).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - } - - @Test - @DisplayName("Test Comp Graph Null Layer") - void testCompGraphNullLayer() { - ComputationGraphConfiguration.GraphBuilder gb = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.01)).seed(42).miniBatch(false).l1(0.2).l2(0.2).updater(Updater.RMSPROP).graphBuilder().addInputs("in").addLayer("L" + 1, new GravesLSTM.Builder().nIn(20).updater(Updater.RMSPROP).nOut(10).weightInit(WeightInit.XAVIER).dropOut(0.4).l1(0.3).activation(Activation.SIGMOID).build(), "in").addLayer("output", new RnnOutputLayer.Builder().nIn(20).nOut(10).activation(Activation.SOFTMAX).weightInit(WeightInit.RELU_UNIFORM).build(), "L" + 1).setOutputs("output"); - ComputationGraphConfiguration conf = gb.build(); - ComputationGraph cg = new ComputationGraph(conf); - cg.init(); - } - - @Test - @DisplayName("Test Predefined Config Values") - void testPredefinedConfigValues() { - double expectedMomentum = 0.9; - double expectedAdamMeanDecay = 0.9; - double expectedAdamVarDecay = 0.999; - double expectedRmsDecay = 0.95; - Distribution expectedDist = new NormalDistribution(0, 1); - double expectedL1 = 0.0; - double expectedL2 = 0.0; - // Nesterovs Updater - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(0.9)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.5).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new Nesterovs(0.3, 0.4)).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - BaseLayer layerConf = (BaseLayer) net.getLayer(0).conf().getLayer(); - assertEquals(expectedMomentum, ((Nesterovs) layerConf.getIUpdater()).getMomentum(), 1e-3); - assertNull(TestUtils.getL1Reg(layerConf.getRegularization())); - assertEquals(0.5, TestUtils.getL2(layerConf), 1e-3); - BaseLayer layerConf1 = (BaseLayer) net.getLayer(1).conf().getLayer(); - assertEquals(0.4, ((Nesterovs) layerConf1.getIUpdater()).getMomentum(), 1e-3); - // Adam Updater - conf = new NeuralNetConfiguration.Builder().updater(new Adam(0.3)).weightInit(new WeightInitDistribution(expectedDist)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.5).l1(0.3).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - layerConf = (BaseLayer) net.getLayer(0).conf().getLayer(); - assertEquals(0.3, TestUtils.getL1(layerConf), 1e-3); - assertEquals(0.5, TestUtils.getL2(layerConf), 1e-3); - layerConf1 = (BaseLayer) net.getLayer(1).conf().getLayer(); - assertEquals(expectedAdamMeanDecay, ((Adam) layerConf1.getIUpdater()).getBeta1(), 1e-3); - assertEquals(expectedAdamVarDecay, ((Adam) layerConf1.getIUpdater()).getBeta2(), 1e-3); - assertEquals(new WeightInitDistribution(expectedDist), layerConf1.getWeightInitFn()); - assertNull(TestUtils.getL1Reg(layerConf1.getRegularization())); - assertNull(TestUtils.getL2Reg(layerConf1.getRegularization())); - // RMSProp Updater - conf = new NeuralNetConfiguration.Builder().updater(new RmsProp(0.3)).list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build()).layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new RmsProp(0.3, 0.4, RmsProp.DEFAULT_RMSPROP_EPSILON)).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - layerConf = (BaseLayer) net.getLayer(0).conf().getLayer(); - assertEquals(expectedRmsDecay, ((RmsProp) layerConf.getIUpdater()).getRmsDecay(), 1e-3); - assertNull(TestUtils.getL1Reg(layerConf.getRegularization())); - assertNull(TestUtils.getL2Reg(layerConf.getRegularization())); - layerConf1 = (BaseLayer) net.getLayer(1).conf().getLayer(); - assertEquals(0.4, ((RmsProp) layerConf1.getIUpdater()).getRmsDecay(), 1e-3); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CNNProcessorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CNNProcessorTest.java deleted file mode 100644 index 9e083f7d9..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CNNProcessorTest.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf.preprocessor; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.*; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - */ -@DisplayName("Cnn Processor Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class CNNProcessorTest extends BaseDL4JTest { - - private static int rows = 28; - - private static int cols = 28; - - private static INDArray in2D = Nd4j.create(DataType.FLOAT, 1, 784); - - private static INDArray in3D = Nd4j.create(DataType.FLOAT, 20, 784, 7); - - private static INDArray in4D = Nd4j.create(DataType.FLOAT, 20, 1, 28, 28); - - @Test - @DisplayName("Test Feed Forward To Cnn Pre Processor") - void testFeedForwardToCnnPreProcessor() { - FeedForwardToCnnPreProcessor convProcessor = new FeedForwardToCnnPreProcessor(rows, cols, 1); - INDArray check2to4 = convProcessor.preProcess(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); - int val2to4 = check2to4.shape().length; - assertTrue(val2to4 == 4); - assertEquals(Nd4j.create(DataType.FLOAT, 1, 1, 28, 28), check2to4); - INDArray check4to4 = convProcessor.preProcess(in4D, -1, LayerWorkspaceMgr.noWorkspaces()); - int val4to4 = check4to4.shape().length; - assertTrue(val4to4 == 4); - assertEquals(Nd4j.create(DataType.FLOAT, 20, 1, 28, 28), check4to4); - } - - @Test - @DisplayName("Test Feed Forward To Cnn Pre Processor 2") - void testFeedForwardToCnnPreProcessor2() { - int[] nRows = { 1, 5, 20 }; - int[] nCols = { 1, 5, 20 }; - int[] nDepth = { 1, 3 }; - int[] nMiniBatchSize = { 1, 5 }; - for (int rows : nRows) { - for (int cols : nCols) { - for (int d : nDepth) { - FeedForwardToCnnPreProcessor convProcessor = new FeedForwardToCnnPreProcessor(rows, cols, d); - for (int miniBatch : nMiniBatchSize) { - long[] ffShape = new long[] { miniBatch, rows * cols * d }; - INDArray rand = Nd4j.rand(ffShape); - INDArray ffInput_c = Nd4j.create(DataType.FLOAT, ffShape, 'c'); - INDArray ffInput_f = Nd4j.create(DataType.FLOAT, ffShape, 'f'); - ffInput_c.assign(rand); - ffInput_f.assign(rand); - assertEquals(ffInput_c, ffInput_f); - // Test forward pass: - INDArray convAct_c = convProcessor.preProcess(ffInput_c, -1, LayerWorkspaceMgr.noWorkspaces()); - INDArray convAct_f = convProcessor.preProcess(ffInput_f, -1, LayerWorkspaceMgr.noWorkspaces()); - long[] convShape = { miniBatch, d, rows, cols }; - assertArrayEquals(convShape, convAct_c.shape()); - assertArrayEquals(convShape, convAct_f.shape()); - assertEquals(convAct_c, convAct_f); - // Check values: - // CNN reshaping (for each example) takes a 1d vector and converts it to 3d - // (4d total, for minibatch data) - // 1d vector is assumed to be rows from channels 0 concatenated, followed by channels 1, etc - for (int ex = 0; ex < miniBatch; ex++) { - for (int r = 0; r < rows; r++) { - for (int c = 0; c < cols; c++) { - for (int depth = 0; depth < d; depth++) { - // pos in vector - int origPosition = depth * (rows * cols) + r * cols + c; - double vecValue = ffInput_c.getDouble(ex, origPosition); - double convValue = convAct_c.getDouble(ex, depth, r, c); - assertEquals(vecValue, convValue, 0.0); - } - } - } - } - // Test backward pass: - // Idea is that backward pass should do opposite to forward pass - INDArray epsilon4_c = Nd4j.create(DataType.FLOAT, convShape, 'c'); - INDArray epsilon4_f = Nd4j.create(DataType.FLOAT, convShape, 'f'); - epsilon4_c.assign(convAct_c); - epsilon4_f.assign(convAct_f); - INDArray epsilon2_c = convProcessor.backprop(epsilon4_c, -1, LayerWorkspaceMgr.noWorkspaces()); - INDArray epsilon2_f = convProcessor.backprop(epsilon4_f, -1, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(ffInput_c, epsilon2_c); - assertEquals(ffInput_c, epsilon2_f); - } - } - } - } - } - - @Test - @DisplayName("Test Feed Forward To Cnn Pre Processor Backprop") - void testFeedForwardToCnnPreProcessorBackprop() { - FeedForwardToCnnPreProcessor convProcessor = new FeedForwardToCnnPreProcessor(rows, cols, 1); - convProcessor.preProcess(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); - INDArray check2to2 = convProcessor.backprop(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); - int val2to2 = check2to2.shape().length; - assertTrue(val2to2 == 2); - assertEquals(Nd4j.create(DataType.FLOAT, 1, 784), check2to2); - } - - @Test - @DisplayName("Test Cnn To Feed Forward Processor") - void testCnnToFeedForwardProcessor() { - CnnToFeedForwardPreProcessor convProcessor = new CnnToFeedForwardPreProcessor(rows, cols, 1); - INDArray check2to4 = convProcessor.backprop(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); - int val2to4 = check2to4.shape().length; - assertTrue(val2to4 == 4); - assertEquals(Nd4j.create(DataType.FLOAT, 1, 1, 28, 28), check2to4); - INDArray check4to4 = convProcessor.backprop(in4D, -1, LayerWorkspaceMgr.noWorkspaces()); - int val4to4 = check4to4.shape().length; - assertTrue(val4to4 == 4); - assertEquals(Nd4j.create(DataType.FLOAT, 20, 1, 28, 28), check4to4); - } - - @Test - @DisplayName("Test Cnn To Feed Forward Pre Processor Backprop") - void testCnnToFeedForwardPreProcessorBackprop() { - CnnToFeedForwardPreProcessor convProcessor = new CnnToFeedForwardPreProcessor(rows, cols, 1); - convProcessor.preProcess(in4D, -1, LayerWorkspaceMgr.noWorkspaces()); - INDArray check2to2 = convProcessor.preProcess(in2D, -1, LayerWorkspaceMgr.noWorkspaces()); - int val2to2 = check2to2.shape().length; - assertTrue(val2to2 == 2); - assertEquals(Nd4j.create(DataType.FLOAT, 1, 784), check2to2); - INDArray check4to2 = convProcessor.preProcess(in4D, -1, LayerWorkspaceMgr.noWorkspaces()); - int val4to2 = check4to2.shape().length; - assertTrue(val4to2 == 2); - assertEquals(Nd4j.create(DataType.FLOAT, 20, 784), check4to2); - } - - @Test - @DisplayName("Test Cnn To Feed Forward Pre Processor 2") - void testCnnToFeedForwardPreProcessor2() { - int[] nRows = { 1, 5, 20 }; - int[] nCols = { 1, 5, 20 }; - int[] nDepth = { 1, 3 }; - int[] nMiniBatchSize = { 1, 5 }; - for (int rows : nRows) { - for (int cols : nCols) { - for (int d : nDepth) { - CnnToFeedForwardPreProcessor convProcessor = new CnnToFeedForwardPreProcessor(rows, cols, d); - for (int miniBatch : nMiniBatchSize) { - long[] convActShape = new long[] { miniBatch, d, rows, cols }; - INDArray rand = Nd4j.rand(convActShape); - INDArray convInput_c = Nd4j.create(DataType.FLOAT, convActShape, 'c'); - INDArray convInput_f = Nd4j.create(DataType.FLOAT, convActShape, 'f'); - convInput_c.assign(rand); - convInput_f.assign(rand); - assertEquals(convInput_c, convInput_f); - // Test forward pass: - INDArray ffAct_c = convProcessor.preProcess(convInput_c, -1, LayerWorkspaceMgr.noWorkspaces()); - INDArray ffAct_f = convProcessor.preProcess(convInput_f, -1, LayerWorkspaceMgr.noWorkspaces()); - long[] ffActShape = { miniBatch, d * rows * cols }; - assertArrayEquals(ffActShape, ffAct_c.shape()); - assertArrayEquals(ffActShape, ffAct_f.shape()); - assertEquals(ffAct_c, ffAct_f); - // Check values: - // CNN reshaping (for each example) takes a 1d vector and converts it to 3d - // (4d total, for minibatch data) - // 1d vector is assumed to be rows from channels 0 concatenated, followed by channels 1, etc - for (int ex = 0; ex < miniBatch; ex++) { - for (int r = 0; r < rows; r++) { - for (int c = 0; c < cols; c++) { - for (int depth = 0; depth < d; depth++) { - // pos in vector after reshape - int vectorPosition = depth * (rows * cols) + r * cols + c; - double vecValue = ffAct_c.getDouble(ex, vectorPosition); - double convValue = convInput_c.getDouble(ex, depth, r, c); - assertEquals(convValue, vecValue, 0.0); - } - } - } - } - // Test backward pass: - // Idea is that backward pass should do opposite to forward pass - INDArray epsilon2_c = Nd4j.create(DataType.FLOAT, ffActShape, 'c'); - INDArray epsilon2_f = Nd4j.create(DataType.FLOAT, ffActShape, 'f'); - epsilon2_c.assign(ffAct_c); - epsilon2_f.assign(ffAct_c); - INDArray epsilon4_c = convProcessor.backprop(epsilon2_c, -1, LayerWorkspaceMgr.noWorkspaces()); - INDArray epsilon4_f = convProcessor.backprop(epsilon2_f, -1, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(convInput_c, epsilon4_c); - assertEquals(convInput_c, epsilon4_f); - } - } - } - } - } - - @Test - @DisplayName("Test Invalid Input Shape") - void testInvalidInputShape() { - NeuralNetConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).miniBatch(true).cacheMode(CacheMode.DEVICE).updater(new Nesterovs(0.9)).gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT); - int[] kernelArray = new int[] { 3, 3 }; - int[] strideArray = new int[] { 1, 1 }; - int[] zeroPaddingArray = new int[] { 0, 0 }; - int processWidth = 4; - // Building the DL4J network - NeuralNetConfiguration.ListBuilder listBuilder = builder.list(); - listBuilder = listBuilder.layer(0, new ConvolutionLayer.Builder(kernelArray, strideArray, zeroPaddingArray).name("cnn1").convolutionMode(ConvolutionMode.Strict).nIn(// 2 input channels - 2).nOut(processWidth).weightInit(WeightInit.XAVIER_UNIFORM).activation(Activation.RELU).biasInit(1e-2).build()); - listBuilder = listBuilder.layer(1, new ConvolutionLayer.Builder(kernelArray, strideArray, zeroPaddingArray).name("cnn2").convolutionMode(ConvolutionMode.Strict).nOut(processWidth).weightInit(WeightInit.XAVIER_UNIFORM).activation(Activation.RELU).biasInit(1e-2).build()); - listBuilder = listBuilder.layer(2, new ConvolutionLayer.Builder(kernelArray, strideArray, zeroPaddingArray).name("cnn3").convolutionMode(ConvolutionMode.Strict).nOut(processWidth).weightInit(WeightInit.XAVIER_UNIFORM).activation(Activation.RELU).build()); - listBuilder = listBuilder.layer(3, new ConvolutionLayer.Builder(kernelArray, strideArray, zeroPaddingArray).name("cnn4").convolutionMode(ConvolutionMode.Strict).nOut(processWidth).weightInit(WeightInit.XAVIER_UNIFORM).activation(Activation.RELU).build()); - listBuilder = listBuilder.layer(4, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).name("output").nOut(1).activation(Activation.TANH).build()); - MultiLayerConfiguration conf = listBuilder.setInputType(InputType.convolutional(20, 10, 2)).build(); - // For some reason, this model works - MultiLayerNetwork niceModel = new MultiLayerNetwork(conf); - niceModel.init(); - // Valid - niceModel.output(Nd4j.create(DataType.FLOAT, 1, 2, 20, 10)); - try { - niceModel.output(Nd4j.create(DataType.FLOAT, 1, 2, 10, 20)); - fail("Expected exception"); - } catch (IllegalStateException e) { - // OK - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CustomPreprocessorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CustomPreprocessorTest.java deleted file mode 100644 index 946af34f4..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/preprocessor/CustomPreprocessorTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf.preprocessor; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.InputPreProcessor; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.preprocessor.custom.MyCustomPreprocessor; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.introspect.AnnotatedClass; -import org.nd4j.shade.jackson.databind.jsontype.NamedType; -import java.util.Collection; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Custom Preprocessor Test") -class CustomPreprocessorTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Custom Preprocessor") - void testCustomPreprocessor() { - // Second: let's create a MultiLayerCofiguration with one, and check JSON and YAML config actually works... - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(10).activation(Activation.SOFTMAX).nOut(10).build()).inputPreProcessor(0, new MyCustomPreprocessor()).build(); - String json = conf.toJson(); - String yaml = conf.toYaml(); - // System.out.println(json); - MultiLayerConfiguration confFromJson = MultiLayerConfiguration.fromJson(json); - assertEquals(conf, confFromJson); - MultiLayerConfiguration confFromYaml = MultiLayerConfiguration.fromYaml(yaml); - assertEquals(conf, confFromYaml); - assertTrue(confFromJson.getInputPreProcess(0) instanceof MyCustomPreprocessor); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphCNN.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphCNN.java deleted file mode 100644 index a9aee733b..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestCompGraphCNN.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.graph; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator; -import org.deeplearning4j.exception.DL4JInvalidConfigException; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.CNN2DFormat; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - - -//@Disabled -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -public class TestCompGraphCNN extends BaseDL4JTest { - - protected ComputationGraphConfiguration conf; - protected ComputationGraph graph; - protected DataSetIterator dataSetIterator; - protected DataSet ds; - - protected static ComputationGraphConfiguration getMultiInputGraphConfig() { - ComputationGraphConfiguration conf = - new NeuralNetConfiguration.Builder() - .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) - .graphBuilder().addInputs("input") - .setInputTypes(InputType.convolutional(32, 32, 3)) - .addLayer("cnn1", - new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(3).nOut(3) - .build(), - "input") - .addLayer("cnn2", - new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(3).nOut(3) - .build(), - "input") - .addLayer("max1", - new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) - .stride(1, 1).kernelSize(2, 2).build(), - "cnn1", "cnn2") - .addLayer("dnn1", new DenseLayer.Builder().nOut(7).build(), "max1") - .addLayer("output", new OutputLayer.Builder().nIn(7).nOut(10).activation(Activation.SOFTMAX).build(), "dnn1") - .setOutputs("output").build(); - - return conf; - } - - protected static DataSetIterator getDS() { - - List list = new ArrayList<>(5); - for (int i = 0; i < 5; i++) { - INDArray f = Nd4j.create(1, 32 * 32 * 3); - INDArray l = Nd4j.create(1, 10); - l.putScalar(i, 1.0); - list.add(new DataSet(f, l)); - } - return new ListDataSetIterator(list, 5); - } - - protected static int getNumParams() { - return 2 * (3 * 1 * 4 * 4 * 3 + 3) + (7 * 14 * 14 * 6 + 7) + (7 * 10 + 10); - } - - @BeforeEach - @Disabled - public void beforeDo() { - conf = getMultiInputGraphConfig(); - graph = new ComputationGraph(conf); - graph.init(); - - dataSetIterator = getDS(); - ds = dataSetIterator.next(); - - } - - @Test - public void testConfigBasic() { - //Check the order. there are 2 possible valid orders here - int[] order = graph.topologicalSortOrder(); - int[] expOrder1 = new int[] {0, 1, 2, 4, 3, 5, 6}; //First of 2 possible valid orders - int[] expOrder2 = new int[] {0, 2, 1, 4, 3, 5, 6}; //Second of 2 possible valid orders - boolean orderOK = Arrays.equals(expOrder1, order) || Arrays.equals(expOrder2, order); - assertTrue(orderOK); - - INDArray params = graph.params(); - assertNotNull(params); - - // confirm param shape is what is expected - int nParams = getNumParams(); - assertEquals(nParams, params.length()); - - INDArray arr = Nd4j.linspace(0, nParams, nParams, DataType.FLOAT).reshape(1, nParams); - assertEquals(nParams, arr.length()); - - // params are set - graph.setParams(arr); - params = graph.params(); - assertEquals(arr, params); - - //Number of inputs and outputs: - assertEquals(1, graph.getNumInputArrays()); - assertEquals(1, graph.getNumOutputArrays()); - - } - - @Test() - public void testCNNComputationGraphKernelTooLarge() { - assertThrows(DL4JInvalidConfigException.class,() -> { - int imageWidth = 23; - int imageHeight = 19; - int nChannels = 1; - int classes = 2; - int numSamples = 200; - - int kernelHeight = 3; - int kernelWidth = imageWidth; - - - DataSet trainInput; - - ComputationGraphConfiguration conf = - new NeuralNetConfiguration.Builder() - .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) - .seed(123).graphBuilder().addInputs("input") - .setInputTypes(InputType.convolutional(nChannels, imageWidth, - imageHeight)) - .addLayer("conv1", new ConvolutionLayer.Builder() - .kernelSize(kernelHeight, kernelWidth).stride(1, 1) - .dataFormat(CNN2DFormat.NCHW) - .nIn(nChannels).nOut(2).weightInit(WeightInit.XAVIER) - .activation(Activation.RELU).build(), "input") - .addLayer("pool1", - new SubsamplingLayer.Builder() - .dataFormat(CNN2DFormat.NCHW) - .poolingType(SubsamplingLayer.PoolingType.MAX) - .kernelSize(imageHeight - kernelHeight + 1, 1) - .stride(1, 1).build(), - "conv1") - .addLayer("output", new OutputLayer.Builder().nOut(classes).activation(Activation.SOFTMAX).build(), "pool1") - .setOutputs("output").build(); - - - ComputationGraph model = new ComputationGraph(conf); - model.init(); - - - INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); - INDArray emptyLables = Nd4j.zeros(numSamples, classes); - - trainInput = new DataSet(emptyFeatures, emptyLables); - - model.fit(trainInput); - }); - - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestSetGetParameters.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestSetGetParameters.java deleted file mode 100644 index 6ad77de04..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/graph/TestSetGetParameters.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.graph; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.*; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -public class TestSetGetParameters extends BaseDL4JTest { - - @Test - public void testInitWithParamsCG() { - - Nd4j.getRandom().setSeed(12345); - - //Create configuration. Doesn't matter if this doesn't actually work for forward/backward pass here - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder() - .addInputs("in").addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in") - .addLayer("1", new GravesLSTM.Builder().nIn(10).nOut(10).build(), "in") - .addLayer("2", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build(), "in") - .addLayer("3", new ConvolutionLayer.Builder().nIn(10).nOut(10).kernelSize(2, 2).stride(2, 2) - .padding(2, 2).build(), "in") - .addLayer("4", new OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "3") - .addLayer("5", new OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "0") - .addLayer("6", new RnnOutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "1", - "2") - .setOutputs("4", "5", "6").build(); - - ComputationGraph net = new ComputationGraph(conf); - net.init(); - INDArray params = net.params(); - - - ComputationGraph net2 = new ComputationGraph(conf); - net2.init(params, true); - - ComputationGraph net3 = new ComputationGraph(conf); - net3.init(params, false); - - assertEquals(params, net2.params()); - assertEquals(params, net3.params()); - - assertFalse(params == net2.params()); //Different objects due to clone - assertTrue(params == net3.params()); //Same object due to clone - - - Map paramsMap = net.paramTable(); - Map paramsMap2 = net2.paramTable(); - Map paramsMap3 = net3.paramTable(); - for (String s : paramsMap.keySet()) { - assertEquals(paramsMap.get(s), paramsMap2.get(s)); - assertEquals(paramsMap.get(s), paramsMap3.get(s)); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/ActivationLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/ActivationLayerTest.java deleted file mode 100644 index e7d9e0417..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/ActivationLayerTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ActivationLayer; -import org.deeplearning4j.nn.conf.layers.AutoEncoder; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.activations.impl.ActivationELU; -import org.nd4j.linalg.activations.impl.ActivationRationalTanh; -import org.nd4j.linalg.activations.impl.ActivationSoftmax; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.List; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - */ -@DisplayName("Activation Layer Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class ActivationLayerTest extends BaseDL4JTest { - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Test - @DisplayName("Test Input Types") - void testInputTypes() { - org.deeplearning4j.nn.conf.layers.ActivationLayer l = new org.deeplearning4j.nn.conf.layers.ActivationLayer.Builder().activation(Activation.RELU).build(); - InputType in1 = InputType.feedForward(20); - InputType in2 = InputType.convolutional(28, 28, 1); - assertEquals(in1, l.getOutputType(0, in1)); - assertEquals(in2, l.getOutputType(0, in2)); - assertNull(l.getPreProcessorForInputType(in1)); - assertNull(l.getPreProcessorForInputType(in2)); - } - - @Test - @DisplayName("Test Dense Activation Layer") - void testDenseActivationLayer() throws Exception { - DataSetIterator iter = new MnistDataSetIterator(2, 2); - DataSet next = iter.next(); - // Run without separate activation layer - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new DenseLayer.Builder().nIn(28 * 28 * 1).nOut(10).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - network.fit(next); - // Run with separate activation layer - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new DenseLayer.Builder().nIn(28 * 28 * 1).nOut(10).activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER).build()).layer(1, new org.deeplearning4j.nn.conf.layers.ActivationLayer.Builder().activation(Activation.RELU).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork network2 = new MultiLayerNetwork(conf2); - network2.init(); - network2.fit(next); - // check parameters - assertEquals(network.getLayer(0).getParam("W"), network2.getLayer(0).getParam("W")); - assertEquals(network.getLayer(1).getParam("W"), network2.getLayer(2).getParam("W")); - assertEquals(network.getLayer(0).getParam("b"), network2.getLayer(0).getParam("b")); - assertEquals(network.getLayer(1).getParam("b"), network2.getLayer(2).getParam("b")); - // check activations - network.init(); - network.setInput(next.getFeatures()); - List activations = network.feedForward(true); - network2.init(); - network2.setInput(next.getFeatures()); - List activations2 = network2.feedForward(true); - assertEquals(activations.get(1).reshape(activations2.get(2).shape()), activations2.get(2)); - assertEquals(activations.get(2), activations2.get(3)); - } - - @Test - @DisplayName("Test Auto Encoder Activation Layer") - void testAutoEncoderActivationLayer() throws Exception { - int minibatch = 3; - int nIn = 5; - int layerSize = 5; - int nOut = 3; - INDArray next = Nd4j.rand(new int[] { minibatch, nIn }); - INDArray labels = Nd4j.zeros(minibatch, nOut); - for (int i = 0; i < minibatch; i++) { - labels.putScalar(i, i % nOut, 1.0); - } - // Run without separate activation layer - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new AutoEncoder.Builder().nIn(nIn).nOut(layerSize).corruptionLevel(0.0).activation(Activation.SIGMOID).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.RECONSTRUCTION_CROSSENTROPY).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - // Labels are necessary for this test: layer activation function affect pretraining results, otherwise - network.fit(next, labels); - // Run with separate activation layer - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new AutoEncoder.Builder().nIn(nIn).nOut(layerSize).corruptionLevel(0.0).activation(Activation.IDENTITY).build()).layer(1, new org.deeplearning4j.nn.conf.layers.ActivationLayer.Builder().activation(Activation.SIGMOID).build()).layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.RECONSTRUCTION_CROSSENTROPY).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).build()).build(); - MultiLayerNetwork network2 = new MultiLayerNetwork(conf2); - network2.init(); - network2.fit(next, labels); - // check parameters - assertEquals(network.getLayer(0).getParam("W"), network2.getLayer(0).getParam("W")); - assertEquals(network.getLayer(1).getParam("W"), network2.getLayer(2).getParam("W")); - assertEquals(network.getLayer(0).getParam("b"), network2.getLayer(0).getParam("b")); - assertEquals(network.getLayer(1).getParam("b"), network2.getLayer(2).getParam("b")); - // check activations - network.init(); - network.setInput(next); - List activations = network.feedForward(true); - network2.init(); - network2.setInput(next); - List activations2 = network2.feedForward(true); - assertEquals(activations.get(1).reshape(activations2.get(2).shape()), activations2.get(2)); - assertEquals(activations.get(2), activations2.get(3)); - } - - @Test - @DisplayName("Test CNN Activation Layer") - void testCNNActivationLayer() throws Exception { - DataSetIterator iter = new MnistDataSetIterator(2, 2); - DataSet next = iter.next(); - // Run without separate activation layer - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(1).nOut(20).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - network.fit(next); - // Run with separate activation layer - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(1).nOut(20).activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER).build()).layer(1, new org.deeplearning4j.nn.conf.layers.ActivationLayer.Builder().activation(Activation.RELU).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork network2 = new MultiLayerNetwork(conf2); - network2.init(); - network2.fit(next); - // check parameters - assertEquals(network.getLayer(0).getParam("W"), network2.getLayer(0).getParam("W")); - assertEquals(network.getLayer(1).getParam("W"), network2.getLayer(2).getParam("W")); - assertEquals(network.getLayer(0).getParam("b"), network2.getLayer(0).getParam("b")); - // check activations - network.init(); - network.setInput(next.getFeatures()); - List activations = network.feedForward(true); - network2.init(); - network2.setInput(next.getFeatures()); - List activations2 = network2.feedForward(true); - assertEquals(activations.get(1).reshape(activations2.get(2).shape()), activations2.get(2)); - assertEquals(activations.get(2), activations2.get(3)); - } - - @Test - @DisplayName("Test Activation Inheritance") - void testActivationInheritance() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).weightInit(WeightInit.XAVIER).activation(Activation.RATIONALTANH).list().layer(new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(new ActivationLayer()).layer(new ActivationLayer.Builder().build()).layer(new ActivationLayer.Builder().activation(Activation.ELU).build()).layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - assertNotNull(((ActivationLayer) network.getLayer(1).conf().getLayer()).getActivationFn()); - assertTrue(((DenseLayer) network.getLayer(0).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); - assertTrue(((ActivationLayer) network.getLayer(1).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); - assertTrue(((ActivationLayer) network.getLayer(2).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); - assertTrue(((ActivationLayer) network.getLayer(3).conf().getLayer()).getActivationFn() instanceof ActivationELU); - assertTrue(((OutputLayer) network.getLayer(4).conf().getLayer()).getActivationFn() instanceof ActivationSoftmax); - } - - @Test - @DisplayName("Test Activation Inheritance CG") - void testActivationInheritanceCG() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).weightInit(WeightInit.XAVIER).activation(Activation.RATIONALTANH).graphBuilder().addInputs("in").addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in").addLayer("1", new ActivationLayer(), "0").addLayer("2", new ActivationLayer.Builder().build(), "1").addLayer("3", new ActivationLayer.Builder().activation(Activation.ELU).build(), "2").addLayer("4", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "3").setOutputs("4").build(); - ComputationGraph network = new ComputationGraph(conf); - network.init(); - assertNotNull(((ActivationLayer) network.getLayer("1").conf().getLayer()).getActivationFn()); - assertTrue(((DenseLayer) network.getLayer("0").conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); - assertTrue(((ActivationLayer) network.getLayer("1").conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); - assertTrue(((ActivationLayer) network.getLayer("2").conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh); - assertTrue(((ActivationLayer) network.getLayer("3").conf().getLayer()).getActivationFn() instanceof ActivationELU); - assertTrue(((OutputLayer) network.getLayer("4").conf().getLayer()).getActivationFn() instanceof ActivationSoftmax); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/AutoEncoderTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/AutoEncoderTest.java deleted file mode 100644 index 645945344..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/AutoEncoderTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.graph.MergeVertex; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.AutoEncoder; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.api.MultiDataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Auto Encoder Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class AutoEncoderTest extends BaseDL4JTest { - - @Test - @DisplayName("Sanity Check Issue 5662") - void sanityCheckIssue5662() { - int mergeSize = 50; - int encdecSize = 25; - int in1Size = 20; - int in2Size = 15; - int hiddenSize = 10; - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in1", "in2").addLayer("1", new DenseLayer.Builder().nOut(mergeSize).build(), "in1").addLayer("2", new DenseLayer.Builder().nOut(mergeSize).build(), "in2").addVertex("merge", new MergeVertex(), "1", "2").addLayer("e", new AutoEncoder.Builder().nOut(encdecSize).corruptionLevel(0.2).build(), "merge").addLayer("hidden", new AutoEncoder.Builder().nOut(hiddenSize).build(), "e").addLayer("decoder", new AutoEncoder.Builder().nOut(encdecSize).corruptionLevel(0.2).build(), "hidden").addLayer("L4", new DenseLayer.Builder().nOut(mergeSize).build(), "decoder").addLayer("out1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(in1Size).build(), "L4").addLayer("out2", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(in2Size).build(), "L4").setOutputs("out1", "out2").setInputTypes(InputType.feedForward(in1Size), InputType.feedForward(in2Size)).build(); - ComputationGraph net = new ComputationGraph(conf); - net.init(); - MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(new INDArray[] { Nd4j.create(1, in1Size), Nd4j.create(1, in2Size) }, new INDArray[] { Nd4j.create(1, in1Size), Nd4j.create(1, in2Size) }); - net.summary(InputType.feedForward(in1Size), InputType.feedForward(in2Size)); - net.fit(new SingletonMultiDataSetIterator(mds)); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/BaseLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/BaseLayerTest.java deleted file mode 100644 index ef2ddac52..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/BaseLayerTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import java.util.HashMap; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Base Layer Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class BaseLayerTest extends BaseDL4JTest { - - protected INDArray weight = Nd4j.create(new double[] { 0.10, -0.20, -0.15, 0.05 }, new int[] { 2, 2 }); - - protected INDArray bias = Nd4j.create(new double[] { 0.5, 0.5 }, new int[] { 1, 2 }); - - protected Map paramTable; - - @BeforeEach - void doBefore() { - paramTable = new HashMap<>(); - paramTable.put("W", weight); - paramTable.put("b", bias); - } - - @Test - @DisplayName("Test Set Existing Params Convolution Single Layer") - void testSetExistingParamsConvolutionSingleLayer() { - Layer layer = configureSingleLayer(); - assertNotEquals(paramTable, layer.paramTable()); - layer.setParamTable(paramTable); - assertEquals(paramTable, layer.paramTable()); - } - - @Test - @DisplayName("Test Set Existing Params Dense Multi Layer") - void testSetExistingParamsDenseMultiLayer() { - MultiLayerNetwork net = configureMultiLayer(); - for (Layer layer : net.getLayers()) { - assertNotEquals(paramTable, layer.paramTable()); - layer.setParamTable(paramTable); - assertEquals(paramTable, layer.paramTable()); - } - } - - public Layer configureSingleLayer() { - int nIn = 2; - int nOut = 2; - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new ConvolutionLayer.Builder().nIn(nIn).nOut(nOut).build()).build(); - val numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - return conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - } - - public MultiLayerNetwork configureMultiLayer() { - int nIn = 2; - int nOut = 2; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(nOut).build()).layer(1, new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - return net; - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/CacheModeTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/CacheModeTest.java deleted file mode 100644 index 853bf75d0..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/CacheModeTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.conf.*; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Cache Mode Test") -class CacheModeTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Conv Cache Mode Simple") - void testConvCacheModeSimple() { - MultiLayerConfiguration conf1 = getConf(CacheMode.NONE); - MultiLayerConfiguration conf2 = getConf(CacheMode.DEVICE); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - INDArray in = Nd4j.rand(3, 28 * 28); - INDArray labels = TestUtils.randomOneHot(3, 10); - INDArray out1 = net1.output(in); - INDArray out2 = net2.output(in); - assertEquals(out1, out2); - assertEquals(net1.params(), net2.params()); - net1.fit(in, labels); - net2.fit(in, labels); - assertEquals(net1.params(), net2.params()); - } - - private static MultiLayerConfiguration getConf(CacheMode cacheMode) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).inferenceWorkspaceMode(WorkspaceMode.ENABLED).trainingWorkspaceMode(WorkspaceMode.ENABLED).seed(12345).cacheMode(cacheMode).list().layer(new ConvolutionLayer.Builder().nOut(3).build()).layer(new ConvolutionLayer.Builder().nOut(3).build()).layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - return conf; - } - - @Test - @DisplayName("Test LSTM Cache Mode Simple") - void testLSTMCacheModeSimple() { - for (boolean graves : new boolean[] { true, false }) { - MultiLayerConfiguration conf1 = getConfLSTM(CacheMode.NONE, graves); - MultiLayerConfiguration conf2 = getConfLSTM(CacheMode.DEVICE, graves); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - INDArray in = Nd4j.rand(new int[] { 3, 3, 10 }); - INDArray labels = TestUtils.randomOneHotTimeSeries(3, 10, 10); - INDArray out1 = net1.output(in); - INDArray out2 = net2.output(in); - assertEquals(out1, out2); - assertEquals(net1.params(), net2.params()); - net1.fit(in, labels); - net2.fit(in, labels); - assertEquals(net1.params(), net2.params()); - } - } - - private static MultiLayerConfiguration getConfLSTM(CacheMode cacheMode, boolean graves) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).inferenceWorkspaceMode(WorkspaceMode.ENABLED).trainingWorkspaceMode(WorkspaceMode.ENABLED).seed(12345).cacheMode(cacheMode).list().layer(graves ? new GravesLSTM.Builder().nIn(3).nOut(3).build() : new LSTM.Builder().nIn(3).nOut(3).build()).layer(graves ? new GravesLSTM.Builder().nIn(3).nOut(3).build() : new LSTM.Builder().nIn(3).nOut(3).build()).layer(new RnnOutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).build()).build(); - return conf; - } - - @Test - @DisplayName("Test Conv Cache Mode Simple CG") - void testConvCacheModeSimpleCG() { - ComputationGraphConfiguration conf1 = getConfCG(CacheMode.NONE); - ComputationGraphConfiguration conf2 = getConfCG(CacheMode.DEVICE); - ComputationGraph net1 = new ComputationGraph(conf1); - net1.init(); - ComputationGraph net2 = new ComputationGraph(conf2); - net2.init(); - INDArray in = Nd4j.rand(3, 28 * 28); - INDArray labels = TestUtils.randomOneHot(3, 10); - INDArray out1 = net1.outputSingle(in); - INDArray out2 = net2.outputSingle(in); - assertEquals(out1, out2); - assertEquals(net1.params(), net2.params()); - net1.fit(new DataSet(in, labels)); - net2.fit(new DataSet(in, labels)); - assertEquals(net1.params(), net2.params()); - } - - private static ComputationGraphConfiguration getConfCG(CacheMode cacheMode) { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).inferenceWorkspaceMode(WorkspaceMode.ENABLED).trainingWorkspaceMode(WorkspaceMode.ENABLED).seed(12345).cacheMode(cacheMode).graphBuilder().addInputs("in").layer("0", new ConvolutionLayer.Builder().nOut(3).build(), "in").layer("1", new ConvolutionLayer.Builder().nOut(3).build(), "0").layer("2", new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).build(), "1").setOutputs("2").setInputTypes(InputType.convolutionalFlat(28, 28, 1)).build(); - return conf; - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/CenterLossOutputLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/CenterLossOutputLayerTest.java deleted file mode 100755 index f58ef892e..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/CenterLossOutputLayerTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; -import java.util.Random; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Center Loss Output Layer Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class CenterLossOutputLayerTest extends BaseDL4JTest { - - private ComputationGraph getGraph(int numLabels, double lambda) { - Nd4j.getRandom().setSeed(12345); - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).dist(new NormalDistribution(0, 1)).updater(new NoOp()).graphBuilder().addInputs("input1").addLayer("l1", new DenseLayer.Builder().nIn(4).nOut(5).activation(Activation.RELU).build(), "input1").addLayer("lossLayer", new CenterLossOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(numLabels).lambda(lambda).activation(Activation.SOFTMAX).build(), "l1").setOutputs("lossLayer").build(); - ComputationGraph graph = new ComputationGraph(conf); - graph.init(); - return graph; - } - - public ComputationGraph getCNNMnistConfig() { - // Number of input channels - int nChannels = 1; - // The number of possible outcomes - int outputNum = 10; - ComputationGraphConfiguration conf = // Training iterations as above - new NeuralNetConfiguration.Builder().seed(12345).l2(0.0005).weightInit(WeightInit.XAVIER).updater(new Nesterovs(0.01, 0.9)).graphBuilder().addInputs("input").setInputTypes(InputType.convolutionalFlat(28, 28, 1)).addLayer("0", new ConvolutionLayer.Builder(5, 5).nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build(), "input").addLayer("1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build(), "0").addLayer("2", new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "1").addLayer("3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build(), "2").addLayer("4", new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build(), "3").addLayer("output", new org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer.Builder(LossFunction.MCXENT).nOut(outputNum).activation(Activation.SOFTMAX).build(), "4").setOutputs("output").build(); - ComputationGraph graph = new ComputationGraph(conf); - graph.init(); - return graph; - } - - @Test - @DisplayName("Test Lambda Conf") - void testLambdaConf() { - double[] lambdas = new double[] { 0.1, 0.01 }; - double[] results = new double[2]; - int numClasses = 2; - INDArray input = Nd4j.rand(150, 4); - INDArray labels = Nd4j.zeros(150, numClasses); - Random r = new Random(12345); - for (int i = 0; i < 150; i++) { - labels.putScalar(i, r.nextInt(numClasses), 1.0); - } - ComputationGraph graph; - for (int i = 0; i < lambdas.length; i++) { - graph = getGraph(numClasses, lambdas[i]); - graph.setInput(0, input); - graph.setLabel(0, labels); - graph.computeGradientAndScore(); - results[i] = graph.score(); - } - assertNotEquals(results[0], results[1]); - } - - @Test - @Disabled - @DisplayName("Test MNIST Config") - void testMNISTConfig() throws Exception { - // Test batch size - int batchSize = 64; - DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345); - ComputationGraph net = getCNNMnistConfig(); - net.init(); - net.setListeners(new ScoreIterationListener(1)); - for (int i = 0; i < 50; i++) { - net.fit(mnistTrain.next()); - Thread.sleep(1000); - } - Thread.sleep(100000); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/DropoutLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/DropoutLayerTest.java deleted file mode 100644 index 14f9c7447..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/DropoutLayerTest.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.InputPreProcessor; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.dropout.Dropout; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.DropoutLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - */ -@DisplayName("Dropout Layer Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class DropoutLayerTest extends BaseDL4JTest { - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Test - @DisplayName("Test Input Types") - void testInputTypes() { - DropoutLayer config = new DropoutLayer.Builder(0.5).build(); - InputType in1 = InputType.feedForward(20); - InputType in2 = InputType.convolutional(28, 28, 1); - assertEquals(in1, config.getOutputType(0, in1)); - assertEquals(in2, config.getOutputType(0, in2)); - assertNull(config.getPreProcessorForInputType(in1)); - assertNull(config.getPreProcessorForInputType(in2)); - } - - @Test - @DisplayName("Test Dropout Layer Without Training") - @Tag(TagNames.LARGE_RESOURCES) - @Tag(TagNames.LONG_TEST) - void testDropoutLayerWithoutTraining() throws Exception { - MultiLayerConfiguration confIntegrated = new NeuralNetConfiguration.Builder().seed(3648).list().layer(0, new ConvolutionLayer.Builder(1, 1).stride(1, 1).nIn(1).nOut(1).dropOut(0.25).activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).dropOut(0.25).nOut(4).build()).setInputType(InputType.convolutionalFlat(2, 2, 1)).build(); - MultiLayerNetwork netIntegrated = new MultiLayerNetwork(confIntegrated); - netIntegrated.init(); - netIntegrated.getLayer(0).setParam("W", Nd4j.eye(1)); - netIntegrated.getLayer(0).setParam("b", Nd4j.zeros(1, 1)); - netIntegrated.getLayer(1).setParam("W", Nd4j.eye(4)); - netIntegrated.getLayer(1).setParam("b", Nd4j.zeros(4, 1)); - MultiLayerConfiguration confSeparate = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(3648).list().layer(0, new DropoutLayer.Builder(0.25).build()).layer(1, new ConvolutionLayer.Builder(1, 1).stride(1, 1).nIn(1).nOut(1).activation(Activation.IDENTITY).weightInit(WeightInit.XAVIER).build()).layer(2, new DropoutLayer.Builder(0.25).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(4).build()).setInputType(InputType.convolutionalFlat(2, 2, 1)).build(); - MultiLayerNetwork netSeparate = new MultiLayerNetwork(confSeparate); - netSeparate.init(); - netSeparate.getLayer(1).setParam("W", Nd4j.eye(1)); - netSeparate.getLayer(1).setParam("b", Nd4j.zeros(1, 1)); - netSeparate.getLayer(3).setParam("W", Nd4j.eye(4)); - netSeparate.getLayer(3).setParam("b", Nd4j.zeros(4, 1)); - // Disable input modification for this test: - for (Layer l : netIntegrated.getLayers()) { - l.allowInputModification(false); - } - for (Layer l : netSeparate.getLayers()) { - l.allowInputModification(false); - } - INDArray in = Nd4j.arange(1, 5).reshape(1, 4); - Nd4j.getRandom().setSeed(12345); - List actTrainIntegrated = netIntegrated.feedForward(in.dup(), true); - Nd4j.getRandom().setSeed(12345); - List actTrainSeparate = netSeparate.feedForward(in.dup(), true); - Nd4j.getRandom().setSeed(12345); - List actTestIntegrated = netIntegrated.feedForward(in.dup(), false); - Nd4j.getRandom().setSeed(12345); - List actTestSeparate = netSeparate.feedForward(in.dup(), false); - // Check masks: - INDArray maskIntegrated = ((Dropout) netIntegrated.getLayer(0).conf().getLayer().getIDropout()).getMask(); - INDArray maskSeparate = ((Dropout) netSeparate.getLayer(0).conf().getLayer().getIDropout()).getMask(); - assertEquals(maskIntegrated, maskSeparate); - assertEquals(actTrainIntegrated.get(1), actTrainSeparate.get(2)); - assertEquals(actTrainIntegrated.get(2), actTrainSeparate.get(4)); - assertEquals(actTestIntegrated.get(1), actTestSeparate.get(2)); - assertEquals(actTestIntegrated.get(2), actTestSeparate.get(4)); - } - - @Test - @DisplayName("Test Dropout Layer With Dense Mnist") - void testDropoutLayerWithDenseMnist() throws Exception { - DataSetIterator iter = new MnistDataSetIterator(2, 2); - DataSet next = iter.next(); - // Run without separate activation layer - MultiLayerConfiguration confIntegrated = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new DenseLayer.Builder().nIn(28 * 28 * 1).nOut(10).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).dropOut(0.25).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork netIntegrated = new MultiLayerNetwork(confIntegrated); - netIntegrated.init(); - netIntegrated.fit(next); - // Run with separate activation layer - MultiLayerConfiguration confSeparate = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new DenseLayer.Builder().nIn(28 * 28 * 1).nOut(10).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new DropoutLayer.Builder(0.25).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork netSeparate = new MultiLayerNetwork(confSeparate); - netSeparate.init(); - netSeparate.fit(next); - // Disable input modification for this test: - for (Layer l : netIntegrated.getLayers()) { - l.allowInputModification(false); - } - for (Layer l : netSeparate.getLayers()) { - l.allowInputModification(false); - } - // check parameters - assertEquals(netIntegrated.getLayer(0).getParam("W"), netSeparate.getLayer(0).getParam("W")); - assertEquals(netIntegrated.getLayer(0).getParam("b"), netSeparate.getLayer(0).getParam("b")); - assertEquals(netIntegrated.getLayer(1).getParam("W"), netSeparate.getLayer(2).getParam("W")); - assertEquals(netIntegrated.getLayer(1).getParam("b"), netSeparate.getLayer(2).getParam("b")); - // check activations - netIntegrated.setInput(next.getFeatures()); - netSeparate.setInput(next.getFeatures()); - Nd4j.getRandom().setSeed(12345); - List actTrainIntegrated = netIntegrated.feedForward(true); - Nd4j.getRandom().setSeed(12345); - List actTrainSeparate = netSeparate.feedForward(true); - assertEquals(actTrainIntegrated.get(1), actTrainSeparate.get(1)); - assertEquals(actTrainIntegrated.get(2), actTrainSeparate.get(3)); - Nd4j.getRandom().setSeed(12345); - List actTestIntegrated = netIntegrated.feedForward(false); - Nd4j.getRandom().setSeed(12345); - List actTestSeparate = netSeparate.feedForward(false); - assertEquals(actTestIntegrated.get(1), actTrainSeparate.get(1)); - assertEquals(actTestIntegrated.get(2), actTestSeparate.get(3)); - } - - @Test - @DisplayName("Test Dropout Layer With Conv Mnist") - void testDropoutLayerWithConvMnist() throws Exception { - // Set to double datatype - MKL-DNN not used for CPU (otherwise different strides due to Dl4J impl permutes) - Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); - DataSetIterator iter = new MnistDataSetIterator(2, 2); - DataSet next = iter.next(); - // Run without separate activation layer - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration confIntegrated = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(1).nOut(20).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).dropOut(0.5).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - // Run with separate activation layer - Nd4j.getRandom().setSeed(12345); - // Manually configure preprocessors - // This is necessary, otherwise CnnToFeedForwardPreprocessor will be in different locatinos - // i.e., dropout on 4d activations in latter, and dropout on 2d activations in former - Map preProcessorMap = new HashMap<>(); - preProcessorMap.put(1, new CnnToFeedForwardPreProcessor(13, 13, 20)); - MultiLayerConfiguration confSeparate = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, new ConvolutionLayer.Builder(4, 4).stride(2, 2).nIn(1).nOut(20).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(1, new DropoutLayer.Builder(0.5).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()).inputPreProcessors(preProcessorMap).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - Nd4j.getRandom().setSeed(12345); - MultiLayerNetwork netIntegrated = new MultiLayerNetwork(confIntegrated); - netIntegrated.init(); - Nd4j.getRandom().setSeed(12345); - MultiLayerNetwork netSeparate = new MultiLayerNetwork(confSeparate); - netSeparate.init(); - assertEquals(netIntegrated.params(), netSeparate.params()); - Nd4j.getRandom().setSeed(12345); - netIntegrated.fit(next); - Nd4j.getRandom().setSeed(12345); - netSeparate.fit(next); - assertEquals(netIntegrated.params(), netSeparate.params()); - // check parameters - assertEquals(netIntegrated.getLayer(0).getParam("W"), netSeparate.getLayer(0).getParam("W")); - assertEquals(netIntegrated.getLayer(0).getParam("b"), netSeparate.getLayer(0).getParam("b")); - assertEquals(netIntegrated.getLayer(1).getParam("W"), netSeparate.getLayer(2).getParam("W")); - assertEquals(netIntegrated.getLayer(1).getParam("b"), netSeparate.getLayer(2).getParam("b")); - // check activations - netIntegrated.setInput(next.getFeatures().dup()); - netSeparate.setInput(next.getFeatures().dup()); - Nd4j.getRandom().setSeed(12345); - List actTrainIntegrated = netIntegrated.feedForward(true); - Nd4j.getRandom().setSeed(12345); - List actTrainSeparate = netSeparate.feedForward(true); - assertEquals(actTrainIntegrated.get(1), actTrainSeparate.get(1)); - assertEquals(actTrainIntegrated.get(2), actTrainSeparate.get(3)); - netIntegrated.setInput(next.getFeatures().dup()); - netSeparate.setInput(next.getFeatures().dup()); - Nd4j.getRandom().setSeed(12345); - List actTestIntegrated = netIntegrated.feedForward(false); - Nd4j.getRandom().setSeed(12345); - List actTestSeparate = netSeparate.feedForward(false); - assertEquals(actTestIntegrated.get(1), actTestSeparate.get(1)); - assertEquals(actTestIntegrated.get(2), actTestSeparate.get(3)); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerTest.java deleted file mode 100644 index 5ba47d148..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerTest.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; -import org.deeplearning4j.nn.transferlearning.TransferLearning; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Frozen Layer Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class FrozenLayerTest extends BaseDL4JTest { - - /* - A model with a few frozen layers == - Model with non frozen layers set with the output of the forward pass of the frozen layers - */ - @Test - @DisplayName("Test Frozen") - void testFrozen() { - DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).activation(Activation.IDENTITY); - FineTuneConfiguration finetune = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build(); - MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(overallConf.clone().list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build()); - modelToFineTune.init(); - List ff = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false); - INDArray asFrozenFeatures = ff.get(2); - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(finetune).setFeatureExtractor(1).build(); - INDArray paramsLastTwoLayers = Nd4j.hstack(modelToFineTune.getLayer(2).params(), modelToFineTune.getLayer(3).params()); - MultiLayerNetwork notFrozen = new MultiLayerNetwork(overallConf.clone().list().layer(0, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build(), paramsLastTwoLayers); - // assertEquals(modelNow.getLayer(2).conf(), notFrozen.getLayer(0).conf()); //Equal, other than names - // assertEquals(modelNow.getLayer(3).conf(), notFrozen.getLayer(1).conf()); //Equal, other than names - // Check: forward pass - INDArray outNow = modelNow.output(randomData.getFeatures()); - INDArray outNotFrozen = notFrozen.output(asFrozenFeatures); - assertEquals(outNow, outNotFrozen); - for (int i = 0; i < 5; i++) { - notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); - modelNow.fit(randomData); - } - INDArray expected = Nd4j.hstack(modelToFineTune.getLayer(0).params(), modelToFineTune.getLayer(1).params(), notFrozen.params()); - INDArray act = modelNow.params(); - assertEquals(expected, act); - } - - @Test - @DisplayName("Clone MLN Frozen") - void cloneMLNFrozen() { - DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).activation(Activation.IDENTITY); - MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(overallConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build()); - modelToFineTune.init(); - INDArray asFrozenFeatures = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false).get(2); - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).setFeatureExtractor(1).build(); - MultiLayerNetwork clonedModel = modelNow.clone(); - // Check json - assertEquals(modelNow.getLayerWiseConfigurations().toJson(), clonedModel.getLayerWiseConfigurations().toJson()); - // Check params - assertEquals(modelNow.params(), clonedModel.params()); - MultiLayerNetwork notFrozen = new MultiLayerNetwork(overallConf.list().layer(0, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build(), Nd4j.hstack(modelToFineTune.getLayer(2).params(), modelToFineTune.getLayer(3).params())); - int i = 0; - while (i < 5) { - notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); - modelNow.fit(randomData); - clonedModel.fit(randomData); - i++; - } - INDArray expectedParams = Nd4j.hstack(modelToFineTune.getLayer(0).params(), modelToFineTune.getLayer(1).params(), notFrozen.params()); - assertEquals(expectedParams, modelNow.params()); - assertEquals(expectedParams, clonedModel.params()); - } - - @Test - @DisplayName("Test Frozen Comp Graph") - void testFrozenCompGraph() { - DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).activation(Activation.IDENTITY); - ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In").addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0").addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1").addLayer("layer3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer2").setOutputs("layer3").build()); - modelToFineTune.init(); - INDArray asFrozenFeatures = modelToFineTune.feedForward(randomData.getFeatures(), false).get("layer1"); - ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune).setFeatureExtractor("layer1").build(); - ComputationGraph notFrozen = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer0In").addLayer("layer1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer0").setOutputs("layer1").build()); - notFrozen.init(); - notFrozen.setParams(Nd4j.hstack(modelToFineTune.getLayer("layer2").params(), modelToFineTune.getLayer("layer3").params())); - int i = 0; - while (i < 5) { - notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); - modelNow.fit(randomData); - i++; - } - assertEquals(Nd4j.hstack(modelToFineTune.getLayer("layer0").params(), modelToFineTune.getLayer("layer1").params(), notFrozen.params()), modelNow.params()); - } - - @Test - @DisplayName("Clone Comp Graph Frozen") - void cloneCompGraphFrozen() { - DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).activation(Activation.IDENTITY); - ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In").addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0").addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1").addLayer("layer3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer2").setOutputs("layer3").build()); - modelToFineTune.init(); - INDArray asFrozenFeatures = modelToFineTune.feedForward(randomData.getFeatures(), false).get("layer1"); - ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune).setFeatureExtractor("layer1").build(); - ComputationGraph clonedModel = modelNow.clone(); - // Check json - assertEquals(clonedModel.getConfiguration().toJson(), modelNow.getConfiguration().toJson()); - // Check params - assertEquals(modelNow.params(), clonedModel.params()); - ComputationGraph notFrozen = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer0In").addLayer("layer1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer0").setOutputs("layer1").build()); - notFrozen.init(); - notFrozen.setParams(Nd4j.hstack(modelToFineTune.getLayer("layer2").params(), modelToFineTune.getLayer("layer3").params())); - int i = 0; - while (i < 5) { - notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); - modelNow.fit(randomData); - clonedModel.fit(randomData); - i++; - } - INDArray expectedParams = Nd4j.hstack(modelToFineTune.getLayer("layer0").params(), modelToFineTune.getLayer("layer1").params(), notFrozen.params()); - assertEquals(expectedParams, modelNow.params()); - assertEquals(expectedParams, clonedModel.params()); - } - - @Test - @DisplayName("Test Frozen Layer Instantiation") - void testFrozenLayerInstantiation() { - // We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if - // they were initialized via the builder - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(1, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayer(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())).layer(1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayer(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())).layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - assertEquals(net1.params(), net2.params()); - String json = conf2.toJson(); - MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json); - assertEquals(conf2, fromJson); - MultiLayerNetwork net3 = new MultiLayerNetwork(fromJson); - net3.init(); - INDArray input = Nd4j.rand(10, 10); - INDArray out2 = net2.output(input); - INDArray out3 = net3.output(input); - assertEquals(out2, out3); - } - - @Test - @DisplayName("Test Frozen Layer Instantiation Comp Graph") - void testFrozenLayerInstantiationCompGraph() { - // We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if - // they were initialized via the builder - ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder().addInputs("in").addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build(), "in").addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build(), "0").addLayer("2", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "1").setOutputs("2").build(); - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder().addInputs("in").addLayer("0", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayer.Builder().layer(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).build(), "in").addLayer("1", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayer.Builder().layer(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).build(), "0").addLayer("2", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "1").setOutputs("2").build(); - ComputationGraph net1 = new ComputationGraph(conf1); - net1.init(); - ComputationGraph net2 = new ComputationGraph(conf2); - net2.init(); - assertEquals(net1.params(), net2.params()); - String json = conf2.toJson(); - ComputationGraphConfiguration fromJson = ComputationGraphConfiguration.fromJson(json); - assertEquals(conf2, fromJson); - ComputationGraph net3 = new ComputationGraph(fromJson); - net3.init(); - INDArray input = Nd4j.rand(10, 10); - INDArray out2 = net2.outputSingle(input); - INDArray out3 = net3.outputSingle(input); - assertEquals(out2, out3); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackpropTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackpropTest.java deleted file mode 100644 index 093c5da91..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/FrozenLayerWithBackpropTest.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.graph.MergeVertex; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; -import org.deeplearning4j.nn.transferlearning.TransferLearning; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Frozen Layer With Backprop Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class FrozenLayerWithBackpropTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Frozen With Backprop Layer Instantiation") - void testFrozenWithBackpropLayerInstantiation() { - // We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if - // they were initialized via the builder - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(1, new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())).layer(1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - assertEquals(net1.params(), net2.params()); - String json = conf2.toJson(); - MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json); - assertEquals(conf2, fromJson); - MultiLayerNetwork net3 = new MultiLayerNetwork(fromJson); - net3.init(); - INDArray input = Nd4j.rand(10, 10); - INDArray out2 = net2.output(input); - INDArray out3 = net3.output(input); - assertEquals(out2, out3); - } - - @Test - @DisplayName("Test Frozen Layer Instantiation Comp Graph") - void testFrozenLayerInstantiationCompGraph() { - // We need to be able to instantitate frozen layers from JSON etc, and have them be the same as if - // they were initialized via the builder - ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder().addInputs("in").addLayer("0", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build(), "in").addLayer("1", new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build(), "0").addLayer("2", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "1").setOutputs("2").build(); - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).graphBuilder().addInputs("in").addLayer("0", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()), "in").addLayer("1", new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(10).nOut(10).activation(Activation.TANH).weightInit(WeightInit.XAVIER).build()), "0").addLayer("2", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build(), "1").setOutputs("2").build(); - ComputationGraph net1 = new ComputationGraph(conf1); - net1.init(); - ComputationGraph net2 = new ComputationGraph(conf2); - net2.init(); - assertEquals(net1.params(), net2.params()); - String json = conf2.toJson(); - ComputationGraphConfiguration fromJson = ComputationGraphConfiguration.fromJson(json); - assertEquals(conf2, fromJson); - ComputationGraph net3 = new ComputationGraph(fromJson); - net3.init(); - INDArray input = Nd4j.rand(10, 10); - INDArray out2 = net2.outputSingle(input); - INDArray out3 = net3.outputSingle(input); - assertEquals(out2, out3); - } - - @Test - @DisplayName("Test Multi Layer Network Frozen Layer Params After Backprop") - void testMultiLayerNetworkFrozenLayerParamsAfterBackprop() { - Nd4j.getRandom().setSeed(12345); - DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1)); - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345).weightInit(WeightInit.XAVIER).updater(new Sgd(2)).list().layer(new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(3).nOut(4).build())).layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(4).nOut(2).build())).layer(new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(2).nOut(1).build())).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf1); - network.init(); - INDArray unfrozenLayerParams = network.getLayer(0).params().dup(); - INDArray frozenLayerParams1 = network.getLayer(1).params().dup(); - INDArray frozenLayerParams2 = network.getLayer(2).params().dup(); - INDArray frozenOutputLayerParams = network.getLayer(3).params().dup(); - for (int i = 0; i < 100; i++) { - network.fit(randomData); - } - assertNotEquals(unfrozenLayerParams, network.getLayer(0).params()); - assertEquals(frozenLayerParams1, network.getLayer(1).params()); - assertEquals(frozenLayerParams2, network.getLayer(2).params()); - assertEquals(frozenOutputLayerParams, network.getLayer(3).params()); - } - - @Test - @DisplayName("Test Computation Graph Frozen Layer Params After Backprop") - void testComputationGraphFrozenLayerParamsAfterBackprop() { - Nd4j.getRandom().setSeed(12345); - DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1)); - String frozenBranchName = "B1-"; - String unfrozenBranchName = "B2-"; - String initialLayer = "initial"; - String frozenBranchUnfrozenLayer0 = frozenBranchName + "0"; - String frozenBranchFrozenLayer1 = frozenBranchName + "1"; - String frozenBranchFrozenLayer2 = frozenBranchName + "2"; - String frozenBranchOutput = frozenBranchName + "Output"; - String unfrozenLayer0 = unfrozenBranchName + "0"; - String unfrozenLayer1 = unfrozenBranchName + "1"; - String unfrozenBranch2 = unfrozenBranchName + "Output"; - ComputationGraphConfiguration computationGraphConf = new NeuralNetConfiguration.Builder().updater(new Sgd(2.0)).seed(12345).graphBuilder().addInputs("input").addLayer(initialLayer, new DenseLayer.Builder().nIn(4).nOut(4).build(), "input").addLayer(frozenBranchUnfrozenLayer0, new DenseLayer.Builder().nIn(4).nOut(3).build(), initialLayer).addLayer(frozenBranchFrozenLayer1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(3).nOut(4).build()), frozenBranchUnfrozenLayer0).addLayer(frozenBranchFrozenLayer2, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(4).nOut(2).build()), frozenBranchFrozenLayer1).addLayer(unfrozenLayer0, new DenseLayer.Builder().nIn(4).nOut(4).build(), initialLayer).addLayer(unfrozenLayer1, new DenseLayer.Builder().nIn(4).nOut(2).build(), unfrozenLayer0).addLayer(unfrozenBranch2, new DenseLayer.Builder().nIn(2).nOut(1).build(), unfrozenLayer1).addVertex("merge", new MergeVertex(), frozenBranchFrozenLayer2, unfrozenBranch2).addLayer(frozenBranchOutput, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(3).nOut(1).build()), "merge").setOutputs(frozenBranchOutput).build(); - ComputationGraph computationGraph = new ComputationGraph(computationGraphConf); - computationGraph.init(); - INDArray unfrozenLayerParams = computationGraph.getLayer(frozenBranchUnfrozenLayer0).params().dup(); - INDArray frozenLayerParams1 = computationGraph.getLayer(frozenBranchFrozenLayer1).params().dup(); - INDArray frozenLayerParams2 = computationGraph.getLayer(frozenBranchFrozenLayer2).params().dup(); - INDArray frozenOutputLayerParams = computationGraph.getLayer(frozenBranchOutput).params().dup(); - for (int i = 0; i < 100; i++) { - computationGraph.fit(randomData); - } - assertNotEquals(unfrozenLayerParams, computationGraph.getLayer(frozenBranchUnfrozenLayer0).params()); - assertEquals(frozenLayerParams1, computationGraph.getLayer(frozenBranchFrozenLayer1).params()); - assertEquals(frozenLayerParams2, computationGraph.getLayer(frozenBranchFrozenLayer2).params()); - assertEquals(frozenOutputLayerParams, computationGraph.getLayer(frozenBranchOutput).params()); - } - - /** - * Frozen layer should have same results as a layer with Sgd updater with learning rate set to 0 - */ - @Test - @DisplayName("Test Frozen Layer Vs Sgd") - void testFrozenLayerVsSgd() { - Nd4j.getRandom().setSeed(12345); - DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1)); - MultiLayerConfiguration confSgd = new NeuralNetConfiguration.Builder().seed(12345).weightInit(WeightInit.XAVIER).updater(new Sgd(2)).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new DenseLayer.Builder().updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).nIn(3).nOut(4).build()).layer(2, new DenseLayer.Builder().updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).nIn(4).nOut(2).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).activation(Activation.TANH).nIn(2).nOut(1).build()).build(); - MultiLayerConfiguration confFrozen = new NeuralNetConfiguration.Builder().seed(12345).weightInit(WeightInit.XAVIER).updater(new Sgd(2)).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(3).nOut(4).build())).layer(2, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(4).nOut(2).build())).layer(3, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(2).nOut(1).build())).build(); - MultiLayerNetwork frozenNetwork = new MultiLayerNetwork(confFrozen); - frozenNetwork.init(); - INDArray unfrozenLayerParams = frozenNetwork.getLayer(0).params().dup(); - INDArray frozenLayerParams1 = frozenNetwork.getLayer(1).params().dup(); - INDArray frozenLayerParams2 = frozenNetwork.getLayer(2).params().dup(); - INDArray frozenOutputLayerParams = frozenNetwork.getLayer(3).params().dup(); - MultiLayerNetwork sgdNetwork = new MultiLayerNetwork(confSgd); - sgdNetwork.init(); - INDArray unfrozenSgdLayerParams = sgdNetwork.getLayer(0).params().dup(); - INDArray frozenSgdLayerParams1 = sgdNetwork.getLayer(1).params().dup(); - INDArray frozenSgdLayerParams2 = sgdNetwork.getLayer(2).params().dup(); - INDArray frozenSgdOutputLayerParams = sgdNetwork.getLayer(3).params().dup(); - for (int i = 0; i < 100; i++) { - frozenNetwork.fit(randomData); - } - for (int i = 0; i < 100; i++) { - sgdNetwork.fit(randomData); - } - assertEquals(frozenNetwork.getLayer(0).params(), sgdNetwork.getLayer(0).params()); - assertEquals(frozenNetwork.getLayer(1).params(), sgdNetwork.getLayer(1).params()); - assertEquals(frozenNetwork.getLayer(2).params(), sgdNetwork.getLayer(2).params()); - assertEquals(frozenNetwork.getLayer(3).params(), sgdNetwork.getLayer(3).params()); - } - - @Test - @DisplayName("Test Computation Graph Vs Sgd") - void testComputationGraphVsSgd() { - Nd4j.getRandom().setSeed(12345); - DataSet randomData = new DataSet(Nd4j.rand(100, 4), Nd4j.rand(100, 1)); - String frozenBranchName = "B1-"; - String unfrozenBranchName = "B2-"; - String initialLayer = "initial"; - String frozenBranchUnfrozenLayer0 = frozenBranchName + "0"; - String frozenBranchFrozenLayer1 = frozenBranchName + "1"; - String frozenBranchFrozenLayer2 = frozenBranchName + "2"; - String frozenBranchOutput = frozenBranchName + "Output"; - String unfrozenLayer0 = unfrozenBranchName + "0"; - String unfrozenLayer1 = unfrozenBranchName + "1"; - String unfrozenBranch2 = unfrozenBranchName + "Output"; - ComputationGraphConfiguration computationGraphConf = new NeuralNetConfiguration.Builder().updater(new Sgd(2.0)).seed(12345).graphBuilder().addInputs("input").addLayer(initialLayer, new DenseLayer.Builder().nIn(4).nOut(4).build(), "input").addLayer(frozenBranchUnfrozenLayer0, new DenseLayer.Builder().nIn(4).nOut(3).build(), initialLayer).addLayer(frozenBranchFrozenLayer1, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(3).nOut(4).build()), frozenBranchUnfrozenLayer0).addLayer(frozenBranchFrozenLayer2, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new DenseLayer.Builder().nIn(4).nOut(2).build()), frozenBranchFrozenLayer1).addLayer(unfrozenLayer0, new DenseLayer.Builder().nIn(4).nOut(4).build(), initialLayer).addLayer(unfrozenLayer1, new DenseLayer.Builder().nIn(4).nOut(2).build(), unfrozenLayer0).addLayer(unfrozenBranch2, new DenseLayer.Builder().nIn(2).nOut(1).build(), unfrozenLayer1).addVertex("merge", new MergeVertex(), frozenBranchFrozenLayer2, unfrozenBranch2).addLayer(frozenBranchOutput, new org.deeplearning4j.nn.conf.layers.misc.FrozenLayerWithBackprop(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.TANH).nIn(3).nOut(1).build()), "merge").setOutputs(frozenBranchOutput).build(); - ComputationGraphConfiguration computationSgdGraphConf = new NeuralNetConfiguration.Builder().updater(new Sgd(2.0)).seed(12345).graphBuilder().addInputs("input").addLayer(initialLayer, new DenseLayer.Builder().nIn(4).nOut(4).build(), "input").addLayer(frozenBranchUnfrozenLayer0, new DenseLayer.Builder().nIn(4).nOut(3).build(), initialLayer).addLayer(frozenBranchFrozenLayer1, new DenseLayer.Builder().updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).nIn(3).nOut(4).build(), frozenBranchUnfrozenLayer0).addLayer(frozenBranchFrozenLayer2, new DenseLayer.Builder().updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).nIn(4).nOut(2).build(), frozenBranchFrozenLayer1).addLayer(unfrozenLayer0, new DenseLayer.Builder().nIn(4).nOut(4).build(), initialLayer).addLayer(unfrozenLayer1, new DenseLayer.Builder().nIn(4).nOut(2).build(), unfrozenLayer0).addLayer(unfrozenBranch2, new DenseLayer.Builder().nIn(2).nOut(1).build(), unfrozenLayer1).addVertex("merge", new MergeVertex(), frozenBranchFrozenLayer2, unfrozenBranch2).addLayer(frozenBranchOutput, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).updater(new Sgd(0.0)).biasUpdater(new Sgd(0.0)).activation(Activation.TANH).nIn(3).nOut(1).build(), "merge").setOutputs(frozenBranchOutput).build(); - ComputationGraph frozenComputationGraph = new ComputationGraph(computationGraphConf); - frozenComputationGraph.init(); - INDArray unfrozenLayerParams = frozenComputationGraph.getLayer(frozenBranchUnfrozenLayer0).params().dup(); - INDArray frozenLayerParams1 = frozenComputationGraph.getLayer(frozenBranchFrozenLayer1).params().dup(); - INDArray frozenLayerParams2 = frozenComputationGraph.getLayer(frozenBranchFrozenLayer2).params().dup(); - INDArray frozenOutputLayerParams = frozenComputationGraph.getLayer(frozenBranchOutput).params().dup(); - ComputationGraph sgdComputationGraph = new ComputationGraph(computationSgdGraphConf); - sgdComputationGraph.init(); - INDArray unfrozenSgdLayerParams = sgdComputationGraph.getLayer(frozenBranchUnfrozenLayer0).params().dup(); - INDArray frozenSgdLayerParams1 = sgdComputationGraph.getLayer(frozenBranchFrozenLayer1).params().dup(); - INDArray frozenSgdLayerParams2 = sgdComputationGraph.getLayer(frozenBranchFrozenLayer2).params().dup(); - INDArray frozenSgdOutputLayerParams = sgdComputationGraph.getLayer(frozenBranchOutput).params().dup(); - for (int i = 0; i < 100; i++) { - frozenComputationGraph.fit(randomData); - } - for (int i = 0; i < 100; i++) { - sgdComputationGraph.fit(randomData); - } - assertEquals(frozenComputationGraph.getLayer(frozenBranchUnfrozenLayer0).params(), sgdComputationGraph.getLayer(frozenBranchUnfrozenLayer0).params()); - assertEquals(frozenComputationGraph.getLayer(frozenBranchFrozenLayer1).params(), sgdComputationGraph.getLayer(frozenBranchFrozenLayer1).params()); - assertEquals(frozenComputationGraph.getLayer(frozenBranchFrozenLayer2).params(), sgdComputationGraph.getLayer(frozenBranchFrozenLayer2).params()); - assertEquals(frozenComputationGraph.getLayer(frozenBranchOutput).params(), sgdComputationGraph.getLayer(frozenBranchOutput).params()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/OutputLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/OutputLayerTest.java deleted file mode 100755 index 5d3c9e7c8..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/OutputLayerTest.java +++ /dev/null @@ -1,339 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.*; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.layers.recurrent.RnnOutputLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.deeplearning4j.optimize.api.TrainingListener; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; -import java.util.Collections; -import java.util.Random; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Output Layer Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class OutputLayerTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Set Params") - void testSetParams() { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).updater(new Sgd(1e-1)).layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.ZERO).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).build(); - long numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - OutputLayer l = (OutputLayer) conf.getLayer().instantiate(conf, Collections.singletonList(new ScoreIterationListener(1)), 0, params, true, params.dataType()); - params = l.params(); - l.setParams(params); - assertEquals(params, l.params()); - } - - @Test - @DisplayName("Test Output Layers Rnn Forward Pass") - void testOutputLayersRnnForwardPass() { - // Test output layer with RNNs ( - // Expect all outputs etc. to be 2d - int nIn = 2; - int nOut = 5; - int layerSize = 4; - int timeSeriesLength = 6; - int miniBatchSize = 3; - Random r = new Random(12345L); - INDArray input = Nd4j.zeros(miniBatchSize, nIn, timeSeriesLength); - for (int i = 0; i < miniBatchSize; i++) { - for (int j = 0; j < nIn; j++) { - for (int k = 0; k < timeSeriesLength; k++) { - input.putScalar(new int[] { i, j, k }, r.nextDouble() - 0.5); - } - } - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize).dist(new NormalDistribution(0, 1)).activation(Activation.TANH).updater(new NoOp()).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).dist(new NormalDistribution(0, 1)).updater(new NoOp()).build()).inputPreProcessor(1, new RnnToFeedForwardPreProcessor()).build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - INDArray out2d = mln.feedForward(input).get(2); - assertArrayEquals(out2d.shape(), new long[] { miniBatchSize * timeSeriesLength, nOut }); - INDArray out = mln.output(input); - assertArrayEquals(out.shape(), new long[] { miniBatchSize * timeSeriesLength, nOut }); - INDArray preout = mln.output(input); - assertArrayEquals(preout.shape(), new long[] { miniBatchSize * timeSeriesLength, nOut }); - // As above, but for RnnOutputLayer. Expect all activations etc. to be 3d - MultiLayerConfiguration confRnn = new NeuralNetConfiguration.Builder().seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize).dist(new NormalDistribution(0, 1)).activation(Activation.TANH).updater(new NoOp()).build()).layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).dist(new NormalDistribution(0, 1)).updater(new NoOp()).build()).build(); - MultiLayerNetwork mlnRnn = new MultiLayerNetwork(confRnn); - mln.init(); - INDArray out3d = mlnRnn.feedForward(input).get(2); - assertArrayEquals(out3d.shape(), new long[] { miniBatchSize, nOut, timeSeriesLength }); - INDArray outRnn = mlnRnn.output(input); - assertArrayEquals(outRnn.shape(), new long[] { miniBatchSize, nOut, timeSeriesLength }); - INDArray preoutRnn = mlnRnn.output(input); - assertArrayEquals(preoutRnn.shape(), new long[] { miniBatchSize, nOut, timeSeriesLength }); - } - - @Test - @DisplayName("Test Rnn Output Layer Inc Edge Cases") - void testRnnOutputLayerIncEdgeCases() { - // Basic test + test edge cases: timeSeriesLength==1, miniBatchSize==1, both - int[] tsLength = { 5, 1, 5, 1 }; - int[] miniBatch = { 7, 7, 1, 1 }; - int nIn = 3; - int nOut = 6; - int layerSize = 4; - FeedForwardToRnnPreProcessor proc = new FeedForwardToRnnPreProcessor(); - for (int t = 0; t < tsLength.length; t++) { - Nd4j.getRandom().setSeed(12345); - int timeSeriesLength = tsLength[t]; - int miniBatchSize = miniBatch[t]; - Random r = new Random(12345L); - INDArray input = Nd4j.zeros(miniBatchSize, nIn, timeSeriesLength); - for (int i = 0; i < miniBatchSize; i++) { - for (int j = 0; j < nIn; j++) { - for (int k = 0; k < timeSeriesLength; k++) { - input.putScalar(new int[] { i, j, k }, r.nextDouble() - 0.5); - } - } - } - INDArray labels3d = Nd4j.zeros(miniBatchSize, nOut, timeSeriesLength); - for (int i = 0; i < miniBatchSize; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - int idx = r.nextInt(nOut); - labels3d.putScalar(new int[] { i, idx, j }, 1.0f); - } - } - INDArray labels2d = proc.backprop(labels3d, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize).dist(new NormalDistribution(0, 1)).activation(Activation.TANH).updater(new NoOp()).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).dist(new NormalDistribution(0, 1)).updater(new NoOp()).build()).inputPreProcessor(1, new RnnToFeedForwardPreProcessor()).build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - INDArray out2d = mln.feedForward(input).get(2); - INDArray out3d = proc.preProcess(out2d, miniBatchSize, LayerWorkspaceMgr.noWorkspaces()); - MultiLayerConfiguration confRnn = new NeuralNetConfiguration.Builder().seed(12345L).list().layer(0, new GravesLSTM.Builder().nIn(nIn).nOut(layerSize).dist(new NormalDistribution(0, 1)).activation(Activation.TANH).updater(new NoOp()).build()).layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).dist(new NormalDistribution(0, 1)).updater(new NoOp()).build()).build(); - MultiLayerNetwork mlnRnn = new MultiLayerNetwork(confRnn); - mlnRnn.init(); - INDArray outRnn = mlnRnn.feedForward(input).get(2); - mln.setLabels(labels2d); - mlnRnn.setLabels(labels3d); - mln.computeGradientAndScore(); - mlnRnn.computeGradientAndScore(); - // score is average over all examples. - // However: OutputLayer version has miniBatch*timeSeriesLength "examples" (after reshaping) - // RnnOutputLayer has miniBatch examples - // Hence: expect difference in scores by factor of timeSeriesLength - double score = mln.score() * timeSeriesLength; - double scoreRNN = mlnRnn.score(); - assertTrue(!Double.isNaN(score)); - assertTrue(!Double.isNaN(scoreRNN)); - double relError = Math.abs(score - scoreRNN) / (Math.abs(score) + Math.abs(scoreRNN)); - System.out.println(relError); - assertTrue(relError < 1e-6); - // Check labels and inputs for output layer: - OutputLayer ol = (OutputLayer) mln.getOutputLayer(); - assertArrayEquals(ol.getInput().shape(), new long[] { miniBatchSize * timeSeriesLength, layerSize }); - assertArrayEquals(ol.getLabels().shape(), new long[] { miniBatchSize * timeSeriesLength, nOut }); - RnnOutputLayer rnnol = (RnnOutputLayer) mlnRnn.getOutputLayer(); - // assertArrayEquals(rnnol.getInput().shape(),new int[]{miniBatchSize,layerSize,timeSeriesLength}); - // Input may be set by BaseLayer methods. Thus input may end up as reshaped 2d version instead of original 3d version. - // Not ideal, but everything else works. - assertArrayEquals(rnnol.getLabels().shape(), new long[] { miniBatchSize, nOut, timeSeriesLength }); - // Check shapes of output for both: - assertArrayEquals(out2d.shape(), new long[] { miniBatchSize * timeSeriesLength, nOut }); - INDArray out = mln.output(input); - assertArrayEquals(out.shape(), new long[] { miniBatchSize * timeSeriesLength, nOut }); - INDArray preout = mln.output(input); - assertArrayEquals(preout.shape(), new long[] { miniBatchSize * timeSeriesLength, nOut }); - INDArray outFFRnn = mlnRnn.feedForward(input).get(2); - assertArrayEquals(outFFRnn.shape(), new long[] { miniBatchSize, nOut, timeSeriesLength }); - INDArray outRnn2 = mlnRnn.output(input); - assertArrayEquals(outRnn2.shape(), new long[] { miniBatchSize, nOut, timeSeriesLength }); - INDArray preoutRnn = mlnRnn.output(input); - assertArrayEquals(preoutRnn.shape(), new long[] { miniBatchSize, nOut, timeSeriesLength }); - } - } - - @Test - @DisplayName("Test Compare Rnn Output Rnn Loss") - void testCompareRnnOutputRnnLoss() { - Nd4j.getRandom().setSeed(12345); - int timeSeriesLength = 4; - int nIn = 5; - int layerSize = 6; - int nOut = 6; - int miniBatchSize = 3; - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345L).updater(new NoOp()).list().layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH).dist(new NormalDistribution(0, 1.0)).updater(new NoOp()).build()).layer(new DenseLayer.Builder().nIn(layerSize).nOut(nOut).activation(Activation.IDENTITY).build()).layer(new RnnLossLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf1); - mln.init(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345L).updater(new NoOp()).list().layer(new LSTM.Builder().nIn(nIn).nOut(layerSize).activation(Activation.TANH).dist(new NormalDistribution(0, 1.0)).updater(new NoOp()).build()).layer(new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(layerSize).nOut(nOut).build()).build(); - MultiLayerNetwork mln2 = new MultiLayerNetwork(conf2); - mln2.init(); - mln2.setParams(mln.params()); - INDArray in = Nd4j.rand(new int[] { miniBatchSize, nIn, timeSeriesLength }); - INDArray out1 = mln.output(in); - INDArray out2 = mln.output(in); - assertEquals(out1, out2); - Random r = new Random(12345); - INDArray labels = Nd4j.create(miniBatchSize, nOut, timeSeriesLength); - for (int i = 0; i < miniBatchSize; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - labels.putScalar(i, r.nextInt(nOut), j, 1.0); - } - } - mln.setInput(in); - mln.setLabels(labels); - mln2.setInput(in); - mln2.setLabels(labels); - mln.computeGradientAndScore(); - mln2.computeGradientAndScore(); - assertEquals(mln.gradient().gradient(), mln2.gradient().gradient()); - assertEquals(mln.score(), mln2.score(), 1e-6); - TestUtils.testModelSerialization(mln); - } - - @Test - @DisplayName("Test Cnn Loss Layer") - void testCnnLossLayer() { - for (WorkspaceMode ws : WorkspaceMode.values()) { - log.info("*** Testing workspace: " + ws); - for (Activation a : new Activation[] { Activation.TANH, Activation.SELU }) { - // Check that (A+identity) is equal to (identity+A), for activation A - // i.e., should get same output and weight gradients for both - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345L).updater(new NoOp()).convolutionMode(ConvolutionMode.Same).inferenceWorkspaceMode(ws).trainingWorkspaceMode(ws).list().layer(new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(Activation.IDENTITY).kernelSize(2, 2).stride(1, 1).dist(new NormalDistribution(0, 1.0)).updater(new NoOp()).build()).layer(new CnnLossLayer.Builder(LossFunction.MSE).activation(a).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345L).updater(new NoOp()).convolutionMode(ConvolutionMode.Same).inferenceWorkspaceMode(ws).trainingWorkspaceMode(ws).list().layer(new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(a).kernelSize(2, 2).stride(1, 1).dist(new NormalDistribution(0, 1.0)).updater(new NoOp()).build()).layer(new CnnLossLayer.Builder(LossFunction.MSE).activation(Activation.IDENTITY).build()).build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf1); - mln.init(); - MultiLayerNetwork mln2 = new MultiLayerNetwork(conf2); - mln2.init(); - mln2.setParams(mln.params()); - INDArray in = Nd4j.rand(new int[] { 3, 3, 5, 5 }); - INDArray out1 = mln.output(in); - INDArray out2 = mln2.output(in); - assertEquals(out1, out2); - INDArray labels = Nd4j.rand(out1.shape()); - mln.setInput(in); - mln.setLabels(labels); - mln2.setInput(in); - mln2.setLabels(labels); - mln.computeGradientAndScore(); - mln2.computeGradientAndScore(); - assertEquals(mln.score(), mln2.score(), 1e-6); - assertEquals(mln.gradient().gradient(), mln2.gradient().gradient()); - // Also check computeScoreForExamples - INDArray in2a = Nd4j.rand(new int[] { 1, 3, 5, 5 }); - INDArray labels2a = Nd4j.rand(new int[] { 1, 4, 5, 5 }); - INDArray in2 = Nd4j.concat(0, in2a, in2a); - INDArray labels2 = Nd4j.concat(0, labels2a, labels2a); - INDArray s = mln.scoreExamples(new DataSet(in2, labels2), false); - assertArrayEquals(new long[] { 2, 1 }, s.shape()); - assertEquals(s.getDouble(0), s.getDouble(1), 1e-6); - TestUtils.testModelSerialization(mln); - } - } - } - - @Test - @DisplayName("Test Cnn Loss Layer Comp Graph") - void testCnnLossLayerCompGraph() { - for (WorkspaceMode ws : WorkspaceMode.values()) { - log.info("*** Testing workspace: " + ws); - for (Activation a : new Activation[] { Activation.TANH, Activation.SELU }) { - // Check that (A+identity) is equal to (identity+A), for activation A - // i.e., should get same output and weight gradients for both - ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(12345L).updater(new NoOp()).convolutionMode(ConvolutionMode.Same).inferenceWorkspaceMode(ws).trainingWorkspaceMode(ws).graphBuilder().addInputs("in").addLayer("0", new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(Activation.IDENTITY).kernelSize(2, 2).stride(1, 1).dist(new NormalDistribution(0, 1.0)).updater(new NoOp()).build(), "in").addLayer("1", new CnnLossLayer.Builder(LossFunction.MSE).activation(a).build(), "0").setOutputs("1").build(); - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345L).updater(new NoOp()).convolutionMode(ConvolutionMode.Same).inferenceWorkspaceMode(ws).trainingWorkspaceMode(ws).graphBuilder().addInputs("in").addLayer("0", new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(a).kernelSize(2, 2).stride(1, 1).dist(new NormalDistribution(0, 1.0)).updater(new NoOp()).build(), "in").addLayer("1", new CnnLossLayer.Builder(LossFunction.MSE).activation(Activation.IDENTITY).build(), "0").setOutputs("1").build(); - ComputationGraph graph = new ComputationGraph(conf1); - graph.init(); - ComputationGraph graph2 = new ComputationGraph(conf2); - graph2.init(); - graph2.setParams(graph.params()); - INDArray in = Nd4j.rand(new int[] { 3, 3, 5, 5 }); - INDArray out1 = graph.outputSingle(in); - INDArray out2 = graph2.outputSingle(in); - assertEquals(out1, out2); - INDArray labels = Nd4j.rand(out1.shape()); - graph.setInput(0, in); - graph.setLabels(labels); - graph2.setInput(0, in); - graph2.setLabels(labels); - graph.computeGradientAndScore(); - graph2.computeGradientAndScore(); - assertEquals(graph.score(), graph2.score(), 1e-6); - assertEquals(graph.gradient().gradient(), graph2.gradient().gradient()); - // Also check computeScoreForExamples - INDArray in2a = Nd4j.rand(new int[] { 1, 3, 5, 5 }); - INDArray labels2a = Nd4j.rand(new int[] { 1, 4, 5, 5 }); - INDArray in2 = Nd4j.concat(0, in2a, in2a); - INDArray labels2 = Nd4j.concat(0, labels2a, labels2a); - INDArray s = graph.scoreExamples(new DataSet(in2, labels2), false); - assertArrayEquals(new long[] { 2, 1 }, s.shape()); - assertEquals(s.getDouble(0), s.getDouble(1), 1e-6); - TestUtils.testModelSerialization(graph); - } - } - } - - @Test - @DisplayName("Test Cnn Output Layer Softmax") - void testCnnOutputLayerSoftmax() { - // Check that softmax is applied channels-wise - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).updater(new NoOp()).convolutionMode(ConvolutionMode.Same).list().layer(new ConvolutionLayer.Builder().nIn(3).nOut(4).activation(Activation.IDENTITY).dist(new NormalDistribution(0, 1.0)).updater(new NoOp()).build()).layer(new CnnLossLayer.Builder(LossFunction.MSE).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray in = Nd4j.rand(new int[] { 2, 3, 4, 5 }); - INDArray out = net.output(in); - double min = out.minNumber().doubleValue(); - double max = out.maxNumber().doubleValue(); - assertTrue(min >= 0 && max <= 1.0); - INDArray sum = out.sum(1); - assertEquals(Nd4j.ones(DataType.FLOAT, 2, 4, 5), sum); - } - - @Test - @DisplayName("Test Output Layer Defaults") - void testOutputLayerDefaults() { - new NeuralNetConfiguration.Builder().list().layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder().nIn(10).nOut(10).build()).build(); - new NeuralNetConfiguration.Builder().list().layer(new org.deeplearning4j.nn.conf.layers.LossLayer.Builder().build()).build(); - new NeuralNetConfiguration.Builder().list().layer(new org.deeplearning4j.nn.conf.layers.CnnLossLayer.Builder().build()).build(); - new NeuralNetConfiguration.Builder().list().layer(new org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer.Builder().build()).build(); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/RepeatVectorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/RepeatVectorTest.java deleted file mode 100644 index 7da5c0ec3..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/RepeatVectorTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.misc.RepeatVector; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.primitives.Pair; -import java.util.Arrays; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Repeat Vector Test") -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -@Tag(TagNames.DL4J_OLD_API) -class RepeatVectorTest extends BaseDL4JTest { - - private int REPEAT = 4; - - private Layer getRepeatVectorLayer() { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).dataType(DataType.DOUBLE).layer(new RepeatVector.Builder(REPEAT).build()).build(); - return conf.getLayer().instantiate(conf, null, 0, null, false, DataType.DOUBLE); - } - - @Test - @DisplayName("Test Repeat Vector") - void testRepeatVector() { - double[] arr = new double[] { 1., 2., 3., 1., 2., 3., 1., 2., 3., 1., 2., 3. }; - INDArray expectedOut = Nd4j.create(arr, new long[] { 1, 3, REPEAT }, 'f'); - INDArray input = Nd4j.create(new double[] { 1., 2., 3. }, new long[] { 1, 3 }); - Layer layer = getRepeatVectorLayer(); - INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertTrue(Arrays.equals(expectedOut.shape(), output.shape())); - assertEquals(expectedOut, output); - INDArray epsilon = Nd4j.ones(1, 3, 4); - Pair out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); - INDArray outEpsilon = out.getSecond(); - INDArray expectedEpsilon = Nd4j.create(new double[] { 4., 4., 4. }, new long[] { 1, 3 }); - assertEquals(expectedEpsilon, outEpsilon); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/TestDropout.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/TestDropout.java deleted file mode 100644 index 2f9a0ce88..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/TestDropout.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.layers; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; - -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.RNG) -public class TestDropout extends BaseDL4JTest { - - @Test - public void testDropoutSimple() throws Exception { - //Testing dropout with a single layer - //Layer input: values should be set to either 0.0 or 2.0x original value - - int nIn = 8; - int nOut = 8; - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .updater(new Sgd()) - .dropOut(0.5).list() - .layer(0, new OutputLayer.Builder().activation(Activation.IDENTITY) - .lossFunction(LossFunctions.LossFunction.MSE).nIn(nIn).nOut(nOut) - .weightInit(WeightInit.XAVIER).build()) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - net.getLayer(0).getParam("W").assign(Nd4j.eye(nIn)); - - int nTests = 15; - - Nd4j.getRandom().setSeed(12345); - int noDropoutCount = 0; - for (int i = 0; i < nTests; i++) { - INDArray in = Nd4j.rand(1, nIn); - INDArray out = Nd4j.rand(1, nOut); - INDArray inCopy = in.dup(); - - List l = net.feedForward(in, true); - - INDArray postDropout = l.get(l.size() - 1); - //Dropout occurred. Expect inputs to be either scaled 2x original, or set to 0.0 (with dropout = 0.5) - for (int j = 0; j < inCopy.length(); j++) { - double origValue = inCopy.getDouble(j); - double doValue = postDropout.getDouble(j); - if (doValue > 0.0) { - //Input was kept -> should be scaled by factor of (1.0/0.5 = 2) - assertEquals(origValue * 2.0, doValue, 0.0001); - } - } - - //Do forward pass - //(1) ensure dropout ISN'T being applied for forward pass at test time - //(2) ensure dropout ISN'T being applied for test time scoring - //If dropout is applied at test time: outputs + score will differ between passes - INDArray in2 = Nd4j.rand(1, nIn); - INDArray out2 = Nd4j.rand(1, nOut); - INDArray outTest1 = net.output(in2, false); - INDArray outTest2 = net.output(in2, false); - INDArray outTest3 = net.output(in2, false); - assertEquals(outTest1, outTest2); - assertEquals(outTest1, outTest3); - - double score1 = net.score(new DataSet(in2, out2), false); - double score2 = net.score(new DataSet(in2, out2), false); - double score3 = net.score(new DataSet(in2, out2), false); - assertEquals(score1, score2, 0.0); - assertEquals(score1, score3, 0.0); - } - - if (noDropoutCount >= nTests / 3) { - //at 0.5 dropout ratio and more than a few inputs, expect only a very small number of instances where - //no dropout occurs, just due to random chance - fail("Too many instances of dropout not being applied"); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsNetMNISTTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsNetMNISTTest.java deleted file mode 100644 index 647bde5c2..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/CapsNetMNISTTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.capsule; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.IOException; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ActivationLayer; -import org.deeplearning4j.nn.conf.layers.CapsuleLayer; -import org.deeplearning4j.nn.conf.layers.CapsuleStrengthLayer; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.LossLayer; -import org.deeplearning4j.nn.conf.layers.PrimaryCapsules; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.evaluation.classification.Evaluation; -import org.nd4j.linalg.activations.impl.ActivationSoftmax; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.lossfunctions.impl.LossNegativeLogLikelihood; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Disabled("AB - ignored due to excessive runtime. Keep for manual debugging when required") -@DisplayName("Caps Net MNIST Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class CapsNetMNISTTest extends BaseDL4JTest { - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Test - @DisplayName("Test Caps Net On MNIST") - void testCapsNetOnMNIST() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).updater(new Adam()).list().layer(new ConvolutionLayer.Builder().nOut(16).kernelSize(9, 9).stride(3, 3).build()).layer(new PrimaryCapsules.Builder(8, 8).kernelSize(7, 7).stride(2, 2).build()).layer(new CapsuleLayer.Builder(10, 16, 3).build()).layer(new CapsuleStrengthLayer.Builder().build()).layer(new ActivationLayer.Builder(new ActivationSoftmax()).build()).layer(new LossLayer.Builder(new LossNegativeLogLikelihood()).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - int rngSeed = 12345; - try { - MnistDataSetIterator mnistTrain = new MnistDataSetIterator(64, true, rngSeed); - MnistDataSetIterator mnistTest = new MnistDataSetIterator(64, false, rngSeed); - for (int i = 0; i < 2; i++) { - model.fit(mnistTrain); - } - Evaluation eval = model.evaluate(mnistTest); - assertTrue(eval.accuracy() > 0.95, "Accuracy not over 95%"); - assertTrue(eval.precision() > 0.95, "Precision not over 95%"); - assertTrue(eval.recall() > 0.95, "Recall not over 95%"); - assertTrue(eval.f1() > 0.95, "F1-score not over 95%"); - } catch (IOException e) { - System.out.println("Could not load MNIST."); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/PrimaryCapsulesTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/PrimaryCapsulesTest.java deleted file mode 100644 index 78d1b1fc0..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/capsule/PrimaryCapsulesTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.capsule; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.PrimaryCapsules; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Primary Capsules Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class PrimaryCapsulesTest extends BaseDL4JTest { - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Test - @DisplayName("Test Output Type") - void testOutputType() { - PrimaryCapsules layer = new PrimaryCapsules.Builder(8, 8).kernelSize(7, 7).stride(2, 2).build(); - InputType in1 = InputType.convolutional(7, 7, 16); - assertEquals(InputType.recurrent(8, 8), layer.getOutputType(0, in1)); - } - - @Test - @DisplayName("Test Input Type") - void testInputType() { - PrimaryCapsules layer = new PrimaryCapsules.Builder(8, 8).kernelSize(7, 7).stride(2, 2).build(); - InputType in1 = InputType.convolutional(7, 7, 16); - layer.setNIn(in1, true); - assertEquals(8, layer.getCapsules()); - assertEquals(8, layer.getCapsuleDimensions()); - } - - @Test - @DisplayName("Test Config") - void testConfig() { - PrimaryCapsules layer1 = new PrimaryCapsules.Builder(8, 10).kernelSize(5, 5).stride(4, 4).useLeakyReLU(0.5).build(); - assertEquals(8, layer1.getCapsuleDimensions()); - assertEquals(10, layer1.getChannels()); - assertArrayEquals(new int[] { 5, 5 }, layer1.getKernelSize()); - assertArrayEquals(new int[] { 4, 4 }, layer1.getStride()); - assertArrayEquals(new int[] { 0, 0 }, layer1.getPadding()); - assertArrayEquals(new int[] { 1, 1 }, layer1.getDilation()); - assertTrue(layer1.isUseRelu()); - assertEquals(0.5, layer1.getLeak(), 0.001); - PrimaryCapsules layer2 = new PrimaryCapsules.Builder(8, 10).kernelSize(5, 5).stride(4, 4).build(); - assertFalse(layer2.isUseRelu()); - PrimaryCapsules layer3 = new PrimaryCapsules.Builder(8, 10).kernelSize(5, 5).stride(4, 4).useReLU().build(); - assertTrue(layer3.isUseRelu()); - assertEquals(0, layer3.getLeak(), 0.001); - } - - @Test - @DisplayName("Test Layer") - void testLayer() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123).list().layer(new PrimaryCapsules.Builder(8, 10).kernelSize(5, 5).stride(4, 4).useLeakyReLU(0.5).build()).setInputType(InputType.convolutional(20, 20, 20)).build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - INDArray emptyFeatures = Nd4j.zeros(64, 20, 20, 20); - long[] shape = model.output(emptyFeatures).shape(); - assertArrayEquals(new long[] { 64, 160, 8 }, shape); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvDataFormatTests.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvDataFormatTests.java deleted file mode 100644 index 47783194e..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvDataFormatTests.java +++ /dev/null @@ -1,1068 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.convolution; - -import lombok.*; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.exception.DL4JInvalidInputException; -import org.deeplearning4j.nn.api.MaskState; -import org.deeplearning4j.nn.conf.*; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.CnnLossLayer; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.ZeroPaddingLayer; -import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.ComposableInputPreProcessor; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.workspace.ArrayType; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.deeplearning4j.util.ConvolutionUtils; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.primitives.Pair; -import org.nd4j.linalg.factory.Nd4jBackend; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -public class ConvDataFormatTests extends BaseDL4JTest { - - - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(DataType dataType : Arrays.asList(new DataType[]{DataType.FLOAT, DataType.DOUBLE})) { - args.add(Arguments.of(dataType,nd4jBackend)); - } - } - return args.stream(); - } - - - @Override - public long getTimeoutMilliseconds() { - return 999999999L; - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testConv2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getConv2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getConv2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getConv2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getConv2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testSubsampling2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSubsampling2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getSubsampling2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getSubsampling2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getSubsampling2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testDepthwiseConv2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getDepthwiseConv2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getDepthwiseConv2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getDepthwiseConv2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getDepthwiseConv2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testSeparableConv2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSeparableConv2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getSeparableConv2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getSeparableConv2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getSeparableConv2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testDeconv2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getDeconv2DNet2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getDeconv2DNet2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getDeconv2DNet2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getDeconv2DNet2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testLRN(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getLrnLayer(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getLrnLayer(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getLrnLayer(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getLrnLayer(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testZeroPaddingLayer(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getZeroPaddingNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getZeroPaddingNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getZeroPaddingNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getZeroPaddingNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testCropping2DLayer(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getCropping2dNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getCropping2dNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getCropping2dNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getCropping2dNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testUpsampling2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getUpsamplingNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getUpsamplingNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getUpsamplingNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getUpsamplingNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testBatchNormNet(DataType dataType,Nd4jBackend backend) { - try { - for(boolean useLogStd : new boolean[]{true, false}) { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = (helpers ? "With helpers" : "No helpers") + " - " + (useLogStd ? "logstd" : "std"); - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getBatchNormNet(dataType,useLogStd, CNN2DFormat.NCHW, true)) - .net2(getBatchNormNet(dataType,useLogStd, CNN2DFormat.NCHW, false)) - .net3(getBatchNormNet(dataType,useLogStd, CNN2DFormat.NHWC, true)) - .net4(getBatchNormNet(dataType,useLogStd, CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testCnnLossLayer(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labelsNHWC = TestUtils.randomOneHot(dataType,2*6*6, 3); - labelsNHWC = labelsNHWC.reshape(2,6,6,3); - INDArray labelsNCHW = labelsNHWC.permute(0,3,1,2).dup(); - - - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getCnnLossNet(CNN2DFormat.NCHW, true, ConvolutionMode.Same)) - .net2(getCnnLossNet(CNN2DFormat.NCHW, false, ConvolutionMode.Same)) - .net3(getCnnLossNet(CNN2DFormat.NHWC, true, ConvolutionMode.Same)) - .net4(getCnnLossNet(CNN2DFormat.NHWC, false, ConvolutionMode.Same)) - .inNCHW(inNCHW) - .labelsNCHW(labelsNCHW) - .labelsNHWC(labelsNHWC) - .testLayerIdx(1) - .nhwcOutput(true) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testSpaceToDepthNet(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSpaceToDepthNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getSpaceToDepthNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getSpaceToDepthNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getSpaceToDepthNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testSpaceToBatchNet(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 16, 16); - INDArray labels = TestUtils.randomOneHot(8, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSpaceToBatchNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getSpaceToBatchNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getSpaceToBatchNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getSpaceToBatchNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testLocallyConnected(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getLocallyConnectedNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getLocallyConnectedNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getLocallyConnectedNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getLocallyConnectedNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - - @MethodSource("org.deeplearning4j.nn.layers.convolution.ConvDataFormatTests#params") - @ParameterizedTest - public void testGlobalPooling(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (PoolingType pt : PoolingType.values()) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + pt + ")" : "No helpers (" + pt + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getGlobalPoolingNet(dataType,CNN2DFormat.NCHW, pt, true)) - .net2(getGlobalPoolingNet(dataType,CNN2DFormat.NCHW, pt, false)) - .net3(getGlobalPoolingNet(dataType,CNN2DFormat.NHWC, pt, true)) - .net4(getGlobalPoolingNet(dataType,CNN2DFormat.NHWC, pt, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - private MultiLayerNetwork getConv2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new ConvolutionLayer.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new ConvolutionLayer.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getSubsampling2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new SubsamplingLayer.Builder() - .kernelSize(2, 2) - .stride(1, 1) - .dataFormat(format) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new SubsamplingLayer.Builder() - .kernelSize(2, 2) - .stride(1, 1) - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getSeparableConv2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new SeparableConvolution2D.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new SeparableConvolution2D.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getDepthwiseConv2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new DepthwiseConvolution2D.Builder() - .depthMultiplier(2) - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new DepthwiseConvolution2D.Builder() - .depthMultiplier(2) - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getLrnLayer(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new LocalResponseNormalization.Builder() - .dataFormat(format) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new LocalResponseNormalization.Builder() - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getZeroPaddingNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new ZeroPaddingLayer.Builder(2,2) - .dataFormat(format).build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new ZeroPaddingLayer.Builder(2,2).build(), - format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getCropping2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new Cropping2D.Builder(2,2) - .dataFormat(format).build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new Cropping2D.Builder(2,2) - .build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getUpsamplingNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new Upsampling2D.Builder(2) - .dataFormat(format).build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new Upsampling2D.Builder(2) - .build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getDeconv2DNet2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new Deconvolution2D.Builder().nOut(2) - .activation(Activation.TANH) - .kernelSize(2,2) - .dataFormat(format) - .stride(2,2) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new Deconvolution2D.Builder().nOut(2) - .activation(Activation.TANH) - .kernelSize(2,2) - .dataFormat(format) - .stride(2,2) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getBatchNormNet(DataType dataType,boolean logStdev, CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new BatchNormalization.Builder() - .useLogStd(logStdev) - .dataFormat(format) - .helperAllowFallback(false) - .nOut(3).build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new BatchNormalization.Builder() - .useLogStd(logStdev) - .helperAllowFallback(false) - .nOut(3).build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getSpaceToDepthNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new SpaceToDepthLayer.Builder() - .blocks(2) - .dataFormat(format) - .build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new SpaceToDepthLayer.Builder() - .blocks(2) - .build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getSpaceToBatchNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new SpaceToBatchLayer.Builder() - .blocks(2, 2) - .dataFormat(format) - .build(), format, ConvolutionMode.Same, InputType.convolutional(16, 16, 3, format)); - } else { - return getNetWithLayer(dataType,new SpaceToBatchLayer.Builder() - .blocks(2, 2) - .build(), format, ConvolutionMode.Same, InputType.convolutional(16, 16, 3, format)); - } - } - - private MultiLayerNetwork getLocallyConnectedNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new LocallyConnected2D.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new LocallyConnected2D.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getNetWithLayer(DataType dataType,Layer layer, CNN2DFormat format, ConvolutionMode cm, InputType inputType) { - NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder() - .dataType(dataType) - .seed(12345) - .convolutionMode(cm) - .list() - .layer(new ConvolutionLayer.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .helperAllowFallback(false) - .build()) - .layer(layer) - .layer(new OutputLayer.Builder().nOut(10) - .activation(Activation.SOFTMAX).build()) - .setInputType(inputType != null ? inputType : InputType.convolutional(12, 12, 3, format)); - - if(format == CNN2DFormat.NHWC && !(layer instanceof GlobalPoolingLayer)){ - //Add a preprocessor due to the differences in how NHWC and NCHW activations are flattened - //DL4J's flattening behaviour matches Keras (hence TF) for import compatibility - builder.inputPreProcessor(2, new ComposableInputPreProcessor(new NHWCToNCHWPreprocessor(), new CnnToFeedForwardPreProcessor())); - } - - MultiLayerNetwork net = new MultiLayerNetwork(builder.build()); - net.init(); - return net; - } - - private MultiLayerNetwork getGlobalPoolingNet(DataType dataType,CNN2DFormat format, PoolingType pt, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new GlobalPoolingLayer.Builder(pt) - .poolingDimensions(format == CNN2DFormat.NCHW ? new int[]{2,3} : new int[]{1,2}) - .build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new GlobalPoolingLayer.Builder(pt) - .build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getCnnLossNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm){ - NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder() - .seed(12345) - .convolutionMode(cm) - .list() - .layer(new ConvolutionLayer.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build()); - if(setOnLayerAlso){ - builder.layer(new CnnLossLayer.Builder() - .format(format).activation(Activation.SOFTMAX).build()); - } else { - builder.layer(new CnnLossLayer.Builder() - .activation(Activation.SOFTMAX).build()); - } - - builder.setInputType(InputType.convolutional(12, 12, 3, format)); - - MultiLayerNetwork net = new MultiLayerNetwork(builder.build()); - net.init(); - return net; - } - - @AllArgsConstructor - @Data - @NoArgsConstructor - @Builder - private static class TestCase { - private String msg; - private MultiLayerNetwork net1; - private MultiLayerNetwork net2; - private MultiLayerNetwork net3; - private MultiLayerNetwork net4; - private INDArray inNCHW; - private INDArray labelsNCHW; - private INDArray labelsNHWC; - private int testLayerIdx; - private boolean nhwcOutput; - } - - public static void testHelper(TestCase tc) { - - tc.net2.params().assign(tc.net1.params()); - tc.net3.params().assign(tc.net1.params()); - tc.net4.params().assign(tc.net1.params()); - - //Test forward pass: - INDArray inNCHW = tc.inNCHW; - INDArray inNHWC = tc.inNCHW.permute(0, 2, 3, 1).dup(); - - INDArray l0_1 = tc.net1.feedForward(inNCHW).get(tc.testLayerIdx + 1); - INDArray l0_2 = tc.net2.feedForward(inNCHW).get(tc.testLayerIdx + 1); - INDArray l0_3 = tc.net3.feedForward(inNHWC).get(tc.testLayerIdx + 1); - INDArray l0_4 = tc.net4.feedForward(inNHWC).get(tc.testLayerIdx + 1); - - assertEquals(l0_1, l0_2,tc.msg); - if(l0_1.rank() == 4) { - assertEquals(l0_1, l0_3.permute(0, 3, 1, 2),tc.msg); - assertEquals(l0_1, l0_4.permute(0, 3, 1, 2),tc.msg); - } else { - assertEquals(l0_1, l0_3,tc.msg); - assertEquals( l0_1, l0_4,tc.msg); - } - - - INDArray out1 = tc.net1.output(inNCHW); - INDArray out2 = tc.net2.output(inNCHW); - INDArray out3 = tc.net3.output(inNHWC); - INDArray out4 = tc.net4.output(inNHWC); - - assertEquals(out1, out2,tc.msg); - if(!tc.nhwcOutput) { - assertEquals(out1, out3,tc.msg); - assertEquals( out1, out4,tc.msg); - } else { - assertEquals(out1, out3.permute(0,3,1,2),tc.msg); //NHWC to NCHW - assertEquals(out1, out4.permute(0,3,1,2),tc.msg); - } - - //Test backprop - Pair p1 = tc.net1.calculateGradients(inNCHW, tc.labelsNCHW, null, null); - Pair p2 = tc.net2.calculateGradients(inNCHW, tc.labelsNCHW, null, null); - Pair p3 = tc.net3.calculateGradients(inNHWC, tc.labelsNHWC, null, null); - Pair p4 = tc.net4.calculateGradients(inNHWC, tc.labelsNHWC, null, null); - - //Inpput gradients - assertEquals( p1.getSecond(), p2.getSecond(),tc.msg); - assertEquals(p1.getSecond(), p3.getSecond().permute(0,3,1,2),tc.msg); //Input gradients for NHWC input are also in NHWC format - assertEquals( p1.getSecond(), p4.getSecond().permute(0,3,1,2),tc.msg); - - List diff12 = differentGrads(p1.getFirst(), p2.getFirst()); - List diff13 = differentGrads(p1.getFirst(), p3.getFirst()); - List diff14 = differentGrads(p1.getFirst(), p4.getFirst()); - assertEquals( 0, diff12.size(),tc.msg + " " + diff12); - assertEquals( 0, diff13.size(),tc.msg + " " + diff13); - assertEquals(0, diff14.size(),tc.msg + " " + diff14); - - assertEquals(p1.getFirst().gradientForVariable(), p2.getFirst().gradientForVariable(),tc.msg); - assertEquals(p1.getFirst().gradientForVariable(), p3.getFirst().gradientForVariable(),tc.msg); - assertEquals( p1.getFirst().gradientForVariable(), p4.getFirst().gradientForVariable(),tc.msg); - - tc.net1.fit(inNCHW, tc.labelsNCHW); - tc.net2.fit(inNCHW, tc.labelsNCHW); - tc.net3.fit(inNHWC, tc.labelsNHWC); - tc.net4.fit(inNHWC, tc.labelsNHWC); - - assertEquals(tc.net1.params(), tc.net2.params(),tc.msg); - assertEquals(tc.net1.params(), tc.net3.params(),tc.msg); - assertEquals(tc.net1.params(), tc.net4.params(),tc.msg); - - //Test serialization - MultiLayerNetwork net1a = TestUtils.testModelSerialization(tc.net1); - MultiLayerNetwork net2a = TestUtils.testModelSerialization(tc.net2); - MultiLayerNetwork net3a = TestUtils.testModelSerialization(tc.net3); - MultiLayerNetwork net4a = TestUtils.testModelSerialization(tc.net4); - - out1 = tc.net1.output(inNCHW); - assertEquals(out1, net1a.output(inNCHW),tc.msg); - assertEquals(out1, net2a.output(inNCHW),tc.msg); - if(!tc.nhwcOutput) { - assertEquals( out1, net3a.output(inNHWC),tc.msg); - assertEquals(out1, net4a.output(inNHWC),tc.msg); - } else { - assertEquals(out1, net3a.output(inNHWC).permute(0,3,1,2),tc.msg); //NHWC to NCHW - assertEquals(out1, net4a.output(inNHWC).permute(0,3,1,2),tc.msg); - } - - } - - private static List differentGrads(Gradient g1, Gradient g2) { - List differs = new ArrayList<>(); - Map m1 = g1.gradientForVariable(); - Map m2 = g2.gradientForVariable(); - for(String s : m1.keySet()){ - INDArray a1 = m1.get(s); - INDArray a2 = m2.get(s); - if(!a1.equals(a2)){ - differs.add(s); - } - } - return differs; - } - - - //Converts NHWC to NCHW activations - @EqualsAndHashCode - private static class NHWCToNCHWPreprocessor implements InputPreProcessor { - - @Override - public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) { - return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.permute(0,3,1,2)); - } - - @Override - public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) { - return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, output.permute(0,2,3,1)); - } - - @Override - public InputPreProcessor clone() { - return this; - } - - @Override - public InputType getOutputType(InputType inputType) { - InputType.InputTypeConvolutional c = (InputType.InputTypeConvolutional) inputType; - return InputType.convolutional(c.getHeight(), c.getWidth(), c.getChannels(), CNN2DFormat.NCHW); - } - - @Override - public Pair feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState, int minibatchSize) { - return null; - } - } - - - @Test - public void testWrongFormatIn() { - - for(CNN2DFormat df : CNN2DFormat.values()) { - for(int i = 0; i < 4; i++) { - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder() - .list(); - switch (i){ - case 0: - b.layer(new ConvolutionLayer.Builder().kernelSize(2,2).nIn(3).nOut(3).dataFormat(df).build()); - b.setInputType(InputType.convolutional(12,12,3,df)); - break; - case 1: - b.layer(new DepthwiseConvolution2D.Builder().kernelSize(2,2).nIn(3).nOut(3).dataFormat(df).build()); - b.setInputType(InputType.convolutional(12,12,3,df)); - break; - case 2: - b.layer(new Deconvolution2D.Builder().dataFormat(df).kernelSize(2,2).nIn(3).nOut(3).build()); - b.setInputType(InputType.convolutional(12,12,3,df)); - break; - case 3: - b.layer(new SeparableConvolution2D.Builder().dataFormat(df).kernelSize(2,2).nIn(3).nOut(3).build()); - b.setInputType(InputType.convolutional(12,12,3,df)); - break; - } - - - MultiLayerNetwork net = new MultiLayerNetwork(b.build()); - net.init(); - - INDArray in; - INDArray wrongFormatIn; - if(df == CNN2DFormat.NCHW){ - in = Nd4j.create(DataType.FLOAT, 5, 3, 12, 12); - wrongFormatIn = Nd4j.create(DataType.FLOAT, 5, 12, 12, 3); - } else { - in = Nd4j.create(DataType.FLOAT, 5, 12, 12, 3); - wrongFormatIn = Nd4j.create(DataType.FLOAT, 5, 3, 12, 12); - } - - net.output(in); - - try { - net.output(wrongFormatIn); - } catch (DL4JInvalidInputException e) { -// e.printStackTrace(); - String msg = e.getMessage(); - assertTrue(msg.contains(ConvolutionUtils.NCHW_NHWC_ERROR_MSG) || msg.contains("input array channels does not match CNN layer configuration"),msg); - } - } - } - - - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerSetupTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerSetupTest.java deleted file mode 100644 index 0b586f900..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerSetupTest.java +++ /dev/null @@ -1,318 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.convolution; - -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.image.recordreader.ImageRecordReader; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToCnnPreProcessor; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.params.BatchNormalizationParamInitializer; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.linalg.util.FeatureUtil; -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * @author Adam Gibson - */ -@DisplayName("Convolution Layer Setup Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.LARGE_RESOURCES) -class ConvolutionLayerSetupTest extends BaseDL4JTest { - - @TempDir - public Path testDir; - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Test - @DisplayName("Test Convolution Layer Setup") - void testConvolutionLayerSetup() { - MultiLayerConfiguration.Builder builder = inComplete(); - builder.setInputType(InputType.convolutionalFlat(28, 28, 1)); - MultiLayerConfiguration completed = complete().build(); - MultiLayerConfiguration test = builder.build(); - assertEquals(completed, test); - } - - @Test - @DisplayName("Test Dense To Output Layer") - void testDenseToOutputLayer() { - Nd4j.getRandom().setSeed(12345); - final int numRows = 76; - final int numColumns = 76; - int nChannels = 3; - int outputNum = 6; - int seed = 123; - // setup the network - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).l1(1e-1).l2(2e-4).dropOut(0.5).miniBatch(true).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list().layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).build()).layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).build()).layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build()).layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(numRows, numColumns, nChannels)); - DataSet d = new DataSet(Nd4j.rand(new int[] { 10, nChannels, numRows, numColumns }), FeatureUtil.toOutcomeMatrix(new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, 6)); - MultiLayerNetwork network = new MultiLayerNetwork(builder.build()); - network.init(); - network.fit(d); - } - - @Test - @DisplayName("Test Mnist Lenet") - void testMnistLenet() throws Exception { - MultiLayerConfiguration.Builder incomplete = incompleteMnistLenet(); - incomplete.setInputType(InputType.convolutionalFlat(28, 28, 1)); - MultiLayerConfiguration testConf = incomplete.build(); - assertEquals(800, ((FeedForwardLayer) testConf.getConf(4).getLayer()).getNIn()); - assertEquals(500, ((FeedForwardLayer) testConf.getConf(5).getLayer()).getNIn()); - // test instantiation - DataSetIterator iter = new MnistDataSetIterator(10, 10); - MultiLayerNetwork network = new MultiLayerNetwork(testConf); - network.init(); - network.fit(iter.next()); - } - - @Test - @DisplayName("Test Multi Channel") - void testMultiChannel() throws Exception { - INDArray in = Nd4j.rand(new int[] { 10, 3, 28, 28 }); - INDArray labels = Nd4j.rand(10, 2); - DataSet next = new DataSet(in, labels); - NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLFW(); - builder.setInputType(InputType.convolutional(28, 28, 3)); - MultiLayerConfiguration conf = builder.build(); - ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(2).getLayer(); - assertEquals(6, layer2.getNIn()); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - network.fit(next); - } - - @Test - @DisplayName("Test LRN") - void testLRN(@TempDir Path testFolder) throws Exception { - List labels = new ArrayList<>(Arrays.asList("Zico", "Ziwang_Xu")); - File dir = testFolder.toFile(); - new ClassPathResource("lfwtest/").copyDirectory(dir); - String rootDir = dir.getAbsolutePath(); - RecordReader reader = new ImageRecordReader(28, 28, 3); - reader.initialize(new FileSplit(new File(rootDir))); - DataSetIterator recordReader = new RecordReaderDataSetIterator(reader, 10, 1, labels.size()); - labels.remove("lfwtest"); - NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLRN(); - builder.setInputType(InputType.convolutional(28, 28, 3)); - MultiLayerConfiguration conf = builder.build(); - ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(3).getLayer(); - assertEquals(6, layer2.getNIn()); - } - - public MultiLayerConfiguration.Builder incompleteLRN() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(3).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list().layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 5, 5 }).nOut(6).build()).layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(new int[] { 2, 2 }).build()).layer(2, new LocalResponseNormalization.Builder().build()).layer(3, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 5, 5 }).nOut(6).build()).layer(4, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(new int[] { 2, 2 }).build()).layer(5, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(2).activation(Activation.SOFTMAX).build()); - return builder; - } - - public MultiLayerConfiguration.Builder incompleteLFW() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(3).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list().layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 5, 5 }).nOut(6).build()).layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(new int[] { 2, 2 }).build()).layer(2, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 5, 5 }).nOut(6).build()).layer(3, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(new int[] { 2, 2 }).build()).layer(4, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nOut(2).build()); - return builder; - } - - public MultiLayerConfiguration.Builder incompleteMnistLenet() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(3).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list().layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 5, 5 }).nIn(1).nOut(20).build()).layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(new int[] { 2, 2 }, new int[] { 2, 2 }).build()).layer(2, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 5, 5 }).nIn(20).nOut(50).build()).layer(3, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(new int[] { 2, 2 }, new int[] { 2, 2 }).build()).layer(4, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nOut(500).build()).layer(5, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nOut(10).build()); - return builder; - } - - public MultiLayerConfiguration mnistLenet() { - MultiLayerConfiguration builder = new NeuralNetConfiguration.Builder().seed(3).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list().layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 5, 5 }).nIn(1).nOut(6).build()).layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(new int[] { 5, 5 }, new int[] { 2, 2 }).build()).layer(2, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 5, 5 }).nIn(1).nOut(6).build()).layer(3, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(new int[] { 5, 5 }, new int[] { 2, 2 }).build()).layer(4, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(150).nOut(10).build()).build(); - return builder; - } - - public MultiLayerConfiguration.Builder inComplete() { - int nChannels = 1; - int outputNum = 10; - int seed = 123; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list().layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 10, 10 }, new int[] { 2, 2 }).nIn(nChannels).nOut(6).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()); - return builder; - } - - public MultiLayerConfiguration.Builder complete() { - final int numRows = 28; - final int numColumns = 28; - int nChannels = 1; - int outputNum = 10; - int seed = 123; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list().layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(new int[] { 10, 10 }, new int[] { 2, 2 }).nIn(nChannels).nOut(6).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(// 216 - 5 * 5 * 1 * 6).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).inputPreProcessor(0, new FeedForwardToCnnPreProcessor(numRows, numColumns, nChannels)).inputPreProcessor(2, new CnnToFeedForwardPreProcessor(5, 5, 6)); - return builder; - } - - @Test - @DisplayName("Test Deconvolution") - void testDeconvolution() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(0, new Deconvolution2D.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()).layer(1, new SubsamplingLayer.Builder().kernelSize(2, 2).padding(1, 1).stride(2, 2).build()).layer(2, new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)); - MultiLayerConfiguration conf = builder.build(); - assertNotNull(conf.getInputPreProcess(2)); - assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); - CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); - assertEquals(29, proc.getInputHeight()); - assertEquals(29, proc.getInputWidth()); - assertEquals(3, proc.getNumChannels()); - assertEquals(29 * 29 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); - } - - @Test - @DisplayName("Test Sub Sampling With Padding") - void testSubSamplingWithPadding() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(0, // (28-2+0)/2+1 = 14 - new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()).layer(1, // (14-2+2)/2+1 = 8 -> 8x8x3 - new SubsamplingLayer.Builder().kernelSize(2, 2).padding(1, 1).stride(2, 2).build()).layer(2, new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)); - MultiLayerConfiguration conf = builder.build(); - assertNotNull(conf.getInputPreProcess(2)); - assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); - CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); - assertEquals(8, proc.getInputHeight()); - assertEquals(8, proc.getInputWidth()); - assertEquals(3, proc.getNumChannels()); - assertEquals(8 * 8 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); - } - - @Test - @DisplayName("Test Upsampling") - void testUpsampling() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(// (28-2+0)/2+1 = 14 - new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()).layer(// 14 * 3 = 42! - new Upsampling2D.Builder().size(3).build()).layer(new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)); - MultiLayerConfiguration conf = builder.build(); - assertNotNull(conf.getInputPreProcess(2)); - assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); - CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); - assertEquals(42, proc.getInputHeight()); - assertEquals(42, proc.getInputWidth()); - assertEquals(3, proc.getNumChannels()); - assertEquals(42 * 42 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); - } - - @Test - @DisplayName("Test Space To Batch") - void testSpaceToBatch() { - int[] blocks = new int[] { 2, 2 }; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(// (28-2+0)/2+1 = 14 - new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()).layer(// Divide space dimensions by blocks, i.e. 14/2 = 7 - new SpaceToBatchLayer.Builder(blocks).build()).layer(new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)); - MultiLayerConfiguration conf = builder.build(); - assertNotNull(conf.getInputPreProcess(2)); - assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); - CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); - assertEquals(7, proc.getInputHeight()); - assertEquals(7, proc.getInputWidth()); - assertEquals(3, proc.getNumChannels()); - } - - @Test - @DisplayName("Test Space To Depth") - void testSpaceToDepth() { - int blocks = 2; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()).layer(new SpaceToDepthLayer.Builder(blocks, SpaceToDepthLayer.DataFormat.NCHW).build()).layer(// nIn of the next layer gets multiplied by 2*2. - new OutputLayer.Builder().nIn(3 * 2 * 2).nOut(3).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)); - MultiLayerConfiguration conf = builder.build(); - assertNotNull(conf.getInputPreProcess(2)); - assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); - CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); - assertEquals(7, proc.getInputHeight()); - assertEquals(7, proc.getInputWidth()); - assertEquals(12, proc.getNumChannels()); - } - - @Test - @DisplayName("Test CNNDBN Multi Layer") - void testCNNDBNMultiLayer() throws Exception { - DataSetIterator iter = new MnistDataSetIterator(2, 2); - DataSet next = iter.next(); - // Run with separate activation layer - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).weightInit(WeightInit.XAVIER).list().layer(0, new ConvolutionLayer.Builder(new int[] { 1, 1 }, new int[] { 1, 1 }).nIn(1).nOut(6).activation(Activation.IDENTITY).build()).layer(1, new BatchNormalization.Builder().build()).layer(2, new ActivationLayer.Builder().activation(Activation.RELU).build()).layer(3, new DenseLayer.Builder().nIn(28 * 28 * 6).nOut(10).activation(Activation.IDENTITY).build()).layer(4, new BatchNormalization.Builder().nOut(10).build()).layer(5, new ActivationLayer.Builder().activation(Activation.RELU).build()).layer(6, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - network.setInput(next.getFeatures()); - INDArray activationsActual = network.output(next.getFeatures()); - assertEquals(10, activationsActual.shape()[1], 1e-2); - network.fit(next); - INDArray actualGammaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.GAMMA); - INDArray actualBetaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.BETA); - assertTrue(actualGammaParam != null); - assertTrue(actualBetaParam != null); - } - - @Test - @DisplayName("Test Separable Conv 2 D") - void testSeparableConv2D() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(new SeparableConvolution2D.Builder(2, 2).depthMultiplier(2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()).layer(// (14-2+2)/2+1 = 8 -> 8x8x3 - new SubsamplingLayer.Builder().kernelSize(2, 2).padding(1, 1).stride(2, 2).build()).layer(2, new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)); - MultiLayerConfiguration conf = builder.build(); - assertNotNull(conf.getInputPreProcess(2)); - assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); - CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); - assertEquals(8, proc.getInputHeight()); - assertEquals(8, proc.getInputWidth()); - assertEquals(3, proc.getNumChannels()); - assertEquals(8 * 8 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); - } - - @Test - @DisplayName("Test Deconv 2 D") - void testDeconv2D() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(new Deconvolution2D.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()).layer(new SubsamplingLayer.Builder().kernelSize(2, 2).padding(1, 1).stride(2, 2).build()).layer(2, new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)); - MultiLayerConfiguration conf = builder.build(); - assertNotNull(conf.getInputPreProcess(2)); - assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor); - CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2); - assertEquals(29, proc.getInputHeight()); - assertEquals(29, proc.getInputWidth()); - assertEquals(3, proc.getNumChannels()); - assertEquals(29 * 29 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerTest.java deleted file mode 100644 index 854fbcf92..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerTest.java +++ /dev/null @@ -1,667 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.convolution; - -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.eval.Evaluation; -import org.deeplearning4j.exception.DL4JException; -import org.deeplearning4j.exception.DL4JInvalidInputException; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.RNNFormat; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.Convolution1DLayer; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.weights.WeightInitNormal; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.enums.RnnDataFormat; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.activations.impl.ActivationSoftmax; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.linalg.convolution.Convolution; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.INDArrayIndex; -import org.nd4j.linalg.indexing.NDArrayIndex; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.linalg.lossfunctions.impl.LossMCXENT; -import java.io.File; -import java.util.Arrays; -import java.util.List; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author Adam Gibson - */ -@DisplayName("Convolution Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -class ConvolutionLayerTest extends BaseDL4JTest { - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Test - @DisplayName("Test Twd First Layer") - void testTwdFirstLayer() throws Exception { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4).updater(new Nesterovs(0.9)).dropOut(0.5).list().layer(0, // 16 filters kernel size 8 stride 4 - new ConvolutionLayer.Builder(8, 8).stride(4, 4).nOut(16).dropOut(0.5).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, // 32 filters kernel size 4 stride 2 - new ConvolutionLayer.Builder(4, 4).stride(2, 2).nOut(32).dropOut(0.5).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(2, // fully connected with 256 rectified units - new DenseLayer.Builder().nOut(256).activation(Activation.RELU).weightInit(WeightInit.XAVIER).dropOut(0.5).build()).layer(3, // output layer - new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS).nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)); - DataSetIterator iter = new MnistDataSetIterator(10, 10); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - DataSet ds = iter.next(); - for (int i = 0; i < 5; i++) { - network.fit(ds); - } - } - - @Test - @DisplayName("Test CNN Sub Combo With Mixed HW") - void testCNNSubComboWithMixedHW() { - int imageHeight = 20; - int imageWidth = 23; - int nChannels = 1; - int classes = 2; - int numSamples = 200; - int kernelHeight = 3; - int kernelWidth = 3; - DataSet trainInput; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, new ConvolutionLayer.Builder(kernelHeight, kernelWidth).stride(1, 1).nOut(2).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new SubsamplingLayer.Builder().poolingType(SubsamplingLayer.PoolingType.MAX).kernelSize(imageHeight - kernelHeight, 1).stride(1, 1).build()).layer(2, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, nChannels)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); - INDArray emptyLables = Nd4j.zeros(numSamples, classes); - trainInput = new DataSet(emptyFeatures, emptyLables); - model.fit(trainInput); - } - - @Test - @DisplayName("Test Causal 1 d") - void testCausal1d() { - Nd4j.getEnvironment().setVerbose(true); - Nd4j.getEnvironment().setDebug(true); - // See: Fixes: https://github.com/eclipse/deeplearning4j/issues/9060 - double learningRate = 1e-3; - long seed = 123; - long timeSteps = 72; - long vectorLength = 64; - long batchSize = 1; - INDArray arr = Nd4j.randn(batchSize, vectorLength, timeSteps); - MultiLayerConfiguration build = new NeuralNetConfiguration.Builder().seed(seed).activation(Activation.RELU).weightInit(// better init - new WeightInitNormal()).updater(new Adam(learningRate)).list().layer(new Convolution1D.Builder().kernelSize(2).rnnDataFormat(RNNFormat.NCW).stride(1).nOut(14).convolutionMode(ConvolutionMode.Causal).dilation(4).build()).layer(new RnnLossLayer.Builder().dataFormat(RNNFormat.NCW).activation(new ActivationSoftmax()).lossFunction(new LossMCXENT()).build()).setInputType(InputType.recurrent(vectorLength, timeSteps, RNNFormat.NCW)).build(); - MultiLayerNetwork network = new MultiLayerNetwork(build); - network.init(); - INDArray output = network.output(arr); - assertArrayEquals(new long[] { 1, 14, 72 }, output.shape()); - System.out.println(output); - } - - @Test - @DisplayName("Test CNN Too Large Kernel") - void testCNNTooLargeKernel() { - assertThrows(DL4JException.class, () -> { - int imageHeight = 20; - int imageWidth = 23; - int nChannels = 1; - int classes = 2; - int numSamples = 200; - int kernelHeight = imageHeight; - int kernelWidth = imageWidth + 1; - DataSet trainInput; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, // (img-kernel+2*padding)/stride + 1: must be >= 1. Therefore: with p=0, kernel <= img size - new ConvolutionLayer.Builder(kernelHeight, kernelWidth).stride(1, 1).nOut(2).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, nChannels)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); - INDArray emptyLables = Nd4j.zeros(numSamples, classes); - trainInput = new DataSet(emptyFeatures, emptyLables); - model.fit(trainInput); - }); - } - - @Test - @DisplayName("Test CNN Zero Stride") - void testCNNZeroStride() { - assertThrows(Exception.class, () -> { - int imageHeight = 20; - int imageWidth = 23; - int nChannels = 1; - int classes = 2; - int numSamples = 200; - int kernelHeight = imageHeight; - int kernelWidth = imageWidth; - DataSet trainInput; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, new ConvolutionLayer.Builder(kernelHeight, kernelWidth).stride(1, 0).nOut(2).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(imageHeight, imageWidth, nChannels)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); - INDArray emptyLables = Nd4j.zeros(numSamples, classes); - trainInput = new DataSet(emptyFeatures, emptyLables); - model.fit(trainInput); - }); - } - - @Test - @DisplayName("Test CNN Bias Init") - void testCNNBiasInit() { - ConvolutionLayer cnn = new ConvolutionLayer.Builder().nIn(1).nOut(3).biasInit(1).build(); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(cnn).build(); - val numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - assertEquals(1, layer.getParam("b").size(0)); - } - - @Test - @DisplayName("Test CNN Input Setup MNIST") - void testCNNInputSetupMNIST() throws Exception { - INDArray input = getMnistData(); - Layer layer = getMNISTConfig(); - layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(input, layer.input()); - assertArrayEquals(input.shape(), layer.input().shape()); - } - - @Test - @DisplayName("Test Feature Map Shape MNIST") - void testFeatureMapShapeMNIST() throws Exception { - int inputWidth = 28; - int[] stride = new int[] { 1, 1 }; - int[] padding = new int[] { 0, 0 }; - int[] kernelSize = new int[] { 9, 9 }; - int nChannelsIn = 1; - int depth = 20; - int featureMapWidth = (inputWidth + padding[1] * 2 - kernelSize[1]) / stride[1] + 1; - INDArray input = getMnistData(); - Layer layer = getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding); - INDArray convActivations = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(featureMapWidth, convActivations.size(2)); - assertEquals(depth, convActivations.size(1)); - } - - @Test - @DisplayName("Test Activate Results Contained") - void testActivateResultsContained() { - Layer layer = getContainedConfig(); - INDArray input = getContainedData(); - INDArray expectedOutput = Nd4j.create(new float[] { 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f, 0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f, 0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f, 0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f, 0.99966465f, 0.99966465f, 0.99966465f }, new int[] { 1, 2, 4, 4 }); - INDArray convActivations = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(expectedOutput.shape(), convActivations.shape()); - assertEquals(expectedOutput, convActivations); - } - - // //////////////////////////////////////////////////////////////////////////////// - private static Layer getCNNConfig(int nIn, int nOut, int[] kernelSize, int[] stride, int[] padding) { - ConvolutionLayer layer = new ConvolutionLayer.Builder(kernelSize, stride, padding).nIn(nIn).nOut(nOut).activation(Activation.SIGMOID).build(); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(layer).build(); - val numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - return conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - } - - public Layer getMNISTConfig() { - int[] kernelSize = new int[] { 9, 9 }; - int[] stride = new int[] { 1, 1 }; - int[] padding = new int[] { 1, 1 }; - int nChannelsIn = 1; - int depth = 20; - return getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding); - } - - public INDArray getMnistData() throws Exception { - int inputWidth = 28; - int inputHeight = 28; - int nChannelsIn = 1; - int nExamples = 5; - DataSetIterator data = new MnistDataSetIterator(nExamples, nExamples); - DataSet mnist = data.next(); - nExamples = mnist.numExamples(); - return mnist.getFeatures().reshape(nExamples, nChannelsIn, inputHeight, inputWidth); - } - - public Layer getContainedConfig() { - int[] kernelSize = new int[] { 2, 2 }; - int[] stride = new int[] { 2, 2 }; - int[] padding = new int[] { 0, 0 }; - int nChannelsIn = 1; - int depth = 2; - INDArray W = Nd4j.create(new double[] { 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 }, new int[] { 2, 1, 2, 2 }); - INDArray b = Nd4j.create(new double[] { 1, 1 }); - Layer layer = getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding); - layer.setParam("W", W); - layer.setParam("b", b); - return layer; - } - - public INDArray getContainedData() { - INDArray ret = Nd4j.create(new float[] { 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4 }, new int[] { 1, 1, 8, 8 }); - return ret; - } - - public INDArray getContainedCol() { - return Nd4j.create(new float[] { 1, 1, 1, 1, 3, 3, 3, 3, 1, 1, 1, 1, 3, 3, 3, 3, 1, 1, 1, 1, 3, 3, 3, 3, 1, 1, 1, 1, 3, 3, 3, 3, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4 }, new int[] { 1, 1, 2, 2, 4, 4 }); - } - - // //////////////////////////////////////////////////////////////////////////////// - @Test - @DisplayName("Test CNNMLN Pretrain") - void testCNNMLNPretrain() throws Exception { - // Note CNN does not do pretrain - int numSamples = 10; - int batchSize = 10; - DataSetIterator mnistIter = new MnistDataSetIterator(batchSize, numSamples, true); - MultiLayerNetwork model = getCNNMLNConfig(false, true); - model.fit(mnistIter); - mnistIter.reset(); - MultiLayerNetwork model2 = getCNNMLNConfig(false, true); - model2.fit(mnistIter); - mnistIter.reset(); - DataSet test = mnistIter.next(); - Evaluation eval = new Evaluation(); - INDArray output = model.output(test.getFeatures()); - eval.eval(test.getLabels(), output); - double f1Score = eval.f1(); - Evaluation eval2 = new Evaluation(); - INDArray output2 = model2.output(test.getFeatures()); - eval2.eval(test.getLabels(), output2); - double f1Score2 = eval2.f1(); - assertEquals(f1Score, f1Score2, 1e-4); - } - - @Test - @DisplayName("Test CNNMLN Backprop") - void testCNNMLNBackprop() throws Exception { - int numSamples = 10; - int batchSize = 10; - DataSetIterator mnistIter = new MnistDataSetIterator(batchSize, numSamples, true); - MultiLayerNetwork model = getCNNMLNConfig(true, false); - model.fit(mnistIter); - MultiLayerNetwork model2 = getCNNMLNConfig(true, false); - model2.fit(mnistIter); - mnistIter.reset(); - DataSet test = mnistIter.next(); - Evaluation eval = new Evaluation(); - INDArray output = model.output(test.getFeatures()); - eval.eval(test.getLabels(), output); - double f1Score = eval.f1(); - Evaluation eval2 = new Evaluation(); - INDArray output2 = model2.output(test.getFeatures()); - eval2.eval(test.getLabels(), output2); - double f1Score2 = eval2.f1(); - assertEquals(f1Score, f1Score2, 1e-4); - } - - @Test - @DisplayName("Test Get Set Params") - void testGetSetParams() { - MultiLayerNetwork net = getCNNMLNConfig(true, false); - INDArray paramsOrig = net.params().dup(); - net.setParams(paramsOrig); - INDArray params2 = net.params(); - assertEquals(paramsOrig, params2); - } - - private static final int kH = 2; - - private static final int kW = 2; - - private static final int[] strides = { 1, 1 }; - - private static final int[] pad = { 0, 0 }; - - private static final int miniBatch = 2; - - private static final int inDepth = 2; - - private static final int height = 3; - - private static final int width = 3; - - private static final int outW = 2; - - private static final int outH = 2; - - private static INDArray getInput() { - /* - ----- Input images ----- - example 0: - channels 0 channels 1 - [ 0 1 2 [ 9 10 11 - 3 4 5 12 13 14 - 6 7 8] 15 16 17] - example 1: - [18 19 20 [27 28 29 - 21 22 23 30 31 32 - 24 25 26] 33 34 35] - */ - INDArray input = Nd4j.create(new int[] { miniBatch, inDepth, height, width }, 'c'); - input.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 } })); - input.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 9, 10, 11 }, { 12, 13, 14 }, { 15, 16, 17 } })); - input.put(new INDArrayIndex[] { NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 18, 19, 20 }, { 21, 22, 23 }, { 24, 25, 26 } })); - input.put(new INDArrayIndex[] { NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 27, 28, 29 }, { 30, 31, 32 }, { 33, 34, 35 } })); - return input; - } - - @Test - @DisplayName("Test Cnn Im 2 Col Reshaping") - void testCnnIm2ColReshaping() { - // This test: a bit unusual in that it tests the *assumptions* of the CNN implementation rather than the implementation itself - // Specifically, it tests the row and column orders after reshaping on im2col is reshaped (both forward and backward pass) - INDArray input = getInput(); - // im2col in the required order: want [outW,outH,miniBatch,depthIn,kH,kW], but need to input [miniBatch,channels,kH,kW,outH,outW] - // given the current im2col implementation - // To get this: create an array of the order we want, permute it to the order required by im2col implementation, and then do im2col on that - // to get old order from required order: permute(2,3,4,5,1,2) - INDArray col = Nd4j.create(new int[] { miniBatch, outH, outW, inDepth, kH, kW }, 'c'); - INDArray col2 = col.permute(0, 3, 4, 5, 1, 2); - Convolution.im2col(input, kH, kW, strides[0], strides[1], pad[0], pad[1], false, col2); - /* - Expected Output, im2col - - example 0 - - channels 0 channels 1 - h0,w0 h0,w1 h0,w0 h0,w1 - 0 1 1 2 9 10 10 11 - 3 4 4 5 12 13 13 14 - - h1,w0 h1,w1 h1,w0 h1,w1 - 3 4 4 5 12 13 13 14 - 6 7 7 8 15 16 16 17 - - - example 1 - - channels 0 channels 1 - h0,w0 h0,w1 h0,w0 h0,w1 - 18 19 19 20 27 28 28 29 - 21 22 22 23 30 31 31 32 - - h1,w0 h1,w1 h1,w0 h1,w1 - 21 22 22 23 30 31 31 32 - 24 25 25 26 33 34 34 35 - */ - // Now, after reshaping im2col to 2d, we expect: - // Rows with order (wOut0,hOut0,mb0), (wOut1,hOut0,mb0), (wOut0,hOut1,mb0), (wOut1,hOut1,mb0), (wOut0,hOut0,mb1), ... - // Columns with order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), ... - INDArray reshapedCol = Shape.newShapeNoCopy(col, new int[] { miniBatch * outH * outW, inDepth * kH * kW }, false); - INDArray exp2d = Nd4j.create(outW * outH * miniBatch, inDepth * kH * kW); - // wOut0,hOut0,mb0 -> both depths, in order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), (d1,kh0,kw1), (d1,kh1,kw0), (d1,kh1,kw1) - exp2d.putRow(0, Nd4j.create(new double[] { 0, 1, 3, 4, 9, 10, 12, 13 })); - // wOut1,hOut0,mb0 - exp2d.putRow(1, Nd4j.create(new double[] { 1, 2, 4, 5, 10, 11, 13, 14 })); - // wOut0,hOut1,mb0 - exp2d.putRow(2, Nd4j.create(new double[] { 3, 4, 6, 7, 12, 13, 15, 16 })); - // wOut1,hOut1,mb0 - exp2d.putRow(3, Nd4j.create(new double[] { 4, 5, 7, 8, 13, 14, 16, 17 })); - // wOut0,hOut0,mb1 - exp2d.putRow(4, Nd4j.create(new double[] { 18, 19, 21, 22, 27, 28, 30, 31 })); - // wOut1,hOut0,mb1 - exp2d.putRow(5, Nd4j.create(new double[] { 19, 20, 22, 23, 28, 29, 31, 32 })); - // wOut0,hOut1,mb1 - exp2d.putRow(6, Nd4j.create(new double[] { 21, 22, 24, 25, 30, 31, 33, 34 })); - // wOut1,hOut1,mb1 - exp2d.putRow(7, Nd4j.create(new double[] { 22, 23, 25, 26, 31, 32, 34, 35 })); - assertEquals(exp2d, reshapedCol); - // Check the same thing for the backprop im2col (different order) - INDArray colBackprop = Nd4j.create(new int[] { miniBatch, outH, outW, inDepth, kH, kW }, 'c'); - INDArray colBackprop2 = colBackprop.permute(0, 3, 4, 5, 1, 2); - Convolution.im2col(input, kH, kW, strides[0], strides[1], pad[0], pad[1], false, colBackprop2); - INDArray reshapedColBackprop = Shape.newShapeNoCopy(colBackprop, new int[] { miniBatch * outH * outW, inDepth * kH * kW }, false); - // Rows with order (mb0,h0,w0), (mb0,h0,w1), (mb0,h1,w0), (mb0,h1,w1), (mb1,h0,w0), (mb1,h0,w1), (mb1,h1,w0), (mb1,h1,w1) - // Columns with order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), ... - INDArray exp2dv2 = Nd4j.create(outW * outH * miniBatch, inDepth * kH * kW); - // wOut0,hOut0,mb0 -> both depths, in order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), (d1,kh0,kw1), (d1,kh1,kw0), (d1,kh1,kw1) - exp2dv2.putRow(0, Nd4j.create(new double[] { 0, 1, 3, 4, 9, 10, 12, 13 })); - // wOut1,hOut0,mb0 - exp2dv2.putRow(1, Nd4j.create(new double[] { 1, 2, 4, 5, 10, 11, 13, 14 })); - // wOut0,hOut1,mb0 - exp2dv2.putRow(2, Nd4j.create(new double[] { 3, 4, 6, 7, 12, 13, 15, 16 })); - // wOut1,hOut1,mb0 - exp2dv2.putRow(3, Nd4j.create(new double[] { 4, 5, 7, 8, 13, 14, 16, 17 })); - // wOut0,hOut0,mb1 - exp2dv2.putRow(4, Nd4j.create(new double[] { 18, 19, 21, 22, 27, 28, 30, 31 })); - // wOut1,hOut0,mb1 - exp2dv2.putRow(5, Nd4j.create(new double[] { 19, 20, 22, 23, 28, 29, 31, 32 })); - // wOut0,hOut1,mb1 - exp2dv2.putRow(6, Nd4j.create(new double[] { 21, 22, 24, 25, 30, 31, 33, 34 })); - // wOut1,hOut1,mb1 - exp2dv2.putRow(7, Nd4j.create(new double[] { 22, 23, 25, 26, 31, 32, 34, 35 })); - assertEquals(exp2dv2, reshapedColBackprop); - } - - @Test - @DisplayName("Test Delta Reshaping") - void testDeltaReshaping() { - // As per above test: testing assumptions of cnn implementation... - // Delta: initially shape [miniBatch,dOut,outH,outW] - // permute to [dOut,miniB,outH,outW] - // then reshape to [dOut,miniB*outH*outW] - // Expect columns of delta2d to be like: (mb0,h0,w0), (mb0,h0,w1), (mb1,h0,w2), (mb0,h1,w0), ... (mb1,...), ..., (mb2,...) - int miniBatch = 3; - int depth = 2; - int outW = 3; - int outH = 3; - /* - ----- Input delta ----- - example 0: - channels 0 channels 1 - [ 0 1 2 [ 9 10 11 - 3 4 5 12 13 14 - 6 7 8] 15 16 17] - example 1: - [18 19 20 [27 28 29 - 21 22 23 30 31 32 - 24 25 26] 33 34 35] - example 2: - [36 37 38 [45 46 47 - 39 40 41 48 49 50 - 42 43 44] 51 52 53] - */ - INDArray deltaOrig = Nd4j.create(new int[] { miniBatch, depth, outH, outW }, 'c'); - deltaOrig.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 } })); - deltaOrig.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 9, 10, 11 }, { 12, 13, 14 }, { 15, 16, 17 } })); - deltaOrig.put(new INDArrayIndex[] { NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 18, 19, 20 }, { 21, 22, 23 }, { 24, 25, 26 } })); - deltaOrig.put(new INDArrayIndex[] { NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 27, 28, 29 }, { 30, 31, 32 }, { 33, 34, 35 } })); - deltaOrig.put(new INDArrayIndex[] { NDArrayIndex.point(2), NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 36, 37, 38 }, { 39, 40, 41 }, { 42, 43, 44 } })); - deltaOrig.put(new INDArrayIndex[] { NDArrayIndex.point(2), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 45, 46, 47 }, { 48, 49, 50 }, { 51, 52, 53 } })); - INDArray deltaPermute = deltaOrig.permute(1, 0, 2, 3).dup('c'); - INDArray delta2d = Shape.newShapeNoCopy(deltaPermute, new int[] { depth, miniBatch * outW * outH }, false); - INDArray exp = Nd4j.create(new double[][] { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 18, 19, 20, 21, 22, 23, 24, 25, 26, 36, 37, 38, 39, 40, 41, 42, 43, // depth0 - 44 }, { 9, 10, 11, 12, 13, 14, 15, 16, 17, 27, 28, 29, 30, 31, 32, 33, 34, 35, 45, 46, 47, 48, 49, 50, 51, 52, // depth1 - 53 } }).castTo(delta2d.dataType()); - assertEquals(exp, delta2d); - } - - @Test - @DisplayName("Test Weight Reshaping") - void testWeightReshaping() { - // Test assumptions of weight reshaping - // Weights: originally c order, shape [outDepth, inDepth, kH, kw] - // permute (3,2,1,0) - int depthOut = 2; - int depthIn = 3; - int kH = 2; - int kW = 2; - /* - ----- Weights ----- - - dOut 0 - - dIn 0 dIn 1 dIn 2 - [ 0 1 [ 4 5 [ 8 9 - 2 3] 6 7] 10 11] - - dOut 1 - - [12 13 [16 17 [20 21 - 14 15] 18 19] 22 23] - */ - INDArray weightOrig = Nd4j.create(new int[] { depthOut, depthIn, kH, kW }, 'c'); - weightOrig.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 0, 1 }, { 2, 3 } })); - weightOrig.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 4, 5 }, { 6, 7 } })); - weightOrig.put(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 8, 9 }, { 10, 11 } })); - weightOrig.put(new INDArrayIndex[] { NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 12, 13 }, { 14, 15 } })); - weightOrig.put(new INDArrayIndex[] { NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 16, 17 }, { 18, 19 } })); - weightOrig.put(new INDArrayIndex[] { NDArrayIndex.point(1), NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.all() }, Nd4j.create(new double[][] { { 20, 21 }, { 22, 23 } })); - INDArray weightPermute = weightOrig.permute(3, 2, 1, 0); - INDArray w2d = Shape.newShapeNoCopy(weightPermute, new int[] { depthIn * kH * kW, depthOut }, true); - assertNotNull(w2d); - // Expected order of weight rows, after reshaping: (kw0,kh0,din0), (kw1,kh0,din0), (kw0,kh1,din0), (kw1,kh1,din0), (kw0,kh0,din1), ... - INDArray wExp = Nd4j.create(new double[][] { { 0, 12 }, { 1, 13 }, { 2, 14 }, { 3, 15 }, { 4, 16 }, { 5, 17 }, { 6, 18 }, { 7, 19 }, { 8, 20 }, { 9, 21 }, { 10, 22 }, { 11, 23 } }).castTo(DataType.FLOAT); - assertEquals(wExp, w2d); - } - - // //////////////////////////////////////////////////////////////////////////////// - private static MultiLayerNetwork getCNNMLNConfig(boolean backprop, boolean pretrain) { - int outputNum = 10; - int seed = 123; - MultiLayerConfiguration.Builder conf = new NeuralNetConfiguration.Builder().seed(seed).optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list().layer(0, new ConvolutionLayer.Builder(new int[] { 10, 10 }).nOut(6).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] { 2, 2 }).stride(1, 1).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)); - MultiLayerNetwork model = new MultiLayerNetwork(conf.build()); - model.init(); - return model; - } - - @Test - @DisplayName("Test 1 d Input Type") - void test1dInputType() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().convolutionMode(ConvolutionMode.Same).list().layer(new Convolution1DLayer.Builder().nOut(3).kernelSize(2).activation(Activation.TANH).build()).layer(new Subsampling1DLayer.Builder().kernelSize(2).stride(2).build()).layer(new Upsampling1D.Builder().size(2).build()).layer(new RnnOutputLayer.Builder().nOut(7).activation(Activation.SOFTMAX).build()).setInputType(InputType.recurrent(10)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - List l = conf.getLayerActivationTypes(InputType.recurrent(10)); - assertEquals(InputType.recurrent(3, -1), l.get(0)); - assertEquals(InputType.recurrent(3, -1), l.get(1)); - assertEquals(InputType.recurrent(3, -1), l.get(2)); - assertEquals(InputType.recurrent(7, -1), l.get(3)); - List l2 = conf.getLayerActivationTypes(InputType.recurrent(10, 6)); - assertEquals(InputType.recurrent(3, 6), l2.get(0)); - assertEquals(InputType.recurrent(3, 3), l2.get(1)); - assertEquals(InputType.recurrent(3, 6), l2.get(2)); - assertEquals(InputType.recurrent(7, 6), l2.get(3)); - INDArray in = Nd4j.create(2, 10, 6); - INDArray out = net.output(in); - assertArrayEquals(new long[] { 2, 7, 6 }, out.shape()); - } - - @Test - @DisplayName("Test Deconv Bad Input") - void testDeconvBadInput() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new Deconvolution2D.Builder().nIn(5).nOut(3).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray badInput = Nd4j.create(DataType.FLOAT, 1, 10, 5, 5); - try { - net.output(badInput); - } catch (DL4JInvalidInputException e) { - String msg = e.getMessage(); - assertTrue( msg.contains("Deconvolution2D") && msg.contains("input") && msg.contains("channels"),msg); - } - } - - @Test - @DisplayName("Test Conv 1 d Causal Allowed") - void testConv1dCausalAllowed() { - new Convolution1DLayer.Builder().convolutionMode(ConvolutionMode.Causal).kernelSize(2).build(); - new Subsampling1DLayer.Builder().convolutionMode(ConvolutionMode.Causal).kernelSize(2).build(); - } - - @Test - @DisplayName("Test Conv 2 d No Causal Allowed") - void testConv2dNoCausalAllowed() { - try { - new ConvolutionLayer.Builder().convolutionMode(ConvolutionMode.Causal).build(); - fail("Expected exception"); - } catch (Throwable t) { - String m = t.getMessage().toLowerCase(); - assertTrue(m.contains("causal") && m.contains("1d"),m); - } - try { - new Deconvolution2D.Builder().convolutionMode(ConvolutionMode.Causal).build(); - fail("Expected exception"); - } catch (Throwable t) { - String m = t.getMessage().toLowerCase(); - assertTrue(m.contains("causal") && m.contains("1d"),m); - } - try { - new DepthwiseConvolution2D.Builder().convolutionMode(ConvolutionMode.Causal).build(); - fail("Expected exception"); - } catch (Throwable t) { - String m = t.getMessage().toLowerCase(); - assertTrue( m.contains("causal") && m.contains("1d"),m); - } - try { - new SeparableConvolution2D.Builder().convolutionMode(ConvolutionMode.Causal).build(); - fail("Expected exception"); - } catch (Throwable t) { - String m = t.getMessage().toLowerCase(); - assertTrue(m.contains("causal") && m.contains("1d"),m); - } - try { - new SubsamplingLayer.Builder().convolutionMode(ConvolutionMode.Causal).build(); - fail("Expected exception"); - } catch (Throwable t) { - String m = t.getMessage().toLowerCase(); - assertTrue( m.contains("causal") && m.contains("1d"),m); - } - } - - @Test - @DisplayName("Test Conv 3 d No Causal Allowed") - void testConv3dNoCausalAllowed() { - try { - new Convolution3D.Builder().convolutionMode(ConvolutionMode.Causal).build(); - fail("Expected exception"); - } catch (Throwable t) { - String m = t.getMessage().toLowerCase(); - assertTrue(m.contains("causal") && m.contains("1d"),m); - } - try { - new Subsampling3DLayer.Builder().convolutionMode(ConvolutionMode.Causal).build(); - fail("Expected exception"); - } catch (Throwable t) { - String m = t.getMessage().toLowerCase(); - assertTrue(m.contains("causal") && m.contains("1d"),m); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/LocallyConnectedLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/LocallyConnectedLayerTest.java deleted file mode 100644 index db73245ff..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/LocallyConnectedLayerTest.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.convolution; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataBuffer; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.buffer.util.DataTypeUtil; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.MultiDataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.Arrays; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * @author Max Pumperla - */ -@DisplayName("Locally Connected Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class LocallyConnectedLayerTest extends BaseDL4JTest { - - @BeforeEach - void before() { - DataTypeUtil.setDTypeForContext(DataType.DOUBLE); - Nd4j.factory().setDType(DataType.DOUBLE); - Nd4j.EPS_THRESHOLD = 1e-4; - } - - @Test - @DisplayName("Test 2 d Forward") - void test2dForward() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4).updater(new Nesterovs(0.9)).dropOut(0.5).list().layer(new LocallyConnected2D.Builder().kernelSize(8, 8).nIn(3).stride(4, 4).nOut(16).dropOut(0.5).convolutionMode(ConvolutionMode.Strict).setInputSize(28, 28).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(// output layer - new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS).nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(28, 28, 3)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - INDArray input = Nd4j.ones(10, 3, 28, 28); - INDArray output = network.output(input, false); - assertArrayEquals(new long[] { 10, 10 }, output.shape()); - } - - @Test - @DisplayName("Test 1 d Forward") - void test1dForward() { - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4).updater(new Nesterovs(0.9)).dropOut(0.5).list().layer(new LocallyConnected1D.Builder().kernelSize(4).nIn(3).stride(1).nOut(16).dropOut(0.5).convolutionMode(ConvolutionMode.Strict).setInputSize(28).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(// output layer - new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS).nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.recurrent(3, 8)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - INDArray input = Nd4j.ones(10, 3, 8); - INDArray output = network.output(input, false); - ; - for (int i = 0; i < 100; i++) { - // TODO: this falls flat for 1000 iterations on my machine - output = network.output(input, false); - } - assertArrayEquals(new long[] { (8 - 4 + 1) * 10, 10 }, output.shape()); - network.fit(input, output); - } - - @Test - @DisplayName("Test Locally Connected") - void testLocallyConnected() { - for (DataType globalDtype : new DataType[] { DataType.DOUBLE, DataType.FLOAT, DataType.HALF }) { - Nd4j.setDefaultDataTypes(globalDtype, globalDtype); - for (DataType networkDtype : new DataType[] { DataType.DOUBLE, DataType.FLOAT, DataType.HALF }) { - assertEquals(globalDtype, Nd4j.dataType()); - assertEquals(globalDtype, Nd4j.defaultFloatingPointType()); - for (int test = 0; test < 2; test++) { - String msg = "Global dtype: " + globalDtype + ", network dtype: " + networkDtype + ", test=" + test; - ComputationGraphConfiguration.GraphBuilder b = new NeuralNetConfiguration.Builder().dataType(networkDtype).seed(123).updater(new NoOp()).weightInit(WeightInit.XAVIER).convolutionMode(ConvolutionMode.Same).graphBuilder(); - INDArray[] in; - INDArray label; - switch(test) { - case 0: - b.addInputs("in").addLayer("1", new LSTM.Builder().nOut(5).build(), "in").addLayer("2", new LocallyConnected1D.Builder().kernelSize(2).nOut(4).build(), "1").addLayer("out", new RnnOutputLayer.Builder().nOut(10).build(), "2").setOutputs("out").setInputTypes(InputType.recurrent(5, 4)); - in = new INDArray[] { Nd4j.rand(networkDtype, 2, 5, 4) }; - label = TestUtils.randomOneHotTimeSeries(2, 10, 4).castTo(networkDtype); - break; - case 1: - b.addInputs("in").addLayer("1", new ConvolutionLayer.Builder().kernelSize(2, 2).nOut(5).convolutionMode(ConvolutionMode.Same).build(), "in").addLayer("2", new LocallyConnected2D.Builder().kernelSize(2, 2).nOut(5).build(), "1").addLayer("out", new OutputLayer.Builder().nOut(10).build(), "2").setOutputs("out").setInputTypes(InputType.convolutional(8, 8, 1)); - in = new INDArray[] { Nd4j.rand(networkDtype, 2, 1, 8, 8) }; - label = TestUtils.randomOneHot(2, 10).castTo(networkDtype); - break; - default: - throw new RuntimeException(); - } - ComputationGraph net = new ComputationGraph(b.build()); - net.init(); - INDArray out = net.outputSingle(in); - assertEquals(networkDtype, out.dataType(),msg); - Map ff = net.feedForward(in, false); - for (Map.Entry e : ff.entrySet()) { - if (e.getKey().equals("in")) - continue; - String s = msg + " - layer: " + e.getKey(); - assertEquals( networkDtype, e.getValue().dataType(),s); - } - net.setInputs(in); - net.setLabels(label); - net.computeGradientAndScore(); - net.fit(new MultiDataSet(in, new INDArray[] { label })); - } - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java deleted file mode 100644 index 25da3b545..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.convolution; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.gradient.DefaultGradient; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.primitives.Pair; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import java.util.Arrays; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * @author Adam Gibson - */ -@DisplayName("Subsampling Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class SubsamplingLayerTest extends BaseDL4JTest { - - private int nExamples = 1; - - // channels & nOut - private int depth = 20; - - private int nChannelsIn = 1; - - private int inputWidth = 28; - - private int inputHeight = 28; - - private int[] kernelSize = new int[] { 2, 2 }; - - private int[] stride = new int[] { 2, 2 }; - - int featureMapWidth = (inputWidth - kernelSize[0]) / stride[0] + 1; - - int featureMapHeight = (inputHeight - kernelSize[1]) / stride[0] + 1; - - private INDArray epsilon = Nd4j.ones(nExamples, depth, featureMapHeight, featureMapWidth); - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Test - @DisplayName("Test Sub Sample Max Activate") - void testSubSampleMaxActivate() throws Exception { - INDArray containedExpectedOut = Nd4j.create(new double[] { 5., 7., 6., 8., 4., 7., 5., 9. }, new long[] { 1, 2, 2, 2 }).castTo(Nd4j.defaultFloatingPointType()); - INDArray containedInput = getContainedData(); - INDArray input = getData(); - Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX); - INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces()); - assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape())); - assertEquals(containedExpectedOut, containedOutput); - INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertTrue(Arrays.equals(new long[] { nExamples, nChannelsIn, featureMapWidth, featureMapHeight }, output.shape())); - // channels retained - assertEquals(nChannelsIn, output.size(1), 1e-4); - } - - @Test - @DisplayName("Test Sub Sample Mean Activate") - void testSubSampleMeanActivate() throws Exception { - INDArray containedExpectedOut = Nd4j.create(new double[] { 2., 4., 3., 5., 3.5, 6.5, 4.5, 8.5 }, new int[] { 1, 2, 2, 2 }).castTo(Nd4j.defaultFloatingPointType()); - INDArray containedInput = getContainedData(); - INDArray input = getData(); - Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG); - INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces()); - assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape())); - assertEquals(containedExpectedOut, containedOutput); - INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertTrue(Arrays.equals(new long[] { nExamples, nChannelsIn, featureMapWidth, featureMapHeight }, output.shape())); - // channels retained - assertEquals(nChannelsIn, output.size(1), 1e-4); - } - - // //////////////////////////////////////////////////////////////////////////////// - @Test - @DisplayName("Test Sub Sample Layer Max Backprop") - void testSubSampleLayerMaxBackprop() throws Exception { - INDArray expectedContainedEpsilonInput = Nd4j.create(new double[] { 1., 1., 1., 1., 1., 1., 1., 1. }, new int[] { 1, 2, 2, 2 }).castTo(Nd4j.defaultFloatingPointType()); - INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] { 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0. }, new int[] { 1, 2, 4, 4 }).castTo(Nd4j.defaultFloatingPointType()); - INDArray input = getContainedData(); - Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX); - layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - Pair containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond()); - assertEquals(null, containedOutput.getFirst().getGradientFor("W")); - assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length); - INDArray input2 = getData(); - layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces()); - long depth = input2.size(1); - epsilon = Nd4j.ones(5, depth, featureMapHeight, featureMapWidth); - Pair out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(input.shape().length, out.getSecond().shape().length); - // channels retained - assertEquals(depth, out.getSecond().size(1)); - } - - @Test - @DisplayName("Test Sub Sample Layer Avg Backprop") - void testSubSampleLayerAvgBackprop() throws Exception { - INDArray expectedContainedEpsilonInput = Nd4j.create(new double[] { 1., 2., 3., 4., 5., 6., 7., 8. }, new int[] { 1, 2, 2, 2 }).castTo(Nd4j.defaultFloatingPointType()); - INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] { 0.25, 0.25, 0.5, 0.5, 0.25, 0.25, 0.5, 0.5, 0.75, 0.75, 1., 1., 0.75, 0.75, 1., 1., 1.25, 1.25, 1.5, 1.5, 1.25, 1.25, 1.5, 1.5, 1.75, 1.75, 2., 2., 1.75, 1.75, 2., 2. }, new int[] { 1, 2, 4, 4 }).castTo(Nd4j.defaultFloatingPointType()); - INDArray input = getContainedData(); - Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG); - layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - Pair containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond()); - assertEquals(null, containedOutput.getFirst().getGradientFor("W")); - assertArrayEquals(expectedContainedEpsilonResult.shape(), containedOutput.getSecond().shape()); - } - - @Test - @DisplayName("Test Sub Sample Layer Sum Backprop") - void testSubSampleLayerSumBackprop() { - assertThrows(UnsupportedOperationException.class, () -> { - Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.SUM); - INDArray input = getData(); - layer.setInput(input, LayerWorkspaceMgr.noWorkspaces()); - layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); - }); - } - - // //////////////////////////////////////////////////////////////////////////////// - private Layer getSubsamplingLayer(SubsamplingLayer.PoolingType pooling) { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123).layer(new SubsamplingLayer.Builder(pooling, new int[] { 2, 2 }).build()).build(); - return conf.getLayer().instantiate(conf, null, 0, null, true, Nd4j.defaultFloatingPointType()); - } - - public INDArray getData() throws Exception { - DataSetIterator data = new MnistDataSetIterator(5, 5); - DataSet mnist = data.next(); - nExamples = mnist.numExamples(); - return mnist.getFeatures().reshape(nExamples, nChannelsIn, inputWidth, inputHeight); - } - - public INDArray getContainedData() { - INDArray ret = Nd4j.create(new double[] { 1., 1., 3., 7., 5., 1., 3., 3., 2., 2., 8., 4., 2., 6., 4., 4., 3., 3., 6., 7., 4., 4., 6., 7., 5., 5., 9., 8., 4., 4., 9., 8. }, new int[] { 1, 2, 4, 4 }).castTo(Nd4j.defaultFloatingPointType()); - return ret; - } - - private Gradient createPrevGradient() { - Gradient gradient = new DefaultGradient(); - INDArray pseudoGradients = Nd4j.ones(nExamples, nChannelsIn, inputHeight, inputWidth); - gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, pseudoGradients); - gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, pseudoGradients); - return gradient; - } - - // //////////////////////////////////////////////////////////////////////////////// - @Test - @DisplayName("Test Sub Too Large Kernel") - void testSubTooLargeKernel() { - assertThrows(Exception.class, () -> { - int imageHeight = 20; - int imageWidth = 23; - int nChannels = 1; - int classes = 2; - int numSamples = 200; - int kernelHeight = 3; - int kernelWidth = 3; - DataSet trainInput; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(kernelHeight, kernelWidth).stride(1, 1).nOut(2).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new SubsamplingLayer.Builder().poolingType(SubsamplingLayer.PoolingType.MAX).kernelSize(imageHeight - kernelHeight + 2, // imageHeight-kernelHeight+1 is ok: full height - 1).stride(1, 1).build()).layer(2, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(imageHeight, imageWidth, nChannels)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels); - INDArray emptyLables = Nd4j.zeros(numSamples, classes); - trainInput = new DataSet(emptyFeatures, emptyLables); - model.fit(trainInput); - }); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayer.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayer.java deleted file mode 100644 index cd75501ae..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/custom/testclasses/CustomLayer.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.layers.custom.testclasses; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.deeplearning4j.nn.api.ParamInitializer; -import org.deeplearning4j.nn.conf.InputPreProcessor; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; -import org.deeplearning4j.nn.conf.memory.LayerMemoryReport; -import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.deeplearning4j.optimize.api.TrainingListener; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -import java.util.Collection; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class CustomLayer extends FeedForwardLayer { - - private final double someCustomParameter; - - public CustomLayer(@JsonProperty("someCustomParameter") double someCustomParameter) { - this.someCustomParameter = someCustomParameter; - this.nIn = 10; - this.nOut = 10; - } - - @Override - public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf, - Collection trainingListeners, int layerIndex, INDArray layerParamsView, - boolean initializeParams, DataType networkDataType) { - CustomLayerImpl ret = new CustomLayerImpl(conf, networkDataType); - ret.setListeners(trainingListeners); - ret.setIndex(layerIndex); - ret.setParamsViewArray(layerParamsView); - Map paramTable = initializer().init(conf, layerParamsView, initializeParams); - ret.setParamTable(paramTable); - ret.setConf(conf); - return ret; - } - - @Override - public ParamInitializer initializer() { - return DefaultParamInitializer.getInstance(); - } - - @Override - public InputType getOutputType(int layerIndex, InputType inputType) { - return InputType.feedForward(10); - } - - @Override - public void setNIn(InputType inputType, boolean override) { - //No op - } - - @Override - public InputPreProcessor getPreProcessorForInputType(InputType inputType) { - return null; - } - - @Override - public LayerMemoryReport getMemoryReport(InputType inputType) { - throw new UnsupportedOperationException(); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayerTest.java deleted file mode 100644 index b546af346..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/feedforward/embedding/EmbeddingLayerTest.java +++ /dev/null @@ -1,555 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.feedforward.embedding; - -import lombok.EqualsAndHashCode; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.RNNFormat; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.EmbeddingLayer; -import org.deeplearning4j.nn.conf.layers.EmbeddingSequenceLayer; -import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.weights.embeddings.EmbeddingInitializer; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.activations.impl.ActivationIdentity; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Random; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Embedding Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class EmbeddingLayerTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Embedding Layer Config") - void testEmbeddingLayerConfig() { - for (boolean hasBias : new boolean[] { true, false }) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new EmbeddingLayer.Builder().hasBias(hasBias).nIn(10).nOut(5).build()).layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - Layer l0 = net.getLayer(0); - assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingLayer.class, l0.getClass()); - assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn()); - assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut()); - INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY); - INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY); - assertArrayEquals(new long[] { 10, 5 }, weights.shape()); - if (hasBias) { - assertArrayEquals(new long[] { 1, 5 }, bias.shape()); - } - } - } - - @Test - @DisplayName("Test Embedding Sequence Layer Config") - void testEmbeddingSequenceLayerConfig() { - int inputLength = 6; - int nIn = 10; - int embeddingDim = 5; - int nout = 4; - for (boolean hasBias : new boolean[] { true, false }) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(new EmbeddingSequenceLayer.Builder().hasBias(hasBias).inputLength(inputLength).nIn(nIn).nOut(embeddingDim).build()).layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nout).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - Layer l0 = net.getLayer(0); - assertEquals(org.deeplearning4j.nn.layers.feedforward.embedding.EmbeddingSequenceLayer.class, l0.getClass()); - assertEquals(10, ((FeedForwardLayer) l0.conf().getLayer()).getNIn()); - assertEquals(5, ((FeedForwardLayer) l0.conf().getLayer()).getNOut()); - INDArray weights = l0.getParam(DefaultParamInitializer.WEIGHT_KEY); - INDArray bias = l0.getParam(DefaultParamInitializer.BIAS_KEY); - assertArrayEquals(new long[] { 10, 5 }, weights.shape()); - if (hasBias) { - assertArrayEquals(new long[] { 1, 5 }, bias.shape()); - } - } - } - - @Test - @DisplayName("Test Embedding Longer Sequences Forward Pass") - void testEmbeddingLongerSequencesForwardPass() { - int nClassesIn = 10; - int inputLength = 6; - int embeddingDim = 5; - int nOut = 4; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(new EmbeddingSequenceLayer.Builder().inputLength(inputLength).hasBias(true).nIn(nClassesIn).nOut(embeddingDim).build()).layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - int batchSize = 3; - INDArray inEmbedding = Nd4j.create(batchSize, inputLength); - Random r = new Random(12345); - for (int i = 0; i < batchSize; i++) { - int classIdx = r.nextInt(nClassesIn); - inEmbedding.putScalar(i, classIdx); - } - INDArray output = net.output(inEmbedding); - assertArrayEquals(new long[] { batchSize, nOut, inputLength }, output.shape()); - } - - @Test - @DisplayName("Test Embedding Single Sequence Forward Pass") - void testEmbeddingSingleSequenceForwardPass() { - int nClassesIn = 10; - int embeddingDim = 5; - int nOut = 4; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(new EmbeddingSequenceLayer.Builder().inputLength(1).hasBias(true).nIn(nClassesIn).nOut(embeddingDim).build()).layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build()).layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build()).inputPreProcessor(0, new RnnToFeedForwardPreProcessor()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net.init(); - net2.init(); - net2.setParams(net.params().dup()); - int batchSize = 3; - INDArray inEmbedding = Nd4j.create(batchSize, 1); - INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, 1); - Random r = new Random(12345); - for (int i = 0; i < batchSize; i++) { - int classIdx = r.nextInt(nClassesIn); - inEmbedding.putScalar(i, classIdx); - inOneHot.putScalar(new int[] { i, classIdx, 0 }, 1.0); - } - List activationsDense = net2.feedForward(inOneHot, false); - List activationEmbedding = net.feedForward(inEmbedding, false); - INDArray actD1 = activationsDense.get(1); - INDArray actE1 = activationEmbedding.get(1).reshape(batchSize, embeddingDim); - assertEquals(actD1, actE1); - INDArray actD2 = activationsDense.get(2); - INDArray actE2 = activationEmbedding.get(2).reshape(batchSize, nOut); - assertEquals(actD2, actE2); - } - - @Test - @DisplayName("Test Embedding Forward Pass") - void testEmbeddingForwardPass() { - // With the same parameters, embedding layer should have same activations as the equivalent one-hot representation - // input with a DenseLayer - int nClassesIn = 10; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build()).layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build()).layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net.init(); - net2.init(); - net2.setParams(net.params().dup()); - int batchSize = 3; - INDArray inEmbedding = Nd4j.create(batchSize, 1); - INDArray inOneHot = Nd4j.create(batchSize, nClassesIn); - Random r = new Random(12345); - for (int i = 0; i < batchSize; i++) { - int classIdx = r.nextInt(nClassesIn); - inEmbedding.putScalar(i, classIdx); - inOneHot.putScalar(new int[] { i, classIdx }, 1.0); - } - List activationsEmbedding = net.feedForward(inEmbedding, false); - List activationsDense = net2.feedForward(inOneHot, false); - for (int i = 1; i < 3; i++) { - INDArray actE = activationsEmbedding.get(i); - INDArray actD = activationsDense.get(i); - assertEquals(actE, actD); - } - } - - @Test - @DisplayName("Test Embedding Backward Pass") - void testEmbeddingBackwardPass() { - // With the same parameters, embedding layer should have same activations as the equivalent one-hot representation - // input with a DenseLayer - int nClassesIn = 10; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4).activation(Activation.SOFTMAX).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(4).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net.init(); - net2.init(); - net2.setParams(net.params().dup()); - int batchSize = 3; - INDArray inEmbedding = Nd4j.create(batchSize, 1); - INDArray inOneHot = Nd4j.create(batchSize, nClassesIn); - INDArray outLabels = Nd4j.create(batchSize, 4); - Random r = new Random(12345); - for (int i = 0; i < batchSize; i++) { - int classIdx = r.nextInt(nClassesIn); - inEmbedding.putScalar(i, classIdx); - inOneHot.putScalar(new int[] { i, classIdx }, 1.0); - int labelIdx = r.nextInt(4); - outLabels.putScalar(new int[] { i, labelIdx }, 1.0); - } - net.setInput(inEmbedding); - net2.setInput(inOneHot); - net.setLabels(outLabels); - net2.setLabels(outLabels); - net.computeGradientAndScore(); - net2.computeGradientAndScore(); - assertEquals(net2.score(), net.score(), 1e-6); - Map gradient = net.gradient().gradientForVariable(); - Map gradient2 = net2.gradient().gradientForVariable(); - assertEquals(gradient.size(), gradient2.size()); - for (String s : gradient.keySet()) { - assertEquals(gradient2.get(s), gradient.get(s)); - } - } - - @Test - @DisplayName("Test Embedding Sequence Backward Pass") - void testEmbeddingSequenceBackwardPass() { - int nClassesIn = 10; - int embeddingDim = 5; - int nOut = 4; - int inputLength = 1; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(new EmbeddingSequenceLayer.Builder().inputLength(inputLength).hasBias(true).nIn(nClassesIn).nOut(embeddingDim).build()).layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build()).setInputType(InputType.recurrent(nClassesIn, inputLength, RNNFormat.NCW)).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).list().layer(new DenseLayer.Builder().nIn(nClassesIn).nOut(embeddingDim).activation(Activation.IDENTITY).build()).layer(new RnnOutputLayer.Builder().nIn(embeddingDim).nOut(nOut).activation(Activation.SOFTMAX).build()).setInputType(InputType.recurrent(nClassesIn, inputLength, RNNFormat.NCW)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net.init(); - net2.init(); - net2.setParams(net.params().dup()); - int batchSize = 3; - INDArray inEmbedding = Nd4j.create(batchSize, 1); - INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, 1); - INDArray outLabels = Nd4j.create(batchSize, 4, 1); - Random r = new Random(1337); - for (int i = 0; i < batchSize; i++) { - int classIdx = r.nextInt(nClassesIn); - inEmbedding.putScalar(i, classIdx); - inOneHot.putScalar(new int[] { i, classIdx, 0 }, 1.0); - int labelIdx = r.nextInt(4); - outLabels.putScalar(new int[] { i, labelIdx, 0 }, 1.0); - } - net.setInput(inEmbedding); - net2.setInput(inOneHot); - net.setLabels(outLabels); - net2.setLabels(outLabels); - net.computeGradientAndScore(); - net2.computeGradientAndScore(); - // System.out.println(net.score() + "\t" + net2.score()); - assertEquals(net2.score(), net.score(), 1e-6); - Map gradient = net.gradient().gradientForVariable(); - Map gradient2 = net2.gradient().gradientForVariable(); - assertEquals(gradient.size(), gradient2.size()); - for (String s : gradient.keySet()) { - assertEquals(gradient2.get(s), gradient.get(s)); - } - } - - @Test - @DisplayName("Test Embedding Layer RNN") - void testEmbeddingLayerRNN() { - int nClassesIn = 10; - int batchSize = 3; - int timeSeriesLength = 8; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().activation(Activation.TANH).dataType(DataType.DOUBLE).list().layer(0, new EmbeddingLayer.Builder().hasBias(true).nIn(nClassesIn).nOut(5).build()).layer(1, new LSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build()).layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4).activation(Activation.SOFTMAX).build()).inputPreProcessor(0, new RnnToFeedForwardPreProcessor()).inputPreProcessor(1, new FeedForwardToRnnPreProcessor()).setInputType(InputType.recurrent(nClassesIn, timeSeriesLength, RNNFormat.NCW)).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).dataType(DataType.DOUBLE).list().layer(0, new DenseLayer.Builder().nIn(nClassesIn).nOut(5).activation(Activation.IDENTITY).build()).layer(1, new LSTM.Builder().nIn(5).nOut(7).activation(Activation.SOFTSIGN).build()).layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(7).nOut(4).activation(Activation.SOFTMAX).build()).inputPreProcessor(0, new RnnToFeedForwardPreProcessor()).inputPreProcessor(1, new FeedForwardToRnnPreProcessor()).setInputType(InputType.recurrent(nClassesIn, timeSeriesLength, RNNFormat.NCW)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net.init(); - net2.init(); - net2.setParams(net.params().dup()); - ; - INDArray inEmbedding = Nd4j.create(batchSize, 1, timeSeriesLength); - INDArray inOneHot = Nd4j.create(batchSize, nClassesIn, timeSeriesLength); - INDArray outLabels = Nd4j.create(batchSize, 4, timeSeriesLength); - Random r = new Random(12345); - for (int i = 0; i < batchSize; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - int classIdx = r.nextInt(nClassesIn); - inEmbedding.putScalar(new int[] { i, 0, j }, classIdx); - inOneHot.putScalar(new int[] { i, classIdx, j }, 1.0); - int labelIdx = r.nextInt(4); - outLabels.putScalar(new int[] { i, labelIdx, j }, 1.0); - } - } - net.setInput(inEmbedding); - net2.setInput(inOneHot); - net.setLabels(outLabels); - net2.setLabels(outLabels); - net.computeGradientAndScore(); - net2.computeGradientAndScore(); - // System.out.println(net.score() + "\t" + net2.score()); - assertEquals(net2.score(), net.score(), 1e-5); - Map gradient = net.gradient().gradientForVariable(); - Map gradient2 = net2.gradient().gradientForVariable(); - assertEquals(gradient.size(), gradient2.size()); - for (String s : gradient.keySet()) { - assertEquals(gradient2.get(s), gradient.get(s)); - } - } - - @Test - @DisplayName("Test Embedding Layer With Masking") - void testEmbeddingLayerWithMasking() { - // Idea: have masking on the input with an embedding and dense layers on input - // Ensure that the parameter gradients for the inputs don't depend on the inputs when inputs are masked - int[] miniBatchSizes = { 1, 2, 5 }; - int nIn = 2; - Random r = new Random(12345); - int numInputClasses = 10; - int timeSeriesLength = 5; - for (DataType maskDtype : new DataType[] { DataType.FLOAT, DataType.DOUBLE, DataType.INT }) { - for (int nExamples : miniBatchSizes) { - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).seed(12345).list().layer(0, new EmbeddingLayer.Builder().hasBias(true).activation(Activation.TANH).nIn(numInputClasses).nOut(5).build()).layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build()).layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build()).layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3).nOut(4).build()).inputPreProcessor(0, new RnnToFeedForwardPreProcessor()).inputPreProcessor(2, new FeedForwardToRnnPreProcessor()).setInputType(InputType.recurrent(numInputClasses, timeSeriesLength, RNNFormat.NCW)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).seed(12345).list().layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(numInputClasses).nOut(5).build()).layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build()).layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build()).layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3).nOut(4).build()).inputPreProcessor(0, new RnnToFeedForwardPreProcessor()).inputPreProcessor(2, new FeedForwardToRnnPreProcessor()).setInputType(InputType.recurrent(numInputClasses, timeSeriesLength, RNNFormat.NCW)).build(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - net2.setParams(net.params().dup()); - INDArray inEmbedding = Nd4j.zeros(nExamples, 1, timeSeriesLength); - INDArray inDense = Nd4j.zeros(nExamples, numInputClasses, timeSeriesLength); - INDArray labels = Nd4j.zeros(nExamples, 4, timeSeriesLength); - for (int i = 0; i < nExamples; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - int inIdx = r.nextInt(numInputClasses); - inEmbedding.putScalar(new int[] { i, 0, j }, inIdx); - inDense.putScalar(new int[] { i, inIdx, j }, 1.0); - int outIdx = r.nextInt(4); - labels.putScalar(new int[] { i, outIdx, j }, 1.0); - } - } - INDArray inputMask = Nd4j.zeros(maskDtype, nExamples, timeSeriesLength); - for (int i = 0; i < nExamples; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - inputMask.putScalar(new int[] { i, j }, (r.nextBoolean() ? 1.0 : 0.0)); - } - } - net.setLayerMaskArrays(inputMask, null); - net2.setLayerMaskArrays(inputMask, null); - List actEmbedding = net.feedForward(inEmbedding, false); - List actDense = net2.feedForward(inDense, false); - for (int i = 1; i < actEmbedding.size(); i++) { - assertEquals(actDense.get(i), actEmbedding.get(i)); - } - net.setLabels(labels); - net2.setLabels(labels); - net.computeGradientAndScore(); - net2.computeGradientAndScore(); - // System.out.println(net.score() + "\t" + net2.score()); - assertEquals(net2.score(), net.score(), 1e-5); - Map gradients = net.gradient().gradientForVariable(); - Map gradients2 = net2.gradient().gradientForVariable(); - assertEquals(gradients.keySet(), gradients2.keySet()); - for (String s : gradients.keySet()) { - assertEquals(gradients2.get(s), gradients.get(s)); - } - } - } - } - - @Test - @DisplayName("Test W 2 V Inits") - void testW2VInits() { - Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT); - for (int i = 0; i < 2; i++) { - INDArray vectors = Nd4j.linspace(1, 15, 15, DataType.FLOAT).reshape(5, 3); - EmbeddingLayer el; - if (i == 0) { - el = new EmbeddingLayer.Builder().weightInit(vectors).build(); - } else { - el = new EmbeddingLayer.Builder().weightInit(new WordVectorsMockup()).build(); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(el).layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(3).nOut(3).build()).layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3).nOut(4).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray w = net.getParam("0_W"); - assertEquals(vectors, w); - TestUtils.testModelSerialization(net); - // Test same thing for embedding sequence layer: - EmbeddingSequenceLayer esl; - if (i == 0) { - esl = new EmbeddingSequenceLayer.Builder().weightInit(vectors).build(); - } else { - esl = new EmbeddingSequenceLayer.Builder().weightInit(new WordVectorsMockup()).build(); - } - conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(esl).layer(new GlobalPoolingLayer()).layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(3).nOut(3).build()).layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3).nOut(4).build()).build(); - net = new MultiLayerNetwork(conf); - net.init(); - w = net.getParam("0_W"); - assertEquals(vectors, w); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Embedding Sequence Layer With Masking") - void testEmbeddingSequenceLayerWithMasking() { - // Idea: have masking on the input with an embedding and dense layers on input - // Ensure that the parameter gradients for the inputs don't depend on the inputs when inputs are masked - int[] miniBatchSizes = { 1, 3 }; - int nIn = 2; - Random r = new Random(12345); - int numInputClasses = 10; - int timeSeriesLength = 5; - for (DataType maskDtype : new DataType[] { DataType.FLOAT, DataType.DOUBLE, DataType.INT }) { - for (DataType inLabelDtype : new DataType[] { DataType.FLOAT, DataType.DOUBLE, DataType.INT }) { - for (int inputRank : new int[] { 2, 3 }) { - for (int nExamples : miniBatchSizes) { - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).seed(12345).list().layer(0, new EmbeddingSequenceLayer.Builder().hasBias(true).activation(Activation.TANH).nIn(numInputClasses).nOut(5).build()).layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build()).layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).build()).layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3).nOut(4).build()).setInputType(InputType.recurrent(numInputClasses, timeSeriesLength, RNNFormat.NCW)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).seed(12345).list().layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(numInputClasses).nOut(5).build()).layer(1, new DenseLayer.Builder().activation(Activation.TANH).nIn(5).nOut(4).build()).layer(2, new LSTM.Builder().activation(Activation.TANH).nIn(4).nOut(3).dataFormat(RNNFormat.NCW).build()).layer(3, new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3).nOut(4).build()).setInputType(InputType.recurrent(numInputClasses, 1, RNNFormat.NCW)).build(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - net2.setParams(net.params().dup()); - INDArray inEmbedding = Nd4j.zeros(inLabelDtype, inputRank == 2 ? new long[] { nExamples, timeSeriesLength } : new long[] { nExamples, 1, timeSeriesLength }); - INDArray inDense = Nd4j.zeros(inLabelDtype, nExamples, numInputClasses, timeSeriesLength); - INDArray labels = Nd4j.zeros(inLabelDtype, nExamples, 4, timeSeriesLength); - for (int i = 0; i < nExamples; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - int inIdx = r.nextInt(numInputClasses); - inEmbedding.putScalar(inputRank == 2 ? new int[] { i, j } : new int[] { i, 0, j }, inIdx); - inDense.putScalar(new int[] { i, inIdx, j }, 1.0); - int outIdx = r.nextInt(4); - labels.putScalar(new int[] { i, outIdx, j }, 1.0); - } - } - INDArray inputMask = Nd4j.zeros(maskDtype, nExamples, timeSeriesLength); - for (int i = 0; i < nExamples; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - inputMask.putScalar(new int[] { i, j }, (r.nextBoolean() ? 1.0 : 0.0)); - } - } - net.setLayerMaskArrays(inputMask, null); - net2.setLayerMaskArrays(inputMask, null); - List actEmbedding = net.feedForward(inEmbedding, false); - List actDense = net2.feedForward(inDense, false); - for (int i = 2; i < actEmbedding.size(); i++) { - // Start from layer 2: EmbeddingSequence is 3d, first dense is 2d (before reshape) - assertEquals(actDense.get(i), actEmbedding.get(i)); - } - net.setLabels(labels); - net2.setLabels(labels); - net.computeGradientAndScore(); - net2.computeGradientAndScore(); - assertEquals(net2.score(), net.score(), 1e-5); - Map gradients = net.gradient().gradientForVariable(); - Map gradients2 = net2.gradient().gradientForVariable(); - assertEquals(gradients.keySet(), gradients2.keySet()); - for (String s : gradients.keySet()) { - assertEquals(gradients2.get(s), gradients.get(s)); - } - } - } - } - } - } - - @EqualsAndHashCode - @DisplayName("Word Vectors Mockup") - private static class WordVectorsMockup implements EmbeddingInitializer { - - @Override - public void loadWeightsInto(INDArray array) { - INDArray vectors = Nd4j.linspace(1, 15, 15, DataType.FLOAT).reshape(5, 3); - array.assign(vectors); - } - - @Override - public long vocabSize() { - return 5; - } - - @Override - public int vectorSize() { - return 3; - } - - @Override - public boolean jsonSerializable() { - return true; - } - } - - @Test - @DisplayName("Test Embedding Default Activation") - void testEmbeddingDefaultActivation() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new EmbeddingLayer.Builder().nIn(10).nOut(10).build()).layer(new EmbeddingSequenceLayer.Builder().nIn(10).nOut(10).build()).build(); - EmbeddingLayer l = (EmbeddingLayer) conf.getConf(0).getLayer(); - assertEquals(new ActivationIdentity(), l.getActivationFn()); - EmbeddingSequenceLayer l2 = (EmbeddingSequenceLayer) conf.getConf(1).getLayer(); - assertEquals(new ActivationIdentity(), l2.getActivationFn()); - } - - @Test - @DisplayName("Test Embedding Weight Init") - void testEmbeddingWeightInit() { - // https://github.com/eclipse/deeplearning4j/issues/8663 - // The embedding layer weight initialization should be independent of the vocabulary size (nIn setting) - for (WeightInit wi : new WeightInit[] { WeightInit.XAVIER, WeightInit.RELU, WeightInit.XAVIER_UNIFORM, WeightInit.LECUN_NORMAL }) { - for (boolean seq : new boolean[] { false, true }) { - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(seq ? new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100).nOut(100).build() : new EmbeddingLayer.Builder().weightInit(wi).nIn(100).nOut(100).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(seq ? new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100).nOut(100).build() : new EmbeddingLayer.Builder().weightInit(wi).nIn(100).nOut(100).build()).build(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder().seed(12345).list().layer(seq ? new EmbeddingSequenceLayer.Builder().weightInit(wi).nIn(100000).nOut(100).build() : new EmbeddingLayer.Builder().weightInit(wi).nIn(100000).nOut(100).build()).build(); - MultiLayerNetwork net3 = new MultiLayerNetwork(conf3); - net3.init(); - INDArray p1 = net.params(); - INDArray p2 = net2.params(); - INDArray p3 = net3.params(); - boolean eq = p1.equalsWithEps(p2, 1e-4); - String str = (seq ? "EmbeddingSequenceLayer" : "EmbeddingLayer") + " - " + wi; - assertTrue(eq,str + " p1/p2 params not equal"); - double m1 = p1.meanNumber().doubleValue(); - double s1 = p1.stdNumber().doubleValue(); - double m3 = p3.meanNumber().doubleValue(); - double s3 = p3.stdNumber().doubleValue(); - assertEquals( m1, m3, 0.1,str); - assertEquals(s1, s3, 0.1,str); - double re = relErr(s1, s3); - assertTrue( re < 0.05,str + " - " + re); - } - } - } - - public static double relErr(double d1, double d2) { - if (d1 == 0.0 && d2 == 0.0) - return 0.0; - return Math.abs(d1 - d2) / (Math.abs(d1) + Math.abs(d2)); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java deleted file mode 100644 index 3e81dac9e..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/normalization/BatchNormalizationTest.java +++ /dev/null @@ -1,580 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.normalization; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.ListDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.Updater; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.BatchNormalization; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.params.BatchNormalizationParamInitializer; -import org.deeplearning4j.nn.transferlearning.FineTuneConfiguration; -import org.deeplearning4j.nn.transferlearning.TransferLearning; -import org.deeplearning4j.nn.updater.MultiLayerUpdater; -import org.deeplearning4j.nn.updater.UpdaterBlock; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.buffer.util.DataTypeUtil; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastAddOp; -import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastDivOp; -import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastMulOp; -import org.nd4j.linalg.api.ops.impl.broadcast.BroadcastSubOp; -import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.NoOpUpdater; -import org.nd4j.linalg.learning.RmsPropUpdater; -import org.nd4j.linalg.learning.config.AdaDelta; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.linalg.ops.transforms.Transforms; -import org.nd4j.common.primitives.Pair; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - */ -@Slf4j -@DisplayName("Batch Normalization Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) -class BatchNormalizationTest extends BaseDL4JTest { - - static { - // Force Nd4j initialization, then set data type to double: - Nd4j.zeros(1); - DataTypeUtil.setDTypeForContext(DataType.DOUBLE); - } - - protected INDArray dnnInput = Nd4j.linspace(0, 31, 32, Nd4j.dataType()).reshape(2, 16); - - protected INDArray dnnEpsilon = Nd4j.linspace(0, 31, 32, Nd4j.dataType()).reshape(2, 16); - - protected INDArray cnnInput = Nd4j.linspace(0, 63, 64, Nd4j.dataType()).reshape(2, 2, 4, 4); - - protected INDArray cnnEpsilon = Nd4j.linspace(0, 63, 64, Nd4j.dataType()).reshape(2, 2, 4, 4); - - @BeforeEach - void doBefore() { - } - - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - - @Test - @DisplayName("Test Dnn Forward Pass") - void testDnnForwardPass() { - int nOut = 10; - Layer l = getLayer(nOut, 0.0, false, -1, -1); - // Gamma, beta, global mean, global var - assertEquals(4 * nOut, l.numParams()); - INDArray randInput = Nd4j.rand(100, nOut); - INDArray output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces()); - INDArray mean = output.mean(0); - INDArray stdev = output.std(false, 0); - // System.out.println(Arrays.toString(mean.data().asFloat())); - assertArrayEquals(new float[nOut], mean.data().asFloat(), 1e-6f); - assertEquals(Nd4j.ones(nOut), stdev); - // If we fix gamma/beta: expect different mean and variance... - double gamma = 2.0; - double beta = 3.0; - l = getLayer(nOut, 0.0, true, gamma, beta); - // Should have only global mean/var parameters - assertEquals(2 * nOut, l.numParams()); - output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces()); - mean = output.mean(0); - stdev = output.std(false, 0); - assertEquals(Nd4j.valueArrayOf(mean.shape(), beta), mean); - assertEquals(Nd4j.valueArrayOf(stdev.shape(), gamma), stdev); - } - - protected static Layer getLayer(int nOut, double epsilon, boolean lockGammaBeta, double gamma, double beta) { - BatchNormalization.Builder b = new BatchNormalization.Builder().nOut(nOut).eps(epsilon); - if (lockGammaBeta) { - b.lockGammaBeta(true).gamma(gamma).beta(beta); - } - BatchNormalization bN = b.build(); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(bN).build(); - long numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = null; - if (numParams > 0) { - params = Nd4j.create(1, numParams); - } - Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true, params == null ? Nd4j.defaultFloatingPointType() : params.dataType()); - if (numParams > 0) { - layer.setBackpropGradientsViewArray(Nd4j.create(1, numParams)); - } - return layer; - } - - @Test - @DisplayName("Test Dnn Forward Backward") - void testDnnForwardBackward() { - double eps = 1e-5; - int nIn = 4; - int minibatch = 2; - Nd4j.getRandom().setSeed(12345); - INDArray input = Nd4j.rand('c', new int[] { minibatch, nIn }); - // TODO: other values for gamma/beta - INDArray gamma = Nd4j.ones(1, nIn); - INDArray beta = Nd4j.zeros(1, nIn); - Layer l = getLayer(nIn, eps, false, -1, -1); - INDArray mean = input.mean(0); - INDArray var = input.var(false, 0); - INDArray xHat = input.subRowVector(mean).divRowVector(Transforms.sqrt(var.add(eps), true)); - INDArray outExpected = xHat.mulRowVector(gamma).addRowVector(beta); - INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces()); - // System.out.println(Arrays.toString(outExpected.data().asDouble())); - // System.out.println(Arrays.toString(out.data().asDouble())); - assertEquals(outExpected, out); - // ------------------------------------------------------------- - // Check backprop - // dL/dy - INDArray epsilon = Nd4j.rand(minibatch, nIn); - INDArray dldgammaExp = epsilon.mul(xHat).sum(true, 0); - INDArray dldbetaExp = epsilon.sum(true, 0); - INDArray dldxhat = epsilon.mulRowVector(gamma); - INDArray dldvar = dldxhat.mul(input.subRowVector(mean)).mul(-0.5).mulRowVector(Transforms.pow(var.add(eps), -3.0 / 2.0, true)).sum(0); - INDArray dldmu = dldxhat.mulRowVector(Transforms.pow(var.add(eps), -1.0 / 2.0, true)).neg().sum(0).add(dldvar.mul(input.subRowVector(mean).mul(-2.0).sum(0).div(minibatch))); - INDArray dldinExp = dldxhat.mulRowVector(Transforms.pow(var.add(eps), -1.0 / 2.0, true)).add(input.subRowVector(mean).mul(2.0 / minibatch).mulRowVector(dldvar)).addRowVector(dldmu.mul(1.0 / minibatch)); - Pair p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); - INDArray dldgamma = p.getFirst().getGradientFor("gamma"); - INDArray dldbeta = p.getFirst().getGradientFor("beta"); - assertEquals(dldgammaExp, dldgamma); - assertEquals(dldbetaExp, dldbeta); - // System.out.println("EPSILONS"); - // System.out.println(Arrays.toString(dldinExp.data().asDouble())); - // System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble())); - assertEquals(dldinExp, p.getSecond()); - } - - @Test - @DisplayName("Test Cnn Forward Pass") - void testCnnForwardPass() { - int nOut = 10; - Layer l = getLayer(nOut, 0.0, false, -1, -1); - // Gamma, beta, global mean, global var - assertEquals(4 * nOut, l.numParams()); - int hw = 15; - Nd4j.getRandom().setSeed(12345); - INDArray randInput = Nd4j.rand(new int[] { 100, nOut, hw, hw }); - INDArray output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(4, output.rank()); - INDArray mean = output.mean(0, 2, 3); - INDArray stdev = output.std(false, 0, 2, 3); - assertArrayEquals(new float[nOut], mean.data().asFloat(), 1e-6f); - assertArrayEquals(Nd4j.ones(1, nOut).data().asFloat(), stdev.data().asFloat(), 1e-6f); - // If we fix gamma/beta: expect different mean and variance... - double gamma = 2.0; - double beta = 3.0; - l = getLayer(nOut, 0.0, true, gamma, beta); - // Should have only global mean/var parameters - assertEquals(2 * nOut, l.numParams()); - output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces()); - mean = output.mean(0, 2, 3); - stdev = output.std(false, 0, 2, 3); - assertEquals(Nd4j.valueArrayOf(mean.shape(), beta), mean); - assertEquals(Nd4j.valueArrayOf(stdev.shape(), gamma), stdev); - } - - @Test - @DisplayName("Test 2 d Vs 4 d") - void test2dVs4d() { - // Idea: 2d and 4d should be the same... - Nd4j.getRandom().setSeed(12345); - int m = 2; - int h = 3; - int w = 3; - int nOut = 2; - INDArray in = Nd4j.rand('c', m * h * w, nOut); - INDArray in4 = in.dup(); - in4 = Shape.newShapeNoCopy(in4, new int[] { m, h, w, nOut }, false); - assertNotNull(in4); - in4 = in4.permute(0, 3, 1, 2).dup(); - INDArray arr = Nd4j.rand(1, m * h * w * nOut).reshape('f', h, w, m, nOut).permute(2, 3, 1, 0); - in4 = arr.assign(in4); - Layer l1 = getLayer(nOut); - Layer l2 = getLayer(nOut); - INDArray out2d = l1.activate(in.dup(), true, LayerWorkspaceMgr.noWorkspaces()); - INDArray out4d = l2.activate(in4.dup(), true, LayerWorkspaceMgr.noWorkspaces()); - INDArray out4dAs2 = out4d.permute(0, 2, 3, 1).dup('c'); - out4dAs2 = Shape.newShapeNoCopy(out4dAs2, new int[] { m * h * w, nOut }, false); - assertEquals(out2d, out4dAs2); - // Test backprop: - INDArray epsilons2d = Nd4j.rand('c', m * h * w, nOut); - INDArray epsilons4d = epsilons2d.dup(); - epsilons4d = Shape.newShapeNoCopy(epsilons4d, new int[] { m, h, w, nOut }, false); - assertNotNull(epsilons4d); - epsilons4d = epsilons4d.permute(0, 3, 1, 2).dup(); - Pair b2d = l1.backpropGradient(epsilons2d, LayerWorkspaceMgr.noWorkspaces()); - Pair b4d = l2.backpropGradient(epsilons4d, LayerWorkspaceMgr.noWorkspaces()); - INDArray e4dAs2d = b4d.getSecond().permute(0, 2, 3, 1).dup('c'); - e4dAs2d = Shape.newShapeNoCopy(e4dAs2d, new int[] { m * h * w, nOut }, false); - assertEquals(b2d.getSecond(), e4dAs2d); - } - - protected static Layer getLayer(int nOut) { - return getLayer(nOut, Nd4j.EPS_THRESHOLD, false, -1, -1); - } - - @Test - @DisplayName("Test Cnn Forward Backward") - void testCnnForwardBackward() { - double eps = 1e-5; - int nIn = 4; - int hw = 3; - int minibatch = 2; - Nd4j.getRandom().setSeed(12345); - INDArray input = Nd4j.rand('c', new int[] { minibatch, nIn, hw, hw }); - // TODO: other values for gamma/beta - INDArray gamma = Nd4j.ones(1, nIn); - INDArray beta = Nd4j.zeros(1, nIn); - Layer l = getLayer(nIn, eps, false, -1, -1); - INDArray mean = input.mean(0, 2, 3); - INDArray var = input.var(false, 0, 2, 3); - INDArray xHat = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1)); - Nd4j.getExecutioner().exec(new BroadcastDivOp(xHat, Transforms.sqrt(var.add(eps), true), xHat, 1)); - INDArray outExpected = Nd4j.getExecutioner().exec(new BroadcastMulOp(xHat, gamma, xHat.dup(), 1)); - Nd4j.getExecutioner().exec(new BroadcastAddOp(outExpected, beta, outExpected, 1)); - INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces()); - // System.out.println(Arrays.toString(outExpected.data().asDouble())); - // System.out.println(Arrays.toString(out.data().asDouble())); - assertEquals(outExpected, out); - // ------------------------------------------------------------- - // Check backprop - // dL/dy - INDArray epsilon = Nd4j.rand('c', new int[] { minibatch, nIn, hw, hw }); - int effectiveMinibatch = minibatch * hw * hw; - INDArray dldgammaExp = epsilon.mul(xHat).sum(0, 2, 3); - dldgammaExp = dldgammaExp.reshape(1, dldgammaExp.length()); - INDArray dldbetaExp = epsilon.sum(0, 2, 3); - dldbetaExp = dldbetaExp.reshape(1, dldbetaExp.length()); - // epsilon.mulRowVector(gamma); - INDArray dldxhat = Nd4j.getExecutioner().exec(new BroadcastMulOp(epsilon, gamma, epsilon.dup(), 1)); - INDArray inputSubMean = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1)); - INDArray dldvar = dldxhat.mul(inputSubMean).mul(-0.5); - dldvar = Nd4j.getExecutioner().exec(new BroadcastMulOp(dldvar, Transforms.pow(var.add(eps), -3.0 / 2.0, true), dldvar.dup(), 1)); - dldvar = dldvar.sum(0, 2, 3); - INDArray dldmu = Nd4j.getExecutioner().exec(new BroadcastMulOp(dldxhat, Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1)).neg().sum(0, 2, 3); - dldmu = dldmu.add(dldvar.mul(inputSubMean.mul(-2.0).sum(0, 2, 3).div(effectiveMinibatch))); - INDArray dldinExp = Nd4j.getExecutioner().exec(new BroadcastMulOp(dldxhat, Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1)); - dldinExp = dldinExp.add(Nd4j.getExecutioner().exec(new BroadcastMulOp(inputSubMean.mul(2.0 / effectiveMinibatch), dldvar, inputSubMean.dup(), 1))); - dldinExp = Nd4j.getExecutioner().exec(new BroadcastAddOp(dldinExp, dldmu.mul(1.0 / effectiveMinibatch), dldinExp.dup(), 1)); - Pair p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); - INDArray dldgamma = p.getFirst().getGradientFor("gamma"); - INDArray dldbeta = p.getFirst().getGradientFor("beta"); - assertEquals(dldgammaExp, dldgamma); - assertEquals(dldbetaExp, dldbeta); - // System.out.println("EPSILONS"); - // System.out.println(Arrays.toString(dldinExp.data().asDouble())); - // System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble())); - assertEquals(dldinExp, p.getSecond()); - } - - @Test - @DisplayName("Test DBNBN Multi Layer") - void testDBNBNMultiLayer() throws Exception { - DataSetIterator iter = new MnistDataSetIterator(2, 2); - DataSet next = iter.next(); - // Run with separate activation layer - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new DenseLayer.Builder().nIn(28 * 28).nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new BatchNormalization.Builder().nOut(10).build()).layer(2, new ActivationLayer.Builder().activation(Activation.RELU).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - network.setInput(next.getFeatures()); - INDArray activationsActual = network.output(next.getFeatures()); - assertEquals(10, activationsActual.shape()[1], 1e-2); - network.fit(next); - INDArray actualGammaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.GAMMA); - INDArray actualBetaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.BETA); - assertTrue(actualGammaParam != null); - assertTrue(actualBetaParam != null); - } - - @Test - @DisplayName("Test CNNBN Activation Combo") - void testCNNBNActivationCombo() throws Exception { - DataSetIterator iter = new MnistDataSetIterator(2, 2); - DataSet next = iter.next(); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).build()).layer(1, new BatchNormalization.Builder().build()).layer(2, new ActivationLayer.Builder().activation(Activation.RELU).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - network.fit(next); - assertNotEquals(null, network.getLayer(0).getParam("W")); - assertNotEquals(null, network.getLayer(0).getParam("b")); - } - - @Test - @DisplayName("Check Serialization") - void checkSerialization() throws Exception { - // Serialize the batch norm network (after training), and make sure we get same activations out as before - // i.e., make sure state is properly stored - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list().layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).build()).layer(1, new BatchNormalization.Builder().build()).layer(2, new ActivationLayer.Builder().activation(Activation.LEAKYRELU).build()).layer(3, new DenseLayer.Builder().nOut(10).activation(Activation.LEAKYRELU).build()).layer(4, new BatchNormalization.Builder().build()).layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSetIterator iter = new MnistDataSetIterator(16, true, 12345); - for (int i = 0; i < 20; i++) { - net.fit(iter.next()); - } - INDArray in = iter.next().getFeatures(); - INDArray out = net.output(in, false); - INDArray out2 = net.output(in, false); - assertEquals(out, out2); - MultiLayerNetwork net2 = TestUtils.testModelSerialization(net); - INDArray outDeser = net2.output(in, false); - assertEquals(out, outDeser); - } - - @Test - @DisplayName("Test Gradient And Updaters") - void testGradientAndUpdaters() throws Exception { - // Global mean/variance are part of the parameter vector. Expect 0 gradient, and no-op updater for these - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.RMSPROP).seed(12345).list().layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).build()).layer(1, new BatchNormalization.Builder().build()).layer(2, new ActivationLayer.Builder().activation(Activation.LEAKYRELU).build()).layer(3, new DenseLayer.Builder().nOut(10).activation(Activation.LEAKYRELU).build()).layer(4, new BatchNormalization.Builder().build()).layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSetIterator iter = new MnistDataSetIterator(16, true, 12345); - DataSet ds = iter.next(); - net.setInput(ds.getFeatures()); - net.setLabels(ds.getLabels()); - net.computeGradientAndScore(); - Gradient g = net.gradient(); - Map map = g.gradientForVariable(); - org.deeplearning4j.nn.api.Updater u = net.getUpdater(); - MultiLayerUpdater mlu = (MultiLayerUpdater) u; - List l = mlu.getUpdaterBlocks(); - assertNotNull(l); - // Conv+bn (RMSProp), No-op (bn), RMSProp (dense, bn), no-op (bn), RMSProp (out) - assertEquals(5, l.size()); - for (UpdaterBlock ub : l) { - List list = ub.getLayersAndVariablesInBlock(); - for (UpdaterBlock.ParamState v : list) { - if (BatchNormalizationParamInitializer.GLOBAL_MEAN.equals(v.getParamName()) || BatchNormalizationParamInitializer.GLOBAL_VAR.equals(v.getParamName()) || BatchNormalizationParamInitializer.GLOBAL_LOG_STD.equals(v.getParamName())) { - assertTrue(ub.getGradientUpdater() instanceof NoOpUpdater); - } else { - assertTrue(ub.getGradientUpdater() instanceof RmsPropUpdater); - } - } - } - } - - @Test - @DisplayName("Check Mean Variance Estimate") - void checkMeanVarianceEstimate() throws Exception { - Nd4j.getRandom().setSeed(12345); - // Check that the internal global mean/variance estimate is approximately correct - for (boolean useLogStd : new boolean[] { true, false }) { - // First, Mnist data as 2d input (NOT taking into account convolution property) - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.RMSPROP).seed(12345).list().layer(0, new BatchNormalization.Builder().nIn(10).nOut(10).eps(1e-5).decay(0.95).useLogStd(useLogStd).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - int minibatch = 32; - List list = new ArrayList<>(); - for (int i = 0; i < 200; i++) { - list.add(new DataSet(Nd4j.rand(minibatch, 10), Nd4j.rand(minibatch, 10))); - } - DataSetIterator iter = new ListDataSetIterator(list); - INDArray expMean = Nd4j.valueArrayOf(new int[] { 1, 10 }, 0.5); - // Expected variance of U(0,1) distribution: 1/12 * (1-0)^2 = 0.0833 - INDArray expVar = Nd4j.valueArrayOf(new int[] { 1, 10 }, 1 / 12.0); - for (int i = 0; i < 10; i++) { - iter.reset(); - net.fit(iter); - } - INDArray estMean = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_MEAN); - INDArray estVar; - if (useLogStd) { - INDArray log10std = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_LOG_STD); - estVar = Nd4j.valueArrayOf(log10std.shape(), 10.0).castTo(log10std.dataType()); - // stdev = 10^(log10(stdev)) - Transforms.pow(estVar, log10std, false); - estVar.muli(estVar); - } else { - estVar = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_VAR); - } - float[] fMeanExp = expMean.data().asFloat(); - float[] fMeanAct = estMean.data().asFloat(); - float[] fVarExp = expVar.data().asFloat(); - float[] fVarAct = estVar.data().asFloat(); - // System.out.println("Mean vs. estimated mean:"); - // System.out.println(Arrays.toString(fMeanExp)); - // System.out.println(Arrays.toString(fMeanAct)); - // - // System.out.println("Var vs. estimated var:"); - // System.out.println(Arrays.toString(fVarExp)); - // System.out.println(Arrays.toString(fVarAct)); - assertArrayEquals(fMeanExp, fMeanAct, 0.02f); - assertArrayEquals(fVarExp, fVarAct, 0.02f); - } - } - - @Test - @DisplayName("Check Mean Variance Estimate CNN") - void checkMeanVarianceEstimateCNN() throws Exception { - for (boolean useLogStd : new boolean[] { true, false }) { - Nd4j.getRandom().setSeed(12345); - // Check that the internal global mean/variance estimate is approximately correct - // First, Mnist data as 2d input (NOT taking into account convolution property) - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.RMSPROP).seed(12345).list().layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(useLogStd).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).nOut(10).build()).setInputType(InputType.convolutional(5, 5, 3)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - int minibatch = 32; - List list = new ArrayList<>(); - for (int i = 0; i < 100; i++) { - list.add(new DataSet(Nd4j.rand(new int[] { minibatch, 3, 5, 5 }), Nd4j.rand(minibatch, 10))); - } - DataSetIterator iter = new ListDataSetIterator(list); - INDArray expMean = Nd4j.valueArrayOf(new int[] { 1, 3 }, 0.5); - // Expected variance of U(0,1) distribution: 1/12 * (1-0)^2 = 0.0833 - INDArray expVar = Nd4j.valueArrayOf(new int[] { 1, 3 }, 1 / 12.0); - for (int i = 0; i < 10; i++) { - iter.reset(); - net.fit(iter); - } - INDArray estMean = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_MEAN); - INDArray estVar; - if (useLogStd) { - INDArray log10std = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_LOG_STD); - estVar = Nd4j.valueArrayOf(log10std.shape(), 10.0).castTo(log10std.dataType()); - // stdev = 10^(log10(stdev)) - Transforms.pow(estVar, log10std, false); - estVar.muli(estVar); - } else { - estVar = net.getLayer(0).getParam(BatchNormalizationParamInitializer.GLOBAL_VAR); - } - float[] fMeanExp = expMean.data().asFloat(); - float[] fMeanAct = estMean.data().asFloat(); - float[] fVarExp = expVar.data().asFloat(); - float[] fVarAct = estVar.data().asFloat(); - // System.out.println("Mean vs. estimated mean:"); - // System.out.println(Arrays.toString(fMeanExp)); - // System.out.println(Arrays.toString(fMeanAct)); - // - // System.out.println("Var vs. estimated var:"); - // System.out.println(Arrays.toString(fVarExp)); - // System.out.println(Arrays.toString(fVarAct)); - assertArrayEquals(fMeanExp, fMeanAct, 0.01f); - assertArrayEquals(fVarExp, fVarAct, 0.01f); - } - } - - @Test - @DisplayName("Check Mean Variance Estimate CNN Compare Modes") - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - void checkMeanVarianceEstimateCNNCompareModes() throws Exception { - Nd4j.getRandom().setSeed(12345); - // Check that the internal global mean/variance estimate is approximately correct - // First, Mnist data as 2d input (NOT taking into account convolution property) - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.RMSPROP).seed(12345).list().layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(false).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).nOut(10).build()).setInputType(InputType.convolutional(5, 5, 3)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.RMSPROP).seed(12345).list().layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(true).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER).activation(Activation.IDENTITY).nOut(10).build()).setInputType(InputType.convolutional(5, 5, 3)).build(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - int minibatch = 32; - for (int i = 0; i < 10; i++) { - DataSet ds = new DataSet(Nd4j.rand(new int[] { minibatch, 3, 5, 5 }), Nd4j.rand(minibatch, 10)); - net.fit(ds); - net2.fit(ds); - INDArray globalVar = net.getParam("0_" + BatchNormalizationParamInitializer.GLOBAL_VAR); - INDArray log10std = net2.getParam("0_" + BatchNormalizationParamInitializer.GLOBAL_LOG_STD); - INDArray globalVar2 = Nd4j.valueArrayOf(log10std.shape(), 10.0).castTo(log10std.dataType()); - // stdev = 10^(log10(stdev)) - Transforms.pow(globalVar2, log10std, false); - globalVar2.muli(globalVar2); - assertEquals(globalVar, globalVar2); - } - } - - @Test - @DisplayName("Test Batch Norm") - void testBatchNorm() throws Exception { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new Adam(1e-3)).activation(Activation.TANH).list().layer(new ConvolutionLayer.Builder().nOut(5).kernelSize(2, 2).build()).layer(new BatchNormalization()).layer(new ConvolutionLayer.Builder().nOut(5).kernelSize(2, 2).build()).layer(new OutputLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSetIterator iter = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(32, true, 12345), 10); - net.fit(iter); - MultiLayerNetwork net2 = new TransferLearning.Builder(net).fineTuneConfiguration(FineTuneConfiguration.builder().updater(new AdaDelta()).build()).removeOutputLayer().addLayer(new BatchNormalization.Builder().nOut(3380).build()).addLayer(new OutputLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nIn(3380).nOut(10).build()).build(); - net2.fit(iter); - } - - @Test - @DisplayName("Test Batch Norm Recurrent Cnn 1 d") - void testBatchNormRecurrentCnn1d() { - // Simple sanity check on CNN1D and RNN layers - for (boolean rnn : new boolean[] { true, false }) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).weightInit(WeightInit.XAVIER).convolutionMode(ConvolutionMode.Same).list().layer(rnn ? new LSTM.Builder().nOut(3).build() : new Convolution1DLayer.Builder().kernelSize(3).stride(1).nOut(3).build()).layer(new BatchNormalization()).layer(new RnnOutputLayer.Builder().nOut(3).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build()).setInputType(InputType.recurrent(3)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray in = Nd4j.rand(new int[] { 1, 3, 5 }); - INDArray label = Nd4j.rand(new int[] { 1, 3, 5 }); - INDArray out = net.output(in); - assertArrayEquals(new long[] { 1, 3, 5 }, out.shape()); - net.fit(in, label); - log.info("OK: {}", (rnn ? "rnn" : "cnn1d")); - } - } - - @Test - @DisplayName("Test Input Validation") - void testInputValidation() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new BatchNormalization.Builder().nIn(10).nOut(10).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray in1 = Nd4j.create(1, 10); - INDArray in2 = Nd4j.create(1, 5); - INDArray out1 = net.output(in1); - try { - INDArray out2 = net.output(in2); - fail(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("expected input")); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/normalization/LocalResponseTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/normalization/LocalResponseTest.java deleted file mode 100644 index f939135e4..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/normalization/LocalResponseTest.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.normalization; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.LocalResponseNormalization; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.primitives.Pair; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - */ -@DisplayName("Local Response Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class LocalResponseTest extends BaseDL4JTest { - - private INDArray x = Nd4j.create(new double[] { 0.88128096, -0.96666986, -0.61832994, 0.26418415, 0.05694608, 0.2950289, 0.99222249, 0.24541704, 0.4219842, 0.96430975, 0.19299535, -0.06658337, -0.27603117, 0.24216647, 0.21834095, 0.03863283, -0.82313406, -0.37236378, -0.77667993, 0.66295379, -0.34406275, -0.25924176, 0.26652309, -0.58964926, -0.46907067, 0.34666502, 0.81208313, -0.17042427, -0.22470538, 0.8348338, 0.50494033, 0.45004508, 0.58735144, -0.87217808, -0.74788797, -0.04363599, 0.72276866, 0.52476895, -0.52383977, 0.1311436, 0.2628099, 0.77274454, 0.86400729, -0.35246921, -0.03399619, -0.502312, 0.42834607, 0.85534132, 0.90083021, 0.24571614, 0.63058525, -0.82919437, 0.57236177, -0.0913529, -0.7102778, 0.81631756, -0.89004314, 0.43995622, -0.26112801, -0.76135367, 0.65180862, -0.54667377, 0.94908774, 0.59298772, 0.36457643, 0.58892179, -0.52951556, 0.31559938, -0.55268252, 0.8272332, 0.37911707, -0.96299696, -0.40717798, 0.43324658, 0.2589654, -0.15605508, 0.96334064, -0.31666604, 0.19781154, 0.09908111, 0.64796048, -0.99037546, 0.67919868, 0.43810204 }, new int[] { 2, 7, 3, 2 }); - - private INDArray activationsExpected = Nd4j.create(new double[] { 0.52397668, -0.57476264, -0.3676528, 0.15707894, 0.03385943, 0.17542371, 0.58992499, 0.14591768, 0.25090647, 0.57335907, 0.11475233, -0.03958985, -0.16411273, 0.14398433, 0.12981956, 0.02297027, -0.48942304, -0.22139823, -0.46177959, 0.39418164, -0.20457059, -0.15413573, 0.15846729, -0.3505919, -0.27889356, 0.20611978, 0.48284137, -0.10133155, -0.13360347, 0.49636194, 0.30022132, 0.26758799, 0.34922296, -0.51858318, -0.4446843, -0.02594452, 0.42974478, 0.31202248, -0.31146204, 0.07797609, 0.15626372, 0.4594543, 0.51370209, -0.20957276, -0.02021335, -0.29866382, 0.25469059, 0.50856382, 0.53558689, 0.14609739, 0.37491882, -0.49301448, 0.34031925, -0.05431537, -0.42228988, 0.48536259, -0.52917528, 0.26157826, -0.15526266, -0.45265958, 0.38753596, -0.32503816, 0.56427884, 0.35256693, 0.21676543, 0.35014921, -0.31483513, 0.18764766, -0.32859638, 0.49183461, 0.22540972, -0.57255536, -0.24210122, 0.25760418, 0.15397197, -0.0927838, 0.57277, -0.18827969, 0.1176173, 0.05891332, 0.38526815, -0.58884346, 0.40383074, 0.26048511 }, new int[] { 2, 7, 3, 2 }); - - private INDArray epsilon = Nd4j.create(new double[] { -0.13515499, 0.96470547, -0.62253004, 0.80172491, -0.97510445, -0.41198033, -0.4790071, 0.07551047, -0.01383764, -0.05797465, 0.21242172, 0.7145375, -0.17809176, -0.11465316, -0.2066526, 0.21950938, 0.4627091, 0.30275798, 0.61443841, 0.75912178, -0.132248, -0.82923287, 0.74962652, -0.88993639, 0.04406403, 0.32096064, -0.46400586, 0.1603231, 0.63007826, 0.10626783, 0.08009516, 0.88297033, 0.11441587, 0.35862735, 0.40441504, -0.60132015, 0.87743825, 0.09792926, 0.92742652, 0.6182847, -0.9602651, -0.19611064, 0.15762019, 0.00339905, -0.9238292, 0.02451134, -0.44294646, -0.5450229, 0.87502575, -0.59481794, 0.65259099, -0.77772689, 0.53300053, 0.11541174, 0.32667685, 0.99437004, -0.04084824, -0.45166185, 0.29513556, 0.53582036, 0.95541358, -0.75714606, -0.63295805, -0.70315111, -0.6553846, -0.78824568, 0.84295344, -0.38352135, -0.04541624, 0.17396702, 0.41530582, 0.11870354, 0.85787249, -0.94597596, 0.05792254, 0.04811822, 0.04847952, -0.82953823, 0.8089835, 0.50185651, -0.88619858, -0.78598201, 0.27489874, 0.63673472 }, new int[] { 2, 7, 3, 2 }); - - private INDArray newEpsilonExpected = Nd4j.create(new double[] { -0.08033668, 0.57355404, -0.37014094, 0.47668865, -0.57978398, -0.24495915, -0.28474802, 0.04490108, -0.00823483, -0.03448687, 0.12630466, 0.42485803, -0.10589627, -0.06816553, -0.12287001, 0.13051508, 0.27510744, 0.18001786, 0.36528736, 0.45133191, -0.07863599, -0.49303374, 0.44571424, -0.52912313, 0.02620371, 0.19082049, -0.27585581, 0.09532529, 0.3746179, 0.06316902, 0.04761803, 0.52497554, 0.06804816, 0.21323238, 0.24044329, -0.35752413, 0.52168733, 0.05821467, 0.55140609, 0.3676247, -0.57095432, -0.11660115, 0.09367896, 0.00202246, -0.54928631, 0.01455687, -0.26336867, -0.3240425, 0.52023786, -0.35366109, 0.3879728, -0.46243483, 0.31692421, 0.06862034, 0.19421607, 0.59124804, -0.0242459, -0.26852599, 0.17547797, 0.31857637, 0.56804365, -0.45020312, -0.37634474, -0.41804832, -0.38966343, -0.4686695, 0.50119156, -0.22802454, -0.02698562, 0.10343311, 0.24693431, 0.0706142, 0.5100745, -0.56245267, 0.03443092, 0.02860913, 0.02883426, -0.49320197, 0.4810102, 0.29840365, -0.5269345, -0.46732581, 0.16344811, 0.37857518 }, new int[] { 2, 7, 3, 2 }); - - private INDArray activationsActual; - - private Layer layer; - - @BeforeEach - void doBefore() { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123).layer(new LocalResponseNormalization.Builder().k(2).n(5).alpha(1e-4).beta(0.75).build()).build(); - layer = new LocalResponseNormalization().instantiate(conf, null, 0, null, false, Nd4j.defaultFloatingPointType()); - activationsActual = layer.activate(x, false, LayerWorkspaceMgr.noWorkspaces()); - } - - @Test - @DisplayName("Test Activate") - void testActivate() { - // Precision is off from the expected results because expected results generated in numpy - assertEquals(activationsExpected, activationsActual); - assertArrayEquals(activationsExpected.shape(), activationsActual.shape()); - } - - @Test - @DisplayName("Test Backprop Gradient") - void testBackpropGradient() { - Pair containedOutput = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(newEpsilonExpected.getDouble(8), containedOutput.getSecond().getDouble(8), 1e-4); - assertEquals(newEpsilonExpected.getDouble(20), containedOutput.getSecond().getDouble(20), 1e-4); - assertEquals(null, containedOutput.getFirst().getGradientFor("W")); - assertArrayEquals(newEpsilonExpected.shape(), containedOutput.getSecond().shape()); - } - - @Test - @DisplayName("Test Regularization") - void testRegularization() { - // Confirm a structure with regularization true will not throw an error - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).l1(0.2).l2(0.1).seed(123).layer(new LocalResponseNormalization.Builder().k(2).n(5).alpha(1e-4).beta(0.75).build()).build(); - } - - @Test - @DisplayName("Test Multi CNN Layer") - void testMultiCNNLayer() throws Exception { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list().layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER).activation(Activation.RELU).build()).layer(1, new LocalResponseNormalization.Builder().build()).layer(2, new DenseLayer.Builder().nOut(2).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(10).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - DataSetIterator iter = new MnistDataSetIterator(2, 2); - DataSet next = iter.next(); - network.fit(next); - } - - @Test - @DisplayName("Test Lrn Manual") - void testLrnManual() { - int wh = 5; - int depth = 6; - int minibatch = 3; - int n = 4; - double k = 2.0; - double alpha = 1e-4; - double beta = 0.75; - INDArray in = Nd4j.rand(new int[] { minibatch, depth, wh, wh }); - INDArray outExp = Nd4j.zeros(minibatch, depth, wh, wh); - for (int m = 0; m < minibatch; m++) { - for (int x = 0; x < wh; x++) { - for (int y = 0; y < wh; y++) { - for (int i = 0; i < depth; i++) { - int jFrom = Math.max(0, i - n / 2); - int jTo = Math.min(depth - 1, i + n / 2); - double sum = 0.0; - for (int j = jFrom; j <= jTo; j++) { - double d = in.getDouble(m, j, x, y); - sum += d * d; - } - double out = in.getDouble(m, i, x, y) / Math.pow(k + alpha * sum, beta); - outExp.putScalar(m, i, x, y, out); - } - } - } - } - LocalResponseNormalization lrn = new LocalResponseNormalization.Builder().build(); - NeuralNetConfiguration nnc = new NeuralNetConfiguration.Builder().layer(lrn).build(); - org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization layer = (org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization) lrn.instantiate(nnc, null, 0, null, false, Nd4j.defaultFloatingPointType()); - INDArray outAct = layer.activate(in, true, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(outExp, outAct); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayerTest.java deleted file mode 100644 index 12f6e0ee3..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/ocnn/OCNNOutputLayerTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.ocnn; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.gradientcheck.GradientCheckUtil; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.deeplearning4j.util.ModelSerializer; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.impl.ActivationIdentity; -import org.nd4j.linalg.activations.impl.ActivationReLU; -import org.nd4j.linalg.activations.impl.ActivationSigmoid; -import org.nd4j.linalg.api.buffer.DataBuffer; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.schedule.ScheduleType; -import org.nd4j.linalg.schedule.StepSchedule; -import java.io.File; -import java.util.UUID; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Ocnn Output Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.FILE_IO) -class OCNNOutputLayerTest extends BaseDL4JTest { - - private static final boolean PRINT_RESULTS = true; - - private static final boolean RETURN_ON_FIRST_FAILURE = false; - - private static final double DEFAULT_EPS = 1e-6; - - private static final double DEFAULT_MAX_REL_ERROR = 1e-3; - - private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; - - @TempDir - public Path testDir; - - static { - Nd4j.setDataType(DataType.DOUBLE); - } - - @Test - @DisplayName("Test Layer") - void testLayer() { - DataSetIterator dataSetIterator = getNormalizedIterator(); - boolean doLearningFirst = true; - MultiLayerNetwork network = getGradientCheckNetwork(2); - DataSet ds = dataSetIterator.next(); - INDArray arr = ds.getFeatures(); - network.setInput(arr); - if (doLearningFirst) { - // Run a number of iterations of learning - network.setInput(arr); - network.setListeners(new ScoreIterationListener(1)); - network.computeGradientAndScore(); - double scoreBefore = network.score(); - for (int j = 0; j < 10; j++) network.fit(ds); - network.computeGradientAndScore(); - double scoreAfter = network.score(); - // Can't test in 'characteristic mode of operation' if not learning - String msg = "testLayer() - score did not (sufficiently) decrease during learning - activationFn=" + "relu" + ", lossFn=" + "ocnn" + ", " + "sigmoid" + ", doLearningFirst=" + doLearningFirst + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")"; - // assertTrue(msg, scoreAfter < scoreBefore); - } - if (PRINT_RESULTS) { - System.out.println("testLayer() - activationFn=" + "relu" + ", lossFn=" + "ocnn" + "sigmoid" + ", doLearningFirst=" + doLearningFirst); - for (int j = 0; j < network.getnLayers(); j++) System.out.println("Layer " + j + " # params: " + network.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(network, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, ds.getFeatures(), ds.getLabels()); - String msg = "testLayer() - activationFn=" + "relu" + ", lossFn=" + "ocnn" + ",=" + "sigmoid" + ", doLearningFirst=" + doLearningFirst; - assertTrue(gradOK,msg); - } - - @Test - @DisplayName("Test Label Probabilities") - void testLabelProbabilities() throws Exception { - Nd4j.getRandom().setSeed(42); - DataSetIterator dataSetIterator = getNormalizedIterator(); - MultiLayerNetwork network = getSingleLayer(); - DataSet next = dataSetIterator.next(); - DataSet filtered = next.filterBy(new int[] { 0, 1 }); - for (int i = 0; i < 10; i++) { - network.setEpochCount(i); - network.getLayerWiseConfigurations().setEpochCount(i); - network.fit(filtered); - } - DataSet anomalies = next.filterBy(new int[] { 2 }); - INDArray output = network.output(anomalies.getFeatures()); - INDArray normalOutput = network.output(anomalies.getFeatures(), false); - assertEquals(output.lt(0.0).castTo(Nd4j.defaultFloatingPointType()).sumNumber().doubleValue(), normalOutput.eq(0.0).castTo(Nd4j.defaultFloatingPointType()).sumNumber().doubleValue(), 1e-1); - // System.out.println("Labels " + anomalies.getLabels()); - // System.out.println("Anomaly output " + normalOutput); - // System.out.println(output); - INDArray normalProbs = network.output(filtered.getFeatures()); - INDArray outputForNormalSamples = network.output(filtered.getFeatures(), false); - System.out.println("Normal probabilities " + normalProbs); - System.out.println("Normal raw output " + outputForNormalSamples); - File tmpFile = new File(testDir.toFile(), "tmp-file-" + UUID.randomUUID().toString()); - ModelSerializer.writeModel(network, tmpFile, true); - tmpFile.deleteOnExit(); - MultiLayerNetwork multiLayerNetwork = ModelSerializer.restoreMultiLayerNetwork(tmpFile); - assertEquals(network.params(), multiLayerNetwork.params()); - assertEquals(network.numParams(), multiLayerNetwork.numParams()); - } - - public DataSetIterator getNormalizedIterator() { - DataSetIterator dataSetIterator = new IrisDataSetIterator(150, 150); - NormalizerStandardize normalizerStandardize = new NormalizerStandardize(); - normalizerStandardize.fit(dataSetIterator); - dataSetIterator.reset(); - dataSetIterator.setPreProcessor(normalizerStandardize); - return dataSetIterator; - } - - private MultiLayerNetwork getSingleLayer() { - int numHidden = 2; - MultiLayerConfiguration configuration = new NeuralNetConfiguration.Builder().seed(12345).weightInit(WeightInit.XAVIER).miniBatch(true).updater(new Adam(0.1)).list(new DenseLayer.Builder().activation(new ActivationReLU()).nIn(4).nOut(2).build(), new org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer.Builder().nIn(2).activation(new ActivationSigmoid()).initialRValue(0.1).nu(0.1).hiddenLayerSize(numHidden).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(configuration); - network.init(); - network.setListeners(new ScoreIterationListener(1)); - return network; - } - - public MultiLayerNetwork getGradientCheckNetwork(int numHidden) { - MultiLayerConfiguration configuration = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).seed(42).updater(new NoOp()).miniBatch(false).list(new DenseLayer.Builder().activation(new ActivationIdentity()).nIn(4).nOut(4).build(), new org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer.Builder().nIn(4).nu(0.002).activation(new ActivationSigmoid()).hiddenLayerSize(numHidden).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(configuration); - network.init(); - return network; - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalTest.java deleted file mode 100644 index d197a2760..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/BidirectionalTest.java +++ /dev/null @@ -1,491 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.recurrent; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator; -import org.deeplearning4j.earlystopping.EarlyStoppingConfiguration; -import org.deeplearning4j.earlystopping.saver.InMemoryModelSaver; -import org.deeplearning4j.earlystopping.scorecalc.DataSetLossCalculator; -import org.deeplearning4j.earlystopping.termination.MaxEpochsTerminationCondition; -import org.deeplearning4j.earlystopping.trainer.EarlyStoppingGraphTrainer; -import org.deeplearning4j.nn.conf.*; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM; -import org.deeplearning4j.nn.conf.layers.GravesLSTM; -import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; -import org.deeplearning4j.nn.conf.layers.recurrent.Bidirectional; -import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; -import org.deeplearning4j.nn.conf.layers.variational.BernoulliReconstructionDistribution; -import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.updater.MultiLayerUpdater; -import org.deeplearning4j.nn.updater.graph.ComputationGraphUpdater; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.util.ModelSerializer; -import org.deeplearning4j.util.TimeSeriesUtils; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.enums.RnnDataFormat; -import org.nd4j.linalg.BaseNd4jTestWithBackends; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.MultiDataSet; -import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.Nd4jBackend; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.primitives.Pair; -import org.deeplearning4j.nn.workspace.ArrayType; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - -import static org.deeplearning4j.nn.conf.RNNFormat.NCW; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Bidirectional Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class BidirectionalTest extends BaseDL4JTest { - - - - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(RNNFormat rnnFormat : RNNFormat.values()) { - args.add(Arguments.of(rnnFormat,nd4jBackend)); - } - } - return args.stream(); - } - - - @DisplayName("Compare Implementations") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.BidirectionalTest#params") - void compareImplementations(RNNFormat rnnDataFormat,Nd4jBackend backend) { - for (WorkspaceMode wsm : WorkspaceMode.values()) { - log.info("*** Starting workspace mode: " + wsm); - // Bidirectional(GravesLSTM) and GravesBidirectionalLSTM should be equivalent, given equivalent params - // Note that GravesBidirectionalLSTM implements ADD mode only - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).trainingWorkspaceMode(wsm).inferenceWorkspaceMode(wsm).updater(new Adam()).list().layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())).layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())).layer(new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).dataFormat(rnnDataFormat).nIn(10).nOut(10).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).trainingWorkspaceMode(wsm).inferenceWorkspaceMode(wsm).updater(new Adam()).list().layer(new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()).layer(new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()).layer(new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).dataFormat(rnnDataFormat).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - assertEquals(net1.numParams(), net2.numParams()); - for (int i = 0; i < 3; i++) { - int n1 = (int) net1.getLayer(i).numParams(); - int n2 = (int) net2.getLayer(i).numParams(); - assertEquals(n1, n2); - } - // Assuming exact same layout here... - net2.setParams(net1.params()); - INDArray in; - if (rnnDataFormat == NCW) { - in = Nd4j.rand(new int[] { 3, 10, 5 }); - } else { - in = Nd4j.rand(new int[] { 3, 5, 10 }); - } - INDArray out1 = net1.output(in); - INDArray out2 = net2.output(in); - assertEquals(out1, out2); - INDArray labels; - if (rnnDataFormat == NCW) { - labels = Nd4j.rand(new int[] { 3, 10, 5 }); - } else { - labels = Nd4j.rand(new int[] { 3, 5, 10 }); - } - net1.setInput(in); - net1.setLabels(labels); - net2.setInput(in); - net2.setLabels(labels); - net1.computeGradientAndScore(); - net2.computeGradientAndScore(); - // Ensure scores are equal: - assertEquals(net1.score(), net2.score(), 1e-6); - // Ensure gradients are equal: - Gradient g1 = net1.gradient(); - Gradient g2 = net2.gradient(); - assertEquals(g1.gradient(), g2.gradient()); - // Ensure updates are equal: - MultiLayerUpdater u1 = (MultiLayerUpdater) net1.getUpdater(); - MultiLayerUpdater u2 = (MultiLayerUpdater) net2.getUpdater(); - assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray()); - u1.update(net1, g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - u2.update(net2, g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(g1.gradient(), g2.gradient()); - assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray()); - // Ensure params are equal, after fitting - net1.fit(in, labels); - net2.fit(in, labels); - INDArray p1 = net1.params(); - INDArray p2 = net2.params(); - assertEquals(p1, p2); - } - } - - @DisplayName("Compare Implementations Comp Graph") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.BidirectionalTest#params") - void compareImplementationsCompGraph(RNNFormat rnnFormat,Nd4jBackend backend) { - // for(WorkspaceMode wsm : WorkspaceMode.values()) { - for (WorkspaceMode wsm : new WorkspaceMode[] { WorkspaceMode.NONE, WorkspaceMode.ENABLED }) { - log.info("*** Starting workspace mode: " + wsm); - // Bidirectional(GravesLSTM) and GravesBidirectionalLSTM should be equivalent, given equivalent params - // Note that GravesBidirectionalLSTM implements ADD mode only - ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).updater(new Adam()).trainingWorkspaceMode(wsm).inferenceWorkspaceMode(wsm).graphBuilder().addInputs("in").layer("0", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).build()), "in").layer("1", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).build()), "0").layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).build(), "1").setOutputs("2").build(); - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).updater(new Adam()).trainingWorkspaceMode(wsm).inferenceWorkspaceMode(wsm).graphBuilder().addInputs("in").layer("0", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build(), "in").layer("1", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build(), "0").layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).build(), "1").setOutputs("2").build(); - ComputationGraph net1 = new ComputationGraph(conf1); - net1.init(); - ComputationGraph net2 = new ComputationGraph(conf2); - net2.init(); - assertEquals(net1.numParams(), net2.numParams()); - for (int i = 0; i < 3; i++) { - int n1 = (int) net1.getLayer(i).numParams(); - int n2 = (int) net2.getLayer(i).numParams(); - assertEquals(n1, n2); - } - // Assuming exact same layout here... - net2.setParams(net1.params()); - INDArray in = Nd4j.rand(new int[] { 3, 10, 5 }); - INDArray out1 = net1.outputSingle(in); - INDArray out2 = net2.outputSingle(in); - assertEquals(out1, out2); - INDArray labels = Nd4j.rand(new int[] { 3, 10, 5 }); - net1.setInput(0, in); - net1.setLabels(labels); - net2.setInput(0, in); - net2.setLabels(labels); - net1.computeGradientAndScore(); - net2.computeGradientAndScore(); - // Ensure scores are equal: - assertEquals(net1.score(), net2.score(), 1e-6); - // Ensure gradients are equal: - Gradient g1 = net1.gradient(); - Gradient g2 = net2.gradient(); - assertEquals(g1.gradient(), g2.gradient()); - // Ensure updates are equal: - ComputationGraphUpdater u1 = net1.getUpdater(); - ComputationGraphUpdater u2 = net2.getUpdater(); - assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray()); - u1.update(g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - u2.update(g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - assertEquals(g1.gradient(), g2.gradient()); - assertEquals(u1.getUpdaterStateViewArray(), u2.getUpdaterStateViewArray()); - // Ensure params are equal, after fitting - net1.fit(new DataSet(in, labels)); - net2.fit(new DataSet(in, labels)); - INDArray p1 = net1.params(); - INDArray p2 = net2.params(); - assertEquals(p1, p2); - } - } - - @DisplayName("Test Serialization") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.BidirectionalTest#params") - void testSerialization(RNNFormat rnnDataFormat,Nd4jBackend backend) throws Exception { - for (WorkspaceMode wsm : WorkspaceMode.values()) { - log.info("*** Starting workspace mode: " + wsm); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).trainingWorkspaceMode(wsm).inferenceWorkspaceMode(wsm).updater(new Adam()).list().layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())).layer(new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())).layer(new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).dataFormat(rnnDataFormat).build()).build(); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - INDArray in; - INDArray labels; - long[] inshape = rnnDataFormat == NCW ? new long[] { 3, 10, 5 } : new long[] { 3, 5, 10 }; - in = Nd4j.rand(inshape); - labels = Nd4j.rand(inshape); - net1.fit(in, labels); - byte[] bytes; - try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - ModelSerializer.writeModel(net1, baos, true); - bytes = baos.toByteArray(); - } - MultiLayerNetwork net2 = ModelSerializer.restoreMultiLayerNetwork(new ByteArrayInputStream(bytes), true); - in = Nd4j.rand(inshape); - labels = Nd4j.rand(inshape); - INDArray out1 = net1.output(in); - INDArray out2 = net2.output(in); - assertEquals(out1, out2); - net1.setInput(in); - net2.setInput(in); - net1.setLabels(labels); - net2.setLabels(labels); - net1.computeGradientAndScore(); - net2.computeGradientAndScore(); - assertEquals(net1.score(), net2.score(), 1e-6); - assertEquals(net1.gradient().gradient(), net2.gradient().gradient()); - } - } - - @DisplayName("Test Serialization Comp Graph") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.BidirectionalTest#params") - void testSerializationCompGraph(RNNFormat rnnDataFormat,Nd4jBackend backend) throws Exception { - for (WorkspaceMode wsm : WorkspaceMode.values()) { - log.info("*** Starting workspace mode: " + wsm); - Nd4j.getRandom().setSeed(12345); - ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().activation(Activation.TANH).weightInit(WeightInit.XAVIER).trainingWorkspaceMode(wsm).inferenceWorkspaceMode(wsm).updater(new Adam()).graphBuilder().addInputs("in").layer("0", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()), "in").layer("1", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()), "0").layer("2", new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).dataFormat(rnnDataFormat).nIn(10).nOut(10).build(), "1").setOutputs("2").build(); - ComputationGraph net1 = new ComputationGraph(conf1); - net1.init(); - long[] inshape = (rnnDataFormat == NCW) ? new long[] { 3, 10, 5 } : new long[] { 3, 5, 10 }; - INDArray in = Nd4j.rand(inshape); - INDArray labels = Nd4j.rand(inshape); - net1.fit(new DataSet(in, labels)); - byte[] bytes; - try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - ModelSerializer.writeModel(net1, baos, true); - bytes = baos.toByteArray(); - } - ComputationGraph net2 = ModelSerializer.restoreComputationGraph(new ByteArrayInputStream(bytes), true); - in = Nd4j.rand(inshape); - labels = Nd4j.rand(inshape); - INDArray out1 = net1.outputSingle(in); - INDArray out2 = net2.outputSingle(in); - assertEquals(out1, out2); - net1.setInput(0, in); - net2.setInput(0, in); - net1.setLabels(labels); - net2.setLabels(labels); - net1.computeGradientAndScore(); - net2.computeGradientAndScore(); - assertEquals(net1.score(), net2.score(), 1e-6); - assertEquals(net1.gradient().gradient(), net2.gradient().gradient()); - } - } - - @DisplayName("Test Simple Bidirectional") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.BidirectionalTest#params") - public void testSimpleBidirectional(RNNFormat rnnDataFormat,Nd4jBackend backend) { - for (WorkspaceMode wsm : WorkspaceMode.values()) { - log.info("*** Starting workspace mode: " + wsm); - Nd4j.getRandom().setSeed(12345); - Bidirectional.Mode[] modes = new Bidirectional.Mode[] { Bidirectional.Mode.CONCAT, Bidirectional.Mode.ADD, Bidirectional.Mode.AVERAGE, Bidirectional.Mode.MUL }; - long[] inshape = rnnDataFormat == NCW ? new long[] { 3, 10, 6 } : new long[] { 3, 6, 10 }; - INDArray in = Nd4j.rand(inshape); - for (Bidirectional.Mode m : modes) { - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).weightInit(WeightInit.XAVIER).trainingWorkspaceMode(wsm).inferenceWorkspaceMode(wsm).updater(new Adam()).list().layer(new Bidirectional(m, new SimpleRnn.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build())).build(); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).weightInit(WeightInit.XAVIER).updater(new Adam()).list().layer(new SimpleRnn.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()).build(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2.clone()); - net2.init(); - MultiLayerNetwork net3 = new MultiLayerNetwork(conf2.clone()); - net3.init(); - net2.setParam("0_W", net1.getParam("0_fW")); - net2.setParam("0_RW", net1.getParam("0_fRW")); - net2.setParam("0_b", net1.getParam("0_fb")); - net3.setParam("0_W", net1.getParam("0_bW")); - net3.setParam("0_RW", net1.getParam("0_bRW")); - net3.setParam("0_b", net1.getParam("0_bb")); - INDArray inReverse = TimeSeriesUtils.reverseTimeSeries(in, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT, rnnDataFormat); - INDArray out1 = net1.output(in); - INDArray out2 = net2.output(in); - INDArray out3 = TimeSeriesUtils.reverseTimeSeries(net3.output(inReverse), LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT, rnnDataFormat); - INDArray outExp; - switch(m) { - case ADD: - outExp = out2.add(out3); - break; - case MUL: - outExp = out2.mul(out3); - break; - case AVERAGE: - outExp = out2.add(out3).muli(0.5); - break; - case CONCAT: - outExp = Nd4j.concat((rnnDataFormat == NCW) ? 1 : 2, out2, out3); - break; - default: - throw new RuntimeException(); - } - assertEquals(outExp, out1,m.toString()); - // Check gradients: - if (m == Bidirectional.Mode.ADD || m == Bidirectional.Mode.CONCAT) { - INDArray eps = Nd4j.rand(inshape); - INDArray eps1; - if (m == Bidirectional.Mode.CONCAT) { - eps1 = Nd4j.concat((rnnDataFormat == NCW) ? 1 : 2, eps, eps); - } else { - eps1 = eps; - } - net1.setInput(in); - net2.setInput(in); - net3.setInput(TimeSeriesUtils.reverseTimeSeries(in, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT, rnnDataFormat)); - net1.feedForward(true, false); - net2.feedForward(true, false); - net3.feedForward(true, false); - Pair p1 = net1.backpropGradient(eps1, LayerWorkspaceMgr.noWorkspaces()); - Pair p2 = net2.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces()); - Pair p3 = net3.backpropGradient(TimeSeriesUtils.reverseTimeSeries(eps, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT, rnnDataFormat), LayerWorkspaceMgr.noWorkspaces()); - Gradient g1 = p1.getFirst(); - Gradient g2 = p2.getFirst(); - Gradient g3 = p3.getFirst(); - for (boolean updates : new boolean[] { false, true }) { - if (updates) { - net1.getUpdater().update(net1, g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - net2.getUpdater().update(net2, g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - net3.getUpdater().update(net3, g3, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - } - assertEquals(g2.gradientForVariable().get("0_W"), g1.gradientForVariable().get("0_fW")); - assertEquals(g2.gradientForVariable().get("0_RW"), g1.gradientForVariable().get("0_fRW")); - assertEquals(g2.gradientForVariable().get("0_b"), g1.gradientForVariable().get("0_fb")); - assertEquals(g3.gradientForVariable().get("0_W"), g1.gradientForVariable().get("0_bW")); - assertEquals(g3.gradientForVariable().get("0_RW"), g1.gradientForVariable().get("0_bRW")); - assertEquals(g3.gradientForVariable().get("0_b"), g1.gradientForVariable().get("0_bb")); - } - } - } - } - } - - @DisplayName("Test Simple Bidirectional Comp Graph") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.BidirectionalTest#params") - void testSimpleBidirectionalCompGraph(RNNFormat rnnDataFormat,Nd4jBackend backend) { - for (WorkspaceMode wsm : WorkspaceMode.values()) { - log.info("*** Starting workspace mode: " + wsm); - Nd4j.getRandom().setSeed(12345); - Bidirectional.Mode[] modes = new Bidirectional.Mode[] { Bidirectional.Mode.CONCAT, Bidirectional.Mode.ADD, Bidirectional.Mode.AVERAGE, Bidirectional.Mode.MUL }; - long[] inshape = rnnDataFormat == NCW ? new long[] { 3, 10, 6 } : new long[] { 3, 6, 10 }; - INDArray in = Nd4j.rand(inshape); - for (Bidirectional.Mode m : modes) { - ComputationGraphConfiguration conf1 = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).weightInit(WeightInit.XAVIER).trainingWorkspaceMode(wsm).inferenceWorkspaceMode(wsm).updater(new Adam()).graphBuilder().addInputs("in").layer("0", new Bidirectional(m, new SimpleRnn.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build()), "in").setOutputs("0").build(); - ComputationGraph net1 = new ComputationGraph(conf1); - net1.init(); - ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).weightInit(WeightInit.XAVIER).updater(new Adam()).graphBuilder().addInputs("in").layer("0", new SimpleRnn.Builder().nIn(10).nOut(10).dataFormat(rnnDataFormat).build(), "in").setOutputs("0").build(); - ComputationGraph net2 = new ComputationGraph(conf2.clone()); - net2.init(); - ComputationGraph net3 = new ComputationGraph(conf2.clone()); - net3.init(); - net2.setParam("0_W", net1.getParam("0_fW")); - net2.setParam("0_RW", net1.getParam("0_fRW")); - net2.setParam("0_b", net1.getParam("0_fb")); - net3.setParam("0_W", net1.getParam("0_bW")); - net3.setParam("0_RW", net1.getParam("0_bRW")); - net3.setParam("0_b", net1.getParam("0_bb")); - INDArray out1 = net1.outputSingle(in); - INDArray out2 = net2.outputSingle(in); - INDArray out3; - INDArray inReverse; - if (rnnDataFormat == RNNFormat.NWC) { - inReverse = TimeSeriesUtils.reverseTimeSeries(in.permute(0, 2, 1), LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT).permute(0, 2, 1); - out3 = net3.outputSingle(inReverse); - out3 = TimeSeriesUtils.reverseTimeSeries(out3.permute(0, 2, 1), LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT).permute(0, 2, 1); - } else { - inReverse = TimeSeriesUtils.reverseTimeSeries(in, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT); - out3 = net3.outputSingle(inReverse); - out3 = TimeSeriesUtils.reverseTimeSeries(out3, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT); - } - INDArray outExp; - switch(m) { - case ADD: - outExp = out2.add(out3); - break; - case MUL: - outExp = out2.mul(out3); - break; - case AVERAGE: - outExp = out2.add(out3).muli(0.5); - break; - case CONCAT: - System.out.println(out2.shapeInfoToString()); - System.out.println(out3.shapeInfoToString()); - outExp = Nd4j.concat((rnnDataFormat == NCW) ? 1 : 2, out2, out3); - break; - default: - throw new RuntimeException(); - } - assertEquals(outExp, out1,m.toString()); - // Check gradients: - if (m == Bidirectional.Mode.ADD || m == Bidirectional.Mode.CONCAT) { - INDArray eps = Nd4j.rand(inshape); - INDArray eps1; - if (m == Bidirectional.Mode.CONCAT) { - eps1 = Nd4j.concat((rnnDataFormat == NCW) ? 1 : 2, eps, eps); - } else { - eps1 = eps; - } - INDArray epsReversed = (rnnDataFormat == NCW) ? TimeSeriesUtils.reverseTimeSeries(eps, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT) : TimeSeriesUtils.reverseTimeSeries(eps.permute(0, 2, 1), LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT).permute(0, 2, 1); - net1.outputSingle(true, false, in); - net2.outputSingle(true, false, in); - net3.outputSingle(true, false, inReverse); - Gradient g1 = net1.backpropGradient(eps1); - Gradient g2 = net2.backpropGradient(eps); - Gradient g3 = net3.backpropGradient(epsReversed); - for (boolean updates : new boolean[] { false, true }) { - if (updates) { - net1.getUpdater().update(g1, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - net2.getUpdater().update(g2, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - net3.getUpdater().update(g3, 0, 0, 3, LayerWorkspaceMgr.noWorkspaces()); - } - assertEquals(g2.gradientForVariable().get("0_W"), g1.gradientForVariable().get("0_fW")); - assertEquals(g2.gradientForVariable().get("0_RW"), g1.gradientForVariable().get("0_fRW")); - assertEquals(g2.gradientForVariable().get("0_b"), g1.gradientForVariable().get("0_fb")); - assertEquals(g3.gradientForVariable().get("0_W"), g1.gradientForVariable().get("0_bW")); - assertEquals(g3.gradientForVariable().get("0_RW"), g1.gradientForVariable().get("0_bRW")); - assertEquals(g3.gradientForVariable().get("0_b"), g1.gradientForVariable().get("0_bb")); - } - } - } - } - } - - @DisplayName("Test Issue 5472") - @MethodSource("org.deeplearning4j.nn.layers.recurrent.BidirectionalTest#params") - @ParameterizedTest - void testIssue5472(RNNFormat rnnDataFormat,Nd4jBackend backend) { - // https://github.com/eclipse/deeplearning4j/issues/5472 - int in = 2; - int out = 2; - ComputationGraphConfiguration.GraphBuilder builder = new NeuralNetConfiguration.Builder().updater(new Adam(0.01)).activation(Activation.RELU).graphBuilder().addInputs("IN").setInputTypes(InputType.recurrent(in)).addLayer("AUTOENCODER", new VariationalAutoencoder.Builder().encoderLayerSizes(64).decoderLayerSizes(64).nOut(7).pzxActivationFunction(Activation.IDENTITY).reconstructionDistribution(new BernoulliReconstructionDistribution(Activation.SIGMOID.getActivationFunction())).build(), "IN").addLayer("RNN", new Bidirectional(Bidirectional.Mode.ADD, new GravesLSTM.Builder().nOut(128).build()), "AUTOENCODER").addLayer("OUT", new RnnOutputLayer.Builder().nOut(out).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "RNN").setOutputs("OUT"); - ComputationGraph net = new ComputationGraph(builder.build()); - net.init(); - MultiDataSetIterator iterator = new SingletonMultiDataSetIterator(new MultiDataSet(Nd4j.create(10, in, 5), Nd4j.create(10, out, 5))); - EarlyStoppingConfiguration.Builder b = new EarlyStoppingConfiguration.Builder<>().epochTerminationConditions(new MaxEpochsTerminationCondition(10)).scoreCalculator(new DataSetLossCalculator(iterator, true)).evaluateEveryNEpochs(1).modelSaver(new InMemoryModelSaver<>()); - EarlyStoppingGraphTrainer earlyStoppingGraphTrainer = new EarlyStoppingGraphTrainer(b.build(), net, iterator, null); - earlyStoppingGraphTrainer.fit(); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTMTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTMTest.java deleted file mode 100644 index 1463d21a5..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesBidirectionalLSTMTest.java +++ /dev/null @@ -1,379 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.recurrent; - -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.CacheMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.RNNFormat; -import org.deeplearning4j.nn.conf.distribution.UniformDistribution; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.params.GravesBidirectionalLSTMParamInitializer; -import org.deeplearning4j.nn.params.GravesLSTMParamInitializer; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.primitives.Pair; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.activations.impl.ActivationSigmoid; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.Nd4jBackend; -import org.nd4j.linalg.learning.config.AdaGrad; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.*; - -@DisplayName("Graves Bidirectional LSTM Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class GravesBidirectionalLSTMTest extends BaseDL4JTest { - - private double score = 0.0; - - - - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(RNNFormat rnnFormat : RNNFormat.values()) { - args.add(Arguments.of(rnnFormat,nd4jBackend)); - } - } - return args.stream(); - } - - @DisplayName("Test Bidirectional LSTM Graves Forward Basic") - @MethodSource("org.deeplearning4j.nn.layers.recurrent.GravesBidirectionalLSTMTest#params") - @ParameterizedTest - void testBidirectionalLSTMGravesForwardBasic(RNNFormat rnnDataFormat,Nd4jBackend backend) { - // Very basic test of forward prop. of LSTM layer with a time series. - // Essentially make sure it doesn't throw any exceptions, and provides output in the correct shape. - int nIn = 13; - int nHiddenUnits = 17; - final NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn).nOut(nHiddenUnits).dataFormat(rnnDataFormat).activation(Activation.TANH).build()).build(); - val numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - final GravesBidirectionalLSTM layer = (GravesBidirectionalLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - // Data: has shape [miniBatchSize,nIn,timeSeriesLength]; - // Output/activations has shape [miniBatchsize,nHiddenUnits,timeSeriesLength]; - if (rnnDataFormat == RNNFormat.NCW) { - final INDArray dataSingleExampleTimeLength1 = Nd4j.ones(1, nIn, 1); - final INDArray activations1 = layer.activate(dataSingleExampleTimeLength1, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations1.shape(), new long[] { 1, nHiddenUnits, 1 }); - final INDArray dataMultiExampleLength1 = Nd4j.ones(10, nIn, 1); - final INDArray activations2 = layer.activate(dataMultiExampleLength1, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations2.shape(), new long[] { 10, nHiddenUnits, 1 }); - final INDArray dataSingleExampleLength12 = Nd4j.ones(1, nIn, 12); - final INDArray activations3 = layer.activate(dataSingleExampleLength12, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations3.shape(), new long[] { 1, nHiddenUnits, 12 }); - final INDArray dataMultiExampleLength15 = Nd4j.ones(10, nIn, 15); - final INDArray activations4 = layer.activate(dataMultiExampleLength15, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations4.shape(), new long[] { 10, nHiddenUnits, 15 }); - } else { - final INDArray dataSingleExampleTimeLength1 = Nd4j.ones(1, 1, nIn); - final INDArray activations1 = layer.activate(dataSingleExampleTimeLength1, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations1.shape(), new long[] { 1, 1, nHiddenUnits }); - final INDArray dataMultiExampleLength1 = Nd4j.ones(10, 1, nIn); - final INDArray activations2 = layer.activate(dataMultiExampleLength1, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations2.shape(), new long[] { 10, 1, nHiddenUnits }); - final INDArray dataSingleExampleLength12 = Nd4j.ones(1, 12, nIn); - final INDArray activations3 = layer.activate(dataSingleExampleLength12, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations3.shape(), new long[] { 1, 12, nHiddenUnits }); - final INDArray dataMultiExampleLength15 = Nd4j.ones(10, 15, nIn); - final INDArray activations4 = layer.activate(dataMultiExampleLength15, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations4.shape(), new long[] { 10, 15, nHiddenUnits }); - } - } - - @DisplayName("Test Bidirectional LSTM Graves Backward Basic") - @MethodSource("org.deeplearning4j.nn.layers.recurrent.GravesBidirectionalLSTMTest#params") - @ParameterizedTest - void testBidirectionalLSTMGravesBackwardBasic(RNNFormat rnnDataFormat,Nd4jBackend backend) { - // Very basic test of backprop for mini-batch + time series - // Essentially make sure it doesn't throw any exceptions, and provides output in the correct shape. - testGravesBackwardBasicHelper(rnnDataFormat,13, 3, 17, 10, 7); - // Edge case: miniBatchSize = 1 - testGravesBackwardBasicHelper(rnnDataFormat,13, 3, 17, 1, 7); - // Edge case: timeSeriesLength = 1 - testGravesBackwardBasicHelper(rnnDataFormat,13, 3, 17, 10, 1); - // Edge case: both miniBatchSize = 1 and timeSeriesLength = 1 - testGravesBackwardBasicHelper(rnnDataFormat,13, 3, 17, 1, 1); - } - - private void testGravesBackwardBasicHelper(RNNFormat rnnDataFormat,int nIn, int nOut, int lstmNHiddenUnits, int miniBatchSize, int timeSeriesLength) { - INDArray inputData = (rnnDataFormat == RNNFormat.NCW) ? Nd4j.ones(miniBatchSize, nIn, timeSeriesLength) : Nd4j.ones(miniBatchSize, timeSeriesLength, nIn); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn).nOut(lstmNHiddenUnits).dataFormat(rnnDataFormat).dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()).build(); - long numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - GravesBidirectionalLSTM lstm = (GravesBidirectionalLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - lstm.setBackpropGradientsViewArray(Nd4j.create(1, conf.getLayer().initializer().numParams(conf))); - // Set input, do a forward pass: - lstm.activate(inputData, false, LayerWorkspaceMgr.noWorkspaces()); - assertNotNull(lstm.input()); - INDArray epsilon = (rnnDataFormat == RNNFormat.NCW) ? Nd4j.ones(miniBatchSize, lstmNHiddenUnits, timeSeriesLength) : Nd4j.ones(miniBatchSize, timeSeriesLength, lstmNHiddenUnits); - Pair out = lstm.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); - Gradient outGradient = out.getFirst(); - INDArray nextEpsilon = out.getSecond(); - INDArray biasGradientF = outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS); - INDArray inWeightGradientF = outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS); - INDArray recurrentWeightGradientF = outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS); - assertNotNull(biasGradientF); - assertNotNull(inWeightGradientF); - assertNotNull(recurrentWeightGradientF); - INDArray biasGradientB = outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS); - INDArray inWeightGradientB = outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS); - INDArray recurrentWeightGradientB = outGradient.getGradientFor(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS); - assertNotNull(biasGradientB); - assertNotNull(inWeightGradientB); - assertNotNull(recurrentWeightGradientB); - assertArrayEquals(biasGradientF.shape(), new long[] { 1, 4 * lstmNHiddenUnits }); - assertArrayEquals(inWeightGradientF.shape(), new long[] { nIn, 4 * lstmNHiddenUnits }); - assertArrayEquals(recurrentWeightGradientF.shape(), new long[] { lstmNHiddenUnits, 4 * lstmNHiddenUnits + 3 }); - assertArrayEquals(biasGradientB.shape(), new long[] { 1, 4 * lstmNHiddenUnits }); - assertArrayEquals(inWeightGradientB.shape(), new long[] { nIn, 4 * lstmNHiddenUnits }); - assertArrayEquals(recurrentWeightGradientB.shape(), new long[] { lstmNHiddenUnits, 4 * lstmNHiddenUnits + 3 }); - assertNotNull(nextEpsilon); - if (rnnDataFormat == RNNFormat.NCW) { - assertArrayEquals(nextEpsilon.shape(), new long[] { miniBatchSize, nIn, timeSeriesLength }); - } else { - assertArrayEquals(nextEpsilon.shape(), new long[] { miniBatchSize, timeSeriesLength, nIn }); - } - // Check update: - for (String s : outGradient.gradientForVariable().keySet()) { - lstm.update(outGradient.getGradientFor(s), s); - } - } - - @DisplayName("Test Graves Bidirectional LSTM Forward Pass Helper") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.GravesBidirectionalLSTMTest#params") - void testGravesBidirectionalLSTMForwardPassHelper(RNNFormat rnnDataFormat,Nd4jBackend backend) throws Exception { - // GravesBidirectionalLSTM.activateHelper() has different behaviour (due to optimizations) when forBackprop==true vs false - // But should otherwise provide identical activations - Nd4j.getRandom().setSeed(12345); - final int nIn = 10; - final int layerSize = 15; - final int miniBatchSize = 4; - final int timeSeriesLength = 7; - final NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn).nOut(layerSize).dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()).build(); - long numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - final GravesBidirectionalLSTM lstm = (GravesBidirectionalLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - final INDArray input = Nd4j.rand(new int[] { miniBatchSize, nIn, timeSeriesLength }); - lstm.setInput(input, LayerWorkspaceMgr.noWorkspaces()); - final INDArray fwdPassFalse = LSTMHelpers.activateHelper(lstm, lstm.conf(), new ActivationSigmoid(), lstm.input(), lstm.getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS), lstm.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS), lstm.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS), false, null, null, false, true, GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS, null, true, null, CacheMode.NONE, LayerWorkspaceMgr.noWorkspaces(), true).fwdPassOutput; - final INDArray[] fwdPassTrue = LSTMHelpers.activateHelper(lstm, lstm.conf(), new ActivationSigmoid(), lstm.input(), lstm.getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS), lstm.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS), lstm.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS), false, null, null, true, true, GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS, null, true, null, CacheMode.NONE, LayerWorkspaceMgr.noWorkspaces(), true).fwdPassOutputAsArrays; - // I have no idea what the heck this does --Ben - for (int i = 0; i < timeSeriesLength; i++) { - final INDArray sliceFalse = fwdPassFalse.tensorAlongDimension(i, 1, 0); - final INDArray sliceTrue = fwdPassTrue[i]; - assertTrue(sliceFalse.equals(sliceTrue)); - } - } - - static private void reverseColumnsInPlace(final INDArray x) { - final long N = x.size(1); - final INDArray x2 = x.dup(); - for (int t = 0; t < N; t++) { - final long b = N - t - 1; - // clone? - x.putColumn(t, x2.getColumn(b)); - } - } - - @DisplayName("Test Get Set Params") - @MethodSource("org.deeplearning4j.nn.layers.recurrent.GravesBidirectionalLSTMTest#params") - @ParameterizedTest - void testGetSetParmas(RNNFormat rnnDataFormat,Nd4jBackend backend) { - final int nIn = 2; - final int layerSize = 3; - final int miniBatchSize = 2; - final int timeSeriesLength = 10; - Nd4j.getRandom().setSeed(12345); - final NeuralNetConfiguration confBidirectional = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn).nOut(layerSize).dataFormat(rnnDataFormat).dist(new UniformDistribution(-0.1, 0.1)).activation(Activation.TANH).build()).build(); - long numParams = confBidirectional.getLayer().initializer().numParams(confBidirectional); - INDArray params = Nd4j.create(1, numParams); - final GravesBidirectionalLSTM bidirectionalLSTM = (GravesBidirectionalLSTM) confBidirectional.getLayer().instantiate(confBidirectional, null, 0, params, true, params.dataType()); - final INDArray sig = (rnnDataFormat == RNNFormat.NCW) ? Nd4j.rand(new int[] { miniBatchSize, nIn, timeSeriesLength }) : Nd4j.rand(new int[] { miniBatchSize, timeSeriesLength, nIn }); - final INDArray act1 = bidirectionalLSTM.activate(sig, false, LayerWorkspaceMgr.noWorkspaces()); - params = bidirectionalLSTM.params(); - bidirectionalLSTM.setParams(params); - final INDArray act2 = bidirectionalLSTM.activate(sig, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(act2.data().asDouble(), act1.data().asDouble(), 1e-8); - } - - @DisplayName("Test Simple Forwards And Backwards Activation") - @MethodSource("org.deeplearning4j.nn.layers.recurrent.GravesBidirectionalLSTMTest#params") - @ParameterizedTest - void testSimpleForwardsAndBackwardsActivation(RNNFormat rnnDataFormat,Nd4jBackend backend) { - final int nIn = 2; - final int layerSize = 3; - final int miniBatchSize = 1; - final int timeSeriesLength = 5; - Nd4j.getRandom().setSeed(12345); - final NeuralNetConfiguration confBidirectional = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().nIn(nIn).nOut(layerSize).dataFormat(rnnDataFormat).dist(new UniformDistribution(-0.1, 0.1)).activation(Activation.TANH).updater(new NoOp()).build()).build(); - final NeuralNetConfiguration confForwards = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(layerSize).dataFormat(rnnDataFormat).weightInit(WeightInit.ZERO).activation(Activation.TANH).build()).build(); - long numParams = confForwards.getLayer().initializer().numParams(confForwards); - INDArray params = Nd4j.create(1, numParams); - long numParamsBD = confBidirectional.getLayer().initializer().numParams(confBidirectional); - INDArray paramsBD = Nd4j.create(1, numParamsBD); - final GravesBidirectionalLSTM bidirectionalLSTM = (GravesBidirectionalLSTM) confBidirectional.getLayer().instantiate(confBidirectional, null, 0, paramsBD, true, params.dataType()); - final GravesLSTM forwardsLSTM = (GravesLSTM) confForwards.getLayer().instantiate(confForwards, null, 0, params, true, params.dataType()); - bidirectionalLSTM.setBackpropGradientsViewArray(Nd4j.create(1, confBidirectional.getLayer().initializer().numParams(confBidirectional))); - forwardsLSTM.setBackpropGradientsViewArray(Nd4j.create(1, confForwards.getLayer().initializer().numParams(confForwards))); - final INDArray sig = (rnnDataFormat == RNNFormat.NCW) ? Nd4j.rand(new int[] { miniBatchSize, nIn, timeSeriesLength }) : Nd4j.rand(new int[] { miniBatchSize, timeSeriesLength, nIn }); - final INDArray sigb = sig.dup(); - if (rnnDataFormat == RNNFormat.NCW) { - reverseColumnsInPlace(sigb.slice(0)); - } else { - reverseColumnsInPlace(sigb.slice(0).permute(1, 0)); - } - final INDArray recurrentWeightsF = bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS); - final INDArray inputWeightsF = bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS); - final INDArray biasWeightsF = bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS); - final INDArray recurrentWeightsF2 = forwardsLSTM.getParam(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY); - final INDArray inputWeightsF2 = forwardsLSTM.getParam(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY); - final INDArray biasWeightsF2 = forwardsLSTM.getParam(GravesLSTMParamInitializer.BIAS_KEY); - // assert that the forwards part of the bidirectional layer is equal to that of the regular LSTM - assertArrayEquals(recurrentWeightsF2.shape(), recurrentWeightsF.shape()); - assertArrayEquals(inputWeightsF2.shape(), inputWeightsF.shape()); - assertArrayEquals(biasWeightsF2.shape(), biasWeightsF.shape()); - forwardsLSTM.setParam(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY, recurrentWeightsF); - forwardsLSTM.setParam(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY, inputWeightsF); - forwardsLSTM.setParam(GravesLSTMParamInitializer.BIAS_KEY, biasWeightsF); - // copy forwards weights to make the forwards activations do the same thing - final INDArray recurrentWeightsB = bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS); - final INDArray inputWeightsB = bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS); - final INDArray biasWeightsB = bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS); - // assert that the forwards and backwards are the same shapes - assertArrayEquals(recurrentWeightsF.shape(), recurrentWeightsB.shape()); - assertArrayEquals(inputWeightsF.shape(), inputWeightsB.shape()); - assertArrayEquals(biasWeightsF.shape(), biasWeightsB.shape()); - // zero out backwards layer - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS, Nd4j.zeros(recurrentWeightsB.shape())); - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS, Nd4j.zeros(inputWeightsB.shape())); - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS, Nd4j.zeros(biasWeightsB.shape())); - forwardsLSTM.setInput(sig, LayerWorkspaceMgr.noWorkspaces()); - // compare activations - final INDArray activation1 = forwardsLSTM.activate(sig, false, LayerWorkspaceMgr.noWorkspaces()).slice(0); - final INDArray activation2 = bidirectionalLSTM.activate(sig, false, LayerWorkspaceMgr.noWorkspaces()).slice(0); - assertArrayEquals(activation1.data().asFloat(), activation2.data().asFloat(), 1e-5f); - final INDArray randSig = (rnnDataFormat == RNNFormat.NCW) ? Nd4j.rand(new int[] { 1, layerSize, timeSeriesLength }) : Nd4j.rand(new int[] { 1, timeSeriesLength, layerSize }); - INDArray randSigBackwards = randSig.dup(); - if (rnnDataFormat == RNNFormat.NCW) { - reverseColumnsInPlace(randSigBackwards.slice(0)); - } else { - reverseColumnsInPlace(randSigBackwards.slice(0).permute(1, 0)); - } - final Pair backprop1 = forwardsLSTM.backpropGradient(randSig, LayerWorkspaceMgr.noWorkspaces()); - final Pair backprop2 = bidirectionalLSTM.backpropGradient(randSig, LayerWorkspaceMgr.noWorkspaces()); - // compare gradients - assertArrayEquals(backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY).dup().data().asFloat(), backprop2.getFirst().getGradientFor(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS).dup().data().asFloat(), 1e-5f); - assertArrayEquals(backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY).dup().data().asFloat(), backprop2.getFirst().getGradientFor(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS).dup().data().asFloat(), 1e-5f); - assertArrayEquals(backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.BIAS_KEY).dup().data().asFloat(), backprop2.getFirst().getGradientFor(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS).dup().data().asFloat(), 1e-5f); - // copy forwards to backwards - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS, bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS)); - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS, bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS)); - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS, bidirectionalLSTM.getParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS)); - // zero out forwards layer - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_FORWARDS, Nd4j.zeros(recurrentWeightsB.shape())); - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_FORWARDS, Nd4j.zeros(inputWeightsB.shape())); - bidirectionalLSTM.setParam(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_FORWARDS, Nd4j.zeros(biasWeightsB.shape())); - // run on reversed signal - final INDArray activation3 = bidirectionalLSTM.activate(sigb, false, LayerWorkspaceMgr.noWorkspaces()).slice(0); - final INDArray activation3Reverse = activation3.dup(); - if (rnnDataFormat == RNNFormat.NCW) { - reverseColumnsInPlace(activation3Reverse); - } else { - reverseColumnsInPlace(activation3Reverse.permute(1, 0)); - } - assertArrayEquals(activation3Reverse.shape(), activation1.shape()); - assertEquals(activation3Reverse, activation1); - // test backprop now - final INDArray refBackGradientReccurrent = backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY); - final INDArray refBackGradientInput = backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY); - final INDArray refBackGradientBias = backprop1.getFirst().getGradientFor(GravesLSTMParamInitializer.BIAS_KEY); - // reverse weights only with backwards signal should yield same result as forwards weights with forwards signal - final Pair backprop3 = bidirectionalLSTM.backpropGradient(randSigBackwards, LayerWorkspaceMgr.noWorkspaces()); - final INDArray backGradientRecurrent = backprop3.getFirst().getGradientFor(GravesBidirectionalLSTMParamInitializer.RECURRENT_WEIGHT_KEY_BACKWARDS); - final INDArray backGradientInput = backprop3.getFirst().getGradientFor(GravesBidirectionalLSTMParamInitializer.INPUT_WEIGHT_KEY_BACKWARDS); - final INDArray backGradientBias = backprop3.getFirst().getGradientFor(GravesBidirectionalLSTMParamInitializer.BIAS_KEY_BACKWARDS); - assertArrayEquals(refBackGradientBias.dup().data().asDouble(), backGradientBias.dup().data().asDouble(), 1e-6); - assertArrayEquals(refBackGradientInput.dup().data().asDouble(), backGradientInput.dup().data().asDouble(), 1e-6); - assertArrayEquals(refBackGradientReccurrent.dup().data().asDouble(), backGradientRecurrent.dup().data().asDouble(), 1e-6); - final INDArray refEpsilon = backprop1.getSecond().dup(); - final INDArray backEpsilon = backprop3.getSecond().dup(); - if (rnnDataFormat == RNNFormat.NCW) { - reverseColumnsInPlace(refEpsilon.slice(0)); - } else { - reverseColumnsInPlace(refEpsilon.slice(0).permute(1, 0)); - } - assertArrayEquals(backEpsilon.dup().data().asDouble(), refEpsilon.dup().data().asDouble(), 1e-6); - } - - @MethodSource("org.deeplearning4j.nn.layers.recurrent.GravesBidirectionalLSTMTest#params") - @DisplayName("Test Serialization") - @ParameterizedTest - void testSerialization(RNNFormat rnnDataFormat,Nd4jBackend backend) { - final MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new AdaGrad(0.1)).l2(0.001).seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).dist(new UniformDistribution(-0.05, 0.05)).build()).layer(1, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).dist(new UniformDistribution(-0.05, 0.05)).build()).layer(2, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(2).build()).build(); - final String json1 = conf1.toJson(); - final MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json1); - final String json2 = conf1.toJson(); - assertEquals(json1, json2); - } - - @DisplayName("Test Gate Activation Fns Sanity Check") - @MethodSource("org.deeplearning4j.nn.layers.recurrent.GravesBidirectionalLSTMTest#params") - @ParameterizedTest - void testGateActivationFnsSanityCheck(RNNFormat rnnDataFormat,Nd4jBackend backend) { - for (String gateAfn : new String[] { "sigmoid", "hardsigmoid" }) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder().gateActivationFunction(gateAfn).activation(Activation.TANH).nIn(2).nOut(2).dataFormat(rnnDataFormat).build()).layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(2).dataFormat(rnnDataFormat).activation(Activation.TANH).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(gateAfn, ((org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM) net.getLayer(0).conf().getLayer()).getGateActivationFn().toString()); - INDArray in = Nd4j.rand(new int[] { 3, 2, 5 }); - INDArray labels = Nd4j.rand(new int[] { 3, 2, 5 }); - if (rnnDataFormat == RNNFormat.NWC) { - in = in.permute(0, 2, 1); - labels = labels.permute(0, 2, 1); - } - net.fit(in, labels); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTMTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTMTest.java deleted file mode 100644 index 8b324fcf0..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/GravesLSTMTest.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.recurrent; - -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.common.config.DL4JClassLoading; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.UniformDistribution; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.params.GravesLSTMParamInitializer; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.INDArrayIndex; -import org.nd4j.linalg.indexing.NDArrayIndex; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.primitives.Pair; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.List; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Graves LSTM Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class GravesLSTMTest extends BaseDL4JTest { - - @Test - @DisplayName("Test LSTM Graves Forward Basic") - void testLSTMGravesForwardBasic() { - // Very basic test of forward prop. of LSTM layer with a time series. - // Essentially make sure it doesn't throw any exceptions, and provides output in the correct shape. - int nIn = 13; - int nHiddenUnits = 17; - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(nHiddenUnits).activation(Activation.TANH).build()).build(); - val numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - GravesLSTM layer = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - // Data: has shape [miniBatchSize,nIn,timeSeriesLength]; - // Output/activations has shape [miniBatchsize,nHiddenUnits,timeSeriesLength]; - INDArray dataSingleExampleTimeLength1 = Nd4j.ones(1, nIn, 1); - INDArray activations1 = layer.activate(dataSingleExampleTimeLength1, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations1.shape(), new long[] { 1, nHiddenUnits, 1 }); - INDArray dataMultiExampleLength1 = Nd4j.ones(10, nIn, 1); - INDArray activations2 = layer.activate(dataMultiExampleLength1, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations2.shape(), new long[] { 10, nHiddenUnits, 1 }); - INDArray dataSingleExampleLength12 = Nd4j.ones(1, nIn, 12); - INDArray activations3 = layer.activate(dataSingleExampleLength12, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations3.shape(), new long[] { 1, nHiddenUnits, 12 }); - INDArray dataMultiExampleLength15 = Nd4j.ones(10, nIn, 15); - INDArray activations4 = layer.activate(dataMultiExampleLength15, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(activations4.shape(), new long[] { 10, nHiddenUnits, 15 }); - } - - @Test - @DisplayName("Test LSTM Graves Backward Basic") - void testLSTMGravesBackwardBasic() { - // Very basic test of backprop for mini-batch + time series - // Essentially make sure it doesn't throw any exceptions, and provides output in the correct shape. - testGravesBackwardBasicHelper(13, 3, 17, 10, 7); - // Edge case: miniBatchSize = 1 - testGravesBackwardBasicHelper(13, 3, 17, 1, 7); - // Edge case: timeSeriesLength = 1 - testGravesBackwardBasicHelper(13, 3, 17, 10, 1); - // Edge case: both miniBatchSize = 1 and timeSeriesLength = 1 - testGravesBackwardBasicHelper(13, 3, 17, 1, 1); - } - - private static void testGravesBackwardBasicHelper(int nIn, int nOut, int lstmNHiddenUnits, int miniBatchSize, int timeSeriesLength) { - INDArray inputData = Nd4j.ones(miniBatchSize, nIn, timeSeriesLength); - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(lstmNHiddenUnits).dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()).build(); - val numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - GravesLSTM lstm = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - lstm.setBackpropGradientsViewArray(Nd4j.create(1, conf.getLayer().initializer().numParams(conf))); - // Set input, do a forward pass: - lstm.activate(inputData, false, LayerWorkspaceMgr.noWorkspaces()); - assertNotNull(lstm.input()); - INDArray epsilon = Nd4j.ones(miniBatchSize, lstmNHiddenUnits, timeSeriesLength); - Pair out = lstm.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); - Gradient outGradient = out.getFirst(); - INDArray nextEpsilon = out.getSecond(); - INDArray biasGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.BIAS_KEY); - INDArray inWeightGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY); - INDArray recurrentWeightGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY); - assertNotNull(biasGradient); - assertNotNull(inWeightGradient); - assertNotNull(recurrentWeightGradient); - assertArrayEquals(biasGradient.shape(), new long[] { 1, 4 * lstmNHiddenUnits }); - assertArrayEquals(inWeightGradient.shape(), new long[] { nIn, 4 * lstmNHiddenUnits }); - assertArrayEquals(recurrentWeightGradient.shape(), new long[] { lstmNHiddenUnits, 4 * lstmNHiddenUnits + 3 }); - assertNotNull(nextEpsilon); - assertArrayEquals(nextEpsilon.shape(), new long[] { miniBatchSize, nIn, timeSeriesLength }); - // Check update: - for (String s : outGradient.gradientForVariable().keySet()) { - lstm.update(outGradient.getGradientFor(s), s); - } - } - - @Test - @DisplayName("Test Graves LSTM Forward Pass Helper") - void testGravesLSTMForwardPassHelper() throws Exception { - // GravesLSTM.activateHelper() has different behaviour (due to optimizations) when forBackprop==true vs false - // But should otherwise provide identical activations - Nd4j.getRandom().setSeed(12345); - int nIn = 10; - int layerSize = 15; - int miniBatchSize = 4; - int timeSeriesLength = 7; - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(layerSize).dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()).build(); - val numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - GravesLSTM lstm = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - INDArray input = Nd4j.rand(new int[] { miniBatchSize, nIn, timeSeriesLength }); - lstm.setInput(input, LayerWorkspaceMgr.noWorkspaces()); - Method actHelper = GravesLSTM.class.getDeclaredMethod("activateHelper", boolean.class, INDArray.class, INDArray.class, boolean.class, LayerWorkspaceMgr.class); - actHelper.setAccessible(true); - // Call activateHelper with both forBackprop == true, and forBackprop == false and compare - Class innerClass = DL4JClassLoading.loadClassByName("org.deeplearning4j.nn.layers.recurrent.FwdPassReturn"); - // GravesLSTM.FwdPassReturn object; want fwdPassOutput INDArray - Object oFalse = actHelper.invoke(lstm, false, null, null, false, LayerWorkspaceMgr.noWorkspacesImmutable()); - // want fwdPassOutputAsArrays object - Object oTrue = actHelper.invoke(lstm, false, null, null, true, LayerWorkspaceMgr.noWorkspacesImmutable()); - Field fwdPassOutput = innerClass.getDeclaredField("fwdPassOutput"); - fwdPassOutput.setAccessible(true); - Field fwdPassOutputAsArrays = innerClass.getDeclaredField("fwdPassOutputAsArrays"); - fwdPassOutputAsArrays.setAccessible(true); - INDArray fwdPassFalse = (INDArray) fwdPassOutput.get(oFalse); - INDArray[] fwdPassTrue = (INDArray[]) fwdPassOutputAsArrays.get(oTrue); - for (int i = 0; i < timeSeriesLength; i++) { - INDArray sliceFalse = fwdPassFalse.tensorAlongDimension(i, 1, 0); - INDArray sliceTrue = fwdPassTrue[i]; - assertTrue(sliceFalse.equals(sliceTrue)); - } - } - - @Test - @DisplayName("Test Single Example") - void testSingleExample() { - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().activation(Activation.TANH).nIn(2).nOut(2).build()).layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(1).activation(Activation.TANH).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray in1 = Nd4j.rand(new int[] { 1, 2, 4 }); - INDArray in2 = Nd4j.rand(new int[] { 1, 2, 5 }); - in2.put(new INDArrayIndex[] { NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4) }, in1); - assertEquals(in1, in2.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4))); - INDArray labels1 = Nd4j.rand(new int[] { 1, 1, 4 }); - INDArray labels2 = Nd4j.create(1, 1, 5); - labels2.put(new INDArrayIndex[] { NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4) }, labels1); - assertEquals(labels1, labels2.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.interval(0, 4))); - INDArray out1 = net.output(in1); - INDArray out2 = net.output(in2); - // System.out.println(Arrays.toString(net.output(in1).data().asFloat())); - // System.out.println(Arrays.toString(net.output(in2).data().asFloat())); - List activations1 = net.feedForward(in1); - List activations2 = net.feedForward(in2); - // for (int i = 0; i < 3; i++) { - // System.out.println("-----\n" + i); - // System.out.println(Arrays.toString(activations1.get(i).dup().data().asDouble())); - // System.out.println(Arrays.toString(activations2.get(i).dup().data().asDouble())); - // - // System.out.println(activations1.get(i)); - // System.out.println(activations2.get(i)); - // } - // Expect first 4 time steps to be indentical... - for (int i = 0; i < 4; i++) { - double d1 = out1.getDouble(i); - double d2 = out2.getDouble(i); - assertEquals(d1, d2, 0.0); - } - } - - @Test - @DisplayName("Test Gate Activation Fns Sanity Check") - void testGateActivationFnsSanityCheck() { - for (String gateAfn : new String[] { "sigmoid", "hardsigmoid" }) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345).list().layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().gateActivationFunction(gateAfn).activation(Activation.TANH).nIn(2).nOut(2).build()).layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(2).activation(Activation.TANH).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(gateAfn, ((org.deeplearning4j.nn.conf.layers.GravesLSTM) net.getLayer(0).conf().getLayer()).getGateActivationFn().toString()); - INDArray in = Nd4j.rand(new int[] { 3, 2, 5 }); - INDArray labels = Nd4j.rand(new int[] { 3, 2, 5 }); - net.fit(in, labels); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayerTest.java deleted file mode 100644 index 08e4455d0..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/recurrent/MaskZeroLayerTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.layers.recurrent; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.RNNFormat; -import org.deeplearning4j.nn.conf.layers.LSTM; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.deeplearning4j.optimize.api.TrainingListener; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.Nd4jBackend; -import org.nd4j.linalg.indexing.NDArrayIndex; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Mask Zero Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class MaskZeroLayerTest extends BaseDL4JTest { - - - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(RNNFormat rnnFormat : RNNFormat.values()) { - args.add(Arguments.of(rnnFormat,nd4jBackend)); - } - } - return args.stream(); - } - - - @DisplayName("Activate") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.MaskZeroLayerTest#params") - void activate(RNNFormat rnnDataFormat,Nd4jBackend backend) { - // GIVEN two examples where some of the timesteps are zero. - INDArray ex1 = Nd4j.create(new double[][] { new double[] { 0, 3, 5 }, new double[] { 0, 0, 2 } }); - INDArray ex2 = Nd4j.create(new double[][] { new double[] { 0, 0, 2 }, new double[] { 0, 0, 2 } }); - // A LSTM which adds one for every non-zero timestep - org.deeplearning4j.nn.conf.layers.LSTM underlying = new org.deeplearning4j.nn.conf.layers.LSTM.Builder().activation(Activation.IDENTITY).gateActivationFunction(Activation.IDENTITY).nIn(2).nOut(1).dataFormat(rnnDataFormat).build(); - NeuralNetConfiguration conf = new NeuralNetConfiguration(); - conf.setLayer(underlying); - INDArray params = Nd4j.zeros(new int[] { 1, 16 }); - // Set the biases to 1. - for (int i = 12; i < 16; i++) { - params.putScalar(i, 1.0); - } - Layer lstm = underlying.instantiate(conf, Collections.emptyList(), 0, params, false, params.dataType()); - double maskingValue = 0.0; - MaskZeroLayer l = new MaskZeroLayer(lstm, maskingValue); - INDArray input = Nd4j.create(Arrays.asList(ex1, ex2), new int[] { 2, 2, 3 }); - if (rnnDataFormat == RNNFormat.NWC) { - input = input.permute(0, 2, 1); - } - // WHEN - INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces()); - if (rnnDataFormat == RNNFormat.NWC) { - out = out.permute(0, 2, 1); - } - // THEN output should only be incremented for the non-zero timesteps - INDArray firstExampleOutput = out.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.all()); - INDArray secondExampleOutput = out.get(NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.all()); - assertEquals(0.0, firstExampleOutput.getDouble(0), 1e-6); - assertEquals(1.0, firstExampleOutput.getDouble(1), 1e-6); - assertEquals(2.0, firstExampleOutput.getDouble(2), 1e-6); - assertEquals(0.0, secondExampleOutput.getDouble(0), 1e-6); - assertEquals(0.0, secondExampleOutput.getDouble(1), 1e-6); - assertEquals(1.0, secondExampleOutput.getDouble(2), 1e-6); - } - - - @DisplayName("Test Serialization") - @ParameterizedTest - @MethodSource("org.deeplearning4j.nn.layers.recurrent.MaskZeroLayerTest#params") - void testSerialization(RNNFormat rnnDataFormat,Nd4jBackend backend) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new org.deeplearning4j.nn.conf.layers.util.MaskZeroLayer.Builder().setMaskValue(0.0).setUnderlying(new LSTM.Builder().nIn(4).nOut(5).dataFormat(rnnDataFormat).build()).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - TestUtils.testModelSerialization(net); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/SameDiffCustomLayerTests.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/SameDiffCustomLayerTests.java deleted file mode 100644 index 6c821f866..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/samediff/SameDiffCustomLayerTests.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.layers.samediff; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.graph.GraphVertex; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.inputs.InvalidInputTypeException; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.samediff.SDLayerParams; -import org.deeplearning4j.nn.conf.layers.samediff.SDVertexParams; -import org.deeplearning4j.nn.conf.layers.samediff.SameDiffVertex; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.autodiff.samediff.SDVariable; -import org.nd4j.autodiff.samediff.SameDiff; -import org.nd4j.common.base.Preconditions; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.nativeblas.NativeOpsHolder; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -@Slf4j -@NativeTag -@Tag(TagNames.SAMEDIFF) -@Tag(TagNames.CUSTOM_FUNCTIONALITY) -public class SameDiffCustomLayerTests extends BaseDL4JTest { - private DataType initialType; - - - @BeforeEach - public void before() { - Nd4j.create(1); - initialType = Nd4j.dataType(); - - Nd4j.setDataType(DataType.DOUBLE); - Nd4j.getRandom().setSeed(123); - } - - @AfterEach - public void after() { - Nd4j.setDataType(initialType); - - NativeOpsHolder.getInstance().getDeviceNativeOps().enableDebugMode(false); - NativeOpsHolder.getInstance().getDeviceNativeOps().enableVerboseMode(false); - } - - @Test - public void testInputValidationSameDiffLayer(){ - assertThrows(IllegalArgumentException.class,() -> { - final MultiLayerConfiguration config = new NeuralNetConfiguration.Builder().list() - .layer(new ValidatingSameDiffLayer()) - .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.SIGMOID).nOut(2).build()) - .setInputType(InputType.feedForward(2)) - .build(); - - final MultiLayerNetwork net = new MultiLayerNetwork(config); - net.init(); - - final INDArray goodInput = Nd4j.rand(1, 2); - final INDArray badInput = Nd4j.rand(2, 2); - - net.fit(goodInput, goodInput); - net.fit(badInput, badInput); - - - }); - - } - - @Test - public void testInputValidationSameDiffVertex(){ - assertThrows(IllegalArgumentException.class,() -> { - final ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder().graphBuilder() - .addVertex("a", new ValidatingSameDiffVertex(), "input") - .addLayer("output", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.SIGMOID).nOut(2).build(), "a") - .addInputs("input") - .setInputTypes(InputType.feedForward(2)) - .setOutputs("output") - .build(); - - final ComputationGraph net = new ComputationGraph(config); - net.init(); - - final INDArray goodInput = Nd4j.rand(1, 2); - final INDArray badInput = Nd4j.rand(2, 2); - - net.fit(new INDArray[]{goodInput}, new INDArray[]{goodInput}); - net.fit(new INDArray[]{badInput}, new INDArray[]{badInput}); - }); - - } - - private class ValidatingSameDiffLayer extends org.deeplearning4j.nn.conf.layers.samediff.SameDiffLayer { - @Override - public void validateInput(INDArray input) { - Preconditions.checkArgument(input.size(0) < 2, "Expected Message"); - } - - @Override - public SDVariable defineLayer(SameDiff sameDiff, SDVariable layerInput, Map paramTable, SDVariable mask) { - return layerInput; - } - - @Override - public void defineParameters(SDLayerParams params) { } - - @Override - public void initializeParameters(Map params) { } - - @Override - public InputType getOutputType(int layerIndex, InputType inputType) { return inputType; } - } - - private class ValidatingSameDiffVertex extends SameDiffVertex { - @Override - public GraphVertex clone() { - return new ValidatingSameDiffVertex(); - } - - @Override - public InputType getOutputType(int layerIndex, InputType... vertexInputs) throws InvalidInputTypeException { - return vertexInputs[0]; - } - - @Override - public void validateInput(INDArray[] input) { - Preconditions.checkArgument(input[0].size(0) < 2, "Expected Message"); - } - - @Override - public SDVariable defineVertex(SameDiff sameDiff, Map layerInput, Map paramTable, Map maskVars) { - return layerInput.get("input"); - } - - @Override - public void defineParametersAndInputs(SDVertexParams params) { - params.defineInputs("input"); - } - - @Override - public void initializeParameters(Map params) {} - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/LargeNetTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/LargeNetTest.java deleted file mode 100644 index 8e99809cb..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/misc/LargeNetTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.misc; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.EmbeddingLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataBuffer; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Disabled -@DisplayName("Large Net Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.FILE_IO) -@Tag(TagNames.WORKSPACES) -class LargeNetTest extends BaseDL4JTest { - - @Disabled - @Test - @DisplayName("Test Large Multi Layer Network") - void testLargeMultiLayerNetwork() { - Nd4j.setDataType(DataType.FLOAT); - // More than 2.1 billion parameters - // 10M classes plus 300 vector size -> 3 billion elements - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new EmbeddingLayer.Builder().nIn(10_000_000).nOut(300).build()).layer(new OutputLayer.Builder().nIn(300).nOut(10).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray params = net.params(); - long paramsLength = params.length(); - long expParamsLength = 10_000_000L * 300 + 300 * 10 + 10; - assertEquals(expParamsLength, paramsLength); - long[] expW = new long[] { 10_000_000, 300 }; - assertArrayEquals(expW, net.getParam("0_W").shape()); - long[] expW1 = new long[] { 300, 10 }; - assertArrayEquals(expW1, net.getParam("1_W").shape()); - long[] expB1 = new long[] { 1, 10 }; - assertArrayEquals(expB1, net.getParam("1_b").shape()); - } - - @Disabled - @Test - @DisplayName("Test Large Comp Graph") - void testLargeCompGraph() { - Nd4j.setDataType(DataType.FLOAT); - // More than 2.1 billion parameters - // 10M classes plus 300 vector size -> 3 billion elements - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").layer("0", new EmbeddingLayer.Builder().nIn(10_000_000).nOut(300).build(), "in").layer("1", new OutputLayer.Builder().nIn(300).nOut(10).activation(Activation.SOFTMAX).build(), "0").setOutputs("1").build(); - ComputationGraph net = new ComputationGraph(conf); - net.init(); - INDArray params = net.params(); - long paramsLength = params.length(); - long expParamsLength = 10_000_000L * 300 + 300 * 10 + 10; - assertEquals(expParamsLength, paramsLength); - long[] expW = new long[] { 10_000_000, 300 }; - assertArrayEquals(expW, net.getParam("0_W").shape()); - long[] expW1 = new long[] { 300, 10 }; - assertArrayEquals(expW1, net.getParam("1_W").shape()); - long[] expB1 = new long[] { 1, 10 }; - assertArrayEquals(expB1, net.getParam("1_b").shape()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/BackPropMLPTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/BackPropMLPTest.java deleted file mode 100644 index 7995ebd26..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/BackPropMLPTest.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.multilayer; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.iter.NdIndexIterator; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.transforms.strict.SigmoidDerivative; -import org.nd4j.linalg.api.ops.impl.transforms.strict.TanhDerivative; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.exception.ND4JArraySizeException; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; -import org.nd4j.linalg.ops.transforms.Transforms; -import java.util.Arrays; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.fail; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Back Prop MLP Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class BackPropMLPTest extends BaseDL4JTest { - - @Test - @DisplayName("Test MLP Trivial") - void testMLPTrivial() { - // Simplest possible case: 1 hidden layer, 1 hidden neuron, batch size of 1. - MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(new int[] { 1 }, Activation.SIGMOID)); - network.setListeners(new ScoreIterationListener(1)); - network.init(); - DataSetIterator iter = new IrisDataSetIterator(1, 10); - while (iter.hasNext()) network.fit(iter.next()); - } - - @Test - @DisplayName("Test MLP") - void testMLP() { - // Simple mini-batch test with multiple hidden layers - MultiLayerConfiguration conf = getIrisMLPSimpleConfig(new int[] { 5, 4, 3 }, Activation.SIGMOID); - // System.out.println(conf); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - DataSetIterator iter = new IrisDataSetIterator(10, 100); - while (iter.hasNext()) { - network.fit(iter.next()); - } - } - - @Test - @DisplayName("Test MLP 2") - void testMLP2() { - // Simple mini-batch test with multiple hidden layers - MultiLayerConfiguration conf = getIrisMLPSimpleConfig(new int[] { 5, 15, 3 }, Activation.TANH); - // System.out.println(conf); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - DataSetIterator iter = new IrisDataSetIterator(12, 120); - while (iter.hasNext()) { - network.fit(iter.next()); - } - } - - @Test - @DisplayName("Test Single Example Weight Updates") - void testSingleExampleWeightUpdates() { - // Simplest possible case: 1 hidden layer, 1 hidden neuron, batch size of 1. - // Manually calculate weight updates (entirely outside of DL4J and ND4J) - // and compare expected and actual weights after backprop - DataSetIterator iris = new IrisDataSetIterator(1, 10); - MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(new int[] { 1 }, Activation.SIGMOID)); - network.init(); - Layer[] layers = network.getLayers(); - final boolean printCalculations = false; - while (iris.hasNext()) { - DataSet data = iris.next(); - INDArray x = data.getFeatures(); - INDArray y = data.getLabels(); - float[] xFloat = asFloat(x); - float[] yFloat = asFloat(y); - // Do forward pass: - // Hidden layer - INDArray l1Weights = layers[0].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); - // Output layer - INDArray l2Weights = layers[1].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); - INDArray l1Bias = layers[0].getParam(DefaultParamInitializer.BIAS_KEY).dup(); - INDArray l2Bias = layers[1].getParam(DefaultParamInitializer.BIAS_KEY).dup(); - float[] l1WeightsFloat = asFloat(l1Weights); - float[] l2WeightsFloat = asFloat(l2Weights); - float l1BiasFloat = l1Bias.getFloat(0); - float[] l2BiasFloatArray = asFloat(l2Bias); - // z=w*x+b - float hiddenUnitPreSigmoid = dotProduct(l1WeightsFloat, xFloat) + l1BiasFloat; - // a=sigma(z) - float hiddenUnitPostSigmoid = sigmoid(hiddenUnitPreSigmoid); - float[] outputPreSoftmax = new float[3]; - // Normally a matrix multiplication here, but only one hidden unit in this trivial example - for (int i = 0; i < 3; i++) { - outputPreSoftmax[i] = hiddenUnitPostSigmoid * l2WeightsFloat[i] + l2BiasFloatArray[i]; - } - float[] outputPostSoftmax = softmax(outputPreSoftmax); - // Do backward pass: - // out-labels - float[] deltaOut = vectorDifference(outputPostSoftmax, yFloat); - // deltaHidden = sigmaPrime(hiddenUnitZ) * sum_k (w_jk * \delta_k); here, only one j - float deltaHidden = 0.0f; - for (int i = 0; i < 3; i++) deltaHidden += l2WeightsFloat[i] * deltaOut[i]; - deltaHidden *= derivOfSigmoid(hiddenUnitPreSigmoid); - // Calculate weight/bias updates: - // dL/dW = delta * (activation of prev. layer) - // dL/db = delta - float[] dLdwOut = new float[3]; - for (int i = 0; i < dLdwOut.length; i++) dLdwOut[i] = deltaOut[i] * hiddenUnitPostSigmoid; - float[] dLdwHidden = new float[4]; - for (int i = 0; i < dLdwHidden.length; i++) dLdwHidden[i] = deltaHidden * xFloat[i]; - float[] dLdbOut = deltaOut; - float dLdbHidden = deltaHidden; - if (printCalculations) { - System.out.println("deltaOut = " + Arrays.toString(deltaOut)); - System.out.println("deltaHidden = " + deltaHidden); - System.out.println("dLdwOut = " + Arrays.toString(dLdwOut)); - System.out.println("dLdbOut = " + Arrays.toString(dLdbOut)); - System.out.println("dLdwHidden = " + Arrays.toString(dLdwHidden)); - System.out.println("dLdbHidden = " + dLdbHidden); - } - // Calculate new parameters: - // w_i = w_i - (learningRate)/(batchSize) * sum_j (dL_j/dw_i) - // b_i = b_i - (learningRate)/(batchSize) * sum_j (dL_j/db_i) - // Which for batch size of one (here) is simply: - // w_i = w_i - learningRate * dL/dW - // b_i = b_i - learningRate * dL/db - float[] expectedL1WeightsAfter = new float[4]; - float[] expectedL2WeightsAfter = new float[3]; - float expectedL1BiasAfter = l1BiasFloat - 0.1f * dLdbHidden; - float[] expectedL2BiasAfter = new float[3]; - for (int i = 0; i < 4; i++) expectedL1WeightsAfter[i] = l1WeightsFloat[i] - 0.1f * dLdwHidden[i]; - for (int i = 0; i < 3; i++) expectedL2WeightsAfter[i] = l2WeightsFloat[i] - 0.1f * dLdwOut[i]; - for (int i = 0; i < 3; i++) expectedL2BiasAfter[i] = l2BiasFloatArray[i] - 0.1f * dLdbOut[i]; - // Finally, do back-prop on network, and compare parameters vs. expected parameters - network.fit(data); - /* INDArray l1WeightsAfter = layers[0].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); //Hidden layer - INDArray l2WeightsAfter = layers[1].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); //Output layer - INDArray l1BiasAfter = layers[0].getParam(DefaultParamInitializer.BIAS_KEY).dup(); - INDArray l2BiasAfter = layers[1].getParam(DefaultParamInitializer.BIAS_KEY).dup(); - float[] l1WeightsFloatAfter = asFloat(l1WeightsAfter); - float[] l2WeightsFloatAfter = asFloat(l2WeightsAfter); - float l1BiasFloatAfter = l1BiasAfter.getFloat(0); - float[] l2BiasFloatAfter = asFloat(l2BiasAfter); - - if( printCalculations) { - System.out.println("Expected L1 weights = " + Arrays.toString(expectedL1WeightsAfter)); - System.out.println("Actual L1 weights = " + Arrays.toString(asFloat(l1WeightsAfter))); - System.out.println("Expected L2 weights = " + Arrays.toString(expectedL2WeightsAfter)); - System.out.println("Actual L2 weights = " + Arrays.toString(asFloat(l2WeightsAfter))); - System.out.println("Expected L1 bias = " + expectedL1BiasAfter); - System.out.println("Actual L1 bias = " + Arrays.toString(asFloat(l1BiasAfter))); - System.out.println("Expected L2 bias = " + Arrays.toString(expectedL2BiasAfter)); - System.out.println("Actual L2 bias = " + Arrays.toString(asFloat(l2BiasAfter))); - } - - - float eps = 1e-4f; - assertArrayEquals(l1WeightsFloatAfter,expectedL1WeightsAfter,eps); - assertArrayEquals(l2WeightsFloatAfter,expectedL2WeightsAfter,eps); - assertEquals(l1BiasFloatAfter,expectedL1BiasAfter,eps); - assertArrayEquals(l2BiasFloatAfter,expectedL2BiasAfter,eps); - */ - // System.out.println("\n\n--------------"); - } - } - - @Test - @DisplayName("Test MLP Gradient Calculation") - void testMLPGradientCalculation() { - testIrisMiniBatchGradients(1, new int[] { 1 }, Activation.SIGMOID); - testIrisMiniBatchGradients(1, new int[] { 5 }, Activation.SIGMOID); - testIrisMiniBatchGradients(12, new int[] { 15, 25, 10 }, Activation.SIGMOID); - testIrisMiniBatchGradients(50, new int[] { 10, 50, 200, 50, 10 }, Activation.TANH); - testIrisMiniBatchGradients(150, new int[] { 30, 50, 20 }, Activation.TANH); - } - - private static void testIrisMiniBatchGradients(int miniBatchSize, int[] hiddenLayerSizes, Activation activationFunction) { - int totalExamples = 10 * miniBatchSize; - if (totalExamples > 150) { - totalExamples = miniBatchSize * (150 / miniBatchSize); - } - if (miniBatchSize > 150) { - fail(); - } - DataSetIterator iris = new IrisDataSetIterator(miniBatchSize, totalExamples); - MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(hiddenLayerSizes, Activation.SIGMOID)); - network.init(); - Layer[] layers = network.getLayers(); - int nLayers = layers.length; - while (iris.hasNext()) { - DataSet data = iris.next(); - INDArray x = data.getFeatures(); - INDArray y = data.getLabels(); - // Do forward pass: - INDArray[] layerWeights = new INDArray[nLayers]; - INDArray[] layerBiases = new INDArray[nLayers]; - for (int i = 0; i < nLayers; i++) { - layerWeights[i] = layers[i].getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); - layerBiases[i] = layers[i].getParam(DefaultParamInitializer.BIAS_KEY).dup(); - } - INDArray[] layerZs = new INDArray[nLayers]; - INDArray[] layerActivations = new INDArray[nLayers]; - for (int i = 0; i < nLayers; i++) { - INDArray layerInput = (i == 0 ? x : layerActivations[i - 1]); - layerZs[i] = layerInput.castTo(layerWeights[i].dataType()).mmul(layerWeights[i]).addiRowVector(layerBiases[i]); - layerActivations[i] = (i == nLayers - 1 ? doSoftmax(layerZs[i].dup()) : doSigmoid(layerZs[i].dup())); - } - // Do backward pass: - INDArray[] deltas = new INDArray[nLayers]; - // Out - labels; shape=[miniBatchSize,nOut]; - deltas[nLayers - 1] = layerActivations[nLayers - 1].sub(y.castTo(layerActivations[nLayers - 1].dataType())); - assertArrayEquals(deltas[nLayers - 1].shape(), new long[] { miniBatchSize, 3 }); - for (int i = nLayers - 2; i >= 0; i--) { - INDArray sigmaPrimeOfZ; - sigmaPrimeOfZ = doSigmoidDerivative(layerZs[i]); - INDArray epsilon = layerWeights[i + 1].mmul(deltas[i + 1].transpose()).transpose(); - deltas[i] = epsilon.mul(sigmaPrimeOfZ); - assertArrayEquals(deltas[i].shape(), new long[] { miniBatchSize, hiddenLayerSizes[i] }); - } - INDArray[] dLdw = new INDArray[nLayers]; - INDArray[] dLdb = new INDArray[nLayers]; - for (int i = 0; i < nLayers; i++) { - INDArray prevActivations = (i == 0 ? x : layerActivations[i - 1]); - // Raw gradients, so not yet divided by mini-batch size (division is done in BaseUpdater) - // Shape: [nIn, nOut] - dLdw[i] = deltas[i].transpose().castTo(prevActivations.dataType()).mmul(prevActivations).transpose(); - // Shape: [1,nOut] - dLdb[i] = deltas[i].sum(true, 0); - int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]); - int nOut = (i < nLayers - 1 ? hiddenLayerSizes[i] : 3); - assertArrayEquals(dLdw[i].shape(), new long[] { nIn, nOut }); - assertArrayEquals(dLdb[i].shape(), new long[] { 1, nOut }); - } - // Calculate and get gradient, compare to expected - network.setInput(x); - network.setLabels(y); - network.computeGradientAndScore(); - Gradient gradient = network.gradientAndScore().getFirst(); - float eps = 1e-4f; - for (int i = 0; i < hiddenLayerSizes.length; i++) { - String wKey = i + "_" + DefaultParamInitializer.WEIGHT_KEY; - String bKey = i + "_" + DefaultParamInitializer.BIAS_KEY; - INDArray wGrad = gradient.getGradientFor(wKey); - INDArray bGrad = gradient.getGradientFor(bKey); - float[] wGradf = asFloat(wGrad); - float[] bGradf = asFloat(bGrad); - float[] expWGradf = asFloat(dLdw[i]); - float[] expBGradf = asFloat(dLdb[i]); - assertArrayEquals(wGradf, expWGradf, eps); - assertArrayEquals(bGradf, expBGradf, eps); - } - } - } - - /** - * Very simple back-prop config set up for Iris. - * Learning Rate = 0.1 - * No regularization, no Adagrad, no momentum etc. One iteration. - */ - private static MultiLayerConfiguration getIrisMLPSimpleConfig(int[] hiddenLayerSizes, Activation activationFunction) { - NeuralNetConfiguration.ListBuilder lb = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).seed(12345L).list(); - for (int i = 0; i < hiddenLayerSizes.length; i++) { - int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]); - lb.layer(i, new DenseLayer.Builder().nIn(nIn).nOut(hiddenLayerSizes[i]).weightInit(WeightInit.XAVIER).activation(activationFunction).build()); - } - lb.layer(hiddenLayerSizes.length, new OutputLayer.Builder(LossFunction.MCXENT).nIn(hiddenLayerSizes[hiddenLayerSizes.length - 1]).nOut(3).weightInit(WeightInit.XAVIER).activation(activationFunction.equals(Activation.IDENTITY) ? Activation.IDENTITY : Activation.SOFTMAX).build()); - return lb.build(); - } - - public static float[] asFloat(INDArray arr) { - long len = arr.length(); - if (len > Integer.MAX_VALUE) - throw new ND4JArraySizeException(); - float[] f = new float[(int) len]; - NdIndexIterator iterator = new NdIndexIterator('c', arr.shape()); - for (int i = 0; i < len; i++) { - f[i] = arr.getFloat(iterator.next()); - } - return f; - } - - public static float dotProduct(float[] x, float[] y) { - float sum = 0.0f; - for (int i = 0; i < x.length; i++) sum += x[i] * y[i]; - return sum; - } - - public static float sigmoid(float in) { - return (float) (1.0 / (1.0 + Math.exp(-in))); - } - - public static float[] sigmoid(float[] in) { - float[] out = new float[in.length]; - for (int i = 0; i < in.length; i++) { - out[i] = sigmoid(in[i]); - } - return out; - } - - public static float derivOfSigmoid(float in) { - // float v = (float)( Math.exp(in) / Math.pow(1+Math.exp(in),2.0) ); - float v = in * (1 - in); - return v; - } - - public static float[] derivOfSigmoid(float[] in) { - float[] out = new float[in.length]; - for (int i = 0; i < in.length; i++) { - out[i] = derivOfSigmoid(in[i]); - } - return out; - } - - public static float[] softmax(float[] in) { - float[] out = new float[in.length]; - float sumExp = 0.0f; - for (int i = 0; i < in.length; i++) { - sumExp += Math.exp(in[i]); - } - for (int i = 0; i < in.length; i++) { - out[i] = (float) Math.exp(in[i]) / sumExp; - } - return out; - } - - public static float[] vectorDifference(float[] x, float[] y) { - float[] out = new float[x.length]; - for (int i = 0; i < x.length; i++) { - out[i] = x[i] - y[i]; - } - return out; - } - - public static INDArray doSoftmax(INDArray input) { - return Transforms.softmax(input, true); - } - - public static INDArray doSigmoid(INDArray input) { - return Transforms.sigmoid(input, true); - } - - public static INDArray doSigmoidDerivative(INDArray input) { - return Nd4j.getExecutioner().exec(new SigmoidDerivative(input.dup())); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java deleted file mode 100644 index 80661a17c..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/MultiLayerTest.java +++ /dev/null @@ -1,1029 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.multilayer; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.SingletonMultiDataSetIterator; -import org.deeplearning4j.eval.Evaluation; -import org.deeplearning4j.exception.DL4JException; -import org.deeplearning4j.nn.api.Model; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.*; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.objdetect.Yolo2OutputLayer; -import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.RnnToCnnPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.RnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.gradient.DefaultGradient; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.layers.BaseOutputLayer; -import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.deeplearning4j.nn.transferlearning.TransferLearning; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.deeplearning4j.optimize.api.BaseTrainingListener; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.deeplearning4j.util.ModelSerializer; -import org.junit.jupiter.api.*;import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.*; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.executioner.OpExecutioner; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.SplitTestAndTrain; -import org.nd4j.linalg.dataset.api.MultiDataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.heartbeat.Heartbeat; -import org.nd4j.linalg.heartbeat.reports.Environment; -import org.nd4j.linalg.heartbeat.reports.Event; -import org.nd4j.linalg.heartbeat.reports.Task; -import org.nd4j.linalg.heartbeat.utils.EnvironmentUtils; -import org.nd4j.linalg.heartbeat.utils.TaskUtils; -import org.nd4j.linalg.indexing.NDArrayIndex; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.primitives.Pair; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.*; -import static org.junit.jupiter.api.Assertions.*; - -import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.jupiter.api.Assertions.assertThrows; - -@Slf4j -@DisplayName("Multi Layer Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -public class MultiLayerTest extends BaseDL4JTest { - - private static OpExecutioner.ProfilingMode origMode; - - @BeforeAll - static void beforeClass() { - origMode = Nd4j.getExecutioner().getProfilingMode(); - } - - @BeforeEach - void before() { - Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC); - } - - @AfterAll - static void afterClass() { - Nd4j.getExecutioner().setProfilingMode(origMode); - } - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Test - @DisplayName("Test Set Params") - void testSetParams() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.TANH).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()).build(); - MultiLayerNetwork network3 = new MultiLayerNetwork(conf); - network3.init(); - INDArray params = network3.params(); - INDArray weights = network3.getLayer(0).getParam(DefaultParamInitializer.WEIGHT_KEY).dup(); - INDArray bias = network3.getLayer(0).getParam(DefaultParamInitializer.BIAS_KEY).dup(); - network3.setParameters(params); - assertEquals(weights, network3.getLayer(0).getParam(DefaultParamInitializer.WEIGHT_KEY)); - assertEquals(bias, network3.getLayer(0).getParam(DefaultParamInitializer.BIAS_KEY)); - INDArray params4 = network3.params(); - assertEquals(params, params4); - } - - @Test - @DisplayName("Test Batch Norm") - void testBatchNorm() { - Nd4j.getRandom().setSeed(123); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(2, new BatchNormalization.Builder().nOut(2).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - network.setListeners(new ScoreIterationListener(1)); - DataSetIterator iter = new IrisDataSetIterator(150, 150); - DataSet next = iter.next(); - next.normalizeZeroMeanZeroUnitVariance(); - SplitTestAndTrain trainTest = next.splitTestAndTrain(110); - network.setLabels(trainTest.getTrain().getLabels()); - network.init(); - for (int i = 0; i < 5; i++) { - network.fit(trainTest.getTrain()); - } - } - - @Test - @DisplayName("Test Back Prop") - void testBackProp() { - Nd4j.getRandom().setSeed(123); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(2, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - network.setListeners(new ScoreIterationListener(1)); - DataSetIterator iter = new IrisDataSetIterator(150, 150); - DataSet next = iter.next(); - next.normalizeZeroMeanZeroUnitVariance(); - SplitTestAndTrain trainTest = next.splitTestAndTrain(110); - network.setInput(trainTest.getTrain().getFeatures()); - network.setLabels(trainTest.getTrain().getLabels()); - network.init(); - for (int i = 0; i < 5; i++) { - network.fit(trainTest.getTrain()); - } - DataSet test = trainTest.getTest(); - Evaluation eval = new Evaluation(); - INDArray output = network.output(test.getFeatures()); - eval.eval(test.getLabels(), output); - log.info("Score " + eval.stats()); - } - - @Test - @DisplayName("Test Gradient With As List") - void testGradientWithAsList() { - MultiLayerNetwork net1 = new MultiLayerNetwork(getConf()); - MultiLayerNetwork net2 = new MultiLayerNetwork(getConf()); - net1.init(); - net2.init(); - DataSet x1 = new IrisDataSetIterator(1, 150).next(); - DataSet all = new IrisDataSetIterator(150, 150).next(); - DataSet x2 = all.asList().get(0); - // x1 and x2 contain identical data - assertArrayEquals(asFloat(x1.getFeatures()), asFloat(x2.getFeatures()), 0.0f); - assertArrayEquals(asFloat(x1.getLabels()), asFloat(x2.getLabels()), 0.0f); - assertEquals(x1, x2); - // Set inputs/outputs so gradient can be calculated: - net1.feedForward(x1.getFeatures()); - net2.feedForward(x2.getFeatures()); - ((BaseOutputLayer) net1.getLayer(1)).setLabels(x1.getLabels()); - ((BaseOutputLayer) net2.getLayer(1)).setLabels(x2.getLabels()); - net1.gradient(); - net2.gradient(); - } - - /** - * This test intended only to test activateSelectedLayers method, it does not involves fully-working AutoEncoder. - */ - @Test - @DisplayName("Test Selected Activations") - void testSelectedActivations() { - // Train DeepAutoEncoder on very limited trainset - final int numRows = 28; - final int numColumns = 28; - int seed = 123; - int numSamples = 3; - int iterations = 1; - int listenerFreq = iterations / 5; - log.info("Load data...."); - float[][] trainingData = new float[numSamples][numColumns * numRows]; - Arrays.fill(trainingData[0], 0.95f); - Arrays.fill(trainingData[1], 0.5f); - Arrays.fill(trainingData[2], 0.05f); - log.info("Build model...."); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed).optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list().layer(0, new DenseLayer.Builder().nIn(numRows * numColumns).nOut(1000).build()).layer(1, new DenseLayer.Builder().nIn(1000).nOut(500).build()).layer(2, new DenseLayer.Builder().nIn(500).nOut(250).build()).layer(3, new DenseLayer.Builder().nIn(250).nOut(100).build()).layer(4, // encoding stops - new DenseLayer.Builder().nIn(100).nOut(30).build()).layer(5, // decoding starts - new DenseLayer.Builder().nIn(30).nOut(100).build()).layer(6, new DenseLayer.Builder().nIn(100).nOut(250).build()).layer(7, new DenseLayer.Builder().nIn(250).nOut(500).build()).layer(8, new DenseLayer.Builder().nIn(500).nOut(1000).build()).layer(9, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(1000).nOut(numRows * numColumns).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - model.addListeners(new ScoreIterationListener(listenerFreq)); - log.info("Train model...."); - int cnt = 0; - while (cnt < numSamples) { - INDArray input = Nd4j.create(trainingData[cnt]).reshape(1, -1); - model.fit(new DataSet(input, input)); - cnt++; - } - // Make two separate selective calls - log.info("Testing full cycle..."); - List comparableResult = model.feedForward(Nd4j.create(trainingData[0], new long[] { 1, trainingData[0].length })); - INDArray encodeResult = model.activateSelectedLayers(0, 4, Nd4j.create(trainingData[0], new long[] { 1, trainingData[0].length })); - log.info("Compare feedForward results with selectedActivation"); - assertEquals(comparableResult.get(5), encodeResult); - INDArray decodeResults = model.activateSelectedLayers(5, 9, encodeResult); - log.info("Decode results: " + decodeResults.columns() + " " + decodeResults); - log.info("Comparable results: " + comparableResult.get(10).columns() + " " + comparableResult.get(10)); - assertEquals(comparableResult.get(10), decodeResults); - } - - private static MultiLayerConfiguration getConf() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).dist(new NormalDistribution(0, 1)).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).dist(new NormalDistribution(0, 1)).build()).build(); - return conf; - } - - public static float[] asFloat(INDArray arr) { - long len = arr.length(); - float[] f = new float[(int) len]; - for (int i = 0; i < len; i++) f[i] = arr.getFloat(i); - return f; - } - - @Test - @DisplayName("Test Feed Forward To Layer") - void testFeedForwardToLayer() { - int nIn = 30; - int nOut = 25; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new Sgd(1e-3)).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(600).dist(new NormalDistribution(0, 1e-5)).build()).layer(1, new DenseLayer.Builder().nIn(600).nOut(250).dist(new NormalDistribution(0, 1e-5)).build()).layer(2, new DenseLayer.Builder().nIn(250).nOut(100).dist(new NormalDistribution(0, 1e-5)).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(100).nOut(25).activation(Activation.SOFTMAX).weightInit(new NormalDistribution(0, 1e-5)).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - INDArray input = Nd4j.rand(5, nIn); - List activations = network.feedForward(input); - // 4 layers + input - assertEquals(5, activations.size()); - List activationsAll = network.feedForwardToLayer(3, input); - assertEquals(activations, activationsAll); - for (int i = 3; i >= 0; i--) { - List activationsPartial = network.feedForwardToLayer(i, input); - // i+2: for layer 3: input + activations of {0,1,2,3} -> 5 total = 3+2 - assertEquals(i + 2, activationsPartial.size()); - for (int j = 0; j <= i; j++) { - INDArray exp = activationsAll.get(j); - INDArray act = activationsPartial.get(j); - assertEquals(exp, act); - } - } - } - - @Test - @DisplayName("Test Backprop Gradient") - void testBackpropGradient() { - // Testing: MultiLayerNetwork.backpropGradient() - // i.e., specifically without an output layer - int nIn = 10; - int nOut = 40; - int miniBatch = 5; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new DenseLayer.Builder().nIn(20).nOut(30).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(2, new DenseLayer.Builder().nIn(30).nOut(nOut).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - Nd4j.getRandom().setSeed(12345); - INDArray eps = Nd4j.rand(miniBatch, nOut); - INDArray input = Nd4j.rand(miniBatch, nIn); - net.setInput(input); - // Need to feed forward before backprop - net.feedForward(true, false); - Pair pair = net.backpropGradient(eps, LayerWorkspaceMgr.noWorkspaces()); - INDArray epsOut = pair.getSecond(); - assertNotNull(epsOut); - assertArrayEquals(new long[] { miniBatch, nIn }, epsOut.shape()); - Gradient g = pair.getFirst(); - Map gradMap = g.gradientForVariable(); - // 3 layers, weight + bias gradients for each - assertEquals(6, gradMap.size()); - String[] expKeys = { "0_" + DefaultParamInitializer.WEIGHT_KEY, "0_" + DefaultParamInitializer.BIAS_KEY, "1_" + DefaultParamInitializer.WEIGHT_KEY, "2_" + DefaultParamInitializer.BIAS_KEY, "2_" + DefaultParamInitializer.WEIGHT_KEY, "2_" + DefaultParamInitializer.BIAS_KEY }; - Set keys = gradMap.keySet(); - for (String s : expKeys) { - assertTrue(keys.contains(s)); - } - /* - System.out.println(pair); - - //Use updater to go from raw gradients -> updates - //Apply learning rate, gradient clipping, adagrad/momentum/rmsprop etc - Updater updater = UpdaterCreator.getUpdater(net); - updater.update(net, g, 0, miniBatch); - - StepFunction stepFunction = new NegativeGradientStepFunction(); - INDArray params = net.params(); - System.out.println(Arrays.toString(params.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 10)).dup().data().asFloat())); - stepFunction.step(params, g.gradient()); - net.setParams(params); //params() may not be in-place - System.out.println(Arrays.toString(params.get(NDArrayIndex.all(), NDArrayIndex.interval(0, 10)).dup().data().asFloat())); - */ - } - - @Test - @DisplayName("Test Layer Names") - void testLayerNames() { - int nIn = 10; - int nOut = 40; - List layerNameList = new ArrayList<>(); - layerNameList.add("dnn1"); - layerNameList.add("dnn2"); - layerNameList.add("dnn3"); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).list().layer(0, new DenseLayer.Builder().name("dnn1").nIn(nIn).nOut(20).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(1, new DenseLayer.Builder().name("dnn2").nIn(20).nOut(30).activation(Activation.RELU).weightInit(WeightInit.XAVIER).build()).layer(2, new DenseLayer.Builder().name("dnn3").nIn(30).nOut(nOut).activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(layerNameList.get(0), net.getLayer(0).conf().getLayer().getLayerName()); - assertEquals(layerNameList, net.getLayerNames()); - BaseLayer b = (BaseLayer) net.getLayer(layerNameList.get(2)).conf().getLayer(); - assertEquals(b.getActivationFn().toString(), "softmax"); - } - - @Test - @DisplayName("Test Score Examples") - void testScoreExamples() { - Nd4j.getRandom().setSeed(12345); - int nIn = 5; - int nOut = 6; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()).layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()).build(); - MultiLayerConfiguration confNoReg = new NeuralNetConfiguration.Builder().seed(12345).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()).layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - MultiLayerNetwork netNoReg = new MultiLayerNetwork(confNoReg); - netNoReg.init(); - netNoReg.setParameters(net.params().dup()); - // Score single example, and compare to scoreExamples: - INDArray input = Nd4j.rand(3, nIn); - INDArray output = Nd4j.rand(3, nOut); - DataSet ds = new DataSet(input, output); - INDArray scoresWithRegularization = net.scoreExamples(ds, true); - INDArray scoresNoRegularization = net.scoreExamples(ds, false); - assertArrayEquals(new long[] { 3, 1 }, scoresWithRegularization.shape()); - assertArrayEquals(new long[] { 3, 1 }, scoresNoRegularization.shape()); - for (int i = 0; i < 3; i++) { - DataSet singleEx = new DataSet(input.getRow(i, true), output.getRow(i, true)); - double score = net.score(singleEx); - double scoreNoReg = netNoReg.score(singleEx); - double scoreUsingScoreExamples = scoresWithRegularization.getDouble(i); - double scoreUsingScoreExamplesNoReg = scoresNoRegularization.getDouble(i); - assertEquals(score, scoreUsingScoreExamples, 1e-4); - assertEquals(scoreNoReg, scoreUsingScoreExamplesNoReg, 1e-4); - // Regularization term increases score - assertTrue(scoreUsingScoreExamples > scoreUsingScoreExamplesNoReg); - // System.out.println(score + "\t" + scoreUsingScoreExamples + "\t|\t" + scoreNoReg + "\t" + scoreUsingScoreExamplesNoReg); - } - } - - @Test - @DisplayName("Test Data Set Score") - void testDataSetScore() { - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).seed(12345L).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.SIGMOID).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray in = Nd4j.create(new double[] { 1.0, 2.0, 3.0, 4.0 }, new long[] { 1, 4 }); - INDArray out = Nd4j.create(new double[] { 1, 0, 0 }, new long[] { 1, 3 }); - double score = net.score(new DataSet(in, out)); - } - - @Test - @DisplayName("Test Data Set Score CNN") - void testDataSetScoreCNN() { - int miniBatch = 3; - int depth = 2; - int width = 3; - int height = 3; - int nOut = 2; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).list().layer(0, new ConvolutionLayer.Builder(2, 2).nOut(1).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(2).build()).setInputType(InputType.convolutionalFlat(height, width, depth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - Nd4j.getRandom().setSeed(12345); - Random r = new Random(12345); - INDArray input = Nd4j.rand(miniBatch, depth * width * height); - INDArray labels = Nd4j.create(miniBatch, nOut); - for (int i = 0; i < miniBatch; i++) { - labels.putScalar(new int[] { i, r.nextInt(nOut) }, 1.0); - } - double score = net.score(new DataSet(input, labels)); - } - - @Test - @DisplayName("Test Predict") - void testPredict() throws Exception { - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).seed(12345L).list().layer(0, new DenseLayer.Builder().nIn(784).nOut(50).activation(Activation.RELU).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(50).nOut(10).build()).setInputType(InputType.convolutional(28, 28, 1)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSetIterator ds = new MnistDataSetIterator(10, 10); - net.fit(ds); - DataSetIterator testDs = new MnistDataSetIterator(1, 1); - DataSet testData = testDs.next(); - testData.setLabelNames(Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - String actualLables = testData.getLabelName(0); - List prediction = net.predict(testData); - assertTrue(actualLables != null); - assertTrue(prediction.get(0) != null); - } - - @Test - @Disabled - @DisplayName("Test Cid") - void testCid() throws Exception { - System.out.println(EnvironmentUtils.buildCId()); - Environment environment = EnvironmentUtils.buildEnvironment(); - environment.setSerialVersionID(EnvironmentUtils.buildCId()); - Task task = TaskUtils.buildTask(Nd4j.create(new double[] { 1, 2, 3, 4, 5, 6 }, new long[] { 1, 6 })); - Heartbeat.getInstance().reportEvent(Event.STANDALONE, environment, task); - Thread.sleep(25000); - } - - @Test - @DisplayName("Test Output") - void testOutput() throws Exception { - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).seed(12345L).list().layer(0, new DenseLayer.Builder().nIn(784).nOut(50).activation(Activation.RELU).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(50).nOut(10).build()).setInputType(InputType.convolutional(28, 28, 1)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSetIterator fullData = new MnistDataSetIterator(1, 2); - net.fit(fullData); - fullData.reset(); - DataSet expectedSet = fullData.next(2); - INDArray expectedOut = net.output(expectedSet.getFeatures(), false); - fullData.reset(); - INDArray actualOut = net.output(fullData); - assertEquals(expectedOut, actualOut); - } - - @Test - @DisplayName("Test Gradient Update") - void testGradientUpdate() throws Exception { - DataSetIterator iter = new IrisDataSetIterator(1, 1); - Gradient expectedGradient = new DefaultGradient(); - expectedGradient.setGradientFor("0_W", Nd4j.ones(4, 5).castTo(DataType.DOUBLE)); - expectedGradient.setGradientFor("0_b", Nd4j.ones(1, 5).castTo(DataType.DOUBLE)); - expectedGradient.setGradientFor("1_W", Nd4j.ones(5, 3).castTo(DataType.DOUBLE)); - expectedGradient.setGradientFor("1_b", Nd4j.ones(1, 3).castTo(DataType.DOUBLE)); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .updater(new Sgd(1.0)).activation(Activation.RELU) - .weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().name("dnn1").nIn(4).nOut(5).build()) - .layer(1, new OutputLayer.Builder().name("output").nIn(5).nOut(3).activation(Activation.SOFTMAX) - .weightInit(WeightInit.XAVIER).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - net.fit(iter.next()); - // TODO validate actual layer gradientView - issue getting var out of BaseLayer w/o adding MLN getter that gets confused with local gradient vars - Gradient actualGradient = net.gradient; - assertNotEquals(expectedGradient.getGradientFor("0_W"), actualGradient.getGradientFor("0_W")); - net.update(expectedGradient); - actualGradient = net.gradient; - assertEquals(expectedGradient.getGradientFor("0_W"), actualGradient.getGradientFor("0_W")); - // Update params with set - net.setParam("0_W", Nd4j.ones(4, 5).castTo(DataType.DOUBLE)); - net.setParam("0_b", Nd4j.ones(1, 5).castTo(DataType.DOUBLE)); - net.setParam("1_W", Nd4j.ones(5, 3).castTo(DataType.DOUBLE)); - net.setParam("1_b", Nd4j.ones(1, 3).castTo(DataType.DOUBLE)); - INDArray actualParams = net.params().castTo(DataType.DOUBLE); - // Confirm params - assertEquals(expectedGradient.gradient(), actualParams); - net.update(expectedGradient); - actualParams = net.params().castTo(DataType.DOUBLE); - assertEquals(Nd4j.ones(1, 43).addi(1).castTo(DataType.DOUBLE), actualParams); - } - - @Test - @DisplayName("Test Cnn Invalid Data") - void testCnnInvalidData() { - assertThrows(DL4JException.class, () -> { - int miniBatch = 3; - int depth = 2; - int width = 5; - int height = 5; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nIn(2).nOut(2).build()).layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(2).build()).setInputType(InputType.convolutional(height, width, depth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // Order: examples, channels, height, width - INDArray inputWrongDepth = Nd4j.rand(new int[] { miniBatch, 5, height, width }); - net.feedForward(inputWrongDepth); - }); - } - - @Test - @DisplayName("Test Applying Pre Train Config And Params") - void testApplyingPreTrainConfigAndParams() { - int nIn = 10; - int nOut = 10; - // Test pretrain true - MultiLayerNetwork aePre = getAeModel(true, nIn, nOut); - int actualNP = (int) aePre.numParams(); - assertEquals(2 * (nIn * nOut + nOut) + nIn, actualNP); - INDArray params = aePre.params(); - // check num params - assertEquals(params.length(), actualNP); - Map paramTable = aePre.paramTable(); - // check vb exists for pretrain layer - assertTrue(paramTable.containsKey("0_vb")); - aePre.setParam("0_vb", Nd4j.ones(10)); - params = aePre.getParam("0_vb"); - // check set params for vb - assertEquals(Nd4j.ones(1, 10), params); - // Test pretrain false, expect same for true because its not changed when applying update - MultiLayerNetwork aeNoPre = getAeModel(false, nIn, nOut); - actualNP = (int) aeNoPre.numParams(); - assertEquals(2 * (nIn * nOut + nOut) + nIn, actualNP); - params = aeNoPre.params(); - assertEquals(params.length(), actualNP); - paramTable = aePre.paramTable(); - assertTrue(paramTable.containsKey("0_vb")); - } - - public MultiLayerNetwork getAeModel(boolean preTrain, int nIn, int nOut) { - MultiLayerConfiguration vae = new NeuralNetConfiguration.Builder().seed(42).updater(new NoOp()).weightInit(WeightInit.UNIFORM).list(new AutoEncoder.Builder().activation(Activation.IDENTITY).nOut(nIn).build(), new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.COSINE_PROXIMITY).activation(Activation.IDENTITY).nOut(nOut).build()).setInputType(InputType.feedForward(nOut)).build(); - MultiLayerNetwork network = new MultiLayerNetwork(vae); - network.init(); - return network; - } - - @Test - @DisplayName("Test Iteration Count And Persistence") - void testIterationCountAndPersistence() throws IOException { - Nd4j.getRandom().setSeed(123); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build(); - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - DataSetIterator iter = new IrisDataSetIterator(50, 150); - assertEquals(0, network.getLayerWiseConfigurations().getIterationCount()); - network.fit(iter); - assertEquals(3, network.getLayerWiseConfigurations().getIterationCount()); - iter.reset(); - network.fit(iter); - assertEquals(6, network.getLayerWiseConfigurations().getIterationCount()); - iter.reset(); - network.fit(iter.next()); - assertEquals(7, network.getLayerWiseConfigurations().getIterationCount()); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ModelSerializer.writeModel(network, baos, true); - byte[] asBytes = baos.toByteArray(); - ByteArrayInputStream bais = new ByteArrayInputStream(asBytes); - MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(bais, true); - assertEquals(7, net.getLayerWiseConfigurations().getIterationCount()); - } - - @Test - @DisplayName("Test Bias L 1 L 2") - void testBiasL1L2() { - Nd4j.getRandom().setSeed(123); - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).weightInit(WeightInit.XAVIER).activation(Activation.TANH).seed(123).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nIn(10).nOut(10).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l1Bias(0.1).l2Bias(0.2).weightInit(WeightInit.XAVIER).activation(Activation.TANH).seed(123).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - BaseLayer bl0 = (BaseLayer) net2.getLayer(0).conf().getLayer(); - assertEquals(0.1, TestUtils.getL1(bl0.getRegularizationBias()), 1e-6); - assertEquals(0.2, TestUtils.getL2(bl0.getRegularizationBias()), 1e-6); - INDArray features = Nd4j.rand(10, 10); - INDArray labels = Nd4j.rand(10, 10); - net2.setParams(net1.params().dup()); - net1.setInput(features); - net1.setLabels(labels); - net2.setInput(features); - net2.setLabels(labels); - net1.computeGradientAndScore(); - net2.computeGradientAndScore(); - double r = net1.calcRegularizationScore(true); - assertEquals(0.0, r, 0.0); - r = net2.calcRegularizationScore(true); - assertEquals(0.0, r, 0.0); - double s1 = net1.score(); - double s2 = net2.score(); - // Biases initialized to 0 -> should initially have same score - assertEquals(s1, s2, 1e-6); - for (int i = 0; i < 10; i++) { - net1.fit(features, labels); - } - net2.setParams(net1.params().dup()); - net1.computeGradientAndScore(); - net2.computeGradientAndScore(); - r = net1.calcRegularizationScore(true); - assertEquals(0.0, r, 0.0); - r = net2.calcRegularizationScore(true); - assertTrue(r > 0.0); - s1 = net1.score(); - s2 = net2.score(); - // Scores should differ due to bias l1/l2 - assertNotEquals(s1, s2, 1e-6); - for (int i = 0; i < 2; i++) { - assertEquals(0.0, net1.getLayer(i).calcRegularizationScore(true), 0.0); - assertTrue(net2.getLayer(i).calcRegularizationScore(true) > 0.0); - } - } - - /* - Summary should pick up preprocessors set manually on inputs as well - */ - @Test - @DisplayName("Test Summary") - void testSummary() { - int V_WIDTH = 130; - int V_HEIGHT = 130; - int V_NFRAMES = 150; - MultiLayerConfiguration confForArchitecture = // l2 regularization on all layers - new NeuralNetConfiguration.Builder().seed(12345).l2(0.001).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list().layer(0, // 3 channels: RGB - new ConvolutionLayer.Builder(10, 10).nIn(3).nOut(30).stride(4, 4).activation(Activation.RELU).weightInit(WeightInit.RELU).updater(Updater.ADAGRAD).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(3, 3).stride(2, 2).build()).layer(2, new ConvolutionLayer.Builder(3, 3).nIn(30).nOut(10).stride(2, 2).activation(Activation.RELU).weightInit(WeightInit.RELU).updater(Updater.ADAGRAD).build()).layer(3, new DenseLayer.Builder().activation(Activation.RELU).nIn(490).nOut(50).weightInit(WeightInit.RELU).updater(Updater.ADAGRAD).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).layer(4, new GravesLSTM.Builder().activation(Activation.SOFTSIGN).nIn(50).nOut(50).weightInit(WeightInit.XAVIER).updater(Updater.ADAGRAD).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).layer(5, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(50).nOut(// 4 possible shapes: circle, square, arc, line - 4).updater(Updater.ADAGRAD).weightInit(WeightInit.XAVIER).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).inputPreProcessor(0, new RnnToCnnPreProcessor(V_HEIGHT, V_WIDTH, 3)).inputPreProcessor(3, new CnnToFeedForwardPreProcessor(7, 7, 10)).inputPreProcessor(4, new FeedForwardToRnnPreProcessor()).backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(V_NFRAMES / 5).tBPTTBackwardLength(V_NFRAMES / 5).build(); - MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(confForArchitecture); - modelExpectedArch.init(); - MultiLayerNetwork modelMow = new TransferLearning.Builder(modelExpectedArch).setFeatureExtractor(2).build(); - // System.out.println(modelExpectedArch.summary()); - // System.out.println(modelMow.summary()); - // System.out.println(modelMow.summary(InputType.recurrent(V_HEIGHT*V_WIDTH*3))); - } - - @Test - @DisplayName("Test Error No Output Layer") - void testErrorNoOutputLayer() { - assertThrows(DL4JException.class, () -> { - MultiLayerConfiguration c = new NeuralNetConfiguration.Builder().list().layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(c); - net.init(); - INDArray f = Nd4j.create(1, 10); - INDArray l = Nd4j.create(1, 10); - net.setInput(f); - net.setLabels(l); - net.computeGradientAndScore(); - }); - } - - @Test - @DisplayName("Test Set Param Table") - void testSetParamTable() { - Nd4j.getRandom().setSeed(123); - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(2, new LSTM.Builder().nIn(2).nOut(2).build()).layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(987).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(2, new LSTM.Builder().nIn(2).nOut(2).build()).layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3).build()).build(); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf1); - net1.init(); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf2); - net2.init(); - assertNotEquals(net1.params(), net2.params()); - assertNotEquals(net1.paramTable(), net2.paramTable()); - net1.setParamTable(net2.paramTable()); - assertEquals(net1.params(), net2.params()); - assertEquals(net1.paramTable(), net2.paramTable()); - } - - @Test - @DisplayName("Test Compare Layer Methods") - void testCompareLayerMethods() { - // Simple test: compare .layer(int, Layer) and .layer(Layer) are identical - MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(123).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(2, new LSTM.Builder().nIn(2).nOut(2).build()).layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3).build()).build(); - MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(123).list().layer(new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER).activation(Activation.TANH).build()).layer(new LSTM.Builder().nIn(2).nOut(2).build()).layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3).build()).build(); - assertEquals(conf1, conf2); - } - - @Test - @DisplayName("Test Epoch Counter") - void testEpochCounter() throws Exception { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(0, net.getLayerWiseConfigurations().getEpochCount()); - DataSetIterator iter = new IrisDataSetIterator(150, 150); - for (int i = 0; i < 4; i++) { - assertEquals(i, net.getLayerWiseConfigurations().getEpochCount()); - net.fit(iter); - assertEquals(i + 1, net.getLayerWiseConfigurations().getEpochCount()); - } - assertEquals(4, net.getLayerWiseConfigurations().getEpochCount()); - MultiLayerNetwork restored = TestUtils.testModelSerialization(net); - assertEquals(4, restored.getLayerWiseConfigurations().getEpochCount()); - } - - @Test - @DisplayName("Test Input Clearance") - void testInputClearance() throws Exception { - // Activations should be cleared - if not, it's possible for out of (workspace) scope arrays to be around - // which can cause a crash - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().convolutionMode(ConvolutionMode.Same).list().layer(new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nIn(1).nOut(1).build()).layer(new SubsamplingLayer.Builder().kernelSize(2, 2).stride(1, 1).build()).layer(new DenseLayer.Builder().nOut(10).build()).layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 1)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray content = Nd4j.create(1, 1, 28, 28); - // Check output: - net.output(content); - for (org.deeplearning4j.nn.api.Layer l : net.getLayers()) { - assertNull(l.input()); - } - // Check feedForward: - net.feedForward(content, false); - for (org.deeplearning4j.nn.api.Layer l : net.getLayers()) { - assertNull(l.input()); - } - } - - @Test - @DisplayName("Test External Errors") - void testExternalErrors() { - // Simple test: same network, but in one case: one less layer (the OutputLayer), where the epsilons are passed in externally - // instead. Should get identical results - for (WorkspaceMode ws : WorkspaceMode.values()) { - log.info("Workspace mode: " + ws); - Nd4j.getRandom().setSeed(12345); - INDArray inData = Nd4j.rand(3, 10); - INDArray outData = Nd4j.rand(3, 10); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration standard = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).trainingWorkspaceMode(ws).inferenceWorkspaceMode(ws).seed(12345).list().layer(new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).build()).build(); - MultiLayerNetwork s = new MultiLayerNetwork(standard); - s.init(); - Nd4j.getRandom().setSeed(12345); - MultiLayerConfiguration external = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).trainingWorkspaceMode(ws).inferenceWorkspaceMode(ws).seed(12345).list().layer(new DenseLayer.Builder().nIn(10).nOut(10).build()).build(); - MultiLayerNetwork e = new MultiLayerNetwork(external); - e.init(); - s.setInput(inData); - s.setLabels(outData); - s.computeGradientAndScore(); - Gradient sGrad = s.gradient(); - s.setInput(inData); - // FF without clearing inputs as we need them later - s.feedForward(true, false); - e.setInput(inData); - // FF without clearing inputs as we need them later - e.feedForward(true, false); - org.deeplearning4j.nn.layers.OutputLayer ol = (org.deeplearning4j.nn.layers.OutputLayer) s.getLayer(1); - Pair olPairStd = ol.backpropGradient(null, LayerWorkspaceMgr.noWorkspaces()); - INDArray olEpsilon = olPairStd.getSecond().detach(); - e.setInput(inData); - e.feedForward(true, false); - Pair extErrorGrad = e.backpropGradient(olEpsilon, LayerWorkspaceMgr.noWorkspaces()); - int nParamsDense = 10 * 10 + 10; - assertEquals(sGrad.gradient().get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(0, nParamsDense)), extErrorGrad.getFirst().gradient()); - Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); - } - } - - @Test - @DisplayName("Test External Errors 2") - void testExternalErrors2() { - Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC); - int nIn = 4; - int nOut = 3; - for (WorkspaceMode ws : WorkspaceMode.values()) { - // System.out.println("***** WORKSPACE: " + ws); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Adam(0.01)).trainingWorkspaceMode(ws).inferenceWorkspaceMode(ws).list().layer(new DenseLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.RELU).build()).layer(new ActivationLayer.Builder().activation(Activation.IDENTITY).build()).inputPreProcessor(0, new RnnToFeedForwardPreProcessor()).inputPreProcessor(1, new FeedForwardToRnnPreProcessor()).build(); - MultiLayerNetwork graph = new MultiLayerNetwork(conf); - graph.init(); - final int minibatch = 5; - final int seqLen = 6; - INDArray param = Nd4j.create(new double[] { 0.54, 0.31, 0.98, -0.30, -0.66, -0.19, -0.29, -0.62, 0.13, -0.32, 0.01, -0.03, 0.00, 0.00, 0.00 }).reshape(1, -1); - graph.setParams(param); - INDArray input = Nd4j.rand(new int[] { minibatch, nIn, seqLen }, 12); - INDArray expected = Nd4j.ones(minibatch, nOut, seqLen); - graph.setInput(input); - INDArray output = graph.feedForward(false, false).get(2); - INDArray error = output.sub(expected); - for (org.deeplearning4j.nn.api.Layer l : graph.getLayers()) { - assertNotNull(l.input()); - assertFalse(l.input().isAttached()); - } - // Compute Gradient - Pair gradient = graph.backpropGradient(error, LayerWorkspaceMgr.noWorkspaces()); - graph.getUpdater().update(graph, gradient.getFirst(), 0, 0, minibatch, LayerWorkspaceMgr.noWorkspaces()); - Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread(); - } - Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.DISABLED); - } - - @Test - @DisplayName("Test Layer Size") - void testLayerSize() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new ConvolutionLayer.Builder().kernelSize(2, 2).nOut(6).build()).layer(new SubsamplingLayer.Builder().kernelSize(2, 2).build()).layer(new DenseLayer.Builder().nOut(30).build()).layer(new OutputLayer.Builder().nOut(13).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutional(28, 28, 3)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - assertEquals(6, net.layerSize(0)); - assertEquals(0, net.layerSize(1)); - assertEquals(30, net.layerSize(2)); - assertEquals(13, net.layerSize(3)); - assertEquals(3, net.layerInputSize(0)); - assertEquals(0, net.layerInputSize(1)); - assertEquals(((FeedForwardLayer) net.getLayer(2).conf().getLayer()).getNIn(), net.layerInputSize(2)); - assertEquals(30, net.layerInputSize(3)); - } - - @Test - @DisplayName("Test Zero Param Net") - void testZeroParamNet() throws Exception { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new SubsamplingLayer.Builder().kernelSize(2, 2).stride(2, 2).build()).layer(new LossLayer.Builder().activation(Activation.SIGMOID).lossFunction(LossFunctions.LossFunction.MSE).build()).setInputType(InputType.convolutionalFlat(28, 28, 1)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSet ds = new MnistDataSetIterator(16, true, 12345).next(); - INDArray out = net.output(ds.getFeatures()); - INDArray labelTemp = Nd4j.create(out.shape()); - ds.setLabels(labelTemp); - net.fit(ds); - MultiLayerNetwork net2 = TestUtils.testModelSerialization(net); - INDArray out2 = net2.output(ds.getFeatures()); - assertEquals(out, out2); - } - - @Test - @DisplayName("Test Input Activation Gradient") - void testInputActivationGradient() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE).seed(12345).activation(Activation.TANH) - .list().layer(new DenseLayer.Builder().nIn(10).nOut(10).build()) - .layer(new OutputLayer.Builder().nIn(10).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray in = Nd4j.rand(1, 10).castTo(DataType.DOUBLE); - INDArray label = Nd4j.rand(1, 10).castTo(DataType.DOUBLE); - Pair p = net.calculateGradients(in, label, null, null); - // Quick gradient check: - double eps = 1e-6; - double maxRelError = 1e-5; - for (int i = 0; i < 10; i++) { - double orig = in.getDouble(i); - in.putScalar(i, orig + eps); - double scorePlus = net.score(new DataSet(in, label)); - in.putScalar(i, orig - eps); - double scoreMinus = net.score(new DataSet(in, label)); - in.putScalar(i, orig); - double expGrad = (scorePlus - scoreMinus) / (2.0 * eps); - double actGrad = p.getSecond().getDouble(i); - double relError = (Math.abs(expGrad - actGrad)) / (Math.abs(expGrad) + Math.abs(actGrad)); - String str = i + " - " + relError + " - exp=" + expGrad + ", act=" + actGrad; - assertTrue(relError < maxRelError,str); - } - } - - @Test - @DisplayName("Test Multi Layer Configuration Activation Types") - void testMultiLayerConfigurationActivationTypes() { - NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder().list().layer(new LSTM.Builder().nOut(6).build()).layer(new LSTM.Builder().nOut(7).build()).layer(new GlobalPoolingLayer()).layer(new OutputLayer.Builder().nOut(8).activation(Activation.SOFTMAX).build()).setInputType(InputType.recurrent(10)); - MultiLayerConfiguration conf = builder.build(); - List outBuilder = builder.getLayerActivationTypes(); - List outConf = conf.getLayerActivationTypes(InputType.recurrent(10)); - List exp = Arrays.asList(InputType.recurrent(6), InputType.recurrent(7), InputType.feedForward(7), InputType.feedForward(8)); - assertEquals(exp, outBuilder); - assertEquals(exp, outConf); - } - - @Test - @DisplayName("Test Multiple Epochs Simple") - - void testMultipleEpochsSimple() { - // Mainly a simple sanity check on the preconditions in the method... - DataSetIterator iter = new IrisDataSetIterator(10, 150); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - net.fit(iter, 3); - ComputationGraph g = net.toComputationGraph(); - g.fit(iter, 3); - } - - @Test - @DisplayName("Test Pretrain Fit Methods") - void testPretrainFitMethods() { - // The fit methods should *not* do layerwise pretraining: - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list().layer(new VariationalAutoencoder.Builder().nIn(10).nOut(10).encoderLayerSizes(10).decoderLayerSizes(10).build()).layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - Set> exp = new HashSet<>(); - exp.add(MultiLayerNetwork.class); - CheckModelsListener listener = new CheckModelsListener(); - net.setListeners(listener); - INDArray f = Nd4j.create(1, 10); - INDArray l = Nd4j.create(1, 10); - DataSet ds = new DataSet(f, l); - MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(f, l); - DataSetIterator iter = new ExistingDataSetIterator(Collections.singletonList(ds)); - net.fit(iter); - assertEquals(exp, listener.getModelClasses()); - net.fit(ds); - assertEquals(exp, listener.getModelClasses()); - net.fit(f, l); - assertEquals(exp, listener.getModelClasses()); - net.fit(f, l, null, null); - assertEquals(exp, listener.getModelClasses()); - net.fit(mds); - assertEquals(exp, listener.getModelClasses()); - net.fit(new SingletonMultiDataSetIterator(mds)); - assertEquals(exp, listener.getModelClasses()); - } - - @Test - @DisplayName("Test IND Array Config Cloning") - void testINDArrayConfigCloning() { - // INDArrays in config should be cloned to avoid threading issues - int mb = 3; - int b = 4; - int c = 3; - int depth = b * (5 + c); - int w = 6; - int h = 6; - INDArray bbPrior = Nd4j.rand(b, 2).muliRowVector(Nd4j.create(new double[] { w, h }).castTo(Nd4j.defaultFloatingPointType())); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l2(0.01).list() - .layer(new ConvolutionLayer.Builder().nIn(depth).nOut(depth).kernelSize(1, 1).build()) - .layer(new Yolo2OutputLayer.Builder().boundingBoxPriors(bbPrior).build()).build(); - MultiLayerConfiguration conf2 = conf.clone(); - INDArray bb1 = ((Yolo2OutputLayer) conf.getConf(1).getLayer()).getBoundingBoxes().castTo(Nd4j.defaultFloatingPointType()); - INDArray bb2 = ((Yolo2OutputLayer) conf2.getConf(1).getLayer()).getBoundingBoxes().castTo(Nd4j.defaultFloatingPointType()); - assertFalse(bb1 == bb2); - assertEquals(bb1, bb2); - } - - @Data - @DisplayName("Check Models Listener") - public static class CheckModelsListener extends BaseTrainingListener { - - private Set> modelClasses = new HashSet<>(); - - @Override - public void iterationDone(Model model, int iteration, int epoch) { - modelClasses.add(model.getClass()); - } - } - - @Test - @DisplayName("Test MLN Updater Blocks") - void testMLNUpdaterBlocks() { - // Check that setting learning rate results in correct rearrangement of updater state within updater blocks - // https://github.com/eclipse/deeplearning4j/issues/6809#issuecomment-463892644 - double lr = 1e-3; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).weightInit(WeightInit.XAVIER).updater(new Adam(lr)).list().layer(new DenseLayer.Builder().nIn(5).nOut(3).build()).layer(new DenseLayer.Builder().nIn(3).nOut(2).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.XENT).nIn(2).nOut(1).activation(Activation.SIGMOID).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray in = Nd4j.rand(1, 5); - INDArray lbl = Nd4j.rand(1, 1); - net.fit(new DataSet(in, lbl)); - INDArray viewArray = net.getUpdater().getStateViewArray(); - INDArray viewArrayCopy = viewArray.dup(); - // Initially updater view array is set out like: - // [m0w, m0b, m1w, m1b, m2w, m2b][v0w, v0b, v1w, v1b, v2w, v2b] - long soFar = 0; - // m0w - INDArray m0w = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 5 * 3)).assign(0); - soFar += 5 * 3; - // m0b - INDArray m0b = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 3)).assign(1); - soFar += 3; - // m1w - INDArray m1w = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 3 * 2)).assign(2); - soFar += 3 * 2; - // m1b - INDArray m1b = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 2)).assign(3); - soFar += 2; - // m2w - INDArray m2w = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 2 * 1)).assign(4); - soFar += 2 * 1; - // m2b - INDArray m2b = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 1)).assign(5); - soFar += 1; - // v0w - INDArray v0w = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 5 * 3)).assign(6); - soFar += 5 * 3; - // v0b - INDArray v0b = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 3)).assign(7); - soFar += 3; - // v1w - INDArray v1w = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 3 * 2)).assign(8); - soFar += 3 * 2; - // v1b - INDArray v1b = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 2)).assign(9); - soFar += 2; - // v2w - INDArray v2w = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 2 * 1)).assign(10); - soFar += 2 * 1; - // v2b - INDArray v2b = viewArray.get(NDArrayIndex.interval(0, 0, true), NDArrayIndex.interval(soFar, soFar + 1)).assign(11); - soFar += 1; - net.setLearningRate(0, 0.0); - // Expect new updater state to look like: - // [m0w, m0b][v0w,v0b], [m1w, m1b, m2w, m2b][v1w, v1b, v2w, v2b] - INDArray exp = Nd4j.concat(1, m0w, m0b, v0w, v0b, m1w, m1b, m2w, m2b, v1w, v1b, v2w, v2b); - INDArray act = net.getUpdater().getStateViewArray(); - // System.out.println(exp); - // System.out.println(act); - assertEquals(exp, act); - // And set layer 1 LR: - net.setLearningRate(1, 0.2); - exp = Nd4j.concat(1, m0w, m0b, v0w, v0b, m1w, m1b, v1w, v1b, m2w, m2b, v2w, v2b); - assertEquals(exp, net.getUpdater().getStateViewArray()); - // Set all back to original LR and check again: - net.setLearningRate(1, lr); - net.setLearningRate(0, lr); - exp = Nd4j.concat(1, m0w, m0b, m1w, m1b, m2w, m2b, v0w, v0b, v1w, v1b, v2w, v2b); - assertEquals(exp, net.getUpdater().getStateViewArray()); - // Finally, training sanity check (if things are wrong, we get -ve values in adam V, which causes NaNs) - net.getUpdater().getStateViewArray().assign(viewArrayCopy); - net.setLearningRate(0, 0.0); - Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.NAN_PANIC); - net.fit(new DataSet(in, lbl)); - Nd4j.getExecutioner().setProfilingMode(OpExecutioner.ProfilingMode.SCOPE_PANIC); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestSetGetParameters.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestSetGetParameters.java deleted file mode 100644 index fffe686e0..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/multilayer/TestSetGetParameters.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.multilayer; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.layers.*; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -public class TestSetGetParameters extends BaseDL4JTest { - - @Test - public void testSetParameters() { - //Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this. - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() - .layer(0, new DenseLayer.Builder().nIn(9).nOut(10) - .dist(new NormalDistribution(0, 1)).build()) - .layer(1, new DenseLayer.Builder().nIn(10).nOut(11) - .dist(new NormalDistribution(0, 1)).build()) - .layer(2, new AutoEncoder.Builder().corruptionLevel(0.5).nIn(11).nOut(12) - .dist(new NormalDistribution(0, 1)).build()) - .layer(3, new OutputLayer.Builder(LossFunction.MSE).nIn(12).nOut(12) - .dist(new NormalDistribution(0, 1)).build()) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - INDArray initParams = net.params().dup(); - Map initParams2 = net.paramTable(); - - net.setParams(net.params()); - - INDArray initParamsAfter = net.params(); - Map initParams2After = net.paramTable(); - - for (String s : initParams2.keySet()) { - assertTrue( initParams2.get(s).equals(initParams2After.get(s)),"Params differ: " + s); - } - - assertEquals(initParams, initParamsAfter); - - //Now, try the other way: get(set(random)) - INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape()); - net.setParams(randomParams.dup()); - - assertEquals(net.params(), randomParams); - } - - @Test - public void testSetParametersRNN() { - //Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this. - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() - .layer(0, new GravesLSTM.Builder().nIn(9).nOut(10) - .dist(new NormalDistribution(0, 1)).build()) - .layer(1, new GravesLSTM.Builder().nIn(10).nOut(11) - .dist(new NormalDistribution(0, 1)).build()) - .layer(2, new RnnOutputLayer.Builder(LossFunction.MSE) - .dist(new NormalDistribution(0, 1)).nIn(11).nOut(12).build()) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - INDArray initParams = net.params().dup(); - Map initParams2 = net.paramTable(); - - net.setParams(net.params()); - - INDArray initParamsAfter = net.params(); - Map initParams2After = net.paramTable(); - - for (String s : initParams2.keySet()) { - assertTrue( initParams2.get(s).equals(initParams2After.get(s)),"Params differ: " + s); - } - - assertEquals(initParams, initParamsAfter); - - //Now, try the other way: get(set(random)) - INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape()); - net.setParams(randomParams.dup()); - - assertEquals(net.params(), randomParams); - } - - @Test - public void testInitWithParams() { - - Nd4j.getRandom().setSeed(12345); - - //Create configuration. Doesn't matter if this doesn't actually work for forward/backward pass here - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list() - .layer(0, new ConvolutionLayer.Builder().nIn(10).nOut(10).kernelSize(2, 2).stride(2, 2) - .padding(2, 2).build()) - .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()) - .layer(2, new GravesLSTM.Builder().nIn(10).nOut(10).build()) - .layer(3, new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).build()) - .layer(4, new OutputLayer.Builder(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(10).build()) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray params = net.params(); - - - MultiLayerNetwork net2 = new MultiLayerNetwork(conf); - net2.init(params, true); - - MultiLayerNetwork net3 = new MultiLayerNetwork(conf); - net3.init(params, false); - - assertEquals(params, net2.params()); - assertEquals(params, net3.params()); - - assertFalse(params == net2.params()); //Different objects due to clone - assertTrue(params == net3.params()); //Same object due to clone - - - Map paramsMap = net.paramTable(); - Map paramsMap2 = net2.paramTable(); - Map paramsMap3 = net3.paramTable(); - for (String s : paramsMap.keySet()) { - assertEquals(paramsMap.get(s), paramsMap2.get(s)); - assertEquals(paramsMap.get(s), paramsMap3.get(s)); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestFrozenLayers.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestFrozenLayers.java deleted file mode 100644 index 52b86b276..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TestFrozenLayers.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.transferlearning; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.layers.FrozenLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.util.LinkedHashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; - -public class TestFrozenLayers extends BaseDL4JTest { - - @Test - public void testFrozenMLN(){ - MultiLayerNetwork orig = getOriginalNet(12345); - - - for(double l1 : new double[]{0.0, 0.3}){ - for( double l2 : new double[]{0.0, 0.4}){ - String msg = "l1=" + l1 + ", l2=" + l2; - - FineTuneConfiguration ftc = new FineTuneConfiguration.Builder() - .updater(new Sgd(0.5)) - .l1(l1) - .l2(l2) - .build(); - - MultiLayerNetwork transfer = new TransferLearning.Builder(orig) - .fineTuneConfiguration(ftc) - .setFeatureExtractor(4) - .removeOutputLayer() - .addLayer(new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MEAN_ABSOLUTE_ERROR).build()) - .build(); - - assertEquals(6, transfer.getnLayers()); - for( int i=0; i<5; i++ ){ - assertTrue( transfer.getLayer(i) instanceof FrozenLayer); - } - - Map paramsBefore = new LinkedHashMap<>(); - for(Map.Entry entry : transfer.paramTable().entrySet()){ - paramsBefore.put(entry.getKey(), entry.getValue().dup()); - } - - for( int i=0; i<20; i++ ){ - INDArray f = Nd4j.rand(new int[]{16,1,28,28}); - INDArray l = Nd4j.rand(new int[]{16,10}); - transfer.fit(f,l); - } - - for(Map.Entry entry : transfer.paramTable().entrySet()){ - String s = msg + " - " + entry.getKey(); - if(entry.getKey().startsWith("5_")){ - //Non-frozen layer - assertNotEquals(paramsBefore.get(entry.getKey()), entry.getValue(), s); - } else { - assertEquals(paramsBefore.get(entry.getKey()), entry.getValue(), s); - } - } - } - } - } - - @Test - public void testFrozenCG(){ - ComputationGraph orig = getOriginalGraph(12345); - - - for(double l1 : new double[]{0.0, 0.3}){ - for( double l2 : new double[]{0.0, 0.4}){ - String msg = "l1=" + l1 + ", l2=" + l2; - - FineTuneConfiguration ftc = new FineTuneConfiguration.Builder() - .updater(new Sgd(0.5)) - .l1(l1) - .l2(l2) - .build(); - - ComputationGraph transfer = new TransferLearning.GraphBuilder(orig) - .fineTuneConfiguration(ftc) - .setFeatureExtractor("4") - .removeVertexAndConnections("5") - .addLayer("5", new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MEAN_ABSOLUTE_ERROR).build(), "4") - .setOutputs("5") - .build(); - - assertEquals(6, transfer.getNumLayers()); - for( int i=0; i<5; i++ ){ - assertTrue( transfer.getLayer(i) instanceof FrozenLayer); - } - - Map paramsBefore = new LinkedHashMap<>(); - for(Map.Entry entry : transfer.paramTable().entrySet()){ - paramsBefore.put(entry.getKey(), entry.getValue().dup()); - } - - for( int i=0; i<20; i++ ){ - INDArray f = Nd4j.rand(new int[]{16,1,28,28}); - INDArray l = Nd4j.rand(new int[]{16,10}); - transfer.fit(new INDArray[]{f},new INDArray[]{l}); - } - - for(Map.Entry entry : transfer.paramTable().entrySet()){ - String s = msg + " - " + entry.getKey(); - if(entry.getKey().startsWith("5_")){ - //Non-frozen layer - assertNotEquals(paramsBefore.get(entry.getKey()), entry.getValue(), s); - } else { - assertEquals(paramsBefore.get(entry.getKey()), entry.getValue(), s); - } - } - } - } - } - - public static MultiLayerNetwork getOriginalNet(int seed){ - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .seed(seed) - .weightInit(WeightInit.XAVIER) - .activation(Activation.TANH) - .convolutionMode(ConvolutionMode.Same) - .updater(new Sgd(0.3)) - .list() - .layer(new ConvolutionLayer.Builder().nOut(3).kernelSize(2,2).stride(1,1).build()) - .layer(new SubsamplingLayer.Builder().kernelSize(2,2).stride(1,1).build()) - .layer(new ConvolutionLayer.Builder().nIn(3).nOut(3).kernelSize(2,2).stride(1,1).build()) - .layer(new DenseLayer.Builder().nOut(64).build()) - .layer(new DenseLayer.Builder().nIn(64).nOut(64).build()) - .layer(new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build()) - .setInputType(InputType.convolutionalFlat(28,28,1)) - .build(); - - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - return net; - } - - public static ComputationGraph getOriginalGraph(int seed){ - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() - .seed(seed) - .weightInit(WeightInit.XAVIER) - .activation(Activation.TANH) - .convolutionMode(ConvolutionMode.Same) - .updater(new Sgd(0.3)) - .graphBuilder() - .addInputs("in") - .layer("0", new ConvolutionLayer.Builder().nOut(3).kernelSize(2,2).stride(1,1).build(), "in") - .layer("1", new SubsamplingLayer.Builder().kernelSize(2,2).stride(1,1).build(), "0") - .layer("2", new ConvolutionLayer.Builder().nIn(3).nOut(3).kernelSize(2,2).stride(1,1).build(), "1") - .layer("3", new DenseLayer.Builder().nOut(64).build(), "2") - .layer("4", new DenseLayer.Builder().nIn(64).nOut(64).build(), "3") - .layer("5", new OutputLayer.Builder().nIn(64).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build(), "4") - .setOutputs("5") - .setInputTypes(InputType.convolutionalFlat(28,28,1)) - .build(); - - - ComputationGraph net = new ComputationGraph(conf); - net.init(); - return net; - } - -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java deleted file mode 100644 index f49c35443..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningCompGraphTest.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.transferlearning; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.constraint.UnitNormConstraint; -import org.deeplearning4j.nn.conf.distribution.ConstantDistribution; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.graph.AttentionVertex; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.weightnoise.DropConnect; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.weights.WeightInitDistribution; -import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.learning.config.RmsProp; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.HashMap; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Transfer Learning Comp Graph Test") -class TransferLearningCompGraphTest extends BaseDL4JTest { - - @Test - @DisplayName("Simple Fine Tune") - void simpleFineTune() { - long rng = 12345L; - DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); - // original conf - ComputationGraphConfiguration confToChange = new NeuralNetConfiguration.Builder().seed(rng).optimizationAlgo(OptimizationAlgorithm.LBFGS).updater(new Nesterovs(0.01, 0.99)).graphBuilder().addInputs("layer0In").setInputTypes(InputType.feedForward(4)).addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In").addLayer("layer1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer0").setOutputs("layer1").build(); - // conf with learning parameters changed - ComputationGraphConfiguration expectedConf = new NeuralNetConfiguration.Builder().seed(rng).updater(new RmsProp(0.2)).graphBuilder().addInputs("layer0In").setInputTypes(InputType.feedForward(4)).addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In").addLayer("layer1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer0").setOutputs("layer1").build(); - ComputationGraph expectedModel = new ComputationGraph(expectedConf); - expectedModel.init(); - ComputationGraph modelToFineTune = new ComputationGraph(expectedConf); - modelToFineTune.init(); - modelToFineTune.setParams(expectedModel.params()); - // model after applying changes with transfer learning - ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune).fineTuneConfiguration(new FineTuneConfiguration.Builder().seed(rng).updater(new RmsProp(0.2)).build()).build(); - // Check json - assertEquals(expectedConf.toJson(), modelNow.getConfiguration().toJson()); - // Check params after fit - modelNow.fit(randomData); - expectedModel.fit(randomData); - assertEquals(modelNow.score(), expectedModel.score(), 1e-8); - assertEquals(modelNow.params(), expectedModel.params()); - } - - @Test - @DisplayName("Test Nout Changes") - void testNoutChanges() { - DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 2)); - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).activation(Activation.IDENTITY); - FineTuneConfiguration fineTuneConfiguration = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).activation(Activation.IDENTITY).build(); - ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "layer0In").addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0").addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1").addLayer("layer3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer2").setOutputs("layer3").build()); - modelToFineTune.init(); - ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune).fineTuneConfiguration(fineTuneConfiguration).nOutReplace("layer3", 2, WeightInit.XAVIER).nOutReplace("layer0", 3, new NormalDistribution(1, 1e-1), WeightInit.XAVIER).build(); - BaseLayer bl0 = ((BaseLayer) modelNow.getLayer("layer0").conf().getLayer()); - BaseLayer bl1 = ((BaseLayer) modelNow.getLayer("layer1").conf().getLayer()); - BaseLayer bl3 = ((BaseLayer) modelNow.getLayer("layer3").conf().getLayer()); - assertEquals(bl0.getWeightInitFn(), new WeightInitDistribution(new NormalDistribution(1, 1e-1))); - assertEquals(bl1.getWeightInitFn(), new WeightInitXavier()); - assertEquals(bl1.getWeightInitFn(), new WeightInitXavier()); - ComputationGraph modelExpectedArch = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(3).build(), "layer0In").addLayer("layer1", new DenseLayer.Builder().nIn(3).nOut(2).build(), "layer0").addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1").addLayer("layer3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(2).build(), "layer2").setOutputs("layer3").build()); - modelExpectedArch.init(); - // modelNow should have the same architecture as modelExpectedArch - assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); - assertArrayEquals(modelExpectedArch.getLayer("layer0").params().shape(), modelNow.getLayer("layer0").params().shape()); - assertArrayEquals(modelExpectedArch.getLayer("layer1").params().shape(), modelNow.getLayer("layer1").params().shape()); - assertArrayEquals(modelExpectedArch.getLayer("layer2").params().shape(), modelNow.getLayer("layer2").params().shape()); - assertArrayEquals(modelExpectedArch.getLayer("layer3").params().shape(), modelNow.getLayer("layer3").params().shape()); - modelNow.setParams(modelExpectedArch.params()); - // fit should give the same results - modelExpectedArch.fit(randomData); - modelNow.fit(randomData); - assertEquals(modelExpectedArch.score(), modelNow.score(), 1e-8); - assertEquals(modelExpectedArch.params(), modelNow.params()); - } - - @Test - @DisplayName("Test Remove And Add") - void testRemoveAndAdd() { - DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).activation(Activation.IDENTITY); - FineTuneConfiguration fineTuneConfiguration = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).activation(Activation.IDENTITY).build(); - ComputationGraph modelToFineTune = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "layer0In").addLayer("layer1", new DenseLayer.Builder().nIn(5).nOut(2).build(), "layer0").addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(3).build(), "layer1").addLayer("layer3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build(), "layer2").setOutputs("layer3").build()); - modelToFineTune.init(); - ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune).fineTuneConfiguration(fineTuneConfiguration).nOutReplace("layer0", 7, WeightInit.XAVIER, WeightInit.XAVIER).nOutReplace("layer2", 5, WeightInit.XAVIER).removeVertexKeepConnections("layer3").addLayer("layer3", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(3).activation(Activation.SOFTMAX).build(), "layer2").build(); - ComputationGraph modelExpectedArch = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").addLayer("layer0", new DenseLayer.Builder().nIn(4).nOut(7).build(), "layer0In").addLayer("layer1", new DenseLayer.Builder().nIn(7).nOut(2).build(), "layer0").addLayer("layer2", new DenseLayer.Builder().nIn(2).nOut(5).build(), "layer1").addLayer("layer3", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(5).nOut(3).build(), "layer2").setOutputs("layer3").build()); - modelExpectedArch.init(); - // modelNow should have the same architecture as modelExpectedArch - assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); - assertArrayEquals(modelExpectedArch.getLayer("layer0").params().shape(), modelNow.getLayer("layer0").params().shape()); - assertArrayEquals(modelExpectedArch.getLayer("layer1").params().shape(), modelNow.getLayer("layer1").params().shape()); - assertArrayEquals(modelExpectedArch.getLayer("layer2").params().shape(), modelNow.getLayer("layer2").params().shape()); - assertArrayEquals(modelExpectedArch.getLayer("layer3").params().shape(), modelNow.getLayer("layer3").params().shape()); - modelNow.setParams(modelExpectedArch.params()); - // fit should give the same results - modelExpectedArch.fit(randomData); - modelNow.fit(randomData); - assertEquals(modelExpectedArch.score(), modelNow.score(), 1e-8); - assertEquals(modelExpectedArch.params(), modelNow.params()); - } - - @Test - @DisplayName("Test All With CNN") - void testAllWithCNN() { - DataSet randomData = new DataSet(Nd4j.rand(10, 28 * 28 * 3).reshape(10, 3, 28, 28), Nd4j.rand(10, 10)); - ComputationGraph modelToFineTune = new ComputationGraph(new NeuralNetConfiguration.Builder().seed(123).weightInit(WeightInit.XAVIER).updater(new Nesterovs(0.01, 0.9)).graphBuilder().addInputs("layer0In").setInputTypes(InputType.convolutionalFlat(28, 28, 3)).addLayer("layer0", new ConvolutionLayer.Builder(5, 5).nIn(3).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build(), "layer0In").addLayer("layer1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build(), "layer0").addLayer("layer2", new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "layer1").addLayer("layer3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build(), "layer2").addLayer("layer4", new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build(), "layer3").addLayer("layer5", new DenseLayer.Builder().activation(Activation.RELU).nOut(250).build(), "layer4").addLayer("layer6", new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(100).activation(Activation.SOFTMAX).build(), "layer5").setOutputs("layer6").build()); - modelToFineTune.init(); - // this will override the learning configuration set in the model - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().seed(456).updater(new Sgd(0.001)); - FineTuneConfiguration fineTuneConfiguration = new FineTuneConfiguration.Builder().seed(456).updater(new Sgd(0.001)).build(); - ComputationGraph modelNow = new TransferLearning.GraphBuilder(modelToFineTune).fineTuneConfiguration(fineTuneConfiguration).setFeatureExtractor("layer1").nOutReplace("layer4", 600, WeightInit.XAVIER).removeVertexAndConnections("layer5").removeVertexAndConnections("layer6").setInputs("layer0In").setInputTypes(InputType.convolutionalFlat(28, 28, 3)).addLayer("layer5", new DenseLayer.Builder().activation(Activation.RELU).nIn(600).nOut(300).build(), "layer4").addLayer("layer6", new DenseLayer.Builder().activation(Activation.RELU).nIn(300).nOut(150).build(), "layer5").addLayer("layer7", new DenseLayer.Builder().activation(Activation.RELU).nIn(150).nOut(50).build(), "layer6").addLayer("layer8", new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nIn(50).nOut(10).build(), "layer7").setOutputs("layer8").build(); - ComputationGraph modelExpectedArch = new ComputationGraph(overallConf.graphBuilder().addInputs("layer0In").setInputTypes(InputType.convolutionalFlat(28, 28, 3)).addLayer("layer0", new FrozenLayer(new ConvolutionLayer.Builder(5, 5).nIn(3).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()), "layer0In").addLayer("layer1", new FrozenLayer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()), "layer0").addLayer("layer2", new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "layer1").addLayer("layer3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build(), "layer2").addLayer("layer4", new DenseLayer.Builder().activation(Activation.RELU).nOut(600).build(), "layer3").addLayer("layer5", new DenseLayer.Builder().activation(Activation.RELU).nOut(300).build(), "layer4").addLayer("layer6", new DenseLayer.Builder().activation(Activation.RELU).nOut(150).build(), "layer5").addLayer("layer7", new DenseLayer.Builder().activation(Activation.RELU).nOut(50).build(), "layer6").addLayer("layer8", new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(10).activation(Activation.SOFTMAX).build(), "layer7").setOutputs("layer8").build()); - modelExpectedArch.init(); - modelExpectedArch.getVertex("layer0").setLayerAsFrozen(); - modelExpectedArch.getVertex("layer1").setLayerAsFrozen(); - assertEquals(modelExpectedArch.getConfiguration().toJson(), modelNow.getConfiguration().toJson()); - modelNow.setParams(modelExpectedArch.params()); - int i = 0; - while (i < 5) { - modelExpectedArch.fit(randomData); - modelNow.fit(randomData); - i++; - } - assertEquals(modelExpectedArch.params(), modelNow.params()); - } - - @Test - @DisplayName("Test Transfer Global Pool") - void testTransferGlobalPool() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new Adam(0.1)).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("blstm1", new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).activation(Activation.TANH).build(), "in").addLayer("pool", new GlobalPoolingLayer.Builder().build(), "blstm1").addLayer("dense", new DenseLayer.Builder().nIn(10).nOut(10).build(), "pool").addLayer("out", new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.IDENTITY).lossFunction(LossFunctions.LossFunction.MSE).build(), "dense").setOutputs("out").build(); - ComputationGraph g = new ComputationGraph(conf); - g.init(); - FineTuneConfiguration fineTuneConfiguration = new FineTuneConfiguration.Builder().seed(12345).updater(new Sgd(0.01)).build(); - ComputationGraph graph = new TransferLearning.GraphBuilder(g).fineTuneConfiguration(fineTuneConfiguration).removeVertexKeepConnections("out").setFeatureExtractor("dense").addLayer("out", new OutputLayer.Builder().updater(new Adam(0.1)).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).nIn(10).nOut(5).build(), "dense").build(); - ComputationGraphConfiguration confExpected = new NeuralNetConfiguration.Builder().seed(12345).updater(new Sgd(0.01)).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in").addLayer("blstm1", new FrozenLayer(new GravesBidirectionalLSTM.Builder().nIn(10).nOut(10).activation(Activation.TANH).build()), "in").addLayer("pool", new FrozenLayer(new GlobalPoolingLayer.Builder().build()), "blstm1").addLayer("dense", new FrozenLayer(new DenseLayer.Builder().nIn(10).nOut(10).build()), "pool").addLayer("out", new OutputLayer.Builder().nIn(10).nOut(5).activation(Activation.SOFTMAX).updater(new Adam(0.1)).lossFunction(LossFunctions.LossFunction.MCXENT).build(), "dense").setOutputs("out").build(); - ComputationGraph modelExpected = new ComputationGraph(confExpected); - modelExpected.init(); - // assertEquals(confExpected, graph.getConfiguration()); - assertEquals(confExpected.toJson(), graph.getConfiguration().toJson()); - } - - @Test - @DisplayName("Test Object Overrides") - void testObjectOverrides() { - // https://github.com/eclipse/deeplearning4j/issues/4368 - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().dropOut(0.5).weightNoise(new DropConnect(0.5)).l2(0.5).constrainWeights(new UnitNormConstraint()).graphBuilder().addInputs("in").addLayer("layer", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in").setOutputs("layer").build(); - ComputationGraph orig = new ComputationGraph(conf); - orig.init(); - FineTuneConfiguration ftc = new FineTuneConfiguration.Builder().dropOut(0).weightNoise(null).constraints(null).l2(0.0).build(); - ComputationGraph transfer = new TransferLearning.GraphBuilder(orig).fineTuneConfiguration(ftc).build(); - DenseLayer l = (DenseLayer) transfer.getLayer(0).conf().getLayer(); - assertNull(l.getIDropout()); - assertNull(l.getWeightNoise()); - assertNull(l.getConstraints()); - assertNull(TestUtils.getL2Reg(l)); - } - - @Test - @DisplayName("Test Transfer Learning Subsequent") - void testTransferLearningSubsequent() { - String inputName = "in"; - String outputName = "out"; - final String firstConv = "firstConv"; - final String secondConv = "secondConv"; - final INDArray input = Nd4j.create(6, 6, 6, 6); - final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder().weightInit(new ConstantDistribution(666)).graphBuilder().addInputs(inputName).setOutputs(outputName).setInputTypes(InputType.inferInputTypes(input)).addLayer(firstConv, new Convolution2D.Builder(3, 3).nOut(10).build(), inputName).addLayer(secondConv, new Convolution2D.Builder(1, 1).nOut(3).build(), firstConv).addLayer(outputName, new OutputLayer.Builder().nOut(2).lossFunction(LossFunctions.LossFunction.MSE).build(), secondConv).build()); - graph.init(); - final ComputationGraph newGraph = new TransferLearning.GraphBuilder(graph).nOutReplace(firstConv, 7, new ConstantDistribution(333)).nOutReplace(secondConv, 3, new ConstantDistribution(111)).removeVertexAndConnections(outputName).addLayer(outputName, new OutputLayer.Builder().nIn(48).nOut(2).lossFunction(LossFunctions.LossFunction.MSE).build(), new CnnToFeedForwardPreProcessor(4, 4, 3), secondConv).setOutputs(outputName).build(); - newGraph.init(); - assertEquals(7, newGraph.layerInputSize(secondConv), "Incorrect # inputs"); - newGraph.outputSingle(input); - } - - @Test - @DisplayName("Test Change N Out N In") - void testChangeNOutNIn() { - final String inputName = "input"; - final String changeNoutName = "changeNout"; - final String poolName = "pool"; - final String afterPoolName = "afterPool"; - final String outputName = "output"; - final INDArray input = Nd4j.create(new long[] { 1, 2, 4, 4 }); - final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder().graphBuilder().addInputs(inputName).setOutputs(outputName).setInputTypes(InputType.inferInputTypes(input)).addLayer(changeNoutName, new Convolution2D.Builder(1, 1).nOut(10).build(), inputName).addLayer(poolName, new SubsamplingLayer.Builder(1, 1).build(), changeNoutName).addLayer(afterPoolName, new Convolution2D.Builder(1, 1).nOut(7).build(), poolName).addLayer(outputName, new OutputLayer.Builder().activation(Activation.SOFTMAX).nOut(2).build(), afterPoolName).build()); - graph.init(); - final ComputationGraph newGraph = new TransferLearning.GraphBuilder(graph).nOutReplace(changeNoutName, 5, WeightInit.XAVIER).nInReplace(afterPoolName, 5, WeightInit.XAVIER).build(); - newGraph.init(); - assertEquals(5, newGraph.layerSize(changeNoutName), "Incorrect number of outputs!"); - assertEquals(5, newGraph.layerInputSize(afterPoolName), "Incorrect number of inputs!"); - newGraph.output(input); - } - - @Test - @DisplayName("Test Transfer Learning Same Diff Layers Graph") - void testTransferLearningSameDiffLayersGraph() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").layer("l0", new LSTM.Builder().nIn(5).nOut(5).build(), "in").layer("l1", new RecurrentAttentionLayer.Builder().nHeads(1).headSize(5).nIn(5).nOut(5).build(), "l0").layer("out", new RnnOutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).build(), "l1").setOutputs("out").build(); - ComputationGraph cg = new ComputationGraph(conf); - cg.init(); - INDArray arr = Nd4j.rand(DataType.FLOAT, 2, 5, 10); - INDArray out = cg.output(arr)[0]; - ComputationGraph cg2 = new TransferLearning.GraphBuilder(cg).removeVertexAndConnections("out").fineTuneConfiguration(FineTuneConfiguration.builder().updater(new Adam(0.01)).build()).removeVertexAndConnections("out").addLayer("newOut", new RnnOutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).build(), "l1").setOutputs("newOut").build(); - cg2.output(arr); - Map m = new HashMap<>(cg.paramTable()); - m.put("newOut_W", m.remove("out_W")); - m.put("newOut_b", m.remove("out_b")); - cg2.setParamTable(m); - Map p1 = cg.paramTable(); - Map p2 = cg2.paramTable(); - for (String s : p1.keySet()) { - INDArray i1 = p1.get(s); - INDArray i2 = p2.get(s.replaceAll("out", "newOut")); - assertEquals(i1, i2,s); - } - INDArray out2 = cg2.outputSingle(arr); - assertEquals(out, out2); - } - - @Test - @DisplayName("Test Transfer Learning Same Diff Layers Graph Vertex") - void testTransferLearningSameDiffLayersGraphVertex() { - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").layer("l0", new LSTM.Builder().nIn(5).nOut(5).build(), "in").addVertex("l1", new AttentionVertex.Builder().nHeads(1).headSize(5).nInKeys(5).nInQueries(5).nInValues(5).nOut(5).build(), "l0", "l0", "l0").layer("out", new RnnOutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).build(), "l1").setOutputs("out").build(); - ComputationGraph cg = new ComputationGraph(conf); - cg.init(); - INDArray arr = Nd4j.rand(DataType.FLOAT, 2, 5, 10); - INDArray out = cg.output(arr)[0]; - ComputationGraph cg2 = new TransferLearning.GraphBuilder(cg).removeVertexAndConnections("out").fineTuneConfiguration(FineTuneConfiguration.builder().updater(new Adam(0.01)).build()).removeVertexAndConnections("out").addLayer("newOut", new RnnOutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).build(), "l1").setOutputs("newOut").build(); - cg2.output(arr); - Map m = new HashMap<>(cg.paramTable()); - m.put("newOut_W", m.remove("out_W")); - m.put("newOut_b", m.remove("out_b")); - cg2.setParamTable(m); - Map p1 = cg.paramTable(); - Map p2 = cg2.paramTable(); - for (String s : p1.keySet()) { - INDArray i1 = p1.get(s); - INDArray i2 = p2.get(s.replaceAll("out", "newOut")); - assertEquals(i1, i2,s); - } - INDArray out2 = cg2.outputSingle(arr); - assertEquals(out, out2); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelperTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelperTest.java deleted file mode 100644 index e38f8ba4d..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningHelperTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.transferlearning; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.graph.MergeVertex; -import org.deeplearning4j.nn.conf.graph.SubsetVertex; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.MultiDataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Transfer Learning Helper Test") -class TransferLearningHelperTest extends BaseDL4JTest { - - @Test - @DisplayName("Tes Unfrozen Subset") - void tesUnfrozenSubset() { - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().seed(124).activation(Activation.IDENTITY).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)); - /* - (inCentre) (inRight) - | | - denseCentre0 | - | | - ,-------- denseCentre1 denseRight0 - / | | - subsetLeft(0-3) denseCentre2 ---- denseRight ---- mergeRight - | | | - denseLeft0 denseCentre3 denseRight1 - | | | - (outLeft) (outCentre) (outRight) - - */ - ComputationGraphConfiguration conf = overallConf.graphBuilder().addInputs("inCentre", "inRight").addLayer("denseCentre0", new DenseLayer.Builder().nIn(10).nOut(9).build(), "inCentre").addLayer("denseCentre1", new DenseLayer.Builder().nIn(9).nOut(8).build(), "denseCentre0").addLayer("denseCentre2", new DenseLayer.Builder().nIn(8).nOut(7).build(), "denseCentre1").addLayer("denseCentre3", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2").addLayer("outCentre", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(7).nOut(4).build(), "denseCentre3").addVertex("subsetLeft", new SubsetVertex(0, 3), "denseCentre1").addLayer("denseLeft0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "subsetLeft").addLayer("outLeft", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(6).build(), "denseLeft0").addLayer("denseRight", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2").addLayer("denseRight0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "inRight").addVertex("mergeRight", new MergeVertex(), "denseRight", "denseRight0").addLayer("denseRight1", new DenseLayer.Builder().nIn(10).nOut(5).build(), "mergeRight").addLayer("outRight", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(5).build(), "denseRight1").setOutputs("outLeft", "outCentre", "outRight").build(); - ComputationGraph modelToTune = new ComputationGraph(conf); - modelToTune.init(); - TransferLearningHelper helper = new TransferLearningHelper(modelToTune, "denseCentre2"); - ComputationGraph modelSubset = helper.unfrozenGraph(); - ComputationGraphConfiguration expectedConf = // inputs are in sorted order - overallConf.graphBuilder().addInputs("denseCentre1", "denseCentre2", "inRight").addLayer("denseCentre3", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2").addLayer("outCentre", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(7).nOut(4).build(), "denseCentre3").addVertex("subsetLeft", new SubsetVertex(0, 3), "denseCentre1").addLayer("denseLeft0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "subsetLeft").addLayer("outLeft", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(6).build(), "denseLeft0").addLayer("denseRight", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2").addLayer("denseRight0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "inRight").addVertex("mergeRight", new MergeVertex(), "denseRight", "denseRight0").addLayer("denseRight1", new DenseLayer.Builder().nIn(10).nOut(5).build(), "mergeRight").addLayer("outRight", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(5).build(), "denseRight1").setOutputs("outLeft", "outCentre", "outRight").build(); - ComputationGraph expectedModel = new ComputationGraph(expectedConf); - expectedModel.init(); - assertEquals(expectedConf.toJson(), modelSubset.getConfiguration().toJson()); - } - - @Test - @DisplayName("Test Fit Un Frozen") - void testFitUnFrozen() { - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.9)).seed(124).activation(Activation.IDENTITY).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT); - ComputationGraphConfiguration conf = overallConf.graphBuilder().addInputs("inCentre", "inRight").addLayer("denseCentre0", new DenseLayer.Builder().nIn(10).nOut(9).build(), "inCentre").addLayer("denseCentre1", new DenseLayer.Builder().nIn(9).nOut(8).build(), "denseCentre0").addLayer("denseCentre2", new DenseLayer.Builder().nIn(8).nOut(7).build(), "denseCentre1").addLayer("denseCentre3", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2").addLayer("outCentre", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(7).nOut(4).build(), "denseCentre3").addVertex("subsetLeft", new SubsetVertex(0, 3), "denseCentre1").addLayer("denseLeft0", new DenseLayer.Builder().nIn(4).nOut(5).build(), "subsetLeft").addLayer("outLeft", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(6).build(), "denseLeft0").addLayer("denseRight", new DenseLayer.Builder().nIn(7).nOut(7).build(), "denseCentre2").addLayer("denseRight0", new DenseLayer.Builder().nIn(2).nOut(3).build(), "inRight").addVertex("mergeRight", new MergeVertex(), "denseRight", "denseRight0").addLayer("denseRight1", new DenseLayer.Builder().nIn(10).nOut(5).build(), "mergeRight").addLayer("outRight", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(5).nOut(5).build(), "denseRight1").setOutputs("outLeft", "outCentre", "outRight").build(); - ComputationGraph modelToTune = new ComputationGraph(conf); - modelToTune.init(); - INDArray inRight = Nd4j.rand(10, 2); - INDArray inCentre = Nd4j.rand(10, 10); - INDArray outLeft = Nd4j.rand(10, 6); - INDArray outRight = Nd4j.rand(10, 5); - INDArray outCentre = Nd4j.rand(10, 4); - MultiDataSet origData = new MultiDataSet(new INDArray[] { inCentre, inRight }, new INDArray[] { outLeft, outCentre, outRight }); - ComputationGraph modelIdentical = modelToTune.clone(); - modelIdentical.getVertex("denseCentre0").setLayerAsFrozen(); - modelIdentical.getVertex("denseCentre1").setLayerAsFrozen(); - modelIdentical.getVertex("denseCentre2").setLayerAsFrozen(); - TransferLearningHelper helper = new TransferLearningHelper(modelToTune, "denseCentre2"); - MultiDataSet featurizedDataSet = helper.featurize(origData); - assertEquals(modelIdentical.getLayer("denseRight0").params(), modelToTune.getLayer("denseRight0").params()); - modelIdentical.fit(origData); - helper.fitFeaturized(featurizedDataSet); - assertEquals(modelIdentical.getLayer("denseCentre0").params(), modelToTune.getLayer("denseCentre0").params()); - assertEquals(modelIdentical.getLayer("denseCentre1").params(), modelToTune.getLayer("denseCentre1").params()); - assertEquals(modelIdentical.getLayer("denseCentre2").params(), modelToTune.getLayer("denseCentre2").params()); - assertEquals(modelIdentical.getLayer("denseCentre3").params(), modelToTune.getLayer("denseCentre3").params()); - assertEquals(modelIdentical.getLayer("outCentre").params(), modelToTune.getLayer("outCentre").params()); - assertEquals(modelIdentical.getLayer("denseRight").conf().toJson(), modelToTune.getLayer("denseRight").conf().toJson()); - assertEquals(modelIdentical.getLayer("denseRight").params(), modelToTune.getLayer("denseRight").params()); - assertEquals(modelIdentical.getLayer("denseRight0").conf().toJson(), modelToTune.getLayer("denseRight0").conf().toJson()); - // assertEquals(modelIdentical.getLayer("denseRight0").params(),modelToTune.getLayer("denseRight0").params()); - assertEquals(modelIdentical.getLayer("denseRight1").params(), modelToTune.getLayer("denseRight1").params()); - assertEquals(modelIdentical.getLayer("outRight").params(), modelToTune.getLayer("outRight").params()); - assertEquals(modelIdentical.getLayer("denseLeft0").params(), modelToTune.getLayer("denseLeft0").params()); - assertEquals(modelIdentical.getLayer("outLeft").params(), modelToTune.getLayer("outLeft").params()); - // log.info(modelIdentical.summary()); - // log.info(helper.unfrozenGraph().summary()); - modelIdentical.summary(); - helper.unfrozenGraph().summary(); - } - - @Test - @DisplayName("Test MLN") - void testMLN() { - DataSet randomData = new DataSet(Nd4j.rand(10, 4), Nd4j.rand(10, 3)); - NeuralNetConfiguration.Builder overallConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).activation(Activation.IDENTITY); - MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(overallConf.clone().list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build()); - modelToFineTune.init(); - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).setFeatureExtractor(1).build(); - List ff = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false); - INDArray asFrozenFeatures = ff.get(2); - TransferLearningHelper helper = new TransferLearningHelper(modelToFineTune, 1); - INDArray paramsLastTwoLayers = Nd4j.hstack(modelToFineTune.getLayer(2).params(), modelToFineTune.getLayer(3).params()); - MultiLayerNetwork notFrozen = new MultiLayerNetwork(overallConf.clone().list().layer(0, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build(), paramsLastTwoLayers); - assertEquals(asFrozenFeatures, helper.featurize(randomData).getFeatures()); - assertEquals(randomData.getLabels(), helper.featurize(randomData).getLabels()); - for (int i = 0; i < 5; i++) { - notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); - helper.fitFeaturized(helper.featurize(randomData)); - modelNow.fit(randomData); - } - INDArray expected = Nd4j.hstack(modelToFineTune.getLayer(0).params(), modelToFineTune.getLayer(1).params(), notFrozen.params()); - INDArray act = modelNow.params(); - assertEquals(expected, act); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java deleted file mode 100644 index e07ea0cfd..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/transferlearning/TransferLearningMLNTest.java +++ /dev/null @@ -1,379 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.transferlearning; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.TestUtils; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.BackpropType; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.constraint.UnitNormConstraint; -import org.deeplearning4j.nn.conf.distribution.ConstantDistribution; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.RnnToCnnPreProcessor; -import org.deeplearning4j.nn.conf.serde.JsonMappers; -import org.deeplearning4j.nn.conf.weightnoise.DropConnect; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.weights.WeightInitDistribution; -import org.deeplearning4j.nn.weights.WeightInitRelu; -import org.deeplearning4j.nn.weights.WeightInitXavier; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.*; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Transfer Learning MLN Test") -class TransferLearningMLNTest extends BaseDL4JTest { - - @Test - @DisplayName("Simple Fine Tune") - void simpleFineTune() { - long rng = 12345L; - Nd4j.getRandom().setSeed(rng); - DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 3)); - // original conf - NeuralNetConfiguration.Builder confToChange = new NeuralNetConfiguration.Builder().seed(rng).optimizationAlgo(OptimizationAlgorithm.LBFGS).updater(new Nesterovs(0.01, 0.99)); - MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(confToChange.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build()); - modelToFineTune.init(); - // model after applying changes with transfer learning - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(new FineTuneConfiguration.Builder().seed(rng).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(// Intent: override both weight and bias LR, unless bias LR is manually set also - new RmsProp(0.5)).l2(0.4).build()).build(); - for (org.deeplearning4j.nn.api.Layer l : modelNow.getLayers()) { - BaseLayer bl = ((BaseLayer) l.conf().getLayer()); - assertEquals(new RmsProp(0.5), bl.getIUpdater()); - } - NeuralNetConfiguration.Builder confSet = new NeuralNetConfiguration.Builder().seed(rng).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new RmsProp(0.5)).l2(0.4); - MultiLayerNetwork expectedModel = new MultiLayerNetwork(confSet.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build()); - expectedModel.init(); - expectedModel.setParams(modelToFineTune.params().dup()); - assertEquals(expectedModel.params(), modelNow.params()); - // Check json - MultiLayerConfiguration expectedConf = expectedModel.getLayerWiseConfigurations(); - assertEquals(expectedConf.toJson(), modelNow.getLayerWiseConfigurations().toJson()); - // Check params after fit - modelNow.fit(randomData); - expectedModel.fit(randomData); - assertEquals(modelNow.score(), expectedModel.score(), 1e-6); - INDArray pExp = expectedModel.params(); - INDArray pNow = modelNow.params(); - assertEquals(pExp, pNow); - } - - @Test - @DisplayName("Test Nout Changes") - void testNoutChanges() { - Nd4j.getRandom().setSeed(12345); - DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 2)); - NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)); - FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build(); - MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(equivalentConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(5).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build()); - modelToFineTune.init(); - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf).nOutReplace(3, 2, WeightInit.XAVIER, WeightInit.XAVIER).nOutReplace(0, 3, WeightInit.XAVIER, new NormalDistribution(1, 1e-1)).build(); - MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(equivalentConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(3).build()).layer(1, new DenseLayer.Builder().nIn(3).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(2).build()).build()); - modelExpectedArch.init(); - // Will fail - expected because of dist and weight init changes - // assertEquals(modelExpectedArch.getLayerWiseConfigurations().toJson(), modelNow.getLayerWiseConfigurations().toJson()); - BaseLayer bl0 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(0).getLayer()); - BaseLayer bl1 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(1).getLayer()); - BaseLayer bl3 = ((BaseLayer) modelNow.getLayerWiseConfigurations().getConf(3).getLayer()); - assertEquals(bl0.getWeightInitFn().getClass(), WeightInitXavier.class); - try { - assertEquals(JsonMappers.getMapper().writeValueAsString(bl1.getWeightInitFn()), JsonMappers.getMapper().writeValueAsString(new WeightInitDistribution(new NormalDistribution(1, 1e-1)))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - assertEquals(bl3.getWeightInitFn(), new WeightInitXavier()); - // modelNow should have the same architecture as modelExpectedArch - assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape()); - modelNow.setParams(modelExpectedArch.params()); - // fit should give the same results - modelExpectedArch.fit(randomData); - modelNow.fit(randomData); - assertEquals(modelExpectedArch.score(), modelNow.score(), 0.000001); - assertEquals(modelExpectedArch.params(), modelNow.params()); - } - - @Test - @DisplayName("Test Remove And Add") - void testRemoveAndAdd() { - Nd4j.getRandom().setSeed(12345); - DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 4), TestUtils.randomOneHot(DataType.FLOAT, 10, 3)); - NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.1)); - FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.1)).build(); - MultiLayerNetwork modelToFineTune = new // overallConf.list() - MultiLayerNetwork(equivalentConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(5).build()).layer(1, new DenseLayer.Builder().nIn(5).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(3).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3).build()).build()); - modelToFineTune.init(); - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf).nOutReplace(0, 7, WeightInit.XAVIER, WeightInit.XAVIER).nOutReplace(2, 5, WeightInit.XAVIER).removeOutputLayer().addLayer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(5).nOut(3).updater(new Sgd(0.5)).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(equivalentConf.list().layer(0, new DenseLayer.Builder().nIn(4).nOut(7).build()).layer(1, new DenseLayer.Builder().nIn(7).nOut(2).build()).layer(2, new DenseLayer.Builder().nIn(2).nOut(5).build()).layer(3, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).updater(new Sgd(0.5)).nIn(5).nOut(3).build()).build()); - modelExpectedArch.init(); - // modelNow should have the same architecture as modelExpectedArch - assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape()); - modelNow.setParams(modelExpectedArch.params()); - // fit should give the same results - modelExpectedArch.fit(randomData); - modelNow.fit(randomData); - double scoreExpected = modelExpectedArch.score(); - double scoreActual = modelNow.score(); - assertEquals(scoreExpected, scoreActual, 1e-4); - assertEquals(modelExpectedArch.params(), modelNow.params()); - } - - @Test - @DisplayName("Test Remove And Processing") - void testRemoveAndProcessing() { - int V_WIDTH = 130; - int V_HEIGHT = 130; - int V_NFRAMES = 150; - MultiLayerConfiguration confForArchitecture = // l2 regularization on all layers - new NeuralNetConfiguration.Builder().seed(12345).l2(0.001).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new AdaGrad(0.4)).list().layer(0, // 3 channels: RGB - new ConvolutionLayer.Builder(10, 10).nIn(3).nOut(30).stride(4, 4).activation(Activation.RELU).weightInit(WeightInit.RELU).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(3, 3).stride(2, 2).build()).layer(2, new ConvolutionLayer.Builder(3, 3).nIn(30).nOut(10).stride(2, 2).activation(Activation.RELU).weightInit(WeightInit.RELU).build()).layer(3, new DenseLayer.Builder().activation(Activation.RELU).nIn(490).nOut(50).weightInit(WeightInit.RELU).updater(new AdaGrad(0.5)).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).layer(4, new GravesLSTM.Builder().activation(Activation.SOFTSIGN).nIn(50).nOut(50).weightInit(WeightInit.XAVIER).updater(new AdaGrad(0.6)).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).layer(5, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(50).nOut(// 4 possible shapes: circle, square, arc, line - 4).weightInit(WeightInit.XAVIER).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).inputPreProcessor(0, new RnnToCnnPreProcessor(V_HEIGHT, V_WIDTH, 3)).inputPreProcessor(3, new CnnToFeedForwardPreProcessor(7, 7, 10)).inputPreProcessor(4, new FeedForwardToRnnPreProcessor()).backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(V_NFRAMES / 5).tBPTTBackwardLength(V_NFRAMES / 5).build(); - MultiLayerNetwork modelExpectedArch = new MultiLayerNetwork(confForArchitecture); - modelExpectedArch.init(); - MultiLayerNetwork modelToTweak = new MultiLayerNetwork(new NeuralNetConfiguration.Builder().seed(12345).updater(new RmsProp(0.1)).list().layer(0, // Only keep the first layer the same - new ConvolutionLayer.Builder(10, 10).nIn(// 3 channels: RGB - 3).nOut(30).stride(4, 4).activation(Activation.RELU).weightInit(WeightInit.RELU).updater(new AdaGrad(0.1)).build()).layer(1, new SubsamplingLayer.Builder(// change kernel size - SubsamplingLayer.PoolingType.MAX).kernelSize(5, 5).stride(2, 2).build()).layer(2, // change here - new ConvolutionLayer.Builder(6, 6).nIn(30).nOut(10).stride(2, 2).activation(Activation.RELU).weightInit(WeightInit.RELU).build()).layer(3, // change here - new DenseLayer.Builder().activation(Activation.RELU).nIn(250).nOut(50).weightInit(WeightInit.RELU).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).updater(new RmsProp(0.01)).build()).layer(4, // change here - new GravesLSTM.Builder().activation(Activation.SOFTSIGN).nIn(50).nOut(25).weightInit(WeightInit.XAVIER).build()).layer(5, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(25).nOut(4).weightInit(WeightInit.XAVIER).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).inputPreProcessor(0, new RnnToCnnPreProcessor(V_HEIGHT, V_WIDTH, 3)).inputPreProcessor(3, new CnnToFeedForwardPreProcessor(5, 5, 10)).inputPreProcessor(4, new FeedForwardToRnnPreProcessor()).backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(V_NFRAMES / 5).tBPTTBackwardLength(V_NFRAMES / 5).build()); - modelToTweak.init(); - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToTweak).fineTuneConfiguration(// l2 regularization on all layers - new FineTuneConfiguration.Builder().seed(12345).l2(0.001).updater(new AdaGrad(0.4)).weightInit(WeightInit.RELU).build()).removeLayersFromOutput(5).addLayer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(3, 3).stride(2, 2).build()).addLayer(new ConvolutionLayer.Builder(3, 3).nIn(30).nOut(10).stride(2, 2).activation(Activation.RELU).weightInit(WeightInit.RELU).build()).addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(490).nOut(50).weightInit(WeightInit.RELU).updater(new AdaGrad(0.5)).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).addLayer(new GravesLSTM.Builder().activation(Activation.SOFTSIGN).nIn(50).nOut(50).weightInit(WeightInit.XAVIER).updater(new AdaGrad(0.6)).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).addLayer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(50).nOut(// 4 possible shapes: circle, square, arc, line - 4).weightInit(WeightInit.XAVIER).gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(10).build()).setInputPreProcessor(3, new CnnToFeedForwardPreProcessor(7, 7, 10)).setInputPreProcessor(4, new FeedForwardToRnnPreProcessor()).build(); - // modelNow should have the same architecture as modelExpectedArch - assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(0).toJson(), modelNow.getLayerWiseConfigurations().getConf(0).toJson()); - // some learning related info the subsampling layer will not be overwritten - // assertTrue(modelExpectedArch.getLayerWiseConfigurations().getConf(1).toJson().equals(modelNow.getLayerWiseConfigurations().getConf(1).toJson())); - assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(2).toJson(), modelNow.getLayerWiseConfigurations().getConf(2).toJson()); - assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(3).toJson(), modelNow.getLayerWiseConfigurations().getConf(3).toJson()); - assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(4).toJson(), modelNow.getLayerWiseConfigurations().getConf(4).toJson()); - assertEquals(modelExpectedArch.getLayerWiseConfigurations().getConf(5).toJson(), modelNow.getLayerWiseConfigurations().getConf(5).toJson()); - assertArrayEquals(modelExpectedArch.params().shape(), modelNow.params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); - // subsampling has no params - // assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(2).params().shape(), modelNow.getLayer(2).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(3).params().shape(), modelNow.getLayer(3).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(4).params().shape(), modelNow.getLayer(4).params().shape()); - assertArrayEquals(modelExpectedArch.getLayer(5).params().shape(), modelNow.getLayer(5).params().shape()); - } - - @Test - @DisplayName("Test All With CNN") - void testAllWithCNN() { - Nd4j.getRandom().setSeed(12345); - DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 28 * 28 * 3).reshape(10, 3, 28, 28), TestUtils.randomOneHot(DataType.FLOAT, 10, 10)); - MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(new NeuralNetConfiguration.Builder().seed(123).weightInit(WeightInit.XAVIER).updater(new Nesterovs(0.01, 0.9)).list().layer(0, new ConvolutionLayer.Builder(5, 5).nIn(3).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(2, new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()).layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()).layer(5, new DenseLayer.Builder().activation(Activation.RELU).nOut(250).build()).layer(6, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(100).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(28, 28, 3)).build()); - modelToFineTune.init(); - // 10x20x12x12 - INDArray asFrozenFeatures = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false).get(2); - NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.2)).optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT); - FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.2)).build(); - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf).setFeatureExtractor(1).nOutReplace(4, 600, WeightInit.XAVIER).removeLayersFromOutput(2).addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(600).nOut(300).build()).addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(300).nOut(150).build()).addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(150).nOut(50).build()).addLayer(new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nIn(50).nOut(10).build()).build(); - MultiLayerNetwork notFrozen = new MultiLayerNetwork(equivalentConf.list().layer(0, new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()).layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(2, new DenseLayer.Builder().activation(Activation.RELU).nOut(600).build()).layer(3, new DenseLayer.Builder().activation(Activation.RELU).nOut(300).build()).layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(150).build()).layer(5, new DenseLayer.Builder().activation(Activation.RELU).nOut(50).build()).layer(6, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(10).activation(Activation.SOFTMAX).build()).setInputType(InputType.convolutionalFlat(12, 12, 20)).build()); - notFrozen.init(); - assertArrayEquals(modelToFineTune.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); - // subsampling has no params - // assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); - assertArrayEquals(notFrozen.getLayer(0).params().shape(), modelNow.getLayer(2).params().shape()); - modelNow.getLayer(2).setParams(notFrozen.getLayer(0).params()); - // subsampling has no params - // assertArrayEquals(notFrozen.getLayer(1).params().shape(), modelNow.getLayer(3).params().shape()); - assertArrayEquals(notFrozen.getLayer(2).params().shape(), modelNow.getLayer(4).params().shape()); - modelNow.getLayer(4).setParams(notFrozen.getLayer(2).params()); - assertArrayEquals(notFrozen.getLayer(3).params().shape(), modelNow.getLayer(5).params().shape()); - modelNow.getLayer(5).setParams(notFrozen.getLayer(3).params()); - assertArrayEquals(notFrozen.getLayer(4).params().shape(), modelNow.getLayer(6).params().shape()); - modelNow.getLayer(6).setParams(notFrozen.getLayer(4).params()); - assertArrayEquals(notFrozen.getLayer(5).params().shape(), modelNow.getLayer(7).params().shape()); - modelNow.getLayer(7).setParams(notFrozen.getLayer(5).params()); - assertArrayEquals(notFrozen.getLayer(6).params().shape(), modelNow.getLayer(8).params().shape()); - modelNow.getLayer(8).setParams(notFrozen.getLayer(6).params()); - int i = 0; - while (i < 3) { - notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); - modelNow.fit(randomData); - i++; - } - INDArray expectedParams = Nd4j.hstack(modelToFineTune.getLayer(0).params(), notFrozen.params()); - assertEquals(expectedParams, modelNow.params()); - } - - @Test - @DisplayName("Test Fine Tune Override") - void testFineTuneOverride() { - // Check that fine-tune overrides are selective - i.e., if I only specify a new LR, only the LR should be modified - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Adam(1e-4)).activation(Activation.TANH).weightInit(WeightInit.RELU).l1(0.1).l2(0.2).list().layer(0, new DenseLayer.Builder().nIn(10).nOut(5).build()).layer(1, new OutputLayer.Builder().nIn(5).nOut(4).activation(Activation.HARDSIGMOID).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - MultiLayerNetwork net2 = new TransferLearning.Builder(net).fineTuneConfiguration(new FineTuneConfiguration.Builder().updater(new Adam(2e-2)).backpropType(// Should be set on MLC - BackpropType.TruncatedBPTT).build()).build(); - // Check original net isn't modified: - BaseLayer l0 = (BaseLayer) net.getLayer(0).conf().getLayer(); - assertEquals(new Adam(1e-4), l0.getIUpdater()); - assertEquals(Activation.TANH.getActivationFunction(), l0.getActivationFn()); - assertEquals(new WeightInitRelu(), l0.getWeightInitFn()); - assertEquals(0.1, TestUtils.getL1(l0), 1e-6); - BaseLayer l1 = (BaseLayer) net.getLayer(1).conf().getLayer(); - assertEquals(new Adam(1e-4), l1.getIUpdater()); - assertEquals(Activation.HARDSIGMOID.getActivationFunction(), l1.getActivationFn()); - assertEquals(new WeightInitRelu(), l1.getWeightInitFn()); - assertEquals(0.2, TestUtils.getL2(l1), 1e-6); - assertEquals(BackpropType.Standard, conf.getBackpropType()); - // Check new net has only the appropriate things modified (i.e., LR) - l0 = (BaseLayer) net2.getLayer(0).conf().getLayer(); - assertEquals(new Adam(2e-2), l0.getIUpdater()); - assertEquals(Activation.TANH.getActivationFunction(), l0.getActivationFn()); - assertEquals(new WeightInitRelu(), l0.getWeightInitFn()); - assertEquals(0.1, TestUtils.getL1(l0), 1e-6); - l1 = (BaseLayer) net2.getLayer(1).conf().getLayer(); - assertEquals(new Adam(2e-2), l1.getIUpdater()); - assertEquals(Activation.HARDSIGMOID.getActivationFunction(), l1.getActivationFn()); - assertEquals(new WeightInitRelu(), l1.getWeightInitFn()); - assertEquals(0.2, TestUtils.getL2(l1), 1e-6); - assertEquals(BackpropType.TruncatedBPTT, net2.getLayerWiseConfigurations().getBackpropType()); - } - - @Test - @DisplayName("Test All With CNN New") - void testAllWithCNNNew() { - Nd4j.getRandom().setSeed(12345); - DataSet randomData = new DataSet(Nd4j.rand(DataType.FLOAT, 10, 28 * 28 * 3).reshape(10, 3, 28, 28), TestUtils.randomOneHot(10, 10)); - MultiLayerNetwork modelToFineTune = new MultiLayerNetwork(new NeuralNetConfiguration.Builder().seed(123).weightInit(WeightInit.XAVIER).updater(new Nesterovs(0.01, 0.9)).list().layer(0, new ConvolutionLayer.Builder(5, 5).nIn(3).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()).layer(1, new SubsamplingLayer.Builder(PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(2, new ConvolutionLayer.Builder(5, 5).stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()).layer(3, new SubsamplingLayer.Builder(PoolingType.MAX).kernelSize(2, 2).stride(2, 2).build()).layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()).layer(5, new DenseLayer.Builder().activation(Activation.RELU).nOut(250).build()).layer(6, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(100).activation(Activation.SOFTMAX).build()).setInputType(// See note below - InputType.convolutionalFlat(28, 28, 3)).build()); - modelToFineTune.init(); - // 10x20x12x12 - INDArray asFrozenFeatures = modelToFineTune.feedForwardToLayer(2, randomData.getFeatures(), false).get(2); - NeuralNetConfiguration.Builder equivalentConf = new NeuralNetConfiguration.Builder().updater(new Sgd(0.2)); - FineTuneConfiguration overallConf = new FineTuneConfiguration.Builder().updater(new Sgd(0.2)).build(); - MultiLayerNetwork modelNow = new TransferLearning.Builder(modelToFineTune).fineTuneConfiguration(overallConf).setFeatureExtractor(1).removeLayersFromOutput(5).addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(12 * 12 * 20).nOut(300).build()).addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(300).nOut(150).build()).addLayer(new DenseLayer.Builder().activation(Activation.RELU).nIn(150).nOut(50).build()).addLayer(new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).activation(Activation.SOFTMAX).nIn(50).nOut(10).build()).setInputPreProcessor(2, new CnnToFeedForwardPreProcessor(12, 12, 20)).build(); - MultiLayerNetwork notFrozen = new MultiLayerNetwork(equivalentConf.list().layer(0, new DenseLayer.Builder().activation(Activation.RELU).nIn(12 * 12 * 20).nOut(300).build()).layer(1, new DenseLayer.Builder().activation(Activation.RELU).nIn(300).nOut(150).build()).layer(2, new DenseLayer.Builder().activation(Activation.RELU).nIn(150).nOut(50).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nIn(50).nOut(10).activation(Activation.SOFTMAX).build()).inputPreProcessor(0, new CnnToFeedForwardPreProcessor(12, 12, 20)).build()); - notFrozen.init(); - assertArrayEquals(modelToFineTune.getLayer(0).params().shape(), modelNow.getLayer(0).params().shape()); - // subsampling has no params - // assertArrayEquals(modelExpectedArch.getLayer(1).params().shape(), modelNow.getLayer(1).params().shape()); - assertArrayEquals(notFrozen.getLayer(0).params().shape(), modelNow.getLayer(2).params().shape()); - modelNow.getLayer(2).setParams(notFrozen.getLayer(0).params()); - assertArrayEquals(notFrozen.getLayer(1).params().shape(), modelNow.getLayer(3).params().shape()); - modelNow.getLayer(3).setParams(notFrozen.getLayer(1).params()); - assertArrayEquals(notFrozen.getLayer(2).params().shape(), modelNow.getLayer(4).params().shape()); - modelNow.getLayer(4).setParams(notFrozen.getLayer(2).params()); - assertArrayEquals(notFrozen.getLayer(3).params().shape(), modelNow.getLayer(5).params().shape()); - modelNow.getLayer(5).setParams(notFrozen.getLayer(3).params()); - int i = 0; - while (i < 3) { - notFrozen.fit(new DataSet(asFrozenFeatures, randomData.getLabels())); - modelNow.fit(randomData); - i++; - } - INDArray expectedParams = Nd4j.hstack(modelToFineTune.getLayer(0).params(), notFrozen.params()); - assertEquals(expectedParams, modelNow.params()); - } - - @Test - @DisplayName("Test Object Overrides") - void testObjectOverrides() { - // https://github.com/eclipse/deeplearning4j/issues/4368 - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dropOut(0.5).weightNoise(new DropConnect(0.5)).l2(0.5).constrainWeights(new UnitNormConstraint()).list().layer(new DenseLayer.Builder().nIn(10).nOut(10).build()).build(); - MultiLayerNetwork orig = new MultiLayerNetwork(conf); - orig.init(); - FineTuneConfiguration ftc = new FineTuneConfiguration.Builder().dropOut(0).weightNoise(null).constraints(null).l2(0.0).build(); - MultiLayerNetwork transfer = new TransferLearning.Builder(orig).fineTuneConfiguration(ftc).build(); - DenseLayer l = (DenseLayer) transfer.getLayer(0).conf().getLayer(); - assertNull(l.getIDropout()); - assertNull(l.getWeightNoise()); - assertNull(l.getConstraints()); - assertNull(TestUtils.getL2Reg(l)); - } - - @Test - @DisplayName("Test Transfer Learning Subsequent") - void testTransferLearningSubsequent() { - final INDArray input = Nd4j.create(6, 6, 6, 6); - final MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder().weightInit(new ConstantDistribution(666)).list().setInputType(InputType.inferInputTypes(input)[0]).layer(new Convolution2D.Builder(3, 3).nOut(10).build()).layer(new Convolution2D.Builder(1, 1).nOut(3).build()).layer(new OutputLayer.Builder().nOut(2).lossFunction(LossFunctions.LossFunction.MSE).build()).build()); - net.init(); - MultiLayerNetwork newGraph = new TransferLearning.Builder(net).fineTuneConfiguration(new FineTuneConfiguration.Builder().build()).nOutReplace(0, 7, new ConstantDistribution(333)).nOutReplace(1, 3, new ConstantDistribution(111)).removeLayersFromOutput(1).addLayer(new OutputLayer.Builder().nIn(48).nOut(2).lossFunction(LossFunctions.LossFunction.MSE).build()).setInputPreProcessor(2, new CnnToFeedForwardPreProcessor(4, 4, 3)).build(); - newGraph.init(); - assertEquals(7, newGraph.layerInputSize(1), "Incorrect # inputs"); - newGraph.output(input); - } - - @Test - @DisplayName("Test Change N Out N In") - void testChangeNOutNIn() { - INDArray input = Nd4j.create(new long[] { 1, 2, 4, 4 }); - MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder().list().setInputType(InputType.inferInputTypes(input)[0]).layer(new Convolution2D.Builder(1, 1).nOut(10).build()).layer(new SubsamplingLayer.Builder(1, 1).build()).layer(new Convolution2D.Builder(1, 1).nOut(7).build()).layer(new OutputLayer.Builder().activation(Activation.SOFTMAX).nOut(2).build()).build()); - net.init(); - final MultiLayerNetwork newNet = new TransferLearning.Builder(net).nOutReplace(0, 5, WeightInit.XAVIER).nInReplace(2, 5, WeightInit.XAVIER).build(); - newNet.init(); - assertEquals(5, newNet.layerSize(0), "Incorrect number of outputs!"); - assertEquals(5, newNet.layerInputSize(2), "Incorrect number of inputs!"); - newNet.output(input); - } - - @Test - @DisplayName("Test Transfer Learning Same Diff Layers") - void testTransferLearningSameDiffLayers() { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).activation(Activation.TANH).updater(new Adam(0.01)).weightInit(WeightInit.XAVIER).list().layer(new LSTM.Builder().nOut(8).build()).layer(new SelfAttentionLayer.Builder().nOut(4).nHeads(2).projectInput(true).build()).layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build()).layer(new OutputLayer.Builder().nOut(2).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).setInputType(InputType.recurrent(4)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - INDArray in = Nd4j.rand(DataType.FLOAT, 3, 4, 5); - INDArray out = net.output(in); - MultiLayerNetwork net2 = new TransferLearning.Builder(net).fineTuneConfiguration(FineTuneConfiguration.builder().updater(new Adam(0.01)).build()).removeLayersFromOutput(1).addLayer(new OutputLayer.Builder().nIn(4).nOut(2).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()).build(); - net2.setParam("3_W", net.getParam("3_W")); - net2.setParam("3_b", net.getParam("3_b")); - Map p1 = net.paramTable(); - Map p2 = net2.paramTable(); - for (String s : p1.keySet()) { - INDArray i1 = p1.get(s); - INDArray i2 = p2.get(s); - assertEquals(i1, i2,s); - } - INDArray out2 = net2.output(in); - assertEquals(out, out2); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/LegacyWeightInitTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/LegacyWeightInitTest.java deleted file mode 100644 index 968985bac..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/LegacyWeightInitTest.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.weights; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.distribution.*; -import org.deeplearning4j.nn.conf.serde.JsonMappers; -import org.junit.jupiter.api.*; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.rng.Random; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.factory.RandomFactory; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import static org.junit.jupiter.api.Assertions.*; - -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Legacy Weight Init Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class LegacyWeightInitTest extends BaseDL4JTest { - - private RandomFactory prevFactory; - - private final static int SEED = 666; - - private final static List distributions = Arrays.asList(new LogNormalDistribution(12.3, 4.56), new BinomialDistribution(3, 0.3), new NormalDistribution(0.666, 0.333), new UniformDistribution(-1.23, 4.56), new OrthogonalDistribution(3.45), new TruncatedNormalDistribution(0.456, 0.123), new ConstantDistribution(666)); - - @BeforeEach - void setRandomFactory() { - prevFactory = Nd4j.randomFactory; - Nd4j.randomFactory = new FixedSeedRandomFactory(prevFactory); - } - - @AfterEach - void resetRandomFactory() { - Nd4j.randomFactory = prevFactory; - } - - /** - * Test that param init is identical to legacy implementation - */ - @Test - @DisplayName("Init Params") - void initParams() { - // To make identity happy - final long[] shape = { 5, 5 }; - final long fanIn = shape[0]; - final long fanOut = shape[1]; - final INDArray inLegacy = Nd4j.create(DataType.DOUBLE,fanIn * fanOut); - final INDArray inTest = inLegacy.dup(); - for (WeightInit legacyWi : WeightInit.values()) { - if (legacyWi != WeightInit.DISTRIBUTION) { - Nd4j.getRandom().setSeed(SEED); - final INDArray expected = WeightInitUtil. - initWeights(fanIn, fanOut, shape, legacyWi, null, inLegacy) - .castTo(DataType.DOUBLE); - Nd4j.getRandom().setSeed(SEED); - final INDArray actual = legacyWi.getWeightInitFunction() - .init(fanIn, fanOut, shape, - WeightInitUtil.DEFAULT_WEIGHT_INIT_ORDER, inTest) - .castTo(DataType.DOUBLE); - assertArrayEquals(shape, actual.shape(),"Incorrect shape for " + legacyWi + "!"); - assertEquals( expected, actual,"Incorrect weight initialization for " + legacyWi + "!"); - } - } - } - - /** - * Test that param init is identical to legacy implementation - */ - @Test - @DisplayName("Init Params From Distribution") - @Execution(ExecutionMode.SAME_THREAD) - @Disabled(TagNames.NEEDS_VERIFY) - void initParamsFromDistribution() { - // To make identity happy - final long[] shape = { 3, 7 }; - final long fanIn = shape[0]; - final long fanOut = shape[1]; - final INDArray inLegacy = Nd4j.create(DataType.DOUBLE,fanIn * fanOut); - final INDArray inTest = inLegacy.dup(); - for (Distribution dist : distributions) { - Nd4j.getRandom().setSeed(SEED); - final INDArray expected = WeightInitUtil - .initWeights(fanIn, fanOut, shape, WeightInit.DISTRIBUTION, - Distributions.createDistribution(dist), inLegacy) - .castTo(DataType.DOUBLE); - final INDArray actual = new WeightInitDistribution(dist) - .init(fanIn, fanOut, shape, WeightInitUtil.DEFAULT_WEIGHT_INIT_ORDER, - inTest).castTo(DataType.DOUBLE); - assertArrayEquals(shape, actual.shape(),"Incorrect shape for " + dist.getClass().getSimpleName() + "!"); - assertEquals( expected, actual,"Incorrect weight initialization for " + dist.getClass().getSimpleName() + "!"); - } - } - - /** - * Test that weight inits can be serialized and de-serialized in JSON format - */ - @Test - @DisplayName("Serialize Deserialize Json") - void serializeDeserializeJson() throws IOException { - // To make identity happy - final long[] shape = { 5, 5 }; - final long fanIn = shape[0]; - final long fanOut = shape[1]; - final ObjectMapper mapper = JsonMappers.getMapper(); - final INDArray inBefore = Nd4j.create(fanIn * fanOut); - final INDArray inAfter = inBefore.dup(); - // Just use to enum to loop over all strategies - for (WeightInit legacyWi : WeightInit.values()) { - if (legacyWi != WeightInit.DISTRIBUTION) { - Nd4j.getRandom().setSeed(SEED); - final IWeightInit before = legacyWi.getWeightInitFunction(); - final INDArray expected = before.init(fanIn, fanOut, shape, inBefore.ordering(), inBefore); - final String json = mapper.writeValueAsString(before); - final IWeightInit after = mapper.readValue(json, IWeightInit.class); - Nd4j.getRandom().setSeed(SEED); - final INDArray actual = after.init(fanIn, fanOut, shape, inAfter.ordering(), inAfter); - assertArrayEquals( shape, actual.shape(),"Incorrect shape for " + legacyWi + "!"); - assertEquals(expected, actual,"Incorrect weight initialization for " + legacyWi + "!"); - } - } - } - - /** - * Test that distribution can be serialized and de-serialized in JSON format - */ - @Test - @DisplayName("Serialize Deserialize Distribution Json") - @Disabled("") - @Tag(TagNames.NEEDS_VERIFY) - void serializeDeserializeDistributionJson() throws IOException { - // To make identity happy - final long[] shape = { 3, 7 }; - final long fanIn = shape[0]; - final long fanOut = shape[1]; - final ObjectMapper mapper = JsonMappers.getMapper(); - final INDArray inBefore = Nd4j.create(fanIn * fanOut); - final INDArray inAfter = inBefore.dup(); - for (Distribution dist : distributions) { - Nd4j.getRandom().setSeed(SEED); - final IWeightInit before = new WeightInitDistribution(dist); - final INDArray expected = before.init(fanIn, fanOut, shape, inBefore.ordering(), inBefore); - final String json = mapper.writeValueAsString(before); - final IWeightInit after = mapper.readValue(json, IWeightInit.class); - Nd4j.getRandom().setSeed(SEED); - final INDArray actual = after.init(fanIn, fanOut, shape, inAfter.ordering(), inAfter); - assertArrayEquals(shape, actual.shape(),"Incorrect shape for " + dist.getClass().getSimpleName() + "!"); - assertEquals(expected, actual,"Incorrect weight initialization for " + dist.getClass().getSimpleName() + "!"); - } - } - - /** - * Test equals and hashcode implementation. Redundant as one can trust Lombok on this?? - */ - @Test - @DisplayName("Equals And Hash Code") - void equalsAndHashCode() { - WeightInit lastInit = WeightInit.values()[WeightInit.values().length - 1]; - for (WeightInit legacyWi : WeightInit.values()) { - if (legacyWi != WeightInit.DISTRIBUTION) { - assertEquals(legacyWi.getWeightInitFunction(), legacyWi.getWeightInitFunction(), "Shall be equal!"); - assertNotEquals(lastInit.getWeightInitFunction(), legacyWi.getWeightInitFunction(), "Shall not be equal!"); - if (legacyWi != WeightInit.NORMAL && legacyWi != WeightInit.LECUN_NORMAL) { - lastInit = legacyWi; - } - } - } - Distribution lastDist = distributions.get(distributions.size() - 1); - for (Distribution distribution : distributions) { - assertEquals(new WeightInitDistribution(distribution), new WeightInitDistribution(distribution.clone()), "Shall be equal!"); - assertNotEquals(new WeightInitDistribution(lastDist), new WeightInitDistribution(distribution), "Shall not be equal!"); - lastDist = distribution; - } - } - - /** - * Assumes RandomFactory will only call no-args constructor while this test runs - */ - @DisplayName("Fixed Seed Random Factory") - private static class FixedSeedRandomFactory extends RandomFactory { - - private final RandomFactory factory; - - private FixedSeedRandomFactory(RandomFactory factory) { - super(factory.getRandom().getClass()); - this.factory = factory; - } - - @Override - public Random getRandom() { - return getNewRandomInstance(SEED); - } - - @Override - public Random getNewRandomInstance() { - return factory.getNewRandomInstance(); - } - - @Override - public Random getNewRandomInstance(long seed) { - return factory.getNewRandomInstance(seed); - } - - @Override - public Random getNewRandomInstance(long seed, long size) { - return factory.getNewRandomInstance(seed, size); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitIdentityTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitIdentityTest.java deleted file mode 100644 index 36187f768..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/weights/WeightInitIdentityTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.weights; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.RNNFormat; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.impl.ActivationIdentity; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Weight Init Identity Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class WeightInitIdentityTest extends BaseDL4JTest { - - /** - * Test identity mapping for 1d convolution - */ - @Test - @Disabled("Ignore for now. Underlying logic changed. Gradient checker passes so implementatin is valid.") - @DisplayName("Test Id Conv 1 D") - void testIdConv1D() { - final INDArray input = Nd4j.randn(DataType.FLOAT, 1, 5, 7); - final String inputName = "input"; - final String conv = "conv"; - final String output = "output"; - final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder().graphBuilder().addInputs(inputName).setOutputs(output).layer(conv, new Convolution1DLayer.Builder(7).convolutionMode(ConvolutionMode.Same).nOut(input.size(1)).weightInit(new WeightInitIdentity()).activation(new ActivationIdentity()).build(), inputName).layer(output, new RnnLossLayer.Builder().activation(new ActivationIdentity()).build(), conv).setInputTypes(InputType.recurrent(5, 7, RNNFormat.NCW)).build()); - graph.init(); - INDArray reshape = graph.outputSingle(input).reshape(input.shape()); - assertEquals(input, reshape, "Mapping was not identity!"); - } - - /** - * Test identity mapping for 2d convolution - */ - @Test - @DisplayName("Test Id Conv 2 D") - void testIdConv2D() { - final INDArray input = Nd4j.randn(DataType.FLOAT, 1, 5, 7, 11); - final String inputName = "input"; - final String conv = "conv"; - final String output = "output"; - final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder().graphBuilder().setInputTypes(InputType.inferInputType(input)).addInputs(inputName).setOutputs(output).layer(conv, new ConvolutionLayer.Builder(3, 5).convolutionMode(ConvolutionMode.Same).nOut(input.size(1)).weightInit(new WeightInitIdentity()).activation(new ActivationIdentity()).build(), inputName).layer(output, new CnnLossLayer.Builder().activation(new ActivationIdentity()).build(), conv).build()); - graph.init(); - assertEquals(input, graph.outputSingle(input), "Mapping was not identity!"); - } - - /** - * Test identity mapping for 3d convolution - */ - @Test - @DisplayName("Test Id Conv 3 D") - void testIdConv3D() { - final INDArray input = Nd4j.randn(DataType.FLOAT, 1, 5, 7, 11, 13); - final String inputName = "input"; - final String conv = "conv"; - final String output = "output"; - final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder().graphBuilder().setInputTypes(InputType.inferInputType(input)).addInputs(inputName).setOutputs(output).layer(conv, new Convolution3D.Builder(3, 7, 5).convolutionMode(ConvolutionMode.Same).dataFormat(Convolution3D.DataFormat.NCDHW).nOut(input.size(1)).weightInit(new WeightInitIdentity()).activation(new ActivationIdentity()).build(), inputName).layer(output, new Cnn3DLossLayer.Builder(Convolution3D.DataFormat.NCDHW).activation(new ActivationIdentity()).build(), conv).build()); - graph.init(); - assertEquals(input, graph.outputSingle(input), "Mapping was not identity!"); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/BackTrackLineSearchTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/BackTrackLineSearchTest.java deleted file mode 100644 index ea8eb9e82..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/BackTrackLineSearchTest.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.optimize.solver; - -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.layers.OutputLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.deeplearning4j.optimize.api.TrainingListener; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.deeplearning4j.optimize.solvers.BackTrackLineSearch; -import org.deeplearning4j.optimize.stepfunctions.DefaultStepFunction; -import org.deeplearning4j.optimize.stepfunctions.NegativeDefaultStepFunction; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.Collections; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * @author Adam Gibson - */ -@DisplayName("Back Track Line Search Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.FILE_IO) -class BackTrackLineSearchTest extends BaseDL4JTest { - - private DataSetIterator irisIter; - - private DataSet irisData; - - @BeforeEach - void before() { - if (irisIter == null) { - irisIter = new IrisDataSetIterator(5, 5); - } - if (irisData == null) { - irisData = irisIter.next(); - irisData.normalizeZeroMeanZeroUnitVariance(); - } - } - - @Test - @DisplayName("Test Single Min Line Search") - void testSingleMinLineSearch() throws Exception { - OutputLayer layer = getIrisLogisticLayerConfig(Activation.SOFTMAX, 100, LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD); - int nParams = (int) layer.numParams(); - layer.setBackpropGradientsViewArray(Nd4j.create(1, nParams)); - layer.setInput(irisData.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); - layer.setLabels(irisData.getLabels()); - layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); - BackTrackLineSearch lineSearch = new BackTrackLineSearch(layer, layer.getOptimizer()); - double step = lineSearch.optimize(layer.params(), layer.gradient().gradient(), layer.gradient().gradient(), LayerWorkspaceMgr.noWorkspacesImmutable()); - assertEquals(1.0, step, 1e-3); - } - - @Test - @DisplayName("Test Single Max Line Search") - void testSingleMaxLineSearch() throws Exception { - double score1, score2; - OutputLayer layer = getIrisLogisticLayerConfig(Activation.SOFTMAX, 100, LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD); - int nParams = (int) layer.numParams(); - layer.setBackpropGradientsViewArray(Nd4j.create(1, nParams)); - layer.setInput(irisData.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); - layer.setLabels(irisData.getLabels()); - layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); - score1 = layer.score(); - BackTrackLineSearch lineSearch = new BackTrackLineSearch(layer, new NegativeDefaultStepFunction(), layer.getOptimizer()); - double step = lineSearch.optimize(layer.params(), layer.gradient().gradient(), layer.gradient().gradient(), LayerWorkspaceMgr.noWorkspacesImmutable()); - assertEquals(1.0, step, 1e-3); - } - - @Test - @DisplayName("Test Mult Min Line Search") - void testMultMinLineSearch() throws Exception { - double score1, score2; - OutputLayer layer = getIrisLogisticLayerConfig(Activation.SOFTMAX, 100, LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD); - int nParams = (int) layer.numParams(); - layer.setBackpropGradientsViewArray(Nd4j.create(1, nParams)); - layer.setInput(irisData.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); - layer.setLabels(irisData.getLabels()); - layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); - score1 = layer.score(); - INDArray origGradient = layer.gradient().gradient().dup(); - NegativeDefaultStepFunction sf = new NegativeDefaultStepFunction(); - BackTrackLineSearch lineSearch = new BackTrackLineSearch(layer, sf, layer.getOptimizer()); - double step = lineSearch.optimize(layer.params(), layer.gradient().gradient(), layer.gradient().gradient(), LayerWorkspaceMgr.noWorkspacesImmutable()); - INDArray currParams = layer.params(); - sf.step(currParams, origGradient, step); - layer.setParams(currParams); - layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); - score2 = layer.score(); - assertTrue(score1 > score2,"score1=" + score1 + ", score2=" + score2); - } - - @Test - @DisplayName("Test Mult Max Line Search") - void testMultMaxLineSearch() throws Exception { - double score1, score2; - irisData.normalizeZeroMeanZeroUnitVariance(); - OutputLayer layer = getIrisLogisticLayerConfig(Activation.SOFTMAX, 100, LossFunctions.LossFunction.MCXENT); - int nParams = (int) layer.numParams(); - layer.setBackpropGradientsViewArray(Nd4j.create(1, nParams)); - layer.setInput(irisData.getFeatures(), LayerWorkspaceMgr.noWorkspaces()); - layer.setLabels(irisData.getLabels()); - layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); - score1 = layer.score(); - INDArray origGradient = layer.gradient().gradient().dup(); - DefaultStepFunction sf = new DefaultStepFunction(); - BackTrackLineSearch lineSearch = new BackTrackLineSearch(layer, sf, layer.getOptimizer()); - double step = lineSearch.optimize(layer.params().dup(), layer.gradient().gradient().dup(), layer.gradient().gradient().dup(), LayerWorkspaceMgr.noWorkspacesImmutable()); - INDArray currParams = layer.params(); - sf.step(currParams, origGradient, step); - layer.setParams(currParams); - layer.computeGradientAndScore(LayerWorkspaceMgr.noWorkspaces()); - score2 = layer.score(); - assertTrue(score1 < score2,"score1 = " + score1 + ", score2 = " + score2); - } - - private static OutputLayer getIrisLogisticLayerConfig(Activation activationFunction, int maxIterations, LossFunctions.LossFunction lossFunction) { - NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345L).miniBatch(true).maxNumLineSearchIterations(maxIterations).layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(lossFunction).nIn(4).nOut(3).activation(activationFunction).weightInit(WeightInit.XAVIER).build()).build(); - val numParams = conf.getLayer().initializer().numParams(conf); - INDArray params = Nd4j.create(1, numParams); - return (OutputLayer) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); - } - - // ///////////////////////////////////////////////////////////////////////// - @Test - @DisplayName("Test Back Track Line Gradient Descent") - void testBackTrackLineGradientDescent() { - OptimizationAlgorithm optimizer = OptimizationAlgorithm.LINE_GRADIENT_DESCENT; - DataSetIterator irisIter = new IrisDataSetIterator(1, 1); - DataSet data = irisIter.next(); - MultiLayerNetwork network = new MultiLayerNetwork(getIrisMultiLayerConfig(Activation.SIGMOID, optimizer)); - network.init(); - TrainingListener listener = new ScoreIterationListener(10); - network.setListeners(Collections.singletonList(listener)); - double oldScore = network.score(data); - for (int i = 0; i < 100; i++) { - network.fit(data.getFeatures(), data.getLabels()); - } - double score = network.score(); - assertTrue(score < oldScore); - } - - @Test - @DisplayName("Test Back Track Line CG") - void testBackTrackLineCG() { - OptimizationAlgorithm optimizer = OptimizationAlgorithm.CONJUGATE_GRADIENT; - DataSet data = irisIter.next(); - data.normalizeZeroMeanZeroUnitVariance(); - MultiLayerNetwork network = new MultiLayerNetwork(getIrisMultiLayerConfig(Activation.RELU, optimizer)); - network.init(); - TrainingListener listener = new ScoreIterationListener(10); - network.setListeners(Collections.singletonList(listener)); - double firstScore = network.score(data); - for (int i = 0; i < 5; i++) { - network.fit(data.getFeatures(), data.getLabels()); - } - double score = network.score(); - assertTrue(score < firstScore); - } - - @Test - @DisplayName("Test Back Track Line LBFGS") - void testBackTrackLineLBFGS() { - OptimizationAlgorithm optimizer = OptimizationAlgorithm.LBFGS; - DataSet data = irisIter.next(); - data.normalizeZeroMeanZeroUnitVariance(); - MultiLayerNetwork network = new MultiLayerNetwork(getIrisMultiLayerConfig(Activation.RELU, optimizer)); - network.init(); - TrainingListener listener = new ScoreIterationListener(10); - network.setListeners(Collections.singletonList(listener)); - double oldScore = network.score(data); - for (int i = 0; i < 5; i++) { - network.fit(data.getFeatures(), data.getLabels()); - } - double score = network.score(); - assertTrue(score < oldScore); - } - - private static MultiLayerConfiguration getIrisMultiLayerConfig(Activation activationFunction, OptimizationAlgorithm optimizer) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().optimizationAlgo(optimizer).updater(new Adam(0.01)).seed(12345L).list().layer(0, new DenseLayer.Builder().nIn(4).nOut(100).weightInit(WeightInit.XAVIER).activation(activationFunction).build()).layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(100).nOut(3).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build()).build(); - return conf; - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java deleted file mode 100644 index adeb00d93..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimize/solver/accumulation/SmartFancyBlockingQueueTest.java +++ /dev/null @@ -1,326 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.optimize.solver.accumulation; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.apache.commons.lang3.RandomUtils; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.optimize.solvers.accumulation.SmartFancyBlockingQueue; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.nd4j.common.util.ThreadUtils; -import org.nd4j.linalg.factory.Nd4j; -import java.util.ArrayList; -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.CyclicBarrier; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import static java.time.Duration.ofMillis; -import static org.junit.jupiter.api.Assertions.assertTimeout; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@Disabled("AB 2019/05/21 - Failing (stuck, causing timeouts) - Issue #7657") -@DisplayName("Smart Fancy Blocking Queue Test") -class SmartFancyBlockingQueueTest extends BaseDL4JTest { - - @Test - @DisplayName("Test SFBQ _ 1") - void testSFBQ_1() { - assertTimeout(ofMillis(120000), () -> { - val queue = new SmartFancyBlockingQueue(8, Nd4j.create(5, 5)); - val array = Nd4j.create(5, 5); - for (int e = 0; e < 6; e++) { - queue.put(Nd4j.create(5, 5).assign(e)); - } - ; - assertEquals(6, queue.size()); - for (int e = 6; e < 10; e++) { - queue.put(Nd4j.create(5, 5).assign(e)); - } - assertEquals(1, queue.size()); - }); - } - - @Test - @DisplayName("Test SFBQ _ 2") - void testSFBQ_2() { - assertTimeout(ofMillis(120000), () -> { - final val queue = new SmartFancyBlockingQueue(1285601, Nd4j.create(5, 5)); - final val barrier = new CyclicBarrier(4); - val threads = new ArrayList(); - for (int e = 0; e < 4; e++) { - val f = e; - val t = new Thread(new Runnable() { - - @Override - public void run() { - int cnt = 0; - while (true) { - while (cnt < 1000) { - if (!queue.isEmpty()) { - if (cnt % 50 == 0) - log.info("Thread {}: [{}]", f, cnt); - val arr = queue.poll(); - assertNotNull(arr); - val local = arr.unsafeDuplication(true); - assertEquals(cnt, local.meanNumber().intValue()); - cnt++; - } - try { - barrier.await(); - if (f == 0) - queue.registerConsumers(4); - barrier.await(); - } catch (InterruptedException e1) { - e1.printStackTrace(); - } catch (BrokenBarrierException e1) { - e1.printStackTrace(); - } - } - break; - } - } - }); - t.setName("reader thread " + f); - t.start(); - threads.add(t); - } - for (int e = 0; e < 1000; e++) { - queue.put(Nd4j.create(5, 5).assign(e)); - Nd4j.getExecutioner().commit(); - } - for (val t : threads) t.join(); - }); - } - - @Test - @DisplayName("Test SFBQ _ 3") - void testSFBQ_3() { - assertTimeout(ofMillis(120000), () -> { - final val queue = new SmartFancyBlockingQueue(1285601, Nd4j.create(5, 5)); - val threads = new ArrayList(); - for (int e = 0; e < 4; e++) { - val f = e; - val t = new Thread(new Runnable() { - - @Override - public void run() { - int cnt = 0; - while (true) { - while (cnt < 1000) { - if (!queue.isEmpty()) { - if (cnt % 50 == 0) - log.info("Thread {}: [{}]", f, cnt); - val arr = queue.poll(); - assertNotNull(arr); - val local = arr.unsafeDuplication(true); - cnt++; - } - } - break; - } - } - }); - t.start(); - threads.add(t); - } - val b = new Thread(new Runnable() { - - @Override - public void run() { - while (true) { - queue.registerConsumers(4); - ThreadUtils.uncheckedSleep(30); - } - } - }); - b.setDaemon(true); - b.start(); - val writers = new ArrayList(); - for (int e = 0; e < 4; e++) { - val t = new Thread(new Runnable() { - - @Override - public void run() { - for (int e = 0; e < 250; e++) { - try { - queue.put(Nd4j.createUninitialized(5, 5).assign(e)); - Thread.sleep(30); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - } - }); - writers.add(t); - t.start(); - } - for (val t : writers) t.join(); - for (val t : threads) t.join(); - }); - } - - @Test - @DisplayName("Test SFBQ _ 4") - void testSFBQ_4() { - assertTimeout(ofMillis(120000), () -> { - final val queue = new SmartFancyBlockingQueue(16, Nd4j.create(5, 5)); - final val barrier = new CyclicBarrier(4); - /* - val m = new Thread(new Runnable() { - @Override - public void run() { - while (true) { - queue.registerConsumers(4); - ThreadUtils.uncheckedSleep(100); - } - } - }); - - - m.setName("master thread"); - m.setDaemon(true); - m.start(); -*/ - val threads = new ArrayList(); - for (int e = 0; e < 4; e++) { - val f = e; - val t = new Thread(new Runnable() { - - @Override - public void run() { - try { - for (int e = 0; e < 100; e++) { - log.info("[Thread {}]: fill phase {}", f, e); - val numUpdates = RandomUtils.nextInt(8, 128); - for (int p = 0; p < numUpdates; p++) { - queue.put(Nd4j.createUninitialized(5, 5)); - } - if (f == 0) - queue.registerConsumers(4); - barrier.await(); - log.info("[Thread {}]: read phase {}", f, e); - while (!queue.isEmpty()) { - val arr = queue.poll(); - assertNotNull(arr); - } - barrier.await(); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (BrokenBarrierException e) { - throw new RuntimeException(e); - } - } - }); - t.setName("worker thread " + f); - t.start(); - threads.add(t); - } - for (val t : threads) t.join(); - }); - } - - @Test - @DisplayName("Test SFBQ _ 5") - void testSFBQ_5() { - assertTimeout(ofMillis(120000), () -> { - final val queue = new SmartFancyBlockingQueue(16, Nd4j.create(5, 5)); - final val barrier = new CyclicBarrier(4); - // writers are just spamming updates every X ms - val writers = new ArrayList(); - for (int e = 0; e < 4; e++) { - val w = new Thread(new Runnable() { - - @Override - public void run() { - while (true) { - try { - val n = RandomUtils.nextInt(8, 64); - for (int i = 1; i < n + 1; i++) { - val arr = Nd4j.createUninitialized(5, 5).assign(i); - Nd4j.getExecutioner().commit(); - queue.put(arr); - } - ThreadUtils.uncheckedSleep(10); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - } - }); - w.setName("writer thread " + e); - w.setDaemon(true); - w.start(); - writers.add(w); - } - // each reader will read 250 updates. supposedly equal :) - final long[] means = new long[4]; - val readers = new ArrayList(); - for (int e = 0; e < 4; e++) { - final int f = e; - means[f] = 0; - val t = new Thread(new Runnable() { - - @Override - public void run() { - try { - int cnt = 0; - int fnt = 0; - while (cnt < 1000) { - if (!queue.isEmpty()) { - while (!queue.isEmpty()) { - val m = queue.poll(); - val arr = m.unsafeDuplication(true); - val mean = arr.meanNumber().longValue(); - assertNotEquals(0, mean,"Failed at cycle: " + cnt); - means[f] += mean; - cnt++; - } - barrier.await(); - } - barrier.await(); - if (f == 0) { - log.info("Read cycle finished"); - queue.registerConsumers(4); - } - barrier.await(); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } catch (BrokenBarrierException e) { - throw new RuntimeException(e); - } - } - }); - t.setName("reader thread " + f); - t.start(); - readers.add(t); - } - for (val t : readers) t.join(); - // all messages should be the same - assertEquals(means[0], means[1]); - assertEquals(means[0], means[2]); - assertEquals(means[0], means[3]); - }); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java deleted file mode 100644 index 6798a2094..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/optimizer/listener/TestListeners.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.optimizer.listener; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.core.storage.StatsStorageRouter; -import org.deeplearning4j.core.storage.listener.RoutingIterationListener; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.Model; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.AutoEncoder; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.optimize.api.BaseTrainingListener; -import org.deeplearning4j.optimize.api.TrainingListener; -import org.deeplearning4j.optimize.listeners.ComposableIterationListener; -import org.deeplearning4j.optimize.listeners.PerformanceListener; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.deeplearning4j.optimize.listeners.TimeIterationListener; -import org.deeplearning4j.optimize.listeners.CheckpointListener; -import org.deeplearning4j.optimize.solvers.BaseOptimizer; - -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.primitives.Triple; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@Slf4j -public class TestListeners extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 90000L; - } - - @Test - public void testSettingListenersUnsupervised() { - //Pretrain layers should get copies of the listeners, in addition to the - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list() - .layer(0, new AutoEncoder.Builder().nIn(10).nOut(10).build()) - .layer(1, new VariationalAutoencoder.Builder().nIn(10).nOut(10).build()).build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - net.setListeners(new ScoreIterationListener(), new TestRoutingListener()); - - for (Layer l : net.getLayers()) { - Collection layerListeners = l.getListeners(); - assertEquals(2, layerListeners.size(),l.getClass().toString()); - TrainingListener[] lArr = layerListeners.toArray(new TrainingListener[2]); - assertTrue(lArr[0] instanceof ScoreIterationListener); - assertTrue(lArr[1] instanceof TestRoutingListener); - } - - Collection netListeners = net.getListeners(); - assertEquals(2, netListeners.size()); - TrainingListener[] lArr = netListeners.toArray(new TrainingListener[2]); - assertTrue(lArr[0] instanceof ScoreIterationListener); - assertTrue(lArr[1] instanceof TestRoutingListener); - - - ComputationGraphConfiguration gConf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in") - .addLayer("0", new AutoEncoder.Builder().nIn(10).nOut(10).build(), "in") - .addLayer("1", new VariationalAutoencoder.Builder().nIn(10).nOut(10).build(), "0") - .setOutputs("1").build(); - ComputationGraph cg = new ComputationGraph(gConf); - cg.init(); - - cg.setListeners(new ScoreIterationListener(), new TestRoutingListener()); - - for (Layer l : cg.getLayers()) { - Collection layerListeners = l.getListeners(); - assertEquals(2, layerListeners.size()); - lArr = layerListeners.toArray(new TrainingListener[2]); - assertTrue(lArr[0] instanceof ScoreIterationListener); - assertTrue(lArr[1] instanceof TestRoutingListener); - } - - netListeners = cg.getListeners(); - assertEquals(2, netListeners.size()); - lArr = netListeners.toArray(new TrainingListener[2]); - assertTrue(lArr[0] instanceof ScoreIterationListener); - assertTrue(lArr[1] instanceof TestRoutingListener); - } - - private static class TestRoutingListener extends BaseTrainingListener implements RoutingIterationListener { - - @Override - public void setStorageRouter(StatsStorageRouter router) {} - - @Override - public StatsStorageRouter getStorageRouter() { - return null; - } - - @Override - public void setWorkerID(String workerID) {} - - @Override - public String getWorkerID() { - return null; - } - - @Override - public void setSessionID(String sessionID) {} - - @Override - public String getSessionID() { - return null; - } - - @Override - public RoutingIterationListener clone() { - return null; - } - - @Override - public void iterationDone(Model model, int iteration, int epoch) {} - } - - - - - - @Test - public void testListenerSerialization(@TempDir Path tempDir) throws Exception { - //Note: not all listeners are (or should be) serializable. But some should be - for Spark etc - - List listeners = new ArrayList<>(); - listeners.add(new ScoreIterationListener()); - listeners.add(new PerformanceListener(1, true, true)); - listeners.add(new TimeIterationListener(10000)); - listeners.add(new ComposableIterationListener(new ScoreIterationListener(), new PerformanceListener(1, true, true))); - listeners.add(new CheckpointListener.Builder(tempDir.toFile()).keepAll().saveEveryNIterations(3).build()); //Doesn't usually need to be serialized, but no reason it can't be... - - - DataSetIterator iter = new IrisDataSetIterator(10, 150); - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .list() - .layer(new OutputLayer.Builder().nIn(4).nOut(3) - .activation(Activation.SOFTMAX) - .lossFunction(LossFunctions.LossFunction.MCXENT).build()) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - net.setListeners(listeners); - - net.fit(iter); - - List listeners2 = new ArrayList<>(); - for(TrainingListener il : listeners){ - log.info("------------------"); - log.info("Testing listener: {}", il); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(baos); - oos.writeObject(il); - byte[] bytes = baos.toByteArray(); - - ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes)); - TrainingListener il2 = (TrainingListener) ois.readObject(); - - listeners2.add(il2); - } - - net.setListeners(listeners2); - net.fit(iter); - } - - - @Test - public void testListenerCalls(){ - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .list() - .layer(new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - TestListener tl = new TestListener(); - net.setListeners(tl); - - DataSetIterator irisIter = new IrisDataSetIterator(50, 150); - - net.fit(irisIter, 2); - - List> exp = new ArrayList<>(); - exp.add(new Triple<>(Call.EPOCH_START, 0, 0)); - exp.add(new Triple<>(Call.ON_FWD, 0, 0)); - exp.add(new Triple<>(Call.ON_BWD, 0, 0)); - exp.add(new Triple<>(Call.ON_GRAD, 0, 0)); - exp.add(new Triple<>(Call.ITER_DONE, 0, 0)); - exp.add(new Triple<>(Call.ON_FWD, 1, 0)); - exp.add(new Triple<>(Call.ON_BWD, 1, 0)); - exp.add(new Triple<>(Call.ON_GRAD, 1, 0)); - exp.add(new Triple<>(Call.ITER_DONE, 1, 0)); - exp.add(new Triple<>(Call.ON_FWD, 2, 0)); - exp.add(new Triple<>(Call.ON_BWD, 2, 0)); - exp.add(new Triple<>(Call.ON_GRAD, 2, 0)); - exp.add(new Triple<>(Call.ITER_DONE, 2, 0)); - exp.add(new Triple<>(Call.EPOCH_END, 3, 0)); //Post updating iter count, pre update epoch count - - exp.add(new Triple<>(Call.EPOCH_START, 3, 1)); - exp.add(new Triple<>(Call.ON_FWD, 3, 1)); - exp.add(new Triple<>(Call.ON_BWD, 3, 1)); - exp.add(new Triple<>(Call.ON_GRAD, 3, 1)); - exp.add(new Triple<>(Call.ITER_DONE, 3, 1)); - exp.add(new Triple<>(Call.ON_FWD, 4, 1)); - exp.add(new Triple<>(Call.ON_BWD, 4, 1)); - exp.add(new Triple<>(Call.ON_GRAD, 4, 1)); - exp.add(new Triple<>(Call.ITER_DONE, 4, 1)); - exp.add(new Triple<>(Call.ON_FWD, 5, 1)); - exp.add(new Triple<>(Call.ON_BWD, 5, 1)); - exp.add(new Triple<>(Call.ON_GRAD, 5, 1)); - exp.add(new Triple<>(Call.ITER_DONE, 5, 1)); - exp.add(new Triple<>(Call.EPOCH_END, 6, 1)); - - - assertEquals(exp, tl.getCalls()); - - - tl = new TestListener(); - - ComputationGraph cg = net.toComputationGraph(); - cg.setListeners(tl); - - cg.fit(irisIter, 2); - - assertEquals(exp, tl.getCalls()); - } - - private static enum Call { - ITER_DONE, - EPOCH_START, - EPOCH_END, - ON_FWD, - ON_GRAD, - ON_BWD - } - - @Data - private static class TestListener implements TrainingListener { - - private List> calls = new ArrayList<>(); - - - @Override - public void iterationDone(Model model, int iteration, int epoch) { - calls.add(new Triple<>(Call.ITER_DONE, iteration, epoch)); - } - - @Override - public void onEpochStart(Model model) { - calls.add(new Triple<>(Call.EPOCH_START, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); - } - - @Override - public void onEpochEnd(Model model) { - calls.add(new Triple<>(Call.EPOCH_END, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); - } - - @Override - public void onForwardPass(Model model, List activations) { - calls.add(new Triple<>(Call.ON_FWD, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); - } - - @Override - public void onForwardPass(Model model, Map activations) { - calls.add(new Triple<>(Call.ON_FWD, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); - } - - @Override - public void onGradientCalculation(Model model) { - calls.add(new Triple<>(Call.ON_GRAD, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); - } - - @Override - public void onBackwardPass(Model model) { - calls.add(new Triple<>(Call.ON_BWD, BaseOptimizer.getIterationCount(model), BaseOptimizer.getEpochCount(model))); - } - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/ParallelExistingMiniBatchDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/ParallelExistingMiniBatchDataSetIteratorTest.java deleted file mode 100644 index eb03506e4..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/ParallelExistingMiniBatchDataSetIteratorTest.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.parallelism; - -import lombok.extern.slf4j.Slf4j; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.io.ClassPathResource; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.callbacks.DataSetDeserializer; -import org.deeplearning4j.datasets.iterator.parallel.FileSplitParallelDataSetIterator; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.common.primitives.Pair; -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import static java.time.Duration.ofMillis; -import static org.junit.jupiter.api.Assertions.assertTimeout; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -/* - @Test - public void testSimpleLoop1() throws Exception { - ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 4); - ExistingMiniBatchDataSetIterator test = new ExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin"); - - - List> pairs = new ArrayList<>(); - - int cnt = 0; - long time1 = System.nanoTime(); - while (iterator.hasNext()) { - DataSet ds = iterator.next(); - long time2 = System.nanoTime(); - assertNotNull(ds); - assertEquals(64, ds.numExamples()); - pairs.add(new Pair(time2 - time1, 0L)); - cnt++; - time1 = System.nanoTime(); - } - assertEquals(26, cnt); - - cnt = 0; - time1 = System.nanoTime(); - while (test.hasNext()) { - DataSet ds = test.next(); - long time2 = System.nanoTime(); - assertNotNull(ds); - assertEquals(64, ds.numExamples()); - pairs.get(cnt).setSecond(time2 - time1); - cnt++; - time1 = System.nanoTime(); - } - - assertEquals(26, cnt); - - for (Pair times: pairs) { - log.info("Parallel: {} ns; Simple: {} ns", times.getFirst(), times.getSecond()); - } - } - - @Test - public void testReset1() throws Exception { - ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 8); - - int cnt = 0; - long time1 = System.nanoTime(); - while (iterator.hasNext()) { - DataSet ds = iterator.next(); - long time2 = System.nanoTime(); - assertNotNull(ds); - assertEquals(64, ds.numExamples()); - cnt++; - - if (cnt == 10) - iterator.reset(); - - time1 = System.nanoTime(); - } - assertEquals(36, cnt); - } - - @Test - public void testWithAdsi1() throws Exception { - ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 8); - AsyncDataSetIterator adsi = new AsyncDataSetIterator(iterator, 8, true); - - int cnt = 0; - long time1 = System.nanoTime(); - while (adsi.hasNext()) { - DataSet ds = adsi.next(); - long time2 = System.nanoTime(); - assertNotNull(ds); - assertEquals(64, ds.numExamples()); - cnt++; - - if (cnt == 10) - adsi.reset(); - - time1 = System.nanoTime(); - } - assertEquals(36, cnt); - } - */ -@DisplayName("Parallel Existing Mini Batch Data Set Iterator Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class ParallelExistingMiniBatchDataSetIteratorTest extends BaseDL4JTest { - - @TempDir - public Path tempDir; - - private static File rootFolder; - - @BeforeEach - void setUp() throws Exception { - if (rootFolder == null) { - rootFolder = tempDir.toFile(); - for (int i = 0; i < 26; i++) { - new ClassPathResource("/datasets/mnist/mnist-train-" + i + ".bin").getTempFileFromArchive(rootFolder); - } - } - } - - @Test - @DisplayName("Test New Simple Loop 1") - void testNewSimpleLoop1() { - assertTimeout(ofMillis(30000), () -> { - FileSplitParallelDataSetIterator fspdsi = new FileSplitParallelDataSetIterator(rootFolder, "mnist-train-%d.bin", new DataSetDeserializer()); - List> pairs = new ArrayList<>(); - long time1 = System.nanoTime(); - int cnt = 0; - while (fspdsi.hasNext()) { - DataSet ds = fspdsi.next(); - long time2 = System.nanoTime(); - pairs.add(new Pair(time2 - time1, 0L)); - assertNotNull(ds); - // imitating processing here - Thread.sleep(10); - cnt++; - time1 = System.nanoTime(); - } - assertEquals(26, cnt); - for (Pair times : pairs) { - log.info("Parallel: {} ns; Simple: {} ns", times.getFirst(), times.getSecond()); - } - }); - } - /* - @Test - public void testSimpleLoop1() throws Exception { - ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 4); - ExistingMiniBatchDataSetIterator test = new ExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin"); - - - List> pairs = new ArrayList<>(); - - int cnt = 0; - long time1 = System.nanoTime(); - while (iterator.hasNext()) { - DataSet ds = iterator.next(); - long time2 = System.nanoTime(); - assertNotNull(ds); - assertEquals(64, ds.numExamples()); - pairs.add(new Pair(time2 - time1, 0L)); - cnt++; - time1 = System.nanoTime(); - } - assertEquals(26, cnt); - - cnt = 0; - time1 = System.nanoTime(); - while (test.hasNext()) { - DataSet ds = test.next(); - long time2 = System.nanoTime(); - assertNotNull(ds); - assertEquals(64, ds.numExamples()); - pairs.get(cnt).setSecond(time2 - time1); - cnt++; - time1 = System.nanoTime(); - } - - assertEquals(26, cnt); - - for (Pair times: pairs) { - log.info("Parallel: {} ns; Simple: {} ns", times.getFirst(), times.getSecond()); - } - } - - @Test - public void testReset1() throws Exception { - ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 8); - - int cnt = 0; - long time1 = System.nanoTime(); - while (iterator.hasNext()) { - DataSet ds = iterator.next(); - long time2 = System.nanoTime(); - assertNotNull(ds); - assertEquals(64, ds.numExamples()); - cnt++; - - if (cnt == 10) - iterator.reset(); - - time1 = System.nanoTime(); - } - assertEquals(36, cnt); - } - - @Test - public void testWithAdsi1() throws Exception { - ParallelExistingMiniBatchDataSetIterator iterator = new ParallelExistingMiniBatchDataSetIterator(rootFolder,"mnist-train-%d.bin", 8); - AsyncDataSetIterator adsi = new AsyncDataSetIterator(iterator, 8, true); - - int cnt = 0; - long time1 = System.nanoTime(); - while (adsi.hasNext()) { - DataSet ds = adsi.next(); - long time2 = System.nanoTime(); - assertNotNull(ds); - assertEquals(64, ds.numExamples()); - cnt++; - - if (cnt == 10) - adsi.reset(); - - time1 = System.nanoTime(); - } - assertEquals(36, cnt); - } - */ -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/RandomTests.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/RandomTests.java deleted file mode 100644 index 3454e4e5d..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/parallelism/RandomTests.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.parallelism; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.api.Model; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.WorkspaceMode; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.RNG) -public class RandomTests extends BaseDL4JTest { - - /** - * In this test we check for equality of model params after initialization in different threads - * - * @throws Exception - */ - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - @Execution(ExecutionMode.SAME_THREAD) - public void testModelInitialParamsEquality1() throws Exception { - final List models = new CopyOnWriteArrayList<>(); - - for (int i = 0; i < 4; i++) { - Thread thread = new Thread(() -> { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(119) // Training iterations as above - .l2(0.0005) - //.learningRateDecayPolicy(LearningRatePolicy.Inverse).lrPolicyDecayRate(0.001).lrPolicyPower(0.75) - .weightInit(WeightInit.XAVIER) - .updater(new Nesterovs(0.01, 0.9)) - .trainingWorkspaceMode(WorkspaceMode.ENABLED).list() - .layer(0, new ConvolutionLayer.Builder(5, 5) - //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied - .nIn(1).stride(1, 1).nOut(20).activation(Activation.IDENTITY) - .build()) - .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) - .kernelSize(2, 2).stride(2, 2).build()) - .layer(2, new ConvolutionLayer.Builder(5, 5) - //Note that nIn need not be specified in later layers - .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build()) - .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) - .kernelSize(2, 2).stride(2, 2).build()) - .layer(4, new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()) - .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) - .nOut(10).activation(Activation.SOFTMAX).build()) - .setInputType(InputType.convolutionalFlat(28, 28, 1)) //See note below - .build(); - - MultiLayerNetwork network = new MultiLayerNetwork(conf); - network.init(); - - models.add(network); - }); - - thread.start(); - thread.join(); - } - - - // at the end of day, model params has to - for (int i = 0; i < models.size(); i++) { - assertEquals(models.get(0).params(), models.get(i).params()); - } - } - - - @Test - public void testRngInitMLN() { - Nd4j.getRandom().setSeed(12345); - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).activation(Activation.TANH) - .weightInit(WeightInit.XAVIER).list() - .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()) - .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2, - new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nIn(10).nOut(10).build()) - .build(); - - String json = conf.toJson(); - - MultiLayerNetwork net1 = new MultiLayerNetwork(conf); - net1.init(); - - MultiLayerNetwork net2 = new MultiLayerNetwork(conf); - net2.init(); - - assertEquals(net1.params(), net2.params()); - - MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json); - - Nd4j.getRandom().setSeed(987654321); - MultiLayerNetwork net3 = new MultiLayerNetwork(fromJson); - net3.init(); - - assertEquals(net1.params(), net3.params()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/SystemPollingTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/SystemPollingTest.java deleted file mode 100644 index 486178f1e..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/perf/listener/SystemPollingTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.perf.listener; - -import org.apache.commons.io.FileUtils; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.core.listener.HardwareMetric; -import org.deeplearning4j.core.listener.SystemPolling; -import org.junit.jupiter.api.Disabled; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.factory.Nd4j; -import java.io.File; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@Disabled("AB 2019/05/24 - Failing on CI - \"Could not initialize class oshi.jna.platform.linux.Libc\" - Issue #7657") -@DisplayName("System Polling Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -class SystemPollingTest extends BaseDL4JTest { - - @TempDir - public Path tempDir; - - @Test - @DisplayName("Test Polling") - void testPolling() throws Exception { - Nd4j.create(1); - File tmpDir = tempDir.toFile(); - SystemPolling systemPolling = new SystemPolling.Builder().outputDirectory(tmpDir).pollEveryMillis(1000).build(); - systemPolling.run(); - Thread.sleep(8000); - systemPolling.stopPolling(); - File[] files = tmpDir.listFiles(); - assertTrue(files != null && files.length > 0); - // System.out.println(Arrays.toString(files)); - String yaml = FileUtils.readFileToString(files[0]); - HardwareMetric fromYaml = HardwareMetric.fromYaml(yaml); - System.out.println(fromYaml); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java deleted file mode 100644 index 76042c8b4..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/regressiontest/customlayer100a/CustomLayer.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.regressiontest.customlayer100a; - -import lombok.Getter; -import lombok.Setter; -import lombok.val; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.ParamInitializer; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; -import org.deeplearning4j.nn.conf.memory.LayerMemoryReport; -import org.deeplearning4j.nn.conf.memory.MemoryReport; -import org.deeplearning4j.nn.params.DefaultParamInitializer; -import org.deeplearning4j.optimize.api.TrainingListener; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.activations.IActivation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; - -import java.util.Collection; -import java.util.Map; - -public class CustomLayer extends FeedForwardLayer { - - private IActivation secondActivationFunction; - - public CustomLayer() { - //We need a no-arg constructor so we can deserialize the configuration from JSON or YAML format - // Without this, you will likely get an exception like the following: - //com.fasterxml.jackson.databind.JsonMappingException: No suitable constructor found for type [simple type, class org.deeplearning4j.examples.misc.customlayers.layer.CustomLayer]: can not instantiate from JSON object (missing default constructor or creator, or perhaps need to add/enable type information?) - } - - private CustomLayer(Builder builder) { - super(builder); - this.secondActivationFunction = builder.secondActivationFunction; - } - - public IActivation getSecondActivationFunction() { - //We also need setter/getter methods for our layer configuration fields (if any) for JSON serialization - return secondActivationFunction; - } - - public void setSecondActivationFunction(IActivation secondActivationFunction) { - //We also need setter/getter methods for our layer configuration fields (if any) for JSON serialization - this.secondActivationFunction = secondActivationFunction; - } - - @Override - public Layer instantiate(NeuralNetConfiguration conf, Collection iterationListeners, - int layerIndex, INDArray layerParamsView, boolean initializeParams, DataType networkDataType) { - //The instantiate method is how we go from the configuration class (i.e., this class) to the implementation class - // (i.e., a CustomLayerImpl instance) - //For the most part, it's the same for each type of layer - - CustomLayerImpl myCustomLayer = new CustomLayerImpl(conf, networkDataType); - myCustomLayer.setListeners(iterationListeners); //Set the iteration listeners, if any - myCustomLayer.setIndex(layerIndex); //Integer index of the layer - - //Parameter view array: In Deeplearning4j, the network parameters for the entire network (all layers) are - // allocated in one big array. The relevant section of this parameter vector is extracted out for each layer, - // (i.e., it's a "view" array in that it's a subset of a larger array) - // This is a row vector, with length equal to the number of parameters in the layer - myCustomLayer.setParamsViewArray(layerParamsView); - - //Initialize the layer parameters. For example, - // Note that the entries in paramTable (2 entries here: a weight array of shape [nIn,nOut] and biases of shape [1,nOut] - // are in turn a view of the 'layerParamsView' array. - Map paramTable = initializer().init(conf, layerParamsView, initializeParams); - myCustomLayer.setParamTable(paramTable); - myCustomLayer.setConf(conf); - return myCustomLayer; - } - - @Override - public ParamInitializer initializer() { - //This method returns the parameter initializer for this type of layer - //In this case, we can use the DefaultParamInitializer, which is the same one used for DenseLayer - //For more complex layers, you may need to implement a custom parameter initializer - //See the various parameter initializers here: - //https://github.com/eclipse/deeplearning4j/tree/master/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/params - - return DefaultParamInitializer.getInstance(); - } - - @Override - public LayerMemoryReport getMemoryReport(InputType inputType) { - //Memory report is used to estimate how much memory is required for the layer, for different configurations - //If you don't need this functionality for your custom layer, you can return a LayerMemoryReport - // with all 0s, or - - //This implementation: based on DenseLayer implementation - InputType outputType = getOutputType(-1, inputType); - - val numParams = initializer().numParams(this); - int updaterStateSize = (int) getIUpdater().stateSize(numParams); - - int trainSizeFixed = 0; - int trainSizeVariable = 0; - if (getIDropout() != null) { - //Assume we dup the input for dropout - trainSizeVariable += inputType.arrayElementsPerExample(); - } - - //Also, during backprop: we do a preOut call -> gives us activations size equal to the output size - // which is modified in-place by activation function backprop - // then we have 'epsilonNext' which is equivalent to input size - trainSizeVariable += outputType.arrayElementsPerExample(); - - return new LayerMemoryReport.Builder(layerName, CustomLayer.class, inputType, outputType) - .standardMemory(numParams, updaterStateSize) - .workingMemory(0, 0, trainSizeFixed, - trainSizeVariable) //No additional memory (beyond activations) for inference - .cacheMemory(MemoryReport.CACHE_MODE_ALL_ZEROS, - MemoryReport.CACHE_MODE_ALL_ZEROS) //No caching in DenseLayer - .build(); - } - - - //Here's an implementation of a builder pattern, to allow us to easily configure the layer - //Note that we are inheriting all of the FeedForwardLayer.Builder options: things like n - public static class Builder extends FeedForwardLayer.Builder { - - @Getter - @Setter - private IActivation secondActivationFunction; - - //This is an example of a custom property in the configuration - - /** - * A custom property used in this custom layer example. See the CustomLayerExampleReadme.md for details - * - * @param secondActivationFunction Second activation function for the layer - */ - public Builder secondActivationFunction(String secondActivationFunction) { - return secondActivationFunction(Activation.fromString(secondActivationFunction)); - } - - /** - * A custom property used in this custom layer example. See the CustomLayerExampleReadme.md for details - * - * @param secondActivationFunction Second activation function for the layer - */ - public Builder secondActivationFunction(Activation secondActivationFunction) { - this.secondActivationFunction = secondActivationFunction.getActivationFunction(); - return this; - } - - @Override - @SuppressWarnings("unchecked") //To stop warnings about unchecked cast. Not required. - public CustomLayer build() { - return new CustomLayer(this); - } - } - -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/ui/UiConnectionInfoTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/ui/UiConnectionInfoTest.java deleted file mode 100644 index a8e8ec3af..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/ui/UiConnectionInfoTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.ui; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.core.ui.UiConnectionInfo; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Ui Connection Info Test") -@NativeTag -@Tag(TagNames.DL4J_OLD_API) -@Tag(TagNames.UI) -class UiConnectionInfoTest extends BaseDL4JTest { - - @BeforeEach - void setUp() throws Exception { - } - - @Test - @DisplayName("Test Get First Part 1") - void testGetFirstPart1() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setPort(8080).build(); - assertEquals(info.getFirstPart(), "http://localhost:8080"); - } - - @Test - @DisplayName("Test Get First Part 2") - void testGetFirstPart2() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().enableHttps(true).setPort(8080).build(); - assertEquals(info.getFirstPart(), "https://localhost:8080"); - } - - @Test - @DisplayName("Test Get First Part 3") - void testGetFirstPart3() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082).build(); - assertEquals(info.getFirstPart(), "https://192.168.1.1:8082"); - } - - @Test - @DisplayName("Test Get Second Part 1") - void testGetSecondPart1() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082).setPath("www-data").build(); - assertEquals(info.getSecondPart(), "/www-data/"); - } - - @Test - @DisplayName("Test Get Second Part 2") - void testGetSecondPart2() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082).setPath("/www-data/tmp/").build(); - assertEquals(info.getSecondPart(), "/www-data/tmp/"); - } - - @Test - @DisplayName("Test Get Second Part 3") - void testGetSecondPart3() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082).setPath("/www-data/tmp").build(); - assertEquals(info.getSecondPart(), "/www-data/tmp/"); - } - - @Test - @DisplayName("Test Get Second Part 4") - void testGetSecondPart4() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082).setPath("/www-data//tmp").build(); - assertEquals(info.getSecondPart(), "/www-data/tmp/"); - } - - @Test - @DisplayName("Test Get Second Part 5") - void testGetSecondPart5() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082).setPath("/www-data//tmp").build(); - assertEquals(info.getSecondPart("alpha"), "/www-data/tmp/alpha/"); - } - - @Test - @DisplayName("Test Get Second Part 6") - void testGetSecondPart6() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082).setPath("//www-data//tmp").build(); - assertEquals(info.getSecondPart("/alpha/"), "/www-data/tmp/alpha/"); - } - - @Test - @DisplayName("Test Get Second Part 7") - void testGetSecondPart7() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(true).setPort(8082).setPath("//www-data//tmp").build(); - assertEquals(info.getSecondPart("/alpha//beta/"), "/www-data/tmp/alpha/beta/"); - } - - @Test - @DisplayName("Test Get Second Part 8") - void testGetSecondPart8() throws Exception { - UiConnectionInfo info = new UiConnectionInfo.Builder().setAddress("192.168.1.1").enableHttps(false).setPort(8082).setPath("/www-data//tmp").build(); - assertEquals(info.getFullAddress(), "http://192.168.1.1:8082/www-data/tmp/"); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ArrayUtilTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ArrayUtilTest.java deleted file mode 100755 index a9c3ecdf0..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ArrayUtilTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.util; - -import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.common.util.ArrayUtil; -import java.util.Arrays; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - */ -@DisplayName("Array Util Test") -@Tag(TagNames.JAVA_ONLY) -class ArrayUtilTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Range") - void testRange() { - int[] range = ArrayUtil.range(0, 2); - int[] test = { 0, 1 }; - assertEquals(true, Arrays.equals(test, range)); - int[] test2 = { -1, 0 }; - int[] range2 = ArrayUtil.range(-1, 1); - assertEquals(true, Arrays.equals(test2, range2)); - } - - @Test - @DisplayName("Test Strides") - void testStrides() { - int[] shape = { 5, 4, 3 }; - int[] cStyleStride = { 12, 3, 1 }; - int[] fortranStyleStride = { 1, 5, 20 }; - int[] fortranStyleTest = ArrayUtil.calcStridesFortran(shape); - int[] cStyleTest = ArrayUtil.calcStrides(shape); - assertEquals(true, Arrays.equals(cStyleStride, cStyleTest)); - assertEquals(true, Arrays.equals(fortranStyleStride, fortranStyleTest)); - int[] shape2 = { 2, 2 }; - int[] cStyleStride2 = { 2, 1 }; - int[] fortranStyleStride2 = { 1, 2 }; - int[] cStyleTest2 = ArrayUtil.calcStrides(shape2); - int[] fortranStyleTest2 = ArrayUtil.calcStridesFortran(shape2); - assertEquals(true, Arrays.equals(cStyleStride2, cStyleTest2)); - assertEquals(true, Arrays.equals(fortranStyleStride2, fortranStyleTest2)); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/CrashReportingUtilTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/CrashReportingUtilTest.java deleted file mode 100644 index 094d443af..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/CrashReportingUtilTest.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.util; - -import org.apache.commons.io.FileUtils; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.EarlyTerminationDataSetIterator; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.PoolingType; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.*; - -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.io.File; -import static org.junit.jupiter.api.Assertions.*; - -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Crash Reporting Util Test") -@NativeTag -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) -class CrashReportingUtilTest extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 120000; - } - - @TempDir - public Path testDir; - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @AfterEach - void after() { - // Reset dir - CrashReportingUtil.crashDumpOutputDirectory(null); - } - - @Test - @DisplayName("Test") - @Disabled - void test() throws Exception { - File dir = testDir.toFile(); - CrashReportingUtil.crashDumpOutputDirectory(dir); - int kernel = 2; - int stride = 1; - int padding = 0; - PoolingType poolingType = PoolingType.MAX; - int inputDepth = 1; - int height = 28; - int width = 28; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()).dist(new NormalDistribution(0, 1)).list().layer(0, new ConvolutionLayer.Builder().kernelSize(kernel, kernel).stride(stride, stride).padding(padding, padding).nIn(inputDepth).nOut(3).build()).layer(1, new SubsamplingLayer.Builder(poolingType).kernelSize(kernel, kernel).stride(stride, stride).padding(padding, padding).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(10).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - net.addListeners(new ScoreIterationListener(1)); - // Test net that hasn't been trained yet - Exception e = new Exception(); - CrashReportingUtil.writeMemoryCrashDump(net, e); - File[] list = dir.listFiles(); - assertNotNull(list); - assertEquals(1, list.length); - String str = FileUtils.readFileToString(list[0]); - // System.out.println(str); - assertTrue(str.contains("Network Information")); - assertTrue(str.contains("Layer Helpers")); - assertTrue(str.contains("JavaCPP")); - assertTrue(str.contains("ScoreIterationListener")); - // Train: - DataSetIterator iter = new EarlyTerminationDataSetIterator(new MnistDataSetIterator(32, true, 12345), 5); - net.fit(iter); - dir = testDir.toFile(); - CrashReportingUtil.crashDumpOutputDirectory(dir); - CrashReportingUtil.writeMemoryCrashDump(net, e); - list = dir.listFiles(); - assertNotNull(list); - assertEquals(1, list.length); - str = FileUtils.readFileToString(list[0]); - assertTrue(str.contains("Network Information")); - assertTrue(str.contains("Layer Helpers")); - assertTrue(str.contains("JavaCPP")); - assertTrue(str.contains("ScoreIterationListener(1)")); - // System.out.println("///////////////////////////////////////////////////////////"); - // System.out.println(str); - // System.out.println("///////////////////////////////////////////////////////////"); - // Also test manual memory info - String mlnMemoryInfo = net.memoryInfo(32, InputType.convolutionalFlat(28, 28, 1)); - // System.out.println("///////////////////////////////////////////////////////////"); - // System.out.println(mlnMemoryInfo); - // System.out.println("///////////////////////////////////////////////////////////"); - assertTrue(mlnMemoryInfo.contains("Network Information")); - assertTrue(mlnMemoryInfo.contains("Layer Helpers")); - assertTrue(mlnMemoryInfo.contains("JavaCPP")); - assertTrue(mlnMemoryInfo.contains("ScoreIterationListener(1)")); - // ////////////////////////////////////// - // Same thing on ComputationGraph: - dir = testDir.toFile(); - CrashReportingUtil.crashDumpOutputDirectory(dir); - ComputationGraph cg = net.toComputationGraph(); - cg.setListeners(new ScoreIterationListener(1)); - // Test net that hasn't been trained yet - CrashReportingUtil.writeMemoryCrashDump(cg, e); - list = dir.listFiles(); - assertNotNull(list); - assertEquals(1, list.length); - str = FileUtils.readFileToString(list[0]); - assertTrue(str.contains("Network Information")); - assertTrue(str.contains("Layer Helpers")); - assertTrue(str.contains("JavaCPP")); - assertTrue(str.contains("ScoreIterationListener(1)")); - // Train: - cg.fit(iter); - dir = testDir.toFile(); - CrashReportingUtil.crashDumpOutputDirectory(dir); - CrashReportingUtil.writeMemoryCrashDump(cg, e); - list = dir.listFiles(); - assertNotNull(list); - assertEquals(1, list.length); - str = FileUtils.readFileToString(list[0]); - assertTrue(str.contains("Network Information")); - assertTrue(str.contains("Layer Helpers")); - assertTrue(str.contains("JavaCPP")); - assertTrue(str.contains("ScoreIterationListener(1)")); - // System.out.println("///////////////////////////////////////////////////////////"); - // System.out.println(str); - // System.out.println("///////////////////////////////////////////////////////////"); - // Also test manual memory info - String cgMemoryInfo = cg.memoryInfo(32, InputType.convolutionalFlat(28, 28, 1)); - // System.out.println("///////////////////////////////////////////////////////////"); - // System.out.println(cgMemoryInfo); - // System.out.println("///////////////////////////////////////////////////////////"); - assertTrue(cgMemoryInfo.contains("Network Information")); - assertTrue(cgMemoryInfo.contains("Layer Helpers")); - assertTrue(cgMemoryInfo.contains("JavaCPP")); - assertTrue(cgMemoryInfo.contains("ScoreIterationListener(1)")); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelGuesserTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelGuesserTest.java deleted file mode 100644 index bc70310d7..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelGuesserTest.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.util; - -import org.apache.commons.compress.utils.IOUtils; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.core.util.ModelGuesser; -import org.deeplearning4j.nn.api.Model; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Disabled; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.preprocessor.Normalizer; -import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.resources.Resources; -import java.io.*; - -import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.*; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@Disabled -@DisplayName("Model Guesser Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class ModelGuesserTest extends BaseDL4JTest { - - @TempDir - public Path testDir; - - - - @Test - @DisplayName("Test Model Guess File") - void testModelGuessFile() throws Exception { - File f = Resources.asFile("modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_model.h5"); - assertTrue(f.exists()); - Model guess1 = ModelGuesser.loadModelGuess(f.getAbsolutePath()); - assertNotNull(guess1); - f = Resources.asFile("modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_1_model.h5"); - assertTrue(f.exists()); - Model guess2 = ModelGuesser.loadModelGuess(f.getAbsolutePath()); - assertNotNull(guess2); - } - - @Test - @DisplayName("Test Model Guess Input Stream") - void testModelGuessInputStream() throws Exception { - File f = Resources.asFile("modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_model.h5"); - assertTrue(f.exists()); - try (InputStream inputStream = new FileInputStream(f)) { - Model guess1 = ModelGuesser.loadModelGuess(inputStream); - assertNotNull(guess1); - } - f = Resources.asFile("modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_1_model.h5"); - assertTrue(f.exists()); - try (InputStream inputStream = new FileInputStream(f)) { - Model guess1 = ModelGuesser.loadModelGuess(inputStream); - assertNotNull(guess1); - } - } - - @Test - @DisplayName("Test Load Normalizers File") - void testLoadNormalizersFile() throws Exception { - MultiLayerNetwork net = getNetwork(); - File tempFile = testDir.resolve("testLoadNormalizersFile.bin").toFile(); - ModelSerializer.writeModel(net, tempFile, true); - NormalizerMinMaxScaler normalizer = new NormalizerMinMaxScaler(0, 1); - normalizer.fit(new DataSet(Nd4j.rand(new int[] { 2, 2 }), Nd4j.rand(new int[] { 2, 2 }))); - ModelSerializer.addNormalizerToModel(tempFile, normalizer); - Model model = ModelGuesser.loadModelGuess(tempFile.getAbsolutePath()); - Normalizer normalizer1 = ModelGuesser.loadNormalizer(tempFile.getAbsolutePath()); - assertEquals(model, net); - assertEquals(normalizer, normalizer1); - } - - @Test - @DisplayName("Test Normalizer In Place") - void testNormalizerInPlace() throws Exception { - MultiLayerNetwork net = getNetwork(); - File tempFile = testDir.resolve("testNormalizerInPlace.bin").toFile(); - NormalizerMinMaxScaler normalizer = new NormalizerMinMaxScaler(0, 1); - normalizer.fit(new DataSet(Nd4j.rand(new int[] { 2, 2 }), Nd4j.rand(new int[] { 2, 2 }))); - ModelSerializer.writeModel(net, tempFile, true, normalizer); - Model model = ModelGuesser.loadModelGuess(tempFile.getAbsolutePath()); - Normalizer normalizer1 = ModelGuesser.loadNormalizer(tempFile.getAbsolutePath()); - assertEquals(model, net); - assertEquals(normalizer, normalizer1); - } - - @Test - @DisplayName("Test Load Normalizers Input Stream") - void testLoadNormalizersInputStream() throws Exception { - MultiLayerNetwork net = getNetwork(); - File tempFile = testDir.resolve("testLoadNormalizersInputStream.bin").toFile(); - ModelSerializer.writeModel(net, tempFile, true); - NormalizerMinMaxScaler normalizer = new NormalizerMinMaxScaler(0, 1); - normalizer.fit(new DataSet(Nd4j.rand(new int[] { 2, 2 }), Nd4j.rand(new int[] { 2, 2 }))); - ModelSerializer.addNormalizerToModel(tempFile, normalizer); - Model model = ModelGuesser.loadModelGuess(tempFile.getAbsolutePath()); - try (InputStream inputStream = new FileInputStream(tempFile)) { - Normalizer normalizer1 = ModelGuesser.loadNormalizer(inputStream); - assertEquals(model, net); - assertEquals(normalizer, normalizer1); - } - } - - @Test - @DisplayName("Test Model Guesser Dl 4 j Model File") - void testModelGuesserDl4jModelFile() throws Exception { - MultiLayerNetwork net = getNetwork(); - File tempFile = testDir.resolve("testModelGuesserDl4jModelFile.bin").toFile(); - ModelSerializer.writeModel(net, tempFile, true); - MultiLayerNetwork network = (MultiLayerNetwork) ModelGuesser.loadModelGuess(tempFile.getAbsolutePath()); - assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson()); - assertEquals(net.params(), network.params()); - assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); - } - - @Test - @DisplayName("Test Model Guesser Dl 4 j Model Input Stream") - void testModelGuesserDl4jModelInputStream() throws Exception { - MultiLayerNetwork net = getNetwork(); - File tempFile = testDir.resolve("testModelGuesserDl4jModelInputStream.bin").toFile(); - ModelSerializer.writeModel(net, tempFile, true); - try (InputStream inputStream = new FileInputStream(tempFile)) { - MultiLayerNetwork network = (MultiLayerNetwork) ModelGuesser.loadModelGuess(inputStream); - assertNotNull(network); - assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson()); - assertEquals(net.params(), network.params()); - assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); - } - } - - @Test - @DisplayName("Test Model Guess Config File") - void testModelGuessConfigFile() throws Exception { - ClassPathResource resource = new ClassPathResource("modelimport/keras/configs/cnn_tf_config.json", ModelGuesserTest.class.getClassLoader()); - File f = getTempFile(resource); - String configFilename = f.getAbsolutePath(); - Object conf = ModelGuesser.loadConfigGuess(configFilename); - assertTrue(conf instanceof MultiLayerConfiguration); - ClassPathResource sequenceResource = new ClassPathResource("/keras/simple/mlp_fapi_multiloss_config.json"); - File f2 = getTempFile(sequenceResource); - Object sequenceConf = ModelGuesser.loadConfigGuess(f2.getAbsolutePath()); - assertTrue(sequenceConf instanceof ComputationGraphConfiguration); - ClassPathResource resourceDl4j = new ClassPathResource("model.json"); - File fDl4j = getTempFile(resourceDl4j); - String configFilenameDl4j = fDl4j.getAbsolutePath(); - Object confDl4j = ModelGuesser.loadConfigGuess(configFilenameDl4j); - assertTrue(confDl4j instanceof ComputationGraphConfiguration); - } - - @Test - @DisplayName("Test Model Guess Config Input Stream") - void testModelGuessConfigInputStream() throws Exception { - ClassPathResource resource = new ClassPathResource("modelimport/keras/configs/cnn_tf_config.json", ModelGuesserTest.class.getClassLoader()); - File f = getTempFile(resource); - try (InputStream inputStream = new FileInputStream(f)) { - Object conf = ModelGuesser.loadConfigGuess(inputStream); - assertTrue(conf instanceof MultiLayerConfiguration); - } - ClassPathResource sequenceResource = new ClassPathResource("/keras/simple/mlp_fapi_multiloss_config.json"); - File f2 = getTempFile(sequenceResource); - try (InputStream inputStream = new FileInputStream(f2)) { - Object sequenceConf = ModelGuesser.loadConfigGuess(inputStream); - assertTrue(sequenceConf instanceof ComputationGraphConfiguration); - } - ClassPathResource resourceDl4j = new ClassPathResource("model.json"); - File fDl4j = getTempFile(resourceDl4j); - try (InputStream inputStream = new FileInputStream(fDl4j)) { - Object confDl4j = ModelGuesser.loadConfigGuess(inputStream); - assertTrue(confDl4j instanceof ComputationGraphConfiguration); - } - } - - private File getTempFile(ClassPathResource classPathResource) throws Exception { - InputStream is = classPathResource.getInputStream(); - File f = testDir.toFile(); - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(f)); - IOUtils.copy(is, bos); - bos.flush(); - bos.close(); - return f; - } - - private MultiLayerNetwork getNetwork() { - int nIn = 5; - int nOut = 6; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()).layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - return net; - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelSerializerTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelSerializerTest.java deleted file mode 100644 index 2f19499d1..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/ModelSerializerTest.java +++ /dev/null @@ -1,361 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.util; - -import lombok.val; -import org.apache.commons.lang3.SerializationUtils; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.dataset.api.preprocessor.Normalizer; -import org.nd4j.linalg.dataset.api.preprocessor.NormalizerMinMaxScaler; -import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.primitives.Pair; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.util.*; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Model Serializer Test") -@Disabled -@NativeTag -@Tag(TagNames.FILE_IO) -class ModelSerializerTest extends BaseDL4JTest { - - @TempDir - public Path tempDir; - - @Test - @DisplayName("Test Write MLN Model") - void testWriteMLNModel() throws Exception { - int nIn = 5; - int nOut = 6; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()).layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(net, tempFile, true); - MultiLayerNetwork network = ModelSerializer.restoreMultiLayerNetwork(tempFile); - assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson()); - assertEquals(net.params(), network.params()); - assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); - } - - @Test - @DisplayName("Test Write Mln Model Input Stream") - void testWriteMlnModelInputStream() throws Exception { - int nIn = 5; - int nOut = 6; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()).layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - File tempFile = tempDir.toFile(); - FileOutputStream fos = new FileOutputStream(tempFile); - ModelSerializer.writeModel(net, fos, true); - // checking adding of DataNormalization to the model file - NormalizerMinMaxScaler scaler = new NormalizerMinMaxScaler(); - DataSetIterator iter = new IrisDataSetIterator(150, 150); - scaler.fit(iter); - ModelSerializer.addNormalizerToModel(tempFile, scaler); - NormalizerMinMaxScaler restoredScaler = ModelSerializer.restoreNormalizerFromFile(tempFile); - assertNotEquals(null, scaler.getMax()); - assertEquals(scaler.getMax(), restoredScaler.getMax()); - assertEquals(scaler.getMin(), restoredScaler.getMin()); - FileInputStream fis = new FileInputStream(tempFile); - MultiLayerNetwork network = ModelSerializer.restoreMultiLayerNetwork(fis); - assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson()); - assertEquals(net.params(), network.params()); - assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); - } - - @Test - @DisplayName("Test Write CG Model") - void testWriteCGModel() throws Exception { - ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).graphBuilder().addInputs("in").addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).activation(Activation.SOFTMAX).build(), "dense").setOutputs("out").build(); - ComputationGraph cg = new ComputationGraph(config); - cg.init(); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(cg, tempFile, true); - ComputationGraph network = ModelSerializer.restoreComputationGraph(tempFile); - assertEquals(network.getConfiguration().toJson(), cg.getConfiguration().toJson()); - assertEquals(cg.params(), network.params()); - assertEquals(cg.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); - } - - @Test - @DisplayName("Test Write CG Model Input Stream") - void testWriteCGModelInputStream() throws Exception { - ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).graphBuilder().addInputs("in").addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).activation(Activation.SOFTMAX).build(), "dense").setOutputs("out").build(); - ComputationGraph cg = new ComputationGraph(config); - cg.init(); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(cg, tempFile, true); - FileInputStream fis = new FileInputStream(tempFile); - ComputationGraph network = ModelSerializer.restoreComputationGraph(fis); - assertEquals(network.getConfiguration().toJson(), cg.getConfiguration().toJson()); - assertEquals(cg.params(), network.params()); - assertEquals(cg.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray()); - } - - private DataSet trivialDataSet() { - INDArray inputs = Nd4j.create(new float[] { 1.0f, 2.0f, 3.0f }, new int[] { 1, 3 }); - INDArray labels = Nd4j.create(new float[] { 4.0f, 5.0f, 6.0f }, new int[] { 1, 3 }); - return new DataSet(inputs, labels); - } - - private ComputationGraph simpleComputationGraph() { - ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder().optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.1)).graphBuilder().addInputs("in").addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).activation(Activation.SOFTMAX).build(), "dense").setOutputs("out").build(); - return new ComputationGraph(config); - } - - @Test - @DisplayName("Test Save Restore Normalizer From Input Stream") - void testSaveRestoreNormalizerFromInputStream() throws Exception { - DataSet dataSet = trivialDataSet(); - NormalizerStandardize norm = new NormalizerStandardize(); - norm.fit(dataSet); - ComputationGraph cg = simpleComputationGraph(); - cg.init(); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(cg, tempFile, true); - ModelSerializer.addNormalizerToModel(tempFile, norm); - FileInputStream fis = new FileInputStream(tempFile); - NormalizerStandardize restored = ModelSerializer.restoreNormalizerFromInputStream(fis); - assertNotEquals(null, restored); - DataSet dataSet2 = dataSet.copy(); - norm.preProcess(dataSet2); - assertNotEquals(dataSet.getFeatures(), dataSet2.getFeatures()); - restored.revert(dataSet2); - assertEquals(dataSet.getFeatures(), dataSet2.getFeatures()); - } - - @Test - @DisplayName("Test Restore Unsaved Normalizer From Input Stream") - void testRestoreUnsavedNormalizerFromInputStream() throws Exception { - DataSet dataSet = trivialDataSet(); - NormalizerStandardize norm = new NormalizerStandardize(); - norm.fit(dataSet); - ComputationGraph cg = simpleComputationGraph(); - cg.init(); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(cg, tempFile, true); - FileInputStream fis = new FileInputStream(tempFile); - NormalizerStandardize restored = ModelSerializer.restoreNormalizerFromInputStream(fis); - assertEquals(null, restored); - } - - @Test - @DisplayName("Test Invalid Loading 1") - void testInvalidLoading1() throws Exception { - ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in").addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2).nOut(3).build(), "dense").setOutputs("out").build(); - ComputationGraph cg = new ComputationGraph(config); - cg.init(); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(cg, tempFile, true); - try { - ModelSerializer.restoreMultiLayerNetwork(tempFile); - fail(); - } catch (Exception e) { - String msg = e.getMessage(); - assertTrue(msg.contains("JSON") && msg.contains("restoreComputationGraph"),msg); - } - } - - @Test - @DisplayName("Test Invalid Loading 2") - void testInvalidLoading2() throws Exception { - int nIn = 5; - int nOut = 6; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list().layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build()).layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - File tempFile = tempDir.resolve("testInvalidLoading2.bin").toFile(); - ModelSerializer.writeModel(net, tempFile, true); - try { - ModelSerializer.restoreComputationGraph(tempFile); - fail(); - } catch (Exception e) { - String msg = e.getMessage(); - assertTrue(msg.contains("JSON") && msg.contains("restoreMultiLayerNetwork"),msg); - } - } - - @Test - @DisplayName("Test Invalid Stream Reuse") - void testInvalidStreamReuse() throws Exception { - int nIn = 5; - int nOut = 6; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).list().layer(new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSet dataSet = trivialDataSet(); - NormalizerStandardize norm = new NormalizerStandardize(); - norm.fit(dataSet); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(net, tempFile, true); - ModelSerializer.addNormalizerToModel(tempFile, norm); - InputStream is = new FileInputStream(tempFile); - ModelSerializer.restoreMultiLayerNetwork(is); - try { - ModelSerializer.restoreNormalizerFromInputStream(is); - fail("Expected exception"); - } catch (Exception e) { - String msg = e.getMessage(); - assertTrue(msg.contains("may have been closed"),msg); - } - try { - ModelSerializer.restoreMultiLayerNetwork(is); - fail("Expected exception"); - } catch (Exception e) { - String msg = e.getMessage(); - assertTrue(msg.contains("may have been closed"),msg); - } - // Also test reading both model and normalizer from stream (correctly) - Pair pair = ModelSerializer.restoreMultiLayerNetworkAndNormalizer(new FileInputStream(tempFile), true); - assertEquals(net.params(), pair.getFirst().params()); - assertNotNull(pair.getSecond()); - } - - @Test - @DisplayName("Test Invalid Stream Reuse CG") - void testInvalidStreamReuseCG() throws Exception { - int nIn = 5; - int nOut = 6; - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).graphBuilder().addInputs("in").layer("0", new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build(), "in").setOutputs("0").build(); - ComputationGraph net = new ComputationGraph(conf); - net.init(); - DataSet dataSet = trivialDataSet(); - NormalizerStandardize norm = new NormalizerStandardize(); - norm.fit(dataSet); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(net, tempFile, true); - ModelSerializer.addNormalizerToModel(tempFile, norm); - InputStream is = new FileInputStream(tempFile); - ModelSerializer.restoreComputationGraph(is); - try { - ModelSerializer.restoreNormalizerFromInputStream(is); - fail("Expected exception"); - } catch (Exception e) { - String msg = e.getMessage(); - assertTrue(msg.contains("may have been closed"),msg); - } - try { - ModelSerializer.restoreComputationGraph(is); - fail("Expected exception"); - } catch (Exception e) { - String msg = e.getMessage(); - assertTrue(msg.contains("may have been closed"),msg); - } - // Also test reading both model and normalizer from stream (correctly) - Pair pair = ModelSerializer.restoreComputationGraphAndNormalizer(new FileInputStream(tempFile), true); - assertEquals(net.params(), pair.getFirst().params()); - assertNotNull(pair.getSecond()); - } - - @Test - @DisplayName("Test Java Serde _ 1") - void testJavaSerde_1() throws Exception { - int nIn = 5; - int nOut = 6; - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).graphBuilder().addInputs("in").layer("0", new OutputLayer.Builder().nIn(nIn).nOut(nOut).build(), "in").setOutputs("0").validateOutputLayerConfig(false).build(); - ComputationGraph net = new ComputationGraph(conf); - net.init(); - DataSet dataSet = trivialDataSet(); - NormalizerStandardize norm = new NormalizerStandardize(); - norm.fit(dataSet); - val b = SerializationUtils.serialize(net); - ComputationGraph restored = SerializationUtils.deserialize(b); - assertEquals(net, restored); - } - - @Test - @DisplayName("Test Java Serde _ 2") - void testJavaSerde_2() throws Exception { - int nIn = 5; - int nOut = 6; - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).list().layer(0, new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build()).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - DataSet dataSet = trivialDataSet(); - NormalizerStandardize norm = new NormalizerStandardize(); - norm.fit(dataSet); - val b = SerializationUtils.serialize(net); - MultiLayerNetwork restored = SerializationUtils.deserialize(b); - assertEquals(net, restored); - } - - @Test - @DisplayName("Test Put Get Object") - void testPutGetObject() throws Exception { - int nIn = 5; - int nOut = 6; - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01).graphBuilder().addInputs("in").layer("0", new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build(), "in").setOutputs("0").build(); - ComputationGraph net = new ComputationGraph(conf); - net.init(); - File tempFile = tempDir.toFile(); - ModelSerializer.writeModel(net, tempFile, true); - List toWrite = Arrays.asList("zero", "one", "two"); - ModelSerializer.addObjectToFile(tempFile, "myLabels", toWrite); - List restored = ModelSerializer.getObjectFromFile(tempFile, "myLabels"); - assertEquals(toWrite, restored); - Map someOtherData = new HashMap<>(); - someOtherData.put("x", new float[] { 0, 1, 2 }); - someOtherData.put("y", Nd4j.linspace(1, 10, 10, Nd4j.dataType())); - ModelSerializer.addObjectToFile(tempFile, "otherData.bin", someOtherData); - Map dataRestored = ModelSerializer.getObjectFromFile(tempFile, "otherData.bin"); - assertEquals(someOtherData.keySet(), dataRestored.keySet()); - assertArrayEquals((float[]) someOtherData.get("x"), (float[]) dataRestored.get("x"), 0f); - assertEquals(someOtherData.get("y"), dataRestored.get("y")); - List entries = ModelSerializer.listObjectsInFile(tempFile); - assertEquals(2, entries.size()); - System.out.println(entries); - assertTrue(entries.contains("myLabels")); - assertTrue(entries.contains("otherData.bin")); - ComputationGraph restoredNet = ModelSerializer.restoreComputationGraph(tempFile); - assertEquals(net.params(), restoredNet.params()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/MovingWindowMatrixTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/MovingWindowMatrixTest.java deleted file mode 100755 index 59b74f467..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/MovingWindowMatrixTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.util; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.core.util.MovingWindowMatrix; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Moving Window Matrix Test") -@NativeTag -@Tag(TagNames.NDARRAY_ETL) -class MovingWindowMatrixTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Moving Window") - void testMovingWindow() { - INDArray ones = Nd4j.ones(4, 4); - org.deeplearning4j.core.util.MovingWindowMatrix m = new org.deeplearning4j.core.util.MovingWindowMatrix(ones, 2, 2); - List windows = m.windows(); - assertEquals(4, windows.size()); - org.deeplearning4j.core.util.MovingWindowMatrix m2 = new MovingWindowMatrix(ones, 2, 2, true); - List windowsRotate = m2.windows(); - assertEquals(16, windowsRotate.size()); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/TimeSeriesUtilsTest.java b/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/TimeSeriesUtilsTest.java deleted file mode 100644 index 36beb47e3..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/java/org/deeplearning4j/util/TimeSeriesUtilsTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.util; - -import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@DisplayName("Time Series Utils Test") -@NativeTag -@Tag(TagNames.FILE_IO) -class TimeSeriesUtilsTest extends BaseDL4JTest { - - @Test - @DisplayName("Test Moving Average") - void testMovingAverage() { - INDArray a = Nd4j.arange(0, 20).castTo(DataType.DOUBLE); - INDArray result = Nd4j.create(new double[] { 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f, 9.5f, 10.5f, 11.5f, 12.5f, 13.5f, 14.5f, 15.5f, 16.5f, 17.5f }); - INDArray movingAvg = TimeSeriesUtils.movingAverage(a, 4); - assertEquals(result, movingAvg); - } -} diff --git a/deeplearning4j/deeplearning4j-core/src/test/resources/junit-platform.properties b/deeplearning4j/deeplearning4j-core/src/test/resources/junit-platform.properties deleted file mode 100644 index 8ec0fbcee..000000000 --- a/deeplearning4j/deeplearning4j-core/src/test/resources/junit-platform.properties +++ /dev/null @@ -1,25 +0,0 @@ -# -# /* -# * ****************************************************************************** -# * * -# * * -# * * This program and the accompanying materials are made available under the -# * * terms of the Apache License, Version 2.0 which is available at -# * * https://www.apache.org/licenses/LICENSE-2.0. -# * * -# * * See the NOTICE file distributed with this work for additional -# * * information regarding copyright ownership. -# * * Unless required by applicable law or agreed to in writing, software -# * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * * License for the specific language governing permissions and limitations -# * * under the License. -# * * -# * * SPDX-License-Identifier: Apache-2.0 -# * ***************************************************************************** -# */ -# -# - -junit.jupiter.execution.parallel.enabled = true -junit.jupiter.execution.parallel.mode.default = concurrent \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-cuda/pom.xml b/deeplearning4j/deeplearning4j-cuda/pom.xml deleted file mode 100644 index 3e929b025..000000000 --- a/deeplearning4j/deeplearning4j-cuda/pom.xml +++ /dev/null @@ -1,127 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-cuda-11.0 - - deeplearning4j-cuda - - - - 11.0 - 8.0 - 1.5.4 - - - - - org.nd4j - nd4j-cuda-${cuda.version} - ${nd4j.version} - - - org.slf4j - slf4j-api - - - ch.qos.logback - logback-classic - test - - - org.nd4j - nd4j-api - ${nd4j.version} - - - org.deeplearning4j - deeplearning4j-core - ${project.version} - - - org.bytedeco - javacpp - ${javacpp.version} - - - org.bytedeco - cuda-platform - ${cuda.version}-${cudnn.version}-${javacpp-presets.cuda.version} - - - org.junit.jupiter - junit-jupiter-api - - - org.junit.jupiter - junit-jupiter-engine - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - org.nd4j - nd4j-common-tests - ${project.version} - test - - - - - - - - - - nd4j-tests-cpu - - - - maven-surefire-plugin - true - - true - - - - - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/BaseCudnnHelper.java b/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/BaseCudnnHelper.java deleted file mode 100644 index f7ac730b4..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/BaseCudnnHelper.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda; - -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.bytedeco.javacpp.*; -import org.nd4j.jita.allocator.impl.AtomicAllocator; -import org.nd4j.linalg.api.buffer.DataBuffer; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.factory.Nd4j; - -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cudnn.*; -import static org.bytedeco.cuda.global.cudart.*; -import static org.bytedeco.cuda.global.cudnn.*; - -/** - * Functionality shared by all cuDNN-based helpers. - * - * @author saudet - */ -@Slf4j -public abstract class BaseCudnnHelper { - - protected static void checkCuda(int error) { - if (error != cudaSuccess) { - throw new RuntimeException("CUDA error = " + error + ": " + cudaGetErrorString(error).getString()); - } - } - - protected static void checkCudnn(int status) { - if (status != CUDNN_STATUS_SUCCESS) { - throw new RuntimeException("cuDNN status = " + status + ": " + cudnnGetErrorString(status).getString()); - } - } - - protected static class CudnnContext extends cudnnContext { - - protected static class Deallocator extends CudnnContext implements Pointer.Deallocator { - Deallocator(CudnnContext c) { - super(c); - } - - @Override - public void deallocate() { - destroyHandles(); - } - } - - public CudnnContext() { - // insure that cuDNN initializes on the same device as ND4J for this thread - Nd4j.create(1); - AtomicAllocator.getInstance(); - // This needs to be called in subclasses: - // createHandles(); - // deallocator(new Deallocator(this)); - } - - public CudnnContext(CudnnContext c) { - super(c); - } - - protected void createHandles() { - checkCudnn(cudnnCreate(this)); - } - - protected void destroyHandles() { - checkCudnn(cudnnDestroy(this)); - } - } - - protected static class DataCache extends Pointer { - - static class Deallocator extends DataCache implements Pointer.Deallocator { - Deallocator(DataCache c) { - super(c); - } - - @Override - public void deallocate() { - checkCuda(cudaFree(this)); - setNull(); - } - } - - static class HostDeallocator extends DataCache implements Pointer.Deallocator { - HostDeallocator(DataCache c) { - super(c); - } - - @Override - public void deallocate() { - checkCuda(cudaFreeHost(this)); - setNull(); - } - } - - public DataCache() {} - - public DataCache(long size) { - position = 0; - limit = capacity = size; - int error = cudaMalloc(this, size); - if (error != cudaSuccess) { - log.warn("Cannot allocate " + size + " bytes of device memory (CUDA error = " + error - + "), proceeding with host memory"); - checkCuda(cudaMallocHost(this, size)); - deallocator(new HostDeallocator(this)); - } else { - deallocator(new Deallocator(this)); - } - } - - public DataCache(DataCache c) { - super(c); - } - } - - protected static class TensorArray extends PointerPointer { - - static class Deallocator extends TensorArray implements Pointer.Deallocator { - Pointer owner; - - Deallocator(TensorArray a, Pointer owner) { - this.address = a.address; - this.capacity = a.capacity; - this.owner = owner; - } - - @Override - public void deallocate() { - for (int i = 0; !isNull() && i < capacity; i++) { - cudnnTensorStruct t = this.get(cudnnTensorStruct.class, i); - checkCudnn(cudnnDestroyTensorDescriptor(t)); - } - if (owner != null) { - owner.deallocate(); - owner = null; - } - setNull(); - } - } - - public TensorArray() {} - - public TensorArray(long size) { - PointerPointer p = new PointerPointer(size); - p.deallocate(false); - this.address = p.address(); - this.limit = p.limit(); - this.capacity = p.capacity(); - - cudnnTensorStruct t = new cudnnTensorStruct(); - for (int i = 0; i < capacity; i++) { - checkCudnn(cudnnCreateTensorDescriptor(t)); - this.put(i, t); - } - deallocator(new Deallocator(this, p)); - } - - public TensorArray(TensorArray a) { - super(a); - } - } - - protected final DataType nd4jDataType; - protected final int dataType; - protected final int dataTypeSize; - // both CUDNN_DATA_HALF and CUDNN_DATA_FLOAT need a float value for alpha and beta - protected final Pointer alpha; - protected final Pointer beta; - protected SizeTPointer sizeInBytes = new SizeTPointer(1); - - public BaseCudnnHelper(@NonNull DataType dataType){ - this.nd4jDataType = dataType; - this.dataType = dataType == DataType.DOUBLE ? CUDNN_DATA_DOUBLE - : dataType == DataType.FLOAT ? CUDNN_DATA_FLOAT : CUDNN_DATA_HALF; - this.dataTypeSize = dataType == DataType.DOUBLE ? 8 : dataType == DataType.FLOAT ? 4 : 2; - // both CUDNN_DATA_HALF and CUDNN_DATA_FLOAT need a float value for alpha and beta - this.alpha = this.dataType == CUDNN_DATA_DOUBLE ? new DoublePointer(1.0) : new FloatPointer(1.0f); - this.beta = this.dataType == CUDNN_DATA_DOUBLE ? new DoublePointer(0.0) : new FloatPointer(0.0f); - } - - public static int toCudnnDataType(DataType type){ - switch (type){ - case DOUBLE: - return CUDNN_DATA_DOUBLE; - case FLOAT: - return CUDNN_DATA_FLOAT; - case INT: - return CUDNN_DATA_INT32; - case HALF: - return CUDNN_DATA_HALF; - default: - throw new RuntimeException("Cannot convert type: " + type); - } - } - - public boolean checkSupported() { - // add general checks here, if any - return true; - } - - - /** - * From CuDNN documentation - - * "Tensors are restricted to having at least 4 dimensions... When working with lower dimensional data, it is - * recommended that the user create a 4Dtensor, and set the size along unused dimensions to 1." - * - * This method implements that - basically appends 1s to the end (shape or stride) to make it length 4, - * or leaves it unmodified if the length is already 4 or more. - * This method can be used for both shape and strides - * - * @param shapeOrStrides - * @return - */ - protected static int[] adaptForTensorDescr(int[] shapeOrStrides){ - if(shapeOrStrides.length >= 4) - return shapeOrStrides; - int[] out = new int[4]; - int i=0; - for(; i backpropGradient(INDArray input, INDArray weights, INDArray bias, INDArray delta, int[] kernel, - int[] strides, int[] pad, INDArray biasGradView, INDArray weightGradView, IActivation afn, - AlgoMode mode, BwdFilterAlgo bwdFilterAlgo, BwdDataAlgo bwdDataAlgo, - ConvolutionMode convolutionMode, int[] dilation, CNN2DFormat format, LayerWorkspaceMgr workspaceMgr) { - - //AB 2020/04/21 - cuDNN does have NHWC support (with limitations) however I have been unable to get it working - // correctly on NHWC data, even after updating all descriptors, tensor format, etc. - //Therefore: all computation here is done in NCHW format only - //As of a future (next?) release we'll likely switch to C++ for cuDNN support - boolean origNHWC = false; - if(format == CNN2DFormat.NHWC){ - input = input.permute(0,3,1,2); //NHWC to NCHW - delta = delta.permute(0,3,1,2); - origNHWC = true; - } - - int TENSOR_FORMAT = CUDNN_TENSOR_NCHW; - - int code; - - val miniBatch = input.size(0); - val outDepth = weights.size(0); - val inDepth = weights.size(1); - val kH = weights.size(2); - val kW = weights.size(3); - - CudnnForwardArgs args = getCudnnForwardArgs(input, kernel, strides, pad, dilation, convolutionMode, null, CNN2DFormat.NCHW); //Note hardcoded NCHW due to above - input = args.getInput(); - val inH = input.size(2); - val inW = input.size(3); - val srcStride = input.stride(); - val outSize = args.getOutSize(); - val outH = outSize[0]; - val outW = outSize[1]; - - if (!Shape.strideDescendingCAscendingF(delta)) { - // apparently not supported by cuDNN - delta = delta.dup(); - } - - val deltaStride = delta.stride(); - int[] algo1 = new int[1]; - int[] algo2 = new int[1]; - - - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - code = cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, (int) miniBatch, (int) inDepth,(int) inH, (int) inW, - (int) srcStride[0], (int) srcStride[1], (int) srcStride[2], (int) srcStride[3]); - checkCudnn(false, "cudnnSetTensor4dDescriptorEx", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - code = cudnnSetTensor4dDescriptorEx(cudnnContext.deltaTensorDesc, dataType, (int) miniBatch, (int) outDepth, (int) outH, (int) outW, - (int) deltaStride[0], (int) deltaStride[1], (int) deltaStride[2], (int) deltaStride[3]); - checkCudnn(false, "cudnnSetTensor4dDescriptorEx", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - code = cudnnSetConvolution2dDescriptor(cudnnContext.convDesc, pad[0], pad[1], strides[0], strides[1], dilation[0], - dilation[1], CUDNN_CROSS_CORRELATION, dataType); - checkCudnn(false, "cudnnSetConvolution2dDescriptor", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - code = cudnnSetFilter4dDescriptor(cudnnContext.filterDesc, dataType, TENSOR_FORMAT, (int) outDepth, (int) inDepth, (int) kH, (int) kW); - checkCudnn(false, "cudnnSetFilter4dDescriptor", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - if (mode == AlgoMode.USER_SPECIFIED && bwdFilterAlgo != null && bwdDataAlgo != null) { - switch (bwdFilterAlgo) { - case ALGO_0: - algo1[0] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0; - break; - case ALGO_1: - algo1[0] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1; - break; - case FFT: - algo1[0] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT; - break; - case ALGO_3: - algo1[0] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_3; - break; - case WINOGRAD: - algo1[0] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD; - break; - case WINOGRAD_NONFUSED: - algo1[0] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED; - break; - case FFT_TILING: - algo1[0] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING; - break; - case COUNT: - algo1[0] = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_COUNT; - break; - default: - throw new IllegalArgumentException("Unknown BwdFilterAlgo: " + bwdFilterAlgo); - } - - switch (bwdDataAlgo) { - case ALGO_0: - algo2[0] = CUDNN_CONVOLUTION_BWD_DATA_ALGO_0; - break; - case ALGO_1: - algo2[0] = CUDNN_CONVOLUTION_BWD_DATA_ALGO_1; - break; - case FFT: - algo2[0] = CUDNN_CONVOLUTION_BWD_DATA_ALGO_FFT; - break; - case FFT_TILING: - algo2[0] = CUDNN_CONVOLUTION_BWD_DATA_ALGO_FFT_TILING; - break; - case WINOGRAD: - algo2[0] = CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD; - break; - case WINOGRAD_NONFUSED: - algo2[0] = CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED; - break; - case COUNT: - algo2[0] = CUDNN_CONVOLUTION_BWD_DATA_ALGO_COUNT; - break; - default: - throw new IllegalArgumentException("Unknown BwdDataAlgo: " + bwdDataAlgo); - } - } else { - /* - code = cudnnGetConvolutionBackwardFilterAlgorithm(cudnnContext, cudnnContext.srcTensorDesc, - cudnnContext.deltaTensorDesc, cudnnContext.convDesc, cudnnContext.filterDesc, - mode == AlgoMode.NO_WORKSPACE ? CUDNN_CONVOLUTION_BWD_FILTER_NO_WORKSPACE - : CUDNN_CONVOLUTION_BWD_FILTER_PREFER_FASTEST, - 0, algo1); - */ - val fa = new cudnnConvolutionBwdFilterAlgoPerf_t(); - val counts = new int[1]; - code = cudnnFindConvolutionBackwardFilterAlgorithm(cudnnContext, cudnnContext.srcTensorDesc, - cudnnContext.deltaTensorDesc, cudnnContext.convDesc, cudnnContext.filterDesc, 1, counts, fa); - algo1[0] = fa.algo(); - - checkCudnn(false, "cudnnGetConvolutionBackwardFilterAlgorithm", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - /* - code = cudnnGetConvolutionBackwardDataAlgorithm(cudnnContext, cudnnContext.filterDesc, - cudnnContext.deltaTensorDesc, cudnnContext.convDesc, cudnnContext.srcTensorDesc, - mode == AlgoMode.NO_WORKSPACE ? CUDNN_CONVOLUTION_BWD_DATA_NO_WORKSPACE - : CUDNN_CONVOLUTION_BWD_DATA_PREFER_FASTEST, - 0, algo2); - */ - - val da = new cudnnConvolutionBwdDataAlgoPerf_t(); - code = cudnnFindConvolutionBackwardDataAlgorithm(cudnnContext, cudnnContext.filterDesc, - cudnnContext.deltaTensorDesc, cudnnContext.convDesc, cudnnContext.srcTensorDesc, 1, counts, da); - - algo2[0] = da.algo(); - checkCudnn(false, "cudnnGetConvolutionBackwardDataAlgorithm", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - } - - if(log.isTraceEnabled()){ - BwdFilterAlgo fa = BwdFilterAlgo.values()[algo1[0]]; - BwdDataAlgo da = BwdDataAlgo.values()[algo2[0]]; - log.trace("CudnnConvolutionHelper backward algorithm selection: mode {}, filter algorithm {}, data algorithm {}", mode, fa, da); - } - - INDArray epsNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, weights.dataType(), new long[] {(int) miniBatch,(int) inDepth, (int) inH, (int) inW}, 'c'); - - val dstStride = epsNext.stride(); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareActionAllWrite(input, weights, weightGradView, - biasGradView, delta, epsNext); - Pointer srcData = allocator.getPointer(input, context); - Pointer filterData = allocator.getPointer(weights, context); - Pointer filterGradData = allocator.getPointer(weightGradView, context); - Pointer biasGradData = allocator.getPointer(biasGradView, context); - Pointer deltaData = allocator.getPointer(delta, context); - Pointer dstData = allocator.getPointer(epsNext, context); - - code = cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream())); - checkCudnn(false, "cudnnSetStream", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - code = cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, (int) miniBatch, (int) inDepth, (int) inH, (int) inW, - (int) dstStride[0], (int) dstStride[1], (int) dstStride[2], (int) dstStride[3]); - checkCudnn(false, "cudnnSetTensor4dDescriptorEx", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - code = cudnnGetConvolutionBackwardFilterWorkspaceSize(cudnnContext, cudnnContext.srcTensorDesc, - cudnnContext.deltaTensorDesc, cudnnContext.convDesc, cudnnContext.filterDesc, algo1[0], - sizeInBytes); - checkCudnn(false, "cudnnGetConvolutionBackwardFilterWorkspaceSize", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - long sizeInBytes1 = sizeInBytes.get(0); - code = cudnnGetConvolutionBackwardDataWorkspaceSize(cudnnContext, cudnnContext.filterDesc, - cudnnContext.deltaTensorDesc, cudnnContext.convDesc, cudnnContext.dstTensorDesc, algo2[0], - sizeInBytes); - checkCudnn(false, "cudnnGetConvolutionBackwardDataWorkspaceSize", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - DataCache workSpace = workspaceMgr.getHelperWorkspace(LayerWorkspaceMgr.CUDNN_WORKSPACE_KEY); - long sizeInBytes2 = sizeInBytes.get(0); - if (workSpace == null || sizeInBytes1 > workSpace.capacity() || sizeInBytes2 > workSpace.capacity()) { - long newSize = Math.max(sizeInBytes1, sizeInBytes2); - if(log.isTraceEnabled()){ - if(workSpace == null){ - log.trace("CudnnConvolutionHelper backpropGradient: Allocating initial workspace of size {} ({})", newSize, - BinaryByteUnit.format(newSize, "#.00")); - } else { - log.trace("CudnnConvolutionHelper backpropGradient: Deallocating workspace of size {} ({}), allocating new workspace of size {} ({})", - workSpace.capacity(), BinaryByteUnit.format(workSpace.capacity(), "#.00"), - newSize, BinaryByteUnit.format(newSize, "#.00")); - } - } - if(workSpace != null) - workSpace.deallocate(); - workSpace = new DataCache(newSize); - workspaceMgr.setHelperWorkspace(LayerWorkspaceMgr.CUDNN_WORKSPACE_KEY, workSpace); - } - - code = cudnnSetTensor4dDescriptor(cudnnContext.biasTensorDesc, TENSOR_FORMAT, dataType, 1, (int) outDepth, 1, 1); - checkCudnn(false, "cudnnSetTensor4dDescriptor", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - code = cudnnConvolutionBackwardBias(cudnnContext, alpha, cudnnContext.deltaTensorDesc, deltaData, beta, - cudnnContext.biasTensorDesc, biasGradData); - checkCudnn(false, "cudnnConvolutionBackwardBias", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - code = cudnnConvolutionBackwardFilter(cudnnContext, alpha, cudnnContext.srcTensorDesc, srcData, - cudnnContext.deltaTensorDesc, deltaData, cudnnContext.convDesc, algo1[0], workSpace, - workSpace.capacity(), beta, cudnnContext.filterDesc, filterGradData); - checkCudnn(false, "cudnnConvolutionBackwardFilter", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - code = cudnnConvolutionBackwardData(cudnnContext, alpha, cudnnContext.filterDesc, filterData, - cudnnContext.deltaTensorDesc, deltaData, cudnnContext.convDesc, algo2[0], workSpace, - workSpace.capacity(), beta, cudnnContext.dstTensorDesc, dstData); - checkCudnn(false, "cudnnConvolutionBackwardData", code, input, weights, null, delta, kernel, strides, pad, mode, null, bwdFilterAlgo, bwdDataAlgo, convolutionMode, dilation); - - allocator.getFlowController().registerActionAllWrite(context, input, weights, weightGradView, biasGradView, - delta, epsNext); - - Gradient retGradient = new DefaultGradient(); - retGradient.setGradientFor(ConvolutionParamInitializer.BIAS_KEY, biasGradView); - retGradient.setGradientFor(ConvolutionParamInitializer.WEIGHT_KEY, weightGradView, 'c'); - - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - - //Note that: if we had to manually pad for SAME mode, we have to 'undo' this manual padding for the epsilon - // we return. The returned epsilon (i.e., dL/dIn array) has to be the same shape as the *original* input. - if(args.isManualPadBottom() || args.isManualPadRight()) { - epsNext = epsNext.get(all(), all(), - interval(0, epsNext.size(2) - (args.isManualPadBottom() ? 1 : 0)), - interval(0, epsNext.size(3) - (args.isManualPadRight() ? 1 : 0))); - } - - if(origNHWC){ - epsNext = epsNext.permute(0,2,3,1); //NCHW to NHWC - } - - return new Pair<>(retGradient, epsNext); - } - - @Override - public INDArray preOutput(INDArray input, INDArray weights, INDArray bias, int[] kernel, int[] strides, int[] pad, - AlgoMode mode, FwdAlgo fwdAlgo, ConvolutionMode convolutionMode, int[] dilation, CNN2DFormat format, - LayerWorkspaceMgr workspaceMgr) { - - //AB 2020/04/21 - cuDNN does have NHWC support (with limitations) however I have been unable to get it working - // correctly on NHWC data, even after updating all descriptors, tensor format, etc. - //Therefore: all computation here is done in NCHW format only - //As of a future (next?) release we'll likely switch to C++ for cuDNN support - boolean origNHWC = false; - if(format == CNN2DFormat.NHWC){ - input = input.permute(0,3,1,2); //NHWC to NCHW - origNHWC = true; - } - - int TENSOR_FORMAT = CUDNN_TENSOR_NCHW; - - int code; - - val miniBatch = input.size(0); - val outDepth = weights.size(0); - val inDepth = weights.size(1); - val kH = weights.size(2); - val kW = weights.size(3); - - CudnnForwardArgs args = getCudnnForwardArgs(input, kernel, strides, pad, dilation, convolutionMode, null, CNN2DFormat.NCHW); //Note hardcoded NCHW due to above - input = args.getInput(); - val inH = input.size(2); - val inW = input.size(3); - val srcStride = input.stride(); - val outSize = args.getOutSize(); - - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - INDArray z = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, weights.dataType(), new long[] {(int) miniBatch, (int) outDepth, outSize[0], outSize[1]}); - - code = cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, (int) miniBatch, (int) inDepth, (int) inH, (int) inW, - (int) srcStride[0], (int) srcStride[1], (int) srcStride[2], (int) srcStride[3]); - checkCudnn(true, "cudnnSetTensor4dDescriptorEx", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - code = cudnnSetFilter4dDescriptor(cudnnContext.filterDesc, dataType, TENSOR_FORMAT, (int) outDepth, (int) inDepth, (int) kH, (int) kW); - checkCudnn(true, "cudnnSetFilter4dDescriptor", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - code = cudnnSetConvolution2dDescriptor(cudnnContext.convDesc, pad[0], pad[1], strides[0], strides[1], dilation[0], - dilation[1], CUDNN_CROSS_CORRELATION, dataType); - checkCudnn(true, "cudnnSetConvolution2dDescriptor", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - - // find dimension of convolution output - // checkCudnn(cudnnGetConvolution2dForwardOutputDim(cudnnContext.convDesc, cudnnContext.srcTensorDesc, cudnnContext.filterDesc, n, c, h, w)); - // INDArray z = Nd4j.createUninitialized(new int[]{n[0],c[0],h[0],w[0]},'c'); - - - int[] algo = new int[1]; - val dstStride = z.stride(); - code = cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, (int) miniBatch, (int) outDepth, (int) outSize[0], - (int) outSize[1], (int) dstStride[0], (int) dstStride[1], (int) dstStride[2], (int) dstStride[3]); - checkCudnn(true, "cudnnSetTensor4dDescriptorEx", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - if (mode == AlgoMode.USER_SPECIFIED && fwdAlgo != null) { - switch (fwdAlgo) { - case IMPLICIT_GEMM: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM; - break; - case IMPLICIT_PRECOMP_GEMM: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM; - break; - case GEMM: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_GEMM; - break; - case DIRECT: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_DIRECT; - break; - case FFT: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_FFT; - break; - case FFT_TILING: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING; - break; - case WINOGRAD: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD; - break; - case WINOGRAD_NONFUSED: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED; - break; - case COUNT: - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_COUNT; - break; - default: - throw new IllegalArgumentException("Unknown FwdAlgo: " + fwdAlgo); - } - } else { - /* - code = cudnnGetConvolutionForwardAlgorithm_v7(cudnnContext, cudnnContext.srcTensorDesc, - cudnnContext.filterDesc, cudnnContext.convDesc, - cudnnContext.dstTensorDesc, mode == AlgoMode.NO_WORKSPACE - ? CUDNN_CONVOLUTION_FWD_ : CUDNN_CONVOLUTION_FWD_PREFER_FASTEST, - 0, algo); - */ - - val cdf = new cudnnConvolutionFwdAlgoPerf_t(); - val count = new int[1]; - code = cudnnFindConvolutionForwardAlgorithm(cudnnContext, cudnnContext.srcTensorDesc, cudnnContext.filterDesc, cudnnContext.convDesc, cudnnContext.dstTensorDesc, 1, count, cdf); - - if(code != CUDNN_STATUS_SUCCESS){ - //If CuDNN can't infer algorithm - try IMPLICIT_GEMM - //Why this specifically? According to the docs, it seems to have the least number of restrictions - // to things like dilation - - OneTimeLogger.warn(log, "Error getting CuDNN forward algorithm - falling back on IMPLICIT_GEMM"); - mode = AlgoMode.USER_SPECIFIED; - fwdAlgo = FwdAlgo.IMPLICIT_GEMM; - algo[0] = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM; - } - - algo[0] = cdf.algo(); - } - - if(log.isTraceEnabled()){ - FwdAlgo a = FwdAlgo.values()[algo[0]]; - log.trace("CudnnConvolutionHelper forward algorithm selection: mode {}, algorithm {}", mode, a); - } - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareAction(z, input, weights, bias); - Pointer srcData = allocator.getPointer(input, context); - Pointer filterData = allocator.getPointer(weights, context); - Pointer biasData = allocator.getPointer(bias, context); - Pointer dstData = allocator.getPointer(z, context); - - code = cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream())); - checkCudnn(true, "cudnnSetStream", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - code = cudnnGetConvolutionForwardWorkspaceSize(cudnnContext, cudnnContext.srcTensorDesc, - cudnnContext.filterDesc, cudnnContext.convDesc, cudnnContext.dstTensorDesc, algo[0], - sizeInBytes); - checkCudnn(true, "cudnnGetConvolutionForwardWorkspaceSize", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - DataCache workSpace = workspaceMgr.getHelperWorkspace(LayerWorkspaceMgr.CUDNN_WORKSPACE_KEY); - if (workSpace == null || sizeInBytes.get(0) > workSpace.capacity()) { - if(log.isTraceEnabled()){ - if(workSpace == null){ - log.trace("CudnnConvolutionHelper preOutput: allocating initial workspace of size {} ({})", - sizeInBytes.get(), BinaryByteUnit.format(sizeInBytes.get(), "#.00")); - } else { - log.trace("CudnnConvolutionHelper preOutput: Deallocating workspace of size {} ({}), allocating new workspace of size {} ({})", - workSpace.capacity(), BinaryByteUnit.format(workSpace.capacity(), "#.00"), - sizeInBytes.get(), BinaryByteUnit.format(sizeInBytes.get(), "#.00")); - } - } - if(workSpace != null) - workSpace.deallocate(); - workSpace = new DataCache(sizeInBytes.get(0)); - workspaceMgr.setHelperWorkspace(LayerWorkspaceMgr.CUDNN_WORKSPACE_KEY, workSpace); - } - code = cudnnConvolutionForward(cudnnContext, alpha, cudnnContext.srcTensorDesc, srcData, - cudnnContext.filterDesc, filterData, cudnnContext.convDesc, algo[0], workSpace, - workSpace.capacity(), beta, cudnnContext.dstTensorDesc, dstData); - checkCudnn(true, "cudnnConvolutionForward", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - - code = cudnnSetTensor4dDescriptor(cudnnContext.biasTensorDesc, TENSOR_FORMAT, dataType, 1, (int) outDepth, 1, 1); - checkCudnn(true, "cudnnSetTensor4dDescriptor", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - code = cudnnAddTensor(cudnnContext, alpha, cudnnContext.biasTensorDesc, biasData, alpha, - cudnnContext.dstTensorDesc, dstData); - checkCudnn(true, "cudnnAddTensor", code, input, weights, bias, null, kernel, strides, pad, mode, fwdAlgo, null, null, convolutionMode, dilation); - - allocator.registerAction(context, z, input, weights, bias); - - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - - if(origNHWC){ - z = z.permute(0,2,3,1); //NCHW to NHWC - } - - return z; - } - - private void checkCudnn(boolean forward, String step, int code, INDArray input, INDArray weights, INDArray bias, INDArray delta, - int[] kernel, int[] strides, int[] pad, - AlgoMode mode, FwdAlgo fwdAlgo, BwdFilterAlgo bwdFilterAlgo, BwdDataAlgo bwdDataAlgo, ConvolutionMode convolutionMode, int[] dilation) { - - if (code != CUDNN_STATUS_SUCCESS) { - StringBuilder sb = new StringBuilder(); - sb.append("CuDNN error = ").append(code).append(": ").append(cudnnGetErrorString(code).getString()) - .append(" during ") - .append(forward ? "forward pass" : "backward pass") - .append(" - step ").append(step) - .append(": inputShape=").append(Arrays.toString(input.shape())) - .append(", weightsShape=").append(Arrays.toString(weights.shape())) - .append(", biasShape=").append(bias == null ? null : Arrays.toString(bias.shape())); - if (!forward) { - sb.append(", gradientShape=").append(Arrays.toString(delta.shape())); - } - sb.append(", kernel=").append(Arrays.toString(kernel)) - .append(", stride=").append(Arrays.toString(strides)) - .append(", padding=").append(Arrays.toString(pad)) - .append(", dilation=").append(Arrays.toString(dilation)) - .append(", AlgoMode=").append(mode); - if (forward) { - sb.append(", fwdAlgo=").append(fwdAlgo); - } else { - sb.append(", bwdFilterAlgo=").append(bwdFilterAlgo) - .append(", bwdDataAlgo=").append(bwdDataAlgo); - } - sb.append(", convolutionMode=").append(convolutionMode); - - throw new RuntimeException(sb.toString()); - } - } - - @Override - public INDArray activate(INDArray z, IActivation afn, boolean training) { - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - INDArray activation = z; - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareAction(z); - Pointer dstData = allocator.getPointer(z, context); - - checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream()))); - switch (afn.toString()) { - case "identity": - break; - case "sigmoid": - checkCudnn(cudnnSetActivationDescriptor(cudnnContext.activationDesc, CUDNN_ACTIVATION_SIGMOID, - CUDNN_PROPAGATE_NAN, 0)); - checkCudnn(cudnnActivationForward(cudnnContext, cudnnContext.activationDesc, alpha, - cudnnContext.dstTensorDesc, dstData, beta, cudnnContext.dstTensorDesc, dstData)); - break; - case "relu": - checkCudnn(cudnnSetActivationDescriptor(cudnnContext.activationDesc, CUDNN_ACTIVATION_RELU, - CUDNN_PROPAGATE_NAN, 0)); - checkCudnn(cudnnActivationForward(cudnnContext, cudnnContext.activationDesc, alpha, - cudnnContext.dstTensorDesc, dstData, beta, cudnnContext.dstTensorDesc, dstData)); - break; - case "tanh": - checkCudnn(cudnnSetActivationDescriptor(cudnnContext.activationDesc, CUDNN_ACTIVATION_TANH, - CUDNN_PROPAGATE_NAN, 0)); - checkCudnn(cudnnActivationForward(cudnnContext, cudnnContext.activationDesc, alpha, - cudnnContext.dstTensorDesc, dstData, beta, cudnnContext.dstTensorDesc, dstData)); - break; - case "softmax": - checkCudnn(cudnnSoftmaxForward(cudnnContext, CUDNN_SOFTMAX_ACCURATE, CUDNN_SOFTMAX_MODE_CHANNEL, alpha, - cudnnContext.dstTensorDesc, dstData, beta, cudnnContext.dstTensorDesc, dstData)); - break; - case "logsoftmax": - checkCudnn(cudnnSoftmaxForward(cudnnContext, CUDNN_SOFTMAX_LOG, CUDNN_SOFTMAX_MODE_CHANNEL, alpha, - cudnnContext.dstTensorDesc, dstData, beta, cudnnContext.dstTensorDesc, dstData)); - break; - default: - activation = null; - } - - allocator.registerAction(context, activation); - - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - - return activation; - } - - /** - * @param poolingType Used when preparing data for subsampling layers ONLY. Null for convolution layers - * @return - */ - public static CudnnForwardArgs getCudnnForwardArgs(INDArray input, int[] kernel, int[] strides, int[] padding, int[] dilation, - ConvolutionMode convolutionMode, PoolingType poolingType, CNN2DFormat format){ - INDArray origInput = input; - - //Check if we need to dup the input: views, non-contiguous, etc. CuDNN also seems to have has issues if strides - // are non-default for C order - even if they *should* be OK otherwise - if(input.isView() || !Shape.hasDefaultStridesForShape(input)){ - input = input.dup('c'); - } - - boolean nchw = format == CNN2DFormat.NCHW; - int hIdx = nchw ? 2 : 1; - int wIdx = nchw ? 3 : 2; - - val inH = input.size(hIdx); - val inW = input.size(wIdx); - - boolean manualPadBottom = false; - boolean manualPadRight = false; - - int[] outSize; - if (convolutionMode == ConvolutionMode.Same) { - outSize = ConvolutionUtils.getOutputSize(input, kernel, strides, null, convolutionMode, dilation, format); //Also performs validation - padding = ConvolutionUtils.getSameModeTopLeftPadding(outSize, new int[] {(int) inH, (int) inW}, kernel, strides, dilation); - int[] padBottomRight = ConvolutionUtils.getSameModeBottomRightPadding(outSize, new int[] {(int) inH, (int) inW}, kernel, strides, dilation); - if(!Arrays.equals(padding, padBottomRight)){ - /* - CuDNN - even as of 7.1 (CUDA 9.1) still doesn't have support for proper SAME mode padding (i.e., asymmetric - padding) - padding can *only* be specified as the same amount for both the top/bottom, and for left/right. - In SAME mode padding, sometimes these are the same - but often they are not. - Note that when they differ, the bottom or right padding will be exactly 1 more than the top or left padding. - As per TF, we'll manually pad here: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/conv_ops.cc#L571-L607 - */ - manualPadBottom = (padding[0] != padBottomRight[0]); - manualPadRight = (padding[1] != padBottomRight[1]); - - //NCHW format - long[] newShape; - if(nchw){ - newShape = new long[]{input.size(0), input.size(1), - input.size(2) + (manualPadBottom ? 1 : 0), - input.size(3) + (manualPadRight ? 1 : 0)}; - } else { - newShape = new long[]{input.size(0), - input.size(1) + (manualPadBottom ? 1 : 0), - input.size(2) + (manualPadRight ? 1 : 0), - input.size(3)}; - } - INDArray newInput; - if(poolingType == null || poolingType != PoolingType.MAX){ - newInput = Nd4j.create(input.dataType(), newShape); - } else { - //For max pooling, we don't want to include the padding in the maximum values. But, CuDNN doesn't knowm - // that these values are padding and hence should be excluded. Instead: We'll use -infinity so that, - // if the 'real' (non-padding) values are all < 0, we take the real value, not the padding value - newInput = Nd4j.valueArrayOf(newShape, Double.NEGATIVE_INFINITY, input.dataType()); - } - - if(nchw){ - newInput.put(new INDArrayIndex[]{all(), all(), interval(0,input.size(2)), - interval(0, input.size(3))}, input); - } else { - newInput.put(new INDArrayIndex[]{all(), interval(0,input.size(1)), - interval(0, input.size(2)), all()}, input); - } - - input = newInput; - //Now: we've manually applied the "extra" bottom/right padding only - if required. Consequently, we - // now have the same amount of padding required for top/bottom, and left/right - which we'll let - // CuDNN handle - } - } else { - outSize = ConvolutionUtils.getOutputSize(input, kernel, strides, padding, convolutionMode, dilation, format); //Also performs validation - } - - return new CudnnForwardArgs(manualPadBottom, manualPadRight, input, origInput, padding, outSize); - } - - - @AllArgsConstructor - @Data - public static class CudnnForwardArgs { - private boolean manualPadBottom; - private boolean manualPadRight; - private INDArray input; - private INDArray origInput; - private int[] padding; - private int[] outSize; - } - - @Override - public Map helperMemoryUse() { - //No memory use other than shared, and the structs (which are small) - return Collections.emptyMap(); - } - -} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/convolution/subsampling/CudnnSubsamplingHelper.java b/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/convolution/subsampling/CudnnSubsamplingHelper.java deleted file mode 100644 index b92810959..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/convolution/subsampling/CudnnSubsamplingHelper.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.convolution.subsampling; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.bytedeco.javacpp.Pointer; -import org.deeplearning4j.nn.conf.CNN2DFormat; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.layers.PoolingType; -import org.deeplearning4j.nn.gradient.DefaultGradient; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.cuda.BaseCudnnHelper; -import org.deeplearning4j.cuda.convolution.CudnnConvolutionHelper; -import org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingHelper; -import org.nd4j.jita.allocator.Allocator; -import org.nd4j.jita.allocator.impl.AtomicAllocator; -import org.nd4j.jita.conf.CudaEnvironment; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.executioner.GridExecutioner; -import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.common.primitives.Pair; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.deeplearning4j.nn.workspace.ArrayType; - -import java.util.Collections; -import java.util.Map; - -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cudnn.*; - -import static org.bytedeco.cuda.global.cudnn.*; -import static org.deeplearning4j.cuda.convolution.CudnnConvolutionHelper.getCudnnForwardArgs; -import static org.nd4j.linalg.indexing.NDArrayIndex.all; -import static org.nd4j.linalg.indexing.NDArrayIndex.interval; - -/** - * cuDNN-based helper for the subsampling layer. - * - * @author saudet - */ -@Slf4j -public class CudnnSubsamplingHelper extends BaseCudnnHelper implements SubsamplingHelper { - - public CudnnSubsamplingHelper(DataType dataType) { - super(dataType); - } - - private static class CudnnSubsamplingContext extends CudnnContext { - - private static class Deallocator extends CudnnSubsamplingContext implements Pointer.Deallocator { - Deallocator(CudnnSubsamplingContext c) { - super(c); - } - - @Override - public void deallocate() { - destroyHandles(); - } - } - - private cudnnTensorStruct srcTensorDesc = new cudnnTensorStruct(), dstTensorDesc = new cudnnTensorStruct(), - deltaTensorDesc = new cudnnTensorStruct(); - private cudnnPoolingStruct poolingDesc = new cudnnPoolingStruct(); - - public CudnnSubsamplingContext() { - createHandles(); - deallocator(new Deallocator(this)); - } - - public CudnnSubsamplingContext(CudnnSubsamplingContext c) { - super(c); - srcTensorDesc = new cudnnTensorStruct(c.srcTensorDesc); - dstTensorDesc = new cudnnTensorStruct(c.dstTensorDesc); - deltaTensorDesc = new cudnnTensorStruct(c.deltaTensorDesc); - poolingDesc = new cudnnPoolingStruct(c.poolingDesc); - } - - @Override - protected void createHandles() { - super.createHandles(); - checkCudnn(cudnnCreateTensorDescriptor(srcTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dstTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(deltaTensorDesc)); - checkCudnn(cudnnCreatePoolingDescriptor(poolingDesc)); - } - - @Override - protected void destroyHandles() { - checkCudnn(cudnnDestroyPoolingDescriptor(poolingDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(srcTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dstTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(deltaTensorDesc)); - super.destroyHandles(); - } - } - - private CudnnSubsamplingContext cudnnContext = new CudnnSubsamplingContext(); - - @Override - public Pair backpropGradient(INDArray input, INDArray epsilon, int[] kernel, int[] strides, - int[] pad, PoolingType poolingType, ConvolutionMode convolutionMode, - int[] dilation, CNN2DFormat format, LayerWorkspaceMgr workspaceMgr) { - if(dilation[0] != 1 || dilation[1] != 1){ - //CuDNN doesn't support dilated subsampling - return null; - } - - boolean nchw = format == CNN2DFormat.NCHW; - int chIdx = nchw ? 1 : 3; - int hIdx = nchw ? 2 : 1; - int wIdx = nchw ? 3 : 2; - - //We require the output as one of the arguments for backprop here - //TODO we could add cache mode support here somehow... - INDArray reduced = activate(input, true, kernel, strides, pad, poolingType, convolutionMode, dilation, format, workspaceMgr); - - val miniBatch = input.size(0); - val depth = input.size(chIdx); - - CudnnConvolutionHelper.CudnnForwardArgs args = getCudnnForwardArgs(input, kernel, strides, pad, dilation, convolutionMode, poolingType, format); - input = args.getInput(); - val inH = input.size(hIdx); - val inW = input.size(wIdx); - val srcStride = input.stride(); - int[] outSize = args.getOutSize(); - int outH = outSize[0]; - int outW = outSize[1]; - - //subsampling doesn't have weights and thus gradients are not calculated for this layer - //only scale and reshape epsilon - Gradient retGradient = new DefaultGradient(); - - //Epsilons in shape: [miniBatch, channels, outH, outW] - //Epsilons out shape: [miniBatch, channels, inH, inW] - - int poolingMode; - switch (poolingType) { - case AVG: - poolingMode = CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING; - break; - case MAX: - poolingMode = CUDNN_POOLING_MAX; - break; - default: - return null; - } - - if (!Shape.hasDefaultStridesForShape(epsilon) || epsilon.isView()) { - // apparently not supported by cuDNN - epsilon = epsilon.dup('c'); - } - - input = input.dup(); - - val deltaStride = epsilon.stride(); - - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, (int) miniBatch, (int) depth, (int) inH, (int) inW, - (int) srcStride[0], (int) srcStride[chIdx], (int) srcStride[hIdx], (int) srcStride[wIdx])); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.deltaTensorDesc, dataType, (int) miniBatch, (int) depth, (int) outH, (int) outW, - (int) deltaStride[0], (int) deltaStride[chIdx], (int) deltaStride[hIdx], (int) deltaStride[wIdx])); - checkCudnn(cudnnSetPooling2dDescriptor(cudnnContext.poolingDesc, poolingMode, CUDNN_PROPAGATE_NAN, kernel[0], - kernel[1], pad[0], pad[1], strides[0], strides[1])); - - long[] outEpsShape = nchw ? new long[] {miniBatch, depth, inH, inW} : new long[] {miniBatch, inH, inW, depth}; - INDArray outEpsilon = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, input.dataType(), outEpsShape, 'c'); - - val dstStride = outEpsilon.stride(); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, (int) miniBatch, (int) depth, (int) inH, (int) inW, - (int) dstStride[0], (int) dstStride[chIdx], (int) dstStride[hIdx], (int) dstStride[wIdx])); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareAction(input, epsilon, reduced, outEpsilon); - Pointer srcData = allocator.getPointer(input, context); - Pointer epsData = allocator.getPointer(epsilon, context); - Pointer zData = allocator.getPointer(reduced, context); - Pointer dstData = allocator.getPointer(outEpsilon, context); - - checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream()))); - checkCudnn(cudnnPoolingBackward(cudnnContext, cudnnContext.poolingDesc, alpha, cudnnContext.deltaTensorDesc, - zData, cudnnContext.deltaTensorDesc, epsData, cudnnContext.srcTensorDesc, srcData, beta, - cudnnContext.dstTensorDesc, dstData)); - - allocator.registerAction(context, outEpsilon, input, epsilon, reduced); - - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - - //Note that: if we had to manually pad for SAME mode, we have to 'undo' this manual padding for the epsilon - // we return. The returned epsilon (i.e., dL/dIn array) has to be the same shape as the *original* input. - if(args.isManualPadBottom() || args.isManualPadRight()) { - if(nchw){ - outEpsilon = outEpsilon.get(all(), all(), - interval(0, outEpsilon.size(2) - (args.isManualPadBottom() ? 1 : 0)), - interval(0, outEpsilon.size(3) - (args.isManualPadRight() ? 1 : 0))); - } else { - outEpsilon = outEpsilon.get(all(), - interval(0, outEpsilon.size(1) - (args.isManualPadBottom() ? 1 : 0)), - interval(0, outEpsilon.size(2) - (args.isManualPadRight() ? 1 : 0)), - all()); - } - } - - return new Pair<>(retGradient, outEpsilon); - } - - - @Override - public INDArray activate(INDArray input, boolean training, int[] kernel, int[] strides, int[] pad, - PoolingType poolingType, ConvolutionMode convolutionMode, int[] dilation, CNN2DFormat format, LayerWorkspaceMgr workspaceMgr) { - if(dilation[0] != 1 || dilation[1] != 1){ - //CuDNN doesn't support dilated subsampling - return null; - } - - boolean nchw = format == CNN2DFormat.NCHW; - int chIdx = nchw ? 1 : 3; - int hIdx = nchw ? 2 : 1; - int wIdx = nchw ? 3 : 2; - - val miniBatch = input.size(0); - val inDepth = input.size(nchw ? 1 : 3); - - CudnnConvolutionHelper.CudnnForwardArgs args = getCudnnForwardArgs(input, kernel, strides, pad, dilation, convolutionMode, poolingType, format); - input = args.getInput(); - val inH = input.size(nchw ? 2 : 1); - val inW = input.size(nchw ? 3 : 2); - val srcStride = input.stride(); - val outSize = args.getOutSize(); - int outH = outSize[0]; - int outW = outSize[1]; - - - int poolingMode; - switch (poolingType) { - case AVG: - poolingMode = CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING; - break; - case MAX: - poolingMode = CUDNN_POOLING_MAX; - break; - default: - return null; - } - - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - checkCudnn(cudnnSetPooling2dDescriptor(cudnnContext.poolingDesc, poolingMode, CUDNN_PROPAGATE_NAN, kernel[0], - kernel[1], pad[0], pad[1], strides[0], strides[1])); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, (int) miniBatch, (int) inDepth, (int) inH, (int) inW, - (int) srcStride[0], (int) srcStride[chIdx], (int) srcStride[hIdx], (int) srcStride[wIdx])); - - long[] outShape = nchw ? new long[] {miniBatch, inDepth, outH, outW} : new long[] {miniBatch, outH, outW, inDepth}; - INDArray reduced = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.dataType(), outShape, 'c'); - - val dstStride = reduced.stride(); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, (int) miniBatch, (int) inDepth, (int) outH, (int) outW, - (int) dstStride[0], (int) dstStride[chIdx], (int) dstStride[hIdx], (int) dstStride[wIdx])); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareAction(input, reduced); - Pointer srcData = allocator.getPointer(input, context); - Pointer dstData = allocator.getPointer(reduced, context); - - checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream()))); - checkCudnn(cudnnPoolingForward(cudnnContext, cudnnContext.poolingDesc, alpha, cudnnContext.srcTensorDesc, - srcData, beta, cudnnContext.dstTensorDesc, dstData)); - - allocator.registerAction(context, reduced, input); - - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - - return reduced; - } - - @Override - public Map helperMemoryUse() { - //No persistent memory use other than the structs (which are small) - return Collections.emptyMap(); - } - -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/dropout/CudnnDropoutHelper.java b/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/dropout/CudnnDropoutHelper.java deleted file mode 100644 index 9b3414d95..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/dropout/CudnnDropoutHelper.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.dropout; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import com.jakewharton.byteunits.BinaryByteUnit; -import org.bytedeco.javacpp.*; -import org.deeplearning4j.nn.conf.dropout.DropoutHelper; -import org.deeplearning4j.cuda.BaseCudnnHelper; -import org.nd4j.jita.allocator.Allocator; -import org.nd4j.jita.allocator.impl.AtomicAllocator; -import org.nd4j.jita.conf.CudaEnvironment; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.common.util.ArrayUtil; - -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cudnn.*; - -import static org.bytedeco.cuda.global.cudnn.*; - -/** - * CuDNN dropout helper - * - * Note that for repeatability between calls (for example, for gradient checks), we need to do two things: - * (a) set the ND4J RNG seed - * (b) clear the rngStates field - * - * @author Alex Black - */ -@Data -@Slf4j -public class CudnnDropoutHelper extends BaseCudnnHelper implements DropoutHelper { - - private static class CudnnDropoutContext extends CudnnContext { - - private static class Deallocator extends CudnnDropoutContext implements Pointer.Deallocator { - Deallocator(CudnnDropoutContext c) { - super(c); - } - - @Override - public void deallocate() { - destroyHandles(); - } - } - - private cudnnTensorStruct xTensorDesc = new cudnnTensorStruct(); //Input - private cudnnTensorStruct dxTensorDesc = new cudnnTensorStruct(); //Grad at input - private cudnnTensorStruct yTensorDesc = new cudnnTensorStruct(); //Output - private cudnnTensorStruct dyTensorDesc = new cudnnTensorStruct(); //Grad at output - private cudnnDropoutStruct dropoutDesc = new cudnnDropoutStruct(); - - public CudnnDropoutContext() { - createHandles(); - deallocator(new Deallocator(this)); - } - - public CudnnDropoutContext(CudnnDropoutContext c) { - super(c); - xTensorDesc = new cudnnTensorStruct(c.xTensorDesc); - dxTensorDesc = new cudnnTensorStruct(c.dxTensorDesc); - yTensorDesc = new cudnnTensorStruct(c.yTensorDesc); - dyTensorDesc = new cudnnTensorStruct(c.dyTensorDesc); - dropoutDesc = new cudnnDropoutStruct(c.dropoutDesc); - } - - @Override - protected void createHandles() { - super.createHandles(); - checkCudnn(cudnnCreateTensorDescriptor(xTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dxTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(yTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dyTensorDesc)); - checkCudnn(cudnnCreateDropoutDescriptor(dropoutDesc)); - } - - @Override - protected void destroyHandles() { - checkCudnn(cudnnDestroyTensorDescriptor(xTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dxTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(yTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dyTensorDesc)); - checkCudnn(cudnnDestroyDropoutDescriptor(dropoutDesc)); - super.destroyHandles(); - } - } - - private CudnnDropoutContext cudnnContext = new CudnnDropoutContext(); - private boolean initializedDescriptor = false; - private DataCache rngStates; //"Pointer to user-allocated GPU memory that will hold random number generator states." - private DataCache mask; //Mask: persistence between forward and backward - private SizeTPointer stateSizeBytesPtr; - private SizeTPointer reserveSizeBytesPtr; - private float lastInitializedP; - - public CudnnDropoutHelper(DataType dataType){ - super(dataType); - } - - @Override - public void applyDropout(INDArray input, INDArray resultArray, double dropoutInputRetainProb) { - float p = (float)(1.0 - dropoutInputRetainProb); //CuDNN uses p = probability of setting to 0. We use p = probability of retaining - - //TODO int cast - int[] inShape = adaptForTensorDescr(ArrayUtil.toInts(input.shape())); - int[] inStride = adaptForTensorDescr(ArrayUtil.toInts(input.stride())); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.xTensorDesc, dataType, inShape.length, inShape, inStride)); - - int[] outShape = adaptForTensorDescr(ArrayUtil.toInts(resultArray.shape())); - int[] outStride = adaptForTensorDescr(ArrayUtil.toInts(resultArray.stride())); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.yTensorDesc, dataType, outShape.length, outShape, outStride)); - - - if(stateSizeBytesPtr == null){ - stateSizeBytesPtr = new SizeTPointer(1); - reserveSizeBytesPtr = new SizeTPointer(1); - } - checkCudnn(cudnnDropoutGetStatesSize(cudnnContext, stateSizeBytesPtr)); - long rngStateSizeBytes = stateSizeBytesPtr.get(); - checkCudnn(cudnnDropoutGetReserveSpaceSize(cudnnContext.xTensorDesc, reserveSizeBytesPtr)); - long maskReserveSizeBytes = reserveSizeBytesPtr.get(); - - if(rngStates == null || rngStates.capacity() < rngStateSizeBytes){ - if(log.isTraceEnabled()){ - if(rngStates == null){ - log.trace("CudnnDropoutHelper: Allocating intial RNG states workspace of size {} ({})", rngStateSizeBytes, - BinaryByteUnit.format(rngStateSizeBytes, "#.00")); - } else { - log.trace("CudnnDropoutHelper: Deallocating RNG states of size {} ({}), allocating new workspace of size {} ({})", - rngStates.capacity(), BinaryByteUnit.format(rngStates.capacity(), "#.00"), - rngStateSizeBytes, BinaryByteUnit.format(rngStateSizeBytes, "#.00")); - } - } - - if(rngStates != null) - rngStates.deallocate(); - //states = "Pointer to user-allocated GPU memory that will hold random number generator states." - rngStates = new DataCache(rngStateSizeBytes); - initializedDescriptor = false; - } - if(mask == null || mask.capacity() < maskReserveSizeBytes){ - if(log.isTraceEnabled()){ - if(mask == null){ - log.trace("CudnnDropoutHelper: Allocating intial mask array of size {} ({})", maskReserveSizeBytes, - BinaryByteUnit.format(maskReserveSizeBytes, "#.00")); - } else { - log.trace("CudnnDropoutHelper: Deallocating mask array of size {} ({}), allocating new mask array of size {} ({})", - mask.capacity(), BinaryByteUnit.format(mask.capacity(), "#.00"), - maskReserveSizeBytes, BinaryByteUnit.format(maskReserveSizeBytes, "#.00")); - } - } - - if(mask != null) - mask.deallocate(); - //mask = "Pointer to user-allocated GPU memory used by this function. It is expected - //that contents of reserveSpace doe not change between cudnnDropoutForward and - //cudnnDropoutBackward calls." - mask = new DataCache(maskReserveSizeBytes); - } - - //Dropout descriptor: (re)initialize if required - if(!initializedDescriptor || p != lastInitializedP) { - if(log.isTraceEnabled()){ - log.trace("CudnnDropoutHelper: (re)initializing dropout descriptor"); - } - //NOTE: cudnnSetDropoutDescriptor has some internal computation/initialization, and hence is expensive to - // call - so we want to call this as infrequently as possible, and cache the result - long seed = Nd4j.getRandom().nextLong(); - lastInitializedP = p; - checkCudnn(cudnnSetDropoutDescriptor(cudnnContext.dropoutDesc, cudnnContext, p, rngStates, rngStates.capacity(), seed)); - initializedDescriptor = true; - } - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareAction(input, resultArray); - Pointer xPtr = allocator.getPointer(input, context); - Pointer yPtr = allocator.getPointer(resultArray, context); - - checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream()))); - checkCudnn(cudnnDropoutForward(cudnnContext, cudnnContext.dropoutDesc, cudnnContext.xTensorDesc, xPtr, - cudnnContext.yTensorDesc, yPtr, mask, mask.capacity())); - - allocator.registerAction(context, input, resultArray); - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - } - - @Override - public void backprop(INDArray gradAtOutput, INDArray gradAtInput) { - int[] gradAtOutShape = adaptForTensorDescr(ArrayUtil.toInts(gradAtOutput.shape())); - int[] gradAtOutStride = adaptForTensorDescr(ArrayUtil.toInts(gradAtOutput.stride())); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.dyTensorDesc, dataType, gradAtOutShape.length, gradAtOutShape, gradAtOutStride)); - - int[] gradAtInShape = adaptForTensorDescr(ArrayUtil.toInts(gradAtInput.shape())); - int[] gradAtInStride = adaptForTensorDescr(ArrayUtil.toInts(gradAtInput.stride())); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.dxTensorDesc, dataType, gradAtInShape.length, gradAtInShape, gradAtInStride)); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareAction(gradAtOutput, gradAtInput); - Pointer dyPtr = allocator.getPointer(gradAtOutput, context); - Pointer dxPtr = allocator.getPointer(gradAtInput, context); - - checkCudnn(cudnnDropoutBackward(cudnnContext, cudnnContext.dropoutDesc, cudnnContext.dyTensorDesc, dyPtr, - cudnnContext.dxTensorDesc, dxPtr, mask, mask.capacity())); - - allocator.registerAction(context, gradAtOutput, gradAtInput); - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/normalization/CudnnBatchNormalizationHelper.java b/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/normalization/CudnnBatchNormalizationHelper.java deleted file mode 100644 index fea813aa0..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/normalization/CudnnBatchNormalizationHelper.java +++ /dev/null @@ -1,384 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.normalization; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.bytedeco.javacpp.Pointer; -import org.deeplearning4j.nn.conf.CNN2DFormat; -import org.deeplearning4j.nn.gradient.DefaultGradient; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.cuda.BaseCudnnHelper; -import org.deeplearning4j.nn.layers.normalization.BatchNormalizationHelper; -import org.deeplearning4j.nn.params.BatchNormalizationParamInitializer; -import org.deeplearning4j.nn.workspace.ArrayType; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.nd4j.jita.allocator.Allocator; -import org.nd4j.jita.allocator.impl.AtomicAllocator; -import org.nd4j.jita.conf.CudaEnvironment; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.memory.MemoryWorkspace; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.executioner.GridExecutioner; -import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.common.primitives.Pair; -import org.nd4j.common.util.ArrayUtil; - -import java.util.HashMap; -import java.util.Map; - -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cudnn.*; - -import static org.bytedeco.cuda.global.cudnn.*; - -/** - * cuDNN-based helper for the batch normalization layer. - * - * @author saudet - */ -@Slf4j -public class CudnnBatchNormalizationHelper extends BaseCudnnHelper implements BatchNormalizationHelper { - - public CudnnBatchNormalizationHelper(DataType dataType) { - super(dataType); - } - - private static class CudnnBatchNormalizationContext extends CudnnContext { - - private static class Deallocator extends CudnnBatchNormalizationContext implements Pointer.Deallocator { - Deallocator(CudnnBatchNormalizationContext c) { - super(c); - } - - @Override - public void deallocate() { - destroyHandles(); - } - } - - private cudnnTensorStruct srcTensorDesc = new cudnnTensorStruct(), dstTensorDesc = new cudnnTensorStruct(), - deltaTensorDesc = new cudnnTensorStruct(), gammaBetaTensorDesc = new cudnnTensorStruct(); - - public CudnnBatchNormalizationContext() { - createHandles(); - deallocator(new Deallocator(this)); - } - - public CudnnBatchNormalizationContext(CudnnBatchNormalizationContext c) { - super(c); - srcTensorDesc = new cudnnTensorStruct(c.srcTensorDesc); - dstTensorDesc = new cudnnTensorStruct(c.dstTensorDesc); - deltaTensorDesc = new cudnnTensorStruct(c.deltaTensorDesc); - gammaBetaTensorDesc = new cudnnTensorStruct(c.gammaBetaTensorDesc); - } - - @Override - protected void createHandles() { - super.createHandles(); - checkCudnn(cudnnCreateTensorDescriptor(srcTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dstTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(deltaTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(gammaBetaTensorDesc)); - } - - @Override - protected void destroyHandles() { - checkCudnn(cudnnDestroyTensorDescriptor(srcTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dstTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(deltaTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(gammaBetaTensorDesc)); - super.destroyHandles(); - } - } - - protected final int batchNormMode = CUDNN_BATCHNORM_SPATIAL; // would need to increase rank of gamma and beta for CUDNN_BATCHNORM_PER_ACTIVATION - - private CudnnBatchNormalizationContext cudnnContext = new CudnnBatchNormalizationContext(); - private INDArray meanCache; - private INDArray varCache; - private double eps; - - public boolean checkSupported(double eps, boolean isFixedGammaBeta) { - boolean supported = checkSupported(); - if (eps < CUDNN_BN_MIN_EPSILON) { - supported = false; - log.warn("Not supported: eps < CUDNN_BN_MIN_EPSILON (" + eps + " < " + CUDNN_BN_MIN_EPSILON + ")"); - } - return supported; - } - - @Override - public Pair backpropGradient(INDArray input, INDArray epsilon, long[] shape, INDArray gamma, INDArray beta, - INDArray dGammaView, INDArray dBetaView, double eps, CNN2DFormat format, LayerWorkspaceMgr layerWorkspaceMgr) { - - boolean nchw = format == CNN2DFormat.NCHW; - - this.eps = eps; - - int cudnnTensorFormat = nchw ? CUDNN_TENSOR_NCHW : CUDNN_TENSOR_NHWC; - int chIdx = nchw ? 1 : 3; - int hIdx = nchw ? 2 : 1; - int wIdx = nchw ? 3 : 2; - - val miniBatch = (int) input.size(0); - val depth = (int) input.size(chIdx); - val inH = (int) input.size(hIdx); - val inW = (int) input.size(wIdx); - - final boolean isHalf = (input.dataType() == DataType.HALF); - INDArray gammaOrig = null; - INDArray dGammaViewOrig = null; - INDArray dBetaViewOrig = null; - if(isHalf) { //Convert FP16 to FP32 if required (CuDNN BN doesn't support FP16 for these params, only for input/output) - gammaOrig = gamma; - dGammaViewOrig = dGammaView; - dBetaViewOrig = dBetaView; - /* - From CuDNN docs: bnScale, resultBnScaleDiff, resultBnBiasDiff, savedMean, savedInvVariance - "Note: The data type of this tensor descriptor must be 'float' for FP16 and FP32 input tensors, and 'double' - for FP64 input tensors." - >> Last 2 are the meanCache and varCache; first 3 are below - */ - gamma = gamma.castTo(DataType.FLOAT); - dGammaView = dGammaView.castTo(DataType.FLOAT); - dBetaView = dBetaView.castTo(DataType.FLOAT); - } - - Gradient retGradient = new DefaultGradient(); - - if (!Shape.hasDefaultStridesForShape(epsilon)) { - // apparently not supported by cuDNN - epsilon = epsilon.dup('c'); - } - - val srcStride = ArrayUtil.toInts(input.stride()); - val deltaStride = ArrayUtil.toInts(epsilon.stride()); - - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, (int) miniBatch, (int) depth, (int) inH, (int) inW, - (int) srcStride[0], (int) srcStride[chIdx], (int) srcStride[hIdx], (int) srcStride[wIdx])); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.deltaTensorDesc, dataType, (int) miniBatch, (int) depth, (int) inH, (int) inW, - (int) deltaStride[0], (int) deltaStride[chIdx], (int) deltaStride[hIdx], (int) deltaStride[wIdx])); - - long[] nextEpsShape = nchw ? new long[] {miniBatch, depth, inH, inW} : new long[] {miniBatch, inH, inW, depth}; - INDArray nextEpsilon = layerWorkspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, input.dataType(), nextEpsShape, 'c'); - val dstStride = ArrayUtil.toInts(nextEpsilon.stride()); - - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, miniBatch, depth, inH, inW, - dstStride[0], dstStride[chIdx], dstStride[hIdx], dstStride[wIdx])); - checkCudnn(cudnnSetTensor4dDescriptor(cudnnContext.gammaBetaTensorDesc, cudnnTensorFormat, toCudnnDataType(gamma.data().dataType()), (int)shape[0], - (int)shape[1], shape.length > 2 ? (int)shape[2] : 1, shape.length > 3 ? (int)shape[3] : 1)); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareActionAllWrite(input, epsilon, nextEpsilon, gamma, - dGammaView, dBetaView); - Pointer srcData = allocator.getPointer(input, context); - Pointer epsData = allocator.getPointer(epsilon, context); - Pointer dstData = allocator.getPointer(nextEpsilon, context); - Pointer gammaData = allocator.getPointer(gamma, context); - Pointer dGammaData = allocator.getPointer(dGammaView, context); - Pointer dBetaData = allocator.getPointer(dBetaView, context); - Pointer meanCacheData = allocator.getPointer(meanCache, context); - Pointer varCacheData = allocator.getPointer(varCache, context); - - checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream()))); - checkCudnn(cudnnBatchNormalizationBackward(cudnnContext, batchNormMode, alpha, this.beta, alpha, alpha, - cudnnContext.srcTensorDesc, srcData, cudnnContext.deltaTensorDesc, epsData, - cudnnContext.dstTensorDesc, dstData, cudnnContext.gammaBetaTensorDesc, gammaData, dGammaData, - dBetaData, eps, meanCacheData, varCacheData)); - - allocator.getFlowController().registerActionAllWrite(context, input, epsilon, nextEpsilon, gamma, dGammaView, - dBetaView); - - retGradient.setGradientFor(BatchNormalizationParamInitializer.GAMMA, dGammaView); - retGradient.setGradientFor(BatchNormalizationParamInitializer.BETA, dBetaView); - - context.syncOldStream(); - - //Convert back and assign, if required: - if(isHalf){ - gammaOrig.assign(gamma.castTo(DataType.HALF)); - dGammaViewOrig.assign(dGammaView.castTo(DataType.HALF)); - dBetaViewOrig.assign(dBetaView.castTo(DataType.HALF)); - } - - return new Pair<>(retGradient, nextEpsilon); - } - - - @Override - public INDArray preOutput(INDArray x, boolean training, long[] shape, INDArray gamma, INDArray beta, INDArray mean, - INDArray var, double decay, double eps, CNN2DFormat format, LayerWorkspaceMgr workspaceMgr) { - boolean nchw = format == CNN2DFormat.NCHW; - int cudnnTensorFormat = nchw ? CUDNN_TENSOR_NCHW : CUDNN_TENSOR_NHWC; - int chIdx = nchw ? 1 : 3; - int hIdx = nchw ? 2 : 1; - int wIdx = nchw ? 3 : 2; - - this.eps = eps; - final boolean isHalf = (x.dataType() == DataType.FLOAT16); - INDArray origGamma = gamma; - INDArray origBeta = beta; - INDArray origMean = mean; - INDArray origVar = var; - if(isHalf) { - gamma = gamma.castTo(DataType.FLOAT); - beta = beta.castTo(DataType.FLOAT); - mean = mean.castTo(DataType.FLOAT); - var = var.castTo(DataType.FLOAT); - } - - //Notation difference between CuDNN and our implementation: - //Us: runningMean = (1-decay) * batchMean + decay * runningMean - //CuDNN: runningMean = decay * batchMean + (1-decay) * runningMean - //i.e., "decay" has a different meaning... - //Disable in-place updating of running mean/variance, so that all parameter changes are done via the update/gradient - // vector. This is necessary for BatchNormalization to be safe to use in distributed gradient sharing settings - decay = 0.0; //From cudnn docs: runningMean = newMean*factor + runningMean*(1-factor). -> 0 = "in-place modification of running mean disabled" - - val miniBatch = (int) x.size(0); - val inDepth = (int) x.size(chIdx); - val inH = (int) x.size(hIdx); - val inW = (int) x.size(wIdx); - - val srcStride = ArrayUtil.toInts(x.stride()); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, miniBatch, inDepth, inH, inW, - srcStride[0], srcStride[chIdx], srcStride[hIdx], srcStride[wIdx])); - - long[] actShape = nchw ? new long[] {miniBatch, inDepth, inH, inW} : new long[] {miniBatch, inH, inW, inDepth}; - INDArray activations = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, x.dataType(), actShape, 'c'); - - val dstStride = ArrayUtil.toInts(activations.stride()); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, miniBatch, inDepth, inH, inW, - dstStride[0], dstStride[chIdx], dstStride[hIdx], dstStride[wIdx])); - - checkCudnn(cudnnSetTensor4dDescriptor(cudnnContext.gammaBetaTensorDesc, cudnnTensorFormat, toCudnnDataType(mean.data().dataType()), (int)shape[0], - (int)shape[1], shape.length > 2 ? (int)shape[2] : 1, shape.length > 3 ? (int)shape[3] : 1)); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = - allocator.getFlowController().prepareActionAllWrite(x, activations, gamma, beta, mean, var); - Pointer srcData = allocator.getPointer(x, context); - Pointer dstData = allocator.getPointer(activations, context); - Pointer gammaData = allocator.getPointer(gamma, context); - Pointer betaData = allocator.getPointer(beta, context); - Pointer meanData = allocator.getPointer(mean, context); - Pointer varData = allocator.getPointer(var, context); - - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream()))); - if (training) { - if(meanCache == null || meanCache.length() < mean.length()){ - try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) { - meanCache = Nd4j.createUninitialized(x.dataType(), mean.length()); - } - if(x.dataType() == DataType.HALF){ - try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) { - meanCache = meanCache.castTo(DataType.FLOAT); - } - } - } - if(varCache == null || varCache.length() < mean.length()){ - try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) { - varCache = Nd4j.createUninitialized(x.dataType(), mean.length()); - } - if(nd4jDataType == DataType.HALF){ - try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) { - varCache = varCache.castTo(DataType.FLOAT); - } - } - } - Pointer meanCacheData = allocator.getPointer(meanCache, context); - Pointer varCacheData = allocator.getPointer(varCache, context); - - checkCudnn(cudnnBatchNormalizationForwardTraining(cudnnContext, batchNormMode, this.alpha, this.beta, - cudnnContext.srcTensorDesc, srcData, cudnnContext.dstTensorDesc, dstData, - cudnnContext.gammaBetaTensorDesc, gammaData, betaData, decay, meanData, varData, eps, - meanCacheData, varCacheData)); - } else { - checkCudnn(cudnnBatchNormalizationForwardInference(cudnnContext, batchNormMode, this.alpha, this.beta, - cudnnContext.srcTensorDesc, srcData, cudnnContext.dstTensorDesc, dstData, - cudnnContext.gammaBetaTensorDesc, gammaData, betaData, meanData, varData, eps)); - } - - allocator.getFlowController().registerActionAllWrite(context, x, activations, gamma, beta, mean, var); - - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - - context.syncOldStream(); - if(training) { - AtomicAllocator.getInstance().getAllocationPoint(meanCache).tickDeviceWrite(); - AtomicAllocator.getInstance().getAllocationPoint(varCache).tickDeviceWrite(); - } - - if(training && isHalf){ - //Update the running mean and variance arrays; also gamma/beta - origMean.assign(mean.castTo(DataType.HALF)); - origVar.assign(var.castTo(DataType.HALF)); - origGamma.assign(gamma.castTo(DataType.HALF)); - origBeta.assign(beta.castTo(DataType.HALF)); - } - - return activations; - } - - @Override - public INDArray getMeanCache(DataType dataType) { - if(dataType == DataType.HALF){ - //Buffer is FP32 - return meanCache.castTo(DataType.HALF); - } - return meanCache; - } - - @Override - public INDArray getVarCache(DataType dataType) { - INDArray ret; - if(dataType == DataType.HALF){ - INDArray vc = varCache.castTo(DataType.HALF); - ret = vc.mul(vc).rdivi(1.0).subi(eps); - } else { - ret = varCache.mul(varCache).rdivi(1.0).subi(eps); - } - if(dataType == DataType.HALF){ - //Buffer is FP32 - return ret.castTo(DataType.HALF); - } - return ret; - } - - - @Override - public Map helperMemoryUse() { - Map memUse = new HashMap<>(); - memUse.put("meanCache", meanCache == null ? 0 : meanCache.length() * meanCache.data().getElementSize()); - memUse.put("varCache", varCache == null ? 0 : varCache.length() * varCache.data().getElementSize()); - return memUse; - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/normalization/CudnnLocalResponseNormalizationHelper.java b/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/normalization/CudnnLocalResponseNormalizationHelper.java deleted file mode 100644 index e0257a3ec..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/normalization/CudnnLocalResponseNormalizationHelper.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.normalization; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.bytedeco.javacpp.Pointer; -import org.deeplearning4j.nn.gradient.DefaultGradient; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.cuda.BaseCudnnHelper; -import org.deeplearning4j.nn.layers.normalization.LocalResponseNormalizationHelper; -import org.nd4j.jita.allocator.Allocator; -import org.nd4j.jita.allocator.impl.AtomicAllocator; -import org.nd4j.jita.conf.CudaEnvironment; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.executioner.GridExecutioner; -import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.common.primitives.Pair; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.deeplearning4j.nn.workspace.ArrayType; -import org.nd4j.common.util.ArrayUtil; - -import java.util.Collections; -import java.util.Map; - -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cudnn.*; - -import static org.bytedeco.cuda.global.cudnn.*; - -/** - * cuDNN-based helper for the local response normalization layer. - * - * @author saudet - */ -@Slf4j -public class CudnnLocalResponseNormalizationHelper extends BaseCudnnHelper implements LocalResponseNormalizationHelper { - - public CudnnLocalResponseNormalizationHelper(DataType dataType) { - super(dataType); - } - - private static class CudnnLocalResponseNormalizationContext extends CudnnContext { - - private static class Deallocator extends CudnnLocalResponseNormalizationContext implements Pointer.Deallocator { - Deallocator(CudnnLocalResponseNormalizationContext c) { - super(c); - } - - @Override - public void deallocate() { - destroyHandles(); - } - } - - private cudnnTensorStruct srcTensorDesc = new cudnnTensorStruct(), dstTensorDesc = new cudnnTensorStruct(), - deltaTensorDesc = new cudnnTensorStruct(); - private cudnnLRNStruct lrnDesc = new cudnnLRNStruct(); - - public CudnnLocalResponseNormalizationContext() { - createHandles(); - deallocator(new Deallocator(this)); - } - - public CudnnLocalResponseNormalizationContext(CudnnLocalResponseNormalizationContext c) { - super(c); - srcTensorDesc = new cudnnTensorStruct(c.srcTensorDesc); - dstTensorDesc = new cudnnTensorStruct(c.dstTensorDesc); - deltaTensorDesc = new cudnnTensorStruct(c.deltaTensorDesc); - lrnDesc = new cudnnLRNStruct(c.lrnDesc); - } - - @Override - protected void createHandles() { - super.createHandles(); - checkCudnn(cudnnCreateTensorDescriptor(srcTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dstTensorDesc)); - checkCudnn(cudnnCreateTensorDescriptor(deltaTensorDesc)); - checkCudnn(cudnnCreateLRNDescriptor(lrnDesc)); - } - - @Override - protected void destroyHandles() { - checkCudnn(cudnnDestroyLRNDescriptor(lrnDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(srcTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dstTensorDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(deltaTensorDesc)); - super.destroyHandles(); - } - } - - private CudnnLocalResponseNormalizationContext cudnnContext = new CudnnLocalResponseNormalizationContext(); - private INDArray activations = null; - - public boolean checkSupported(double k, double n, double alpha, double beta) { - boolean supported = checkSupported(); - if (n < CUDNN_LRN_MIN_N) { - supported = false; - log.warn("Not supported: n < CUDNN_LRN_MIN_N (" + n + " < " + CUDNN_LRN_MIN_N + ")"); - } - if (n > CUDNN_LRN_MAX_N) { - supported = false; - log.warn("Not supported: n > CUDNN_LRN_MAX_N (" + n + " > " + CUDNN_LRN_MAX_N + ")"); - } - if (k < CUDNN_LRN_MIN_K) { - supported = false; - log.warn("Not supported: k < CUDNN_LRN_MIN_K (" + k + " < " + CUDNN_LRN_MIN_K + ")"); - } - if (beta < CUDNN_LRN_MIN_BETA) { - supported = false; - log.warn("Not supported: beta < CUDNN_LRN_MIN_BETA (" + beta + " < " + CUDNN_LRN_MIN_BETA + ")"); - } - return supported; - } - - @Override - public Pair backpropGradient(INDArray input, INDArray epsilon, double k, double n, double alpha, - double beta, LayerWorkspaceMgr workspaceMgr) { - val miniBatch = (int) input.size(0); - val depth = (int) input.size(1); - val inH = (int) input.size(2); - val inW = (int) input.size(3); - - Gradient retGradient = new DefaultGradient(); - - if (!Shape.hasDefaultStridesForShape(epsilon)) { - // apparently not supported by cuDNN - epsilon = epsilon.dup('c'); - } - - val srcStride = ArrayUtil.toInts(input.stride()); - val deltaStride = ArrayUtil.toInts(epsilon.stride()); - - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, miniBatch, depth, inH, inW, - srcStride[0], srcStride[1], srcStride[2], srcStride[3])); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.deltaTensorDesc, dataType, miniBatch, depth, inH, inW, - deltaStride[0], deltaStride[1], deltaStride[2], deltaStride[3])); - checkCudnn(cudnnSetLRNDescriptor(cudnnContext.lrnDesc, (int) n, alpha, beta, k)); - - INDArray nextEpsilon = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, input.dataType(), new long[] {miniBatch, depth, inH, inW}, 'c'); - - val dstStride = ArrayUtil.toInts(nextEpsilon.stride()); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, miniBatch, depth, inH, inW, - dstStride[0], dstStride[1], dstStride[2], dstStride[3])); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = - allocator.getFlowController().prepareActionAllWrite(input, epsilon, activations, nextEpsilon); - Pointer srcData = allocator.getPointer(input, context); - Pointer epsData = allocator.getPointer(epsilon, context); - Pointer zData = allocator.getPointer(activations, context); - Pointer dstData = allocator.getPointer(nextEpsilon, context); - - checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream()))); - checkCudnn(cudnnLRNCrossChannelBackward(cudnnContext, cudnnContext.lrnDesc, CUDNN_LRN_CROSS_CHANNEL_DIM1, - this.alpha, cudnnContext.deltaTensorDesc, zData, cudnnContext.deltaTensorDesc, epsData, - cudnnContext.srcTensorDesc, srcData, this.beta, cudnnContext.dstTensorDesc, dstData)); - - allocator.getFlowController().registerActionAllWrite(context, input, epsilon, activations, nextEpsilon); - - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - - return new Pair<>(retGradient, nextEpsilon); - } - - - @Override - public INDArray activate(INDArray input, boolean training, double k, double n, double alpha, double beta, LayerWorkspaceMgr workspaceMgr) { - val miniBatch = (int) input.size(0); - val inDepth = (int) input.size(1); - val inH = (int) input.size(2); - val inW = (int) input.size(3); - - if(!Shape.hasDefaultStridesForShape(input)){ - input = input.dup('c'); - } - - val srcStride = ArrayUtil.toInts(input.stride()); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, miniBatch, inDepth, inH, inW, - srcStride[0], srcStride[1], srcStride[2], srcStride[3])); - - activations = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.dataType(), new long[] {miniBatch, inDepth, inH, inW}, 'c'); - - val dstStride = ArrayUtil.toInts(activations.stride()); - checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, miniBatch, inDepth, inH, inW, - dstStride[0], dstStride[1], dstStride[2], dstStride[3])); - checkCudnn(cudnnSetLRNDescriptor(cudnnContext.lrnDesc, (int) n, alpha, beta, k)); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareActionAllWrite(input, activations); - Pointer srcData = allocator.getPointer(input, context); - Pointer dstData = allocator.getPointer(activations, context); - - if (Nd4j.getExecutioner() instanceof GridExecutioner) - ((GridExecutioner) Nd4j.getExecutioner()).flushQueue(); - - checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream()))); - checkCudnn(cudnnLRNCrossChannelForward(cudnnContext, cudnnContext.lrnDesc, CUDNN_LRN_CROSS_CHANNEL_DIM1, - this.alpha, cudnnContext.srcTensorDesc, srcData, this.beta, cudnnContext.dstTensorDesc, - dstData)); - - allocator.getFlowController().registerActionAllWrite(context, input, activations); - - if (CudaEnvironment.getInstance().getConfiguration().isDebug()) - context.syncOldStream(); - - return activations; - } - - @Override - public Map helperMemoryUse() { - //No persistent memory use other than the structs (which are small) - return Collections.emptyMap(); - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/recurrent/CudnnLSTMHelper.java b/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/recurrent/CudnnLSTMHelper.java deleted file mode 100644 index 120078d07..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/main/java/org/deeplearning4j/cuda/recurrent/CudnnLSTMHelper.java +++ /dev/null @@ -1,659 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.recurrent; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import com.jakewharton.byteunits.BinaryByteUnit; -import org.bytedeco.javacpp.Pointer; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.gradient.DefaultGradient; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.cuda.BaseCudnnHelper; -import org.deeplearning4j.nn.layers.recurrent.FwdPassReturn; -import org.deeplearning4j.nn.layers.recurrent.LSTMHelper; -import org.nd4j.jita.allocator.Allocator; -import org.nd4j.jita.allocator.impl.AtomicAllocator; -import org.nd4j.linalg.activations.IActivation; -import org.nd4j.linalg.activations.impl.ActivationSigmoid; -import org.nd4j.linalg.activations.impl.ActivationTanH; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.shape.Shape; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.jcublas.context.CudaContext; -import org.nd4j.common.primitives.Pair; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.deeplearning4j.nn.workspace.ArrayType; - -import java.util.HashMap; -import java.util.Map; - -import org.bytedeco.cuda.cudart.*; -import org.bytedeco.cuda.cudnn.*; -import static org.bytedeco.cuda.global.cudart.*; -import static org.bytedeco.cuda.global.cudnn.*; - -/** - * cuDNN-based helper for the recurrent LSTM layer (no peephole connections). - * - * @author saudet - */ -@Slf4j -public class CudnnLSTMHelper extends BaseCudnnHelper implements LSTMHelper { - - public CudnnLSTMHelper(DataType dataType) { - super(dataType); - } - - private static class CudnnLSTMContext extends CudnnContext { - - private static class Deallocator extends CudnnLSTMContext implements Pointer.Deallocator { - Deallocator(CudnnLSTMContext c) { - super(c); - } - - @Override - public void deallocate() { - destroyHandles(); - } - } - - private cudnnTensorStruct hxDesc = new cudnnTensorStruct(), cxDesc = new cudnnTensorStruct(); - private cudnnTensorStruct hyDesc = new cudnnTensorStruct(), cyDesc = new cudnnTensorStruct(); - private cudnnTensorStruct dhxDesc = new cudnnTensorStruct(), dcxDesc = new cudnnTensorStruct(); - private cudnnTensorStruct dhyDesc = new cudnnTensorStruct(), dcyDesc = new cudnnTensorStruct(); - - private cudnnFilterStruct wDesc = new cudnnFilterStruct(), dwDesc = new cudnnFilterStruct(); - private cudnnFilterStruct linLayerMatDesc = new cudnnFilterStruct(), linLayerBiasDesc = new cudnnFilterStruct(); - - private cudnnRNNStruct rnnDesc = new cudnnRNNStruct(); - private cudnnDropoutStruct dropoutDesc = new cudnnDropoutStruct(); - private cudnnActivationStruct activationDesc = new cudnnActivationStruct(); - - public CudnnLSTMContext() { - createHandles(); - deallocator(new Deallocator(this)); - } - - public CudnnLSTMContext(CudnnLSTMContext c) { - super(c); - hxDesc = new cudnnTensorStruct(c.hxDesc); - cxDesc = new cudnnTensorStruct(c.cxDesc); - hyDesc = new cudnnTensorStruct(c.hyDesc); - cyDesc = new cudnnTensorStruct(c.cyDesc); - dhxDesc = new cudnnTensorStruct(c.dhxDesc); - dcxDesc = new cudnnTensorStruct(c.dcxDesc); - dhyDesc = new cudnnTensorStruct(c.dhyDesc); - dcyDesc = new cudnnTensorStruct(c.dcyDesc); - - wDesc = new cudnnFilterStruct(c.wDesc); - dwDesc = new cudnnFilterStruct(c.dwDesc); - linLayerMatDesc = new cudnnFilterStruct(c.linLayerMatDesc); - linLayerBiasDesc = new cudnnFilterStruct(c.linLayerBiasDesc); - - rnnDesc = new cudnnRNNStruct(c.rnnDesc); - dropoutDesc = new cudnnDropoutStruct(c.dropoutDesc); - activationDesc = new cudnnActivationStruct(c.activationDesc); - } - - @Override - protected void createHandles() { - super.createHandles(); - - checkCudnn(cudnnCreateTensorDescriptor(hxDesc)); - checkCudnn(cudnnCreateTensorDescriptor(cxDesc)); - checkCudnn(cudnnCreateTensorDescriptor(hyDesc)); - checkCudnn(cudnnCreateTensorDescriptor(cyDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dhxDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dcxDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dhyDesc)); - checkCudnn(cudnnCreateTensorDescriptor(dcyDesc)); - - checkCudnn(cudnnCreateFilterDescriptor(wDesc)); - checkCudnn(cudnnCreateFilterDescriptor(dwDesc)); - checkCudnn(cudnnCreateFilterDescriptor(linLayerMatDesc)); - checkCudnn(cudnnCreateFilterDescriptor(linLayerBiasDesc)); - - checkCudnn(cudnnCreateRNNDescriptor(rnnDesc)); - checkCudnn(cudnnCreateDropoutDescriptor(dropoutDesc)); - checkCudnn(cudnnCreateActivationDescriptor(activationDesc)); - } - - @Override - protected void destroyHandles() { - checkCudnn(cudnnDestroyActivationDescriptor(activationDesc)); - checkCudnn(cudnnDestroyDropoutDescriptor(dropoutDesc)); - checkCudnn(cudnnDestroyRNNDescriptor(rnnDesc)); - - checkCudnn(cudnnDestroyFilterDescriptor(wDesc)); - checkCudnn(cudnnDestroyFilterDescriptor(dwDesc)); - checkCudnn(cudnnDestroyFilterDescriptor(linLayerMatDesc)); - checkCudnn(cudnnDestroyFilterDescriptor(linLayerBiasDesc)); - - checkCudnn(cudnnDestroyTensorDescriptor(hxDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(cxDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(hyDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(cyDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dhxDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dcxDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dhyDesc)); - checkCudnn(cudnnDestroyTensorDescriptor(dcyDesc)); - - super.destroyHandles(); - } - } - - // These constants might eventually become variable parameters... - protected static final int NUM_LAYERS = 1; - protected static final float DROPOUT = 0; - protected static final boolean BIDIRECTIONAL = false; - protected static final int RNN_MODE = CUDNN_LSTM; - protected static final int NUM_LINEAR_LAYERS = 8; // CUDNN_LSTM - - private CudnnLSTMContext cudnnContext = new CudnnLSTMContext(); - private TensorArray xDesc = new TensorArray(); - private TensorArray yDesc = new TensorArray(); - private TensorArray dxDesc = new TensorArray(); - private TensorArray dyDesc = new TensorArray(); - private DataCache stateSpace = new DataCache(); - private DataCache reserveSpace = new DataCache(); - private DataCache weightsSpace = new DataCache(); - - private boolean initializedDropoutDescriptor = false; - - private static INDArray toCOrder(INDArray arr) { - if (arr.isView() || arr.ordering() != 'c' || !Shape.strideDescendingCAscendingF(arr)) { - arr = arr.dup('c'); - } - return arr; - } - - @Override - public boolean checkSupported(IActivation gateActivationFn, IActivation activationFn, - boolean hasPeepholeConnections) { - boolean supported = checkSupported(); - if (!(gateActivationFn instanceof ActivationSigmoid)) { - supported = false; - log.warn("Not supported: Gate activation functions != ActivationSigmoid"); - } - if (!(activationFn instanceof ActivationTanH)) { - supported = false; - log.warn("Not supported: Layer activation functions != ActivationTanH"); - } - if (hasPeepholeConnections) { - supported = false; - log.warn("Not supported: LSTM layers with peephole connections"); - } - return supported; - } - - @Override - public Pair backpropGradient(final NeuralNetConfiguration conf, - final IActivation gateActivationFn, final INDArray input, final INDArray recurrentWeights, //Shape: [hiddenLayerSize,4*hiddenLayerSize+3]; order: [wI,wF,wO,wG,wFF,wOO,wGG] - final INDArray inputWeights, //Shape: [n^(L-1),4*hiddenLayerSize]; order: [wi,wf,wo,wg] - final INDArray epsilon, final boolean truncatedBPTT, final int tbpttBackwardLength, - final FwdPassReturn fwdPass, final boolean forwards, final String inputWeightKey, - final String recurrentWeightKey, final String biasWeightKey, - final Map gradientViews, INDArray maskArray, //Input mask: should only be used with bidirectional RNNs + variable length - final boolean hasPeepholeConnections, //True for GravesLSTM, false for LSTM - final LayerWorkspaceMgr workspaceMgr) { - - //Expect errors to have shape: [miniBatchSize,n^(L+1),timeSeriesLength] - val hiddenLayerSize = recurrentWeights.size(0); //i.e., n^L - val prevLayerSize = inputWeights.size(0); //n^(L-1) - val inputLayerSize = input.size(1); - val miniBatchSize = epsilon.size(0); - boolean is2dInput = epsilon.rank() < 3; //Edge case: T=1 may have shape [miniBatchSize,n^(L+1)], equiv. to [miniBatchSize,n^(L+1),1] - long timeSeriesLength = (is2dInput ? 1 : epsilon.size(2)); - - INDArray x = toCOrder(input.permute(2, 0, 1)); - INDArray dy = toCOrder(epsilon.permute(2, 0, 1)); - INDArray dx = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, inputWeights.dataType(), new long[] {timeSeriesLength, miniBatchSize, prevLayerSize}, 'c'); - - INDArray iwGradientsOut = gradientViews.get(inputWeightKey); - INDArray rwGradientsOut = gradientViews.get(recurrentWeightKey); //Order: {I,F,O,G} - INDArray bGradientsOut = gradientViews.get(biasWeightKey); - - INDArray outputActivations = toCOrder(fwdPass.fwdPassOutput.permute(2, 0, 1)); - INDArray prevStepMemCellState = toCOrder(fwdPass.prevMemCell); - INDArray prevStepActivations = toCOrder(fwdPass.prevAct); - - Nd4j.getExecutioner().commit(); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareActionAllWrite(x, dy, dx, outputActivations, - prevStepMemCellState, prevStepActivations, iwGradientsOut, rwGradientsOut, bGradientsOut); - Pointer xData = allocator.getPointer(x, context); - Pointer dyData = allocator.getPointer(dy, context); - Pointer dxData = allocator.getPointer(dx, context); - Pointer outputActivationsData = allocator.getPointer(outputActivations, context); - Pointer prevMemCellStateData = allocator.getPointer(prevStepMemCellState, context); - Pointer prevStepActivationsData = allocator.getPointer(prevStepActivations, context); - Pointer iwGradientsOutData = allocator.getPointer(iwGradientsOut, context); - Pointer rwGradientsOutData = allocator.getPointer(rwGradientsOut, context); - Pointer bGradientsOutData = allocator.getPointer(bGradientsOut, context); - - CUstream_st stream = new CUstream_st(context.getCublasStream()); - checkCudnn(cudnnSetStream(cudnnContext, stream)); - - if (truncatedBPTT) { - val endIdx = Math.max(0, timeSeriesLength - tbpttBackwardLength) * miniBatchSize * hiddenLayerSize; - xData.position(endIdx * dataTypeSize); - dyData.position(endIdx * (BIDIRECTIONAL ? 2 : 1) * dataTypeSize); - outputActivationsData.position(endIdx * (BIDIRECTIONAL ? 2 : 1) * dataTypeSize); - timeSeriesLength = (int) Math.min(timeSeriesLength, tbpttBackwardLength); - } - - cudnnTensorStruct xDesc0 = xDesc.get(cudnnTensorStruct.class, 0); - - DataCache workSpace = workspaceMgr.getHelperWorkspace(LayerWorkspaceMgr.CUDNN_WORKSPACE_KEY); - checkCudnn(cudnnRNNBackwardData(cudnnContext, cudnnContext.rnnDesc, (int) timeSeriesLength, yDesc, - outputActivationsData, dyDesc, dyData, cudnnContext.dhyDesc, null, cudnnContext.dcyDesc, null, - cudnnContext.wDesc, weightsSpace, cudnnContext.hxDesc, prevStepActivationsData, //hx: initial hidden state of RNN - cudnnContext.cxDesc, prevMemCellStateData, //cx: initial cell state of RNN - dxDesc, dxData, //dx: gradient at input of each time step - cudnnContext.dhxDesc, null, //dhx: gradient at initial hidden state of RNN - cudnnContext.dcxDesc, null, //dcx: Gradient at initial cell state - workSpace, workSpace.limit(), reserveSpace, reserveSpace.limit())); - - // cudnnRNNBackwardWeights adds to the data in dW. - checkCuda(cudaMemsetAsync(weightsSpace, 0, weightsSpace.limit(), stream)); - - checkCudnn(cudnnRNNBackwardWeights(cudnnContext, cudnnContext.rnnDesc, (int) timeSeriesLength, xDesc, xData, //Input data - cudnnContext.hxDesc, prevStepActivationsData, //Initial hidden state - yDesc, outputActivationsData, //Output data - workSpace, workSpace.limit(), cudnnContext.dwDesc, weightsSpace, reserveSpace, - reserveSpace.limit())); - - int[] dataType = new int[1]; - int[] format = new int[1]; - int[] nbDims = new int[1]; - int[] filterDimA = new int[3]; - Pointer linLayerMat = new Pointer(); - Pointer linLayerBias = new Pointer(); - - for (int layer = 0; layer < NUM_LAYERS * (BIDIRECTIONAL ? 2 : 1); layer++) { - for (int linLayerID = 0; linLayerID < NUM_LINEAR_LAYERS; linLayerID++) { - checkCudnn(cudnnGetRNNLinLayerMatrixParams(cudnnContext, cudnnContext.rnnDesc, layer, xDesc0, - cudnnContext.wDesc, weightsSpace, linLayerID, cudnnContext.linLayerMatDesc, - linLayerMat)); - - checkCudnn(cudnnGetFilterNdDescriptor(cudnnContext.linLayerMatDesc, 3, dataType, format, nbDims, - filterDimA)); - - checkCudnn(cudnnGetRNNLinLayerBiasParams(cudnnContext, cudnnContext.rnnDesc, layer, xDesc0, - cudnnContext.wDesc, weightsSpace, linLayerID, cudnnContext.linLayerBiasDesc, - linLayerBias)); - - checkCudnn(cudnnGetFilterNdDescriptor(cudnnContext.linLayerBiasDesc, 3, dataType, format, nbDims, - filterDimA)); - - // our data is in "new, forget, output, and input gates" order (aka IFOG), each kind of weight packed together - int position = 0; - long size = 0; - Pointer data = null; - switch (linLayerID) { - case 0: - data = iwGradientsOutData; - position = 3; - size = inputLayerSize; - break; // input gate - case 1: - data = iwGradientsOutData; - position = 1; - size = inputLayerSize; - break; // forget gate - case 2: - data = iwGradientsOutData; - position = 0; - size = inputLayerSize; - break; // new gate (input modulation gate) - case 3: - data = iwGradientsOutData; - position = 2; - size = inputLayerSize; - break; // output gate - case 4: - data = rwGradientsOutData; - position = 3; - size = hiddenLayerSize; - break; // input gate - case 5: - data = rwGradientsOutData; - position = 1; - size = hiddenLayerSize; - break; // forget gate - case 6: - data = rwGradientsOutData; - position = 0; - size = hiddenLayerSize; - break; // new gate (input modulation gate) - case 7: - data = rwGradientsOutData; - position = 2; - size = hiddenLayerSize; - break; // output gate - default: - throw new RuntimeException(); - } - checkCuda(cudaMemcpyAsync(data.position(position * size * hiddenLayerSize * dataTypeSize), linLayerMat, - size * hiddenLayerSize * dataTypeSize, cudaMemcpyDeviceToDevice, stream)); - if (linLayerID < 4) { - checkCuda(cudaMemcpyAsync(bGradientsOutData.position(position * hiddenLayerSize * dataTypeSize), - linLayerBias, hiddenLayerSize * dataTypeSize, cudaMemcpyDeviceToDevice, stream)); - } - } - } - - allocator.getFlowController().registerActionAllWrite(context, x, dy, dx, outputActivations, - prevStepMemCellState, prevStepActivations, iwGradientsOut, rwGradientsOut, bGradientsOut); - - Gradient retGradient = new DefaultGradient(); - retGradient.gradientForVariable().put(inputWeightKey, iwGradientsOut); - retGradient.gradientForVariable().put(recurrentWeightKey, rwGradientsOut); - retGradient.gradientForVariable().put(biasWeightKey, bGradientsOut); - - INDArray epsilonNext = dx.permute(1, 2, 0); //i.e., what would be W^L*(delta^L)^T. Shape: [m,n^(L-1),T] - - return new Pair<>(retGradient, epsilonNext); - } - - @Override - public FwdPassReturn activate(final Layer layer, final NeuralNetConfiguration conf, - final IActivation gateActivationFn, //Activation function for the gates - sigmoid or hard sigmoid (must be found in range 0 to 1) - INDArray input, final INDArray recurrentWeights, //Shape: [hiddenLayerSize,4*hiddenLayerSize+3]; order: [wI,wF,wO,wG,wFF,wOO,wGG] - final INDArray inputWeights, //Shape: [n^(L-1),4*hiddenLayerSize]; order: [wi,wf,wo,wg] - final INDArray biases, //Shape: [4,hiddenLayerSize]; order: [bi,bf,bo,bg]^T - final boolean training, final INDArray prevOutputActivations, final INDArray prevMemCellState, - boolean forBackprop, boolean forwards, final String inputWeightKey, INDArray maskArray, //Input mask: should only be used with bidirectional RNNs + variable length - final boolean hasPeepholeConnections, //True for GravesLSTM, false for LSTM - final LayerWorkspaceMgr workspaceMgr) { - - boolean is2dInput = input.rank() < 3; //Edge case of T=1, may have shape [m,nIn], equiv. to [m,nIn,1] - val timeSeriesLength = (is2dInput ? 1 : input.size(2)); - val hiddenLayerSize = recurrentWeights.size(0); - val miniBatchSize = input.size(0); - val inputLayerSize = input.size(1); - - INDArray x = toCOrder(input.permute(2, 0, 1)); - INDArray linInputWeights = inputWeights; - INDArray linRecurrentWeights = recurrentWeights; - INDArray linBiases = biases; - - INDArray prevAct = toCOrder(prevOutputActivations); - INDArray prevMemCell = toCOrder(prevMemCellState); - - INDArray outputActivations = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, - inputWeights.dataType(), new long[] {timeSeriesLength, miniBatchSize, hiddenLayerSize * (BIDIRECTIONAL ? 2 : 1)}, 'c'); - INDArray finalMemCellState = Nd4j.createUninitialized( inputWeights.dataType(), - new long[] {/*numLayers * (bidirectional ? 2 : 1),*/ miniBatchSize, hiddenLayerSize}, 'c'); - INDArray finalStepActivations = Nd4j.createUninitialized( inputWeights.dataType(), - new long[] {/*numLayers * (bidirectional ? 2 : 1),*/ miniBatchSize, hiddenLayerSize}, 'c'); - - FwdPassReturn toReturn = new FwdPassReturn(); - toReturn.prevAct = prevAct; - toReturn.prevMemCell = prevMemCell; - - Nd4j.getExecutioner().commit(); - - - - if (timeSeriesLength > xDesc.capacity()) { - xDesc.deallocate(); - xDesc = new TensorArray(timeSeriesLength); - } - if (timeSeriesLength > yDesc.capacity()) { - yDesc.deallocate(); - yDesc = new TensorArray(timeSeriesLength); - } - if (timeSeriesLength > dxDesc.capacity()) { - dxDesc.deallocate(); - dxDesc = new TensorArray(timeSeriesLength); - } - if (timeSeriesLength > dyDesc.capacity()) { - dyDesc.deallocate(); - dyDesc = new TensorArray(timeSeriesLength); - } - - for (int i = 0; i < timeSeriesLength; i++) { - int[] dimA = {(int) miniBatchSize, (int) inputLayerSize, 1}; - int[] strideA = {(int) dimA[2] * dimA[1], dimA[2], 1}; - - checkCudnn(cudnnSetTensorNdDescriptor(xDesc.get(cudnnTensorStruct.class, i), dataType, 3, dimA, strideA)); - checkCudnn(cudnnSetTensorNdDescriptor(dxDesc.get(cudnnTensorStruct.class, i), dataType, 3, dimA, strideA)); - - int[] dimB = {(int) miniBatchSize, (int) hiddenLayerSize * (BIDIRECTIONAL ? 2 : 1), 1}; - int[] strideB = {dimB[2] * dimB[1], dimB[2], 1}; - - checkCudnn(cudnnSetTensorNdDescriptor(yDesc.get(cudnnTensorStruct.class, i), dataType, 3, dimB, strideB)); - checkCudnn(cudnnSetTensorNdDescriptor(dyDesc.get(cudnnTensorStruct.class, i), dataType, 3, dimB, strideB)); - } - - int[] dimC = {NUM_LAYERS * (BIDIRECTIONAL ? 2 : 1), (int) miniBatchSize, (int) hiddenLayerSize}; - int[] strideC = {dimC[2] * dimC[1], dimC[2], 1}; - - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.hxDesc, dataType, 3, dimC, strideC)); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.cxDesc, dataType, 3, dimC, strideC)); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.hyDesc, dataType, 3, dimC, strideC)); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.cyDesc, dataType, 3, dimC, strideC)); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.dhxDesc, dataType, 3, dimC, strideC)); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.dcxDesc, dataType, 3, dimC, strideC)); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.dhyDesc, dataType, 3, dimC, strideC)); - checkCudnn(cudnnSetTensorNdDescriptor(cudnnContext.dcyDesc, dataType, 3, dimC, strideC)); - - checkCudnn(cudnnDropoutGetStatesSize(cudnnContext, sizeInBytes)); - long stateSize = sizeInBytes.get(0); - if (stateSize > stateSpace.capacity()) { - stateSpace.deallocate(); - stateSpace = new DataCache(stateSize); - } - stateSpace.limit(stateSize); - - if(!initializedDropoutDescriptor) { - checkCudnn(cudnnSetDropoutDescriptor(cudnnContext.dropoutDesc, cudnnContext, DROPOUT, stateSpace, stateSize, - Nd4j.getRandom().getSeed())); - } - - checkCudnn(cudnnSetRNNDescriptor_v6(cudnnContext, cudnnContext.rnnDesc, (int) hiddenLayerSize, NUM_LAYERS, cudnnContext.dropoutDesc, - CUDNN_LINEAR_INPUT, BIDIRECTIONAL ? CUDNN_BIDIRECTIONAL : CUDNN_UNIDIRECTIONAL, RNN_MODE, - CUDNN_RNN_ALGO_STANDARD, dataType)); - - cudnnTensorStruct xDesc0 = xDesc.get(cudnnTensorStruct.class, 0); - checkCudnn(cudnnGetRNNParamsSize(cudnnContext, cudnnContext.rnnDesc, xDesc0, sizeInBytes, dataType)); - long weightsSize = sizeInBytes.get(0); - if (weightsSize > weightsSpace.capacity()) { - weightsSpace.deallocate(); - weightsSpace = new DataCache(weightsSize); - } - weightsSpace.limit(weightsSize); - - int[] dimW = {(int) weightsSize / dataTypeSize, 1, 1}; - - checkCudnn(cudnnSetFilterNdDescriptor(cudnnContext.wDesc, dataType, CUDNN_TENSOR_NCHW, 3, dimW)); - checkCudnn(cudnnSetFilterNdDescriptor(cudnnContext.dwDesc, dataType, CUDNN_TENSOR_NCHW, 3, dimW)); - - checkCudnn(cudnnGetRNNWorkspaceSize(cudnnContext, cudnnContext.rnnDesc, (int) timeSeriesLength, xDesc, sizeInBytes)); - long workSize = sizeInBytes.get(0); - DataCache workSpace = workspaceMgr.getHelperWorkspace(LayerWorkspaceMgr.CUDNN_WORKSPACE_KEY); - if (workSpace == null || workSize > workSpace.capacity()) { - if(log.isTraceEnabled()){ - if(workSpace == null){ - log.trace("CudnnLSTMHelper activate: Allocating initial workspace of size {} ({})", workSize, - BinaryByteUnit.format(workSize, "#.00")); - } else { - log.trace("CudnnLSTMHelper activate: Deallocating workspace of size {} ({}), allocating new workspace of size {} ({})", - workSpace.capacity(), BinaryByteUnit.format(workSpace.capacity(), "#.00"), - workSize, BinaryByteUnit.format(workSize, "#.00")); - } - } - if(workSpace != null) - workSpace.deallocate(); - workSpace = new DataCache(workSize); - workspaceMgr.setHelperWorkspace(LayerWorkspaceMgr.CUDNN_WORKSPACE_KEY, workSpace); - } - workSpace.limit(workSize); - - checkCudnn(cudnnGetRNNTrainingReserveSize(cudnnContext, cudnnContext.rnnDesc, (int) timeSeriesLength, xDesc, - sizeInBytes)); - long reserveSize = sizeInBytes.get(0); - if (reserveSize > reserveSpace.capacity()) { - reserveSpace.deallocate(); - reserveSpace = new DataCache(reserveSize); - } - reserveSpace.limit(reserveSize); - - Allocator allocator = AtomicAllocator.getInstance(); - CudaContext context = allocator.getFlowController().prepareActionAllWrite(x, linInputWeights, - linRecurrentWeights, linBiases, prevAct, prevMemCell, outputActivations, finalMemCellState, - finalStepActivations); - Pointer xData = allocator.getPointer(x, context); - Pointer linInputWeightsData = allocator.getPointer(linInputWeights, context); - Pointer linRecurrentWeightsData = allocator.getPointer(linRecurrentWeights, context); - Pointer linBiasesData = allocator.getPointer(linBiases, context); - Pointer prevActData = allocator.getPointer(prevAct, context); - Pointer prevMemCellData = allocator.getPointer(prevMemCell, context); - Pointer outputActivationsData = allocator.getPointer(outputActivations, context); - Pointer finalMemCellStateData = allocator.getPointer(finalMemCellState, context); - Pointer finalTimeStepActivationsData = allocator.getPointer(finalStepActivations, context); - - CUstream_st stream = new CUstream_st(context.getCublasStream()); - checkCudnn(cudnnSetStream(cudnnContext, stream)); - - checkCuda(cudaMemsetAsync(weightsSpace, 0, weightsSpace.limit(), stream)); - - int[] dataType = new int[1]; - int[] format = new int[1]; - int[] nbDims = new int[1]; - int[] filterDimA = new int[3]; - Pointer linLayerMat = new Pointer(); - Pointer linLayerBias = new Pointer(); - - for (int layerNum = 0; layerNum < NUM_LAYERS * (BIDIRECTIONAL ? 2 : 1); layerNum++) { - for (int linLayerID = 0; linLayerID < NUM_LINEAR_LAYERS; linLayerID++) { - checkCudnn(cudnnGetRNNLinLayerMatrixParams(cudnnContext, cudnnContext.rnnDesc, layerNum, xDesc0, - cudnnContext.wDesc, weightsSpace, linLayerID, cudnnContext.linLayerMatDesc, - linLayerMat)); - - checkCudnn(cudnnGetFilterNdDescriptor(cudnnContext.linLayerMatDesc, 3, dataType, format, nbDims, - filterDimA)); - - checkCudnn(cudnnGetRNNLinLayerBiasParams(cudnnContext, cudnnContext.rnnDesc, layerNum, xDesc0, - cudnnContext.wDesc, weightsSpace, linLayerID, cudnnContext.linLayerBiasDesc, - linLayerBias)); - - checkCudnn(cudnnGetFilterNdDescriptor(cudnnContext.linLayerBiasDesc, 3, dataType, format, nbDims, - filterDimA)); - - // our data is in "new, forget, output, and input gates" order (aka IFOG), each kind of weight packed together - int position = 0; - long size = 0; - Pointer data = null; - switch (linLayerID) { - case 0: - data = linInputWeightsData; - position = 3; - size = inputLayerSize; - break; // input gate - case 1: - data = linInputWeightsData; - position = 1; - size = inputLayerSize; - break; // forget gate - case 2: - data = linInputWeightsData; - position = 0; - size = inputLayerSize; - break; // new gate - case 3: - data = linInputWeightsData; - position = 2; - size = inputLayerSize; - break; // output gate - case 4: - data = linRecurrentWeightsData; - position = 3; - size = hiddenLayerSize; - break; // input gate - case 5: - data = linRecurrentWeightsData; - position = 1; - size = hiddenLayerSize; - break; // forget gate - case 6: - data = linRecurrentWeightsData; - position = 0; - size = hiddenLayerSize; - break; // new gate - case 7: - data = linRecurrentWeightsData; - position = 2; - size = hiddenLayerSize; - break; // output gate - default: - throw new RuntimeException(); - } - checkCuda(cudaMemcpyAsync(linLayerMat, data.position(position * size * hiddenLayerSize * dataTypeSize), - size * hiddenLayerSize * dataTypeSize, cudaMemcpyDeviceToDevice, stream)); - if (linLayerID < 4) { - checkCuda(cudaMemcpyAsync(linLayerBias, - linBiasesData.position(position * hiddenLayerSize * dataTypeSize), - hiddenLayerSize * dataTypeSize, cudaMemcpyDeviceToDevice, stream)); - } - } - } - - if (training) { - checkCudnn(cudnnRNNForwardTraining(cudnnContext, cudnnContext.rnnDesc, (int) timeSeriesLength, xDesc, xData, - cudnnContext.hxDesc, prevActData, cudnnContext.cxDesc, prevMemCellData, cudnnContext.wDesc, - weightsSpace, yDesc, outputActivationsData, cudnnContext.hyDesc, - finalTimeStepActivationsData, cudnnContext.cyDesc, finalMemCellStateData, workSpace, - workSpace.limit(), reserveSpace, reserveSpace.limit())); - } else { - checkCudnn(cudnnRNNForwardInference(cudnnContext, cudnnContext.rnnDesc, (int) timeSeriesLength, xDesc, xData, - cudnnContext.hxDesc, prevActData, cudnnContext.cxDesc, prevMemCellData, cudnnContext.wDesc, - weightsSpace, yDesc, outputActivationsData, cudnnContext.hyDesc, - finalTimeStepActivationsData, cudnnContext.cyDesc, finalMemCellStateData, workSpace, - workSpace.limit())); - } - - allocator.getFlowController().registerActionAllWrite(context, x, linInputWeights, linRecurrentWeights, - linBiases, prevAct, prevMemCell, outputActivations, finalMemCellState, finalStepActivations); - - toReturn.fwdPassOutput = outputActivations.permute(1, 2, 0); - toReturn.lastAct = finalStepActivations; - toReturn.lastMemCell = finalMemCellState; - toReturn.prevAct = prevAct; - toReturn.prevMemCell = prevMemCell; - - return toReturn; - } - - @Override - public Map helperMemoryUse() { - Map memUse = new HashMap<>(); - memUse.put("stateStace", stateSpace.capacity()); - memUse.put("reserveSpace", reserveSpace.capacity()); - memUse.put("weightsSpace", weightsSpace.capacity()); - return memUse; - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/CuDNNTestUtils.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/CuDNNTestUtils.java deleted file mode 100644 index 9674dee67..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/CuDNNTestUtils.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda; - -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.layers.convolution.ConvolutionLayer; -import org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer; -import org.deeplearning4j.nn.layers.normalization.BatchNormalization; -import org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization; -import org.deeplearning4j.nn.layers.recurrent.LSTM; -import org.nd4j.common.base.Preconditions; -import org.nd4j.common.tests.tags.NativeTag; - -import java.lang.reflect.Field; - -/** - * Test utility methods specific to CuDNN - * - * @author Alex Black - */ -@NativeTag -public class CuDNNTestUtils { - - private CuDNNTestUtils(){ } - - public static void removeHelpers(Layer[] layers) throws Exception { - for(Layer l : layers){ - - if(l instanceof ConvolutionLayer){ - Field f1 = ConvolutionLayer.class.getDeclaredField("helper"); - f1.setAccessible(true); - f1.set(l, null); - } else if(l instanceof SubsamplingLayer){ - Field f2 = SubsamplingLayer.class.getDeclaredField("helper"); - f2.setAccessible(true); - f2.set(l, null); - } else if(l instanceof BatchNormalization) { - Field f3 = BatchNormalization.class.getDeclaredField("helper"); - f3.setAccessible(true); - f3.set(l, null); - } else if(l instanceof LSTM){ - Field f4 = LSTM.class.getDeclaredField("helper"); - f4.setAccessible(true); - f4.set(l, null); - } else if(l instanceof LocalResponseNormalization){ - Field f5 = LocalResponseNormalization.class.getDeclaredField("helper"); - f5.setAccessible(true); - f5.set(l, null); - } - - - if(l.getHelper() != null){ - throw new IllegalStateException("Did not remove helper for layer: " + l.getClass().getSimpleName()); - } - } - } - - public static void assertHelpersPresent(Layer[] layers) throws Exception { - for(Layer l : layers){ - //Don't use instanceof here - there are sub conv subclasses - if(l.getClass() == ConvolutionLayer.class || l instanceof SubsamplingLayer || l instanceof BatchNormalization || l instanceof LSTM){ - Preconditions.checkNotNull(l.getHelper(), l.conf().getLayer().getLayerName()); - } - } - } - - public static void assertHelpersAbsent(Layer[] layers) throws Exception { - for(Layer l : layers){ - Preconditions.checkState(l.getHelper() == null, l.conf().getLayer().getLayerName()); - } - } - -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/TestDataTypes.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/TestDataTypes.java deleted file mode 100644 index 7bcb3aa37..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/TestDataTypes.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.BatchNormalization; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; - -@Slf4j -@NativeTag -public class TestDataTypes extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 180000L; - } - - @Test - public void testDataTypesSimple() throws Exception { - - Map outMapTrain = new HashMap<>(); - Map outMapTest = new HashMap<>(); - for(DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) { - Nd4j.setDefaultDataTypes(globalDtype, globalDtype); - for(DataType netDType : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) { - log.info("Starting test: global dtype = {}, net dtype = {}", globalDtype, netDType); - assertEquals(globalDtype, Nd4j.dataType()); - assertEquals(globalDtype, Nd4j.defaultFloatingPointType()); - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .updater(new Sgd(1e-2)) - .dataType(netDType) - .convolutionMode(ConvolutionMode.Same) - .activation(Activation.TANH) - .seed(12345) - .weightInit(WeightInit.XAVIER) - .list() - .layer(new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nOut(3).build()) - .layer(new SubsamplingLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).build()) - .layer(new BatchNormalization.Builder().eps(1e-3).build()) - .layer(new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).padding(0, 0).nOut(3).build()) - .layer(new OutputLayer.Builder().nOut(10).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()) - .setInputType(InputType.convolutionalFlat(28, 28, 1)) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - - Field f1 = org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.class.getDeclaredField("helper"); - f1.setAccessible(true); - - Field f2 = org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer.class.getDeclaredField("helper"); - f2.setAccessible(true); - - Field f3 = org.deeplearning4j.nn.layers.normalization.BatchNormalization.class.getDeclaredField("helper"); - f3.setAccessible(true); - - assertNotNull(f1.get(net.getLayer(0))); - assertNotNull(f2.get(net.getLayer(1))); - assertNotNull(f3.get(net.getLayer(2))); - assertNotNull(f1.get(net.getLayer(3))); - - DataSet ds = new MnistDataSetIterator(32, true, 12345).next(); - - //Simple sanity checks: - //System.out.println("STARTING FIT"); - net.fit(ds); - net.fit(ds); - - //System.out.println("STARTING OUTPUT"); - INDArray outTrain = net.output(ds.getFeatures(), false); - INDArray outTest = net.output(ds.getFeatures(), true); - - outMapTrain.put(netDType, outTrain.castTo(DataType.DOUBLE)); - outMapTest.put(netDType, outTest.castTo(DataType.DOUBLE)); - } - } - - Nd4j.setDataType(DataType.DOUBLE); - INDArray fp64Train = outMapTrain.get(DataType.DOUBLE); - INDArray fp32Train = outMapTrain.get(DataType.FLOAT).castTo(DataType.DOUBLE); - INDArray fp16Train = outMapTrain.get(DataType.HALF).castTo(DataType.DOUBLE); - - boolean eq64_32 = fp64Train.equalsWithEps(fp32Train, 1e-3); - boolean eq64_16 = fp64Train.equalsWithEps(fp16Train, 1e-2); - - if(!eq64_32){ - System.out.println("FP64/32"); - System.out.println("fp64Train:\n" + fp64Train); - System.out.println("fp32Train:\n" + fp32Train); - } - - if(!eq64_16){ - System.out.println("FP64/16"); - System.out.println("fp64Train:\n" + fp64Train); - System.out.println("fp16Train:\n" + fp16Train); - } - - assertTrue(eq64_32); - assertTrue(eq64_16); - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/TestUtils.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/TestUtils.java deleted file mode 100644 index c33ecc6a9..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/TestUtils.java +++ /dev/null @@ -1,318 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda; - -import org.apache.commons.compress.utils.IOUtils; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.layers.BaseLayer; -import org.deeplearning4j.nn.conf.layers.samediff.AbstractSameDiffLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.layers.convolution.ConvolutionLayer; -import org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer; -import org.deeplearning4j.nn.layers.normalization.BatchNormalization; -import org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization; -import org.deeplearning4j.nn.layers.recurrent.LSTM; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.util.ModelSerializer; -import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.random.impl.BernoulliDistribution; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.regularization.L1Regularization; -import org.nd4j.linalg.learning.regularization.L2Regularization; -import org.nd4j.linalg.learning.regularization.Regularization; -import org.nd4j.linalg.learning.regularization.WeightDecay; - -import java.io.*; -import java.lang.reflect.Field; -import java.util.List; -import java.util.Random; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -public class TestUtils { - - public static MultiLayerNetwork testModelSerialization(MultiLayerNetwork net){ - - MultiLayerNetwork restored; - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ModelSerializer.writeModel(net, baos, true); - byte[] bytes = baos.toByteArray(); - - ByteArrayInputStream bais = new ByteArrayInputStream(bytes); - restored = ModelSerializer.restoreMultiLayerNetwork(bais, true); - - assertEquals(net.getLayerWiseConfigurations(), restored.getLayerWiseConfigurations()); - assertEquals(net.params(), restored.params()); - } catch (IOException e){ - //Should never happen - throw new RuntimeException(e); - } - - //Also check the MultiLayerConfiguration is serializable (required by Spark etc) - MultiLayerConfiguration conf = net.getLayerWiseConfigurations(); - serializeDeserializeJava(conf); - - return restored; - } - - public static ComputationGraph testModelSerialization(ComputationGraph net){ - ComputationGraph restored; - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ModelSerializer.writeModel(net, baos, true); - byte[] bytes = baos.toByteArray(); - - ByteArrayInputStream bais = new ByteArrayInputStream(bytes); - restored = ModelSerializer.restoreComputationGraph(bais, true); - - assertEquals(net.getConfiguration(), restored.getConfiguration()); - assertEquals(net.params(), restored.params()); - } catch (IOException e){ - //Should never happen - throw new RuntimeException(e); - } - - //Also check the ComputationGraphConfiguration is serializable (required by Spark etc) - ComputationGraphConfiguration conf = net.getConfiguration(); - serializeDeserializeJava(conf); - - return restored; - } - - private static T serializeDeserializeJava(T object){ - byte[] bytes; - try(ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos)){ - oos.writeObject(object); - oos.close(); - bytes = baos.toByteArray(); - } catch (IOException e){ - //Should never happen - throw new RuntimeException(e); - } - - T out; - try(ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))){ - out = (T)ois.readObject(); - } catch (IOException | ClassNotFoundException e){ - throw new RuntimeException(e); - } - - assertEquals(object, out); - return out; - } - - public static INDArray randomOneHot(long examples, long nOut){ - return randomOneHot(examples, nOut, new Random(12345)); - } - - public static INDArray randomOneHot(DataType dataType, long examples, long nOut){ - return randomOneHot(dataType, examples, nOut, new Random(12345)); - } - - public static INDArray randomOneHot(long examples, long nOut, long rngSeed){ - return randomOneHot(examples, nOut, new Random(rngSeed)); - } - - public static INDArray randomOneHot(long examples, long nOut, Random rng) { - return randomOneHot(Nd4j.defaultFloatingPointType(), examples,nOut, rng); - } - - public static INDArray randomOneHot(DataType dataType, long examples, long nOut, Random rng){ - INDArray arr = Nd4j.create(dataType, examples, nOut); - for( int i=0; i l){ - for(Regularization r : l){ - if(r instanceof L1Regularization){ - return (L1Regularization) r; - } - } - return null; - } - - public static L2Regularization getL2Reg(BaseLayer baseLayer){ - return getL2Reg(baseLayer.getRegularization()); - } - - public static L2Regularization getL2Reg(List l){ - for(Regularization r : l){ - if(r instanceof L2Regularization){ - return (L2Regularization) r; - } - } - return null; - } - - public static WeightDecay getWeightDecayReg(BaseLayer bl){ - return getWeightDecayReg(bl.getRegularization()); - } - - public static WeightDecay getWeightDecayReg(List l){ - for(Regularization r : l){ - if(r instanceof WeightDecay){ - return (WeightDecay) r; - } - } - return null; - } - - public static double getL1(BaseLayer layer) { - List l = layer.getRegularization(); - return getL1(l); - } - - public static double getL1(List l){ - L1Regularization l1Reg = null; - for(Regularization reg : l){ - if(reg instanceof L1Regularization) - l1Reg = (L1Regularization) reg; - } - assertNotNull(l1Reg); - return l1Reg.getL1().valueAt(0,0); - } - - public static double getL2(BaseLayer layer) { - List l = layer.getRegularization(); - return getL2(l); - } - - public static double getL2(List l){ - L2Regularization l2Reg = null; - for(Regularization reg : l){ - if(reg instanceof L2Regularization) - l2Reg = (L2Regularization) reg; - } - assertNotNull(l2Reg); - return l2Reg.getL2().valueAt(0,0); - } - - public static double getL1(AbstractSameDiffLayer layer){ - return getL1(layer.getRegularization()); - } - - public static double getL2(AbstractSameDiffLayer layer){ - return getL2(layer.getRegularization()); - } - - public static double getWeightDecay(BaseLayer layer) { - return getWeightDecayReg(layer.getRegularization()).getCoeff().valueAt(0,0); - } - - public static void removeHelper(Layer layer) throws Exception { - removeHelpers(new Layer[]{layer}); - } - - public static void removeHelpers(Layer[] layers) throws Exception { - for(Layer l : layers){ - - if(l instanceof ConvolutionLayer){ - Field f1 = ConvolutionLayer.class.getDeclaredField("helper"); - f1.setAccessible(true); - f1.set(l, null); - } else if(l instanceof SubsamplingLayer){ - Field f2 = SubsamplingLayer.class.getDeclaredField("helper"); - f2.setAccessible(true); - f2.set(l, null); - } else if(l instanceof BatchNormalization) { - Field f3 = BatchNormalization.class.getDeclaredField("helper"); - f3.setAccessible(true); - f3.set(l, null); - } else if(l instanceof LSTM){ - Field f4 = LSTM.class.getDeclaredField("helper"); - f4.setAccessible(true); - f4.set(l, null); - } else if(l instanceof LocalResponseNormalization){ - Field f5 = LocalResponseNormalization.class.getDeclaredField("helper"); - f5.setAccessible(true); - f5.set(l, null); - } - - - if(l.getHelper() != null){ - throw new IllegalStateException("Did not remove helper for layer: " + l.getClass().getSimpleName()); - } - } - } - - public static void assertHelperPresent(Layer layer){ - - } - - public static void assertHelpersPresent(Layer[] layers) throws Exception { - for(Layer l : layers){ - //Don't use instanceof here - there are sub conv subclasses - if(l.getClass() == ConvolutionLayer.class || l instanceof SubsamplingLayer || l instanceof BatchNormalization || l instanceof LSTM){ - Preconditions.checkNotNull(l.getHelper(), l.conf().getLayer().getLayerName()); - } - } - } - - public static void assertHelpersAbsent(Layer[] layers) throws Exception { - for(Layer l : layers){ - Preconditions.checkState(l.getHelper() == null, l.conf().getLayer().getLayerName()); - } - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/ValidateCuDNN.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/ValidateCuDNN.java deleted file mode 100644 index 0d64b8c73..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/ValidateCuDNN.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.WorkspaceMode; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.layers.convolution.ConvolutionLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.cuda.util.CuDNNValidationUtil; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.linalg.activations.IActivation; -import org.nd4j.linalg.activations.impl.ActivationELU; -import org.nd4j.linalg.activations.impl.ActivationIdentity; -import org.nd4j.linalg.activations.impl.ActivationSoftmax; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.transforms.any.IsMax; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Nesterovs; -import org.nd4j.linalg.lossfunctions.impl.LossNegativeLogLikelihood; -import org.nd4j.linalg.schedule.ScheduleType; -import org.nd4j.linalg.schedule.StepSchedule; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -@Slf4j -@NativeTag -public class ValidateCuDNN extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 360000L; - } - - @Test - public void validateConvLayers() { - Nd4j.getRandom().setSeed(12345); - - int numClasses = 10; - //imageHeight,imageWidth,channels - int imageHeight = 64; - int imageWidth = 64; - int channels = 3; - IActivation activation = new ActivationIdentity(); - MultiLayerConfiguration multiLayerConfiguration = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .weightInit(WeightInit.XAVIER).seed(42) - .activation(new ActivationELU()) - .updater(new Nesterovs(1e-3, 0.9)) - .list( - new Convolution2D.Builder().nOut(16) - .kernelSize(4, 4).biasInit(0.0) - .stride(2, 2).build(), - new ActivationLayer.Builder().activation(activation).build(), - new Pooling2D.Builder() - .poolingType(SubsamplingLayer.PoolingType.MAX) - .kernelSize(3, 3).stride(2, 2) - .build(), - new Convolution2D.Builder().nOut(256) - .kernelSize(5, 5).padding(2, 2) - .biasInit(0.0) - .stride(1, 1).build(), - new ActivationLayer.Builder().activation(activation).build(), - new Pooling2D.Builder() - .poolingType(SubsamplingLayer.PoolingType.MAX) - .kernelSize(3, 3).stride(2, 2) - .build(), - new Convolution2D.Builder().nOut(16) - .kernelSize(3, 3).padding(1, 1) - .biasInit(0.0) - .stride(1, 1).build(), - new ActivationLayer.Builder().activation(activation).build(), - new Convolution2D.Builder().nOut(16) - .kernelSize(3, 3).padding(1, 1) - .stride(1, 1).build(), - new ActivationLayer.Builder().activation(activation).build(), - new Pooling2D.Builder() - .poolingType(SubsamplingLayer.PoolingType.MAX) - .kernelSize(3, 3).stride(2, 2) - .build(), - new DenseLayer.Builder() - .nOut(64) - .biasInit(0.0) - .build(), - new ActivationLayer.Builder().activation(activation).build(), - new OutputLayer.Builder().activation(new ActivationSoftmax()) - .lossFunction(new LossNegativeLogLikelihood()) - .nOut(numClasses) - .biasInit(0.0) - .build()) - .setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, channels)) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(multiLayerConfiguration); - net.init(); - - int[] fShape = new int[]{8, channels, imageHeight, imageWidth}; - int[] lShape = new int[]{8, numClasses}; - - List> classesToTest = new ArrayList<>(); - classesToTest.add(ConvolutionLayer.class); - classesToTest.add(org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer.class); - - validateLayers(net, classesToTest, true, fShape, lShape, CuDNNValidationUtil.MAX_REL_ERROR, CuDNNValidationUtil.MIN_ABS_ERROR); - } - - @Test - public void validateConvLayersSimpleBN() { - //Test ONLY BN - no other CuDNN functionality (i.e., DL4J impls for everything else) - Nd4j.getRandom().setSeed(12345); - - int minibatch = 8; - int numClasses = 10; - //imageHeight,imageWidth,channels - int imageHeight = 48; - int imageWidth = 48; - int channels = 3; - IActivation activation = new ActivationIdentity(); - MultiLayerConfiguration multiLayerConfiguration = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .weightInit(WeightInit.XAVIER).seed(42) - .activation(new ActivationELU()) - .updater(Nesterovs.builder() - .momentum(0.9) - .learningRateSchedule(new StepSchedule( - ScheduleType.EPOCH, - 1e-2, - 0.1, - 20)).build()).list( - new Convolution2D.Builder().nOut(96) - .kernelSize(11, 11).biasInit(0.0) - .stride(4, 4).build(), - new ActivationLayer.Builder().activation(activation).build(), - new BatchNormalization.Builder().build(), - new Pooling2D.Builder() - .poolingType(SubsamplingLayer.PoolingType.MAX) - .kernelSize(3, 3).stride(2, 2) - .build(), - new DenseLayer.Builder() - .nOut(128) - .biasInit(0.0) - .build(), - new ActivationLayer.Builder().activation(activation).build(), - new OutputLayer.Builder().activation(new ActivationSoftmax()) - .lossFunction(new LossNegativeLogLikelihood()) - .nOut(numClasses) - .biasInit(0.0) - .build()) - .setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, channels)) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(multiLayerConfiguration); - net.init(); - - int[] fShape = new int[]{minibatch, channels, imageHeight, imageWidth}; - int[] lShape = new int[]{minibatch, numClasses}; - - List> classesToTest = new ArrayList<>(); - classesToTest.add(org.deeplearning4j.nn.layers.normalization.BatchNormalization.class); - - validateLayers(net, classesToTest, false, fShape, lShape, CuDNNValidationUtil.MAX_REL_ERROR, CuDNNValidationUtil.MIN_ABS_ERROR); - } - - public void validateConvLayersLRN() { - //Test ONLY LRN - no other CuDNN functionality (i.e., DL4J impls for everything else) - Nd4j.getRandom().setSeed(12345); - - int minibatch = 8; - int numClasses = 10; - //imageHeight,imageWidth,channels - int imageHeight = 48; - int imageWidth = 48; - int channels = 3; - IActivation activation = new ActivationIdentity(); - MultiLayerConfiguration multiLayerConfiguration = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .weightInit(WeightInit.XAVIER).seed(42) - .activation(new ActivationELU()) - .updater(Nesterovs.builder() - .momentum(0.9) - .learningRateSchedule(new StepSchedule( - ScheduleType.EPOCH, - 1e-2, - 0.1, - 20)).build()).list( - new Convolution2D.Builder().nOut(96) - .kernelSize(11, 11).biasInit(0.0) - .stride(4, 4).build(), - new ActivationLayer.Builder().activation(activation).build(), - new LocalResponseNormalization.Builder() - .alpha(1e-3).beta(0.75).k(2) - .n(5).build(), - new Pooling2D.Builder() - .poolingType(SubsamplingLayer.PoolingType.MAX) - .kernelSize(3, 3).stride(2, 2) - .build(), - new Convolution2D.Builder().nOut(256) - .kernelSize(5, 5).padding(2, 2) - .biasInit(0.0) - .stride(1, 1).build(), - new ActivationLayer.Builder().activation(activation).build(), - new OutputLayer.Builder().activation(new ActivationSoftmax()) - .lossFunction(new LossNegativeLogLikelihood()) - .nOut(numClasses) - .biasInit(0.0) - .build()) - .setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, channels)) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(multiLayerConfiguration); - net.init(); - - int[] fShape = new int[]{minibatch, channels, imageHeight, imageWidth}; - int[] lShape = new int[]{minibatch, numClasses}; - - List> classesToTest = new ArrayList<>(); - classesToTest.add(org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization.class); - - validateLayers(net, classesToTest, false, fShape, lShape, 1e-2, 1e-2); - } - - public static void validateLayers(MultiLayerNetwork net, List> classesToTest, boolean testAllCudnnPresent, int[] fShape, int[] lShape, double maxRE, double minAbsErr) { - - for (WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) { - - net.getLayerWiseConfigurations().setTrainingWorkspaceMode(wsm); - net.getLayerWiseConfigurations().setInferenceWorkspaceMode(wsm); - - Nd4j.getRandom().setSeed(12345); - INDArray features = Nd4j.rand(fShape); - INDArray labels = Nd4j.rand(lShape); - labels = Nd4j.exec(new IsMax(labels, 1))[0].castTo(features.dataType()); - - List testCaseList = new ArrayList<>(); - - List dataSets = new ArrayList<>(); - for (int i = 0; i < 6; i++) { - INDArray f = Nd4j.rand(fShape); - INDArray l = Nd4j.rand(lShape); - l = Nd4j.exec(new IsMax(l, 1))[0].castTo(features.dataType()); - dataSets.add(new DataSet(f, l)); - } - DataSetIterator iter = new ExistingDataSetIterator(dataSets); - - for (Class c : classesToTest) { - String name = "WS=" + wsm + ", testCudnnFor=" + c.getSimpleName(); - testCaseList.add(CuDNNValidationUtil.TestCase.builder() - .testName(name) - .allowCudnnHelpersForClasses(Collections.>singletonList(c)) - .testForward(true) - .testScore(true) - .testBackward(true) - .testTraining(true) - .trainFirst(false) - .features(features) - .labels(labels) - .data(iter) - .maxRE(maxRE) - .minAbsErr(minAbsErr) - .build()); - } - - if(testAllCudnnPresent) { - testCaseList.add(CuDNNValidationUtil.TestCase.builder() - .testName("WS=" + wsm + ", ALL CLASSES") - .allowCudnnHelpersForClasses(classesToTest) - .testForward(true) - .testScore(true) - .testBackward(true) - .trainFirst(false) - .features(features) - .labels(labels) - .data(iter) - .maxRE(maxRE) - .minAbsErr(minAbsErr) - .build()); - } - - for (CuDNNValidationUtil.TestCase tc : testCaseList) { - log.info("Running test: " + tc.getTestName()); - CuDNNValidationUtil.validateMLN(net, tc); - } - } - } - -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/convolution/ConvDataFormatTests.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/convolution/ConvDataFormatTests.java deleted file mode 100644 index 38e22b988..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/convolution/ConvDataFormatTests.java +++ /dev/null @@ -1,1040 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.convolution; - -import lombok.*; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.cuda.CuDNNTestUtils; -import org.deeplearning4j.cuda.TestUtils; -import org.deeplearning4j.nn.api.MaskState; -import org.deeplearning4j.nn.conf.CNN2DFormat; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.InputPreProcessor; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.ComposableInputPreProcessor; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.workspace.ArrayType; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.BaseNd4jTestWithBackends; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.primitives.Pair; -import org.nd4j.linalg.factory.Nd4jBackend; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@NativeTag -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -public class ConvDataFormatTests extends BaseDL4JTest { - - - - public static Stream params() { - List args = new ArrayList<>(); - for(Nd4jBackend nd4jBackend : BaseNd4jTestWithBackends.BACKENDS) { - for(DataType dataType : new DataType[]{DataType.FLOAT, DataType.DOUBLE}) { - args.add(Arguments.of(dataType,nd4jBackend)); - } - } - - return args.stream(); - } - - @ParameterizedTest - @MethodSource("params") - public void testConv2d(DataType dataType, Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getConv2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getConv2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getConv2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getConv2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testSubsampling2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSubsampling2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getSubsampling2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getSubsampling2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getSubsampling2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testDepthwiseConv2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getDepthwiseConv2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getDepthwiseConv2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getDepthwiseConv2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getDepthwiseConv2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testSeparableConv2d(DataType dataType, Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSeparableConv2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getSeparableConv2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getSeparableConv2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getSeparableConv2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testDeconv2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getDeconv2DNet2dNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getDeconv2DNet2dNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getDeconv2DNet2dNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getDeconv2DNet2dNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testLRN(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getLrnLayer(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getLrnLayer(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getLrnLayer(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getLrnLayer(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testZeroPaddingLayer(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getZeroPaddingNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getZeroPaddingNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getZeroPaddingNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getZeroPaddingNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testCropping2DLayer(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getCropping2dNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getCropping2dNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getCropping2dNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getCropping2dNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testUpsampling2d(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getUpsamplingNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getUpsamplingNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getUpsamplingNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getUpsamplingNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testBatchNormNet(DataType dataType,Nd4jBackend backend) { - try { - for(boolean useLogStd : new boolean[]{true, false}) { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = (helpers ? "With helpers" : "No helpers") + " - " + (useLogStd ? "logstd" : "std"); - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getBatchNormNet(dataType,useLogStd, CNN2DFormat.NCHW, true)) - .net2(getBatchNormNet(dataType,useLogStd, CNN2DFormat.NCHW, false)) - .net3(getBatchNormNet(dataType,useLogStd, CNN2DFormat.NHWC, true)) - .net4(getBatchNormNet(dataType,useLogStd, CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testCnnLossLayer(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labelsNHWC = TestUtils.randomOneHot(dataType,2*6*6, 3); - labelsNHWC = labelsNHWC.reshape(2,6,6,3); - INDArray labelsNCHW = labelsNHWC.permute(0,3,1,2).dup(); - - - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getCnnLossNet(CNN2DFormat.NCHW, true, ConvolutionMode.Same)) - .net2(getCnnLossNet(CNN2DFormat.NCHW, false, ConvolutionMode.Same)) - .net3(getCnnLossNet(CNN2DFormat.NHWC, true, ConvolutionMode.Same)) - .net4(getCnnLossNet(CNN2DFormat.NHWC, false, ConvolutionMode.Same)) - .inNCHW(inNCHW) - .labelsNCHW(labelsNCHW) - .labelsNHWC(labelsNHWC) - .testLayerIdx(1) - .nhwcOutput(true) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testSpaceToDepthNet(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSpaceToDepthNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getSpaceToDepthNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getSpaceToDepthNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getSpaceToDepthNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testSpaceToBatchNet(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers" : "No helpers"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 16, 16); - INDArray labels = TestUtils.randomOneHot(8, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getSpaceToBatchNet(dataType,CNN2DFormat.NCHW, true)) - .net2(getSpaceToBatchNet(dataType,CNN2DFormat.NCHW, false)) - .net3(getSpaceToBatchNet(dataType,CNN2DFormat.NHWC, true)) - .net4(getSpaceToBatchNet(dataType,CNN2DFormat.NHWC, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testLocallyConnected(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same}) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + cm + ")" : "No helpers (" + cm + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getLocallyConnectedNet(dataType,CNN2DFormat.NCHW, true, cm)) - .net2(getLocallyConnectedNet(dataType,CNN2DFormat.NCHW, false, cm)) - .net3(getLocallyConnectedNet(dataType,CNN2DFormat.NHWC, true, cm)) - .net4(getLocallyConnectedNet(dataType,CNN2DFormat.NHWC, false, cm)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .helpers(helpers) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - @ParameterizedTest - @MethodSource("params") - public void testGlobalPooling(DataType dataType,Nd4jBackend backend) { - try { - for (boolean helpers : new boolean[]{false, true}) { - for (PoolingType pt : PoolingType.values()) { - Nd4j.getRandom().setSeed(12345); - Nd4j.getEnvironment().allowHelpers(helpers); - String msg = helpers ? "With helpers (" + pt + ")" : "No helpers (" + pt + ")"; - System.out.println(" --- " + msg + " ---"); - - INDArray inNCHW = Nd4j.rand(dataType, 2, 3, 12, 12); - INDArray labels = TestUtils.randomOneHot(2, 10); - - TestCase tc = TestCase.builder() - .msg(msg) - .net1(getGlobalPoolingNet(dataType,CNN2DFormat.NCHW, pt, true)) - .net2(getGlobalPoolingNet(dataType,CNN2DFormat.NCHW, pt, false)) - .net3(getGlobalPoolingNet(dataType,CNN2DFormat.NHWC, pt, true)) - .net4(getGlobalPoolingNet(dataType,CNN2DFormat.NHWC, pt, false)) - .inNCHW(inNCHW) - .labelsNCHW(labels) - .labelsNHWC(labels) - .testLayerIdx(1) - .build(); - - testHelper(tc); - } - } - } finally { - Nd4j.getEnvironment().allowHelpers(true); - } - } - - private MultiLayerNetwork getConv2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new ConvolutionLayer.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new ConvolutionLayer.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getSubsampling2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new SubsamplingLayer.Builder() - .kernelSize(2, 2) - .stride(1, 1) - .dataFormat(format) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new SubsamplingLayer.Builder() - .kernelSize(2, 2) - .stride(1, 1) - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getSeparableConv2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new SeparableConvolution2D.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new SeparableConvolution2D.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getDepthwiseConv2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new DepthwiseConvolution2D.Builder() - .depthMultiplier(2) - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new DepthwiseConvolution2D.Builder() - .depthMultiplier(2) - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getLrnLayer(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new LocalResponseNormalization.Builder() - .dataFormat(format) - .helperAllowFallback(false) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new LocalResponseNormalization.Builder() - .helperAllowFallback(false) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getZeroPaddingNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new ZeroPaddingLayer.Builder(2,2) - .dataFormat(format).build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new ZeroPaddingLayer.Builder(2,2).build(), - format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getCropping2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new Cropping2D.Builder(2,2) - .dataFormat(format).build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new Cropping2D.Builder(2,2) - .build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getUpsamplingNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new Upsampling2D.Builder(2) - .dataFormat(format).build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new Upsampling2D.Builder(2) - .build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getDeconv2DNet2dNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new Deconvolution2D.Builder().nOut(2) - .activation(Activation.TANH) - .kernelSize(2,2) - .stride(2,2) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new Deconvolution2D.Builder().nOut(2) - .activation(Activation.TANH) - .kernelSize(2,2) - .stride(2,2) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getBatchNormNet(DataType dataType,boolean logStdev, CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new BatchNormalization.Builder() - .useLogStd(logStdev) - .dataFormat(format) - .helperAllowFallback(false) - .nOut(3).build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new BatchNormalization.Builder() - .useLogStd(logStdev) - .helperAllowFallback(false) - .nOut(3).build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getSpaceToDepthNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new SpaceToDepthLayer.Builder() - .blocks(2) - .dataFormat(format) - .build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new SpaceToDepthLayer.Builder() - .blocks(2) - .build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getSpaceToBatchNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new SpaceToBatchLayer.Builder() - .blocks(2, 2) - .dataFormat(format) - .build(), format, ConvolutionMode.Same, InputType.convolutional(16, 16, 3, format)); - } else { - return getNetWithLayer(dataType,new SpaceToBatchLayer.Builder() - .blocks(2, 2) - .build(), format, ConvolutionMode.Same, InputType.convolutional(16, 16, 3, format)); - } - } - - private MultiLayerNetwork getLocallyConnectedNet(DataType dataType,CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new LocallyConnected2D.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .build(), format, cm, null); - } else { - return getNetWithLayer(dataType,new LocallyConnected2D.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .nOut(3) - .build(), format, cm, null); - } - } - - private MultiLayerNetwork getGlobalPoolingNet(DataType dataType,CNN2DFormat format, PoolingType pt, boolean setOnLayerAlso) { - if (setOnLayerAlso) { - return getNetWithLayer(dataType,new GlobalPoolingLayer.Builder(pt) - .poolingDimensions(format == CNN2DFormat.NCHW ? new int[]{2,3} : new int[]{1,2}) - .build(), format, ConvolutionMode.Same, null); - } else { - return getNetWithLayer(dataType,new GlobalPoolingLayer.Builder(pt) - .build(), format, ConvolutionMode.Same, null); - } - } - - private MultiLayerNetwork getCnnLossNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm){ - NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder() - .seed(12345) - .convolutionMode(cm) - .list() - .layer(new ConvolutionLayer.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build()); - if(setOnLayerAlso){ - builder.layer(new CnnLossLayer.Builder().format(format).activation(Activation.SOFTMAX).build()); - } else { - builder.layer(new CnnLossLayer.Builder().activation(Activation.SOFTMAX).build()); - } - - builder.setInputType(InputType.convolutional(12, 12, 3, format)); - - MultiLayerNetwork net = new MultiLayerNetwork(builder.build()); - net.init(); - return net; - } - - private MultiLayerNetwork getNetWithLayer(DataType dataType,Layer layer, CNN2DFormat format, ConvolutionMode cm, InputType inputType) { - NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder() - .dataType(dataType) - .seed(12345) - .convolutionMode(cm) - .list() - .layer(new ConvolutionLayer.Builder() - .kernelSize(3, 3) - .stride(2, 2) - .activation(Activation.TANH) - .dataFormat(format) - .nOut(3) - .helperAllowFallback(false) - .build()) - .layer(layer) - .layer(new OutputLayer.Builder().activation(Activation.SOFTMAX).nOut(10).build()) - .setInputType(inputType != null ? inputType : InputType.convolutional(12, 12, 3, format)); - - if(format == CNN2DFormat.NHWC && !(layer instanceof GlobalPoolingLayer)){ - //Add a preprocessor due to the differences in how NHWC and NCHW activations are flattened - //DL4J's flattening behaviour matches Keras (hence TF) for import compatibility - builder.inputPreProcessor(2, new ComposableInputPreProcessor(new NHWCToNCHWPreprocessor(), new CnnToFeedForwardPreProcessor())); - } - - MultiLayerNetwork net = new MultiLayerNetwork(builder.build()); - net.init(); - return net; - } - - @AllArgsConstructor - @Data - @NoArgsConstructor - @Builder - private static class TestCase { - private String msg; - private MultiLayerNetwork net1; - private MultiLayerNetwork net2; - private MultiLayerNetwork net3; - private MultiLayerNetwork net4; - private INDArray inNCHW; - private INDArray labelsNCHW; - private INDArray labelsNHWC; - private int testLayerIdx; - private boolean nhwcOutput; - private boolean helpers; - } - - public static void testHelper(TestCase tc) { - - if(!tc.helpers){ - try { - CuDNNTestUtils.removeHelpers(tc.net1.getLayers()); - CuDNNTestUtils.removeHelpers(tc.net2.getLayers()); - CuDNNTestUtils.removeHelpers(tc.net3.getLayers()); - CuDNNTestUtils.removeHelpers(tc.net4.getLayers()); - } catch (Throwable t){ - throw new RuntimeException(t); - } - } - - - tc.net2.params().assign(tc.net1.params()); - tc.net3.params().assign(tc.net1.params()); - tc.net4.params().assign(tc.net1.params()); - - //Test forward pass: - INDArray inNCHW = tc.inNCHW; - INDArray inNHWC = tc.inNCHW.permute(0, 2, 3, 1).dup(); - - INDArray l0_1 = tc.net1.feedForward(inNCHW).get(tc.testLayerIdx + 1); - INDArray l0_2 = tc.net2.feedForward(inNCHW).get(tc.testLayerIdx + 1); - INDArray l0_3 = tc.net3.feedForward(inNHWC).get(tc.testLayerIdx + 1); - INDArray l0_4 = tc.net4.feedForward(inNHWC).get(tc.testLayerIdx + 1); - - assertEquals(l0_1, l0_2, tc.msg); - if(l0_1.rank() == 4) { - assertEquals(l0_1, l0_3.permute(0, 3, 1, 2), tc.msg); - assertEquals(l0_1, l0_4.permute(0, 3, 1, 2), tc.msg); - } else { - assertEquals(l0_1, l0_3, tc.msg); - assertEquals(l0_1, l0_4, tc.msg); - } - - - INDArray out1 = tc.net1.output(inNCHW); - INDArray out2 = tc.net2.output(inNCHW); - INDArray out3 = tc.net3.output(inNHWC); - INDArray out4 = tc.net4.output(inNHWC); - - assertEquals(out1, out2, tc.msg); - if(!tc.nhwcOutput) { - assertEquals(out1, out3, tc.msg); - assertEquals(out1, out4, tc.msg); - } else { - assertEquals(out1, out3.permute(0,3,1,2), tc.msg); //NHWC to NCHW - assertEquals(out1, out4.permute(0,3,1,2), tc.msg); - } - - //Test backprop - Pair p1 = tc.net1.calculateGradients(inNCHW, tc.labelsNCHW, null, null); - Pair p2 = tc.net2.calculateGradients(inNCHW, tc.labelsNCHW, null, null); - Pair p3 = tc.net3.calculateGradients(inNHWC, tc.labelsNHWC, null, null); - Pair p4 = tc.net4.calculateGradients(inNHWC, tc.labelsNHWC, null, null); - - //Inpput gradients - assertEquals(p1.getSecond(), p2.getSecond(), tc.msg); - assertEquals(p1.getSecond(), p3.getSecond().permute(0,3,1,2), tc.msg); //Input gradients for NHWC input are also in NHWC format - assertEquals(p1.getSecond(), p4.getSecond().permute(0,3,1,2), tc.msg); - - List diff12 = differentGrads(p1.getFirst(), p2.getFirst()); - List diff13 = differentGrads(p1.getFirst(), p3.getFirst()); - List diff14 = differentGrads(p1.getFirst(), p4.getFirst()); - assertEquals(0, diff12.size(),tc.msg + " " + diff12); - assertEquals( 0, diff13.size(),tc.msg + " " + diff13); - assertEquals(0, diff14.size(),tc.msg + " " + diff14); - - assertEquals(p1.getFirst().gradientForVariable(), p2.getFirst().gradientForVariable(), tc.msg); - assertEquals(p1.getFirst().gradientForVariable(), p3.getFirst().gradientForVariable(), tc.msg); - assertEquals(p1.getFirst().gradientForVariable(), p4.getFirst().gradientForVariable(), tc.msg); - - tc.net1.fit(inNCHW, tc.labelsNCHW); - tc.net2.fit(inNCHW, tc.labelsNCHW); - tc.net3.fit(inNHWC, tc.labelsNHWC); - tc.net4.fit(inNHWC, tc.labelsNHWC); - - assertEquals(tc.net1.params(), tc.net2.params(), tc.msg); - assertEquals(tc.net1.params(), tc.net3.params(), tc.msg); - assertEquals(tc.net1.params(), tc.net4.params(), tc.msg); - - //Test serialization - MultiLayerNetwork net1a = TestUtils.testModelSerialization(tc.net1); - MultiLayerNetwork net2a = TestUtils.testModelSerialization(tc.net2); - MultiLayerNetwork net3a = TestUtils.testModelSerialization(tc.net3); - MultiLayerNetwork net4a = TestUtils.testModelSerialization(tc.net4); - - if(!tc.helpers){ - try { - CuDNNTestUtils.removeHelpers(net1a.getLayers()); - CuDNNTestUtils.removeHelpers(net2a.getLayers()); - CuDNNTestUtils.removeHelpers(net3a.getLayers()); - CuDNNTestUtils.removeHelpers(net4a.getLayers()); - } catch (Throwable t){ - throw new RuntimeException(t); - } - } - - out1 = tc.net1.output(inNCHW); - assertEquals(out1, net1a.output(inNCHW), tc.msg); - assertEquals(out1, net2a.output(inNCHW), tc.msg); - if(!tc.nhwcOutput) { - assertEquals(out1, net3a.output(inNHWC), tc.msg); - assertEquals(out1, net4a.output(inNHWC), tc.msg); - } else { - assertEquals(out1, net3a.output(inNHWC).permute(0,3,1,2), tc.msg); //NHWC to NCHW - assertEquals(out1, net4a.output(inNHWC).permute(0,3,1,2), tc.msg); - } - - } - - private static List differentGrads(Gradient g1, Gradient g2){ - List differs = new ArrayList<>(); - Map m1 = g1.gradientForVariable(); - Map m2 = g2.gradientForVariable(); - for(String s : m1.keySet()){ - INDArray a1 = m1.get(s); - INDArray a2 = m2.get(s); - if(!a1.equals(a2)){ - differs.add(s); - } - } - return differs; - } - - //Converts NHWC to NCHW activations - @EqualsAndHashCode - private static class NHWCToNCHWPreprocessor implements InputPreProcessor { - - @Override - public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) { - return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.permute(0,3,1,2)); - } - - @Override - public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) { - return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, output.permute(0,2,3,1)); - } - - @Override - public InputPreProcessor clone() { - return this; - } - - - @Override - public InputType getOutputType(InputType inputType) { - InputType.InputTypeConvolutional c = (InputType.InputTypeConvolutional) inputType; - return InputType.convolutional(c.getHeight(), c.getWidth(), c.getChannels(), CNN2DFormat.NCHW); - } - - @Override - public Pair feedForwardMaskArray(INDArray maskArray, MaskState currentMaskState, int minibatchSize) { - return null; - } - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/convolution/TestConvolution.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/convolution/TestConvolution.java deleted file mode 100644 index cf533e147..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/convolution/TestConvolution.java +++ /dev/null @@ -1,376 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.convolution; - -import org.apache.commons.io.FileUtils; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.cuda.CuDNNTestUtils; -import org.deeplearning4j.cuda.TestUtils; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.Distribution; -import org.deeplearning4j.nn.conf.distribution.GaussianDistribution; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.DenseLayer; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; - -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.resources.Resources; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.io.ClassPathResource; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.learning.config.Sgd; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.nd4j.common.primitives.Pair; - -import java.io.File; -import java.lang.reflect.Field; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Created by Alex on 15/11/2016. - */ -public class TestConvolution extends BaseDL4JTest { - - - @Override - public long getTimeoutMilliseconds() { - return 240000L; - } - - @Test - public void testSameModeActivationSizes() { - int inH = 3; - int inW = 4; - int inDepth = 3; - int minibatch = 5; - - int sH = 2; - int sW = 2; - int kH = 3; - int kW = 3; - - org.deeplearning4j.nn.conf.layers.Layer[] l = new org.deeplearning4j.nn.conf.layers.Layer[2]; - l[0] = new ConvolutionLayer.Builder().nOut(4).kernelSize(kH, kW).stride(sH, sW).build(); - l[1] = new SubsamplingLayer.Builder().kernelSize(kH, kW).stride(sH, sW).build(); - - for (int i = 0; i < l.length; i++) { - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().convolutionMode(ConvolutionMode.Same) - .list().layer(0, l[i]).layer(1, new OutputLayer.Builder().nOut(3).build()) - .setInputType(InputType.convolutional(inH, inW, inDepth)).build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - INDArray inData = Nd4j.create(minibatch, inDepth, inH, inW); - List activations = net.feedForward(inData); - INDArray actL0 = activations.get(1); - - int outH = (int) Math.ceil(inH / ((double) sH)); - int outW = (int) Math.ceil(inW / ((double) sW)); - - System.out.println(Arrays.toString(actL0.shape())); - assertArrayEquals(new long[]{minibatch, (i == 0 ? 4 : inDepth), outH, outW}, actL0.shape()); - } - } - - - @Test - public void testCompareCudnnStandardOutputsVsMode() throws Exception { - - ConvolutionMode[] cm = - new ConvolutionMode[]{ConvolutionMode.Strict, ConvolutionMode.Truncate, ConvolutionMode.Same}; - - for (ConvolutionMode c : cm) { - for (ConvolutionLayer.AlgoMode a : new ConvolutionLayer.AlgoMode[]{ConvolutionLayer.AlgoMode.NO_WORKSPACE, ConvolutionLayer.AlgoMode.PREFER_FASTEST}) { - for (boolean conv : new boolean[]{true, false}) { - String msg = c + " - " + a + " - " + (conv ? "conv" : "subsampling"); - System.out.println(msg); - - org.deeplearning4j.nn.conf.layers.Layer l; - if (conv) { - l = new ConvolutionLayer.Builder().nOut(4).kernelSize(4, 4).stride(2, 2).build(); - } else { - l = new SubsamplingLayer.Builder().kernelSize(4, 4).stride(2, 2).build(); - } - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .seed(12345) - .l2(0.0005).updater(new Sgd(0.01)).weightInit(WeightInit.XAVIER).convolutionMode(c).cudnnAlgoMode(a).list() - .layer(0, l) - .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) - .nOut(10).activation(Activation.SOFTMAX).build()) - .setInputType(InputType.convolutionalFlat(28, 28, 1)) //See note below - .build(); - if (conv) { - assertEquals(a, ((ConvolutionLayer) l).getCudnnAlgoMode()); - } - - Nd4j.getRandom().setSeed(12345); - MultiLayerNetwork net1 = new MultiLayerNetwork(conf); - net1.init(); - net1.initGradientsView(); - - Nd4j.getRandom().setSeed(12345); - MultiLayerNetwork net2 = new MultiLayerNetwork(conf); - net2.init(); - net2.initGradientsView(); - - Layer layerCudnn = net1.getLayer(0); - Layer layerStandard = net2.getLayer(0); - - Field f = layerStandard.getClass().getDeclaredField("helper"); - f.setAccessible(true); - f.set(layerStandard, null); - - if (f.get(layerCudnn) == null) - throw new RuntimeException(); - if (f.get(layerStandard) != null) - throw new RuntimeException(); - - - INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{1, 1, 20, 20}); //(20-4+0)/2 +1 = 9 - - INDArray outCudnn = layerCudnn.activate(in, false, LayerWorkspaceMgr.noWorkspaces()); - INDArray outStd = layerStandard.activate(in, false, LayerWorkspaceMgr.noWorkspaces()); - - assertEquals(outStd, outCudnn, msg); - - - //Check backprop: - INDArray epsilon = Nd4j.rand(DataType.DOUBLE, outStd.shape()); - Pair pCudnn = layerCudnn.backpropGradient(epsilon.dup(), LayerWorkspaceMgr.noWorkspaces()); - Pair pStd = layerStandard.backpropGradient(epsilon.dup(), LayerWorkspaceMgr.noWorkspaces()); - -// System.out.println(Arrays.toString(pStd.getSecond().data().asFloat())); -// System.out.println(Arrays.toString(pCudnn.getSecond().data().asFloat())); - - INDArray epsOutStd = pStd.getSecond(); - INDArray epsOutCudnn = pCudnn.getSecond(); - - assertTrue(epsOutStd.equalsWithEps(epsOutCudnn, 1e-4), msg); - - if (conv) { - INDArray gradStd = pStd.getFirst().gradient(); - INDArray gradCudnn = pCudnn.getFirst().gradient(); - - assertTrue(gradStd.equalsWithEps(gradCudnn, 1e-4), msg); - } - } - } - } - } - - - @Test - public void validateXceptionImport(@TempDir Path testDir) throws Exception { - File dir = testDir.toFile(); - File fSource = Resources.asFile("modelimport/keras/examples/xception/xception_tf_keras_2.h5"); - File fExtracted = new File(dir, "xception_tf_keras_2.h5" ); - FileUtils.copyFile(fSource, fExtracted); - - int inSize = 256; - ComputationGraph model = KerasModelImport.importKerasModelAndWeights( fExtracted.getAbsolutePath(), new int[]{inSize, inSize, 3}, false); - model = model.convertDataType(DataType.DOUBLE); - - INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{1, inSize, inSize, 3}); //Keras import model -> NHWC - - CuDNNTestUtils.assertHelpersPresent(model.getLayers()); - Map withCudnn = model.feedForward(in, false); - - CuDNNTestUtils.removeHelpers(model.getLayers()); - CuDNNTestUtils.assertHelpersAbsent(model.getLayers()); - Map noCudnn = model.feedForward(in, false); - - assertEquals(withCudnn.keySet(), noCudnn.keySet()); - - for(String s : withCudnn.keySet()) { - assertEquals(withCudnn.get(s), noCudnn.get(s), s); - } - } - - - @Test - public void testCudnnDilation(){ - //Sanity check on dilated conv execution - int[] k = new int[]{2,3,4,5}; - int[] d = new int[]{1,2,3,4}; - - for( int[] inputSize : new int[][]{{10,1,28,28}, {3,3,224,224}}) { - for (int i = 0; i < k.length; i++) { - for(ConvolutionMode cm : new ConvolutionMode[]{ConvolutionMode.Same, ConvolutionMode.Truncate}) { - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .convolutionMode(ConvolutionMode.Same) - .list() - .layer(new ConvolutionLayer.Builder().kernelSize(k[i], k[i]).dilation(d[i], d[i]).nOut(3).build()) - .layer(new SubsamplingLayer.Builder().kernelSize(k[i], k[i]).dilation(d[i], d[i]).build()) - .layer(new OutputLayer.Builder().nOut(10).build()) - .setInputType(InputType.convolutional(inputSize[3], inputSize[2], inputSize[1])) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - INDArray in = Nd4j.create(inputSize); - net.output(in); - } - } - } - } - - - @Test - public void testGradientNorm() throws Exception { - - int height = 100; - int width = 100; - int channels = 1; - int numLabels = 10; - - for( int batchSize : new int[]{1, 32}) { - - long seed = 12345; - double nonZeroBias = 1; - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .seed(seed) - .dataType(DataType.DOUBLE) - .dist(new NormalDistribution(0.0, 0.01)) - .activation(Activation.RELU) - .updater(new Adam(5e-3)) - //.biasUpdater(new Nesterovs(new StepSchedule(ScheduleType.ITERATION, 2e-2, 0.1, 20000), 0.9)) - .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) - .l2(5 * 1e-4) - .list() - .layer(convInit("cnn1", channels, 96, new int[]{11, 11}, new int[]{4, 4}, - new int[]{3, 3}, 0)) - .layer(maxPool("maxpool1", new int[]{3, 3})) - .layer(conv5x5("cnn2", 256, new int[]{1, 1}, new int[]{2, 2}, nonZeroBias)) - .layer(maxPool("maxpool2", new int[]{3, 3})) - .layer(conv3x3("cnn3", 384, 0)) - .layer(conv3x3("cnn4", 384, nonZeroBias)) - .layer(conv3x3("cnn5", 256, nonZeroBias)) - .layer(maxPool("maxpool3", new int[]{3, 3})) - .layer(fullyConnected("ffn1", 4096, nonZeroBias, new GaussianDistribution(0, 0.005))) - .layer(fullyConnected("ffn2", 4096, nonZeroBias, new GaussianDistribution(0, 0.005))) - .layer(new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) - .name("output") - .nOut(numLabels) - .activation(Activation.SOFTMAX) - .build()) - .setInputType(InputType.convolutional(height, width, channels)) - .build(); - - - MultiLayerNetwork netNoCudnn = new MultiLayerNetwork(conf.clone()); - netNoCudnn.init(); - MultiLayerNetwork netWithCudnn = new MultiLayerNetwork(conf.clone()); - netWithCudnn.init(); - - CuDNNTestUtils.removeHelpers(netNoCudnn.getLayers()); - - - - Nd4j.getRandom().setSeed(12345); - for( int j=0; j<3; j++ ) { -// System.out.println("j=" + j); - INDArray f = Nd4j.rand(new int[]{batchSize, channels, height, width}); - INDArray l = TestUtils.randomOneHot(batchSize, numLabels); - - netNoCudnn.fit(f, l); - netWithCudnn.fit(f, l); - - assertEquals(netNoCudnn.score(), netWithCudnn.score(), 1e-5); - - for (Map.Entry e : netNoCudnn.paramTable().entrySet()) { - boolean pEq = e.getValue().equalsWithEps(netWithCudnn.paramTable().get(e.getKey()), 1e-4); -// int idx = e.getKey().indexOf("_"); -// int layerNum = Integer.parseInt(e.getKey().substring(0, idx)); - //System.out.println(e.getKey() + " - " + pEq + " - " + netNoCudnn.getLayer(layerNum).getClass().getSimpleName()); - assertTrue(pEq); - } - - boolean eq = netNoCudnn.params().equalsWithEps(netWithCudnn.params(), 1e-4); - assertTrue(eq); - } - } - } - - - private static ConvolutionLayer convInit(String name, int in, int out, int[] kernel, int[] stride, - int[] pad, double bias) { - return new ConvolutionLayer.Builder(kernel, stride, pad).name(name) - .nIn(in) - .nOut(out) - .biasInit(bias) - .build(); - } - - private static ConvolutionLayer conv3x3(String name, int out, double bias) { - return new ConvolutionLayer.Builder(new int[] { 3, 3 }, new int[] { 1, 1 }, - new int[] { 1, 1 }).name(name).nOut(out).biasInit(bias).build(); - } - - private static ConvolutionLayer conv5x5(String name, int out, int[] stride, int[] pad, - double bias) { - return new ConvolutionLayer.Builder(new int[] { 5, 5 }, stride, pad).name(name) - .nOut(out) - .biasInit(bias) - .build(); - } - - private static SubsamplingLayer maxPool(String name, int[] kernel) { - return new SubsamplingLayer.Builder(kernel, new int[] { 2, 2 }).name(name).build(); - } - - private static DenseLayer fullyConnected(String name, int out, double bias, Distribution dist) { - return new DenseLayer.Builder().name(name) - .nOut(out) - .biasInit(bias) - .dist(dist) - .build(); - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/gradientcheck/CNNGradientCheckTest.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/gradientcheck/CNNGradientCheckTest.java deleted file mode 100644 index 46355b74e..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/gradientcheck/CNNGradientCheckTest.java +++ /dev/null @@ -1,742 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.cuda.gradientcheck; - -import lombok.val; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.cuda.TestUtils; -import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; -import org.deeplearning4j.gradientcheck.GradientCheckUtil; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.util.Arrays; -import static org.deeplearning4j.nn.conf.ConvolutionMode.Same; -import static org.deeplearning4j.nn.conf.ConvolutionMode.Truncate; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * Created by nyghtowl on 9/1/15. - */ -@NativeTag -@DisplayName("Cnn Gradient Check Test") -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -class CNNGradientCheckTest extends BaseDL4JTest { - - private static final boolean PRINT_RESULTS = true; - - private static final boolean RETURN_ON_FIRST_FAILURE = false; - - private static final double DEFAULT_EPS = 1e-6; - - private static final double DEFAULT_MAX_REL_ERROR = 1e-3; - - private static final double DEFAULT_MIN_ABS_ERROR = 1e-8; - - static { - Nd4j.setDataType(DataType.DOUBLE); - } - - @Override - public long getTimeoutMilliseconds() { - return 180000L; - } - - @Test - @DisplayName("Test Gradient CNNMLN") - - void testGradientCNNMLN() { - // Parameterized test, testing combinations of: - // (a) activation function - // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') - // (c) Loss function (with specified output activations) - Activation[] activFns = { Activation.SIGMOID, Activation.TANH }; - // If true: run some backprop steps first - boolean[] characteristic = { false, true }; - LossFunctions.LossFunction[] lossFunctions = { LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE }; - // i.e., lossFunctions[i] used with outputActivations[i] here - Activation[] outputActivations = { Activation.SOFTMAX, Activation.TANH }; - DataSet ds = new IrisDataSetIterator(150, 150).next(); - ds.normalizeZeroMeanZeroUnitVariance(); - INDArray input = ds.getFeatures(); - INDArray labels = ds.getLabels(); - for (Activation afn : activFns) { - for (boolean doLearningFirst : characteristic) { - for (int i = 0; i < lossFunctions.length; i++) { - LossFunctions.LossFunction lf = lossFunctions[i]; - Activation outputActivation = outputActivations[i]; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp()).weightInit(WeightInit.XAVIER).seed(12345L).list().layer(0, new ConvolutionLayer.Builder(1, 1).nOut(6).activation(afn).cudnnAllowFallback(false).build()).layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).build()).setInputType(InputType.convolutionalFlat(1, 4, 1)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - String name = new Object() { - }.getClass().getEnclosingMethod().getName(); - if (doLearningFirst) { - // Run a number of iterations of learning - mln.setInput(ds.getFeatures()); - mln.setLabels(ds.getLabels()); - mln.computeGradientAndScore(); - double scoreBefore = mln.score(); - for (int j = 0; j < 10; j++) mln.fit(ds); - mln.computeGradientAndScore(); - double scoreAfter = mln.score(); - // Can't test in 'characteristic mode of operation' if not learning - String msg = name + " - score did not (sufficiently) decrease during learning - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")"; - assertTrue(scoreAfter < 0.8 * scoreBefore, msg); - } - if (PRINT_RESULTS) { - System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst); - } - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - } - } - - @Test - @DisplayName("Test Gradient CNNL 1 L 2 MLN") - void testGradientCNNL1L2MLN() { - // Parameterized test, testing combinations of: - // (a) activation function - // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') - // (c) Loss function (with specified output activations) - DataSet ds = new IrisDataSetIterator(150, 150).next(); - ds.normalizeZeroMeanZeroUnitVariance(); - INDArray input = ds.getFeatures(); - INDArray labels = ds.getLabels(); - // use l2vals[i] with l1vals[i] - double[] l2vals = { 0.4, 0.0, 0.4, 0.4 }; - double[] l1vals = { 0.0, 0.0, 0.5, 0.0 }; - double[] biasL2 = { 0.0, 0.0, 0.0, 0.2 }; - double[] biasL1 = { 0.0, 0.0, 0.6, 0.0 }; - Activation[] activFns = { Activation.SIGMOID, Activation.TANH, Activation.ELU, Activation.SOFTPLUS }; - // If true: run some backprop steps first - boolean[] characteristic = { false, true, false, true }; - LossFunctions.LossFunction[] lossFunctions = { LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE, LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE }; - // i.e., lossFunctions[i] used with outputActivations[i] here - Activation[] outputActivations = { Activation.SOFTMAX, Activation.TANH, Activation.SOFTMAX, Activation.IDENTITY }; - for (int i = 0; i < l2vals.length; i++) { - Activation afn = activFns[i]; - boolean doLearningFirst = characteristic[i]; - LossFunctions.LossFunction lf = lossFunctions[i]; - Activation outputActivation = outputActivations[i]; - double l2 = l2vals[i]; - double l1 = l1vals[i]; - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).l2(l2).l1(l1).l2Bias(biasL2[i]).l1Bias(biasL1[i]).optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).seed(12345L).list().layer(0, new ConvolutionLayer.Builder(new int[] { 1, 1 }).nIn(1).nOut(6).weightInit(WeightInit.XAVIER).activation(afn).updater(new NoOp()).build()).layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).weightInit(WeightInit.XAVIER).updater(new NoOp()).build()).setInputType(InputType.convolutionalFlat(1, 4, 1)); - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - String testName = new Object() { - }.getClass().getEnclosingMethod().getName(); - if (doLearningFirst) { - // Run a number of iterations of learning - mln.setInput(ds.getFeatures()); - mln.setLabels(ds.getLabels()); - mln.computeGradientAndScore(); - double scoreBefore = mln.score(); - for (int j = 0; j < 10; j++) mln.fit(ds); - mln.computeGradientAndScore(); - double scoreAfter = mln.score(); - // Can't test in 'characteristic mode of operation' if not learning - String msg = testName + "- score did not (sufficiently) decrease during learning - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst + " (before=" + scoreBefore + ", scoreAfter=" + scoreAfter + ")"; - assertTrue(scoreAfter < 0.8 * scoreBefore, msg); - } - if (PRINT_RESULTS) { - System.out.println(testName + "- activationFn=" + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation + ", doLearningFirst=" + doLearningFirst); - // for (int j = 0; j < mln.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK); - TestUtils.testModelSerialization(mln); - } - } - - @Test - @DisplayName("Test Cnn With Space To Depth") - void testCnnWithSpaceToDepth() { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int minibatchSize = 2; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int blocks = 2; - String[] activations = { "sigmoid" }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - for (String afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).list().layer(new ConvolutionLayer.Builder(kernel).nIn(inputDepth).hasBias(false).cudnnAllowFallback(false).nOut(1).build()).layer(new SpaceToDepthLayer.Builder(blocks, SpaceToDepthLayer.DataFormat.NCHW).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2 * 2 * 4).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - } - - @Test - @DisplayName("Test Cnn With Space To Batch") - void testCnnWithSpaceToBatch() { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int[] minibatchSizes = { 2, 4 }; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int[] blocks = { 1, 1 }; - String[] activations = { "sigmoid", "tanh" }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - for (String afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).list().layer(new ConvolutionLayer.Builder(kernel).nIn(inputDepth).cudnnAllowFallback(false).nOut(3).build()).layer(// trivial space to batch - new SpaceToBatchLayer.Builder(blocks).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(4 * 4 * 3).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - } - } - - @Test - @DisplayName("Test Cnn With Upsampling") - void testCnnWithUpsampling() { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int[] minibatchSizes = { 1, 3 }; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int size = 2; - for (int minibatchSize : minibatchSizes) { - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).dist(new NormalDistribution(0, 1)).list().layer(new ConvolutionLayer.Builder(kernel, stride, padding).nIn(inputDepth).nOut(3).build()).layer(// output: 4*2 =8 -> 8x8x3 - new Upsampling2D.Builder().size(size).build()).layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(8 * 8 * 3).nOut(4).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "Upsampling - minibatch=" + minibatchSize; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Cnn With Subsampling") - void testCnnWithSubsampling() { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int[] minibatchSizes = { 1, 3 }; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int pnorm = 2; - Activation[] activations = { Activation.SIGMOID, Activation.TANH }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).dist(new NormalDistribution(0, 1)).list().layer(0, new ConvolutionLayer.Builder(kernel, stride, padding).nIn(inputDepth).cudnnAllowFallback(false).nOut(3).build()).layer(1, new SubsamplingLayer.Builder(poolingType).cudnnAllowFallback(false).kernelSize(kernel).stride(stride).padding(padding).pnorm(pnorm).build()).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3 * 3 * 3).nOut(4).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - } - } - - @Test - @DisplayName("Test Cnn With Subsampling V 2") - void testCnnWithSubsamplingV2() { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int[] minibatchSizes = { 1, 3 }; - int width = 5; - int height = 5; - int inputDepth = 1; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int pNorm = 3; - Activation[] activations = { Activation.SIGMOID, Activation.TANH }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM }; - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).dist(new NormalDistribution(0, 1)).list().layer(0, new ConvolutionLayer.Builder(kernel, stride, padding).nIn(inputDepth).cudnnAllowFallback(false).nOut(3).build()).layer(1, new SubsamplingLayer.Builder(poolingType).kernelSize(kernel).stride(stride).padding(padding).cudnnAllowFallback(false).pnorm(pNorm).build()).layer(2, new ConvolutionLayer.Builder(kernel, stride, padding).cudnnAllowFallback(false).nIn(3).nOut(2).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2 * 2 * 2).nOut(4).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - } - } - - @Test - @DisplayName("Test Cnn Multi Layer") - void testCnnMultiLayer() { - int nOut = 2; - int[] minibatchSizes = { 1, 2, 5 }; - int width = 5; - int height = 5; - int[] inputDepths = { 1, 2, 4 }; - Activation[] activations = { Activation.SIGMOID, Activation.TANH }; - SubsamplingLayer.PoolingType[] poolingTypes = new SubsamplingLayer.PoolingType[] { SubsamplingLayer.PoolingType.MAX, SubsamplingLayer.PoolingType.AVG }; - Nd4j.getRandom().setSeed(12345); - for (int inputDepth : inputDepths) { - for (Activation afn : activations) { - for (SubsamplingLayer.PoolingType poolingType : poolingTypes) { - for (int minibatchSize : minibatchSizes) { - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new NoOp()).dataType(DataType.DOUBLE).activation(afn).list().layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).cudnnAllowFallback(false).padding(0, 0).nIn(inputDepth).nOut(2).build()).layer(1, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(2, 2).cudnnAllowFallback(false).stride(1, 1).padding(0, 0).build()).layer(2, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(2, 2).cudnnAllowFallback(false).stride(1, 1).padding(0, 0).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2 * 2 * 2).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - assertEquals(ConvolutionMode.Truncate, ((ConvolutionLayer) conf.getConf(0).getLayer()).getConvolutionMode()); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // for (int i = 0; i < 4; i++) { - // System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - // } - String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn=" + afn; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - } - } - } - - @Test - @DisplayName("Test Cnn Same Padding Mode") - void testCnnSamePaddingMode() { - int nOut = 2; - int[] minibatchSizes = { 1, 3, 3, 2, 1, 2 }; - // Same padding mode: insensitive to exact input size... - int[] heights = new int[] { 4, 5, 6, 5, 4, 4 }; - int[] kernelSizes = new int[] { 2, 3, 2, 3, 2, 3 }; - int[] inputDepths = { 1, 2, 4, 3, 2, 3 }; - int width = 5; - Nd4j.getRandom().setSeed(12345); - for (int i = 0; i < minibatchSizes.length; i++) { - int inputDepth = inputDepths[i]; - int minibatchSize = minibatchSizes[i]; - int height = heights[i]; - int k = kernelSizes[i]; - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(Same).list().layer(0, new ConvolutionLayer.Builder().name("layer 0").kernelSize(k, k).stride(1, 1).padding(0, 0).nIn(inputDepth).nOut(2).build()).layer(1, new SubsamplingLayer.Builder().poolingType(SubsamplingLayer.PoolingType.MAX).kernelSize(k, k).stride(1, 1).padding(0, 0).build()).layer(2, new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(k, k).stride(1, 1).padding(0, 0).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // for (int j = 0; j < net.getLayers().length; j++) { - // System.out.println("nParams, layer " + j + ": " + net.getLayer(j).numParams()); - // } - String msg = "Minibatch=" + minibatchSize + ", inDepth=" + inputDepth + ", height=" + height + ", kernelSize=" + k; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Cnn Same Padding Mode Strided") - void testCnnSamePaddingModeStrided() { - int nOut = 2; - int[] minibatchSizes = { 1, 3 }; - int width = 16; - int height = 16; - int[] kernelSizes = new int[] { 2, 3 }; - int[] strides = { 1, 2, 3 }; - int[] inputDepths = { 1, 3 }; - Nd4j.getRandom().setSeed(12345); - for (int inputDepth : inputDepths) { - for (int minibatchSize : minibatchSizes) { - for (int stride : strides) { - for (int k : kernelSizes) { - for (boolean convFirst : new boolean[] { true, false }) { - INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - Layer convLayer = new ConvolutionLayer.Builder().name("layer 0").kernelSize(k, k).stride(stride, stride).padding(0, 0).nIn(inputDepth).nOut(2).build(); - Layer poolLayer = new SubsamplingLayer.Builder().poolingType(SubsamplingLayer.PoolingType.MAX).kernelSize(k, k).stride(stride, stride).padding(0, 0).build(); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(Same).list().layer(0, convFirst ? convLayer : poolLayer).layer(1, convFirst ? poolLayer : convLayer).layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // for (int i = 0; i < net.getLayers().length; i++) { - // System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - // } - String msg = "Minibatch=" + minibatchSize + ", inDepth=" + inputDepth + ", height=" + height + ", kernelSize=" + k + ", stride = " + stride + ", convLayer first = " + convFirst; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(128)); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - } - } - } - } - - @Test - @DisplayName("Test Cnn Zero Padding Layer") - void testCnnZeroPaddingLayer() { - Nd4j.getRandom().setSeed(12345); - int nOut = 4; - int width = 6; - int height = 6; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int[] minibatchSizes = { 1, 3, 2 }; - int[] inputDepths = { 1, 3, 2 }; - int[][] zeroPadLayer = new int[][] { { 0, 0, 0, 0 }, { 1, 1, 0, 0 }, { 2, 2, 2, 2 } }; - for (int i = 0; i < minibatchSizes.length; i++) { - int minibatchSize = minibatchSizes[i]; - int inputDepth = inputDepths[i]; - int[] zeroPad = zeroPadLayer[i]; - INDArray input = Nd4j.rand(DataType.DOUBLE, new int[] { minibatchSize, inputDepth, height, width }); - INDArray labels = TestUtils.randomOneHot(minibatchSize, nOut); - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()).dataType(DataType.DOUBLE).dist(new NormalDistribution(0, 1)).list().layer(0, new ConvolutionLayer.Builder(kernel, stride, padding).nIn(inputDepth).nOut(3).build()).layer(1, new ZeroPaddingLayer.Builder(zeroPad).build()).layer(2, new ConvolutionLayer.Builder(kernel, stride, padding).nIn(3).nOut(3).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(4).build()).setInputType(InputType.convolutional(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // Check zero padding activation shape - org.deeplearning4j.nn.layers.convolution.ZeroPaddingLayer zpl = (org.deeplearning4j.nn.layers.convolution.ZeroPaddingLayer) net.getLayer(1); - val expShape = new long[] { minibatchSize, inputDepth, height + zeroPad[0] + zeroPad[1], width + zeroPad[2] + zeroPad[3] }; - INDArray out = zpl.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(expShape, out.shape()); - String msg = "minibatch=" + minibatchSize + ", channels=" + inputDepth + ", zeroPad = " + Arrays.toString(zeroPad); - if (PRINT_RESULTS) { - System.out.println(msg); - // for (int j = 0; j < net.getnLayers(); j++) - // System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams()); - } - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Deconvolution 2 D") - void testDeconvolution2D() { - int nOut = 2; - int[] minibatchSizes = new int[] { 1, 3, 3, 1, 3 }; - int[] kernelSizes = new int[] { 1, 1, 1, 3, 3 }; - int[] strides = { 1, 1, 2, 2, 2 }; - int[] dilation = { 1, 2, 1, 2, 2 }; - Activation[] activations = new Activation[] { Activation.SIGMOID, Activation.TANH, Activation.SIGMOID, Activation.SIGMOID, Activation.SIGMOID }; - ConvolutionMode[] cModes = new ConvolutionMode[] { Same, Same, Truncate, Truncate, Truncate }; - int width = 7; - int height = 7; - int inputDepth = 3; - Nd4j.getRandom().setSeed(12345); - for (int i = 0; i < minibatchSizes.length; i++) { - int minibatchSize = minibatchSizes[i]; - int k = kernelSizes[i]; - int s = strides[i]; - int d = dilation[i]; - ConvolutionMode cm = cModes[i]; - Activation act = activations[i]; - int w = d * width; - int h = d * height; - INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int j = 0; j < minibatchSize; j++) { - labels.putScalar(new int[] { j, j % nOut }, 1.0); - } - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(act).list().layer(new Deconvolution2D.Builder().name("deconvolution_2D_layer").kernelSize(k, k).stride(s, s).dilation(d, d).convolutionMode(cm).nIn(inputDepth).nOut(nOut).build()); - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // for (int j = 0; j < net.getLayers().length; j++) { - // System.out.println("nParams, layer " + j + ": " + net.getLayer(j).numParams()); - // } - String msg = " - mb=" + minibatchSize + ", k=" + k + ", s=" + s + ", d=" + d + ", cm=" + cm; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(100)); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Depthwise Conv 2 D") - void testDepthwiseConv2D() { - int nIn = 3; - int depthMultiplier = 2; - int nOut = nIn * depthMultiplier; - int width = 5; - int height = 5; - Nd4j.getRandom().setSeed(12345); - int[] ks = new int[] { 1, 3, 3, 1, 3 }; - int[] ss = new int[] { 1, 1, 1, 2, 2 }; - ConvolutionMode[] cms = new ConvolutionMode[] { Truncate, Truncate, Truncate, Truncate, Truncate }; - int[] mb = new int[] { 1, 1, 1, 3, 3 }; - for (int t = 0; t < ks.length; t++) { - int k = ks[t]; - int s = ss[t]; - ConvolutionMode cm = cms[t]; - int minibatchSize = mb[t]; - INDArray input = Nd4j.rand(minibatchSize, width * height * nIn); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(cm).list().layer(new Convolution2D.Builder().kernelSize(1, 1).stride(1, 1).nIn(nIn).nOut(nIn).build()).layer(new DepthwiseConvolution2D.Builder().name("depth-wise conv 2D layer").cudnnAllowFallback(false).kernelSize(k, k).stride(s, s).depthMultiplier(depthMultiplier).nIn(nIn).build()); - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(height, width, nIn)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // for (int i = 0; i < net.getLayers().length; i++) { - // System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - // } - String msg = " - mb=" + minibatchSize + ", k=" + k + ", nIn=" + nIn + ", depthMul=" + depthMultiplier + ", s=" + s + ", cm=" + cm; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(256)); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Separable Conv 2 D") - void testSeparableConv2D() { - int nOut = 2; - int[] minibatchSizes = new int[] { 1, 3 }; - int width = 6; - int height = 6; - int inputDepth = 3; - Nd4j.getRandom().setSeed(12345); - int[] ks = new int[] { 1, 3, 3, 1, 3 }; - int[] ss = new int[] { 1, 1, 1, 2, 2 }; - int[] ds = new int[] { 1, 1, 2, 2, 2 }; - ConvolutionMode[] cms = new ConvolutionMode[] { Truncate, Truncate, Truncate, Truncate, Truncate }; - int[] mb = new int[] { 1, 1, 1, 3, 3 }; - for (int t = 0; t < ks.length; t++) { - int k = ks[t]; - int s = ss[t]; - int d = ds[t]; - ConvolutionMode cm = cms[t]; - int minibatchSize = mb[t]; - // Use larger input with larger dilation values (to avoid invalid config) - int w = d * width; - int h = d * height; - INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(cm).list().layer(new SeparableConvolution2D.Builder().name("Separable conv 2D layer").kernelSize(k, k).stride(s, s).dilation(d, d).depthMultiplier(3).nIn(inputDepth).nOut(2).build()); - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // for (int i = 0; i < net.getLayers().length; i++) { - // System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - // } - String msg = " - mb=" + minibatchSize + ", k=" + k + ", s=" + s + ", d=" + d + ", cm=" + cm; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(50)); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Cnn Dilated") - void testCnnDilated() { - int nOut = 2; - int minibatchSize = 2; - int width = 8; - int height = 8; - int inputDepth = 2; - Nd4j.getRandom().setSeed(12345); - boolean[] sub = new boolean[] { true, true, false, true, false }; - int[] stride = new int[] { 1, 1, 1, 2, 2 }; - int[] kernel = new int[] { 2, 3, 3, 3, 3 }; - int[] ds = new int[] { 2, 2, 3, 3, 2 }; - ConvolutionMode[] cms = new ConvolutionMode[] { Same, Truncate, Truncate, Same, Truncate }; - for (int t = 0; t < sub.length; t++) { - boolean subsampling = sub[t]; - int s = stride[t]; - int k = kernel[t]; - int d = ds[t]; - ConvolutionMode cm = cms[t]; - // Use larger input with larger dilation values (to avoid invalid config) - int w = d * width; - int h = d * height; - INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345).dataType(DataType.DOUBLE).updater(new NoOp()).activation(Activation.TANH).convolutionMode(cm).list().layer(new ConvolutionLayer.Builder().name("layer 0").kernelSize(k, k).stride(s, s).dilation(d, d).nIn(inputDepth).nOut(2).build()); - if (subsampling) { - b.layer(new SubsamplingLayer.Builder().poolingType(SubsamplingLayer.PoolingType.MAX).kernelSize(k, k).stride(s, s).dilation(d, d).build()); - } else { - b.layer(new ConvolutionLayer.Builder().nIn(2).nOut(2).kernelSize(k, k).stride(s, s).dilation(d, d).build()); - } - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // for (int i = 0; i < net.getLayers().length; i++) { - // System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - // } - String msg = (subsampling ? "subsampling" : "conv") + " - mb=" + minibatchSize + ", k=" + k + ", s=" + s + ", d=" + d + ", cm=" + cm; - System.out.println(msg); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - - @Test - @DisplayName("Test Cropping 2 D Layer") - void testCropping2DLayer() { - Nd4j.getRandom().setSeed(12345); - int nOut = 2; - int[] minibatchSizes = { 1, 3 }; - int width = 12; - int height = 11; - int[] inputDepths = { 1, 3 }; - int[] kernel = { 2, 2 }; - int[] stride = { 1, 1 }; - int[] padding = { 0, 0 }; - int[][] cropTestCases = new int[][] { { 0, 0, 0, 0 }, { 1, 1, 0, 0 }, { 2, 2, 2, 2 }, { 1, 2, 3, 4 } }; - for (int inputDepth : inputDepths) { - for (int minibatchSize : minibatchSizes) { - INDArray input = Nd4j.rand(new int[] { minibatchSize, inputDepth, height, width }); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[] { i, i % nOut }, 1.0); - } - for (int[] crop : cropTestCases) { - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().dataType(DataType.DOUBLE).updater(new NoOp()).convolutionMode(ConvolutionMode.Same).weightInit(new NormalDistribution(0, 1)).list().layer(new ConvolutionLayer.Builder(kernel, stride, padding).nIn(inputDepth).nOut(2).build()).layer(new Cropping2D(crop)).layer(new ConvolutionLayer.Builder(kernel, stride, padding).nIn(2).nOut(2).build()).layer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.AVG).kernelSize(3, 3).stride(3, 3).build()).layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(nOut).build()).setInputType(InputType.convolutional(height, width, inputDepth)).build(); - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - // Check cropping activation shape - org.deeplearning4j.nn.layers.convolution.Cropping2DLayer cl = (org.deeplearning4j.nn.layers.convolution.Cropping2DLayer) net.getLayer(1); - val expShape = new long[] { minibatchSize, inputDepth, height - crop[0] - crop[1], width - crop[2] - crop[3] }; - INDArray out = cl.activate(input, false, LayerWorkspaceMgr.noWorkspaces()); - assertArrayEquals(expShape, out.shape()); - String msg = "minibatch=" + minibatchSize + ", channels=" + inputDepth + ", zeroPad = " + Arrays.toString(crop); - if (PRINT_RESULTS) { - System.out.println(msg); - } - boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(input).labels(labels).subset(true).maxPerParam(160)); - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(net); - } - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/gradientcheck/CuDNNGradientChecks.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/gradientcheck/CuDNNGradientChecks.java deleted file mode 100644 index 94329ccc1..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/gradientcheck/CuDNNGradientChecks.java +++ /dev/null @@ -1,714 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.gradientcheck; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.cuda.TestUtils; -import org.deeplearning4j.gradientcheck.GradientCheckUtil; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.OptimizationAlgorithm; -import org.deeplearning4j.nn.conf.ConvolutionMode; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.distribution.UniformDistribution; -import org.deeplearning4j.nn.conf.dropout.Dropout; -import org.deeplearning4j.nn.conf.dropout.IDropout; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.layers.convolution.ConvolutionHelper; -import org.deeplearning4j.cuda.convolution.CudnnConvolutionHelper; -import org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingHelper; -import org.deeplearning4j.cuda.dropout.CudnnDropoutHelper; -import org.deeplearning4j.nn.layers.normalization.BatchNormalizationHelper; -import org.deeplearning4j.cuda.normalization.CudnnBatchNormalizationHelper; -import org.deeplearning4j.cuda.normalization.CudnnLocalResponseNormalizationHelper; -import org.deeplearning4j.nn.layers.normalization.LocalResponseNormalizationHelper; -import org.deeplearning4j.cuda.recurrent.CudnnLSTMHelper; -import org.deeplearning4j.nn.layers.recurrent.LSTMHelper; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.junit.jupiter.api.Test; -import org.nd4j.common.function.Consumer; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.buffer.util.DataTypeUtil; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Random; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Created by Alex on 09/09/2016. - */ -@Slf4j -@NativeTag -public class CuDNNGradientChecks extends BaseDL4JTest { - - private static final boolean PRINT_RESULTS = true; - private static final boolean RETURN_ON_FIRST_FAILURE = false; - private static final double DEFAULT_EPS = 1e-5; - private static final double DEFAULT_MAX_REL_ERROR = 1e-2; - private static final double DEFAULT_MIN_ABS_ERROR = 1e-6; - - static { - DataTypeUtil.setDTypeForContext(DataType.DOUBLE); - } - - @Override - public long getTimeoutMilliseconds() { - return 180000L; - } - - @Test - public void testConvolutional() throws Exception { - - //Parameterized test, testing combinations of: - // (a) activation function - // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation') - // (c) Loss function (with specified output activations) - Activation[] activFns = {Activation.SIGMOID, Activation.TANH}; - boolean[] characteristic = {false, true}; //If true: run some backprop steps first - - int[] minibatchSizes = {1, 4}; - int width = 6; - int height = 6; - int inputDepth = 2; - int nOut = 3; - - Field f = org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.class.getDeclaredField("helper"); - f.setAccessible(true); - - Random r = new Random(12345); - for (Activation afn : activFns) { - for (boolean doLearningFirst : characteristic) { - for (int minibatchSize : minibatchSizes) { - - INDArray input = Nd4j.rand(new int[] {minibatchSize, inputDepth, height, width}); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(i, r.nextInt(nOut), 1.0); - } - - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT) - .dist(new UniformDistribution(-1, 1)) - .updater(new NoOp()).seed(12345L).list() - .layer(0, new ConvolutionLayer.Builder(2, 2).stride(2, 2).padding(1, 1).nOut(3) - .activation(afn).build()) - .layer(1, new ConvolutionLayer.Builder(2, 2).stride(2, 2).padding(0, 0).nOut(3) - .activation(afn).build()) - .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nOut(nOut).build()) - .setInputType(InputType.convolutional(height, width, inputDepth)) - ; - - MultiLayerConfiguration conf = builder.build(); - - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - - org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c0 = - (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) mln.getLayer(0); - ConvolutionHelper ch0 = (ConvolutionHelper) f.get(c0); - assertTrue(ch0 instanceof CudnnConvolutionHelper); - - org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c1 = - (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) mln.getLayer(1); - ConvolutionHelper ch1 = (ConvolutionHelper) f.get(c1); - assertTrue(ch1 instanceof CudnnConvolutionHelper); - - //------------------------------- - //For debugging/comparison to no-cudnn case: set helper field to null - // f.set(c0, null); - // f.set(c1, null); - // assertNull(f.get(c0)); - // assertNull(f.get(c1)); - //------------------------------- - - - String name = new Object() {}.getClass().getEnclosingMethod().getName(); - - if (doLearningFirst) { - //Run a number of iterations of learning - mln.setInput(input); - mln.setLabels(labels); - mln.computeGradientAndScore(); - double scoreBefore = mln.score(); - for (int j = 0; j < 10; j++) - mln.fit(input, labels); - mln.computeGradientAndScore(); - double scoreAfter = mln.score(); - //Can't test in 'characteristic mode of operation' if not learning - String msg = name + " - score did not (sufficiently) decrease during learning - activationFn=" - + afn + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore - + ", scoreAfter=" + scoreAfter + ")"; - assertTrue(scoreAfter < 0.8 * scoreBefore, msg); - } - - if (PRINT_RESULTS) { - System.out.println(name + " - activationFn=" + afn + ", doLearningFirst=" + doLearningFirst); - for (int j = 0; j < mln.getnLayers(); j++) - System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, - DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - - assertTrue(gradOK); - } - } - } - } - - - @Test - public void testConvolutionalNoBias() throws Exception { - int[] minibatchSizes = {1, 4}; - int width = 6; - int height = 6; - int inputDepth = 2; - int nOut = 3; - - Field f = org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.class.getDeclaredField("helper"); - f.setAccessible(true); - - Random r = new Random(12345); - for (int minibatchSize : minibatchSizes) { - for (boolean convHasBias : new boolean[]{true, false}) { - - INDArray input = Nd4j.rand(new int[]{minibatchSize, inputDepth, height, width}); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(i, r.nextInt(nOut), 1.0); - } - - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .dist(new UniformDistribution(-1, 1)) - .updater(new NoOp()).seed(12345L) - .list() - .layer(0, new ConvolutionLayer.Builder(2, 2).stride(2, 2).padding(1, 1).nOut(3) - .hasBias(convHasBias) - .activation(Activation.TANH).build()) - .layer(1, new ConvolutionLayer.Builder(2, 2).stride(2, 2).padding(0, 0).nOut(3) - .hasBias(convHasBias) - .activation(Activation.TANH).build()) - .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nOut(nOut).build()) - .setInputType(InputType.convolutional(height, width, inputDepth)) - ; - - MultiLayerConfiguration conf = builder.build(); - - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - - org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c0 = - (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) mln.getLayer(0); - ConvolutionHelper ch0 = (ConvolutionHelper) f.get(c0); - assertTrue(ch0 instanceof CudnnConvolutionHelper); - - org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c1 = - (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) mln.getLayer(1); - ConvolutionHelper ch1 = (ConvolutionHelper) f.get(c1); - assertTrue(ch1 instanceof CudnnConvolutionHelper); - - - String name = new Object() {}.getClass().getEnclosingMethod().getName() + ", minibatch = " - + minibatchSize + ", convHasBias = " + convHasBias; - - if (PRINT_RESULTS) { - System.out.println(name); - for (int j = 0; j < mln.getnLayers(); j++) - System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, - DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - - assertTrue(gradOK, name); - } - } - } - - @Test - public void testBatchNormCnn() throws Exception { - //Note: CuDNN batch norm supports 4d only, as per 5.1 (according to api reference documentation) - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int depth = 1; - int hw = 4; - int nOut = 4; - INDArray input = Nd4j.rand(new int[] {minibatch, depth, hw, hw}); - INDArray labels = Nd4j.zeros(minibatch, nOut); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - labels.putScalar(i, r.nextInt(nOut), 1.0); - } - - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp()) - .dataType(DataType.DOUBLE) - .seed(12345L) - .dist(new NormalDistribution(0, 2)).list() - .layer(0, new ConvolutionLayer.Builder().kernelSize(2, 2).stride(1, 1).nIn(depth).nOut(2) - .activation(Activation.IDENTITY).build()) - .layer(1, new BatchNormalization.Builder().build()) - .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build()) - .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nOut(nOut).build()) - .setInputType(InputType.convolutional(hw, hw, depth)); - - MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); - mln.init(); - - Field f = org.deeplearning4j.nn.layers.normalization.BatchNormalization.class.getDeclaredField("helper"); - f.setAccessible(true); - - org.deeplearning4j.nn.layers.normalization.BatchNormalization b = - (org.deeplearning4j.nn.layers.normalization.BatchNormalization) mln.getLayer(1); - BatchNormalizationHelper bn = (BatchNormalizationHelper) f.get(b); - assertTrue(bn instanceof CudnnBatchNormalizationHelper); - - //------------------------------- - //For debugging/comparison to no-cudnn case: set helper field to null - // f.set(b, null); - // assertNull(f.get(b)); - //------------------------------- - - if (PRINT_RESULTS) { - for (int j = 0; j < mln.getnLayers(); j++) - System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - - //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - //i.e., runningMean = decay * runningMean + (1-decay) * batchMean - //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, - DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels, null, null, false, -1, excludeParams, null); - - assertTrue(gradOK); - } - - @Test - public void testLRN() throws Exception { - - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int depth = 6; - int hw = 5; - int nOut = 4; - INDArray input = Nd4j.rand(new int[] {minibatch, depth, hw, hw}); - INDArray labels = Nd4j.zeros(minibatch, nOut); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - labels.putScalar(i, r.nextInt(nOut), 1.0); - } - - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp()) - .dataType(DataType.DOUBLE) - .seed(12345L) - .dist(new NormalDistribution(0, 2)).list() - .layer(0, new ConvolutionLayer.Builder().nOut(6).kernelSize(2, 2).stride(1, 1) - .activation(Activation.TANH).build()) - .layer(1, new LocalResponseNormalization.Builder().build()) - .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nOut(nOut).build()) - .setInputType(InputType.convolutional(hw, hw, depth)); - - MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); - mln.init(); - - Field f = org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization.class - .getDeclaredField("helper"); - f.setAccessible(true); - - org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization l = - (org.deeplearning4j.nn.layers.normalization.LocalResponseNormalization) mln.getLayer(1); - LocalResponseNormalizationHelper lrn = (LocalResponseNormalizationHelper) f.get(l); - assertTrue(lrn instanceof CudnnLocalResponseNormalizationHelper); - - //------------------------------- - //For debugging/comparison to no-cudnn case: set helper field to null - // f.set(l, null); - // assertNull(f.get(l)); - //------------------------------- - - if (PRINT_RESULTS) { - for (int j = 0; j < mln.getnLayers(); j++) - System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, - DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - - assertTrue(gradOK); - } - - @Test - public void testLSTM() throws Exception { - - Nd4j.getRandom().setSeed(12345); - int minibatch = 4; - int inputSize = 3; - int lstmLayerSize = 4; - int timeSeriesLength = 3; - int nOut = 4; - INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength}); - INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - labels.putScalar(i, r.nextInt(nOut), j, 1.0); - } - } - - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .updater(new NoOp()).seed(12345L) - .dist(new NormalDistribution(0, 2)).list() - .layer(0, new LSTM.Builder().nIn(input.size(1)).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build()) - ; - - MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); - mln.init(); - - Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper"); - f.setAccessible(true); - - org.deeplearning4j.nn.layers.recurrent.LSTM l = (org.deeplearning4j.nn.layers.recurrent.LSTM) mln.getLayer(1); - LSTMHelper helper = (LSTMHelper) f.get(l); - assertTrue(helper instanceof CudnnLSTMHelper); - - //------------------------------- - //For debugging/comparison to no-cudnn case: set helper field to null - // f.set(l, null); - // assertNull(f.get(l)); - //------------------------------- - - if (PRINT_RESULTS) { - for (int j = 0; j < mln.getnLayers(); j++) - System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, - DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels, null, null, true, 32, null, null); - - assertTrue(gradOK); - } - - - @Test - public void testLSTM2() throws Exception { - - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int inputSize = 3; - int lstmLayerSize = 4; - int timeSeriesLength = 3; - int nOut = 2; - INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength}); - INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - labels.putScalar(i, r.nextInt(nOut), j, 1.0); - } - } - - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .updater(new NoOp()).seed(12345L) - .dist(new NormalDistribution(0, 2)).list() - .layer(0, new LSTM.Builder().nIn(input.size(1)).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build()) - ; - - MultiLayerNetwork mln = new MultiLayerNetwork(builder.build()); - mln.init(); - - Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper"); - f.setAccessible(true); - - org.deeplearning4j.nn.layers.recurrent.LSTM l = (org.deeplearning4j.nn.layers.recurrent.LSTM) mln.getLayer(1); - LSTMHelper helper = (LSTMHelper) f.get(l); - assertTrue(helper instanceof CudnnLSTMHelper); - - //------------------------------- - //For debugging/comparison to no-cudnn case: set helper field to null - // f.set(l, null); - // assertNull(f.get(l)); - //------------------------------- - - if (PRINT_RESULTS) { - for (int j = 0; j < mln.getnLayers(); j++) - System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams()); - } - - boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, - DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - - assertTrue(gradOK); - } - - - @Test - public void testCnnDilated() throws Exception { - int nOut = 2; - - int minibatchSize = 3; - int width = 8; - int height = 8; - int inputDepth = 3; - - - Nd4j.getRandom().setSeed(12345); - - Field f = org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.class.getDeclaredField("helper"); - f.setAccessible(true); - - Field f2 = org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer.class.getDeclaredField("helper"); - f2.setAccessible(true); - - int[] kernelSizes = new int[]{2, 3, 2}; - int[] strides = {1, 2, 2}; - int[] dilation = {2, 3, 2}; - ConvolutionMode[] cModes = new ConvolutionMode[]{ConvolutionMode.Truncate, ConvolutionMode.Same, ConvolutionMode.Truncate}; - - for (boolean subsampling : new boolean[]{false, true}) { - for (int t = 0; t < kernelSizes.length; t++) { - int k = kernelSizes[t]; - int s = strides[t]; - int d = dilation[t]; - ConvolutionMode cm = cModes[t]; - - //Use larger input with larger dilation values (to avoid invalid config) - int w = d * width; - int h = d * height; - - INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth); - INDArray labels = Nd4j.zeros(minibatchSize, nOut); - for (int i = 0; i < minibatchSize; i++) { - labels.putScalar(new int[]{i, i % nOut}, 1.0); - } - - NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345) - .dataType(DataType.DOUBLE) - .updater(new NoOp()) - .activation(Activation.TANH).convolutionMode(cm).list() - .layer(new ConvolutionLayer.Builder().name("layer 0") - .kernelSize(k, k) - .stride(s, s) - .dilation(d, d) - .nIn(inputDepth).nOut(2).build()); - if (subsampling) { - b.layer(new SubsamplingLayer.Builder() - .poolingType(SubsamplingLayer.PoolingType.MAX) - .kernelSize(k, k) - .stride(s, s) - .dilation(d, d) - .build()); - } else { - b.layer(new ConvolutionLayer.Builder().nIn(2).nOut(2) - .kernelSize(k, k) - .stride(s, s) - .dilation(d, d) - .build()); - } - - MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nOut(nOut).build()) - .setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c0 = - (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) net.getLayer(0); - ConvolutionHelper ch0 = (ConvolutionHelper) f.get(c0); - assertTrue(ch0 instanceof CudnnConvolutionHelper); - - if (subsampling) { - org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer s1 = - (org.deeplearning4j.nn.layers.convolution.subsampling.SubsamplingLayer) net.getLayer(1); - SubsamplingHelper sh1 = (SubsamplingHelper) f2.get(s1); - assertTrue(sh1 instanceof SubsamplingHelper); - } else { - org.deeplearning4j.nn.layers.convolution.ConvolutionLayer c1 = - (org.deeplearning4j.nn.layers.convolution.ConvolutionLayer) net.getLayer(1); - ConvolutionHelper ch1 = (ConvolutionHelper) f.get(c1); - assertTrue(ch1 instanceof CudnnConvolutionHelper); - } - - for (int i = 0; i < net.getLayers().length; i++) { - System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams()); - } - - String msg = (subsampling ? "subsampling" : "conv") + " - mb=" + minibatchSize + ", k=" - + k + ", s=" + s + ", d=" + d + ", cm=" + cm; - System.out.println(msg); - - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, - DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels); - - assertTrue(gradOK, msg); - } - } - } - - - @Test - public void testDropout() { - int minibatch = 2; - - for (boolean cnn : new boolean[]{false, true}) { - Nd4j.getRandom().setSeed(12345); - IDropout dropout = new Dropout(0.6); - - NeuralNetConfiguration.ListBuilder builder = new NeuralNetConfiguration.Builder() - .seed(12345) - .dataType(DataType.DOUBLE) - .dist(new NormalDistribution(0, 1)) - .convolutionMode(ConvolutionMode.Same) - .dropOut(dropout) - .activation(Activation.TANH) - .updater(new NoOp()) - .list(); - - if (cnn) { - builder.layer(new ConvolutionLayer.Builder().kernelSize(2, 2).stride(2, 2).nOut(2).build()); - builder.layer(new ConvolutionLayer.Builder().kernelSize(2, 2).stride(2, 2).nOut(2).build()); - builder.setInputType(InputType.convolutional(8, 8, 2)); - } else { - builder.layer(new DenseLayer.Builder().nOut(8).build()); - builder.layer(new DenseLayer.Builder().nOut(8).build()); - builder.setInputType(InputType.feedForward(6)); - } - builder.layer(new OutputLayer.Builder().nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()); - MultiLayerConfiguration conf = builder.build(); - - MultiLayerNetwork mln = new MultiLayerNetwork(conf); - mln.init(); - - INDArray f; - if (cnn) { - f = Nd4j.rand(new int[]{minibatch, 2, 8, 8}).muli(10).subi(5); - } else { - f = Nd4j.rand(minibatch, 6).muli(10).subi(5); - } - INDArray l = TestUtils.randomOneHot(minibatch, 3); - - mln.output(f, true); - - for (Layer layer : mln.getLayers()) { - Dropout d = (Dropout) layer.conf().getLayer().getIDropout(); - assertNotNull(d); - CudnnDropoutHelper h = (CudnnDropoutHelper) d.getHelper(); - assertNotNull(h); - } - - String msg = (cnn ? "CNN" : "Dense") + ": " + dropout.getClass().getSimpleName(); - - //Consumer function to enforce CuDNN RNG repeatability - otherwise will fail due to randomness (inconsistent - // dropout mask between forward passes) - Consumer c = new Consumer() { - @Override - public void accept(MultiLayerNetwork net) { - Nd4j.getRandom().setSeed(12345); - for(Layer l : net.getLayers()){ - Dropout d = (Dropout) l.conf().getLayer().getIDropout(); - if(d != null){ - ((CudnnDropoutHelper)d.getHelper()).setMask(null); - ((CudnnDropoutHelper)d.getHelper()).setRngStates(null); - } - } - } - }; - - log.info("*** Starting test: " + msg + " ***"); - boolean gradOK = GradientCheckUtil.checkGradients( - new GradientCheckUtil.MLNConfig().net(mln).epsilon(DEFAULT_EPS) - .maxRelError(DEFAULT_MAX_REL_ERROR).minAbsoluteError(DEFAULT_MIN_ABS_ERROR) - .print(PRINT_RESULTS ? GradientCheckUtil.PrintMode.ZEROS : GradientCheckUtil.PrintMode.FAILURES_ONLY) - .exitOnFirstError(RETURN_ON_FIRST_FAILURE) - .input(f).labels(l).callEachIter(c)); - - assertTrue(gradOK, msg); - TestUtils.testModelSerialization(mln); - } - } - - - @Test - public void testDenseBatchNorm(){ - - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .dataType(DataType.DOUBLE) - .seed(12345) - .weightInit(WeightInit.XAVIER) - .updater(new NoOp()) - .list() - .layer(new DenseLayer.Builder().nIn(5).nOut(5).activation(Activation.TANH).build()) - .layer(new BatchNormalization.Builder().nOut(5).build()) - .layer(new OutputLayer.Builder().nIn(5).nOut(5).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build()) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - INDArray in = Nd4j.rand(3, 5); - INDArray labels = TestUtils.randomOneHot(3, 5); - - //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc - //i.e., runningMean = decay * runningMean + (1-decay) * batchMean - //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter" - Set excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev")); - boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR, - DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, in, labels, null, null, false, -1, excludeParams, null); - - assertTrue(gradOK); - - TestUtils.testModelSerialization(net); - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/lstm/ValidateCudnnDropout.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/lstm/ValidateCudnnDropout.java deleted file mode 100644 index 3905215f0..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/lstm/ValidateCudnnDropout.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.lstm; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.cuda.dropout.CudnnDropoutHelper; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.conditions.Conditions; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@NativeTag -public class ValidateCudnnDropout extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 180000L; - } - - @Test - public void testCudnnDropoutSimple() { - for (int[] shape : new int[][]{{10, 10}, {5, 2, 5, 2}}) { - - Nd4j.getRandom().setSeed(12345); - INDArray in = Nd4j.ones(shape); - double pRetain = 0.25; - double valueIfKept = 1.0 / pRetain; - - CudnnDropoutHelper d = new CudnnDropoutHelper(DataType.DOUBLE); - - INDArray out = Nd4j.createUninitialized(shape); - d.applyDropout(in, out, pRetain); - - int countZero = Nd4j.getExecutioner().execAndReturn(new MatchCondition(out, Conditions.equals(0.0))).z().getInt(0); - int countNonDropped = Nd4j.getExecutioner().execAndReturn(new MatchCondition(out, Conditions.equals(valueIfKept))).z().getInt(0); -// System.out.println(countZero); -// System.out.println(countNonDropped); - - assertTrue(countZero >= 5 && countZero <= 90, String.valueOf(countZero)); - assertTrue(countNonDropped >= 5 && countNonDropped <= 95, String.valueOf(countNonDropped)); - assertEquals(100, countZero + countNonDropped); - - //Test repeatability: - for (int i = 0; i < 10; i++) { - Nd4j.getRandom().setSeed(12345); - d.setRngStates(null); - d.setMask(null); - - INDArray outNew = Nd4j.createUninitialized(shape); - d.applyDropout(in, outNew, pRetain); - - assertEquals(out, outNew); - } - - //Test backprop: - INDArray gradAtOut = Nd4j.ones(shape); - INDArray gradAtInput = Nd4j.createUninitialized(shape); - d.backprop(gradAtOut, gradAtInput); - Nd4j.getExecutioner().commit(); - - //If dropped: expect 0. Otherwise: expect 1/pRetain, i.e., output for 1s input - assertEquals(out, gradAtInput); - } - } - -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/lstm/ValidateCudnnLSTM.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/lstm/ValidateCudnnLSTM.java deleted file mode 100644 index d4d450bc9..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/lstm/ValidateCudnnLSTM.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.lstm; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.cuda.TestUtils; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.*; -import org.deeplearning4j.nn.conf.distribution.NormalDistribution; -import org.deeplearning4j.nn.conf.layers.LSTM; -import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; -import org.deeplearning4j.nn.gradient.Gradient; -import org.deeplearning4j.cuda.recurrent.CudnnLSTMHelper; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.concurrency.AffinityManager; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.NoOp; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.lang.reflect.Field; -import java.util.Map; -import java.util.Random; -import java.util.function.Supplier; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Created by Alex on 18/07/2017. - */ -public class ValidateCudnnLSTM extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return 180000L; - } - - @Test - public void validateImplSimple() throws Exception { - - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int inputSize = 3; - int lstmLayerSize = 4; - int timeSeriesLength = 3; - int nOut = 2; - INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength}); - INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - labels.putScalar(i, r.nextInt(nOut), j, 1.0); - } - } - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().inferenceWorkspaceMode(WorkspaceMode.NONE) - .trainingWorkspaceMode(WorkspaceMode.NONE).updater(new NoOp()) - .seed(12345L) - .dist(new NormalDistribution(0, 2)).list() - .layer(0, new LSTM.Builder().nIn(input.size(1)).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(1, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build()) - .build(); - - MultiLayerNetwork mln1 = new MultiLayerNetwork(conf.clone()); - mln1.init(); - - MultiLayerNetwork mln2 = new MultiLayerNetwork(conf.clone()); - mln2.init(); - - - assertEquals(mln1.params(), mln2.params()); - - Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper"); - f.setAccessible(true); - - Layer l0 = mln1.getLayer(0); - f.set(l0, null); - assertNull(f.get(l0)); - - l0 = mln2.getLayer(0); - assertTrue(f.get(l0) instanceof CudnnLSTMHelper); - - - INDArray out1 = mln1.output(input); - INDArray out2 = mln2.output(input); - - assertEquals(out1, out2); - - - mln1.setInput(input); - mln1.setLabels(labels); - - mln2.setInput(input); - mln2.setLabels(labels); - - mln1.computeGradientAndScore(); - mln2.computeGradientAndScore(); - - assertEquals(mln1.score(), mln2.score(), 1e-5); - - Gradient g1 = mln1.gradient(); - Gradient g2 = mln2.gradient(); - - for (Map.Entry entry : g1.gradientForVariable().entrySet()) { - INDArray exp = entry.getValue(); - INDArray act = g2.gradientForVariable().get(entry.getKey()); - - //System.out.println(entry.getKey() + "\t" + exp.equals(act)); - } - - assertEquals(mln1.getFlattenedGradients(), mln2.getFlattenedGradients()); - } - - @Test - public void validateImplMultiLayer() throws Exception { - - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int inputSize = 3; - int lstmLayerSize = 4; - int timeSeriesLength = 3; - int nOut = 2; - INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength}); - INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength); - Random r = new Random(12345); - for (int i = 0; i < minibatch; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - labels.putScalar(i, r.nextInt(nOut), j, 1.0); - } - } - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()) - .dataType(DataType.DOUBLE) - .inferenceWorkspaceMode(WorkspaceMode.NONE).trainingWorkspaceMode(WorkspaceMode.NONE) - .seed(12345L) - .dist(new NormalDistribution(0, 2)).list() - .layer(0, new LSTM.Builder().nIn(input.size(1)).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build()) - .build(); - - MultiLayerNetwork mln1 = new MultiLayerNetwork(conf.clone()); - mln1.init(); - - MultiLayerNetwork mln2 = new MultiLayerNetwork(conf.clone()); - mln2.init(); - - - assertEquals(mln1.params(), mln2.params()); - - Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper"); - f.setAccessible(true); - - Layer l0 = mln1.getLayer(0); - Layer l1 = mln1.getLayer(1); - f.set(l0, null); - f.set(l1, null); - assertNull(f.get(l0)); - assertNull(f.get(l1)); - - l0 = mln2.getLayer(0); - l1 = mln2.getLayer(1); - assertTrue(f.get(l0) instanceof CudnnLSTMHelper); - assertTrue(f.get(l1) instanceof CudnnLSTMHelper); - - - INDArray out1 = mln1.output(input); - INDArray out2 = mln2.output(input); - - assertEquals(out1, out2); - - for (int x = 0; x < 10; x++) { - input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength}); - labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength); - for (int i = 0; i < minibatch; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - labels.putScalar(i, r.nextInt(nOut), j, 1.0); - } - } - - mln1.setInput(input); - mln1.setLabels(labels); - - mln2.setInput(input); - mln2.setLabels(labels); - - mln1.computeGradientAndScore(); - mln2.computeGradientAndScore(); - - assertEquals(mln1.score(), mln2.score(), 1e-5); - - assertEquals(mln1.getFlattenedGradients(), mln2.getFlattenedGradients()); - - mln1.fit(new DataSet(input, labels)); - mln2.fit(new DataSet(input, labels)); - - assertEquals("Iteration: " + x, mln1.params(), (Supplier) mln2.params()); - } - } - - - - @Test - public void validateImplMultiLayerTBPTT() throws Exception { - - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int inputSize = 3; - int lstmLayerSize = 4; - int timeSeriesLength = 23; - int tbpttLength = 5; - int nOut = 2; - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()) - .inferenceWorkspaceMode(WorkspaceMode.NONE).trainingWorkspaceMode(WorkspaceMode.NONE) - .seed(12345L) - .dist(new NormalDistribution(0, 2)).list() - .layer(0, new LSTM.Builder().nIn(inputSize).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build()) - .backpropType(BackpropType.TruncatedBPTT) - .tBPTTLength(tbpttLength).build(); - - MultiLayerNetwork mln1 = new MultiLayerNetwork(conf.clone()); - mln1.init(); - - MultiLayerNetwork mln2 = new MultiLayerNetwork(conf.clone()); - mln2.init(); - - - assertEquals(mln1.params(), mln2.params()); - - Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper"); - f.setAccessible(true); - - Layer l0 = mln1.getLayer(0); - Layer l1 = mln1.getLayer(1); - f.set(l0, null); - f.set(l1, null); - assertNull(f.get(l0)); - assertNull(f.get(l1)); - - l0 = mln2.getLayer(0); - l1 = mln2.getLayer(1); - assertTrue(f.get(l0) instanceof CudnnLSTMHelper); - assertTrue(f.get(l1) instanceof CudnnLSTMHelper); - - Random r = new Random(123456); - for (int x = 0; x < 1; x++) { - INDArray input = Nd4j.rand(new int[] {minibatch, inputSize, timeSeriesLength}); - INDArray labels = Nd4j.zeros(minibatch, nOut, timeSeriesLength); - for (int i = 0; i < minibatch; i++) { - for (int j = 0; j < timeSeriesLength; j++) { - labels.putScalar(i, r.nextInt(nOut), j, 1.0); - } - } - - DataSet ds = new DataSet(input, labels); - mln1.fit(ds); - mln2.fit(ds); - } - - assertEquals(mln1.params(), mln2.params()); - } - - @Test - public void validateImplMultiLayerRnnTimeStep() throws Exception { - - for(WorkspaceMode wsm : new WorkspaceMode[]{WorkspaceMode.NONE, WorkspaceMode.ENABLED}) { - Nd4j.getRandom().setSeed(12345); - int minibatch = 10; - int inputSize = 3; - int lstmLayerSize = 4; - int timeSeriesLength = 3; - int tbpttLength = 5; - int nOut = 2; - - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new NoOp()) - .inferenceWorkspaceMode(WorkspaceMode.NONE).trainingWorkspaceMode(WorkspaceMode.NONE) - .cacheMode(CacheMode.NONE).seed(12345L) - .dist(new NormalDistribution(0, 2)).list() - .layer(0, new LSTM.Builder().nIn(inputSize).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(1, new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize) - .gateActivationFunction(Activation.SIGMOID).activation(Activation.TANH).build()) - .layer(2, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX).nIn(lstmLayerSize).nOut(nOut).build()) - .backpropType(BackpropType.TruncatedBPTT) - .tBPTTLength(tbpttLength).build(); - - MultiLayerNetwork mln1 = new MultiLayerNetwork(conf.clone()); - mln1.init(); - - MultiLayerNetwork mln2 = new MultiLayerNetwork(conf.clone()); - mln2.init(); - - - assertEquals(mln1.params(), mln2.params()); - - Field f = org.deeplearning4j.nn.layers.recurrent.LSTM.class.getDeclaredField("helper"); - f.setAccessible(true); - - Layer l0 = mln1.getLayer(0); - Layer l1 = mln1.getLayer(1); - f.set(l0, null); - f.set(l1, null); - assertNull(f.get(l0)); - assertNull(f.get(l1)); - - l0 = mln2.getLayer(0); - l1 = mln2.getLayer(1); - assertTrue(f.get(l0) instanceof CudnnLSTMHelper); - assertTrue(f.get(l1) instanceof CudnnLSTMHelper); - - Random r = new Random(12345); - for (int x = 0; x < 5; x++) { - INDArray input = Nd4j.rand(new int[]{minibatch, inputSize, timeSeriesLength}); - - INDArray step1 = mln1.rnnTimeStep(input); - INDArray step2 = mln2.rnnTimeStep(input); - - assertEquals("Step: " + x, step1, (Supplier) step2); - } - - assertEquals(mln1.params(), mln2.params()); - - //Also check fit (mainly for workspaces sanity check): - INDArray in = Nd4j.rand(new int[]{minibatch, inputSize, 3 * tbpttLength}); - INDArray label = TestUtils.randomOneHotTimeSeries(minibatch, nOut, 3 * tbpttLength); - for( int i=0; i<3; i++ ){ - mln1.fit(in, label); - mln2.fit(in, label); - } - } - } -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/util/CuDNNValidationUtil.java b/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/util/CuDNNValidationUtil.java deleted file mode 100644 index d96b3e124..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/java/org/deeplearning4j/cuda/util/CuDNNValidationUtil.java +++ /dev/null @@ -1,344 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.cuda.util; - -import it.unimi.dsi.fastutil.doubles.DoubleArrayList; -import lombok.*; -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.conf.layers.ConvolutionLayer; -import org.deeplearning4j.nn.conf.layers.SubsamplingLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.optimize.listeners.CollectScoresListener; -import org.nd4j.common.base.Preconditions; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.BooleanIndexing; -import org.nd4j.linalg.indexing.conditions.Conditions; -import org.nd4j.linalg.ops.transforms.Transforms; - -import java.lang.reflect.Field; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; - -@Slf4j -public class CuDNNValidationUtil { - - public static final double MAX_REL_ERROR = 1e-4; -// public static final double MAX_REL_ERROR = 1e-3; -// public static final double MIN_ABS_ERROR = 1e-3; - public static final double MIN_ABS_ERROR = 1e-5; - - @AllArgsConstructor - @NoArgsConstructor - @Data - @Builder - public static class TestCase { - private String testName; - private List> allowCudnnHelpersForClasses; - @Builder.Default private boolean testForward = true; - @Builder.Default private boolean testScore = true; - @Builder.Default private boolean testBackward = true; - @Builder.Default private boolean testTraining = true; - @Builder.Default private boolean trainFirst = false; - @Builder.Default private double maxRE = MAX_REL_ERROR; - @Builder.Default private double minAbsErr = MIN_ABS_ERROR; - INDArray features; - INDArray labels; - private DataSetIterator data; - } - - public static void validateMLN(MultiLayerNetwork netOrig, TestCase t){ - assertNotNull(t.getAllowCudnnHelpersForClasses()); - assertFalse(t.getAllowCudnnHelpersForClasses().isEmpty()); - - //Don't allow fallback: - for(Layer l : netOrig.getLayers()){ - org.deeplearning4j.nn.conf.layers.Layer lConf = l.conf().getLayer(); - if(lConf instanceof ConvolutionLayer){ - ((ConvolutionLayer) lConf).setCudnnAllowFallback(false); - } else if(lConf instanceof SubsamplingLayer){ - ((SubsamplingLayer) lConf).setCudnnAllowFallback(false); - } - } - - - MultiLayerNetwork net1NoCudnn = new MultiLayerNetwork(netOrig.getLayerWiseConfigurations().clone()); - net1NoCudnn.init(); - log.info("Removing all CuDNN helpers from network copy 1"); - removeHelpers(net1NoCudnn.getLayers(), null); - - MultiLayerNetwork net2With = new MultiLayerNetwork(netOrig.getLayerWiseConfigurations().clone()); - net2With.init(); - net2With.params().assign(netOrig.params()); - log.info("Removing all except for specified CuDNN helpers from network copy 2: " + t.getAllowCudnnHelpersForClasses()); - removeHelpers(net2With.getLayers(), t.getAllowCudnnHelpersForClasses()); - - - - - if(t.isTrainFirst()){ - Preconditions.checkState(t.getData() != null, "Test data iterator is "); - log.info("Validation - training first..."); - log.info("*** NOT YET IMPLEMENTED***"); - - - } - - if(t.isTestForward()){ - Preconditions.checkNotNull(t.getFeatures(), "Features are not set (null)"); - - for (boolean train : new boolean[]{false, true}) { - assertEquals(net1NoCudnn.params(), net2With.params()); - String s = "Feed forward test - " + t.getTestName() + " - " + (train ? "Train: " : "Test: "); - List ff1 = net1NoCudnn.feedForward(t.getFeatures(), train); - List ff2 = net2With.feedForward(t.getFeatures(), train); - List paramKeys = new ArrayList<>(net1NoCudnn.paramTable().keySet()); - Collections.sort(paramKeys); - for (String p : paramKeys) { - INDArray p1 = net1NoCudnn.getParam(p); - INDArray p2 = net2With.getParam(p); - INDArray re = relError(p1, p2, t.minAbsErr); - double maxRE = re.maxNumber().doubleValue(); - if (maxRE >= t.maxRE) { - System.out.println("Failed param values: parameter " + p + " - No CuDNN vs. with CuDNN - train=" + train); - System.out.println(p1); - System.out.println(p2); - } - assertTrue(maxRE < t.maxRE, s + " - param changed during forward pass: " + p); - } - - for( int i=0; i= t.maxRE){ - double d1 = arr1.dup('c').getDouble(idx); - double d2 = arr2.dup('c').getDouble(idx); - System.out.println("Different values at index " + idx + ": " + d1 + ", " + d2 + " - RE = " + maxRE); - } - assertTrue(maxRE < t.maxRE, s + layerName + " - max RE: " + maxRE); - log.info("Forward pass, max relative error: " + layerName + " - " + maxRE); - } - - INDArray out1 = net1NoCudnn.output(t.getFeatures(), train); - INDArray out2 = net2With.output(t.getFeatures(), train); - INDArray relError = relError(out1, out2, t.minAbsErr); - double maxRE = relError.maxNumber().doubleValue(); - log.info(s + "Output, max relative error: " + maxRE); - - assertEquals(net1NoCudnn.params(), net2With.params()); //Check that forward pass does not modify params - assertTrue(maxRE < t.maxRE, s + "Max RE: " + maxRE); - } - } - - - if(t.isTestScore()) { - Preconditions.checkNotNull(t.getFeatures(), "Features are not set (null)"); - Preconditions.checkNotNull(t.getLabels(), "Labels are not set (null)"); - - log.info("Validation - checking scores"); - double s1 = net1NoCudnn.score(new DataSet(t.getFeatures(), t.getLabels())); - double s2 = net2With.score(new DataSet(t.getFeatures(), t.getLabels())); - - double re = relError(s1, s2); - String s = "Relative error: " + re; - assertTrue(re < t.maxRE, s); - } - - if(t.isTestBackward()) { - Preconditions.checkNotNull(t.getFeatures(), "Features are not set (null)"); - Preconditions.checkNotNull(t.getLabels(), "Labels are not set (null)"); - log.info("Validation - checking backward pass"); - - //Check gradients - net1NoCudnn.setInput(t.getFeatures()); - net1NoCudnn.setLabels(t.getLabels()); - - net2With.setInput(t.getFeatures()); - net2With.setLabels(t.getLabels()); - - net1NoCudnn.computeGradientAndScore(); - net2With.computeGradientAndScore(); - - List paramKeys = new ArrayList<>(net1NoCudnn.paramTable().keySet()); - Collections.sort(paramKeys); - for(String p : paramKeys){ - INDArray g1 = net1NoCudnn.gradient().gradientForVariable().get(p); - INDArray g2 = net2With.gradient().gradientForVariable().get(p); - - if(g1 == null || g2 == null){ - throw new RuntimeException("Null gradients"); - } - - INDArray re = relError(g1, g2, t.minAbsErr); - double maxRE = re.maxNumber().doubleValue(); - if (maxRE >= t.maxRE) { - System.out.println("Failed param values: no CuDNN vs. with CuDNN - parameter: " + p); - System.out.println(Arrays.toString(g1.dup().data().asFloat())); - System.out.println(Arrays.toString(g2.dup().data().asFloat())); - } else { - System.out.println("OK: " + p); - } - assertTrue(maxRE < t.maxRE, "Gradients are not equal: " + p + ": maxRE=" + maxRE); - } - } - - if(t.isTestTraining()){ - Preconditions.checkNotNull(t.getData(), "DataSetIterator is not set (null)"); - log.info("Testing run-to-run consistency of training with CuDNN"); - - net2With = new MultiLayerNetwork(netOrig.getLayerWiseConfigurations().clone()); - net2With.init(); - net2With.params().assign(netOrig.params()); - log.info("Removing all except for specified CuDNN helpers from network copy 2: " + t.getAllowCudnnHelpersForClasses()); - removeHelpers(net2With.getLayers(), t.getAllowCudnnHelpersForClasses()); - - CollectScoresListener listener = new CollectScoresListener(1); - net2With.setListeners(listener); - net2With.fit(t.getData()); - - for( int i=0; i<2; i++ ) { - - net2With = new MultiLayerNetwork(netOrig.getLayerWiseConfigurations().clone()); - net2With.init(); - net2With.params().assign(netOrig.params()); - log.info("Removing all except for specified CuDNN helpers from network copy 2: " + t.getAllowCudnnHelpersForClasses()); - removeHelpers(net2With.getLayers(), t.getAllowCudnnHelpersForClasses()); - - CollectScoresListener listener2 = new CollectScoresListener(1); - net2With.setListeners(listener2); - net2With.fit(t.getData()); - - DoubleArrayList listOrig = listener.getListScore(); - DoubleArrayList listNew = listener2.getListScore(); - - assertEquals(listOrig.size(), listNew.size()); - for (int j = 0; j < listOrig.size(); j++) { - double d1 = listOrig.get(j); - double d2 = listNew.get(j); - double re = relError(d1, d2); - String msg = "Scores at iteration " + j + " - relError = " + re + ", score1 = " + d1 + ", score2 = " + d2; - assertTrue(re < t.maxRE, msg); - System.out.println("j=" + j + ", d1 = " + d1 + ", d2 = " + d2); - } - } - } - } - - private static void removeHelpers(Layer[] layers, List> keepHelpersFor){ - - Map, Integer> map = new HashMap<>(); - for(Layer l : layers){ - Field f; - try{ - f = l.getClass().getDeclaredField("helper"); - } catch (Exception e){ - //OK, may not be a CuDNN supported layer - continue; - } - - f.setAccessible(true); - boolean keepAndAssertPresent = false; - if(keepHelpersFor != null) { - for (Class c : keepHelpersFor) { - if(c.isAssignableFrom(l.getClass())){ - keepAndAssertPresent = true; - break; - } - } - } - try { - if (keepAndAssertPresent) { - Object o = f.get(l); - assertNotNull(o); - } else { - f.set(l, null); - Integer i = map.get(l.getClass()); - if(i == null){ - i = 0; - } - map.put(l.getClass(), i+1); - } - } catch (IllegalAccessException e){ - throw new RuntimeException(e); - } - } - - for(Map.Entry,Integer> c : map.entrySet()){ - System.out.println("Removed " + c.getValue() + " CuDNN helpers instances from layer " + c.getKey()); - } - } - - private static double relError(double d1, double d2){ - Preconditions.checkState(!Double.isNaN(d1), "d1 is NaN"); - Preconditions.checkState(!Double.isNaN(d2), "d2 is NaN"); - if(d1 == 0.0 && d2 == 0.0){ - return 0.0; - } - - return Math.abs(d1-d2) / (Math.abs(d1) + Math.abs(d2)); - } - - private static INDArray relError(@NonNull INDArray a1, @NonNull INDArray a2, double minAbsError){ - long numNaN1 = Nd4j.getExecutioner().exec(new MatchCondition(a1, Conditions.isNan(), Integer.MAX_VALUE)).getInt(0); - long numNaN2 = Nd4j.getExecutioner().exec(new MatchCondition(a2, Conditions.isNan(), Integer.MAX_VALUE)).getInt(0); - Preconditions.checkState(numNaN1 == 0, "Array 1 has NaNs"); - Preconditions.checkState(numNaN2 == 0, "Array 2 has NaNs"); - - -// INDArray isZero1 = a1.eq(0.0); -// INDArray isZero2 = a2.eq(0.0); -// INDArray bothZero = isZero1.muli(isZero2); - - INDArray abs1 = Transforms.abs(a1, true); - INDArray abs2 = Transforms.abs(a2, true); - INDArray absDiff = Transforms.abs(a1.sub(a2), false); - - //abs(a1-a2) < minAbsError ? 1 : 0 - INDArray greaterThanMinAbs = Transforms.abs(a1.sub(a2), false); - BooleanIndexing.replaceWhere(greaterThanMinAbs, 0.0, Conditions.lessThan(minAbsError)); - BooleanIndexing.replaceWhere(greaterThanMinAbs, 1.0, Conditions.greaterThan(0.0)); - - INDArray result = absDiff.divi(abs1.add(abs2)); - //Only way to have NaNs given there weren't any in original : both 0s - BooleanIndexing.replaceWhere(result, 0.0, Conditions.isNan()); - //Finally, set to 0 if less than min abs error, or unchanged otherwise - result.muli(greaterThanMinAbs); - -// double maxRE = result.maxNumber().doubleValue(); -// if(maxRE > t.maxRe){ -// System.out.println(); -// } - return result; - } - -} diff --git a/deeplearning4j/deeplearning4j-cuda/src/test/resources/logback.xml b/deeplearning4j/deeplearning4j-cuda/src/test/resources/logback.xml deleted file mode 100644 index 6be67561e..000000000 --- a/deeplearning4j/deeplearning4j-cuda/src/test/resources/logback.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - logs/application.log - - %logger{15} - %message%n%xException{5} - - - - - - - %logger{15} - %message%n%xException{5} - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/pom.xml b/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/pom.xml deleted file mode 100644 index 45ee5100b..000000000 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-datasets/pom.xml +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-data - 1.0.0-SNAPSHOT - - - deeplearning4j-datasets - jar - - deeplearning4j-datasets - - - - org.datavec - datavec-data-image - ${datavec.version} - - - org.deeplearning4j - deeplearning4j-datavec-iterators - ${project.version} - - - org.deeplearning4j - deeplearning4j-common - ${project.version} - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/pom.xml b/deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/pom.xml deleted file mode 100644 index 748a10c50..000000000 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-datavec-iterators/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-data - 1.0.0-SNAPSHOT - - - deeplearning4j-datavec-iterators - jar - - deeplearning4j-datavec-iterators - - - - org.datavec - datavec-api - ${datavec.version} - - - org.nd4j - nd4j-api - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml b/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml deleted file mode 100644 index 10ce9a8ce..000000000 --- a/deeplearning4j/deeplearning4j-data/deeplearning4j-utility-iterators/pom.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-data - 1.0.0-SNAPSHOT - - - deeplearning4j-utility-iterators - jar - - deeplearning4j-utility-iterators - - - - org.nd4j - nd4j-api - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-data/pom.xml b/deeplearning4j/deeplearning4j-data/pom.xml deleted file mode 100644 index d7033d3ad..000000000 --- a/deeplearning4j/deeplearning4j-data/pom.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - 4.0.0 - - - deeplearning4j-parent - org.deeplearning4j - 1.0.0-SNAPSHOT - - - deeplearning4j-data - pom - - deeplearning4j-data - - - deeplearning4j-datavec-iterators - deeplearning4j-datasets - deeplearning4j-utility-iterators - - - - - - org.nd4j - nd4j-api - ${nd4j.version} - - - - - - - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - - - diff --git a/deeplearning4j/deeplearning4j-dataimport-solrj/pom.xml b/deeplearning4j/deeplearning4j-dataimport-solrj/pom.xml index 899a81a17..3383a8a83 100644 --- a/deeplearning4j/deeplearning4j-dataimport-solrj/pom.xml +++ b/deeplearning4j/deeplearning4j-dataimport-solrj/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - org.deeplearning4j + net.brutex.ai deeplearning4j-parent 1.0.0-SNAPSHOT @@ -45,12 +45,8 @@ org.apache.maven.plugins maven-surefire-plugin - ${cpu.core.count} - false - false - - -Ddtype=float -Dfile.encoding=UTF-8 - -Dtest.solr.allowed.securerandom=NativePRNG -Xmx${test.heap.size} -Dorg.bytedeco.javacpp.maxphysicalbytes=${test.offheap.size} -Dorg.bytedeco.javacpp.maxbytes=${test.offheap.size} + -Ddtype=float -Dfile.encoding=UTF-8 -Xmx8g + -Dtest.solr.allowed.securerandom=NativePRNG @@ -74,26 +70,14 @@ slf4j-api - org.nd4j + net.brutex.ai nd4j-api - ${nd4j.version} - - - org.deeplearning4j - deeplearning4j-nn ${project.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test + net.brutex.ai + deeplearning4j-nn + ${project.version} org.apache.solr @@ -113,27 +97,26 @@ test + - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - + test-nd4j-native - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} + net.brutex.ai + nd4j-native + ${project.version} test + + + + test-nd4j-cuda-11.2 + - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} + net.brutex.ai + nd4j-cuda-${cuda.version} + ${project.version} test diff --git a/deeplearning4j/deeplearning4j-dataimport-solrj/src/test/java/org/deeplearning4j/nn/dataimport/solr/client/solrj/io/stream/TupleStreamDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-dataimport-solrj/src/test/java/org/deeplearning4j/nn/dataimport/solr/client/solrj/io/stream/TupleStreamDataSetIteratorTest.java index 3d773d8b8..5c21d354a 100644 --- a/deeplearning4j/deeplearning4j-dataimport-solrj/src/test/java/org/deeplearning4j/nn/dataimport/solr/client/solrj/io/stream/TupleStreamDataSetIteratorTest.java +++ b/deeplearning4j/deeplearning4j-dataimport-solrj/src/test/java/org/deeplearning4j/nn/dataimport/solr/client/solrj/io/stream/TupleStreamDataSetIteratorTest.java @@ -17,10 +17,12 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.dataimport.solr.client.solrj.io.stream; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.carrotsearch.randomizedtesting.ThreadFilter; + import java.security.SecureRandom; import java.util.ArrayList; import java.util.Collections; @@ -35,160 +37,214 @@ import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.junit.jupiter.api.*; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction; import org.nd4j.linalg.api.memory.provider.BasicWorkspaceManager; import org.nd4j.rng.deallocator.NativeRandomDeallocator; -import org.junit.jupiter.api.extension.ExtendWith; -@ThreadLeakFilters(defaultFilters = true, filters = { TupleStreamDataSetIteratorTest.PrivateDeallocatorThreadsFilter.class }) -@DisplayName("Tuple Stream Data Set Iterator Test") -@Tag(TagNames.SOLR) -@Tag(TagNames.DIST_SYSTEMS) -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -@Disabled("Permissions issue") -@Tag(TagNames.NEEDS_VERIFY) -class TupleStreamDataSetIteratorTest extends SolrCloudTestCase { +@ThreadLeakFilters(defaultFilters = true, filters = { + TupleStreamDataSetIteratorTest.PrivateDeallocatorThreadsFilter.class +}) +public class TupleStreamDataSetIteratorTest extends SolrCloudTestCase { - static { - /* + static { + /* This is a hack around the backend-dependent nature of secure random implementations though we can set the secure random algorithm in our pom.xml files (via maven surefire and test.solr.allowed.securerandom) there isn't a mechanism that is completely platform independent. By setting it there (for example, to NativePRNG) that makes it pass on some platforms like Linux but fails on some JVMs on Windows For testing purposes, we don't need strict guarantees around RNG, hence we don't want to enforce the RNG algorithm */ - String algorithm = new SecureRandom().getAlgorithm(); - System.setProperty("test.solr.allowed.securerandom", algorithm); + String algorithm = new SecureRandom().getAlgorithm(); + System.setProperty("test.solr.allowed.securerandom", algorithm); + } + + public static class PrivateDeallocatorThreadsFilter implements ThreadFilter { + /** + * Reject deallocator threads over whose cleanup this test has no control. + */ + @Override + public boolean reject(Thread thread) { + final ThreadGroup threadGroup = thread.getThreadGroup(); + final String threadGroupName = (threadGroup == null ? null : threadGroup.getName()); + + if (threadGroupName != null && + threadGroupName.endsWith(TupleStreamDataSetIteratorTest.class.getSimpleName())) { + + final String threadName = thread.getName(); + if (threadName.startsWith(NativeRandomDeallocator.DeallocatorThreadNamePrefix) || + threadName.toLowerCase().contains("deallocator") || + threadName.equals(BasicWorkspaceManager.WorkspaceDeallocatorThreadName)) { + return true; + } + } + + return false; } + } - @DisplayName("Private Deallocator Threads Filter") - static class PrivateDeallocatorThreadsFilter implements ThreadFilter { + private static int numDocs = 0; - /** - * Reject deallocator threads over whose cleanup this test has no control. - */ - @Override - public boolean reject(Thread thread) { - final ThreadGroup threadGroup = thread.getThreadGroup(); - final String threadGroupName = (threadGroup == null ? null : threadGroup.getName()); - if (threadGroupName != null && threadGroupName.endsWith(TupleStreamDataSetIteratorTest.class.getSimpleName())) { - final String threadName = thread.getName(); - if (threadName.startsWith(NativeRandomDeallocator.DeallocatorThreadNamePrefix) || threadName.toLowerCase().contains("deallocator") || threadName.equals(BasicWorkspaceManager.WorkspaceDeallocatorThreadName)) { - return true; - } + @BeforeAll + public static void setupCluster() throws Exception { + + final int numShards = 2; + final int numReplicas = 2; + final int maxShardsPerNode = 1; + final int nodeCount = (numShards*numReplicas + (maxShardsPerNode-1))/maxShardsPerNode; + + // create and configure cluster + configureCluster(nodeCount) + .addConfig("conf", configset("mini")) + .configure(); + + // create an empty collection + CollectionAdminRequest.createCollection("mySolrCollection", "conf", numShards, numReplicas) + .setMaxShardsPerNode(maxShardsPerNode) + .process(cluster.getSolrClient()); + + // compose an update request + final UpdateRequest updateRequest = new UpdateRequest(); + + final List docIds = new ArrayList(); + for (int phase = 1; phase <= 2; ++phase) { + int docIdsIdx = 0; + + if (phase == 2) { + Collections.shuffle(docIds); + } + + final int increment = 32; + + for (int b = 0; b <= 256; b += increment) { + if (256 == b) b--; + for (int g = 0; g <= 256; g += increment) { + if (256 == g) g--; + for (int r = 0; r <= 256; r += increment) { + if (256 == r) r--; + + if (phase == 1) { + docIds.add(docIds.size()+1); + continue; } - return false; + + final float luminance = (b*0.0722f + g*0.7152f + r*0.2126f)/(255*3.0f); // https://en.wikipedia.org/wiki/Luma_(video) + + final SolrInputDocument doc = sdoc("id", Integer.toString(docIds.get(docIdsIdx++)), + "channel_b_f", Float.toString(b/255f), + "channel_g_f", Float.toString(g/255f), + "channel_r_f", Float.toString(r/255f), + "luminance_f", Float.toString(luminance)); + + updateRequest.add(doc); + ++numDocs; + + } } + } } - private static int numDocs = 0; + // make the update request + updateRequest.commit(cluster.getSolrClient(), "mySolrCollection"); + } - @BeforeAll - static void setupCluster() throws Exception { - final int numShards = 2; - final int numReplicas = 2; - final int maxShardsPerNode = 1; - final int nodeCount = (numShards * numReplicas + (maxShardsPerNode - 1)) / maxShardsPerNode; - // create and configure cluster - configureCluster(nodeCount).addConfig("conf", configset("mini")).configure(); - // create an empty collection - CollectionAdminRequest.createCollection("mySolrCollection", "conf", numShards, numReplicas).setMaxShardsPerNode(maxShardsPerNode).process(cluster.getSolrClient()); - // compose an update request - final UpdateRequest updateRequest = new UpdateRequest(); - final List docIds = new ArrayList<>(); - for (int phase = 1; phase <= 2; ++phase) { - int docIdsIdx = 0; - if (phase == 2) { - Collections.shuffle(docIds); - } - final int increment = 32; - for (int b = 0; b <= 256; b += increment) { - if (256 == b) - b--; - for (int g = 0; g <= 256; g += increment) { - if (256 == g) - g--; - for (int r = 0; r <= 256; r += increment) { - if (256 == r) - r--; - if (phase == 1) { - docIds.add(docIds.size() + 1); - continue; - } - // https://en.wikipedia.org/wiki/Luma_(video) - final float luminance = (b * 0.0722f + g * 0.7152f + r * 0.2126f) / (255 * 3.0f); - final SolrInputDocument doc = sdoc("id", Integer.toString(docIds.get(docIdsIdx++)), "channel_b_f", Float.toString(b / 255f), "channel_g_f", Float.toString(g / 255f), "channel_r_f", Float.toString(r / 255f), "luminance_f", Float.toString(luminance)); - updateRequest.add(doc); - ++numDocs; - } - } - } - } - // make the update request - updateRequest.commit(cluster.getSolrClient(), "mySolrCollection"); + private static class CountingIterationListener extends ScoreIterationListener { + + private int numIterationsDone = 0; + + public CountingIterationListener() { + super(1); } - @DisplayName("Counting Iteration Listener") - private static class CountingIterationListener extends ScoreIterationListener { - - private int numIterationsDone = 0; - - public CountingIterationListener() { - super(1); - } - - public int numIterationsDone() { - return numIterationsDone; - } - - @Override - public void iterationDone(Model model, int iteration, int epoch) { - super.iterationDone(model, iteration, epoch); - ++numIterationsDone; - } + public int numIterationsDone() { + return numIterationsDone; } - @Test - @DisplayName("Iterate Test") - void iterateTest() throws Exception { - doIterateTest(true); - doIterateTest(false); + @Override + public void iterationDone(Model model, int iteration, int epoch) { + super.iterationDone(model, iteration, epoch); + ++numIterationsDone; } - private void doIterateTest(boolean withIdKey) throws Exception { - try (final TupleStreamDataSetIterator tsdsi = new TupleStreamDataSetIterator(123, /* batch */ - (withIdKey ? "greeting" : null), /* idKey */ - new String[] { "pie" }, new String[] { "answer" }, "tuple(greeting=\"hello world\",pie=3.14,answer=42)", null)) { - assertTrue(tsdsi.hasNext()); - final DataSet ds = tsdsi.next(); - assertEquals(1, ds.getFeatures().length()); - assertEquals(3.14f, ds.getFeatures().getFloat(0), 0.0f); - assertEquals(1, ds.getLabels().length()); - assertEquals(42f, ds.getLabels().getFloat(0), 0.0f); - assertFalse(tsdsi.hasNext()); - } - } + } - @Test - @DisplayName("Model Fit Test") - void modelFitTest() throws Exception { - final MultiLayerNetwork model = new MultiLayerNetwork(new NeuralNetConfiguration.Builder().list(new OutputLayer.Builder(LossFunction.MSE).nIn(3).nOut(1).weightInit(WeightInit.ONES).activation(Activation.IDENTITY).build()).build()); - model.init(); - int batch = 1; - for (int ii = 1; ii <= 5; ++ii) { - final CountingIterationListener listener = new CountingIterationListener(); - model.setListeners(listener); - batch *= 2; - try (final TupleStreamDataSetIterator tsdsi = new TupleStreamDataSetIterator(batch, "id", /* idKey */ - new String[] { "channel_b_f", "channel_g_f", "channel_r_f" }, new String[] { "luminance_f" }, "search(mySolrCollection," + "q=\"id:*\"," + "fl=\"id,channel_b_f,channel_g_f,channel_r_f,luminance_f\"," + "sort=\"id asc\"," + "qt=\"/export\")", cluster.getZkClient().getZkServerAddress())) { - model.fit(tsdsi); - } - assertEquals("numIterationsDone=" + listener.numIterationsDone() + " numDocs=" + numDocs + " batch=" + batch, (numDocs + (batch - 1)) / batch, listener.numIterationsDone()); - } + @Test + public void iterateTest() throws Exception { + doIterateTest(true); + doIterateTest(false); + } + + private void doIterateTest(boolean withIdKey) throws Exception { + + try (final TupleStreamDataSetIterator + tsdsi = new TupleStreamDataSetIterator( + 123 /* batch */, + (withIdKey ? "greeting" : null) /* idKey */, + new String[] { "pie" }, + new String[] { "answer" }, + "tuple(greeting=\"hello world\",pie=3.14,answer=42)", + null)) { + + assertTrue(tsdsi.hasNext()); + final DataSet ds = tsdsi.next(); + + assertEquals(1, ds.getFeatures().length()); + assertEquals(3.14f, ds.getFeatures().getFloat(0), 0.0f); + + assertEquals(1, ds.getLabels().length()); + assertEquals(42f, ds.getLabels().getFloat(0), 0.0f); + + assertFalse(tsdsi.hasNext()); } + } + + @Test + public void modelFitTest() throws Exception { + + final MultiLayerNetwork model = new MultiLayerNetwork( + new NeuralNetConfiguration.Builder() + .list( + new OutputLayer.Builder(LossFunction.MSE) + .nIn(3) + .nOut(1) + .weightInit(WeightInit.ONES) + .activation(Activation.IDENTITY) + .build() + ) + + + .build() + ); + model.init(); + + int batch = 1; + for (int ii=1; ii<=5; ++ii) { + final CountingIterationListener listener = new CountingIterationListener(); + model.setListeners(listener); + batch *= 2; + + try (final TupleStreamDataSetIterator tsdsi = + new TupleStreamDataSetIterator( + batch, + "id" /* idKey */, + new String[] { "channel_b_f", "channel_g_f", "channel_r_f" }, + new String[] { "luminance_f" }, + "search(mySolrCollection," + + "q=\"id:*\"," + + "fl=\"id,channel_b_f,channel_g_f,channel_r_f,luminance_f\"," + + "sort=\"id asc\"," + + "qt=\"/export\")", + cluster.getZkClient().getZkServerAddress())) { + + model.fit(tsdsi); + } + + assertEquals("numIterationsDone="+listener.numIterationsDone()+" numDocs="+numDocs+" batch="+batch, + (numDocs+(batch-1))/batch, listener.numIterationsDone()); + } + } + } diff --git a/deeplearning4j/deeplearning4j-graph/pom.xml b/deeplearning4j/deeplearning4j-graph/pom.xml index 8b14d9916..db0997f91 100644 --- a/deeplearning4j/deeplearning4j-graph/pom.xml +++ b/deeplearning4j/deeplearning4j-graph/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - org.deeplearning4j + net.brutex.ai deeplearning4j-scaleout 1.0.0-SNAPSHOT ../deeplearning4j-scaleout/pom.xml @@ -36,7 +36,7 @@ - org.deeplearning4j + net.brutex.ai deeplearning4j-core ${project.version} @@ -44,18 +44,6 @@ org.threadly threadly ${threadly.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test ch.qos.logback @@ -63,37 +51,10 @@ test - org.deeplearning4j + net.brutex.ai deeplearning4j-common-tests ${project.version} test - - - - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - - diff --git a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/data/TestGraphLoading.java b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/data/TestGraphLoading.java index 6f2922f7a..5fe2f4cc0 100644 --- a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/data/TestGraphLoading.java +++ b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/data/TestGraphLoading.java @@ -29,25 +29,19 @@ import org.deeplearning4j.graph.data.impl.DelimitedVertexLoader; import org.deeplearning4j.graph.graph.Graph; import org.deeplearning4j.graph.vertexfactory.StringVertexFactory; import org.deeplearning4j.graph.vertexfactory.VertexFactory; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; import java.io.IOException; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -@Disabled("Permissions issues on CI") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) +@Timeout(10) public class TestGraphLoading extends BaseDL4JTest { - @Test() - @Timeout(10000) + @Test public void testEdgeListGraphLoading() throws IOException { ClassPathResource cpr = new ClassPathResource("deeplearning4j-graph/testgraph_7vertices.txt"); @@ -67,8 +61,7 @@ public class TestGraphLoading extends BaseDL4JTest { } } - @Test() - @Timeout(10000) + @Test public void testGraphLoading() throws IOException { ClassPathResource cpr = new ClassPathResource("deeplearning4j-graph/simplegraph.txt"); @@ -111,8 +104,7 @@ public class TestGraphLoading extends BaseDL4JTest { } } - @Test() - @Timeout(10000) + @Test public void testGraphLoadingWithVertices() throws IOException { ClassPathResource verticesCPR = new ClassPathResource("deeplearning4j-graph/test_graph_vertices.txt"); diff --git a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/data/TestGraphLoadingWeighted.java b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/data/TestGraphLoadingWeighted.java index d5f9f2cc1..78bf78ee1 100644 --- a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/data/TestGraphLoadingWeighted.java +++ b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/data/TestGraphLoadingWeighted.java @@ -28,26 +28,20 @@ import org.deeplearning4j.graph.data.impl.WeightedEdgeLineProcessor; import org.deeplearning4j.graph.graph.Graph; import org.deeplearning4j.graph.vertexfactory.StringVertexFactory; import org.deeplearning4j.graph.vertexfactory.VertexFactory; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.TagNames; import java.io.IOException; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; -@Disabled("Permissions issues on CI") -@Tag(TagNames.JAVA_ONLY) -@Tag(TagNames.FILE_IO) +@Timeout(10) public class TestGraphLoadingWeighted extends BaseDL4JTest { - @Test() - @Timeout(10000) + @Test public void testWeightedDirected() throws IOException { String path = new ClassPathResource("deeplearning4j-graph/WeightedGraph.txt").getTempFileFromArchive().getAbsolutePath(); @@ -87,8 +81,7 @@ public class TestGraphLoadingWeighted extends BaseDL4JTest { } - @Test() - @Timeout(10000) + @Test public void testWeightedDirectedV2() throws Exception { String path = new ClassPathResource("deeplearning4j-graph/WeightedGraph.txt").getTempFileFromArchive().getAbsolutePath(); diff --git a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/graph/TestGraph.java b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/graph/TestGraph.java index 0cabea99a..e4f1af7a2 100644 --- a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/graph/TestGraph.java +++ b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/graph/TestGraph.java @@ -27,25 +27,22 @@ import org.deeplearning4j.graph.data.GraphLoader; import org.deeplearning4j.graph.iterator.RandomWalkIterator; import org.deeplearning4j.graph.iterator.WeightedRandomWalkIterator; import org.deeplearning4j.graph.vertexfactory.VertexFactory; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.HashSet; import java.util.List; import java.util.Set; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.FILE_IO) + public class TestGraph extends BaseDL4JTest { - @Test() - @Timeout(10000) + @Test + @Timeout(10) public void testSimpleGraph() { Graph graph = new Graph<>(10, false, new VFactory()); @@ -99,8 +96,8 @@ public class TestGraph extends BaseDL4JTest { } - @Test() - @Timeout(10000) + @Test + @Timeout(10) public void testRandomWalkIterator() { Graph graph = new Graph<>(10, false, new VFactory()); assertEquals(10, graph.numVertices()); @@ -143,8 +140,8 @@ public class TestGraph extends BaseDL4JTest { assertEquals(10, startIdxSet.size()); } - @Test() - @Timeout(10000) + @Test + @Timeout(10) public void testWeightedRandomWalkIterator() throws Exception { //Load a directed, weighted graph from file diff --git a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/DeepWalkGradientCheck.java b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/DeepWalkGradientCheck.java index 7d88ee06a..c18775e1c 100644 --- a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/DeepWalkGradientCheck.java +++ b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/DeepWalkGradientCheck.java @@ -26,9 +26,9 @@ import org.deeplearning4j.graph.graph.Graph; import org.deeplearning4j.graph.iterator.GraphWalkIterator; import org.deeplearning4j.graph.iterator.RandomWalkIterator; import org.deeplearning4j.graph.models.embeddings.InMemoryGraphLookupTable; -import org.junit.jupiter.api.*; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -38,9 +38,7 @@ import java.io.IOException; import static org.junit.jupiter.api.Assertions.*; -@Disabled("Permissions issues on CI") -@NativeTag -@Tag(TagNames.FILE_IO) +@Timeout(10) public class DeepWalkGradientCheck extends BaseDL4JTest { public static final double epsilon = 1e-8; @@ -52,8 +50,7 @@ public class DeepWalkGradientCheck extends BaseDL4JTest { Nd4j.setDataType(DataType.DOUBLE); } - @Test() - @Timeout(10000) + @Test public void checkGradients() throws IOException { ClassPathResource cpr = new ClassPathResource("deeplearning4j-graph/testgraph_7vertices.txt"); @@ -200,8 +197,7 @@ public class DeepWalkGradientCheck extends BaseDL4JTest { - @Test() - @Timeout(60000) + @Test @Timeout(60) public void checkGradients2() throws IOException { double minAbsError = 1e-5; diff --git a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/TestDeepWalk.java b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/TestDeepWalk.java index fdf0e9119..d1b1bccc9 100644 --- a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/TestDeepWalk.java +++ b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/TestDeepWalk.java @@ -33,38 +33,32 @@ import org.deeplearning4j.graph.models.GraphVectors; import org.deeplearning4j.graph.models.loader.GraphVectorSerializer; import org.deeplearning4j.graph.vertexfactory.StringVertexFactory; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.io.ClassPathResource; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.Random; import static org.junit.jupiter.api.Assertions.*; -@Disabled("Permissions issues on CI") -@NativeTag -@Tag(TagNames.FILE_IO) +@Timeout(200) public class TestDeepWalk extends BaseDL4JTest { + @TempDir + public File testDir; @Override public long getTimeoutMilliseconds() { return 120_000L; //Increase timeout due to intermittently slow CI machines } - @Test() - @Timeout(60000) + @Test public void testBasic() throws IOException { //Very basic test. Load graph, build tree, call fit, make sure it doesn't throw any exceptions @@ -102,8 +96,7 @@ public class TestDeepWalk extends BaseDL4JTest { } } - @Test() - @Timeout(180000) + @Test public void testParallel() { IGraph graph = generateRandomGraph(30, 4); @@ -137,8 +130,7 @@ public class TestDeepWalk extends BaseDL4JTest { } - @Test() - @Timeout(60000) + @Test public void testVerticesNearest() { int nVertices = 20; @@ -183,9 +175,8 @@ public class TestDeepWalk extends BaseDL4JTest { } } - @Test() - @Timeout(60000) - public void testLoadingSaving(@TempDir Path testDir) throws IOException { + @Test + public void testLoadingSaving() throws IOException { String out = "dl4jdwtestout.txt"; int nVertices = 20; @@ -199,7 +190,7 @@ public class TestDeepWalk extends BaseDL4JTest { deepWalk.fit(graph, 10); - File f = new File(testDir.toFile(),out); + File f = new File (testDir, out); GraphVectorSerializer.writeGraphVectors(deepWalk, f.getAbsolutePath()); GraphVectors vectors = @@ -221,8 +212,7 @@ public class TestDeepWalk extends BaseDL4JTest { } } - @Test() - @Timeout(180000) + @Test public void testDeepWalk13Vertices() throws IOException { int nVertices = 13; @@ -258,8 +248,7 @@ public class TestDeepWalk extends BaseDL4JTest { deepWalk.getVertexVector(i); } - @Test() - @Timeout(60000) + @Test public void testDeepWalkWeightedParallel() throws IOException { //Load graph diff --git a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/TestGraphHuffman.java b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/TestGraphHuffman.java index 0c95e9bf2..110434952 100644 --- a/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/TestGraphHuffman.java +++ b/deeplearning4j/deeplearning4j-graph/src/test/java/org/deeplearning4j/graph/models/deepwalk/TestGraphHuffman.java @@ -21,23 +21,19 @@ package org.deeplearning4j.graph.models.deepwalk; import org.deeplearning4j.BaseDL4JTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.Arrays; import java.util.HashSet; import java.util.Set; import static org.junit.jupiter.api.Assertions.*; -@NativeTag -@Tag(TagNames.FILE_IO) + +@Timeout(10) public class TestGraphHuffman extends BaseDL4JTest { - @Test() - @Timeout(10000) + @Test public void testGraphHuffman() { //Simple test case from Weiss - Data Structires and Algorithm Analysis in Java 3ed pg436 //Huffman code is non-unique, but length of code for each node is same for all Huffman codes diff --git a/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/pom.xml b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/pom.xml new file mode 100644 index 000000000..1e2039426 --- /dev/null +++ b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/pom.xml @@ -0,0 +1,75 @@ + + + + + deeplearning4j-manifold + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + deeplearning4j-tsne + jar + + deeplearning4j-tsne + http://maven.apache.org + + + UTF-8 + + + + + net.brutex.ai + nearestneighbor-core + ${project.version} + + + net.brutex.ai + deeplearning4j-nn + ${project.version} + + + org.projectlombok + lombok + ${lombok.version} + provided + + + net.brutex.ai + nd4j-api + ${project.version} + + + + net.brutex.ai + deeplearning4j-common-tests + ${project.version} + test + + + + + + test-nd4j-native + + + test-nd4j-cuda-10.2 + + + diff --git a/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java new file mode 100644 index 000000000..35122d29d --- /dev/null +++ b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/BarnesHutTsne.java @@ -0,0 +1,1063 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.plot; + + +import com.google.common.util.concurrent.AtomicDouble; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.clustering.algorithm.Distance; +import org.deeplearning4j.clustering.sptree.DataPoint; +import org.deeplearning4j.clustering.sptree.SpTree; +import org.deeplearning4j.clustering.vptree.VPTree; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.WorkspaceMode; +import org.deeplearning4j.nn.gradient.DefaultGradient; +import org.deeplearning4j.nn.gradient.Gradient; +import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr; +import org.deeplearning4j.optimize.api.ConvexOptimizer; +import org.deeplearning4j.optimize.api.TrainingListener; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.memory.conf.WorkspaceConfiguration; +import org.nd4j.linalg.api.memory.enums.*; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.BooleanIndexing; +import org.nd4j.linalg.indexing.conditions.Conditions; +import org.nd4j.linalg.learning.legacy.AdaGrad; +import org.nd4j.common.primitives.Pair; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.*; + +import static org.nd4j.linalg.factory.Nd4j.*; +import static org.nd4j.linalg.ops.transforms.Transforms.pow; +import static org.nd4j.linalg.ops.transforms.Transforms.sign; + + +/** + * Barnes hut algorithm for TSNE, uses a dual tree approximation approach. + * Work based on: + * http://lvdmaaten.github.io/tsne/ + * For hight dimensions, it's recommended to reduce the dimension up to 50 using another method (PCA or other) + * @author Adam Gibson + */ +@Slf4j +@Data +public class BarnesHutTsne implements Model { + + + public final static String workspaceCache = "LOOP_CACHE"; + public final static String workspaceExternal = "LOOP_EXTERNAL"; + + + protected int maxIter = 1000; + protected double realMin = Nd4j.EPS_THRESHOLD; + protected double initialMomentum = 0.5; + protected double finalMomentum = 0.8; + protected double minGain = 1e-2; + protected double momentum = initialMomentum; + protected int switchMomentumIteration = 250; + protected boolean normalize = true; + protected boolean usePca = false; + protected int stopLyingIteration = 250; + protected double tolerance = 1e-5; + protected double learningRate = 500; + protected AdaGrad adaGrad; + protected boolean useAdaGrad = true; + protected double perplexity = 30; + //protected INDArray gains,yIncs; + protected INDArray Y; + private int N; + private double theta; + private INDArray rows; + private INDArray cols; + private INDArray vals; + private String simiarlityFunction = "cosinesimilarity"; + private boolean invert = true; + private INDArray x; + private int numDimensions = 0; + public final static String Y_GRAD = "yIncs"; + private SpTree tree; + private INDArray gains; + @Setter + private INDArray yIncs; + private int vpTreeWorkers; + protected transient TrainingListener trainingListener; + protected WorkspaceMode workspaceMode; + private Initializer initializer; + + protected final static WorkspaceConfiguration workspaceConfigurationExternal = WorkspaceConfiguration.builder() + .initialSize(0).overallocationLimit(0.3).policyLearning(LearningPolicy.FIRST_LOOP) + .policyReset(ResetPolicy.BLOCK_LEFT).policySpill(SpillPolicy.REALLOCATE) + .policyAllocation(AllocationPolicy.OVERALLOCATE).build(); + + protected WorkspaceConfiguration workspaceConfigurationFeedForward = WorkspaceConfiguration.builder().initialSize(0) + .overallocationLimit(0.2).policyReset(ResetPolicy.BLOCK_LEFT) + .policyLearning(LearningPolicy.OVER_TIME).policySpill(SpillPolicy.REALLOCATE) + .policyAllocation(AllocationPolicy.OVERALLOCATE).build(); + + public final static WorkspaceConfiguration workspaceConfigurationCache = WorkspaceConfiguration.builder() + .overallocationLimit(0.2).policyReset(ResetPolicy.BLOCK_LEFT).cyclesBeforeInitialization(3) + .policyMirroring(MirroringPolicy.FULL).policySpill(SpillPolicy.REALLOCATE) + .policyLearning(LearningPolicy.OVER_TIME).build(); + + + public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter, + double realMin, double initialMomentum, double finalMomentum, double momentum, + int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance, + double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener, + double minGain,int vpTreeWorkers) { + this(numDimensions, simiarlityFunction, theta, invert, maxIter, realMin, initialMomentum, finalMomentum, + momentum, switchMomentumIteration, normalize, stopLyingIteration, tolerance, learningRate, + useAdaGrad, perplexity, TrainingListener, minGain, vpTreeWorkers, WorkspaceMode.NONE, null); + } + + public BarnesHutTsne(int numDimensions, String simiarlityFunction, double theta, boolean invert, int maxIter, + double realMin, double initialMomentum, double finalMomentum, double momentum, + int switchMomentumIteration, boolean normalize, int stopLyingIteration, double tolerance, + double learningRate, boolean useAdaGrad, double perplexity, TrainingListener TrainingListener, + double minGain,int vpTreeWorkers, WorkspaceMode workspaceMode, INDArray staticInput) { + this.maxIter = maxIter; + this.realMin = realMin; + this.initialMomentum = initialMomentum; + this.finalMomentum = finalMomentum; + this.momentum = momentum; + this.normalize = normalize; + this.useAdaGrad = useAdaGrad; + this.stopLyingIteration = stopLyingIteration; + this.learningRate = learningRate; + this.switchMomentumIteration = switchMomentumIteration; + this.tolerance = tolerance; + this.perplexity = perplexity; + this.minGain = minGain; + this.numDimensions = numDimensions; + this.simiarlityFunction = simiarlityFunction; + this.theta = theta; + this.trainingListener = TrainingListener; + this.invert = invert; + this.vpTreeWorkers = vpTreeWorkers; + this.workspaceMode = workspaceMode; + if(this.workspaceMode == null) + this.workspaceMode = WorkspaceMode.NONE; + initializer = (staticInput != null) ? new Initializer(staticInput) : new Initializer(); + } + + + public String getSimiarlityFunction() { + return simiarlityFunction; + } + + public void setSimiarlityFunction(String simiarlityFunction) { + this.simiarlityFunction = simiarlityFunction; + } + + public boolean isInvert() { + return invert; + } + + public void setInvert(boolean invert) { + this.invert = invert; + } + + public double getTheta() { + return theta; + } + + public double getPerplexity() { + return perplexity; + } + + public int getNumDimensions() { + return numDimensions; + } + + public void setNumDimensions(int numDimensions) { + this.numDimensions = numDimensions; + } + + /** + * Convert data to probability + * co-occurrences (aka calculating the kernel) + * @param d the data to convert + * @param perplexity the perplexity of the model + * @return the probabilities of co-occurrence + */ + public INDArray computeGaussianPerplexity(final INDArray d, double perplexity) { + N = d.rows(); + + final int k = (int) (3 * perplexity); + if (N - 1 < 3 * perplexity) + throw new IllegalStateException("Perplexity " + perplexity + "is too large for number of samples " + N); + + + rows = zeros(DataType.INT, 1, N + 1); + cols = zeros(DataType.INT, 1, N * k); + vals = zeros(d.dataType(), N * k); + + for (int n = 0; n < N; n++) + rows.putScalar(n + 1, rows.getDouble(n) + k); + + final double enthropy = Math.log(perplexity); + VPTree tree = new VPTree(d, simiarlityFunction, vpTreeWorkers,invert); + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + log.info("Calculating probabilities of data similarities..."); + for (int i = 0; i < N; i++) { + if (i % 500 == 0) + log.info("Handled " + i + " records"); + + double betaMin = -Double.MAX_VALUE; + double betaMax = Double.MAX_VALUE; + List results = new ArrayList<>(); + List distances = new ArrayList<>(); + tree.search(d.getRow(i), k + 1, results, distances, false, true); + double betas = 1.0; + + if(results.size() == 0){ + throw new IllegalStateException("Search returned no values for vector " + i + + " - similarity \"" + simiarlityFunction + "\" may not be defined (for example, vector is" + + " all zeros with cosine similarity)"); + } + + Double[] dists = new Double[distances.size()]; + distances.toArray(dists); + INDArray cArr = Nd4j.createFromArray(dists).castTo(d.dataType()); //VPTree.buildFromData(results); + + INDArray currP = null; + int tries = 0; + boolean found = false; + //binary search + while (!found && tries < 200) { + Pair pair = computeGaussianKernel(cArr, betas, k); + currP = pair.getFirst(); + double hDiff = pair.getSecond() - enthropy; + + if (hDiff < tolerance && -hDiff < tolerance) + found = true; + else { + if (hDiff > 0) { + betaMin = betas; + + if (betaMax == Double.MAX_VALUE || betaMax == -Double.MAX_VALUE) + betas *= 2; + else + betas = (betas + betaMax) / 2.0; + } else { + betaMax = betas; + if (betaMin == -Double.MAX_VALUE || betaMin == Double.MAX_VALUE) + betas /= 2.0; + else + betas = (betas + betaMin) / 2.0; + } + + tries++; + } + } + + currP.divi(currP.sumNumber().doubleValue() + Double.MIN_VALUE); + INDArray indices = Nd4j.create(1, k + 1); + for (int j = 0; j < indices.length(); j++) { + if (j >= results.size()) + break; + indices.putScalar(j, results.get(j).getIndex()); + } + + for (int l = 0; l < k; l++) { + cols.putScalar(rows.getInt(i) + l, indices.getDouble(l + 1)); + vals.putScalar(rows.getInt(i) + l, currP.getDouble(l)); + } + } + } + return vals; + } + + @Override + public INDArray input() { + return x; + } + + @Override + public ConvexOptimizer getOptimizer() { + return null; + } + + @Override + public INDArray getParam(String param) { + return null; + } + + @Override + public void addListeners(TrainingListener... listener) { + // no-op + } + + @Override + public Map paramTable() { + return null; + } + + @Override + public Map paramTable(boolean backprapParamsOnly) { + return null; + } + + @Override + public void setParamTable(Map paramTable) { + + } + + @Override + public void setParam(String key, INDArray val) { + + } + + @Override + public void clear() {} + + @Override + public void applyConstraints(int iteration, int epoch) { + //No op + } + + /* compute the gradient given the current solution, the probabilities and the constant */ + protected Pair gradient(INDArray p) { + throw new UnsupportedOperationException(); + } + + + @Data + @AllArgsConstructor + static class SymResult { + INDArray rows; + INDArray cols; + INDArray vals; + } + + /** + * Symmetrize the value matrix + * @param rowP + * @param colP + * @param valP + * @return + */ + public SymResult symmetrized(INDArray rowP, INDArray colP, INDArray valP) { + INDArray rowCounts = Nd4j.create(DataType.INT, N); + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + for (int n = 0; n < N; n++) { + int begin = rowP.getInt(n); + int end = rowP.getInt(n + 1); + for (int i = begin; i < end; i++) { + boolean present = false; + for (int m = rowP.getInt(colP.getInt(i)); m < rowP.getInt(colP.getInt(i) + 1); m++) + if (colP.getInt(m) == n) { + present = true; + } + + if (present) + rowCounts.putScalar(n, rowCounts.getInt(n) + 1); + + else { + rowCounts.putScalar(n, rowCounts.getInt(n) + 1); + rowCounts.putScalar(colP.getInt(i), rowCounts.getInt(colP.getInt(i)) + 1); + } + } + } + + int numElements = rowCounts.sumNumber().intValue(); + INDArray offset = Nd4j.create(DataType.INT, N); + INDArray symRowP = Nd4j.zeros(DataType.INT, N + 1); + INDArray symColP = Nd4j.create(DataType.INT, numElements); + INDArray symValP = Nd4j.create(valP.dataType(), numElements); + + for (int n = 0; n < N; n++) + symRowP.putScalar(n + 1, symRowP.getInt(n) + rowCounts.getInt(n)); + + for (int n = 0; n < N; n++) { + for (int i = rowP.getInt(n); i < rowP.getInt(n + 1); i++) { + boolean present = false; + for (int m = rowP.getInt(colP.getInt(i)); m < rowP.getInt(colP.getInt(i)+1); m++) { + if (colP.getInt(m) == n) { + present = true; + if (n <= colP.getInt(i)) { + // make sure we do not add elements twice + symColP.putScalar(symRowP.getInt(n) + offset.getInt(n), colP.getInt(i)); + symColP.putScalar(symRowP.getInt(colP.getInt(i)) + offset.getInt(colP.getInt(i)), n); + symValP.putScalar(symRowP.getInt(n) + offset.getInt(n), + valP.getDouble(i) + valP.getDouble(m)); + symValP.putScalar(symRowP.getInt(colP.getInt(i)) + offset.getInt(colP.getInt(i)), + valP.getDouble(i) + valP.getDouble(m)); + } + } + } + + // If (colP[i], n) is not present, there is no addition involved + if (!present) { + int colPI = colP.getInt(i); + symColP.putScalar(symRowP.getInt(n) + offset.getInt(n), colPI); + symColP.putScalar(symRowP.getInt(colP.getInt(i)) + offset.getInt(colPI), n); + symValP.putScalar(symRowP.getInt(n) + offset.getInt(n), valP.getDouble(i)); + symValP.putScalar(symRowP.getInt(colPI) + offset.getInt(colPI), valP.getDouble(i)); + } + + // Update offsets + if (!present || (present && n <= colP.getInt(i))) { + offset.putScalar(n, offset.getInt(n) + 1); + int colPI = colP.getInt(i); + if (colPI != n) + offset.putScalar(colPI, offset.getInt(colPI) + 1); + } + } + } + + // Divide the result by two + symValP.divi(2.0D); + return new SymResult(symRowP, symColP, symValP); + + } + + + } + + /** + * Computes a gaussian kernel + * given a vector of squared distance distances + * + * @param distances + * @param beta + * @return + */ + public Pair computeGaussianKernel(INDArray distances, double beta, int k) { + // Compute Gaussian kernel row + INDArray currP = Nd4j.create(distances.dataType(), k); + for (int m = 0; m < k; m++) { + currP.putScalar(m, Math.exp(-beta * distances.getDouble(m + 1))); + } + + double sum = currP.sumNumber().doubleValue() + Double.MIN_VALUE; + double h = 0.0; + for (int m = 0; m < k; m++) + h += beta * (distances.getDouble(m + 1) * currP.getDouble(m)); + + h = (h / sum) + Math.log(sum); + + return new Pair<>(currP, h); + } + + + /** + * Init the model + */ + @Override + public void init() { + + } + + /** + * Set the trainingListeners for the ComputationGraph (and all layers in the network) + * + * @param listeners + */ + @Override + public void setListeners(Collection listeners) { + + } + + /** + * Set the trainingListeners for the ComputationGraph (and all layers in the network) + * + * @param listeners + */ + @Override + public void setListeners(TrainingListener... listeners) { + + } + + private int calculateOutputLength() { + int ret = 0; + + INDArray rowCounts = Nd4j.create(N); + for (int n = 0; n < N; n++) { + int begin = rows.getInt(n); + int end = rows.getInt(n + 1); + for (int i = begin; i < end; i++) { + boolean present = false; + for (int m = rows.getInt(cols.getInt(i)); m < rows.getInt(cols.getInt(i) + 1); m++) { + if (cols.getInt(m) == n) { + present = true; + } + } + if (present) + rowCounts.putScalar(n, rowCounts.getDouble(n) + 1); + + else { + rowCounts.putScalar(n, rowCounts.getDouble(n) + 1); + rowCounts.putScalar(cols.getInt(i), rowCounts.getDouble(cols.getInt(i)) + 1); + } + } + } + ret = rowCounts.sum(Integer.MAX_VALUE).getInt(0); + return ret; + } + + public class Initializer { + + private INDArray staticData; + + public Initializer() {} + + public Initializer(INDArray input) { + this.staticData = input; + } + + public INDArray initData() { + if (staticData != null) + return staticData.dup(); + return randn(x.dataType(), x.rows(), numDimensions).muli(1e-3f); + } + } + + public static void zeroMean(INDArray input) { + INDArray means = input.mean(0); + input.subiRowVector(means); + } + + @Override + public void fit() { + if (theta == 0.0) { + log.debug("theta == 0, using decomposed version, might be slow"); + Tsne decomposedTsne = new Tsne(maxIter, realMin, initialMomentum, finalMomentum, minGain, momentum, + switchMomentumIteration, normalize, usePca, stopLyingIteration, tolerance, learningRate, + useAdaGrad, perplexity); + Y = decomposedTsne.calculate(x, numDimensions, perplexity); + } else { + //output + if (Y == null) { + Y = initializer.initData(); + } + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + x.divi(x.maxNumber()); + + computeGaussianPerplexity(x, perplexity); + /*INDArray outRows = Nd4j.create(new int[]{rows.rows(), rows.columns()}, DataType.INT); + BarnesHutSymmetrize op = new BarnesHutSymmetrize(rows, cols, vals, N, outRows); + Nd4j.getExecutioner().exec(op); + INDArray output = op.getSymmetrizedValues(); + INDArray outCols = op.getSymmetrizedCols(); + vals = output.divi(vals.sum(Integer.MAX_VALUE)); + rows = outRows; + cols = outCols;*/ + + SymResult result = symmetrized(rows, cols, vals); + vals = result.vals.divi(result.vals.sumNumber().doubleValue()); + rows = result.rows; + cols = result.cols; + //lie about gradient + vals.muli(12); + for (int i = 0; i < maxIter; i++) { + step(vals, i); + zeroMean(Y); + if (i == switchMomentumIteration) + momentum = finalMomentum; + if (i == stopLyingIteration) + vals.divi(12); + + + if (trainingListener != null) { + trainingListener.iterationDone(this, i, 0); + } + } + } + } + } + + @Override + public void update(Gradient gradient) { + } + + /** + * An individual iteration + * @param p the probabilities that certain points + * are near each other + * @param i the iteration (primarily for debugging purposes) + */ + public void step(INDArray p, int i) { + update(gradient().getGradientFor(Y_GRAD), Y_GRAD); + } + + static double sign_tsne(double x) { return (x == .0 ? .0 : (x < .0 ? -1.0 : 1.0)); } + + + @Override + public void update(INDArray gradient, String paramType) { + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + INDArray yGrads = gradient; +; if (gains == null) + gains = Y.ulike().assign(1.0); + + //Nd4j.getExecutioner().exec(new BarnesHutGains(gains, gains, yGrads, yIncs)); + // Copied from Reference + for (int i = 0; i < yGrads.rows(); ++i) { + for (int j = 0; j < yGrads.columns(); ++j) { + if (sign_tsne(yGrads.getDouble(i,j)) == sign_tsne(yIncs.getDouble(i,j))) { + gains.putScalar(new int[]{i,j}, gains.getDouble(i,j)*0.8); + } + else { + gains.putScalar(new int[]{i,j}, gains.getDouble(i,j)+0.2); + } + } + } + BooleanIndexing.replaceWhere(gains, minGain, Conditions.lessThan(minGain)); + + Y.addi(yIncs); + INDArray gradChange = gains.mul(yGrads); + + if (useAdaGrad) { + if (adaGrad == null) { + adaGrad = new AdaGrad(gradient.shape(), learningRate); + adaGrad.setStateViewArray(Nd4j.zeros(gradient.shape()).reshape(1, gradChange.length()), + gradChange.shape(), gradient.ordering(), true); + } + + gradChange = adaGrad.getGradient(gradChange, 0); + + } else { + gradChange.muli(learningRate); + } + yIncs.muli(momentum).subi(gradChange); + } + } + + + /** + * Save the model as a file with a csv format, adding the label as the last column. + * @param labels + * @param path the path to write + * @throws IOException + */ + public void saveAsFile(List labels, String path) throws IOException { + try (BufferedWriter write = new BufferedWriter(new FileWriter(new File(path)))) { + for (int i = 0; i < Y.rows(); i++) { + if (i >= labels.size()) + break; + String word = labels.get(i); + if (word == null) + continue; + StringBuilder sb = new StringBuilder(); + INDArray wordVector = Y.getRow(i); + for (int j = 0; j < wordVector.length(); j++) { + sb.append(wordVector.getDouble(j)); + if (j < wordVector.length() - 1) + sb.append(","); + } + + sb.append(","); + sb.append(word); + sb.append("\n"); + write.write(sb.toString()); + + } + write.flush(); + } + } + + public void saveAsFile(String path) throws IOException { + try (BufferedWriter write = new BufferedWriter(new FileWriter(new File(path)))) { + for (int i = 0; i < Y.rows(); i++) { + StringBuilder sb = new StringBuilder(); + INDArray wordVector = Y.getRow(i); + for (int j = 0; j < wordVector.length(); j++) { + sb.append(wordVector.getDouble(j)); + if (j < wordVector.length() - 1) + sb.append(","); + } + sb.append("\n"); + write.write(sb.toString()); + } + write.flush(); + } + } + /** + * Plot tsne + * + * @param matrix the matrix to plot + * @param nDims the number + * @param labels + * @param path the path to write + * @throws IOException + * @deprecated use {@link #fit(INDArray)} and {@link #saveAsFile(List, String)} instead. + */ + @Deprecated + public void plot(INDArray matrix, int nDims, List labels, String path) throws IOException { + fit(matrix, nDims); + saveAsFile(labels, path); + } + + + @Override + public double score() { + + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + + // Get estimate of normalization term + INDArray buff = Nd4j.create(numDimensions); + AtomicDouble sum_Q = new AtomicDouble(0.0); + for (int n = 0; n < N; n++) + tree.computeNonEdgeForces(n, theta, buff, sum_Q); + + // Loop over all edges to compute t-SNE error + double C = .0; + INDArray linear = Y; + for (int n = 0; n < N; n++) { + int begin = rows.getInt(n); + int end = rows.getInt(n + 1); + int ind1 = n; + for (int i = begin; i < end; i++) { + int ind2 = cols.getInt(i); + linear.slice(ind1).subi(linear.slice(ind2), buff); + + double Q = pow(buff, 2).sumNumber().doubleValue(); + Q = (1.0 / (1.0 + Q)) / sum_Q.doubleValue(); + C += vals.getDouble(i) * Math.log(vals.getDouble(i) + Nd4j.EPS_THRESHOLD) + / (Q + Nd4j.EPS_THRESHOLD); + } + } + + return C; + + } + + } + + @Override + public void computeGradientAndScore(LayerWorkspaceMgr workspaceMgr) { + + } + + @Override + public INDArray params() { + return null; + } + + @Override + public long numParams() { + return 0; + } + + @Override + public long numParams(boolean backwards) { + return 0; + } + + @Override + public void setParams(INDArray params) { + + } + + @Override + public void setParamsViewArray(INDArray params) { + throw new UnsupportedOperationException(); + } + + @Override + public INDArray getGradientsViewArray() { + throw new UnsupportedOperationException(); + } + + @Override + public void setBackpropGradientsViewArray(INDArray gradients) { + throw new UnsupportedOperationException(); + } + + + public void fit(INDArray data) { + this.x = data; + fit(); + } + + @Override + public void fit(INDArray data, LayerWorkspaceMgr workspaceMgr){ + fit(data); + } + + /** + * Change the dimensions with + * + * @deprecated Use {@link #fit(INDArray)} + */ + @Deprecated + public void fit(INDArray data, int nDims) { + this.x = data; + this.numDimensions = nDims; + fit(); + } + + @Override + public Gradient gradient() { + /*MemoryWorkspace workspace = + workspaceMode == WorkspaceMode.NONE ? new DummyWorkspace() + : Nd4j.getWorkspaceManager().getWorkspaceForCurrentThread( + workspaceConfigurationExternal, + workspaceExternal); + + + try (MemoryWorkspace ws = workspace.notifyScopeEntered())*/ { + + + if (yIncs == null) + yIncs = Y.like(); + if (gains == null) + gains = Y.ulike().assign(1.0D); + + AtomicDouble sumQ = new AtomicDouble(0); + /* Calculate gradient based on barnes hut approximation with positive and negative forces */ + INDArray posF = Y.like(); + INDArray negF = Y.like(); + + tree = new SpTree(Y); + + tree.computeEdgeForces(rows, cols, vals, N, posF); + for (int n = 0; n < N; n++) { + INDArray temp = negF.slice(n); + tree.computeNonEdgeForces(n, theta, temp, sumQ); + } + INDArray dC = posF.subi(negF.divi(sumQ)); + + Gradient ret = new DefaultGradient(); + ret.gradientForVariable().put(Y_GRAD, dC); + return ret; + } + } + + @Override + public Pair gradientAndScore() { + return new Pair<>(gradient(), score()); + } + + @Override + public int batchSize() { + return 0; + } + + @Override + public NeuralNetConfiguration conf() { + return null; + } + + @Override + public void setConf(NeuralNetConfiguration conf) { + + } + + /** + * Return the matrix reduce to the NDim. + */ + public INDArray getData() { + return Y; + } + + public void setData(INDArray data) { + this.Y = data; + } + + // TODO: find better solution for test + public void setN(int N) { + this.N = N; + } + + public static class Builder { + private int maxIter = 1000; + private double realMin = 1e-12f; + private double initialMomentum = 5e-1f; + private double finalMomentum = 8e-1f; + private double momentum = 5e-1f; + private int switchMomentumIteration = 100; + private boolean normalize = true; + private int stopLyingIteration = 100; + private double tolerance = 1e-5f; + private double learningRate = 1e-1f; + private boolean useAdaGrad = false; + private double perplexity = 30; + private double minGain = 1e-2f; + private double theta = 0.5; + private boolean invert = true; + private int numDim = 2; + private String similarityFunction = Distance.EUCLIDEAN.toString(); + private int vpTreeWorkers = 1; + protected WorkspaceMode workspaceMode = WorkspaceMode.NONE; + + private INDArray staticInput; + + public Builder vpTreeWorkers(int vpTreeWorkers) { + this.vpTreeWorkers = vpTreeWorkers; + return this; + } + + public Builder staticInit(INDArray staticInput) { + this.staticInput = staticInput; + return this; + } + + public Builder minGain(double minGain) { + this.minGain = minGain; + return this; + } + + public Builder perplexity(double perplexity) { + this.perplexity = perplexity; + return this; + } + + public Builder useAdaGrad(boolean useAdaGrad) { + this.useAdaGrad = useAdaGrad; + return this; + } + + public Builder learningRate(double learningRate) { + this.learningRate = learningRate; + return this; + } + + + public Builder tolerance(double tolerance) { + this.tolerance = tolerance; + return this; + } + + public Builder stopLyingIteration(int stopLyingIteration) { + this.stopLyingIteration = stopLyingIteration; + return this; + } + + public Builder normalize(boolean normalize) { + this.normalize = normalize; + return this; + } + + public Builder setMaxIter(int maxIter) { + this.maxIter = maxIter; + return this; + } + + public Builder setRealMin(double realMin) { + this.realMin = realMin; + return this; + } + + public Builder setInitialMomentum(double initialMomentum) { + this.initialMomentum = initialMomentum; + return this; + } + + public Builder setFinalMomentum(double finalMomentum) { + this.finalMomentum = finalMomentum; + return this; + } + + public Builder setMomentum(double momentum) { + this.momentum = momentum; + return this; + } + + public Builder setSwitchMomentumIteration(int switchMomentumIteration) { + this.switchMomentumIteration = switchMomentumIteration; + return this; + } + + + public Builder similarityFunction(String similarityFunction) { + this.similarityFunction = similarityFunction; + return this; + } + + public Builder invertDistanceMetric(boolean invert) { + this.invert = invert; + return this; + } + + public Builder theta(double theta) { + this.theta = theta; + return this; + } + + public Builder numDimension(int numDim) { + this.numDim = numDim; + return this; + } + + public Builder workspaceMode(WorkspaceMode workspaceMode){ + this.workspaceMode = workspaceMode; + return this; + } + + public BarnesHutTsne build() { + return new BarnesHutTsne(numDim, similarityFunction, theta, invert, maxIter, realMin, initialMomentum, + finalMomentum, momentum, switchMomentumIteration, normalize, stopLyingIteration, tolerance, + learningRate, useAdaGrad, perplexity, null, minGain, vpTreeWorkers, workspaceMode, staticInput); + } + + } + + + @Override + public void close(){ + //No-op + } +} diff --git a/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/Tsne.java b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/Tsne.java new file mode 100644 index 000000000..ce092eba9 --- /dev/null +++ b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/main/java/org/deeplearning4j/plot/Tsne.java @@ -0,0 +1,436 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.plot; + +import com.google.common.primitives.Ints; +import org.apache.commons.math3.util.FastMath; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dimensionalityreduction.PCA; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.indexing.BooleanIndexing; +import org.nd4j.linalg.indexing.INDArrayIndex; +import org.nd4j.linalg.indexing.SpecifiedIndex; +import org.nd4j.linalg.indexing.conditions.Conditions; +import org.nd4j.linalg.learning.legacy.AdaGrad; +import org.nd4j.common.primitives.Pair; +import org.nd4j.common.util.ArrayUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.nd4j.linalg.factory.Nd4j.*; +import static org.nd4j.linalg.ops.transforms.Transforms.*; + +/** + * dl4j port of original t-sne algorithm described/implemented by van der Maaten and Hinton + * + * + * @author raver119@gmail.com + * @author Adam Gibson + */ +public class Tsne { + protected int maxIter = 1000; + protected double realMin = Nd4j.EPS_THRESHOLD; + protected double initialMomentum = 0.5; + protected double finalMomentum = 0.8; + protected double minGain = 1e-2; + protected double momentum = initialMomentum; + protected int switchMomentumIteration = 100; + protected boolean normalize = true; + protected boolean usePca = false; + protected int stopLyingIteration = 250; + protected double tolerance = 1e-5; + protected double learningRate = 500; + protected AdaGrad adaGrad; + protected boolean useAdaGrad = true; + protected double perplexity = 30; + //protected INDArray gains,yIncs; + protected INDArray Y; + + protected static final Logger logger = LoggerFactory.getLogger(Tsne.class); + + + public Tsne(final int maxIter, final double realMin, final double initialMomentum, final double finalMomentum, + final double minGain, final double momentum, final int switchMomentumIteration, + final boolean normalize, final boolean usePca, final int stopLyingIteration, final double tolerance, + final double learningRate, final boolean useAdaGrad, final double perplexity) { + this.maxIter = maxIter; + this.realMin = realMin; + this.initialMomentum = initialMomentum; + this.finalMomentum = finalMomentum; + this.minGain = minGain; + this.momentum = momentum; + this.switchMomentumIteration = switchMomentumIteration; + this.normalize = normalize; + this.usePca = usePca; + this.stopLyingIteration = stopLyingIteration; + this.tolerance = tolerance; + this.learningRate = learningRate; + this.useAdaGrad = useAdaGrad; + this.perplexity = perplexity; + this.init(); + } + + protected void init() { + + } + + public INDArray calculate(INDArray X, int targetDimensions, double perplexity) { + // pca hook + if (usePca) { + X = PCA.pca(X, Math.min(50, X.columns()), normalize); + } else if (normalize) { + X.subi(X.min(Integer.MAX_VALUE)); + X = X.divi(X.max(Integer.MAX_VALUE)); + X = X.subiRowVector(X.mean(0)); + } + + + int n = X.rows(); + // FIXME: this is wrong, another distribution required here + Y = Nd4j.randn(X.dataType(), X.rows(), targetDimensions); + INDArray dY = Nd4j.zeros(n, targetDimensions); + INDArray iY = Nd4j.zeros(n, targetDimensions); + INDArray gains = Nd4j.ones(n, targetDimensions); + + boolean stopLying = false; + logger.debug("Y:Shape is = " + Arrays.toString(Y.shape())); + + // compute P-values + INDArray P = x2p(X, tolerance, perplexity); + + // do training + for (int i = 0; i < maxIter; i++) { + INDArray sumY = pow(Y, 2).sum(1).transpose(); + + //Student-t distribution + //also un normalized q + // also known as num in original implementation + INDArray qu = Y.mmul(Y.transpose()).muli(-2).addiRowVector(sumY).transpose().addiRowVector(sumY).addi(1) + .rdivi(1); + + // doAlongDiagonal(qu,new Zero()); + + INDArray Q = qu.div(qu.sumNumber().doubleValue()); + BooleanIndexing.replaceWhere(Q, 1e-12, Conditions.lessThan(1e-12)); + + INDArray PQ = P.sub(Q).muli(qu); + + logger.debug("PQ shape is: " + Arrays.toString(PQ.shape())); + logger.debug("PQ.sum(1) shape is: " + Arrays.toString(PQ.sum(1).shape())); + + dY = diag(PQ.sum(1)).subi(PQ).mmul(Y).muli(4); + + + if (i < switchMomentumIteration) { + momentum = initialMomentum; + } else { + momentum = finalMomentum; + } + + gains = gains.add(.2).muli(dY.cond(Conditions.greaterThan(0)).neq(iY.cond(Conditions.greaterThan(0)))) + .addi(gains.mul(0.8).muli(dY.cond(Conditions.greaterThan(0)) + .eq(iY.cond(Conditions.greaterThan(0))))); + + BooleanIndexing.replaceWhere(gains, minGain, Conditions.lessThan(minGain)); + + INDArray gradChange = gains.mul(dY); + + gradChange.muli(learningRate); + + iY.muli(momentum).subi(gradChange); + + double cost = P.mul(log(P.div(Q), false)).sumNumber().doubleValue(); + logger.info("Iteration [" + i + "] error is: [" + cost + "]"); + + Y.addi(iY); + // Y.addi(iY).subiRowVector(Y.mean(0)); + INDArray tiled = Nd4j.tile(Y.mean(0), new int[] {Y.rows(), 1}); + Y.subi(tiled); + + if (!stopLying && (i > maxIter / 2 || i >= stopLyingIteration)) { + P.divi(4); + stopLying = true; + } + } + return Y; + } + + public INDArray diag(INDArray ds) { + boolean isLong = ds.rows() > ds.columns(); + INDArray sliceZero = ds.slice(0); + int dim = Math.max(ds.columns(), ds.rows()); + INDArray result = Nd4j.create(dim, dim); + for (int i = 0; i < dim; i++) { + INDArray sliceSrc = ds.slice(i); + INDArray sliceDst = result.slice(i); + for (int j = 0; j < dim; j++) { + if (i == j) { + if (isLong) + sliceDst.putScalar(j, sliceSrc.getDouble(0)); + else + sliceDst.putScalar(j, sliceZero.getDouble(i)); + } + } + } + + return result; + } + + public void plot(INDArray matrix, int nDims, List labels, String path) throws IOException { + + calculate(matrix, nDims, perplexity); + + BufferedWriter write = new BufferedWriter(new FileWriter(new File(path), true)); + + for (int i = 0; i < Y.rows(); i++) { + if (i >= labels.size()) + break; + String word = labels.get(i); + if (word == null) + continue; + StringBuilder sb = new StringBuilder(); + INDArray wordVector = Y.getRow(i); + for (int j = 0; j < wordVector.length(); j++) { + sb.append(wordVector.getDouble(j)); + if (j < wordVector.length() - 1) + sb.append(","); + } + + sb.append(","); + sb.append(word); + sb.append(" "); + + sb.append("\n"); + write.write(sb.toString()); + + } + + write.flush(); + write.close(); + } + + /** + * Computes a gaussian kernel + * given a vector of squared distance distances + * + * @param d the data + * @param beta + * @return + */ + public Pair hBeta(INDArray d, double beta) { + INDArray P = exp(d.neg().muli(beta)); + double sumP = P.sumNumber().doubleValue(); + double logSumP = FastMath.log(sumP); + Double H = logSumP + ((beta * (d.mul(P).sumNumber().doubleValue())) / sumP); + P.divi(sumP); + return new Pair<>(H, P); + } + + /** + * This method build probabilities for given source data + * + * @param X + * @param tolerance + * @param perplexity + * @return + */ + private INDArray x2p(final INDArray X, double tolerance, double perplexity) { + int n = X.rows(); + final INDArray p = zeros(n, n); + final INDArray beta = ones(n, 1); + final double logU = Math.log(perplexity); + + INDArray sumX = pow(X, 2).sum(1); + + logger.debug("sumX shape: " + Arrays.toString(sumX.shape())); + + INDArray times = X.mmul(X.transpose()).muli(-2); + + logger.debug("times shape: " + Arrays.toString(times.shape())); + + INDArray prodSum = times.transpose().addiColumnVector(sumX); + + logger.debug("prodSum shape: " + Arrays.toString(prodSum.shape())); + + INDArray D = X.mmul(X.transpose()).mul(-2) // thats times + .transpose().addColumnVector(sumX) // thats prodSum + .addRowVector(sumX.transpose()); // thats D + + logger.info("Calculating probabilities of data similarities..."); + logger.debug("Tolerance: " + tolerance); + for (int i = 0; i < n; i++) { + if (i % 500 == 0 && i > 0) + logger.info("Handled [" + i + "] records out of [" + n + "]"); + + double betaMin = Double.NEGATIVE_INFINITY; + double betaMax = Double.POSITIVE_INFINITY; + int[] vals = Ints.concat(ArrayUtil.range(0, i), ArrayUtil.range(i + 1, n)); + INDArrayIndex[] range = new INDArrayIndex[] {new SpecifiedIndex(vals)}; + + INDArray row = D.slice(i).get(range); + Pair pair = hBeta(row, beta.getDouble(i)); + //INDArray hDiff = pair.getFirst().sub(logU); + double hDiff = pair.getFirst() - logU; + int tries = 0; + + //while hdiff > tolerance + while (Math.abs(hDiff) > tolerance && tries < 50) { + //if hdiff > 0 + if (hDiff > 0) { + betaMin = beta.getDouble(i); + if (Double.isInfinite(betaMax)) + beta.putScalar(i, beta.getDouble(i) * 2.0); + else + beta.putScalar(i, (beta.getDouble(i) + betaMax) / 2.0); + } else { + betaMax = beta.getDouble(i); + if (Double.isInfinite(betaMin)) + beta.putScalar(i, beta.getDouble(i) / 2.0); + else + beta.putScalar(i, (beta.getDouble(i) + betaMin) / 2.0); + } + + pair = hBeta(row, beta.getDouble(i)); + hDiff = pair.getFirst() - logU; + tries++; + } + p.slice(i).put(range, pair.getSecond()); + } + + + //dont need data in memory after + logger.info("Mean value of sigma " + sqrt(beta.rdiv(1)).mean(Integer.MAX_VALUE)); + BooleanIndexing.replaceWhere(p, 1e-12, Conditions.isNan()); + + //set 0 along the diagonal + INDArray permute = p.transpose(); + + INDArray pOut = p.add(permute); + + pOut.divi(pOut.sumNumber().doubleValue() + 1e-6); + + pOut.muli(4); + + BooleanIndexing.replaceWhere(pOut, 1e-12, Conditions.lessThan(1e-12)); + //ensure no nans + + return pOut; + } + + + public static class Builder { + protected int maxIter = 1000; + protected double realMin = 1e-12f; + protected double initialMomentum = 5e-1f; + protected double finalMomentum = 8e-1f; + protected double momentum = 5e-1f; + protected int switchMomentumIteration = 100; + protected boolean normalize = true; + protected boolean usePca = false; + protected int stopLyingIteration = 100; + protected double tolerance = 1e-5f; + protected double learningRate = 1e-1f; + protected boolean useAdaGrad = false; + protected double perplexity = 30; + protected double minGain = 1e-1f; + + + public Builder minGain(double minGain) { + this.minGain = minGain; + return this; + } + + public Builder perplexity(double perplexity) { + this.perplexity = perplexity; + return this; + } + + public Builder useAdaGrad(boolean useAdaGrad) { + this.useAdaGrad = useAdaGrad; + return this; + } + + public Builder learningRate(double learningRate) { + this.learningRate = learningRate; + return this; + } + + + public Builder tolerance(double tolerance) { + this.tolerance = tolerance; + return this; + } + + public Builder stopLyingIteration(int stopLyingIteration) { + this.stopLyingIteration = stopLyingIteration; + return this; + } + + public Builder usePca(boolean usePca) { + this.usePca = usePca; + return this; + } + + public Builder normalize(boolean normalize) { + this.normalize = normalize; + return this; + } + + public Builder setMaxIter(int maxIter) { + this.maxIter = maxIter; + return this; + } + + public Builder setRealMin(double realMin) { + this.realMin = realMin; + return this; + } + + public Builder setInitialMomentum(double initialMomentum) { + this.initialMomentum = initialMomentum; + return this; + } + + public Builder setFinalMomentum(double finalMomentum) { + this.finalMomentum = finalMomentum; + return this; + } + + public Builder setMomentum(double momentum) { + this.momentum = momentum; + return this; + } + + public Builder setSwitchMomentumIteration(int switchMomentumIteration) { + this.switchMomentumIteration = switchMomentumIteration; + return this; + } + + public Tsne build() { + return new Tsne(maxIter, realMin, initialMomentum, finalMomentum, minGain, momentum, + switchMomentumIteration, normalize, usePca, stopLyingIteration, tolerance, learningRate, + useAdaGrad, perplexity); + } + } +} diff --git a/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/test/java/org/deeplearning4j/plot/Test6058.java b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/test/java/org/deeplearning4j/plot/Test6058.java new file mode 100644 index 000000000..de88c6851 --- /dev/null +++ b/deeplearning4j/deeplearning4j-manifold/deeplearning4j-tsne/src/test/java/org/deeplearning4j/plot/Test6058.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.plot; + +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class Test6058 extends BaseDL4JTest { + + @Test + public void test() throws Exception { + //All zero input -> cosine similarity isn't defined + //https://github.com/deeplearning4j/deeplearning4j/issues/6058 + val iterations = 10; + val cacheList = new ArrayList(); + + int nWords = 100; + for(int i=0; i cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words +// +// //STEP 2: Turn text input into a list of words +// log.info("Load & Vectorize data...."); +// File wordFile = new ClassPathResource("deeplearning4j-tsne/words.txt").getFile(); //Open the file +// //Get the data of all unique word vectors +// Pair vectors = WordVectorSerializer.loadTxt(wordFile); +// VocabCache cache = vectors.getSecond(); +// INDArray weights = vectors.getFirst().getSyn0(); //seperate weights of unique words into their own list +// +// for(int i = 0; i < cache.numWords(); i++) //seperate strings of words into their own list +// cacheList.add(cache.wordAtIndex(i)); +// +// //STEP 3: build a dual-tree tsne to use later +// log.info("Build model...."); +// BarnesHutTsne tsne = new BarnesHutTsne.Builder() +// .setMaxIter(iterations).theta(0.5) +// .normalize(false) +// .learningRate(500) +// .useAdaGrad(false) +// .workspaceMode(wsm) +// .build(); +// +// //STEP 4: establish the tsne values and save them to a file +// log.info("Store TSNE Coordinates for Plotting...."); +// String outputFile = "target/archive-tmp/tsne-standard-coords.csv"; +// (new File(outputFile)).getParentFile().mkdirs(); +// +// tsne.fit(weights); +// tsne.saveAsFile(cacheList, outputFile); +// +// +// } +// } +// +//} diff --git a/deeplearning4j/deeplearning4j-manifold/pom.xml b/deeplearning4j/deeplearning4j-manifold/pom.xml new file mode 100644 index 000000000..d8ff07dd5 --- /dev/null +++ b/deeplearning4j/deeplearning4j-manifold/pom.xml @@ -0,0 +1,47 @@ + + + + + deeplearning4j-parent + net.brutex.ai + 1.0.0-SNAPSHOT + + 4.0.0 + + deeplearning4j-manifold + pom + + deeplearning4j-manifold + + deeplearning4j-tsne + + + + + + + + + + test-nd4j-native + + + test-nd4j-cuda-10.2 + + + diff --git a/deeplearning4j/deeplearning4j-modelexport-solr/pom.xml b/deeplearning4j/deeplearning4j-modelexport-solr/pom.xml index a4fe90b4f..a1cc1eee5 100644 --- a/deeplearning4j/deeplearning4j-modelexport-solr/pom.xml +++ b/deeplearning4j/deeplearning4j-modelexport-solr/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - org.deeplearning4j + net.brutex.ai deeplearning4j-parent 1.0.0-SNAPSHOT @@ -41,9 +41,7 @@ org.apache.maven.plugins maven-surefire-plugin - false - - -Ddtype=float -Dfile.encoding=UTF-8 -Xmx${test.heap.size} + -Ddtype=float -Dfile.encoding=UTF-8 -Xmx8g -Dtest.solr.allowed.securerandom=NativePRNG @@ -281,23 +279,10 @@ - org.deeplearning4j + net.brutex.ai deeplearning4j-core ${project.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - org.apache.solr solr-test-framework @@ -308,25 +293,23 @@ - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - + test-nd4j-native - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} + net.brutex.ai + nd4j-native + ${project.version} test + + + + test-nd4j-cuda-11.2 + - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} + net.brutex.ai + nd4j-cuda-${cuda.version} + ${project.version} test diff --git a/deeplearning4j/deeplearning4j-modelexport-solr/src/test/java/org/deeplearning4j/nn/modelexport/solr/handler/ModelTupleStreamIntegrationTest.java b/deeplearning4j/deeplearning4j-modelexport-solr/src/test/java/org/deeplearning4j/nn/modelexport/solr/handler/ModelTupleStreamIntegrationTest.java index faf8f3209..7c0505605 100644 --- a/deeplearning4j/deeplearning4j-modelexport-solr/src/test/java/org/deeplearning4j/nn/modelexport/solr/handler/ModelTupleStreamIntegrationTest.java +++ b/deeplearning4j/deeplearning4j-modelexport-solr/src/test/java/org/deeplearning4j/nn/modelexport/solr/handler/ModelTupleStreamIntegrationTest.java @@ -17,11 +17,13 @@ * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ + package org.deeplearning4j.nn.modelexport.solr.handler; import java.io.File; import java.nio.file.Path; import java.security.SecureRandom; + import com.carrotsearch.randomizedtesting.ThreadFilter; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.solr.client.solrj.io.Tuple; @@ -38,154 +40,224 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.layers.OutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; import org.deeplearning4j.util.ModelSerializer; -import org.junit.jupiter.api.*; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.lossfunctions.LossFunctions; import org.nd4j.linalg.api.memory.provider.BasicWorkspaceManager; import org.nd4j.rng.deallocator.NativeRandomDeallocator; -import org.junit.jupiter.api.extension.ExtendWith; -@ThreadLeakFilters(defaultFilters = true, filters = { ModelTupleStreamIntegrationTest.PrivateDeallocatorThreadsFilter.class }) -@DisplayName("Model Tuple Stream Integration Test") -@Disabled("Timeout issue") -@Tag(TagNames.SOLR) -@Tag(TagNames.DIST_SYSTEMS) -class ModelTupleStreamIntegrationTest extends SolrCloudTestCase { +@ThreadLeakFilters(defaultFilters = true, filters = { + ModelTupleStreamIntegrationTest.PrivateDeallocatorThreadsFilter.class +}) +public class ModelTupleStreamIntegrationTest extends SolrCloudTestCase { - static { - /* + static { + /* This is a hack around the backend-dependent nature of secure random implementations though we can set the secure random algorithm in our pom.xml files (via maven surefire and test.solr.allowed.securerandom) there isn't a mechanism that is completely platform independent. By setting it there (for example, to NativePRNG) that makes it pass on some platforms like Linux but fails on some JVMs on Windows For testing purposes, we don't need strict guarantees around RNG, hence we don't want to enforce the RNG algorithm */ - String algorithm = new SecureRandom().getAlgorithm(); - System.setProperty("test.solr.allowed.securerandom", algorithm); - } + String algorithm = new SecureRandom().getAlgorithm(); + System.setProperty("test.solr.allowed.securerandom", algorithm); + } - @DisplayName("Private Deallocator Threads Filter") - static class PrivateDeallocatorThreadsFilter implements ThreadFilter { - /** - * Reject deallocator threads over whose cleanup this test has no control. - */ - @Override - public boolean reject(Thread thread) { - final ThreadGroup threadGroup = thread.getThreadGroup(); - final String threadGroupName = (threadGroup == null ? null : threadGroup.getName()); - if (threadGroupName != null && threadGroupName.endsWith(ModelTupleStreamIntegrationTest.class.getSimpleName())) { - final String threadName = thread.getName(); - if (threadName.startsWith(NativeRandomDeallocator.DeallocatorThreadNamePrefix) || threadName.toLowerCase().contains("deallocator") || threadName.equals(BasicWorkspaceManager.WorkspaceDeallocatorThreadName)) { - return true; - } - } - return false; + public static class PrivateDeallocatorThreadsFilter implements ThreadFilter { + /** + * Reject deallocator threads over whose cleanup this test has no control. + */ + @Override + public boolean reject(Thread thread) { + final ThreadGroup threadGroup = thread.getThreadGroup(); + final String threadGroupName = (threadGroup == null ? null : threadGroup.getName()); + + if (threadGroupName != null && + threadGroupName.endsWith(ModelTupleStreamIntegrationTest.class.getSimpleName())) { + + final String threadName = thread.getName(); + if (threadName.startsWith(NativeRandomDeallocator.DeallocatorThreadNamePrefix) || + threadName.toLowerCase().contains("deallocator") || + threadName.equals(BasicWorkspaceManager.WorkspaceDeallocatorThreadName)) { + return true; } + } + + return false; + } + } + + final private static String MY_COLLECTION_NAME = "mySolrCollection"; + final private static String MY_SERIALIZED_MODEL_FILENAME = "mySerializedModel"; + + @BeforeAll + public static void setupCluster() throws Exception { + + final Path configsetPath = configset("mini-expressible"); + + // create and serialize model + { + final Model model = buildModel(); + final File serializedModelFile = configsetPath + .resolve(MY_SERIALIZED_MODEL_FILENAME) + .toFile(); + ModelSerializer.writeModel(model, serializedModelFile.getPath(), false); } - final private static String MY_COLLECTION_NAME = "mySolrCollection"; + final String configName = "conf"; + final int numShards = 2; + final int numReplicas = 2; + final int maxShardsPerNode = 1; + final int nodeCount = (numShards*numReplicas + (maxShardsPerNode-1))/maxShardsPerNode; - final private static String MY_SERIALIZED_MODEL_FILENAME = "mySerializedModel"; + // create and configure cluster + configureCluster(nodeCount) + .addConfig(configName, configsetPath) + .configure(); - @BeforeAll - static void setupCluster() throws Exception { - final Path configsetPath = configset("mini-expressible"); - // create and serialize model - { - final Model model = buildModel(); - final File serializedModelFile = configsetPath.resolve(MY_SERIALIZED_MODEL_FILENAME).toFile(); - ModelSerializer.writeModel(model, serializedModelFile.getPath(), false); + // create an empty collection + CollectionAdminRequest.createCollection(MY_COLLECTION_NAME, configName, numShards, numReplicas) + .setMaxShardsPerNode(maxShardsPerNode) + .process(cluster.getSolrClient()); + + // compose an update request + final UpdateRequest updateRequest = new UpdateRequest(); + + // add some documents + updateRequest.add( + sdoc("id", "green", + "channel_b_f", "0", + "channel_g_f", "255", + "channel_r_f", "0")); + updateRequest.add( + sdoc("id", "black", + "channel_b_f", "0", + "channel_g_f", "0", + "channel_r_f", "0")); + updateRequest.add( + sdoc("id", "yellow", + "channel_b_f", "0", + "channel_g_f", "255", + "channel_r_f", "255")); + + // make the update request + updateRequest.commit(cluster.getSolrClient(), MY_COLLECTION_NAME); + } + + private static Model buildModel() throws Exception { + + final int numInputs = 3; + final int numOutputs = 2; + + final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list( + new OutputLayer.Builder() + .nIn(numInputs) + .nOut(numOutputs) + .activation(Activation.IDENTITY) + .lossFunction(LossFunctions.LossFunction.MSE) + .build() + ) + .build(); + + final MultiLayerNetwork model = new MultiLayerNetwork(conf); + model.init(); + + final float[] floats = new float[]{ +1, +1, +1, -1, -1, -1, 0, 0 }; + // positive weight for first output, negative weight for second output, no biases + assertEquals((numInputs+1)*numOutputs, floats.length); + + final INDArray params = Nd4j.create(floats); + model.setParams(params); + + return model; + } + + private void doTest(String expr, String[] expectedIds, Object[] expectedLefts, Object[] expectedRights) throws Exception { + ModifiableSolrParams paramsLoc = new ModifiableSolrParams(); + paramsLoc.set("expr", expr); + paramsLoc.set("qt", "/stream"); + + String url = cluster.getRandomJetty(random()).getBaseUrl().toString()+"/"+MY_COLLECTION_NAME; + + + TupleStream tupleStream = new SolrStream(url, paramsLoc); + + StreamContext context = new StreamContext(); + tupleStream.setStreamContext(context); + + try { + tupleStream.open(); + + for (int ii=0; ii floatsList(int numFloats) { + final List floatsList = new ArrayList(); + final float[] floats0 = new float[numFloats]; + final float[] floats1 = new float[numFloats]; + for (int ii=0; ii floatsList(int numFloats) { - final List floatsList = new ArrayList(); - final float[] floats0 = new float[numFloats]; - final float[] floats1 = new float[numFloats]; - for (int ii = 0; ii < numFloats; ++ii) { - floats0[ii] = 0f; - floats1[ii] = 1f; } - floatsList.add(floats0); - floatsList.add(floats1); - return floatsList; + } } + assertEquals(50, testsCount); + } - @Test - @DisplayName("Test") - @Disabled("Permissions issues on CI") - void test() throws Exception { - int testsCount = 0; - for (int numInputs = 1; numInputs <= 5; ++numInputs) { - for (int numOutputs = 1; numOutputs <= 5; ++numOutputs) { - for (Model model : new Model[] { buildMultiLayerNetworkModel(numInputs, numOutputs), buildComputationGraphModel(numInputs, numOutputs) }) { - doTest(model, numInputs, numOutputs); - ++testsCount; - } - } - } - assertEquals(50, testsCount); - } + private void doTest(Model originalModel, int numInputs, int numOutputs) throws Exception { - private void doTest(Model originalModel, int numInputs, int numOutputs) throws Exception { - final Path tempDirPath = Files.createTempDirectory(null); - final File tempDirFile = tempDirPath.toFile(); - tempDirFile.deleteOnExit(); - final SolrResourceLoader solrResourceLoader = new SolrResourceLoader(tempDirPath); - final File tempFile = File.createTempFile("prefix", "suffix", tempDirFile); - tempFile.deleteOnExit(); - final String serializedModelFileName = tempFile.getPath(); - ModelSerializer.writeModel(originalModel, serializedModelFileName, false); - final Model restoredModel = ModelGuesser.loadModelGuess(serializedModelFileName); - final StreamContext streamContext = new StreamContext(); - final SolrClientCache solrClientCache = new SolrClientCache(); - streamContext.setSolrClientCache(solrClientCache); - final String[] inputKeys = new String[numInputs]; - final String inputKeysList = fillArray(inputKeys, "input", ","); - final String[] outputKeys = new String[numOutputs]; - final String outputKeysList = fillArray(outputKeys, "output", ","); - for (final float[] floats : floatsList(numInputs)) { - final String inputValuesList; - { - final StringBuilder sb = new StringBuilder(); - for (int ii = 0; ii < inputKeys.length; ++ii) { - if (0 < ii) - sb.append(','); - sb.append(inputKeys[ii]).append('=').append(floats[ii]); - } - inputValuesList = sb.toString(); - } - final StreamFactory streamFactory = new SolrDefaultStreamFactory().withSolrResourceLoader(solrResourceLoader).withFunctionName("model", ModelTupleStream.class); - final StreamExpression streamExpression = StreamExpressionParser.parse("model(" + "tuple(" + inputValuesList + ")" + ",serializedModelFileName=\"" + serializedModelFileName + "\"" + ",inputKeys=\"" + inputKeysList + "\"" + ",outputKeys=\"" + outputKeysList + "\"" + ")"); - final TupleStream tupleStream = streamFactory.constructStream(streamExpression); - tupleStream.setStreamContext(streamContext); - assertTrue(tupleStream instanceof ModelTupleStream); - final ModelTupleStream modelTupleStream = (ModelTupleStream) tupleStream; - modelTupleStream.open(); - { - final Tuple tuple1 = modelTupleStream.read(); - assertNotNull(tuple1); - assertFalse(tuple1.EOF); - for (int ii = 0; ii < outputKeys.length; ++ii) { - final INDArray inputs = Nd4j.create(new float[][] { floats }); - final double originalScore = NetworkUtils.output((Model) originalModel, inputs).getDouble(ii); - final double restoredScore = NetworkUtils.output((Model) restoredModel, inputs).getDouble(ii); - assertEquals(originalScore, restoredScore, 1e-5,originalModel.getClass().getSimpleName() + " (originalScore-restoredScore)=" + (originalScore - restoredScore)); - final Double outputValue = tuple1.getDouble(outputKeys[ii]); - assertNotNull(outputValue); - final double tupleScore = outputValue.doubleValue(); - assertEquals(originalScore, tupleScore, 1e-5,originalModel.getClass().getSimpleName() + " (originalScore-tupleScore[" + ii + "])=" + (originalScore - tupleScore)); - } - final Tuple tuple2 = modelTupleStream.read(); - assertNotNull(tuple2); - assertTrue(tuple2.EOF); - } - modelTupleStream.close(); - doToExpressionTest(streamExpression, modelTupleStream.toExpression(streamFactory), inputKeys.length); - doToExplanationTest(modelTupleStream.toExplanation(streamFactory)); - } - } + final Path tempDirPath = Files.createTempDirectory(null); + final File tempDirFile = tempDirPath.toFile(); + tempDirFile.deleteOnExit(); - private static void doToExpressionTest(StreamExpression streamExpression, StreamExpressionParameter streamExpressionParameter, int inputKeysLength) { - assertTrue(streamExpressionParameter instanceof StreamExpression); - // tuple(input1=1,input2=2) and tuple(input2=2,input1=1) are equivalent - // but StreamExpression equals does not consider them equal. - if (inputKeysLength == 1) { - assertEquals(streamExpression, (StreamExpression) streamExpressionParameter); - } - } + final SolrResourceLoader solrResourceLoader = new SolrResourceLoader(tempDirPath); - private static void doToExplanationTest(Explanation explanation) { - final Map explanationMap = new TreeMap(); - explanation.toMap(explanationMap); - assertTrue(explanation instanceof StreamExplanation); - assertNotNull(explanationMap.remove("children")); - assertNotNull(explanationMap.remove("expression")); - assertNotNull(explanationMap.remove("expressionNodeId")); - assertEquals(ExpressionType.STREAM_DECORATOR, explanationMap.remove("expressionType")); - assertEquals(explanationMap.remove("functionName"), "model"); - assertEquals(ModelTupleStream.class.getName(), explanationMap.remove("implementingClass")); - assertTrue(explanationMap.isEmpty(),explanationMap.toString()); - } + final File tempFile = File.createTempFile("prefix", "suffix", tempDirFile); + tempFile.deleteOnExit(); - /** - * Fills an existing array using prefix and delimiter, e.g. - * input: arr = [ "", "", "" ] prefix="value" delimiter="," - * output: arr = [ "value1", "value2", "value3" ] - * return: "value1,value2,value3" - */ - private static String fillArray(String[] arr, final String prefix, final String delimiter) { + final String serializedModelFileName = tempFile.getPath(); + + ModelSerializer.writeModel(originalModel, serializedModelFileName, false); + + final Model restoredModel = ModelGuesser.loadModelGuess(serializedModelFileName); + + final StreamContext streamContext = new StreamContext(); + final SolrClientCache solrClientCache = new SolrClientCache(); + streamContext.setSolrClientCache(solrClientCache); + + final String[] inputKeys = new String[numInputs]; + final String inputKeysList = fillArray(inputKeys, "input", ","); + + final String[] outputKeys = new String[numOutputs]; + final String outputKeysList = fillArray(outputKeys, "output", ","); + + for (final float[] floats : floatsList(numInputs)) { + + final String inputValuesList; + { final StringBuilder sb = new StringBuilder(); - for (int ii = 0; ii < arr.length; ++ii) { - arr[ii] = prefix + Integer.toString(ii + 1); - if (0 < ii) - sb.append(delimiter); - sb.append(arr[ii]); + for (int ii=0; ii { - String modelPath = "modelimport/keras/examples/foo/bar.h5"; - importEndModelTest(tempDir,modelPath, null, true, true, false, false); - }); - } - - /** - * MNIST MLP tests - */ - @Test - @DisplayName("Import Mnist Mlp Tf Keras 1") - void importMnistMlpTfKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false); - } - - @Test - @DisplayName("Import Mnist Mlp Th Keras 1") - void importMnistMlpThKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_th_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_th_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, false, true, false, false); - } - - @Test - @DisplayName("Import Mnist Mlp Tf Keras 2") - void importMnistMlpTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/mnist_mlp/mnist_mlp_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false); - } - - @Test - @DisplayName("Import Mnist Mlp Reshape Tf Keras 1") - void importMnistMlpReshapeTfKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/mnist_mlp_reshape/mnist_mlp_reshape_tf_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/mnist_mlp_reshape/mnist_mlp_reshape_tf_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, true, false); - } - - /** - * MNIST CNN tests - */ - @Test - @DisplayName("Import Mnist Cnn Tf Keras 1") - void importMnistCnnTfKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, false, false, false); - } - - @Test - @DisplayName("Import Mnist Cnn Th Keras 1") - void importMnistCnnThKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_th_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_th_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, false, true, true, false); - } - - @Test - @DisplayName("Import Mnist Cnn Tf Keras 2") - void importMnistCnnTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/mnist_cnn/mnist_cnn_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, true, false); - } - - /** - * IMDB Embedding and LSTM test - */ - @Test - @DisplayName("Import Imdb Lstm Tf Keras 1") - void importImdbLstmTfKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_tf_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_tf_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false, true, null, null); - } - - @Test - @DisplayName("Import Imdb Lstm Th Keras 1") - void importImdbLstmThKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_th_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_th_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false, true, null, null); - } - - @Test - @DisplayName("Import Imdb Lstm Tf Keras 2") - void importImdbLstmTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false, true, null, null); - } - - @Test - @DisplayName("Import Imdb Lstm Th Keras 2") - void importImdbLstmThKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_th_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/imdb_lstm/imdb_lstm_th_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, false, true, false, false, true, null, null); - } - - /** - * IMDB LSTM fasttext - */ - // TODO: prediction checks fail due to globalpooling for fasttext, very few grads fail as well - @Test - @DisplayName("Import Imdb Fasttext Tf Keras 1") - void importImdbFasttextTfKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_tf_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_tf_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, false, false, false, false); - } - - @Test - @DisplayName("Import Imdb Fasttext Th Keras 1") - void importImdbFasttextThKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_th_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_th_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, false, false, false, false); - } - - @Test - @DisplayName("Import Imdb Fasttext Tf Keras 2") - void importImdbFasttextTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/imdb_fasttext/imdb_fasttext_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, false, false, false); - } - - /** - * Simple LSTM (return sequences = false) into Dense layer test - */ - @Test - @DisplayName("Import Simple Lstm Tf Keras 1") - void importSimpleLstmTfKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/simple_lstm/simple_lstm_tf_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/simple_lstm/simple_lstm_tf_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false); - } - - @Test - @DisplayName("Import Simple Lstm Th Keras 1") - void importSimpleLstmThKeras1(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/simple_lstm/simple_lstm_th_keras_1_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/simple_lstm/simple_lstm_th_keras_1_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false); - } - - @Test - @DisplayName("Import Simple Lstm Tf Keras 2") - void importSimpleLstmTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/simple_lstm/simple_lstm_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/simple_lstm/simple_lstm_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, false, false, false); - } - - /** - * Simple LSTM (return sequences = true) into flatten into Dense layer test - */ - @Test - @DisplayName("Import Simple Flatten Lstm Tf Keras 2") - void importSimpleFlattenLstmTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/simple_flatten_lstm/simple_flatten_lstm_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/simple_flatten_lstm/" + "simple_flatten_lstm_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false); - } - - /** - * Simple RNN (return sequences = true) into flatten into Dense layer test - */ - @Test - @DisplayName("Import Simple Flatten Rnn Tf Keras 2") - void importSimpleFlattenRnnTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/simple_flatten_rnn/simple_flatten_rnn_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/simple_flatten_rnn/" + "simple_flatten_rnn_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false, true, null, null); - } - - /** - * Simple RNN (return sequences = false) into Dense layer test - */ - @Test - @DisplayName("Import Simple Rnn Tf Keras 2") - void importSimpleRnnTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/simple_rnn/simple_rnn_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/simple_rnn/" + "simple_rnn_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, false, false); - } - - /** - * CNN without bias test - */ - @Test - @DisplayName("Import Cnn No Bias Tf Keras 2") - void importCnnNoBiasTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/cnn_no_bias/mnist_cnn_no_bias_tf_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/cnn_no_bias/mnist_cnn_no_bias_tf_keras_2_inputs_and_outputs.h5"; - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, true, false); - } - - @Test - @DisplayName("Import Sparse Xent") - void importSparseXent(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/simple_sparse_xent/simple_sparse_xent_mlp_keras_2_model.h5"; - String inputsOutputPath = "modelimport/keras/examples/simple_sparse_xent/simple_sparse_xent_mlp_keras_2_inputs_and_outputs.h5"; - MultiLayerNetwork net = importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, true, true); - Layer outLayer = net.getOutputLayer(); - assertTrue(outLayer instanceof org.deeplearning4j.nn.layers.LossLayer); - LossLayer llConf = (LossLayer) outLayer.getConfig(); - assertEquals(new LossSparseMCXENT(), llConf.getLossFn()); - } - - /** - * GAN import tests - */ - @Test - @DisplayName("Import Dcgan Mnist Discriminator") - void importDcganMnistDiscriminator(@TempDir Path tempDir) throws Exception { - importSequentialModelH5Test(tempDir,"modelimport/keras/examples/mnist_dcgan/dcgan_discriminator_epoch_50.h5"); - } - - @Test - @Disabled("Neither keras or tfkeras can load this.") - @DisplayName("Import Dcgan Mnist Generator") - void importDcganMnistGenerator(@TempDir Path tempDir) throws Exception { - importSequentialModelH5Test(tempDir,"modelimport/keras/examples/mnist_dcgan/dcgan_generator_epoch_50.h5"); - } - - /** - * Auxillary classifier GAN import test - */ - @Test - @DisplayName("Import Acgan Discriminator") - void importAcganDiscriminator(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/acgan/acgan_discriminator_1_epochs.h5"); - // NHWC - INDArray input = Nd4j.create(10, 28, 28, 1); - INDArray[] output = model.output(input); - } - - // AB 2020/04/22 Ignored until Keras model import updated to use NHWC support - @Test - @DisplayName("Import Acgan Generator") - void importAcganGenerator(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/acgan/acgan_generator_1_epochs.h5"); - // System.out.println(model.summary()) ; - INDArray latent = Nd4j.create(10, 100); - INDArray label = Nd4j.create(10, 1); - INDArray[] output = model.output(latent, label); - } - - @Test - @DisplayName("Import Acgan Combined") - void importAcganCombined(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/acgan/acgan_combined_1_epochs.h5"); - // TODO: imports, but incorrectly. Has only one input, should have two. - } - - /** - * Deep convolutional GAN import test - */ - @Test - @DisplayName("Import Dcgan Discriminator") - void importDcganDiscriminator(@TempDir Path tempDir) throws Exception { - importSequentialModelH5Test(tempDir,"modelimport/keras/examples/gans/dcgan_discriminator.h5"); - } - - @Test - @DisplayName("Import Dcgan Generator") - void importDcganGenerator(@TempDir Path tempDir) throws Exception { - importSequentialModelH5Test(tempDir,"modelimport/keras/examples/gans/dcgan_generator.h5"); - } - - /** - * Wasserstein GAN import test - */ - @Test - @DisplayName("Import Wgan Discriminator") - void importWganDiscriminator(@TempDir Path tempDir) throws Exception { - for (int i = 0; i < 100; i++) { - // run a few times to make sure HDF5 doesn't crash - importSequentialModelH5Test(tempDir,"modelimport/keras/examples/gans/wgan_discriminator.h5"); - } - } - - @Test - @DisplayName("Import Wgan Generator") - void importWganGenerator(@TempDir Path tempDir) throws Exception { - importSequentialModelH5Test(tempDir,"modelimport/keras/examples/gans/wgan_generator.h5"); - } - - @Test - @DisplayName("Import Cnn 1 d") - void importCnn1d(@TempDir Path tempDir) throws Exception { - importSequentialModelH5Test(tempDir,"modelimport/keras/examples/cnn1d/cnn1d_flatten_tf_keras2.h5"); - } - - /** - * DGA classifier test - */ - @Test - @DisplayName("Import Dga Classifier") - void importDgaClassifier(@TempDir Path tempDir) throws Exception { - importSequentialModelH5Test(tempDir,"modelimport/keras/examples/dga_classifier/keras2_dga_classifier_tf_model.h5"); - } - - /** - * Reshape flat input into 3D to fit into an LSTM model - */ - @Test - @DisplayName("Import Flat Into LSTM") - void importFlatIntoLSTM(@TempDir Path tempDir) throws Exception { - importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/reshape_to_rnn/reshape_model.h5"); - } - - /** - * Functional LSTM test - */ - @Test - @DisplayName("Import Functional Lstm Tf Keras 2") - void importFunctionalLstmTfKeras2(@TempDir Path tempDir) throws Exception { - String modelPath = "modelimport/keras/examples/functional_lstm/lstm_functional_tf_keras_2.h5"; - // No training enabled - ComputationGraph graphNoTrain = importFunctionalModelH5Test(tempDir,modelPath, null, false); - System.out.println(graphNoTrain.summary()); - // Training enabled - ComputationGraph graph = importFunctionalModelH5Test(tempDir,modelPath, null, true); - System.out.println(graph.summary()); - // Make predictions - int miniBatch = 32; - // NWC format - with nIn=4, seqLength = 10 - INDArray input = Nd4j.ones(miniBatch, 10, 4); - INDArray[] out = graph.output(input); - // Fit model - graph.fit(new INDArray[] { input }, out); - } - - /** - * U-Net - */ - @Test - @DisplayName("Import Unet Tf Keras 2") - void importUnetTfKeras2(@TempDir Path tempDir) throws Exception { - importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/unet/unet_keras_2_tf.h5", null, true); - } - - /** - * ResNet50 - */ - @Test - @DisplayName("Import Resnet 50") - void importResnet50(@TempDir Path tempDir) throws Exception { - importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5"); - } - - /** - * DenseNet - */ - @Test - @DisplayName("Import Dense Net") - void importDenseNet(@TempDir Path tempDir) throws Exception { - importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/densenet/densenet121_tf_keras_2.h5"); - } - - /** - * SqueezeNet - */ - @Test - @DisplayName("Import Squeeze Net") - void importSqueezeNet(@TempDir Path tempDir) throws Exception { - importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/squeezenet/squeezenet.h5"); - } - - /** - * MobileNet - */ - @Test - @DisplayName("Import Mobile Net") - void importMobileNet(@TempDir Path tempDir) throws Exception { - ComputationGraph graph = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/mobilenet/alternative.hdf5"); - INDArray input = Nd4j.ones(10, 299, 299, 3); - graph.output(input); - } - - /** - * InceptionV3 Keras 2 no top - */ - @Test - @DisplayName("Import Inception Keras 2") - void importInceptionKeras2(@TempDir Path tempDir) throws Exception { - int[] inputShape = new int[] { 299, 299, 3 }; - ComputationGraph graph = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/inception/inception_tf_keras_2.h5", inputShape, false); - // TF = channels last = NHWC - INDArray input = Nd4j.ones(10, 299, 299, 3); - graph.output(input); - System.out.println(graph.summary()); - } - - /** - * InceptionV3 - */ - @Test - @DisplayName("Import Inception") - // note this is actually keras 1 and its input dimension ordering is channels first - // Takes unreasonably long, but works - void importInception(@TempDir Path tempDir) throws Exception { - ComputationGraph graph = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/inception/inception_v3_complete.h5"); - // TH = channels first = NCHW - INDArray input = Nd4j.ones(10, 3, 299, 299); - graph.output(input); - System.out.println(graph.summary()); - } - - /** - * Inception V4 - */ - @Test - @Disabled - @DisplayName("Import Inception V 4") - // Model and weights have about 170mb, too large for test resources and also too excessive to enable as unit test - void importInceptionV4(@TempDir Path testDir) throws Exception { - String modelUrl = DL4JResources.getURLString("models/inceptionv4_keras_imagenet_weightsandconfig.h5"); - File kerasFile = testDir.resolve("inceptionv4_keras_imagenet_weightsandconfig.h5").toFile(); - if (!kerasFile.exists()) { - FileUtils.copyURLToFile(new URL(modelUrl), kerasFile); - kerasFile.deleteOnExit(); - } - int[] inputShape = new int[] { 299, 299, 3 }; - ComputationGraph graph = importFunctionalModelH5Test(testDir,kerasFile.getAbsolutePath(), inputShape, false); - // System.out.println(graph.summary()); - } - - /** - * Xception - */ - @Test - @DisplayName("Import Xception") - void importXception(@TempDir Path tempDir) throws Exception { - int[] inputShape = new int[] { 299, 299, 3 }; - ComputationGraph graph = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/xception/xception_tf_keras_2.h5", inputShape, false); - } - - /** - * Seq2seq model - */ - @Test - @DisplayName("Import Seq 2 Seq") - // does not work yet, needs DL4J enhancements - void importSeq2Seq(@TempDir Path tempDir) throws Exception { - importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/seq2seq/full_model_seq2seq_5549.h5"); - } - - /** - * Import all AlphaGo Zero model variants, i.e. - * - Dual residual architecture - * - Dual convolutional architecture - * - Separate (policy and value) residual architecture - * - Separate (policy and value) convolutional architecture - */ - // AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last - @Test - @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") - @DisplayName("Import Sep Conv Policy") - void importSepConvPolicy(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/agz/sep_conv_policy.h5"); - INDArray input = Nd4j.create(32, 19, 19, 10); - model.output(input); - } - - // AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last - @Test - @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") - @DisplayName("Import Sep Res Policy") - void importSepResPolicy(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/agz/sep_res_policy.h5"); - INDArray input = Nd4j.create(32, 19, 19, 10); - model.output(input); - } - - // AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last - @Test - @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") - @DisplayName("Import Sep Conv Value") - void importSepConvValue(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/agz/sep_conv_value.h5"); - INDArray input = Nd4j.create(32, 19, 19, 10); - model.output(input); - } - - @Test - @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") - @DisplayName("Import Sep Res Value") - void importSepResValue(@TempDir Path tempDir) throws Exception { - String filePath = "C:\\Users\\agibs\\Documents\\GitHub\\keras1-import-test\\sep_res_value.h5"; - KerasModelBuilder builder = new KerasModel().modelBuilder().modelHdf5Filename(filePath).enforceTrainingConfig(false); - KerasModel model = builder.buildModel(); - ComputationGraph compGraph = model.getComputationGraph(); - // ComputationGraph model = importFunctionalModelH5Test("modelimport/keras/examples/agz/sep_res_value.h5"); - INDArray input = Nd4j.create(32, 19, 19, 10); - compGraph.output(input); - } - - // AB 20200427 Bad keras model - Keras JSON has input shape [null, 10, 19, 19] (i.e., NCHW) but all layers are set to channels_last - @Test - @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") - @DisplayName("Import Dual Res") - void importDualRes(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/agz/dual_res.h5"); - INDArray input = Nd4j.create(32, 19, 19, 10); - model.output(input); - } - - @Test - @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") - @DisplayName("Import Dual Conv") - void importDualConv(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/agz/dual_conv.h5"); - INDArray input = Nd4j.create(32, 19, 19, 10); - model.output(input); - } - - /** - * MTCNN - */ - @Test - @DisplayName("Import MTCNN") - void importMTCNN(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/48net_complete.h5"); - } - - @Test - @Disabled("Data and channel layout mismatch. We don't support permuting the weights yet.") - @DisplayName("Test NCHWNWHC Change Import Model") - void testNCHWNWHCChangeImportModel(@TempDir Path tempDir) throws Exception { - ComputationGraph computationGraph = importFunctionalModelH5Test(tempDir,"modelimport/keras/weights/simpleconv2d_model.hdf5"); - computationGraph.output(Nd4j.zeros(1, 1, 28, 28)); - } - - @Test - @DisplayName("Import MTCNN 2 D") - // TODO: fails, since we can't use OldSoftMax on >2D data (here: convolution layer) - // TODO: also related to #6339, fix this together - void importMTCNN2D(@TempDir Path tempDir) throws Exception { - ComputationGraph model = importFunctionalModelH5Test(tempDir,"modelimport/keras/examples/12net.h5", new int[] { 24, 24, 3 }, false); - INDArray input = Nd4j.create(10, 24, 24, 3); - model.output(input); - // System.out.println(model.summary()); - } - - /** - * Masking layers (simple Masking into LSTM) - */ - @Test - @DisplayName("Test Masking Zero Value") - void testMaskingZeroValue(@TempDir Path tempDir) throws Exception { - MultiLayerNetwork model = importSequentialModelH5Test(tempDir,"modelimport/keras/examples/masking/masking_zero_lstm.h5"); - model.summary(); - } - - @Test - @DisplayName("Test Masking Two Value") - void testMaskingTwoValue(@TempDir Path tempDir) throws Exception { - MultiLayerNetwork model = importSequentialModelH5Test(tempDir,"modelimport/keras/examples/masking/masking_two_lstm.h5"); - model.summary(); - } - - @Test - @DisplayName("Test Causal Conv 1 D") - void testCausalConv1D(@TempDir Path tempDir) throws Exception { - String[] names = new String[] { "causal_conv1d_k2_s1_d1_cl_model.h5", "causal_conv1d_k2_s1_d2_cl_model.h5", "causal_conv1d_k2_s2_d1_cl_model.h5", "causal_conv1d_k2_s3_d1_cl_model.h5", "causal_conv1d_k3_s1_d1_cl_model.h5", "causal_conv1d_k3_s1_d2_cl_model.h5", "causal_conv1d_k3_s2_d1_cl_model.h5", "causal_conv1d_k3_s3_d1_cl_model.h5", "causal_conv1d_k4_s1_d1_cl_model.h5", "causal_conv1d_k4_s1_d2_cl_model.h5", "causal_conv1d_k4_s2_d1_cl_model.h5", "causal_conv1d_k4_s3_d1_cl_model.h5" }; - for (String name : names) { - System.out.println("Starting test: " + name); - String modelPath = "modelimport/keras/examples/causal_conv1d/" + name; - String inputsOutputPath = "modelimport/keras/examples/causal_conv1d/" + (name.substring(0, name.length() - "model.h5".length()) + "inputs_and_outputs.h5"); - // TODO: - /** - * Difference in weights. Same elements, but loaded differently. Likely acceptable difference. Need to confirm though. - */ - MultiLayerNetwork net = importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, true, true, false, null, null); - Layer l = net.getLayer(0); - Convolution1DLayer c1d = (Convolution1DLayer) l.getConfig(); - assertEquals(ConvolutionMode.Causal, c1d.getConvolutionMode()); - } - } - - @Test - @DisplayName("Test Conv 1 D") - void testConv1D(@TempDir Path tempDir) throws Exception { - String[] names = new String[] { "conv1d_k2_s1_d1_cf_same_model.h5", "conv1d_k2_s1_d1_cf_valid_model.h5", "conv1d_k2_s1_d1_cl_same_model.h5", "conv1d_k2_s1_d1_cl_valid_model.h5", "conv1d_k2_s1_d2_cf_same_model.h5", "conv1d_k2_s1_d2_cf_valid_model.h5", "conv1d_k2_s1_d2_cl_same_model.h5", "conv1d_k2_s1_d2_cl_valid_model.h5", "conv1d_k2_s2_d1_cf_same_model.h5", "conv1d_k2_s2_d1_cf_valid_model.h5", "conv1d_k2_s2_d1_cl_same_model.h5", "conv1d_k2_s2_d1_cl_valid_model.h5", "conv1d_k2_s3_d1_cf_same_model.h5", "conv1d_k2_s3_d1_cf_valid_model.h5", "conv1d_k2_s3_d1_cl_same_model.h5", "conv1d_k2_s3_d1_cl_valid_model.h5", "conv1d_k3_s1_d1_cf_same_model.h5", "conv1d_k3_s1_d1_cf_valid_model.h5", "conv1d_k3_s1_d1_cl_same_model.h5", "conv1d_k3_s1_d1_cl_valid_model.h5", "conv1d_k3_s1_d2_cf_same_model.h5", "conv1d_k3_s1_d2_cf_valid_model.h5", "conv1d_k3_s1_d2_cl_same_model.h5", "conv1d_k3_s1_d2_cl_valid_model.h5", "conv1d_k3_s2_d1_cf_same_model.h5", "conv1d_k3_s2_d1_cf_valid_model.h5", "conv1d_k3_s2_d1_cl_same_model.h5", "conv1d_k3_s2_d1_cl_valid_model.h5", "conv1d_k3_s3_d1_cf_same_model.h5", "conv1d_k3_s3_d1_cf_valid_model.h5", "conv1d_k3_s3_d1_cl_same_model.h5", "conv1d_k3_s3_d1_cl_valid_model.h5", "conv1d_k4_s1_d1_cf_same_model.h5", "conv1d_k4_s1_d1_cf_valid_model.h5", "conv1d_k4_s1_d1_cl_same_model.h5", "conv1d_k4_s1_d1_cl_valid_model.h5", "conv1d_k4_s1_d2_cf_same_model.h5", "conv1d_k4_s1_d2_cf_valid_model.h5", "conv1d_k4_s1_d2_cl_same_model.h5", "conv1d_k4_s1_d2_cl_valid_model.h5", "conv1d_k4_s2_d1_cf_same_model.h5", "conv1d_k4_s2_d1_cf_valid_model.h5", "conv1d_k4_s2_d1_cl_same_model.h5", "conv1d_k4_s2_d1_cl_valid_model.h5", "conv1d_k4_s3_d1_cf_same_model.h5", "conv1d_k4_s3_d1_cf_valid_model.h5", "conv1d_k4_s3_d1_cl_same_model.h5", "conv1d_k4_s3_d1_cl_valid_model.h5" }; - for (String name : names) { - System.out.println("Starting test: " + name); - String modelPath = "modelimport/keras/examples/conv1d/" + name; - String inputsOutputPath = "modelimport/keras/examples/conv1d/" + (name.substring(0, name.length() - "model.h5".length()) + "inputs_and_outputs.h5"); - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, true, true, false, null, // f, f2); - null); - } - } - - @Test - @DisplayName("Test Activation Layers") - void testActivationLayers(@TempDir Path tempDir) throws Exception { - String[] names = new String[] { "ELU_0_model.h5", "LeakyReLU_0_model.h5", "ReLU_0_model.h5", "ReLU_1_model.h5", "ReLU_2_model.h5", "ReLU_3_model.h5", "Softmax_0_model.h5", "ThresholdReLU_0_model.h5" }; - for (String name : names) { - System.out.println("Starting test: " + name); - String modelPath = "modelimport/keras/examples/activations/" + name; - String inputsOutputPath = "modelimport/keras/examples/activations/" + (name.substring(0, name.length() - "model.h5".length()) + "inputs_and_outputs.h5"); - importEndModelTest(tempDir,modelPath, inputsOutputPath, true, true, true, true, false, null, null); - } - } - - private ComputationGraph importFunctionalModelH5Test(Path tempDir,String modelPath) throws Exception { - return importFunctionalModelH5Test(tempDir,modelPath, null, false); - } - - private ComputationGraph importFunctionalModelH5Test(Path tempDir,String modelPath, int[] inputShape, boolean train) throws Exception { - File modelFile; - try (InputStream is = Resources.asStream(modelPath)) { - modelFile = createTempFile(tempDir,TEMP_MODEL_FILENAME, H5_EXTENSION); - Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - } - KerasModelBuilder builder = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()).enforceTrainingConfig(train); - if (inputShape != null) { - builder.inputShape(inputShape); - } - KerasModel model = builder.buildModel(); - return model.getComputationGraph(); - } - - private MultiLayerNetwork importSequentialModelH5Test(Path tempDir,String modelPath) throws Exception { - return importSequentialModelH5Test(tempDir,modelPath, null); - } - - private MultiLayerNetwork importSequentialModelH5Test(Path tempDir,String modelPath, int[] inputShape) throws Exception { - try (InputStream is = Resources.asStream(modelPath)) { - File modelFile = createTempFile(tempDir,TEMP_MODEL_FILENAME, H5_EXTENSION); - Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - KerasModelBuilder builder = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()).enforceTrainingConfig(false); - if (inputShape != null) { - builder.inputShape(inputShape); - } - KerasSequentialModel model = builder.buildSequential(); - return model.getMultiLayerNetwork(); - } - } - - public MultiLayerNetwork importEndModelTest(Path tempDir,String modelPath, String inputsOutputsPath, boolean tfOrdering, boolean checkPredictions, boolean checkGradients, boolean enforceTrainingConfig) throws Exception { - return importEndModelTest(tempDir,modelPath, inputsOutputsPath, tfOrdering, checkPredictions, checkGradients, true, enforceTrainingConfig, null, null); - } - - public MultiLayerNetwork importEndModelTest(Path tempDir,String modelPath, String inputsOutputsPath, boolean tfOrdering, boolean checkPredictions, boolean checkGradients, boolean enforceTrainingConfig, boolean checkAuc, Function inputPreProc, BiFunction expectedPreProc) throws Exception { - MultiLayerNetwork model; - try (InputStream is = Resources.asStream(modelPath)) { - File modelFile = createTempFile(tempDir,TEMP_MODEL_FILENAME, H5_EXTENSION); - Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - KerasSequentialModel kerasModel = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()).enforceTrainingConfig(enforceTrainingConfig).buildSequential(); - model = kerasModel.getMultiLayerNetwork(); - } - File outputsFile = createTempFile(tempDir,TEMP_OUTPUTS_FILENAME, H5_EXTENSION); - try (InputStream is = Resources.asStream(inputsOutputsPath)) { - Files.copy(is, outputsFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - } - try (Hdf5Archive outputsArchive = new Hdf5Archive(outputsFile.getAbsolutePath())) { - if (checkPredictions) { - INDArray input = getInputs(outputsArchive, tfOrdering)[0]; - if (inputPreProc != null) - input = inputPreProc.apply(input); - Map activationsKeras = getActivations(outputsArchive, tfOrdering); - for (int i = 0; i < model.getLayers().length; i++) { - String layerName = model.getLayerNames().get(i); - if (activationsKeras.containsKey(layerName)) { - INDArray activationsDl4j = model.feedForwardToLayer(i, input, false).get(i + 1); - long[] shape = activationsDl4j.shape(); - INDArray exp = activationsKeras.get(layerName); - Nd4j.getExecutioner().enableDebugMode(true); - Nd4j.getExecutioner().enableVerboseMode(true); - if (expectedPreProc != null) - exp = expectedPreProc.apply(layerName, exp); - compareINDArrays(layerName, exp, activationsDl4j, EPS); - } - } - INDArray predictionsKeras = getPredictions(outputsArchive, tfOrdering)[0]; - INDArray predictionsDl4j = model.output(input, false); - if (expectedPreProc != null) - predictionsKeras = expectedPreProc.apply("output", predictionsKeras); - compareINDArrays("predictions", predictionsKeras, predictionsDl4j, EPS); - INDArray outputs = getOutputs(outputsArchive, true)[0]; - if (outputs.rank() == 1) { - outputs = outputs.reshape(outputs.length(), 1); - } - val nOut = (int) outputs.size(-1); - if (checkAuc) - compareMulticlassAUC("predictions", outputs, predictionsKeras, predictionsDl4j, nOut, EPS); - } - if (checkGradients && !SKIP_GRAD_CHECKS) { - Random r = new Random(12345); - INDArray input = getInputs(outputsArchive, tfOrdering)[0]; - INDArray predictionsDl4j = model.output(input, false); - // Infer one-hot labels... this probably won't work for all - INDArray testLabels = Nd4j.create(predictionsDl4j.shape()); - if (testLabels.rank() == 2) { - for (int i = 0; i < testLabels.size(0); i++) { - testLabels.putScalar(i, r.nextInt((int) testLabels.size(1)), 1.0); - } - } else if (testLabels.rank() == 3) { - for (int i = 0; i < testLabels.size(0); i++) { - for (int j = 0; j < testLabels.size(1); j++) { - testLabels.putScalar(i, j, r.nextInt((int) testLabels.size(1)), 1.0); - } - } - } else { - throw new RuntimeException("Cannot gradient check 4d output array"); - } - checkGradients(model, input, testLabels); - } - } - return model; - } - - private static INDArray[] getInputs(Hdf5Archive archive, boolean tensorFlowImageDimOrdering) throws Exception { - List inputNames = (List) KerasModelUtils.parseJsonString(archive.readAttributeAsJson(GROUP_ATTR_INPUTS)).get(GROUP_ATTR_INPUTS); - INDArray[] inputs = new INDArray[inputNames.size()]; - for (int i = 0; i < inputNames.size(); i++) { - inputs[i] = archive.readDataSet(inputNames.get(i), GROUP_ATTR_INPUTS); - } - return inputs; - } - - private static Map getActivations(Hdf5Archive archive, boolean tensorFlowImageDimOrdering) throws Exception { - Map activations = new HashMap<>(); - for (String layerName : archive.getDataSets(GROUP_ACTIVATIONS)) { - INDArray activation = archive.readDataSet(layerName, GROUP_ACTIVATIONS); - activations.put(layerName, activation); - } - return activations; - } - - private static INDArray[] getOutputs(Hdf5Archive archive, boolean tensorFlowImageDimOrdering) throws Exception { - List outputNames = (List) KerasModelUtils.parseJsonString(archive.readAttributeAsJson(GROUP_ATTR_OUTPUTS)).get(GROUP_ATTR_OUTPUTS); - INDArray[] outputs = new INDArray[outputNames.size()]; - for (int i = 0; i < outputNames.size(); i++) { - outputs[i] = archive.readDataSet(outputNames.get(i), GROUP_ATTR_OUTPUTS); - } - return outputs; - } - - private static INDArray[] getPredictions(Hdf5Archive archive, boolean tensorFlowImageDimOrdering) throws Exception { - List outputNames = (List) KerasModelUtils.parseJsonString(archive.readAttributeAsJson(GROUP_ATTR_OUTPUTS)).get(GROUP_ATTR_OUTPUTS); - INDArray[] predictions = new INDArray[outputNames.size()]; - for (int i = 0; i < outputNames.size(); i++) { - predictions[i] = archive.readDataSet(outputNames.get(i), GROUP_PREDICTIONS); - } - return predictions; - } - - private static void compareINDArrays(String label, INDArray expected, INDArray actual, double eps) { - if (!expected.equalShapes(actual)) { - throw new IllegalStateException("Shapes do not match for \"" + label + "\": got " + Arrays.toString(expected.shape()) + " vs " + Arrays.toString(actual.shape())); - } - INDArray diff = expected.sub(actual.castTo(expected.dataType())); - double min = diff.minNumber().doubleValue(); - double max = diff.maxNumber().doubleValue(); - log.info(label + ": " + expected.equalsWithEps(actual, eps) + ", " + min + ", " + max); - double threshold = 1e-7; - double aAbsMax = Math.max(Math.abs(expected.minNumber().doubleValue()), Math.abs(expected.maxNumber().doubleValue())); - double bAbsMax = Math.max(Math.abs(actual.minNumber().doubleValue()), Math.abs(actual.maxNumber().doubleValue())); - // skip too small absolute inputs - if (Math.abs(aAbsMax) > threshold && Math.abs(bAbsMax) > threshold) { - boolean eq = expected.equalsWithEps(actual.castTo(expected.dataType()), eps); - if (!eq) { - System.out.println("Expected: " + Arrays.toString(expected.shape()) + ", actual: " + Arrays.toString(actual.shape())); - System.out.println("Expected:\n" + expected); - System.out.println("Actual: \n" + actual); - } - assertTrue(eq,"Output differs: " + label); - } - } - - private static void compareMulticlassAUC(String label, INDArray target, INDArray a, INDArray b, int nbClasses, double eps) { - ROCMultiClass evalA = new ROCMultiClass(100); - evalA.eval(target, a); - double avgAucA = evalA.calculateAverageAUC(); - ROCMultiClass evalB = new ROCMultiClass(100); - evalB.eval(target, b); - double avgAucB = evalB.calculateAverageAUC(); - assertEquals(avgAucA, avgAucB, EPS); - double[] aucA = new double[nbClasses]; - double[] aucB = new double[nbClasses]; - if (nbClasses > 1) { - for (int i = 0; i < nbClasses; i++) { - aucA[i] = evalA.calculateAUC(i); - aucB[i] = evalB.calculateAUC(i); - } - assertArrayEquals(aucA, aucB, EPS); - } - } - - public static void checkGradients(MultiLayerNetwork net, INDArray input, INDArray labels) { - double eps = 1e-6; - double max_rel_error = 1e-3; - double min_abs_error = 1e-8; - MultiLayerNetwork netToTest; - if (net.getOutputLayer() instanceof IOutputLayer) { - netToTest = net; - } else { - org.deeplearning4j.nn.conf.layers.Layer l; - if (labels.rank() == 2) { - l = new LossLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).build(); - } else { - // Rank 3 - l = new RnnOutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).nIn(labels.size(1)).nOut(labels.size(1)).build(); - } - netToTest = new TransferLearning.Builder(net).fineTuneConfiguration(new FineTuneConfiguration.Builder().updater(new NoOp()).dropOut(0.0).build()).addLayer(l).build(); - } - log.info("Num params: " + net.numParams()); - for (Layer l : netToTest.getLayers()) { - // Remove any dropout manually - until this is fixed: - // https://github.com/eclipse/deeplearning4j/issues/4368 - l.conf().getLayer().setIDropout(null); - // Also swap out activation functions... this is a bit of a hack, but should make the net gradient checkable... - if (l.conf().getLayer() instanceof FeedForwardLayer) { - FeedForwardLayer ffl = (FeedForwardLayer) l.conf().getLayer(); - IActivation activation = ffl.getActivationFn(); - if (activation instanceof ActivationReLU || activation instanceof ActivationLReLU) { - ffl.setActivationFn(new ActivationSoftPlus()); - } else if (activation instanceof ActivationHardTanH) { - ffl.setActivationFn(new ActivationTanH()); - } - } - } - Nd4j.setDataType(DataType.DOUBLE); - boolean passed = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(netToTest).input(input).labels(labels).subset(true).maxPerParam(9)); - assertTrue(passed, "Gradient check failed"); - } - - private File createTempFile(Path testDir,String prefix, String suffix) throws IOException { - File ret = new File(testDir.toFile(),prefix + "-" + System.nanoTime() + suffix); - ret.createNewFile(); - ret.deleteOnExit(); - return ret; - } -} diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000PredictTest.java b/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000PredictTest.java deleted file mode 100644 index e394e188c..000000000 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000PredictTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.modelimport.keras.e2e; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.modelimport.keras.KerasLayer; -import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; -import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasSpaceToDepth; -import org.deeplearning4j.nn.transferlearning.TransferLearning; -import org.deeplearning4j.util.ModelSerializer; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.api.preprocessor.ImagePreProcessingScaler; -import org.nd4j.linalg.factory.Nd4j; -import java.io.File; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; - -@Slf4j -@DisplayName("Keras Yolo 9000 Predict Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasYolo9000PredictTest extends BaseDL4JTest { - - private static final String DL4J_MODEL_FILE_NAME = "."; - - private static ImagePreProcessingScaler IMAGE_PREPROCESSING_SCALER = new ImagePreProcessingScaler(0, 1); - - @Test - @Disabled("Need to manually download file for ylo.") - @DisplayName("Test Yolo Prediction Import") - void testYoloPredictionImport() throws Exception { - int HEIGHT = 416; - int WIDTH = 416; - INDArray indArray = Nd4j.create(HEIGHT, WIDTH, 3); - IMAGE_PREPROCESSING_SCALER.transform(indArray); - KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); - String h5_FILENAME = "modelimport/keras/examples/yolo/yolo-voc.h5"; - ComputationGraph graph = KerasModelImport.importKerasModelAndWeights(h5_FILENAME, false); - double[][] priorBoxes = { { 1.3221, 1.73145 }, { 3.19275, 4.00944 }, { 5.05587, 8.09892 }, { 9.47112, 4.84053 }, { 11.2364, 10.0071 } }; - INDArray priors = Nd4j.create(priorBoxes); - ComputationGraph model = new TransferLearning.GraphBuilder(graph).addLayer("outputs", new org.deeplearning4j.nn.conf.layers.objdetect.Yolo2OutputLayer.Builder().boundingBoxPriors(priors).build(), "conv2d_23").setOutputs("outputs").build(); - ModelSerializer.writeModel(model, DL4J_MODEL_FILE_NAME, false); - ComputationGraph computationGraph = ModelSerializer.restoreComputationGraph(new File(DL4J_MODEL_FILE_NAME)); - System.out.println(computationGraph.summary(InputType.convolutional(416, 416, 3))); - INDArray results = computationGraph.outputSingle(indArray); - } -} diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000Test.java b/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000Test.java deleted file mode 100644 index 70aea9015..000000000 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/e2e/KerasYolo9000Test.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.modelimport.keras.e2e; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.modelimport.keras.KerasLayer; -import org.deeplearning4j.nn.modelimport.keras.KerasModel; -import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasSpaceToDepth; -import org.junit.jupiter.api.Disabled; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.resources.Resources; -import java.io.File; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; -import org.junit.jupiter.api.DisplayName; -import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -@Slf4j -@DisplayName("Keras Yolo 9000 Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasYolo9000Test extends BaseDL4JTest { - - private static final String TEMP_MODEL_FILENAME = "tempModel"; - - private static final String H5_EXTENSION = ".h5"; - - @TempDir - public Path testDir; - - @Disabled - @Test - @DisplayName("Test Custom Layer Yolo Import") - // TODO: yolo and yolo-voc output are too large for github, find smaller equivalents - void testCustomLayerYoloImport() throws Exception { - KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); - String modelPath = "modelimport/keras/examples/yolo/yolo.h5"; - try (InputStream is = Resources.asStream(modelPath)) { - File modelFile = testDir.resolve(TEMP_MODEL_FILENAME + System.currentTimeMillis() + H5_EXTENSION).toFile(); - Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - ComputationGraph model = new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()).enforceTrainingConfig(false).buildModel().getComputationGraph(); - System.out.println(model.summary()); - } - } -} diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/flatten/KerasFlatten3dTest.java b/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/flatten/KerasFlatten3dTest.java deleted file mode 100644 index cdf240e3a..000000000 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/layers/flatten/KerasFlatten3dTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.modelimport.keras.layers.flatten; - -import org.deeplearning4j.nn.conf.InputPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.Cnn3DToFeedForwardPreProcessor; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.graph.vertex.GraphVertex; -import org.deeplearning4j.nn.graph.vertex.impl.PreprocessorVertex; -import org.deeplearning4j.nn.modelimport.keras.KerasModelImport; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.io.ClassPathResource; -import java.io.InputStream; -import static org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.extension.ExtendWith; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -@DisplayName("Keras Flatten 3 d Test") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -class KerasFlatten3dTest { - - @Test - @DisplayName("Test Flatten 3 d") - void testFlatten3d() throws Exception { - ClassPathResource classPathResource = new ClassPathResource("modelimport/keras/weights/flatten_3d.hdf5"); - try (InputStream inputStream = classPathResource.getInputStream()) { - ComputationGraph computationGraph = KerasModelImport.importKerasModelAndWeights(inputStream); - assertNotNull(computationGraph); - assertEquals(3, computationGraph.getVertices().length); - GraphVertex[] vertices = computationGraph.getVertices(); - assertTrue(vertices[1] instanceof PreprocessorVertex); - PreprocessorVertex preprocessorVertex = (PreprocessorVertex) vertices[1]; - InputPreProcessor preProcessor = preprocessorVertex.getPreProcessor(); - assertTrue(preProcessor instanceof Cnn3DToFeedForwardPreProcessor); - Cnn3DToFeedForwardPreProcessor cnn3DToFeedForwardPreProcessor = (Cnn3DToFeedForwardPreProcessor) preProcessor; - assertTrue(cnn3DToFeedForwardPreProcessor.isNCDHW()); - assertEquals(10, cnn3DToFeedForwardPreProcessor.getInputDepth()); - assertEquals(10, cnn3DToFeedForwardPreProcessor.getInputHeight()); - assertEquals(1, cnn3DToFeedForwardPreProcessor.getNumChannels()); - assertEquals(10, cnn3DToFeedForwardPreProcessor.getInputWidth()); - System.out.println(cnn3DToFeedForwardPreProcessor); - } - } -} diff --git a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/weights/KerasWeightSettingTests.java b/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/weights/KerasWeightSettingTests.java deleted file mode 100644 index 525b68da0..000000000 --- a/deeplearning4j/deeplearning4j-modelimport/src/test/java/org/deeplearning4j/nn/modelimport/keras/weights/KerasWeightSettingTests.java +++ /dev/null @@ -1,373 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.modelimport.keras.weights; - -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.modelimport.keras.KerasLayer; -import org.deeplearning4j.nn.modelimport.keras.KerasModel; -import org.deeplearning4j.nn.modelimport.keras.layers.convolutional.KerasSpaceToDepth; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.resources.Resources; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.Arrays; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - -@Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.KERAS) -@NativeTag -public class KerasWeightSettingTests extends BaseDL4JTest { - - - @Override - public long getTimeoutMilliseconds() { - return 9999999L; - } - - @Test - public void testSimpleLayersWithWeights(@TempDir Path tempDir) throws Exception { - int[] kerasVersions = new int[]{1, 2}; - String[] backends = new String[]{"tensorflow", "theano"}; - - for (int version : kerasVersions) { - for (String backend : backends) { - String densePath = "modelimport/keras/weights/dense_" + backend + "_" + version + ".h5"; - importDense(tempDir,densePath); - - String conv2dPath = "modelimport/keras/weights/conv2d_" + backend + "_" + version + ".h5"; - importConv2D(tempDir,conv2dPath); - - if (version == 2 && backend.equals("tensorflow")) { // TODO should work for theano - String conv2dReshapePath = "modelimport/keras/weights/conv2d_reshape_" - + backend + "_" + version + ".h5"; - System.out.println(backend + "_" + version); - importConv2DReshape(tempDir,conv2dReshapePath); - } - - if (version == 2) { - String conv1dFlattenPath = "modelimport/keras/weights/embedding_conv1d_flatten_" - + backend + "_" + version + ".h5"; - importConv1DFlatten(tempDir,conv1dFlattenPath); - } - - String lstmPath = "modelimport/keras/weights/lstm_" + backend + "_" + version + ".h5"; - importLstm(tempDir,lstmPath); - - String embeddingLstmPath = "modelimport/keras/weights/embedding_lstm_" - + backend + "_" + version + ".h5"; - importEmbeddingLstm(tempDir,embeddingLstmPath); - - - if (version == 2) { - String embeddingConv1dExtendedPath = "modelimport/keras/weights/embedding_conv1d_extended_" - + backend + "_" + version + ".h5"; - importEmbeddingConv1DExtended(tempDir,embeddingConv1dExtendedPath); - } - - if (version == 2) { - String embeddingConv1dPath = "modelimport/keras/weights/embedding_conv1d_" - + backend + "_" + version + ".h5"; - importEmbeddingConv1D(tempDir,embeddingConv1dPath); - } - - String simpleRnnPath = "modelimport/keras/weights/simple_rnn_" + backend + "_" + version + ".h5"; - importSimpleRnn(tempDir,simpleRnnPath); - - String bidirectionalLstmPath = "modelimport/keras/weights/bidirectional_lstm_" - + backend + "_" + version + ".h5"; - importBidirectionalLstm(tempDir,bidirectionalLstmPath); - - String bidirectionalLstmNoSequencesPath = - "modelimport/keras/weights/bidirectional_lstm_no_return_sequences_" - + backend + "_" + version + ".h5"; - importBidirectionalLstm(tempDir,bidirectionalLstmNoSequencesPath); - - if (version == 2 && backend.equals("tensorflow")) { - String batchToConv2dPath = "modelimport/keras/weights/batch_to_conv2d_" - + backend + "_" + version + ".h5"; - importBatchNormToConv2D(tempDir,batchToConv2dPath); - } - - if (backend.equals("tensorflow") && version == 2) { // TODO should work for theano - String simpleSpaceToBatchPath = "modelimport/keras/weights/space_to_depth_simple_" - + backend + "_" + version + ".h5"; - importSimpleSpaceToDepth(tempDir,simpleSpaceToBatchPath); - } - - if (backend.equals("tensorflow") && version == 2) { - String graphSpaceToBatchPath = "modelimport/keras/weights/space_to_depth_graph_" - + backend + "_" + version + ".h5"; - importGraphSpaceToDepth(tempDir,graphSpaceToBatchPath); - } - - if (backend.equals("tensorflow") && version == 2) { - String sepConvPath = "modelimport/keras/weights/sepconv2d_" + backend + "_" + version + ".h5"; - importSepConv2D(tempDir,sepConvPath); - } - } - } - } - - private void logSuccess(String modelPath) { - log.info("***** Successfully imported " + modelPath); - } - - private void importDense(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, true); - - INDArray weights = model.getLayer(0).getParam("W"); - val weightShape = weights.shape(); - assertEquals(4, weightShape[0]); - assertEquals(6, weightShape[1]); - - INDArray bias = model.getLayer(0).getParam("b"); - assertEquals(6, bias.length()); - logSuccess(modelPath); - } - - private void importSepConv2D(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - - INDArray depthWeights = model.getLayer(0).getParam("W"); - val depthWeightShape = depthWeights.shape(); - - long depthMult = 2; - long kernel = 3; - long nIn = 5; - long nOut = 6; - - assertEquals(depthMult, depthWeightShape[0]); - assertEquals(nIn, depthWeightShape[1]); - assertEquals(kernel, depthWeightShape[2]); - assertEquals(kernel, depthWeightShape[3]); - - INDArray weights = model.getLayer(0).getParam("pW"); - val weightShape = weights.shape(); - - - assertEquals(nOut, weightShape[0]); - assertEquals(nIn * depthMult, weightShape[1]); - assertEquals(1, weightShape[2]); - assertEquals(1, weightShape[3]); - - INDArray bias = model.getLayer(0).getParam("b"); - assertEquals(6, bias.length()); - - INDArray input = Nd4j.ones(1, 3, 4, 5); //NHWC - INDArray output = model.output(input); - - assertArrayEquals(new long[] {1, 1, 2, 6}, output.shape()); //NHWC - - logSuccess(modelPath); - } - - private void importConv2D(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - - INDArray weights = model.getLayer(0).getParam("W"); - val weightShape = weights.shape(); - assertEquals(6, weightShape[0]); - assertEquals(5, weightShape[1]); - assertEquals(3, weightShape[2]); - assertEquals(3, weightShape[3]); - - INDArray bias = model.getLayer(0).getParam("b"); - assertEquals(6,bias.length()); - logSuccess(modelPath); - } - - - private void importConv2DReshape(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - - - int nOut = 12; - int mb = 10; - ; - int[] inShape = new int[]{5, 5, 5}; - INDArray input = Nd4j.zeros(mb, inShape[0], inShape[1], inShape[2]); - INDArray output = model.output(input); - assertArrayEquals(new long[]{mb, nOut}, output.shape()); - logSuccess(modelPath); - } - - private void importConv1DFlatten(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - - int nOut = 6; - int inputLength = 10; - int mb = 42; - int kernel = 3; - - INDArray input = Nd4j.zeros(mb, inputLength); - INDArray output = model.output(input); - if(modelPath.contains("tensorflow")) - assertArrayEquals(new long[]{mb, inputLength - kernel + 1, nOut}, output.shape()); //NWC - else if(modelPath.contains("theano")) { - assertArrayEquals(new long[]{mb, nOut,inputLength - kernel + 1}, output.shape()); //NCW - - } - logSuccess(modelPath); - } - - private void importBatchNormToConv2D(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - model.summary(); - logSuccess(modelPath); - } - - private void importSimpleSpaceToDepth(Path tempDir,String modelPath) throws Exception { - KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - - INDArray input = Nd4j.zeros(10, 6, 6, 4); - INDArray output = model.output(input); - assertArrayEquals(new long[]{10, 3, 3, 16}, output.shape()); - logSuccess(modelPath); - } - - private void importGraphSpaceToDepth(Path tempDir,String modelPath) throws Exception { - KerasLayer.registerCustomLayer("Lambda", KerasSpaceToDepth.class); - ComputationGraph model = loadComputationalGraph(tempDir,modelPath, false); - -// INDArray input[] = new INDArray[]{Nd4j.zeros(10, 4, 6, 6), Nd4j.zeros(10, 16, 3, 3)}; - INDArray input[] = new INDArray[]{Nd4j.zeros(10, 6, 6, 4), Nd4j.zeros(10, 3, 3, 16)}; - INDArray[] output = model.output(input); - log.info(Arrays.toString(output[0].shape())); - assertArrayEquals(new long[]{10, 3, 3, 32}, output[0].shape()); - logSuccess(modelPath); - } - - private void importLstm(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - model.summary(); - // TODO: check weights - logSuccess(modelPath); - } - - private void importEmbeddingLstm(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - - int nIn = 4; - int nOut = 6; - int outputDim = 5; - int inputLength = 10; - int mb = 42; - - INDArray embeddingWeight = model.getLayer(0).getParam("W"); - val embeddingWeightShape = embeddingWeight.shape(); - assertEquals(nIn, embeddingWeightShape[0]); - assertEquals(outputDim, embeddingWeightShape[1]); - - INDArray inEmbedding = Nd4j.zeros(mb, inputLength); - INDArray output = model.output(inEmbedding); - assertArrayEquals(new long[]{mb, inputLength, nOut}, output.shape()); //NWC format - logSuccess(modelPath); - } - - private void importEmbeddingConv1DExtended(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - logSuccess(modelPath); - } - - private void importEmbeddingConv1D(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - - int nIn = 4; - int nOut = 6; - int outputDim = 5; - int inputLength = 10; - int kernel = 3; - int mb = 42; - - INDArray embeddingWeight = model.getLayer(0).getParam("W"); - val embeddingWeightShape = embeddingWeight.shape(); - assertEquals(nIn, embeddingWeightShape[0]); - assertEquals(outputDim, embeddingWeightShape[1]); - - INDArray inEmbedding = Nd4j.zeros(mb, inputLength); - INDArray output = model.output(inEmbedding); - if(modelPath.contains("tensorflow")) - assertArrayEquals(new long[]{mb, inputLength - kernel + 1, nOut}, output.shape()); //NWC - else if(modelPath.contains("theano")) - assertArrayEquals(new long[]{mb, nOut,inputLength - kernel + 1}, output.shape()); //NCC - - logSuccess(modelPath); - } - - private void importSimpleRnn(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - model.summary(); - logSuccess(modelPath); - // TODO: check weights - } - - private void importBidirectionalLstm(Path tempDir,String modelPath) throws Exception { - MultiLayerNetwork model = loadMultiLayerNetwork(tempDir,modelPath, false); - model.summary(); - logSuccess(modelPath); - // TODO: check weights - } - - private MultiLayerNetwork loadMultiLayerNetwork(Path tempDir, String modelPath, boolean training) throws Exception { - File modelFile = createTempFile(tempDir,"temp", ".h5"); - try(InputStream is = Resources.asStream(modelPath)) { - Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - return new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) - .enforceTrainingConfig(training).buildSequential().getMultiLayerNetwork(); - } - } - - private ComputationGraph loadComputationalGraph(Path tempDir,String modelPath, boolean training) throws Exception { - File modelFile = createTempFile(tempDir,"temp", ".h5"); - try(InputStream is = Resources.asStream(modelPath)) { - Files.copy(is, modelFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - return new KerasModel().modelBuilder().modelHdf5Filename(modelFile.getAbsolutePath()) - .enforceTrainingConfig(training).buildModel().getComputationGraph(); - } - } - - private File createTempFile(Path tempDir,String prefix, String suffix) throws IOException { - File createTempFile = Files.createTempFile(tempDir,prefix + "-" + System.nanoTime(),suffix).toFile(); - return createTempFile; - } - -} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/pom.xml b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/pom.xml deleted file mode 100644 index dcadbfa19..000000000 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/pom.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-nlp-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-nlp - - - 0.4 - - - - - org.nd4j - nd4j-native-api - ${nd4j.version} - - - commons-lang - commons-lang - ${commons-lang.version} - - - org.deeplearning4j - deeplearning4j-core - ${project.version} - - - org.threadly - threadly - ${threadly.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - org.hamcrest - hamcrest-core - 1.3 - test - - - org.mockito - mockito-core - ${mockito.version} - test - - - ch.qos.logback - logback-classic - test - - - org.apache.commons - commons-lang3 - ${commonslang.version} - - - com.github.vinhkhuc - jfasttext - ${jfasttext.version} - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java deleted file mode 100644 index 834c24f80..000000000 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/Word2Vec.java +++ /dev/null @@ -1,717 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.models.word2vec; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import lombok.Getter; -import lombok.NonNull; -import org.deeplearning4j.models.embeddings.WeightLookupTable; -import org.deeplearning4j.models.embeddings.learning.ElementsLearningAlgorithm; -import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration; -import org.deeplearning4j.models.embeddings.reader.ModelUtils; -import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; -import org.deeplearning4j.models.sequencevectors.SequenceVectors; -import org.deeplearning4j.models.sequencevectors.interfaces.SequenceIterator; -import org.deeplearning4j.models.sequencevectors.interfaces.VectorsListener; -import org.deeplearning4j.models.sequencevectors.iterators.AbstractSequenceIterator; -import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer; -import org.deeplearning4j.models.word2vec.wordstore.VocabCache; -import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; -import org.deeplearning4j.text.documentiterator.DocumentIterator; -import org.deeplearning4j.text.documentiterator.LabelAwareIterator; -import org.deeplearning4j.text.sentenceiterator.SentenceIterator; -import org.deeplearning4j.text.sentenceiterator.StreamLineIterator; -import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; -import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; -import org.nd4j.shade.jackson.databind.type.CollectionType; - -import java.io.IOException; -import java.util.*; - -public class Word2Vec extends SequenceVectors { - private static final long serialVersionUID = 78249242142L; - - protected transient SentenceIterator sentenceIter; - @Getter - protected transient TokenizerFactory tokenizerFactory; - - /** - * This method defines TokenizerFactory instance to be using during model building - * - * @param tokenizerFactory TokenizerFactory instance - */ - public void setTokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) { - this.tokenizerFactory = tokenizerFactory; - - if (sentenceIter != null) { - SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(sentenceIter) - .tokenizerFactory(this.tokenizerFactory).build(); - this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build(); - } - } - - /** - * This method defines SentenceIterator instance, that will be used as training corpus source - * - * @param iterator SentenceIterator instance - */ - public void setSentenceIterator(@NonNull SentenceIterator iterator) { - //if (tokenizerFactory == null) throw new IllegalStateException("Please call setTokenizerFactory() prior to setSentenceIter() call."); - - if (tokenizerFactory != null) { - SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(iterator) - .tokenizerFactory(tokenizerFactory) - .allowMultithreading(configuration == null || configuration.isAllowParallelTokenization()) - .build(); - this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build(); - } else - log.error("Please call setTokenizerFactory() prior to setSentenceIter() call."); - } - - /** - * This method defines SequenceIterator instance, that will be used as training corpus source. - * Main difference with other iterators here: it allows you to pass already tokenized Sequence for training - * - * @param iterator - */ - public void setSequenceIterator(@NonNull SequenceIterator iterator) { - this.iterator = iterator; - } - - private static ObjectMapper mapper = null; - private static final Object lock = new Object(); - - private static ObjectMapper mapper() { - if (mapper == null) { - synchronized (lock) { - if (mapper == null) { - mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - return mapper; - } - } - } - return mapper; - } - - private static final String CLASS_FIELD = "@class"; - private static final String VOCAB_LIST_FIELD = "VocabCache"; - - public String toJson() throws JsonProcessingException { - - JsonObject retVal = new JsonObject(); - ObjectMapper mapper = mapper(); - - retVal.addProperty(CLASS_FIELD, mapper.writeValueAsString(this.getClass().getName())); - - if (this.vocab instanceof AbstractCache) { - retVal.addProperty(VOCAB_LIST_FIELD, ((AbstractCache) this.vocab).toJson()); - } - - return retVal.toString(); - } - - public static Word2Vec fromJson(String jsonString) throws IOException { - - Word2Vec ret = new Word2Vec(); - - JsonParser parser = new JsonParser(); - JsonObject json = parser.parse(jsonString).getAsJsonObject(); - - VocabCache cache = AbstractCache.fromJson(json.get(VOCAB_LIST_FIELD).getAsString()); - - ret.setVocab(cache); - return ret; - } - - public static class Builder extends SequenceVectors.Builder { - protected SentenceIterator sentenceIterator; - protected LabelAwareIterator labelAwareIterator; - protected TokenizerFactory tokenizerFactory; - protected boolean allowParallelTokenization = true; - - - public Builder() { - - } - - /** - * This method has no effect for Word2Vec - * - * @param vec existing WordVectors model - * @return - */ - @Override - protected Builder useExistingWordVectors(@NonNull WordVectors vec) { - return this; - } - - public Builder(@NonNull VectorsConfiguration configuration) { - super(configuration); - this.allowParallelTokenization = configuration.isAllowParallelTokenization(); - } - - public Builder iterate(@NonNull DocumentIterator iterator) { - this.sentenceIterator = new StreamLineIterator.Builder(iterator).setFetchSize(100).build(); - return this; - } - - /** - * This method used to feed SentenceIterator, that contains training corpus, into ParagraphVectors - * - * @param iterator - * @return - */ - public Builder iterate(@NonNull SentenceIterator iterator) { - this.sentenceIterator = iterator; - return this; - } - - /** - * This method defines TokenizerFactory to be used for strings tokenization during training - * PLEASE NOTE: If external VocabCache is used, the same TokenizerFactory should be used to keep derived tokens equal. - * - * @param tokenizerFactory - * @return - */ - public Builder tokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) { - this.tokenizerFactory = tokenizerFactory; - return this; - } - - /** - * This method used to feed SequenceIterator, that contains training corpus, into ParagraphVectors - * - * @param iterator - * @return - */ - @Override - public Builder iterate(@NonNull SequenceIterator iterator) { - super.iterate(iterator); - return this; - } - - /** - * This method used to feed LabelAwareIterator, that is usually used - * - * @param iterator - * @return - */ - public Builder iterate(@NonNull LabelAwareIterator iterator) { - this.labelAwareIterator = iterator; - return this; - } - - /** - * This method defines mini-batch size - * @param batchSize - * @return - */ - @Override - public Builder batchSize(int batchSize) { - super.batchSize(batchSize); - return this; - } - - /** - * This method defines number of iterations done for each mini-batch during training - * @param iterations - * @return - */ - @Override - public Builder iterations(int iterations) { - super.iterations(iterations); - return this; - } - - /** - * This method defines number of epochs (iterations over whole training corpus) for training - * @param numEpochs - * @return - */ - @Override - public Builder epochs(int numEpochs) { - super.epochs(numEpochs); - return this; - } - - /** - * This method defines number of dimensions for output vectors - * @param layerSize - * @return - */ - @Override - public Builder layerSize(int layerSize) { - super.layerSize(layerSize); - return this; - } - - /** - * This method defines initial learning rate for model training - * - * @param learningRate - * @return - */ - @Override - public Builder learningRate(double learningRate) { - super.learningRate(learningRate); - return this; - } - - /** - * This method defines minimal word frequency in training corpus. All words below this threshold will be removed prior model training - * - * @param minWordFrequency - * @return - */ - @Override - public Builder minWordFrequency(int minWordFrequency) { - super.minWordFrequency(minWordFrequency); - return this; - } - - /** - * This method defines minimal learning rate value for training - * - * @param minLearningRate - * @return - */ - @Override - public Builder minLearningRate(double minLearningRate) { - super.minLearningRate(minLearningRate); - return this; - } - - /** - * This method defines whether model should be totally wiped out prior building, or not - * - * @param reallyReset - * @return - */ - @Override - public Builder resetModel(boolean reallyReset) { - super.resetModel(reallyReset); - return this; - } - - /** - * This method sets vocabulary limit during construction. - * - * Default value: 0. Means no limit - * - * @param limit - * @return - */ - @Override - public Builder limitVocabularySize(int limit) { - super.limitVocabularySize(limit); - return this; - } - - /** - * This method allows to define external VocabCache to be used - * - * @param vocabCache - * @return - */ - @Override - public Builder vocabCache(@NonNull VocabCache vocabCache) { - super.vocabCache(vocabCache); - return this; - } - - /** - * This method allows to define external WeightLookupTable to be used - * - * @param lookupTable - * @return - */ - @Override - public Builder lookupTable(@NonNull WeightLookupTable lookupTable) { - super.lookupTable(lookupTable); - return this; - } - - /** - * This method defines whether subsampling should be used or not - * - * @param sampling set > 0 to subsampling argument, or 0 to disable - * @return - */ - @Override - public Builder sampling(double sampling) { - super.sampling(sampling); - return this; - } - - /** - * This method defines whether adaptive gradients should be used or not - * - * @param reallyUse - * @return - */ - @Override - public Builder useAdaGrad(boolean reallyUse) { - super.useAdaGrad(reallyUse); - return this; - } - - /** - * This method defines whether negative sampling should be used or not - * - * PLEASE NOTE: If you're going to use negative sampling, you might want to disable HierarchicSoftmax, which is enabled by default - * - * Default value: 0 - * - * @param negative set > 0 as negative sampling argument, or 0 to disable - * @return - */ - @Override - public Builder negativeSample(double negative) { - super.negativeSample(negative); - return this; - } - - /** - * This method defines stop words that should be ignored during training - * - * @param stopList - * @return - */ - @Override - public Builder stopWords(@NonNull List stopList) { - super.stopWords(stopList); - return this; - } - - /** - * This method is hardcoded to TRUE, since that's whole point of Word2Vec - * - * @param trainElements - * @return - */ - @Override - public Builder trainElementsRepresentation(boolean trainElements) { - throw new IllegalStateException("You can't change this option for Word2Vec"); - } - - /** - * This method is hardcoded to FALSE, since that's whole point of Word2Vec - * - * @param trainSequences - * @return - */ - @Override - public Builder trainSequencesRepresentation(boolean trainSequences) { - throw new IllegalStateException("You can't change this option for Word2Vec"); - } - - /** - * This method defines stop words that should be ignored during training - * - * @param stopList - * @return - */ - @Override - public Builder stopWords(@NonNull Collection stopList) { - super.stopWords(stopList); - return this; - } - - /** - * This method defines context window size - * - * @param windowSize - * @return - */ - @Override - public Builder windowSize(int windowSize) { - super.windowSize(windowSize); - return this; - } - - /** - * This method defines random seed for random numbers generator - * @param randomSeed - * @return - */ - @Override - public Builder seed(long randomSeed) { - super.seed(randomSeed); - return this; - } - - /** - * This method defines maximum number of concurrent threads available for training - * - * @param numWorkers - * @return - */ - @Override - public Builder workers(int numWorkers) { - super.workers(numWorkers); - return this; - } - - /** - * Sets ModelUtils that gonna be used as provider for utility methods: similarity(), wordsNearest(), accuracy(), etc - * - * @param modelUtils model utils to be used - * @return - */ - @Override - public Builder modelUtils(@NonNull ModelUtils modelUtils) { - super.modelUtils(modelUtils); - return this; - } - - /** - * This method allows to use variable window size. In this case, every batch gets processed using one of predefined window sizes - * - * @param windows - * @return - */ - @Override - public Builder useVariableWindow(int... windows) { - super.useVariableWindow(windows); - return this; - } - - /** - * This method allows you to specify SequenceElement that will be used as UNK element, if UNK is used - * - * @param element - * @return - */ - @Override - public Builder unknownElement(VocabWord element) { - super.unknownElement(element); - return this; - } - - /** - * This method allows you to specify, if UNK word should be used internally - * - * @param reallyUse - * @return - */ - @Override - public Builder useUnknown(boolean reallyUse) { - super.useUnknown(reallyUse); - if (this.unknownElement == null) { - this.unknownElement(new VocabWord(1.0, Word2Vec.DEFAULT_UNK)); - } - return this; - } - - /** - * This method sets VectorsListeners for this SequenceVectors model - * - * @param vectorsListeners - * @return - */ - @Override - public Builder setVectorsListeners(@NonNull Collection> vectorsListeners) { - super.setVectorsListeners(vectorsListeners); - return this; - } - - @Override - public Builder elementsLearningAlgorithm(@NonNull String algorithm) { - super.elementsLearningAlgorithm(algorithm); - return this; - } - - @Override - public Builder elementsLearningAlgorithm(@NonNull ElementsLearningAlgorithm algorithm) { - super.elementsLearningAlgorithm(algorithm); - return this; - } - - /** - * This method enables/disables parallel tokenization. - * - * Default value: TRUE - * @param allow - * @return - */ - public Builder allowParallelTokenization(boolean allow) { - this.allowParallelTokenization = allow; - return this; - } - - /** - * This method ebables/disables periodical vocab truncation during construction - * - * Default value: disabled - * - * @param reallyEnable - * @return - */ - @Override - public Builder enableScavenger(boolean reallyEnable) { - super.enableScavenger(reallyEnable); - return this; - } - - /** - * This method enables/disables Hierarchic softmax - * - * Default value: enabled - * - * @param reallyUse - * @return - */ - @Override - public Builder useHierarchicSoftmax(boolean reallyUse) { - super.useHierarchicSoftmax(reallyUse); - return this; - } - - @Override - public Builder usePreciseWeightInit(boolean reallyUse) { - super.usePreciseWeightInit(reallyUse); - return this; - } - - @Override - public Builder usePreciseMode(boolean reallyUse) { - super.usePreciseMode(reallyUse); - return this; - } - - @Override - public Builder intersectModel(@NonNull SequenceVectors vectors, boolean isLocked) { - super.intersectModel(vectors, isLocked); - return this; - } - - public Word2Vec build() { - presetTables(); - - Word2Vec ret = new Word2Vec(); - - if (sentenceIterator != null) { - if (tokenizerFactory == null) - tokenizerFactory = new DefaultTokenizerFactory(); - - SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(sentenceIterator) - .tokenizerFactory(tokenizerFactory).allowMultithreading(allowParallelTokenization) - .build(); - this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build(); - } - - if (this.labelAwareIterator != null) { - if (tokenizerFactory == null) - tokenizerFactory = new DefaultTokenizerFactory(); - - SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(labelAwareIterator) - .tokenizerFactory(tokenizerFactory).allowMultithreading(allowParallelTokenization) - .build(); - this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build(); - } - - ret.numEpochs = this.numEpochs; - ret.numIterations = this.iterations; - ret.vocab = this.vocabCache; - ret.minWordFrequency = this.minWordFrequency; - ret.learningRate.set(this.learningRate); - ret.minLearningRate = this.minLearningRate; - ret.sampling = this.sampling; - ret.negative = this.negative; - ret.layerSize = this.layerSize; - ret.batchSize = this.batchSize; - ret.learningRateDecayWords = this.learningRateDecayWords; - ret.window = this.window; - ret.resetModel = this.resetModel; - ret.useAdeGrad = this.useAdaGrad; - ret.stopWords = this.stopWords; - ret.workers = this.workers; - ret.useUnknown = this.useUnknown; - ret.unknownElement = this.unknownElement; - ret.variableWindows = this.variableWindows; - ret.seed = this.seed; - ret.enableScavenger = this.enableScavenger; - ret.vocabLimit = this.vocabLimit; - - if (ret.unknownElement == null) - ret.unknownElement = new VocabWord(1.0,SequenceVectors.DEFAULT_UNK); - - - ret.iterator = this.iterator; - ret.lookupTable = this.lookupTable; - ret.tokenizerFactory = this.tokenizerFactory; - ret.modelUtils = this.modelUtils; - - ret.elementsLearningAlgorithm = this.elementsLearningAlgorithm; - ret.sequenceLearningAlgorithm = this.sequenceLearningAlgorithm; - - ret.intersectModel = this.intersectVectors; - ret.lockFactor = this.lockFactor; - - this.configuration.setLearningRate(this.learningRate); - this.configuration.setLayersSize(layerSize); - this.configuration.setHugeModelExpected(hugeModelExpected); - this.configuration.setWindow(window); - this.configuration.setMinWordFrequency(minWordFrequency); - this.configuration.setIterations(iterations); - this.configuration.setSeed(seed); - this.configuration.setBatchSize(batchSize); - this.configuration.setLearningRateDecayWords(learningRateDecayWords); - this.configuration.setMinLearningRate(minLearningRate); - this.configuration.setSampling(this.sampling); - this.configuration.setUseAdaGrad(useAdaGrad); - this.configuration.setNegative(negative); - this.configuration.setEpochs(this.numEpochs); - this.configuration.setStopList(this.stopWords); - this.configuration.setVariableWindows(variableWindows); - this.configuration.setUseHierarchicSoftmax(this.useHierarchicSoftmax); - this.configuration.setPreciseWeightInit(this.preciseWeightInit); - this.configuration.setModelUtils(this.modelUtils.getClass().getCanonicalName()); - this.configuration.setAllowParallelTokenization(this.allowParallelTokenization); - this.configuration.setPreciseMode(this.preciseMode); - - if (tokenizerFactory != null) { - this.configuration.setTokenizerFactory(tokenizerFactory.getClass().getCanonicalName()); - if (tokenizerFactory.getTokenPreProcessor() != null) - this.configuration.setTokenPreProcessor( - tokenizerFactory.getTokenPreProcessor().getClass().getCanonicalName()); - } - - ret.configuration = this.configuration; - - // we hardcode - ret.trainSequenceVectors = false; - ret.trainElementsVectors = true; - - ret.eventListeners = this.vectorsListeners; - - - return ret; - } - } -} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java deleted file mode 100644 index e508f1ab7..000000000 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/fasttext/FastTextTest.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.models.fasttext; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; -import org.deeplearning4j.models.word2vec.Word2Vec; -import org.deeplearning4j.text.sentenceiterator.BasicLineIterator; -import org.deeplearning4j.text.sentenceiterator.SentenceIterator; -import org.junit.jupiter.api.Disabled; - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - - -import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.primitives.Pair; -import org.nd4j.common.resources.Resources; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.hamcrest.CoreMatchers.hasItems; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.*; - -@Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) -public class FastTextTest extends BaseDL4JTest { - - - - private File inputFile = Resources.asFile("models/fasttext/data/labeled_data.txt"); - private File supModelFile = Resources.asFile("models/fasttext/supervised.model.bin"); - private File cbowModelFile = Resources.asFile("models/fasttext/cbow.model.bin"); - private File supervisedVectors = Resources.asFile("models/fasttext/supervised.model.vec"); - - - @Test - public void testTrainSupervised(@TempDir Path testDir) throws IOException { - - File output = testDir.toFile(); - - FastText fastText = - FastText.builder().supervised(true). - inputFile(inputFile.getAbsolutePath()). - outputFile(output.getAbsolutePath()).build(); - log.info("\nTraining supervised model ...\n"); - fastText.fit(); - } - - @Test - public void testTrainSkipgram(@TempDir Path testDir) throws IOException { - - File output = testDir.toFile(); - - FastText fastText = - FastText.builder().skipgram(true). - inputFile(inputFile.getAbsolutePath()). - outputFile(output.getAbsolutePath()).build(); - log.info("\nTraining supervised model ...\n"); - fastText.fit(); - } - - @Test - public void testTrainSkipgramWithBuckets(@TempDir Path testDir) throws IOException { - - File output = Files.createTempFile(testDir,"newFile","bin").toFile(); - - FastText fastText = - FastText.builder().skipgram(true). - bucket(150). - inputFile(inputFile.getAbsolutePath()). - outputFile(output.getAbsolutePath()).build(); - log.info("\nTraining supervised model ...\n"); - fastText.fit(); - } - - @Test - public void testTrainCBOW(@TempDir Path testDir) throws IOException { - - File output = Files.createTempFile(testDir,"newFile","bin").toFile(); - - FastText fastText = - FastText.builder().cbow(true). - inputFile(inputFile.getAbsolutePath()). - outputFile(output.getAbsolutePath()).build(); - log.info("\nTraining supervised model ...\n"); - fastText.fit(); - } - - @Test - public void tesLoadCBOWModel() { - - FastText fastText = new FastText(cbowModelFile); - fastText.test(cbowModelFile); - - assertEquals(19, fastText.vocab().numWords()); - assertEquals("enjoy", fastText.vocab().wordAtIndex(fastText.vocab().numWords() - 1)); - - double[] expected = {5.040466203354299E-4, 0.001005030469968915, 2.8882650076411664E-4, -6.413314840756357E-4, -1.78931062691845E-4, -0.0023157168179750443, -0.002215880434960127, 0.00274421414360404, -1.5344757412094623E-4, 4.6274057240225375E-4, -1.4383681991603225E-4, 3.7832374800927937E-4, 2.523412986192852E-4, 0.0018913350068032742, -0.0024741862434893847, -4.976555937901139E-4, 0.0039220210164785385, -0.001781729981303215, -6.010578363202512E-4, -0.00244093406945467, -7.98621098510921E-4, -0.0010007203090935946, -0.001640203408896923, 7.897148607298732E-4, 9.131592814810574E-4, -0.0013367272913455963, -0.0014030139427632093, -7.755287806503475E-4, -4.2878396925516427E-4, 6.912827957421541E-4, -0.0011824817629531026, -0.0036014916840940714, 0.004353308118879795, -7.073904271237552E-5, -9.646290563978255E-4, -0.0031849315855652094, 2.3360115301329643E-4, -2.9103990527801216E-4, -0.0022990566212683916, -0.002393763978034258, -0.001034979010000825, -0.0010725988540798426, 0.0018285386031493545, -0.0013178540393710136, -1.6632364713586867E-4, -1.4665909475297667E-5, 5.445032729767263E-4, 2.999933494720608E-4, -0.0014367225812748075, -0.002345481887459755, 0.001117417006753385, -8.688368834555149E-4, -0.001830018823966384, 0.0013242220738902688, -8.880519890226424E-4, -6.888324278406799E-4, -0.0036394784692674875, 0.002179111586883664, -1.7201311129610986E-4, 0.002365073887631297, 0.002688770182430744, 0.0023955567739903927, 0.001469283364713192, 0.0011803617235273123, 5.871498142369092E-4, -7.099180947989225E-4, 7.518937345594168E-4, -8.599072461947799E-4, -6.600041524507105E-4, -0.002724145073443651, -8.365285466425121E-4, 0.0013173354091122746, 0.001083166105672717, 0.0014539906987920403, -3.1698777456767857E-4, -2.387022686889395E-4, 1.9560157670639455E-4, 0.0020277926232665777, -0.0012741144746541977, -0.0013026101514697075, -1.5212174912448972E-4, 0.0014194383984431624, 0.0012500399025157094, 0.0013362085446715355, 3.692879108712077E-4, 4.319801155361347E-5, 0.0011261265026405454, 0.0017244465416297317, 5.564604725805111E-5, 0.002170475199818611, 0.0014707016525790095, 0.001303741242736578, 0.005553730763494968, -0.0011097051901742816, -0.0013661726843565702, 0.0014100460102781653, 0.0011811562580987811, -6.622733199037611E-4, 7.860265322960913E-4, -9.811905911192298E-4}; - assertArrayEquals(expected, fastText.getWordVector("enjoy"), 2e-3); - } - - @Test - public void testPredict() { - String text = "I like soccer"; - - FastText fastText = new FastText(supModelFile); - assertEquals(48, fastText.vocab().numWords()); - assertEquals("association", fastText.vocab().wordAtIndex(fastText.vocab().numWords() - 1)); - - double[] expected = {-0.006423053797334433, 0.007660661358386278, 0.006068876478821039, -0.004772625397890806, -0.007143457420170307, -0.007735592778772116, -0.005607823841273785, -0.00836215727031231, 0.0011235733982175589, 2.599214785732329E-4, 0.004131870809942484, 0.007203693501651287, 0.0016768622444942594, 0.008694255724549294, -0.0012487826170399785, -0.00393667770549655, -0.006292815785855055, 0.0049359360709786415, -3.356488887220621E-4, -0.009407570585608482, -0.0026168026961386204, -0.00978928804397583, 0.0032913016621023417, -0.0029464277904480696, -0.008649969473481178, 8.056449587456882E-4, 0.0043088337406516075, -0.008980576880276203, 0.008716211654245853, 0.0073893265798687935, -0.007388216909021139, 0.003814412746578455, -0.005518500227481127, 0.004668557550758123, 0.006603693123906851, 0.003820829326286912, 0.007174000144004822, -0.006393063813447952, -0.0019381389720365405, -0.0046371882781386375, -0.006193376146256924, -0.0036685809027403593, 7.58899434003979E-4, -0.003185075242072344, -0.008330358192324638, 3.3206873922608793E-4, -0.005389622412621975, 0.009706716984510422, 0.0037855932023376226, -0.008665262721478939, -0.0032511046156287193, 4.4134497875347733E-4, -0.008377416990697384, -0.009110655635595322, 0.0019723298028111458, 0.007486093323677778, 0.006400121841579676, 0.00902814231812954, 0.00975200068205595, 0.0060582347214221954, -0.0075621469877660275, 1.0270809434587136E-4, -0.00673140911385417, -0.007316927425563335, 0.009916870854794979, -0.0011407854035496712, -4.502215306274593E-4, -0.007612560410052538, 0.008726916275918484, -3.0280642022262327E-5, 0.005529289599508047, -0.007944817654788494, 0.005593308713287115, 0.003423960180953145, 4.1348213562741876E-4, 0.009524818509817123, -0.0025129399728029966, -0.0030074280221015215, -0.007503866218030453, -0.0028124507516622543, -0.006841592025011778, -2.9375351732596755E-4, 0.007195258513092995, -0.007775942329317331, 3.951996040996164E-4, -0.006887971889227629, 0.0032655203249305487, -0.007975360378623009, -4.840183464693837E-6, 0.004651934839785099, 0.0031739831902086735, 0.004644941072911024, -0.007461248897016048, 0.003057275665923953, 0.008903342299163342, 0.006857945583760738, 0.007567950990051031, 0.001506582135334611, 0.0063307867385447025, 0.005645462777465582}; - assertArrayEquals(expected, fastText.getWordVector("association"), 2e-3); - - String label = fastText.predict(text); - assertEquals("__label__soccer", label); - } - - @Test() - public void testIllegalState() { - assertThrows(IllegalStateException.class,() -> { - String text = "I like soccer"; - - FastText fastText = new FastText(supModelFile); - assertEquals(48, fastText.vocab().numWords()); - assertEquals("association", fastText.vocab().wordAtIndex(fastText.vocab().numWords() - 1)); - - double[] expected = {-0.006423053797334433, 0.007660661358386278, 0.006068876478821039, -0.004772625397890806, -0.007143457420170307, -0.007735592778772116, -0.005607823841273785, -0.00836215727031231, 0.0011235733982175589, 2.599214785732329E-4, 0.004131870809942484, 0.007203693501651287, 0.0016768622444942594, 0.008694255724549294, -0.0012487826170399785, -0.00393667770549655, -0.006292815785855055, 0.0049359360709786415, -3.356488887220621E-4, -0.009407570585608482, -0.0026168026961386204, -0.00978928804397583, 0.0032913016621023417, -0.0029464277904480696, -0.008649969473481178, 8.056449587456882E-4, 0.0043088337406516075, -0.008980576880276203, 0.008716211654245853, 0.0073893265798687935, -0.007388216909021139, 0.003814412746578455, -0.005518500227481127, 0.004668557550758123, 0.006603693123906851, 0.003820829326286912, 0.007174000144004822, -0.006393063813447952, -0.0019381389720365405, -0.0046371882781386375, -0.006193376146256924, -0.0036685809027403593, 7.58899434003979E-4, -0.003185075242072344, -0.008330358192324638, 3.3206873922608793E-4, -0.005389622412621975, 0.009706716984510422, 0.0037855932023376226, -0.008665262721478939, -0.0032511046156287193, 4.4134497875347733E-4, -0.008377416990697384, -0.009110655635595322, 0.0019723298028111458, 0.007486093323677778, 0.006400121841579676, 0.00902814231812954, 0.00975200068205595, 0.0060582347214221954, -0.0075621469877660275, 1.0270809434587136E-4, -0.00673140911385417, -0.007316927425563335, 0.009916870854794979, -0.0011407854035496712, -4.502215306274593E-4, -0.007612560410052538, 0.008726916275918484, -3.0280642022262327E-5, 0.005529289599508047, -0.007944817654788494, 0.005593308713287115, 0.003423960180953145, 4.1348213562741876E-4, 0.009524818509817123, -0.0025129399728029966, -0.0030074280221015215, -0.007503866218030453, -0.0028124507516622543, -0.006841592025011778, -2.9375351732596755E-4, 0.007195258513092995, -0.007775942329317331, 3.951996040996164E-4, -0.006887971889227629, 0.0032655203249305487, -0.007975360378623009, -4.840183464693837E-6, 0.004651934839785099, 0.0031739831902086735, 0.004644941072911024, -0.007461248897016048, 0.003057275665923953, 0.008903342299163342, 0.006857945583760738, 0.007567950990051031, 0.001506582135334611, 0.0063307867385447025, 0.005645462777465582}; - assertArrayEquals(expected, fastText.getWordVector("association"), 2e-3); - - String label = fastText.predict(text); - fastText.wordsNearest("test",1); - }); - - } - - @Test - public void testPredictProbability() { - String text = "I like soccer"; - - FastText fastText = new FastText(supModelFile); - - Pair result = fastText.predictProbability(text); - assertEquals("__label__soccer", result.getFirst()); - assertEquals(-0.6930, result.getSecond(), 2e-3); - - assertEquals(48, fastText.vocabSize()); - assertEquals(0.0500, fastText.getLearningRate(), 2e-3); - assertEquals(100, fastText.getDimension()); - assertEquals(5, fastText.getContextWindowSize()); - assertEquals(5, fastText.getEpoch()); - assertEquals(5, fastText.getNegativesNumber()); - assertEquals(1, fastText.getWordNgrams()); - assertEquals("softmax", fastText.getLossName()); - assertEquals("sup", fastText.getModelName()); - assertEquals(0, fastText.getNumberOfBuckets()); - } - - @Test - public void testVocabulary() { - FastText fastText = new FastText(supModelFile); - assertEquals(48, fastText.vocab().numWords()); - assertEquals(48, fastText.vocabSize()); - - String[] expected = {"", ".", "is", "game", "the", "soccer", "?", "football", "3", "12", "takes", "usually", "A", "US", - "in", "popular", "most", "hours", "and", "clubs", "minutes", "Do", "you", "like", "Is", "your", "favorite", "games", - "Premier", "Soccer", "a", "played", "by", "two", "teams", "of", "eleven", "players", "The", "Football", "League", "an", - "English", "professional", "league", "for", "men's", "association"}; - - for (int i = 0; i < fastText.vocabSize(); ++i) { - assertEquals(expected[i], fastText.vocab().wordAtIndex(i)); - } - } - - @Test - public void testLoadIterator() throws FileNotFoundException { - SentenceIterator iter = new BasicLineIterator(inputFile.getAbsolutePath()); - FastText - .builder() - .supervised(true) - .iterator(iter) - .build() - .loadIterator(); - } - - @Test() - public void testState() { - assertThrows(IllegalStateException.class,() -> { - FastText fastText = new FastText(); - fastText.predict("something"); - }); - - } - - @Test - public void testPretrainedVectors(@TempDir Path testDir) throws IOException { - File output = new File(testDir.toFile(),"newfile.bin"); - output.deleteOnExit(); - FastText fastText = FastText - .builder() - .supervised(true) - .inputFile(inputFile.getAbsolutePath()) - .pretrainedVectorsFile(supervisedVectors.getAbsolutePath()) - .outputFile(output.getAbsolutePath()) - .build(); - - log.info("\nTraining supervised model ...\n"); - fastText.fit(); - } - - @Test - @Disabled("Similarities seem arbitrary, needs verification") - @Tag(TagNames.NEEDS_VERIFY) - public void testWordsStatistics(@TempDir Path testDir) throws IOException { - File output = Files.createTempFile(testDir,"output","bin").toFile(); - - FastText fastText = FastText - .builder() - .supervised(true) - .inputFile(inputFile.getAbsolutePath()) - .outputFile(output.getAbsolutePath()) - .build(); - - log.info("\nTraining supervised model ...\n"); - fastText.fit(); - - File file = new File(output.getAbsolutePath() + ".vec"); - Word2Vec word2Vec = WordVectorSerializer.readAsCsv(file); - - assertEquals(48, word2Vec.getVocab().numWords()); - assertEquals( 0.12572339177131653, word2Vec.similarity("Football", "teams"), 2e-3); - assertEquals( -0.10597872734069824, word2Vec.similarity("professional", "minutes"), 2e-3); - assertEquals( Double.NaN, word2Vec.similarity("java","cpp"), 0.0); - //assertThat(word2Vec.wordsNearest("association", 3), hasItems("Football", "Soccer", "men's")); - } - - @Test - public void testWordsNativeStatistics() { - FastText fastText = new FastText(); - fastText.loadPretrainedVectors(supervisedVectors); - - log.info("\nTraining supervised model ...\n"); - - assertEquals(48, fastText.vocab().numWords()); - assertThat(fastText.wordsNearest("association", 3), hasItems("most","eleven","hours")); - assertEquals(0.1657, fastText.similarity("Football", "teams"), 2e-3); - assertEquals(0.3661, fastText.similarity("professional", "minutes"), 2e-3); - assertEquals(Double.NaN, fastText.similarity("java","cpp"), 0.0); - } -} diff --git a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectorsTest.java b/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectorsTest.java deleted file mode 100644 index d6ce0c674..000000000 --- a/deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp/src/test/java/org/deeplearning4j/models/paragraphvectors/ParagraphVectorsTest.java +++ /dev/null @@ -1,1250 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.models.paragraphvectors; - - -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.LineIterator; -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.models.sequencevectors.sequence.Sequence; -import org.deeplearning4j.models.sequencevectors.transformers.impl.SentenceTransformer; -import org.deeplearning4j.models.sequencevectors.transformers.impl.iterables.BasicTransformerIterator; -import org.deeplearning4j.text.sentenceiterator.*; - - -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.common.io.ClassPathResource; -import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; -import org.deeplearning4j.models.embeddings.learning.impl.elements.SkipGram; -import org.deeplearning4j.models.embeddings.learning.impl.sequence.DBOW; -import org.deeplearning4j.models.embeddings.learning.impl.sequence.DM; -import org.deeplearning4j.models.embeddings.loader.VectorsConfiguration; -import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; -import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; -import org.deeplearning4j.models.word2vec.VocabWord; -import org.deeplearning4j.models.word2vec.Word2Vec; -import org.deeplearning4j.models.word2vec.wordstore.VocabCache; -import org.deeplearning4j.models.word2vec.wordstore.inmemory.AbstractCache; -import org.deeplearning4j.models.word2vec.wordstore.inmemory.InMemoryLookupCache; -import org.deeplearning4j.text.documentiterator.FileLabelAwareIterator; -import org.deeplearning4j.text.documentiterator.LabelAwareIterator; -import org.deeplearning4j.text.documentiterator.LabelledDocument; -import org.deeplearning4j.text.documentiterator.LabelsSource; -import org.deeplearning4j.text.sentenceiterator.interoperability.SentenceIteratorConverter; -import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; -import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; -import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.io.CollectionUtils; -import org.nd4j.linalg.factory.Nd4jBackend; -import org.nd4j.linalg.ops.transforms.Transforms; -import org.nd4j.common.util.SerializationUtils; -import org.nd4j.common.resources.Resources; - -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.*; - -@Slf4j -@Tag(TagNames.FILE_IO) -@NativeTag -public class ParagraphVectorsTest extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return isIntegrationTests() ? 600_000 : 240_000; - } - - - @Override - public DataType getDataType() { - return DataType.FLOAT; - } - - @Override - public DataType getDefaultFPDataType() { - return DataType.FLOAT; - } - - - - /** - * This test checks, how vocab is built using SentenceIterator provided, without labels. - * - * @throws Exception - */ - @Timeout(2400000) - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testParagraphVectorsVocabBuilding1() throws Exception { - File file = Resources.asFile("/big/raw_sentences.txt"); - SentenceIterator iter = new BasicLineIterator(file); //UimaSentenceIterator.createWithPath(file.getAbsolutePath()); - - int numberOfLines = 0; - while (iter.hasNext()) { - iter.nextSentence(); - numberOfLines++; - } - - iter.reset(); - - InMemoryLookupCache cache = new InMemoryLookupCache(false); - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - // LabelsSource source = new LabelsSource("DOC_"); - - ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(5).layerSize(100) - // .labelsGenerator(source) - .windowSize(5).iterate(iter).vocabCache(cache).tokenizerFactory(t).build(); - - vec.buildVocab(); - - LabelsSource source = vec.getLabelsSource(); - - - //VocabCache cache = vec.getVocab(); - log.info("Number of lines in corpus: " + numberOfLines); - assertEquals(numberOfLines, source.getLabels().size()); - assertEquals(97162, source.getLabels().size()); - - assertNotEquals(null, cache); - assertEquals(97406, cache.numWords()); - - // proper number of words for minWordsFrequency = 1 is 244 - assertEquals(244, cache.numWords() - source.getLabels().size()); - } - - /** - * This test doesn't really cares about actual results. We only care about equality between live model & restored models - * - * @throws Exception - */ - @Timeout(3000000) - @Tag(TagNames.LONG_TEST) - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testParagraphVectorsModelling1(Nd4jBackend backend) throws Exception { - File file = Resources.asFile("/big/raw_sentences.txt"); - SentenceIterator iter = new BasicLineIterator(file); - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - LabelsSource source = new LabelsSource("DOC_"); - - ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(5).seed(119).epochs(1) - .layerSize(150).learningRate(0.025).labelsSource(source).windowSize(5) - .sequenceLearningAlgorithm(new DM()).iterate(iter).trainWordVectors(true) - .usePreciseWeightInit(true) - .batchSize(8192) - .tokenizerFactory(t).workers(4).sampling(0).build(); - - vec.fit(); - - VocabCache cache = vec.getVocab(); - - File fullFile = File.createTempFile("paravec", "tests"); - fullFile.deleteOnExit(); - - INDArray originalSyn1_17 = ((InMemoryLookupTable) vec.getLookupTable()).getSyn1().getRow(17, true).dup(); - - WordVectorSerializer.writeParagraphVectors(vec, fullFile); - - int cnt1 = cache.wordFrequency("day"); - int cnt2 = cache.wordFrequency("me"); - - assertNotEquals(1, cnt1); - assertNotEquals(1, cnt2); - assertNotEquals(cnt1, cnt2); - - assertEquals(97406, cache.numWords()); - - assertTrue(vec.hasWord("DOC_16392")); - assertTrue(vec.hasWord("DOC_3720")); - - List result = new ArrayList<>(vec.nearestLabels(vec.getWordVectorMatrix("DOC_16392"), 10)); - System.out.println("nearest labels: " + result); - for (String label : result) { - System.out.println(label + "/DOC_16392: " + vec.similarity(label, "DOC_16392")); - } - assertTrue(result.contains("DOC_16392")); - //assertTrue(result.contains("DOC_21383")); - - - - /* - We have few lines that contain pretty close words invloved. - These sentences should be pretty close to each other in vector space - */ - // line 3721: This is my way . - // line 6348: This is my case . - // line 9836: This is my house . - // line 12493: This is my world . - // line 16393: This is my work . - - // this is special sentence, that has nothing common with previous sentences - // line 9853: We now have one . - - double similarityD = vec.similarity("day", "night"); - log.info("day/night similarity: " + similarityD); - - if (similarityD < 0.0) { - log.info("Day: " + Arrays.toString(vec.getWordVectorMatrix("day").dup().data().asDouble())); - log.info("Night: " + Arrays.toString(vec.getWordVectorMatrix("night").dup().data().asDouble())); - } - - - List labelsOriginal = vec.labelsSource.getLabels(); - - double similarityW = vec.similarity("way", "work"); - log.info("way/work similarity: " + similarityW); - - double similarityH = vec.similarity("house", "world"); - log.info("house/world similarity: " + similarityH); - - double similarityC = vec.similarity("case", "way"); - log.info("case/way similarity: " + similarityC); - - double similarity1 = vec.similarity("DOC_9835", "DOC_12492"); - log.info("9835/12492 similarity: " + similarity1); - // assertTrue(similarity1 > 0.7d); - - double similarity2 = vec.similarity("DOC_3720", "DOC_16392"); - log.info("3720/16392 similarity: " + similarity2); - // assertTrue(similarity2 > 0.7d); - - double similarity3 = vec.similarity("DOC_6347", "DOC_3720"); - log.info("6347/3720 similarity: " + similarity3); - // assertTrue(similarity2 > 0.7d); - - // likelihood in this case should be significantly lower - double similarityX = vec.similarity("DOC_3720", "DOC_9852"); - log.info("3720/9852 similarity: " + similarityX); - assertTrue(similarityX < 0.5d); - - File tempFile = File.createTempFile("paravec", "ser"); - tempFile.deleteOnExit(); - - INDArray day = vec.getWordVectorMatrix("day").dup(); - - /* - Testing txt serialization - */ - File tempFile2 = File.createTempFile("paravec", "ser"); - tempFile2.deleteOnExit(); - - WordVectorSerializer.writeWordVectors(vec, tempFile2); - - ParagraphVectors vec3 = WordVectorSerializer.readParagraphVectorsFromText(tempFile2); - - INDArray day3 = vec3.getWordVectorMatrix("day").dup(); - - List labelsRestored = vec3.labelsSource.getLabels(); - - assertEquals(day, day3); - - assertEquals(labelsOriginal.size(), labelsRestored.size()); - - /* - Testing binary serialization - */ - SerializationUtils.saveObject(vec, tempFile); - - - ParagraphVectors vec2 = SerializationUtils.readObject(tempFile); - INDArray day2 = vec2.getWordVectorMatrix("day").dup(); - - List labelsBinary = vec2.labelsSource.getLabels(); - - assertEquals(day, day2); - - tempFile.delete(); - - - assertEquals(labelsOriginal.size(), labelsBinary.size()); - - INDArray original = vec.getWordVectorMatrix("DOC_16392").dup(); - INDArray originalPreserved = original.dup(); - INDArray inferredA1 = vec.inferVector("This is my work ."); - INDArray inferredB1 = vec.inferVector("This is my work ."); - - double cosAO1 = Transforms.cosineSim(inferredA1.dup(), original.dup()); - double cosAB1 = Transforms.cosineSim(inferredA1.dup(), inferredB1.dup()); - - log.info("Cos O/A: {}", cosAO1); - log.info("Cos A/B: {}", cosAB1); - log.info("Inferred: {}", inferredA1); - // assertTrue(cosAO1 > 0.45); - assertTrue(cosAB1 > 0.95); - - //assertArrayEquals(inferredA.data().asDouble(), inferredB.data().asDouble(), 0.01); - - ParagraphVectors restoredVectors = WordVectorSerializer.readParagraphVectors(fullFile); - restoredVectors.setTokenizerFactory(t); - - INDArray restoredSyn1_17 = ((InMemoryLookupTable) restoredVectors.getLookupTable()).getSyn1().getRow(17, true).dup(); - - assertEquals(originalSyn1_17, restoredSyn1_17); - - INDArray originalRestored = vec.getWordVectorMatrix("DOC_16392").dup(); - - assertEquals(originalPreserved, originalRestored); - - INDArray inferredA2 = restoredVectors.inferVector("This is my work ."); - INDArray inferredB2 = restoredVectors.inferVector("This is my work ."); - INDArray inferredC2 = restoredVectors.inferVector("world way case ."); - - double cosAO2 = Transforms.cosineSim(inferredA2.dup(), original.dup()); - double cosAB2 = Transforms.cosineSim(inferredA2.dup(), inferredB2.dup()); - double cosAAX = Transforms.cosineSim(inferredA1.dup(), inferredA2.dup()); - double cosAC2 = Transforms.cosineSim(inferredC2.dup(), inferredA2.dup()); - - log.info("Cos A2/B2: {}", cosAB2); - log.info("Cos A1/A2: {}", cosAAX); - log.info("Cos O/A2: {}", cosAO2); - log.info("Cos C2/A2: {}", cosAC2); - - log.info("Vector: {}", Arrays.toString(inferredA1.data().asFloat())); - - log.info("cosAO2: {}", cosAO2); - - // assertTrue(cosAO2 > 0.45); - assertTrue(cosAB2 > 0.95); - assertTrue(cosAAX > 0.95); - } - - - @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testParagraphVectorsDM() throws Exception { - File file = Resources.asFile("/big/raw_sentences.txt"); - SentenceIterator iter = new BasicLineIterator(file); - - AbstractCache cache = new AbstractCache.Builder().build(); - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - LabelsSource source = new LabelsSource("DOC_"); - - ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(2).seed(119).epochs(1) - .layerSize(100).learningRate(0.025).labelsSource(source).windowSize(5).iterate(iter) - .trainWordVectors(true).vocabCache(cache).tokenizerFactory(t).negativeSample(0) - .useHierarchicSoftmax(true).sampling(0).workers(1).usePreciseWeightInit(true) - .sequenceLearningAlgorithm(new DM()).build(); - - vec.fit(); - - - int cnt1 = cache.wordFrequency("day"); - int cnt2 = cache.wordFrequency("me"); - - assertNotEquals(1, cnt1); - assertNotEquals(1, cnt2); - assertNotEquals(cnt1, cnt2); - - double simDN = vec.similarity("day", "night"); - log.info("day/night similariry: {}", simDN); - - double similarity1 = vec.similarity("DOC_9835", "DOC_12492"); - log.info("9835/12492 similarity: " + similarity1); - // assertTrue(similarity1 > 0.2d); - - double similarity2 = vec.similarity("DOC_3720", "DOC_16392"); - log.info("3720/16392 similarity: " + similarity2); - // assertTrue(similarity2 > 0.2d); - - double similarity3 = vec.similarity("DOC_6347", "DOC_3720"); - log.info("6347/3720 similarity: " + similarity3); - // assertTrue(similarity3 > 0.6d); - - double similarityX = vec.similarity("DOC_3720", "DOC_9852"); - log.info("3720/9852 similarity: " + similarityX); - if(isIntegrationTests()) { - assertTrue(similarityX < 0.5d); - } - - - // testing DM inference now - - INDArray original = vec.getWordVectorMatrix("DOC_16392").dup(); - INDArray inferredA1 = vec.inferVector("This is my work"); - INDArray inferredB1 = vec.inferVector("This is my work ."); - - double cosAO1 = Transforms.cosineSim(inferredA1.dup(), original.dup()); - double cosAB1 = Transforms.cosineSim(inferredA1.dup(), inferredB1.dup()); - - log.info("Cos O/A: {}", cosAO1); - log.info("Cos A/B: {}", cosAB1); - } - - - @Timeout(300000) - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testParagraphVectorsDBOW() throws Exception { - skipUnlessIntegrationTests(); - - File file = Resources.asFile("/big/raw_sentences.txt"); - SentenceIterator iter = new BasicLineIterator(file); - - AbstractCache cache = new AbstractCache.Builder().build(); - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - LabelsSource source = new LabelsSource("DOC_"); - - ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(5).seed(119).epochs(1) - .layerSize(100).learningRate(0.025).labelsSource(source).windowSize(5).iterate(iter) - .trainWordVectors(true).vocabCache(cache).tokenizerFactory(t).negativeSample(0) - .allowParallelTokenization(true).useHierarchicSoftmax(true).sampling(0).workers(4) - .usePreciseWeightInit(true).sequenceLearningAlgorithm(new DBOW()).build(); - - vec.fit(); - - assertFalse(((InMemoryLookupTable)vec.getLookupTable()).getSyn0().isAttached()); - assertFalse(((InMemoryLookupTable)vec.getLookupTable()).getSyn1().isAttached()); - - int cnt1 = cache.wordFrequency("day"); - int cnt2 = cache.wordFrequency("me"); - - assertNotEquals(1, cnt1); - assertNotEquals(1, cnt2); - assertNotEquals(cnt1, cnt2); - - double simDN = vec.similarity("day", "night"); - log.info("day/night similariry: {}", simDN); - - double similarity1 = vec.similarity("DOC_9835", "DOC_12492"); - log.info("9835/12492 similarity: " + similarity1); - // assertTrue(similarity1 > 0.2d); - - double similarity2 = vec.similarity("DOC_3720", "DOC_16392"); - log.info("3720/16392 similarity: " + similarity2); - // assertTrue(similarity2 > 0.2d); - - double similarity3 = vec.similarity("DOC_6347", "DOC_3720"); - log.info("6347/3720 similarity: " + similarity3); - // assertTrue(similarity3 > 0.6d); - - double similarityX = vec.similarity("DOC_3720", "DOC_9852"); - log.info("3720/9852 similarity: " + similarityX); - assertTrue(similarityX < 0.5d); - - - // testing DM inference now - - INDArray original = vec.getWordVectorMatrix("DOC_16392").dup(); - INDArray inferredA1 = vec.inferVector("This is my work"); - INDArray inferredB1 = vec.inferVector("This is my work ."); - INDArray inferredC1 = vec.inferVector("This is my day"); - INDArray inferredD1 = vec.inferVector("This is my night"); - - log.info("A: {}", Arrays.toString(inferredA1.data().asFloat())); - log.info("C: {}", Arrays.toString(inferredC1.data().asFloat())); - - assertNotEquals(inferredA1, inferredC1); - - double cosAO1 = Transforms.cosineSim(inferredA1.dup(), original.dup()); - double cosAB1 = Transforms.cosineSim(inferredA1.dup(), inferredB1.dup()); - double cosAC1 = Transforms.cosineSim(inferredA1.dup(), inferredC1.dup()); - double cosCD1 = Transforms.cosineSim(inferredD1.dup(), inferredC1.dup()); - - log.info("Cos O/A: {}", cosAO1); - log.info("Cos A/B: {}", cosAB1); - log.info("Cos A/C: {}", cosAC1); - log.info("Cos C/D: {}", cosCD1); - - } - - @Test() - @Timeout(300000) - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testParagraphVectorsWithWordVectorsModelling1() throws Exception { - String backend = Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"); - if(!isIntegrationTests() && "CUDA".equalsIgnoreCase(backend)) { - skipUnlessIntegrationTests(); //Skip CUDA except for integration tests due to very slow test speed - } - - File file = Resources.asFile("/big/raw_sentences.txt"); - SentenceIterator iter = new BasicLineIterator(file); - - // InMemoryLookupCache cache = new InMemoryLookupCache(false); - AbstractCache cache = new AbstractCache.Builder().build(); - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - LabelsSource source = new LabelsSource("DOC_"); - - ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).iterations(3).epochs(1).layerSize(100) - .learningRate(0.025).labelsSource(source).windowSize(5).iterate(iter).trainWordVectors(true) - .vocabCache(cache).tokenizerFactory(t).sampling(0).build(); - - vec.fit(); - - - int cnt1 = cache.wordFrequency("day"); - int cnt2 = cache.wordFrequency("me"); - - assertNotEquals(1, cnt1); - assertNotEquals(1, cnt2); - assertNotEquals(cnt1, cnt2); - - /* - We have few lines that contain pretty close words invloved. - These sentences should be pretty close to each other in vector space - */ - // line 3721: This is my way . - // line 6348: This is my case . - // line 9836: This is my house . - // line 12493: This is my world . - // line 16393: This is my work . - - // this is special sentence, that has nothing common with previous sentences - // line 9853: We now have one . - - assertTrue(vec.hasWord("DOC_3720")); - - double similarityD = vec.similarity("day", "night"); - log.info("day/night similarity: " + similarityD); - - double similarityW = vec.similarity("way", "work"); - log.info("way/work similarity: " + similarityW); - - double similarityH = vec.similarity("house", "world"); - log.info("house/world similarity: " + similarityH); - - double similarityC = vec.similarity("case", "way"); - log.info("case/way similarity: " + similarityC); - - double similarity1 = vec.similarity("DOC_9835", "DOC_12492"); - log.info("9835/12492 similarity: " + similarity1); - // assertTrue(similarity1 > 0.7d); - - double similarity2 = vec.similarity("DOC_3720", "DOC_16392"); - log.info("3720/16392 similarity: " + similarity2); - // assertTrue(similarity2 > 0.7d); - - double similarity3 = vec.similarity("DOC_6347", "DOC_3720"); - log.info("6347/3720 similarity: " + similarity3); - // assertTrue(similarity2 > 0.7d); - - // likelihood in this case should be significantly lower - // however, since corpus is small, and weight initialization is random-based, sometimes this test CAN fail - double similarityX = vec.similarity("DOC_3720", "DOC_9852"); - log.info("3720/9852 similarity: " + similarityX); - assertTrue(similarityX < 0.5d); - - - double sim119 = vec.similarityToLabel("This is my case .", "DOC_6347"); - double sim120 = vec.similarityToLabel("This is my case .", "DOC_3720"); - log.info("1/2: " + sim119 + "/" + sim120); - //assertEquals(similarity3, sim119, 0.001); - } - - - /** - * This test is not indicative. - * there's no need in this test within travis, use it manually only for problems detection - * - * @throws Exception - */ - @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testParagraphVectorsReducedLabels1(@TempDir Path testDir) throws Exception { - val tempDir = testDir.toFile(); - ClassPathResource resource = new ClassPathResource("/labeled"); - resource.copyDirectory(tempDir); - - LabelAwareIterator iter = new FileLabelAwareIterator.Builder().addSourceFolder(tempDir).build(); - - TokenizerFactory t = new DefaultTokenizerFactory(); - - /** - * Please note: text corpus is REALLY small, and some kind of "results" could be received with HIGH epochs number, like 30. - * But there's no reason to keep at that high - */ - - ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).epochs(3).layerSize(100) - .stopWords(new ArrayList()).windowSize(5).iterate(iter).tokenizerFactory(t).build(); - - vec.fit(); - - //WordVectorSerializer.writeWordVectors(vec, "vectors.txt"); - - INDArray w1 = vec.lookupTable().vector("I"); - INDArray w2 = vec.lookupTable().vector("am"); - INDArray w3 = vec.lookupTable().vector("sad."); - - INDArray words = Nd4j.create(3, vec.lookupTable().layerSize()); - - words.putRow(0, w1); - words.putRow(1, w2); - words.putRow(2, w3); - - - INDArray mean = words.isMatrix() ? words.mean(0) : words; - - log.info("Mean" + Arrays.toString(mean.dup().data().asDouble())); - log.info("Array" + Arrays.toString(vec.lookupTable().vector("negative").dup().data().asDouble())); - - double simN = Transforms.cosineSim(mean, vec.lookupTable().vector("negative")); - log.info("Similarity negative: " + simN); - - - double simP = Transforms.cosineSim(mean, vec.lookupTable().vector("neutral")); - log.info("Similarity neutral: " + simP); - - double simV = Transforms.cosineSim(mean, vec.lookupTable().vector("positive")); - log.info("Similarity positive: " + simV); - } - - - @Test() - @Timeout(300000) - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testParallelIterator() throws IOException { - TokenizerFactory factory = new DefaultTokenizerFactory(); - SentenceIterator iterator = new BasicLineIterator(Resources.asFile("big/raw_sentences.txt")); - - SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(iterator).allowMultithreading(true) - .tokenizerFactory(factory).build(); - - BasicTransformerIterator iter = (BasicTransformerIterator)transformer.iterator(); - for (int i = 0; i < 100; ++i) { - int cnt = 0; - long counter = 0; - Sequence sequence = null; - while (iter.hasNext()) { - sequence = iter.next(); - counter += sequence.size(); - cnt++; - } - iter.reset(); - assertEquals(757172, counter); - } - } - - @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testIterator(@TempDir Path testDir) throws IOException { - val folder_labeled = new File(testDir.toFile(),"labeled"); - val folder_unlabeled = new File(testDir.toFile(),"unlabeled"); - assertTrue(folder_labeled.mkdirs()); - assertTrue(folder_labeled.mkdirs()); - new ClassPathResource("/paravec/labeled/").copyDirectory(folder_labeled); - new ClassPathResource("/paravec/unlabeled/").copyDirectory(folder_unlabeled); - - - FileLabelAwareIterator labelAwareIterator = new FileLabelAwareIterator.Builder() - .addSourceFolder(folder_labeled).build(); - - File resource_sentences = Resources.asFile("/big/raw_sentences.txt"); - SentenceIterator iter = new BasicLineIterator(resource_sentences); - - int i = 0; - for (; i < 10; ++i) { - int j = 0; - int labels = 0; - int words = 0; - while (labelAwareIterator.hasNextDocument()) { - ++j; - LabelledDocument document = labelAwareIterator.nextDocument(); - labels += document.getLabels().size(); - List lst = document.getReferencedContent(); - if (!CollectionUtils.isEmpty(lst)) - words += lst.size(); - } - labelAwareIterator.reset(); - //System.out.println(words + " " + labels + " " + j); - assertEquals(0, words); - assertEquals(30, labels); - assertEquals(30, j); - j = 0; - while (iter.hasNext()) { - ++j; - iter.nextSentence(); - } - assertEquals(97162, j); - iter.reset(); - } - - } - - /* - In this test we'll build w2v model, and will use it's vocab and weights for ParagraphVectors. - there's no need in this test within travis, use it manually only for problems detection - */ - @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testParagraphVectorsOverExistingWordVectorsModel(@TempDir Path testDir) throws Exception { - String backend = Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"); - if(!isIntegrationTests() && "CUDA".equalsIgnoreCase(backend)) { - skipUnlessIntegrationTests(); //Skip CUDA except for integration tests due to very slow test speed - } - - // we build w2v from multiple sources, to cover everything - File resource_sentences = Resources.asFile("/big/raw_sentences.txt"); - - val folder_mixed = testDir.toFile(); - ClassPathResource resource_mixed = new ClassPathResource("paravec/"); - resource_mixed.copyDirectory(folder_mixed); - - SentenceIterator iter = new AggregatingSentenceIterator.Builder() - .addSentenceIterator(new BasicLineIterator(resource_sentences)) - .addSentenceIterator(new FileSentenceIterator(folder_mixed)).build(); - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - Word2Vec wordVectors = new Word2Vec.Builder().seed(119).minWordFrequency(1).batchSize(250).iterations(1).epochs(1) - .learningRate(0.025).layerSize(150).minLearningRate(0.001) - .elementsLearningAlgorithm(new SkipGram()).useHierarchicSoftmax(true).windowSize(5) - .allowParallelTokenization(true) - .workers(1) - .iterate(iter).tokenizerFactory(t).build(); - - wordVectors.fit(); - - VocabWord day_A = wordVectors.getVocab().tokenFor("day"); - - INDArray vector_day1 = wordVectors.getWordVectorMatrix("day").dup(); - - // At this moment we have ready w2v model. It's time to use it for ParagraphVectors - - val folder_labeled = new File(testDir.toFile(),"labeled"); - val folder_unlabeled = new File(testDir.toFile(),"unlabeled"); - new ClassPathResource("/paravec/labeled/").copyDirectory(folder_labeled); - new ClassPathResource("/paravec/unlabeled/").copyDirectory(folder_unlabeled); - - - FileLabelAwareIterator labelAwareIterator = new FileLabelAwareIterator.Builder() - .addSourceFolder(folder_labeled).build(); - - - // documents from this iterator will be used for classification - FileLabelAwareIterator unlabeledIterator = new FileLabelAwareIterator.Builder() - .addSourceFolder(folder_unlabeled).build(); - - - // we're building classifier now, with pre-built w2v model passed in - ParagraphVectors paragraphVectors = new ParagraphVectors.Builder().seed(119).iterate(labelAwareIterator) - .learningRate(0.025).minLearningRate(0.001).iterations(10).epochs(1).layerSize(150) - .tokenizerFactory(t).sequenceLearningAlgorithm(new DBOW()).useHierarchicSoftmax(true) - .allowParallelTokenization(true) - .workers(1) - .trainWordVectors(false).useExistingWordVectors(wordVectors).build(); - - paragraphVectors.fit(); - - VocabWord day_B = paragraphVectors.getVocab().tokenFor("day"); - - assertEquals(day_A.getIndex(), day_B.getIndex()); - - /* - double similarityD = wordVectors.similarity("day", "night"); - log.info("day/night similarity: " + similarityD); - assertTrue(similarityD > 0.5d); - */ - - INDArray vector_day2 = paragraphVectors.getWordVectorMatrix("day").dup(); - double crossDay = arraysSimilarity(vector_day1, vector_day2); - - log.info("Day1: " + vector_day1); - log.info("Day2: " + vector_day2); - log.info("Cross-Day similarity: " + crossDay); - log.info("Cross-Day similiarity 2: " + Transforms.cosineSim(Transforms.unitVec(vector_day1), Transforms.unitVec(vector_day2))); - - assertTrue(crossDay > 0.9d); - - /** - * - * Here we're checking cross-vocabulary equality - * - */ - /* - Random rnd = new Random(); - VocabCache cacheP = paragraphVectors.getVocab(); - VocabCache cacheW = wordVectors.getVocab(); - for (int x = 0; x < 1000; x++) { - int idx = rnd.nextInt(cacheW.numWords()); - - String wordW = cacheW.wordAtIndex(idx); - String wordP = cacheP.wordAtIndex(idx); - - assertEquals(wordW, wordP); - - INDArray arrayW = wordVectors.getWordVectorMatrix(wordW); - INDArray arrayP = paragraphVectors.getWordVectorMatrix(wordP); - - double simWP = Transforms.cosineSim(arrayW, arrayP); - assertTrue(simWP >= 0.9); - } - */ - - log.info("Zfinance: " + paragraphVectors.getWordVectorMatrix("Zfinance")); - log.info("Zhealth: " + paragraphVectors.getWordVectorMatrix("Zhealth")); - log.info("Zscience: " + paragraphVectors.getWordVectorMatrix("Zscience")); - - assertTrue(unlabeledIterator.hasNext()); - LabelledDocument document = unlabeledIterator.nextDocument(); - - log.info("Results for document '" + document.getLabel() + "'"); - - List results = new ArrayList<>(paragraphVectors.predictSeveral(document, 3)); - for (String result : results) { - double sim = paragraphVectors.similarityToLabel(document, result); - log.info("Similarity to [" + result + "] is [" + sim + "]"); - } - - String topPrediction = paragraphVectors.predict(document); - assertEquals("Z"+document.getLabel(), topPrediction); - } - - /* - Left as reference implementation, before stuff was changed in w2v - */ - @Deprecated - private double arraysSimilarity(@NonNull INDArray array1, @NonNull INDArray array2) { - if (array1.equals(array2)) - return 1.0; - - INDArray vector = Transforms.unitVec(array1); - INDArray vector2 = Transforms.unitVec(array2); - - if (vector == null || vector2 == null) - return -1; - - return Transforms.cosineSim(vector, vector2); - - } - - /** - * Special test to check d2v inference against pre-trained gensim model and - */ - @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testGensimEquality() throws Exception { - - INDArray expA = Nd4j.create(new double[] {-0.02461922, -0.00801059, -0.01821643, 0.0167951, 0.02240154, - -0.00414107, -0.0022868, 0.00278438, -0.00651088, -0.02066556, -0.01045411, -0.02853066, - 0.00153375, 0.02707097, -0.00754221, -0.02795872, -0.00275301, -0.01455731, -0.00981289, - 0.01557207, -0.005259, 0.00355505, 0.01503531, -0.02185878, 0.0339283, -0.05049067, 0.02849454, - -0.01242505, 0.00438659, -0.03037345, 0.01866657, -0.00740161, -0.01850279, 0.00851284, - -0.01774663, -0.01976997, -0.03317627, 0.00372983, 0.01313218, -0.00041131, 0.00089357, - -0.0156924, 0.01278253, -0.01596088, -0.01415407, -0.01795845, 0.00558284, -0.00529536, - -0.03508032, 0.00725479, -0.01910841, -0.0008098, 0.00614283, -0.00926585, 0.01761538, - -0.00272953, -0.01483113, 0.02062481, -0.03134528, 0.03416841, -0.0156226, -0.01418961, - -0.00817538, 0.01848741, 0.00444605, 0.01090323, 0.00746163, -0.02490317, 0.00835013, - 0.01091823, -0.0177979, 0.0207753, -0.00854185, 0.04269911, 0.02786852, 0.00179449, 0.00303065, - -0.00127148, -0.01589409, -0.01110292, 0.01736244, -0.01177608, 0.00110929, 0.01790557, - -0.01800732, 0.00903072, 0.00210271, 0.0103053, -0.01508116, 0.00336775, 0.00319031, - -0.00982859, 0.02409827, -0.0079536, 0.01347831, -0.02555985, 0.00282605, 0.00350526, - -0.00471707, -0.00592073, -0.01009063, -0.02396305, 0.02643895, -0.05487461, -0.01710705, - -0.0082839, 0.01322765, 0.00098093, 0.01707118, 0.00290805, 0.03256396, 0.00277155, 0.00350602, - 0.0096487, -0.0062662, 0.0331796, -0.01758772, 0.0295204, 0.00295053, -0.00670782, 0.02172252, - 0.00172433, 0.0122977, -0.02401575, 0.01179839, -0.01646545, -0.0242724, 0.01318037, - -0.00745518, -0.00400624, -0.01735787, 0.01627645, 0.04445697, -0.0189355, 0.01315041, - 0.0131585, 0.01770667, -0.00114554, 0.00581599, 0.00745188, -0.01318868, -0.00801476, - -0.00884938, 0.00084786, 0.02578231, -0.01312729, -0.02047793, 0.00485749, -0.00342519, - -0.00744475, 0.01180929, 0.02871456, 0.01483848, -0.00696516, 0.02003011, -0.01721076, - -0.0124568, -0.0114492, -0.00970469, 0.01971609, 0.01599673, -0.01426137, 0.00808409, - -0.01431519, 0.01187332, 0.00144421, -0.00459554, 0.00384032, 0.00866845, 0.00265177, - -0.01003456, 0.0289338, 0.00353483, -0.01664903, -0.03050662, 0.01305057, -0.0084294, - -0.01615093, -0.00897918, 0.00768479, 0.02155688, 0.01594496, 0.00034328, -0.00557031, - -0.00256555, 0.03939554, 0.00274235, 0.001288, 0.02933025, 0.0070212, -0.00573742, 0.00883708, - 0.00829396, -0.01100356, -0.02653269, -0.01023274, 0.03079773, -0.00765917, 0.00949703, - 0.01212146, -0.01362515, -0.0076843, -0.00290596, -0.01707907, 0.02899382, -0.00089925, - 0.01510732, 0.02378234, -0.00947305, 0.0010998, -0.00558241, 0.00057873, 0.01098226, - -0.02019168, -0.013942, -0.01639287, -0.00675588, -0.00400709, -0.02914054, -0.00433462, - 0.01551765, -0.03552055, 0.01681101, -0.00629782, -0.01698086, 0.01891401, 0.03597684, - 0.00888052, -0.01587857, 0.00935822, 0.00931327, -0.0128156, 0.05170929, -0.01811879, - 0.02096679, 0.00897546, 0.00132624, -0.01796336, 0.01888563, -0.01142226, -0.00805926, - 0.00049782, -0.02151541, 0.00747257, 0.023373, -0.00198183, 0.02968843, 0.00443042, -0.00328569, - -0.04200815, 0.01306543, -0.01608924, -0.01604842, 0.03137267, 0.0266054, 0.00172526, - -0.01205696, 0.00047532, 0.00321026, 0.00671424, 0.01710422, -0.01129941, 0.00268044, - -0.01065434, -0.01107133, 0.00036135, -0.02991677, 0.02351665, -0.00343891, -0.01736755, - -0.00100577, -0.00312481, -0.01083809, 0.00387084, 0.01136449, 0.01675043, -0.01978249, - -0.00765182, 0.02746241, -0.01082247, -0.01587164, 0.01104732, -0.00878782, -0.00497555, - -0.00186257, -0.02281011, 0.00141792, 0.00432851, -0.01290263, -0.00387155, 0.00802639, - -0.00761913, 0.01508144, 0.02226428, 0.0107248, 0.01003709, 0.01587571, 0.00083492, -0.01632052, - -0.00435973}); - INDArray expB = Nd4j.create(new double[] {-0.02465764, 0.00756337, -0.0268607, 0.01588023, 0.01580242, - -0.00150542, 0.00116652, 0.0021577, -0.00754891, -0.02441176, -0.01271976, -0.02015191, - 0.00220599, 0.03722657, -0.01629612, -0.02779619, -0.01157856, -0.01937938, -0.00744667, - 0.01990043, -0.00505888, 0.00573646, 0.00385467, -0.0282531, 0.03484593, -0.05528606, - 0.02428633, -0.01510474, 0.00153177, -0.03637344, 0.01747423, -0.00090738, -0.02199888, - 0.01410434, -0.01710641, -0.01446697, -0.04225266, 0.00262217, 0.00871943, 0.00471594, - 0.0101348, -0.01991908, 0.00874325, -0.00606416, -0.01035323, -0.01376545, 0.00451507, - -0.01220307, -0.04361237, 0.00026028, -0.02401881, 0.00580314, 0.00238946, -0.01325974, - 0.01879044, -0.00335623, -0.01631887, 0.02222102, -0.02998703, 0.03190075, -0.01675236, - -0.01799807, -0.01314015, 0.01950069, 0.0011723, 0.01013178, 0.01093296, -0.034143, 0.00420227, - 0.01449351, -0.00629987, 0.01652851, -0.01286825, 0.03314656, 0.03485073, 0.01120341, - 0.01298241, 0.0019494, -0.02420256, -0.0063762, 0.01527091, -0.00732881, 0.0060427, 0.019327, - -0.02068196, 0.00876712, 0.00292274, 0.01312969, -0.01529114, 0.0021757, -0.00565621, - -0.01093122, 0.02758765, -0.01342688, 0.01606117, -0.02666447, 0.00541112, 0.00375426, - -0.00761796, 0.00136015, -0.01169962, -0.03012749, 0.03012953, -0.05491332, -0.01137303, - -0.01392103, 0.01370098, -0.00794501, 0.0248435, 0.00319645, 0.04261713, -0.00364211, - 0.00780485, 0.01182583, -0.00647098, 0.03291231, -0.02515565, 0.03480943, 0.00119836, - -0.00490694, 0.02615346, -0.00152456, 0.00196142, -0.02326461, 0.00603225, -0.02414703, - -0.02540966, 0.0072112, -0.01090273, -0.00505061, -0.02196866, 0.00515245, 0.04981546, - -0.02237269, -0.00189305, 0.0169786, 0.01782372, -0.00430022, 0.00551226, 0.00293861, - -0.01337168, -0.00302476, -0.01869966, 0.00270757, 0.03199976, -0.01614617, -0.02716484, - 0.01560035, -0.01312686, -0.01604082, 0.01347521, 0.03229654, 0.00707219, -0.00588392, - 0.02444809, -0.01068742, -0.0190814, -0.00556385, -0.00462766, 0.01283929, 0.02001247, - -0.00837629, -0.00041943, -0.02298774, 0.00874839, 0.00434907, -0.00963332, 0.00476905, - 0.00793049, -0.00212557, -0.01839353, 0.03345517, 0.00838255, -0.0157447, -0.0376134, - 0.01059611, -0.02323246, -0.01326356, -0.01116734, 0.00598869, 0.0211626, 0.01872963, - -0.0038276, -0.01208279, -0.00989125, 0.04147648, 0.00181867, -0.00369355, 0.02312465, - 0.0048396, 0.00564515, 0.01317832, -0.0057621, -0.01882041, -0.02869064, -0.00670661, - 0.02585443, -0.01108428, 0.01411031, 0.01204507, -0.01244726, -0.00962342, -0.00205239, - -0.01653971, 0.02871559, -0.00772978, 0.0214524, 0.02035478, -0.01324312, 0.00169302, - -0.00064739, 0.00531795, 0.01059279, -0.02455794, -0.00002782, -0.0068906, -0.0160858, - -0.0031842, -0.02295724, 0.01481094, 0.01769004, -0.02925742, 0.02050495, -0.00029003, - -0.02815636, 0.02467367, 0.03419458, 0.00654938, -0.01847546, 0.00999932, 0.00059222, - -0.01722176, 0.05172159, -0.01548486, 0.01746444, 0.007871, 0.0078471, -0.02414417, 0.01898077, - -0.01470176, -0.00299465, 0.00368212, -0.02474656, 0.01317451, 0.03706085, -0.00032923, - 0.02655881, 0.0013586, -0.0120303, -0.05030316, 0.0222294, -0.0070967, -0.02150935, 0.03254268, - 0.01369857, 0.00246183, -0.02253576, -0.00551247, 0.00787363, 0.01215617, 0.02439827, - -0.01104699, -0.00774596, -0.01898127, -0.01407653, 0.00195514, -0.03466602, 0.01560903, - -0.01239944, -0.02474852, 0.00155114, 0.00089324, -0.01725949, -0.00011816, 0.00742845, - 0.01247074, -0.02467943, -0.00679623, 0.01988366, -0.00626181, -0.02396477, 0.01052101, - -0.01123178, -0.00386291, -0.00349261, -0.02714747, -0.00563315, 0.00228767, -0.01303677, - -0.01971108, 0.00014759, -0.00346399, 0.02220698, 0.01979946, -0.00526076, 0.00647453, - 0.01428513, 0.00223467, -0.01690172, -0.0081715}); - - VectorsConfiguration configuration = new VectorsConfiguration(); - - configuration.setIterations(5); - configuration.setLearningRate(0.01); - configuration.setUseHierarchicSoftmax(true); - configuration.setNegative(0); - - Word2Vec w2v = WordVectorSerializer.readWord2VecFromText( - new File("/home/raver119/Downloads/gensim_models_for_dl4j/word"), - new File("/home/raver119/Downloads/gensim_models_for_dl4j/hs"), - new File("/home/raver119/Downloads/gensim_models_for_dl4j/hs_code"), - new File("/home/raver119/Downloads/gensim_models_for_dl4j/hs_mapping"), configuration); - - TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory(); - tokenizerFactory.setTokenPreProcessor(new CommonPreprocessor()); - - - assertNotEquals(null, w2v.getLookupTable()); - assertNotEquals(null, w2v.getVocab()); - - ParagraphVectors d2v = new ParagraphVectors.Builder(configuration).useExistingWordVectors(w2v) - .sequenceLearningAlgorithm(new DM()).tokenizerFactory(tokenizerFactory) - .resetModel(false).build(); - - - assertNotEquals(null, d2v.getLookupTable()); - assertNotEquals(null, d2v.getVocab()); - - assertTrue(d2v.getVocab() == w2v.getVocab()); - assertTrue(d2v.getLookupTable() == w2v.getLookupTable()); - - String textA = "Donald Trump referred to President Obama as “your president” during the first presidential debate on Monday, much to many people’s chagrin on social media. Trump, made the reference after saying that the greatest threat facing the world is nuclear weapons. He then turned to Hillary Clinton and said, “Not global warming like you think and your President thinks,” referring to Obama."; - - String textB = "The comment followed Trump doubling down on his false claims about the so-called birther conspiracy theory about Obama. People following the debate were immediately angered that Trump implied Obama is not his president."; - - String textC = "practice of trust owned Trump for example indeed and conspiracy between provoke"; - - INDArray arrayA = d2v.inferVector(textA); - INDArray arrayB = d2v.inferVector(textB); - INDArray arrayC = d2v.inferVector(textC); - - assertNotEquals(null, arrayA); - assertNotEquals(null, arrayB); - - Transforms.unitVec(arrayA); - Transforms.unitVec(arrayB); - - Transforms.unitVec(expA); - Transforms.unitVec(expB); - - double simX = Transforms.cosineSim(arrayA, arrayB); - double simC = Transforms.cosineSim(arrayA, arrayC); - double simB = Transforms.cosineSim(arrayB, expB); - - log.info("SimilarityX: {}", simX); - log.info("SimilarityC: {}", simC); - log.info("SimilarityB: {}", simB); - } - - @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testDirectInference(@TempDir Path testDir) throws Exception { - boolean isIntegration = isIntegrationTests(); - File resource = Resources.asFile("/big/raw_sentences.txt"); - SentenceIterator sentencesIter = getIterator(isIntegration, resource); - - ClassPathResource resource_mixed = new ClassPathResource("paravec/"); - File local_resource_mixed = testDir.toFile(); - resource_mixed.copyDirectory(local_resource_mixed); - SentenceIterator iter = new AggregatingSentenceIterator.Builder() - .addSentenceIterator(sentencesIter) - .addSentenceIterator(new FileSentenceIterator(local_resource_mixed)).build(); - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - Word2Vec wordVectors = new Word2Vec.Builder().minWordFrequency(1).batchSize(250).iterations(1).epochs(1) - .learningRate(0.025).layerSize(150).minLearningRate(0.001) - .elementsLearningAlgorithm(new SkipGram()).useHierarchicSoftmax(true).windowSize(5) - .iterate(iter).tokenizerFactory(t).build(); - - wordVectors.fit(); - - ParagraphVectors pv = new ParagraphVectors.Builder().tokenizerFactory(t).iterations(10) - .useHierarchicSoftmax(true).trainWordVectors(true).useExistingWordVectors(wordVectors) - .negativeSample(0).sequenceLearningAlgorithm(new DM()).build(); - - INDArray vec1 = pv.inferVector("This text is pretty awesome"); - INDArray vec2 = pv.inferVector("Fantastic process of crazy things happening inside just for history purposes"); - - log.info("vec1/vec2: {}", Transforms.cosineSim(vec1, vec2)); - } - - @Test - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testGoogleModelForInference() throws Exception { - WordVectors googleVectors = WordVectorSerializer.readWord2VecModel(new File("/ext/GoogleNews-vectors-negative300.bin.gz")); - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - ParagraphVectors pv = - new ParagraphVectors.Builder().tokenizerFactory(t).iterations(10).useHierarchicSoftmax(false) - .trainWordVectors(false).iterations(10).useExistingWordVectors(googleVectors) - .negativeSample(10).sequenceLearningAlgorithm(new DM()).build(); - - INDArray vec1 = pv.inferVector("This text is pretty awesome"); - INDArray vec2 = pv.inferVector("Fantastic process of crazy things happening inside just for history purposes"); - - log.info("vec1/vec2: {}", Transforms.cosineSim(vec1, vec2)); - } - - @Test() - @Timeout(300000) - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testHash() { - VocabWord w1 = new VocabWord(1.0, "D1"); - VocabWord w2 = new VocabWord(1.0, "Bo"); - - - - log.info("W1 > Short hash: {}; Long hash: {}", w1.getLabel().hashCode(), w1.getStorageId()); - log.info("W2 > Short hash: {}; Long hash: {}", w2.getLabel().hashCode(), w2.getStorageId()); - - assertNotEquals(w1.getStorageId(), w2.getStorageId()); - } - - - /** - * This is very long test, to track memory consumption over time - * - * @throws Exception - */ - @Tag(TagNames.LONG_TEST) - @ParameterizedTest - @MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs") - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - @Disabled("Takes too long for CI") - @Tag(TagNames.NEEDS_VERIFY) - public void testsParallelFit1(Nd4jBackend backend) throws Exception { - final File file = Resources.asFile("big/raw_sentences.txt"); - - for (int i = 0; i < 1000; i++) { - List threads = new ArrayList<>(); - for (int t = 0; t < 3; t++) { - threads.add(new Thread(() -> { - try { - TokenizerFactory t1 = new DefaultTokenizerFactory(); - - LabelsSource source = new LabelsSource("DOC_"); - - SentenceIteratorConverter sic = - new SentenceIteratorConverter(new BasicLineIterator(file), source); - - ParagraphVectors vec = new ParagraphVectors.Builder().seed(42) - //.batchSize(10) - .minWordFrequency(1).iterations(1).epochs(5).layerSize(100) - .learningRate(0.05) - //.labelsSource(source) - .windowSize(5).trainWordVectors(true).allowParallelTokenization(false) - //.vocabCache(cache) - .tokenizerFactory(t1).workers(1).iterate(sic).build(); - - vec.fit(); - } catch (Exception e) { - throw new RuntimeException(e); - } - })); - } - - for (Thread t : threads) { - t.start(); - } - - for (Thread t : threads) { - t.join(); - } - } - } - - @Test() - @Timeout(300000) - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testJSONSerialization() { - ParagraphVectors paragraphVectors = new ParagraphVectors.Builder().build(); - AbstractCache cache = new AbstractCache.Builder().build(); - - val words = new VocabWord[3]; - words[0] = new VocabWord(1.0, "word"); - words[1] = new VocabWord(2.0, "test"); - words[2] = new VocabWord(3.0, "tester"); - - for (int i = 0; i < words.length; ++i) { - cache.addToken(words[i]); - cache.addWordToIndex(i, words[i].getLabel()); - } - paragraphVectors.setVocab(cache); - - String json = null; - Word2Vec unserialized = null; - try { - json = paragraphVectors.toJson(); - log.info("{}", json.toString()); - - unserialized = ParagraphVectors.fromJson(json); - } catch (Exception e) { - log.error("",e); - fail(); - } - - assertEquals(cache.totalWordOccurrences(), ((ParagraphVectors) unserialized).getVocab().totalWordOccurrences()); - assertEquals(cache.totalNumberOfDocs(), ((ParagraphVectors) unserialized).getVocab().totalNumberOfDocs()); - - for (int i = 0; i < words.length; ++i) { - val cached = cache.wordAtIndex(i); - val restored = ((ParagraphVectors) unserialized).getVocab().wordAtIndex(i); - assertNotNull(cached); - assertEquals(cached, restored); - } - } - - @Test() - @Timeout(300000) - @Tag(TagNames.LONG_TEST) - @Tag(TagNames.LARGE_RESOURCES) - public void testDoubleFit() throws Exception { - boolean isIntegration = isIntegrationTests(); - File resource = Resources.asFile("/big/raw_sentences.txt"); - SentenceIterator iter = getIterator(isIntegration, resource); - - - TokenizerFactory t = new DefaultTokenizerFactory(); - t.setTokenPreProcessor(new CommonPreprocessor()); - - LabelsSource source = new LabelsSource("DOC_"); - - val builder = new ParagraphVectors.Builder(); - ParagraphVectors vec = builder.minWordFrequency(1).iterations(5).seed(119).epochs(1) - .layerSize(150).learningRate(0.025).labelsSource(source).windowSize(5) - .sequenceLearningAlgorithm(new DM()).iterate(iter).trainWordVectors(true) - .usePreciseWeightInit(true) - .batchSize(8192) - .allowParallelTokenization(false) - .tokenizerFactory(t).workers(1).sampling(0).build(); - - vec.fit(); - long num1 = vec.vocab().totalNumberOfDocs(); - - vec.fit(); - System.out.println(vec.vocab().totalNumberOfDocs()); - long num2 = vec.vocab().totalNumberOfDocs(); - - assertEquals(num1, num2); - } - - public static SentenceIterator getIterator(boolean isIntegration, File file) throws IOException { - return getIterator(isIntegration, file, 500); - } - - public static SentenceIterator getIterator(boolean isIntegration, File file, int linesForUnitTest) throws IOException { - if(isIntegration){ - return new BasicLineIterator(file); - } else { - List lines = new ArrayList<>(); - try(InputStream is = new BufferedInputStream(new FileInputStream(file))){ - LineIterator lineIter = IOUtils.lineIterator(is, StandardCharsets.UTF_8); - try{ - for( int i=0; i - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-nlp-parent - pom - - - deeplearning4j-nlp - - - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - - - - - maven-surefire-plugin - ${maven-surefire-plugin.version} - true - - - - 0 - - - false - - false - false - false - 1 - - - - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - org.deeplearning4j - deeplearning4j-cuda-11.0 - ${nd4j.version} - - - - - diff --git a/deeplearning4j/deeplearning4j-nn/pom.xml b/deeplearning4j/deeplearning4j-nn/pom.xml deleted file mode 100644 index 32d58ea30..000000000 --- a/deeplearning4j/deeplearning4j-nn/pom.xml +++ /dev/null @@ -1,154 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-nn - - deeplearning4j-nn - - - - org.deeplearning4j - deeplearning4j-utility-iterators - ${project.version} - - - org.lucee - oswego-concurrent - 1.3.4 - - - org.deeplearning4j - deeplearning4j-common - ${project.version} - - - org.projectlombok - lombok - ${lombok.version} - provided - - - commons-io - commons-io - ${commonsio.version} - - - - org.nd4j - nd4j-api - ${nd4j.version} - - - org.nd4j - nd4j-native-api - ${nd4j.version} - - - org.nd4j - nd4j-common - ${nd4j.version} - - - com.google.code.gson - gson - ${gson.version} - - - - org.nd4j - jackson - ${nd4j.version} - - - - com.github.oshi - oshi-core - ${oshi.version} - - - ch.qos.logback - logback-classic - test - - - it.unimi.dsi - fastutil - ${fastutil.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - - - diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/BaseEvaluation.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/BaseEvaluation.java deleted file mode 100644 index ee86245a5..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/BaseEvaluation.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.eval; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import org.nd4j.common.primitives.AtomicBoolean; -import org.nd4j.common.primitives.AtomicDouble; -import org.nd4j.common.primitives.serde.JsonDeserializerAtomicBoolean; -import org.nd4j.common.primitives.serde.JsonDeserializerAtomicDouble; -import org.nd4j.common.primitives.serde.JsonSerializerAtomicBoolean; -import org.nd4j.common.primitives.serde.JsonSerializerAtomicDouble; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; -import org.nd4j.shade.jackson.databind.module.SimpleModule; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; - -@Deprecated -@EqualsAndHashCode -public abstract class BaseEvaluation extends org.nd4j.evaluation.BaseEvaluation { - - @Getter - private static ObjectMapper objectMapper = configureMapper(new ObjectMapper()); - @Getter - private static ObjectMapper yamlMapper = configureMapper(new ObjectMapper(new YAMLFactory())); - - private static ObjectMapper configureMapper(ObjectMapper ret) { - ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, false); - ret.enable(SerializationFeature.INDENT_OUTPUT); - SimpleModule atomicModule = new SimpleModule(); - atomicModule.addSerializer(AtomicDouble.class,new JsonSerializerAtomicDouble()); - atomicModule.addSerializer(AtomicBoolean.class,new JsonSerializerAtomicBoolean()); - atomicModule.addDeserializer(AtomicDouble.class,new JsonDeserializerAtomicDouble()); - atomicModule.addDeserializer(AtomicBoolean.class,new JsonDeserializerAtomicBoolean()); - ret.registerModule(atomicModule); - //Serialize fields only, not using getters - ret.setVisibilityChecker(ret.getSerializationConfig().getDefaultVisibilityChecker() - .withFieldVisibility(JsonAutoDetect.Visibility.ANY) - .withGetterVisibility(JsonAutoDetect.Visibility.NONE) - .withSetterVisibility(JsonAutoDetect.Visibility.NONE) - .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)); - return ret; - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ConfusionMatrix.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ConfusionMatrix.java deleted file mode 100755 index 050fbe829..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/ConfusionMatrix.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.eval; - -import org.nd4j.shade.guava.collect.HashMultiset; -import org.nd4j.shade.guava.collect.Multiset; -import lombok.Getter; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -@Deprecated -public class ConfusionMatrix> extends org.nd4j.evaluation.classification.ConfusionMatrix { - - /** - * @deprecated Use {@link org.nd4j.evaluation.classification.ConfusionMatrix} - */ - @Deprecated - public ConfusionMatrix(List classes) { - super(classes); - } - - /** - * @deprecated Use {@link org.nd4j.evaluation.classification.ConfusionMatrix} - */ - @Deprecated - public ConfusionMatrix() { - super(); - } - - /** - * @deprecated Use {@link org.nd4j.evaluation.classification.ConfusionMatrix} - */ - @Deprecated - public ConfusionMatrix(ConfusionMatrix other) { - super(other); - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationCalibration.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationCalibration.java deleted file mode 100644 index bb3843ce1..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/EvaluationCalibration.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.eval; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -@Deprecated -@Getter -@EqualsAndHashCode(callSuper = true) -public class EvaluationCalibration extends org.nd4j.evaluation.classification.EvaluationCalibration implements org.deeplearning4j.eval.IEvaluation { - - /** - * @deprecated Use {@link org.nd4j.evaluation.classification.EvaluationCalibration} - */ - @Deprecated - public EvaluationCalibration() { - super(); - } - - /** - * @deprecated Use {@link org.nd4j.evaluation.classification.EvaluationCalibration} - */ - @Deprecated - public EvaluationCalibration(int reliabilityDiagNumBins, int histogramNumBins) { - super(reliabilityDiagNumBins, histogramNumBins); - } - - /** - * @deprecated Use {@link org.nd4j.evaluation.classification.EvaluationCalibration} - */ - @Deprecated - public EvaluationCalibration(@JsonProperty("reliabilityDiagNumBins") int reliabilityDiagNumBins, - @JsonProperty("histogramNumBins") int histogramNumBins, - @JsonProperty("excludeEmptyBins") boolean excludeEmptyBins) { - super(reliabilityDiagNumBins, histogramNumBins, excludeEmptyBins); - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/IEvaluation.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/IEvaluation.java deleted file mode 100644 index 63653f7d9..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/IEvaluation.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.eval; - -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; - -@Deprecated -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY) -public interface IEvaluation extends org.nd4j.evaluation.IEvaluation { - -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java deleted file mode 100644 index e3a381397..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/Histogram.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.eval.curves; - -import lombok.Data; -import org.nd4j.evaluation.curves.BaseHistogram; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -@Deprecated -@Data -public class Histogram extends org.nd4j.evaluation.curves.Histogram { - - /** - * @deprecated Use {@link org.nd4j.evaluation.curves.Histogram} - */ - public Histogram(@JsonProperty("title") String title, @JsonProperty("lower") double lower, - @JsonProperty("upper") double upper, @JsonProperty("binCounts") int[] binCounts) { - super(title, lower, upper, binCounts); - } - - /** - * @deprecated Use {@link org.nd4j.evaluation.curves.Histogram} - */ - public static Histogram fromJson(String json) { - return BaseHistogram.fromJson(json, Histogram.class); - } - - /** - * @deprecated Use {@link org.nd4j.evaluation.curves.Histogram} - */ - public static Histogram fromYaml(String yaml) { - return BaseHistogram.fromYaml(yaml, Histogram.class); - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/PrecisionRecallCurve.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/PrecisionRecallCurve.java deleted file mode 100644 index c42648bcd..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/PrecisionRecallCurve.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.eval.curves; - -import org.nd4j.shade.guava.base.Preconditions; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -import java.util.Arrays; - -@Deprecated -@Data -@EqualsAndHashCode(callSuper = true) -public class PrecisionRecallCurve extends org.nd4j.evaluation.curves.PrecisionRecallCurve{ - - /** - * @deprecated Use {@link org.nd4j.evaluation.curves.ReliabilityDiagram} - */ - @Deprecated - public PrecisionRecallCurve(@JsonProperty("threshold") double[] threshold, - @JsonProperty("precision") double[] precision, @JsonProperty("recall") double[] recall, - @JsonProperty("tpCount") int[] tpCount, @JsonProperty("fpCount") int[] fpCount, - @JsonProperty("fnCount") int[] fnCount, @JsonProperty("totalCount") int totalCount) { - super(threshold, precision, recall, tpCount, fpCount, fnCount, totalCount); - } - - public static class Point extends org.nd4j.evaluation.curves.PrecisionRecallCurve.Point{ - public Point(int idx, double threshold, double precision, double recall) { - super(idx, threshold, precision, recall); - } - } - - public static class Confusion extends org.nd4j.evaluation.curves.PrecisionRecallCurve.Confusion{ - public Confusion(org.nd4j.evaluation.curves.PrecisionRecallCurve.Point point, int tpCount, int fpCount, int fnCount, int tnCount) { - super(point, tpCount, fpCount, fnCount, tnCount); - } - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/ReliabilityDiagram.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/ReliabilityDiagram.java deleted file mode 100644 index cb02a0a4d..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/ReliabilityDiagram.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.eval.curves; - -import lombok.NonNull; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -@Deprecated -public class ReliabilityDiagram extends org.nd4j.evaluation.curves.ReliabilityDiagram { - - /** - * @deprecated Use {@link org.nd4j.evaluation.curves.ReliabilityDiagram} - */ - @Deprecated - public ReliabilityDiagram(@JsonProperty("title") String title, - @NonNull @JsonProperty("meanPredictedValueX") double[] meanPredictedValueX, - @NonNull @JsonProperty("fractionPositivesY") double[] fractionPositivesY) { - super(title, meanPredictedValueX, fractionPositivesY); - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java deleted file mode 100644 index de464eea5..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/eval/curves/RocCurve.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.eval.curves; - -import org.nd4j.shade.guava.base.Preconditions; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -@Deprecated -@Data -@EqualsAndHashCode(exclude = {"auc"}, callSuper = false) -public class RocCurve extends org.nd4j.evaluation.curves.RocCurve { - - /** - * @deprecated Use {@link org.nd4j.evaluation.curves.RocCurve} - */ - @Deprecated - public RocCurve(@JsonProperty("threshold") double[] threshold, @JsonProperty("fpr") double[] fpr, - @JsonProperty("tpr") double[] tpr) { - super(threshold, fpr, tpr); - } - - - /** - * @deprecated Use {@link org.nd4j.evaluation.curves.RocCurve} - */ - @Deprecated - public static RocCurve fromJson(String json) { - return fromJson(json, RocCurve.class); - } - - /** - * @deprecated Use {@link org.nd4j.evaluation.curves.RocCurve} - */ - @Deprecated - public static RocCurve fromYaml(String yaml) { - return fromYaml(yaml, RocCurve.class); - } - -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/DataFormat.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/DataFormat.java deleted file mode 100644 index 2849b9402..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/DataFormat.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ -package org.deeplearning4j.nn.conf; - -import org.deeplearning4j.nn.conf.serde.format.DataFormatDeserializer; -import org.deeplearning4j.nn.conf.serde.format.DataFormatSerializer; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; - -@JsonSerialize(using = DataFormatSerializer.class) -@JsonDeserialize(using = DataFormatDeserializer.class) -public interface DataFormat { -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/BinomialDistribution.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/BinomialDistribution.java deleted file mode 100644 index ecba1b3ef..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/BinomialDistribution.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.distribution; - -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -public class BinomialDistribution extends Distribution { - - private static final long serialVersionUID = 7407024251874318749L; - - private final int numberOfTrials; - private double probabilityOfSuccess; - - /** - * Create a distribution - * - * @param numberOfTrials the number of trials - * @param probabilityOfSuccess the probability of success - */ - @JsonCreator - public BinomialDistribution(@JsonProperty("numberOfTrials") int numberOfTrials, - @JsonProperty("probabilityOfSuccess") double probabilityOfSuccess) { - this.numberOfTrials = numberOfTrials; - this.probabilityOfSuccess = probabilityOfSuccess; - } - - public double getProbabilityOfSuccess() { - return probabilityOfSuccess; - } - - public void setProbabilityOfSuccess(double probabilityOfSuccess) { - this.probabilityOfSuccess = probabilityOfSuccess; - } - - public int getNumberOfTrials() { - return numberOfTrials; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + numberOfTrials; - long temp; - temp = Double.doubleToLongBits(probabilityOfSuccess); - result = prime * result + (int) (temp ^ (temp >>> 32)); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - BinomialDistribution other = (BinomialDistribution) obj; - if (numberOfTrials != other.numberOfTrials) - return false; - if (Double.doubleToLongBits(probabilityOfSuccess) != Double.doubleToLongBits(other.probabilityOfSuccess)) - return false; - return true; - } - - public String toString() { - return "BinomialDistribution(" + "numberOfTrials=" + numberOfTrials + ", probabilityOfSuccess=" - + probabilityOfSuccess + ')'; - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/GaussianDistribution.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/GaussianDistribution.java deleted file mode 100644 index e50c9b15c..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/GaussianDistribution.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.distribution; - -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -@Deprecated -public class GaussianDistribution extends NormalDistribution { - - /** - * Create a gaussian distribution (equivalent to normal) - * with the given mean and std - * - * @param mean the mean - * @param std the standard deviation - */ - @JsonCreator - public GaussianDistribution(@JsonProperty("mean") double mean, @JsonProperty("std") double std) { - super(mean, std); - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/LogNormalDistribution.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/LogNormalDistribution.java deleted file mode 100644 index 204b737ac..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/LogNormalDistribution.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.distribution; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -/** - * A log-normal distribution, with two parameters: mean and standard deviation. - * Note: the mean and standard deviation are for the logarithm of the values. - * Put another way: if X~LogN(M,S), then mean(log(X))=M, and stdev(log(X))=S - * - */ -@EqualsAndHashCode(callSuper = false) -@Data -public class LogNormalDistribution extends Distribution { - - private double mean, std; - - /** - * Create a log-normal distribution - * with the given mean and std - * - * @param mean the mean - * @param std the standard deviation - */ - @JsonCreator - public LogNormalDistribution(@JsonProperty("mean") double mean, @JsonProperty("std") double std) { - this.mean = mean; - this.std = std; - } - - public String toString() { - return "LogNormalDistribution(" + "mean=" + mean + ", std=" + std + ')'; - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/NormalDistribution.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/NormalDistribution.java deleted file mode 100644 index 42c62c0b9..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/NormalDistribution.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.distribution; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -/** - * A normal (Gaussian) distribution, with two parameters: mean and standard deviation - * - */ -@EqualsAndHashCode(callSuper = false) -@Data -public class NormalDistribution extends Distribution { - - private double mean, std; - - /** - * Create a normal distribution - * with the given mean and std - * - * @param mean the mean - * @param std the standard deviation - */ - @JsonCreator - public NormalDistribution(@JsonProperty("mean") double mean, @JsonProperty("std") double std) { - this.mean = mean; - this.std = std; - } - - public double getMean() { - return mean; - } - - public void setMean(double mean) { - this.mean = mean; - } - - public double getStd() { - return std; - } - - public void setStd(double std) { - this.std = std; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - long temp; - temp = Double.doubleToLongBits(mean); - result = prime * result + (int) (temp ^ (temp >>> 32)); - temp = Double.doubleToLongBits(std); - result = prime * result + (int) (temp ^ (temp >>> 32)); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - NormalDistribution other = (NormalDistribution) obj; - if (Double.doubleToLongBits(mean) != Double.doubleToLongBits(other.mean)) - return false; - if (Double.doubleToLongBits(std) != Double.doubleToLongBits(other.std)) - return false; - return true; - } - - public String toString() { - return "NormalDistribution(" + "mean=" + mean + ", std=" + std + ')'; - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/TruncatedNormalDistribution.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/TruncatedNormalDistribution.java deleted file mode 100644 index 28b95025a..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/TruncatedNormalDistribution.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.distribution; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -@EqualsAndHashCode(callSuper = false) -@Data -public class TruncatedNormalDistribution extends Distribution { - - private double mean, std; - - /** - * Create a truncated normal distribution - * with the given mean and std - * - * @param mean the mean - * @param std the standard deviation - */ - @JsonCreator - public TruncatedNormalDistribution(@JsonProperty("mean") double mean, @JsonProperty("std") double std) { - this.mean = mean; - this.std = std; - } - - public String toString() { - return "TruncatedNormalDistribution(" + "mean=" + mean + ", std=" + std + ')'; - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/UniformDistribution.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/UniformDistribution.java deleted file mode 100644 index ade1b7ffa..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/distribution/UniformDistribution.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.distribution; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.apache.commons.math3.exception.NumberIsTooLargeException; -import org.apache.commons.math3.exception.util.LocalizedFormats; -import org.nd4j.shade.jackson.annotation.JsonCreator; -import org.nd4j.shade.jackson.annotation.JsonProperty; - -/** - * A uniform distribution, with two parameters: lower and upper - i.e., U(lower,upper) - * - */ -@EqualsAndHashCode(callSuper = false) -@Data -public class UniformDistribution extends Distribution { - - private double upper, lower; - - /** - * Create a uniform real distribution using the given lower and upper - * bounds. - * - * @param lower Lower bound of this distribution (inclusive). - * @param upper Upper bound of this distribution (exclusive). - * @throws NumberIsTooLargeException if {@code lower >= upper}. - */ - @JsonCreator - public UniformDistribution(@JsonProperty("lower") double lower, @JsonProperty("upper") double upper) - throws NumberIsTooLargeException { - if (lower >= upper) { - throw new NumberIsTooLargeException(LocalizedFormats.LOWER_BOUND_NOT_BELOW_UPPER_BOUND, lower, upper, - false); - } - this.lower = lower; - this.upper = upper; - } - - public String toString() { - return "UniformDistribution(lower=" + lower + ", upper=" + upper + ")"; - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InputType.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InputType.java deleted file mode 100644 index c3317e4ea..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/inputs/InputType.java +++ /dev/null @@ -1,522 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.inputs; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.nn.conf.DataFormat; -import org.deeplearning4j.nn.conf.RNNFormat; -import org.deeplearning4j.nn.conf.CNN2DFormat; -import org.deeplearning4j.nn.conf.layers.Convolution3D; -import org.nd4j.common.base.Preconditions; -import org.nd4j.common.util.OneTimeLogger; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonIgnore; -import org.nd4j.shade.jackson.annotation.JsonInclude; -import org.nd4j.shade.jackson.annotation.JsonProperty; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; - -import java.io.Serializable; -import java.util.Arrays; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class") -@Slf4j -public abstract class InputType implements Serializable { - - /** - * The type of activations in/out of a given GraphVertex
- * FF: Standard feed-foward (2d minibatch, 1d per example) data
- * RNN: Recurrent neural network (3d minibatch) time series data
- * CNN: 2D Convolutional neural network (4d minibatch, [miniBatchSize, channels, height, width]) - * CNNFlat: Flattened 2D conv net data (2d minibatch, [miniBatchSize, height * width * channels]) - * CNN3D: 3D convolutional neural network (5d minibatch, [miniBatchSize, channels, height, width, channels]) - */ - public enum Type { - FF, RNN, CNN, CNNFlat, CNN3D - } - - public static CNN2DFormat getDefaultCNN2DFormat() { - return defaultCNN2DFormat; - } - - public static void setDefaultCNN2DFormat(CNN2DFormat defaultCNN2DFormat) { - InputType.defaultCNN2DFormat = defaultCNN2DFormat; - } - - private static CNN2DFormat defaultCNN2DFormat = CNN2DFormat.NCHW; - - @JsonIgnore - public abstract Type getType(); - - @Override - public abstract String toString(); - - @JsonIgnore - public abstract long arrayElementsPerExample(); - - /** - * Returns the shape of this InputType - * - * @param includeBatchDim Whether to include minibatch in the return shape array - * @return int[] - */ - @JsonIgnore - public abstract long[] getShape(boolean includeBatchDim); - - /** - * Returns the shape of this InputType without minibatch dimension in the returned array - * - * @return int[] - */ - public long[] getShape() { - return getShape(false); - } - - /** - * InputType for feed forward network data - * - * @param size The size of the activations - * @return InputTypeFeedForward - */ - public static InputType feedForward(long size) { - return new InputTypeFeedForward(size, null); - } - - public static InputType feedForward(long size, DataFormat timeDistributedFormat) { - return new InputTypeFeedForward(size,timeDistributedFormat); - } - - /** - * InputType for recurrent neural network (time series) data - * - * @param size The size of the activations - * @return InputTypeRecurrent - */ - public static InputType recurrent(long size) { - return new InputTypeRecurrent(size); - } - - /** - * InputType for recurrent neural network (time series) data - * - * @param size The size of the activations - * @param timeSeriesLength Length of the input time series - * @return InputTypeRecurrent - */ - public static InputType recurrent(long size, long timeSeriesLength) { - return new InputTypeRecurrent(size, timeSeriesLength, RNNFormat.NCW); - } - - public static InputType recurrent(long size, RNNFormat format){ - return new InputTypeRecurrent(size, format); - } - - public static InputType recurrent(long size, long timeSeriesLength, RNNFormat format){ - return new InputTypeRecurrent(size, timeSeriesLength, format); - } - /** - * Input type for convolutional (CNN) data, that is 4d with shape [miniBatchSize, channels, height, width]. - * For CNN data that has been flattened, use {@link #convolutionalFlat(long, long, long)} - * - * @param height height of the input - * @param width Width of the input - * @param depth Depth, or number of channels - * @return InputTypeConvolutional - */ - public static InputType convolutional(long height, long width, long depth) { - return convolutional(height, width, depth, getDefaultCNN2DFormat()); - } - - public static InputType convolutional(long height, long width, long depth, CNN2DFormat format){ - return new InputTypeConvolutional(height, width, depth, format); - } - - /** - * Input type for 3D convolutional (CNN3D) data in NDHWC format, that is 5d with shape - * [miniBatchSize, depth, height, width, channels]. - * - * @param height height of the input - * @param width Width of the input - * @param depth Depth of the input - * @param channels Number of channels of the input - * @return InputTypeConvolutional3D - * @deprecated Use {@link #convolutional3D(Convolution3D.DataFormat, long, long, long, long)} - */ - @Deprecated - public static InputType convolutional3D(long depth, long height, long width, long channels) { - return convolutional3D(Convolution3D.DataFormat.NDHWC, depth, height, width, channels); - } - - /** - * Input type for 3D convolutional (CNN3D) 5d data:
- * If NDHWC format [miniBatchSize, depth, height, width, channels]
- * If NDCWH - * - * @param height height of the input - * @param width Width of the input - * @param depth Depth of the input - * @param channels Number of channels of the input - * @return InputTypeConvolutional3D - */ - public static InputType convolutional3D(Convolution3D.DataFormat dataFormat, long depth, long height, long width, long channels) { - return new InputTypeConvolutional3D(dataFormat, depth, height, width, channels); - } - - /** - * Input type for convolutional (CNN) data, where the data is in flattened (row vector) format. - * Expect data with shape [miniBatchSize, height * width * channels]. For CNN data in 4d format, - * use {@link #convolutional(long, long, long)} - * - * @param height Height of the (unflattened) data represented by this input type - * @param width Width of the (unflattened) data represented by this input type - * @param depth Depth of the (unflattened) data represented by this input type - * @return InputTypeConvolutionalFlat - */ - public static InputType convolutionalFlat(long height, long width, long depth) { - return new InputTypeConvolutionalFlat(height, width, depth); - } - - - @NoArgsConstructor - @Getter - @EqualsAndHashCode(callSuper = false) - public static class InputTypeFeedForward extends InputType { - private long size; - private DataFormat timeDistributedFormat; - - public InputTypeFeedForward(@JsonProperty("size") long size, @JsonProperty("timeDistributedFormat") DataFormat timeDistributedFormat) { - if(size <= 0) { - OneTimeLogger.warn(log,"Assigning a size of zero. This is normally only valid in model import cases with unknown dimensions."); - } - this.size = size; - this.timeDistributedFormat = timeDistributedFormat; - } - - @Override - public Type getType() { - return Type.FF; - } - - @Override - public String toString() { - return "InputTypeFeedForward(" + size + (timeDistributedFormat != null ? "," + timeDistributedFormat : "") + ")"; - } - - @Override - public long arrayElementsPerExample() { - return size; - } - - @Override - public long[] getShape(boolean includeBatchDim) { - if(includeBatchDim) return new long[]{-1, size}; - else return new long[]{size}; - } - } - - @NoArgsConstructor - @Getter - @EqualsAndHashCode(callSuper = false) - public static class InputTypeRecurrent extends InputType { - private long size; - private long timeSeriesLength; - private RNNFormat format = RNNFormat.NCW; - public InputTypeRecurrent(long size) { - this(size, -1); - } - public InputTypeRecurrent(long size, long timeSeriesLength){ - this(size, timeSeriesLength, RNNFormat.NCW); - } - - public InputTypeRecurrent(long size, RNNFormat format){ - this(size, -1, format); - } - public InputTypeRecurrent(@JsonProperty("size") long size, - @JsonProperty("timeSeriesLength") long timeSeriesLength, - @JsonProperty("format") RNNFormat format) { - this.size = size; - this.timeSeriesLength = timeSeriesLength; - this.format = format; - } - - @Override - public Type getType() { - return Type.RNN; - } - - @Override - public String toString() { - if (timeSeriesLength > 0) { - return "InputTypeRecurrent(" + size + ",timeSeriesLength=" + timeSeriesLength + ",format=" + format + ")"; - } else { - return "InputTypeRecurrent(" + size + ",format=" + format + ")"; - } - } - - @Override - public long arrayElementsPerExample() { - if (timeSeriesLength <= 0) { - throw new IllegalStateException("Cannot calculate number of array elements per example: " - + "time series length is not set. Use InputType.recurrent(int size, int timeSeriesLength) instead?"); - } - return timeSeriesLength * size; - } - - @Override - public long[] getShape(boolean includeBatchDim) { - if (includeBatchDim){ - if (format == RNNFormat.NCW) { - return new long[]{-1, size, timeSeriesLength}; - } - else{ - return new long[]{-1, timeSeriesLength, size}; - } - - } - else{ - if (format == RNNFormat.NCW) { - return new long[]{size, timeSeriesLength}; - } - else{ - return new long[]{timeSeriesLength, size}; - } - } - } - } - - @NoArgsConstructor - @Data - @EqualsAndHashCode(callSuper = false) - public static class InputTypeConvolutional extends InputType { - private long height; - private long width; - private long channels; - private CNN2DFormat format = CNN2DFormat.NCHW; //Default for JSON deserialization of older configurations - - public InputTypeConvolutional(@JsonProperty("height") long height, @JsonProperty("width") long width, - @JsonProperty("channels") long channels, @JsonProperty("format") CNN2DFormat format) { - if(height <= 0) { - OneTimeLogger.warn(log,"Assigning height of 0. Normally this is not valid. Exceptions for this are generally related" + - "to model import and unknown dimensions"); - } - - if(width <= 0) { - OneTimeLogger.warn(log,"Assigning height of 0. Normally this is not valid. Exceptions for this are generally related" + - "to model import and unknown dimensions"); - } - - if(width <= 0) { - OneTimeLogger.warn(log,"Assigning width of 0. Normally this is not valid. Exceptions for this are generally related" + - "to model import and unknown dimensions"); - } - - if(channels <= 0) { - OneTimeLogger.warn(log,"Assigning width of 0. Normally this is not valid. Exceptions for this are generally related" + - "to model import and unknown dimensions"); - } - - - this.height = height; - this.width = width; - this.channels = channels; - if(format != null) - this.format = format; - } - - public InputTypeConvolutional(long height, long width, long channels) { - this(height, width, channels, CNN2DFormat.NCHW); - } - - /** - * Return the number of channels / depth for this 2D convolution. This method has been deprecated, - * for consistency purposes, use getChannels() instead. - * - * @return number of channels, i.e. depth for 2D convolutions - */ - @Deprecated - public long getDepth() { - return channels; - } - - /** - * Set the number of channels / depth for this 2D convolution. This method has been deprecated, - * for consistency purposes, use setChannels(channels) instead. - * - **/ - @Deprecated - public void setDepth(long depth) { - this.channels = depth; - } - - @Override - public Type getType() { - return Type.CNN; - } - - @Override - public String toString() { - return "InputTypeConvolutional(h=" + height + ",w=" + width + ",c=" + channels + "," + format + ")"; - } - - @Override - public long arrayElementsPerExample() { - return height * width * channels; - } - - @Override - public long[] getShape(boolean includeBatchDim) { - if(format == CNN2DFormat.NCHW){ - if(includeBatchDim) return new long[]{-1, channels, height, width}; - else return new long[]{channels, height, width}; - } else { - if(includeBatchDim) return new long[]{-1, height, width, channels}; - else return new long[]{height, width, channels}; - } - } - } - - @NoArgsConstructor - @Data - @EqualsAndHashCode(callSuper = false) - public static class InputTypeConvolutional3D extends InputType { - private Convolution3D.DataFormat dataFormat; - private long depth; - private long height; - private long width; - private long channels; - - public InputTypeConvolutional3D(@JsonProperty("dataFormat") Convolution3D.DataFormat dataFormat, - @JsonProperty("depth") long depth, @JsonProperty("height") long height, @JsonProperty("width") long width, @JsonProperty("channels") long channels) { - this.dataFormat = dataFormat; - this.depth = depth; - this.height = height; - this.width = width; - this.channels = channels; - } - - @Override - public Type getType() { - return Type.CNN3D; - } - - @Override - public String toString() { - return "InputTypeConvolutional3D(format=" + dataFormat + ",d=" + depth + ",h=" + height + ",w=" + width + ",c=" + channels + ")"; - } - - @Override - public long arrayElementsPerExample() { - return height * width * depth * channels; - } - - @Override - public long[] getShape(boolean includeBatchDim) { - if(dataFormat == Convolution3D.DataFormat.NDHWC){ - if(includeBatchDim) return new long[]{-1, depth, height, width, channels}; - else return new long[]{depth, height, width, channels}; - } else { - if(includeBatchDim) return new long[]{-1, channels, depth, height, width}; - else return new long[]{channels, depth, height, width}; - } - } - } - - @NoArgsConstructor - @Data - @EqualsAndHashCode(callSuper = false) - public static class InputTypeConvolutionalFlat extends InputType { - private long height; - private long width; - private long depth; - - public InputTypeConvolutionalFlat(@JsonProperty("height") long height, @JsonProperty("width") long width, @JsonProperty("depth") long depth) { - this.height = height; - this.width = width; - this.depth = depth; - } - - @Override - public Type getType() { - return Type.CNNFlat; - } - - public long getFlattenedSize() { - return height * width * depth; - } - - public InputType getUnflattenedType() { - return InputType.convolutional(height, width, depth); - } - - @Override - public String toString() { - return "InputTypeConvolutionalFlat(h=" + height + ",w=" + width + ",d=" + depth + ")"; - } - - @Override - public long arrayElementsPerExample() { - return height * width * depth; - } - - @Override - public long[] getShape(boolean includeBatchDim) { - if(includeBatchDim) return new long[]{-1, depth, height, width}; - else return new long[]{depth, height, width}; - } - } - - - public static InputType inferInputType(INDArray inputArray) { - //Note: ConvolutionalFlat and FeedForward look identical... but either should work OK if using something - // like FeedForwardToCnnPreProcessor - - switch (inputArray.rank()) { - case 2: - return InputType.feedForward(inputArray.size(1)); - case 3: - return InputType.recurrent(inputArray.size(1), (int) inputArray.size(2)); - case 4: - //Order: [minibatch, channels, height, width] -> [h, w, c] - return InputType.convolutional(inputArray.size(2), (int) inputArray.size(3), (int) inputArray.size(1)); - case 5: - //Order: [minibatch, channels, depth, height, width] -> [d, h, w, c] - return InputType.convolutional3D(inputArray.size(2), (int) inputArray.size(3), - (int) inputArray.size(4), (int) inputArray.size(1)); - default: - throw new IllegalArgumentException( - "Cannot infer input type for array with shape: " + Arrays.toString(inputArray.shape())); - } - } - - public static InputType[] inferInputTypes(INDArray... inputArrays) { - InputType[] out = new InputType[inputArrays.length]; - for (int i = 0; i < inputArrays.length; i++) { - out[i] = inferInputType(inputArrays[i]); - } - - return out; - } - -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/Bidirectional.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/Bidirectional.java deleted file mode 100644 index b20772ade..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/recurrent/Bidirectional.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.layers.recurrent; - -import lombok.*; -import org.deeplearning4j.nn.api.ParamInitializer; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.InputPreProcessor; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.RNNFormat; -import org.deeplearning4j.nn.conf.inputs.InputType; -import org.deeplearning4j.nn.conf.layers.BaseRecurrentLayer; -import org.deeplearning4j.nn.conf.layers.FeedForwardLayer; -import org.deeplearning4j.nn.conf.layers.Layer; -import org.deeplearning4j.nn.conf.layers.wrapper.BaseWrapperLayer; -import org.deeplearning4j.nn.conf.memory.LayerMemoryReport; -import org.deeplearning4j.nn.layers.recurrent.BidirectionalLayer; -import org.deeplearning4j.nn.params.BidirectionalParamInitializer; -import org.deeplearning4j.optimize.api.TrainingListener; -import org.deeplearning4j.util.TimeSeriesUtils; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.learning.config.IUpdater; -import org.nd4j.linalg.learning.regularization.Regularization; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; - -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import static org.nd4j.linalg.indexing.NDArrayIndex.interval; -import static org.nd4j.linalg.indexing.NDArrayIndex.point; - -@NoArgsConstructor -@Data -@EqualsAndHashCode(callSuper = true, exclude = {"initializer"}) -@JsonIgnoreProperties({"initializer"}) -public class Bidirectional extends Layer { - - /** - * This Mode enumeration defines how the activations for the forward and backward networks should be combined.
- * ADD: out = forward + backward (elementwise addition)
MUL: out = forward * backward (elementwise - * multiplication)
AVERAGE: out = 0.5 * (forward + backward)
CONCAT: Concatenate the activations.
Where - * 'forward' is the activations for the forward RNN, and 'backward' is the activations for the backward RNN. In all - * cases except CONCAT, the output activations size is the same size as the standard RNN that is being wrapped by - * this layer. In the CONCAT case, the output activations size (dimension 1) is 2x larger than the standard RNN's - * activations array. - */ - public enum Mode { - ADD, MUL, AVERAGE, CONCAT - } - - private Layer fwd; - private Layer bwd; - private Mode mode; - private transient BidirectionalParamInitializer initializer; - - private Bidirectional(Bidirectional.Builder builder) { - super(builder); - } - - /** - * Create a Bidirectional wrapper, with the default Mode (CONCAT) for the specified layer - * - * @param layer layer to wrap - */ - public Bidirectional(@NonNull Layer layer) { - this(Mode.CONCAT, layer); - } - - /** - * Create a Bidirectional wrapper for the specified layer - * - * @param mode Mode to use to combine activations. See {@link Mode} for details - * @param layer layer to wrap - */ - public Bidirectional(@NonNull Mode mode, @NonNull Layer layer) { - if (!(layer instanceof BaseRecurrentLayer || layer instanceof LastTimeStep - || layer instanceof BaseWrapperLayer)) { - throw new IllegalArgumentException("Cannot wrap a non-recurrent layer: " - + "config must extend BaseRecurrentLayer or LastTimeStep " + "Got class: " - + layer.getClass()); - } - this.fwd = layer; - this.bwd = layer.clone(); - this.mode = mode; - } - - public long getNOut() { - if (this.fwd instanceof LastTimeStep) { - return ((FeedForwardLayer) ((LastTimeStep) this.fwd).getUnderlying()).getNOut(); - } else { - return ((FeedForwardLayer) this.fwd).getNOut(); - } - } - - public long getNIn() { - if (this.fwd instanceof LastTimeStep) { - return ((FeedForwardLayer) ((LastTimeStep) this.fwd).getUnderlying()).getNIn(); - } else { - return ((FeedForwardLayer) this.fwd).getNIn(); - } - } - - public RNNFormat getRNNDataFormat(){ - return TimeSeriesUtils.getFormatFromRnnLayer(fwd); - } - - @Override - public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf, - Collection trainingListeners, int layerIndex, INDArray layerParamsView, - boolean initializeParams, DataType networkDataType) { - NeuralNetConfiguration c1 = conf.clone(); - NeuralNetConfiguration c2 = conf.clone(); - c1.setLayer(fwd); - c2.setLayer(bwd); - - long n = layerParamsView.length() / 2; - INDArray fp = layerParamsView.get(interval(0,0,true), interval(0, n)); - INDArray bp = layerParamsView.get(interval(0,0,true), interval(n, 2 * n)); - org.deeplearning4j.nn.api.Layer f = fwd.instantiate(c1, trainingListeners, layerIndex, fp, initializeParams, networkDataType); - - org.deeplearning4j.nn.api.Layer b = bwd.instantiate(c2, trainingListeners, layerIndex, bp, initializeParams, networkDataType); - - BidirectionalLayer ret = new BidirectionalLayer(conf, f, b, layerParamsView); - Map paramTable = initializer().init(conf, layerParamsView, initializeParams); - ret.setParamTable(paramTable); - ret.setConf(conf); - - return ret; - } - - @Override - public ParamInitializer initializer() { - if (initializer == null) { - initializer = new BidirectionalParamInitializer(this); - } - return initializer; - } - - @Override - public InputType getOutputType(int layerIndex, InputType inputType) { - InputType outOrig = fwd.getOutputType(layerIndex, inputType); - - if (fwd instanceof LastTimeStep) { - InputType.InputTypeFeedForward ff = (InputType.InputTypeFeedForward) outOrig; - if (mode == Mode.CONCAT) { - return InputType.feedForward(2 * ff.getSize()); - } else { - return ff; - } - } else { - InputType.InputTypeRecurrent r = (InputType.InputTypeRecurrent) outOrig; - if (mode == Mode.CONCAT) { - return InputType.recurrent(2 * r.getSize(), getRNNDataFormat()); - } else { - return r; - } - } - } - - @Override - public void setNIn(InputType inputType, boolean override) { - fwd.setNIn(inputType, override); - bwd.setNIn(inputType, override); - } - - @Override - public InputPreProcessor getPreProcessorForInputType(InputType inputType) { - return fwd.getPreProcessorForInputType(inputType); - } - - @Override - public List getRegularizationByParam(String paramName){ - //Strip forward/backward prefix from param name - return fwd.getRegularizationByParam(paramName.substring(1)); - } - - @Override - public boolean isPretrainParam(String paramName) { - return fwd.isPretrainParam(paramName.substring(1)); - } - - /** - * Get the updater for the given parameter. Typically the same updater will be used for all updaters, but this is - * not necessarily the case - * - * @param paramName Parameter name - * @return IUpdater for the parameter - */ - public IUpdater getUpdaterByParam(String paramName) { - String sub = paramName.substring(1); - return fwd.getUpdaterByParam(sub); - } - - @Override - public GradientNormalization getGradientNormalization() { - return fwd.getGradientNormalization(); - } - - @Override - public double getGradientNormalizationThreshold() { - return fwd.getGradientNormalizationThreshold(); - } - - @Override - public void setLayerName(String layerName) { - this.layerName = layerName; - fwd.setLayerName(layerName); - bwd.setLayerName(layerName); - } - - @Override - public LayerMemoryReport getMemoryReport(InputType inputType) { - LayerMemoryReport lmr = fwd.getMemoryReport(inputType); - lmr.scale(2); //Double all memory use - return lmr; - } - - @AllArgsConstructor - @Getter - @Setter - public static class Builder extends Layer.Builder { - - private Mode mode; - private Layer layer; - - public void setLayer(Layer layer) { - rnnLayer(layer); - } - - public Builder mode(Mode mode) { - this.setMode(mode); - return this; - } - - public Builder rnnLayer(Layer layer) { - if (!(layer instanceof BaseRecurrentLayer || layer instanceof LastTimeStep - || layer instanceof BaseWrapperLayer)) { - throw new IllegalArgumentException("Cannot wrap a non-recurrent layer: " - + "config must extend BaseRecurrentLayer or LastTimeStep " + "Got class: " - + layer.getClass()); - } - this.setLayer(layer); - return this; - } - - @SuppressWarnings("unchecked") - public Bidirectional build() { - return new Bidirectional(this); - } - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/JsonMappers.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/JsonMappers.java deleted file mode 100644 index 0dd62fc15..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/JsonMappers.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.serde; - -import lombok.extern.slf4j.Slf4j; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.serde.legacy.LegacyJsonFormat; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.databind.*; -import org.nd4j.shade.jackson.databind.cfg.MapperConfig; -import org.nd4j.shade.jackson.databind.deser.BeanDeserializerModifier; -import org.nd4j.shade.jackson.databind.introspect.Annotated; -import org.nd4j.shade.jackson.databind.introspect.AnnotatedClass; -import org.nd4j.shade.jackson.databind.introspect.AnnotationMap; -import org.nd4j.shade.jackson.databind.introspect.JacksonAnnotationIntrospector; -import org.nd4j.shade.jackson.databind.jsontype.TypeResolverBuilder; -import org.nd4j.shade.jackson.databind.module.SimpleModule; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; - -@Slf4j -public class JsonMappers { - - private static ObjectMapper jsonMapper = new ObjectMapper(); - private static ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); - - private static ObjectMapper legacyMapper; - - static { - configureMapper(jsonMapper); - configureMapper(yamlMapper); - } - - /** - * @return The default/primary ObjectMapper for deserializing JSON network configurations in DL4J - */ - public static ObjectMapper getMapper(){ - return jsonMapper; - } - - public static synchronized ObjectMapper getLegacyMapper(){ - if(legacyMapper == null){ - legacyMapper = LegacyJsonFormat.getMapper100alpha(); - configureMapper(legacyMapper); - } - return legacyMapper; - } - - /** - * @return The default/primary ObjectMapper for deserializing network configurations in DL4J (YAML format) - */ - public static ObjectMapper getMapperYaml() { - return yamlMapper; - } - - private static void configureMapper(ObjectMapper ret) { - ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); - ret.enable(SerializationFeature.INDENT_OUTPUT); - - SimpleModule customDeserializerModule = new SimpleModule(); - customDeserializerModule.setDeserializerModifier(new BeanDeserializerModifier() { - @Override - public JsonDeserializer modifyDeserializer(DeserializationConfig config, BeanDescription beanDesc, - JsonDeserializer deserializer) { - //Use our custom deserializers to handle backward compatibility for updaters -> IUpdater - if (beanDesc.getBeanClass() == MultiLayerConfiguration.class) { - return new MultiLayerConfigurationDeserializer(deserializer); - } else if (beanDesc.getBeanClass() == ComputationGraphConfiguration.class) { - return new ComputationGraphConfigurationDeserializer(deserializer); - } - return deserializer; - } - }); - - ret.registerModule(customDeserializerModule); - } -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyJsonFormat.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyJsonFormat.java deleted file mode 100644 index 8726e3bc7..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/serde/legacy/LegacyJsonFormat.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.nn.conf.serde.legacy; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import org.deeplearning4j.nn.conf.InputPreProcessor; -import org.deeplearning4j.nn.conf.graph.*; -import org.deeplearning4j.nn.conf.graph.rnn.DuplicateToTimeSeriesVertex; -import org.deeplearning4j.nn.conf.graph.rnn.LastTimeStepVertex; -import org.deeplearning4j.nn.conf.graph.rnn.ReverseTimeSeriesVertex; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.layers.convolutional.Cropping1D; -import org.deeplearning4j.nn.conf.layers.convolutional.Cropping2D; -import org.deeplearning4j.nn.conf.layers.misc.ElementWiseMultiplicationLayer; -import org.deeplearning4j.nn.conf.layers.misc.FrozenLayer; -import org.deeplearning4j.nn.conf.layers.objdetect.Yolo2OutputLayer; -import org.deeplearning4j.nn.conf.layers.recurrent.Bidirectional; -import org.deeplearning4j.nn.conf.layers.recurrent.SimpleRnn; -import org.deeplearning4j.nn.conf.layers.util.MaskLayer; -import org.deeplearning4j.nn.conf.layers.util.MaskZeroLayer; -import org.deeplearning4j.nn.conf.layers.variational.*; -import org.deeplearning4j.nn.conf.preprocessor.*; -import org.nd4j.linalg.activations.IActivation; -import org.nd4j.linalg.activations.impl.*; -import org.nd4j.linalg.lossfunctions.ILossFunction; -import org.nd4j.linalg.lossfunctions.impl.*; -import org.nd4j.shade.jackson.annotation.JsonSubTypes; -import org.nd4j.shade.jackson.annotation.JsonTypeInfo; -import org.nd4j.shade.jackson.databind.ObjectMapper; - -public class LegacyJsonFormat { - - private LegacyJsonFormat(){ } - - /** - * Get a mapper (minus general config) suitable for loading old format JSON - 1.0.0-alpha and before - * @return Object mapper - */ - public static ObjectMapper getMapper100alpha(){ - //After 1.0.0-alpha, we switched from wrapper object to @class for subtype information - ObjectMapper om = new ObjectMapper(); - - om.addMixIn(InputPreProcessor.class, InputPreProcessorMixin.class); - om.addMixIn(GraphVertex.class, GraphVertexMixin.class); - om.addMixIn(Layer.class, LayerMixin.class); - om.addMixIn(ReconstructionDistribution.class, ReconstructionDistributionMixin.class); - om.addMixIn(IActivation.class, IActivationMixin.class); - om.addMixIn(ILossFunction.class, ILossFunctionMixin.class); - - return om; - } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = CnnToFeedForwardPreProcessor.class, name = "cnnToFeedForward"), - @JsonSubTypes.Type(value = CnnToRnnPreProcessor.class, name = "cnnToRnn"), - @JsonSubTypes.Type(value = ComposableInputPreProcessor.class, name = "composableInput"), - @JsonSubTypes.Type(value = FeedForwardToCnnPreProcessor.class, name = "feedForwardToCnn"), - @JsonSubTypes.Type(value = FeedForwardToRnnPreProcessor.class, name = "feedForwardToRnn"), - @JsonSubTypes.Type(value = RnnToFeedForwardPreProcessor.class, name = "rnnToFeedForward"), - @JsonSubTypes.Type(value = RnnToCnnPreProcessor.class, name = "rnnToCnn")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class InputPreProcessorMixin { } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = ElementWiseVertex.class, name = "ElementWiseVertex"), - @JsonSubTypes.Type(value = MergeVertex.class, name = "MergeVertex"), - @JsonSubTypes.Type(value = SubsetVertex.class, name = "SubsetVertex"), - @JsonSubTypes.Type(value = LayerVertex.class, name = "LayerVertex"), - @JsonSubTypes.Type(value = LastTimeStepVertex.class, name = "LastTimeStepVertex"), - @JsonSubTypes.Type(value = ReverseTimeSeriesVertex.class, name = "ReverseTimeSeriesVertex"), - @JsonSubTypes.Type(value = DuplicateToTimeSeriesVertex.class, name = "DuplicateToTimeSeriesVertex"), - @JsonSubTypes.Type(value = PreprocessorVertex.class, name = "PreprocessorVertex"), - @JsonSubTypes.Type(value = StackVertex.class, name = "StackVertex"), - @JsonSubTypes.Type(value = UnstackVertex.class, name = "UnstackVertex"), - @JsonSubTypes.Type(value = L2Vertex.class, name = "L2Vertex"), - @JsonSubTypes.Type(value = ScaleVertex.class, name = "ScaleVertex"), - @JsonSubTypes.Type(value = L2NormalizeVertex.class, name = "L2NormalizeVertex")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class GraphVertexMixin{ } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = AutoEncoder.class, name = "autoEncoder"), - @JsonSubTypes.Type(value = ConvolutionLayer.class, name = "convolution"), - @JsonSubTypes.Type(value = Convolution1DLayer.class, name = "convolution1d"), - @JsonSubTypes.Type(value = GravesLSTM.class, name = "gravesLSTM"), - @JsonSubTypes.Type(value = LSTM.class, name = "LSTM"), - @JsonSubTypes.Type(value = GravesBidirectionalLSTM.class, name = "gravesBidirectionalLSTM"), - @JsonSubTypes.Type(value = OutputLayer.class, name = "output"), - @JsonSubTypes.Type(value = CenterLossOutputLayer.class, name = "CenterLossOutputLayer"), - @JsonSubTypes.Type(value = RnnOutputLayer.class, name = "rnnoutput"), - @JsonSubTypes.Type(value = LossLayer.class, name = "loss"), - @JsonSubTypes.Type(value = DenseLayer.class, name = "dense"), - @JsonSubTypes.Type(value = SubsamplingLayer.class, name = "subsampling"), - @JsonSubTypes.Type(value = Subsampling1DLayer.class, name = "subsampling1d"), - @JsonSubTypes.Type(value = BatchNormalization.class, name = "batchNormalization"), - @JsonSubTypes.Type(value = LocalResponseNormalization.class, name = "localResponseNormalization"), - @JsonSubTypes.Type(value = EmbeddingLayer.class, name = "embedding"), - @JsonSubTypes.Type(value = ActivationLayer.class, name = "activation"), - @JsonSubTypes.Type(value = VariationalAutoencoder.class, name = "VariationalAutoencoder"), - @JsonSubTypes.Type(value = DropoutLayer.class, name = "dropout"), - @JsonSubTypes.Type(value = GlobalPoolingLayer.class, name = "GlobalPooling"), - @JsonSubTypes.Type(value = ZeroPaddingLayer.class, name = "zeroPadding"), - @JsonSubTypes.Type(value = ZeroPadding1DLayer.class, name = "zeroPadding1d"), - @JsonSubTypes.Type(value = FrozenLayer.class, name = "FrozenLayer"), - @JsonSubTypes.Type(value = Upsampling2D.class, name = "Upsampling2D"), - @JsonSubTypes.Type(value = Yolo2OutputLayer.class, name = "Yolo2OutputLayer"), - @JsonSubTypes.Type(value = RnnLossLayer.class, name = "RnnLossLayer"), - @JsonSubTypes.Type(value = CnnLossLayer.class, name = "CnnLossLayer"), - @JsonSubTypes.Type(value = Bidirectional.class, name = "Bidirectional"), - @JsonSubTypes.Type(value = SimpleRnn.class, name = "SimpleRnn"), - @JsonSubTypes.Type(value = ElementWiseMultiplicationLayer.class, name = "ElementWiseMult"), - @JsonSubTypes.Type(value = MaskLayer.class, name = "MaskLayer"), - @JsonSubTypes.Type(value = MaskZeroLayer.class, name = "MaskZeroLayer"), - @JsonSubTypes.Type(value = Cropping1D.class, name = "Cropping1D"), - @JsonSubTypes.Type(value = Cropping2D.class, name = "Cropping2D")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class LayerMixin {} - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = GaussianReconstructionDistribution.class, name = "Gaussian"), - @JsonSubTypes.Type(value = BernoulliReconstructionDistribution.class, name = "Bernoulli"), - @JsonSubTypes.Type(value = ExponentialReconstructionDistribution.class, name = "Exponential"), - @JsonSubTypes.Type(value = CompositeReconstructionDistribution.class, name = "Composite"), - @JsonSubTypes.Type(value = LossFunctionWrapper.class, name = "LossWrapper")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class ReconstructionDistributionMixin {} - - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = ActivationCube.class, name = "Cube"), - @JsonSubTypes.Type(value = ActivationELU.class, name = "ELU"), - @JsonSubTypes.Type(value = ActivationHardSigmoid.class, name = "HardSigmoid"), - @JsonSubTypes.Type(value = ActivationHardTanH.class, name = "HardTanh"), - @JsonSubTypes.Type(value = ActivationIdentity.class, name = "Identity"), - @JsonSubTypes.Type(value = ActivationLReLU.class, name = "LReLU"), - @JsonSubTypes.Type(value = ActivationRationalTanh.class, name = "RationalTanh"), - @JsonSubTypes.Type(value = ActivationRectifiedTanh.class, name = "RectifiedTanh"), - @JsonSubTypes.Type(value = ActivationSELU.class, name = "SELU"), - @JsonSubTypes.Type(value = ActivationSwish.class, name = "SWISH"), - @JsonSubTypes.Type(value = ActivationReLU.class, name = "ReLU"), - @JsonSubTypes.Type(value = ActivationRReLU.class, name = "RReLU"), - @JsonSubTypes.Type(value = ActivationSigmoid.class, name = "Sigmoid"), - @JsonSubTypes.Type(value = ActivationSoftmax.class, name = "Softmax"), - @JsonSubTypes.Type(value = ActivationSoftPlus.class, name = "SoftPlus"), - @JsonSubTypes.Type(value = ActivationSoftSign.class, name = "SoftSign"), - @JsonSubTypes.Type(value = ActivationTanH.class, name = "TanH")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class IActivationMixin {} - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) - @JsonSubTypes(value = {@JsonSubTypes.Type(value = LossBinaryXENT.class, name = "BinaryXENT"), - @JsonSubTypes.Type(value = LossCosineProximity.class, name = "CosineProximity"), - @JsonSubTypes.Type(value = LossHinge.class, name = "Hinge"), - @JsonSubTypes.Type(value = LossKLD.class, name = "KLD"), - @JsonSubTypes.Type(value = LossMAE.class, name = "MAE"), - @JsonSubTypes.Type(value = LossL1.class, name = "L1"), - @JsonSubTypes.Type(value = LossMAPE.class, name = "MAPE"), - @JsonSubTypes.Type(value = LossMCXENT.class, name = "MCXENT"), - @JsonSubTypes.Type(value = LossMSE.class, name = "MSE"), - @JsonSubTypes.Type(value = LossL2.class, name = "L2"), - @JsonSubTypes.Type(value = LossMSLE.class, name = "MSLE"), - @JsonSubTypes.Type(value = LossNegativeLogLikelihood.class, name = "NegativeLogLikelihood"), - @JsonSubTypes.Type(value = LossPoisson.class, name = "Poisson"), - @JsonSubTypes.Type(value = LossSquaredHinge.class, name = "SquaredHinge"), - @JsonSubTypes.Type(value = LossFMeasure.class, name = "FMeasure")}) - @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class ILossFunctionMixin {} -} diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/CheckpointListener.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/CheckpointListener.java deleted file mode 100644 index b96a7ce9f..000000000 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/optimize/listeners/CheckpointListener.java +++ /dev/null @@ -1,652 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.optimize.listeners; - -import org.nd4j.shade.guava.io.Files; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; -import org.deeplearning4j.nn.api.Model; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.optimize.api.BaseTrainingListener; -import org.deeplearning4j.util.ModelSerializer; -import org.nd4j.common.base.Preconditions; - -import java.io.*; -import java.nio.charset.Charset; -import java.util.*; -import java.util.concurrent.TimeUnit; - -@Slf4j -public class CheckpointListener extends BaseTrainingListener implements Serializable { - - private enum KeepMode {ALL, LAST, LAST_AND_EVERY}; - private static final String[] MODEL_TYPES = new String[]{"MultiLayerNetwork", "ComputationGraph", "Model"}; - - private File rootDir; - private KeepMode keepMode; - private int keepLast; - private int keepEvery; - private boolean logSaving; - private boolean deleteExisting; - - private Integer saveEveryNEpochs; - private Integer saveEveryNIterations; - private boolean saveEveryNIterSinceLast; - private Long saveEveryAmount; - private TimeUnit saveEveryUnit; - private Long saveEveryMs; - private boolean saveEverySinceLast; - - private int lastCheckpointNum = -1; - private File checkpointRecordFile; - - private Checkpoint lastCheckpoint; - private long startTime = -1; - private int startIter = -1; - private Long lastSaveEveryMsNoSinceLast; - - private CheckpointListener(Builder builder){ - this.rootDir = builder.rootDir; - this.keepMode = builder.keepMode; - this.keepLast = builder.keepLast; - this.keepEvery = builder.keepEvery; - this.logSaving = builder.logSaving; - this.deleteExisting = builder.deleteExisting; - - this.saveEveryNEpochs = builder.saveEveryNEpochs; - this.saveEveryNIterations = builder.saveEveryNIterations; - this.saveEveryNIterSinceLast = builder.saveEveryNIterSinceLast; - this.saveEveryAmount = builder.saveEveryAmount; - this.saveEveryUnit = builder.saveEveryUnit; - this.saveEverySinceLast = builder.saveEverySinceLast; - - if(saveEveryAmount != null){ - saveEveryMs = TimeUnit.MILLISECONDS.convert(saveEveryAmount, saveEveryUnit); - } - - this.checkpointRecordFile = new File(rootDir, "checkpointInfo.txt"); - if(this.checkpointRecordFile.exists() && this.checkpointRecordFile.length() > 0){ - - if(deleteExisting){ - //Delete any files matching: - //"checkpoint_" + checkpointNum + "_" + modelType + ".zip"; - this.checkpointRecordFile.delete(); - File[] files = rootDir.listFiles(); - if(files != null && files.length > 0){ - for(File f : files){ - String name = f.getName(); - if(name.startsWith("checkpoint_") && (name.endsWith("MultiLayerNetwork.zip") || name.endsWith("ComputationGraph.zip"))){ - f.delete(); - } - } - } - } else { - throw new IllegalStateException("Detected existing checkpoint files at directory " + rootDir.getAbsolutePath() + - ". Use deleteExisting(true) to delete existing checkpoint files when present."); - } - } - } - - @Override - public void onEpochEnd(Model model) { - int epochsDone = getEpoch(model) + 1; - if(saveEveryNEpochs != null && epochsDone > 0 && epochsDone % saveEveryNEpochs == 0){ - //Save: - saveCheckpoint(model); - } - //General saving conditions: don't need to check here - will check in iterationDone - } - - @Override - public void iterationDone(Model model, int iteration, int epoch) { - if (startTime < 0) { - startTime = System.currentTimeMillis(); - startIter = iteration; - return; - } - - //Check iterations saving condition: - if(saveEveryNIterations != null){ - if(saveEveryNIterSinceLast){ - //Consider last saved model when deciding whether to save - long lastSaveIter = (lastCheckpoint != null ? lastCheckpoint.getIteration() : startIter); - if(iteration - lastSaveIter >= saveEveryNIterations){ - saveCheckpoint(model); - return; - } - } else { - //Same every N iterations, regardless of saving time - if(iteration > 0 && iteration % saveEveryNIterations == 0){ - saveCheckpoint(model); - return; - } - } - } - - //Check time saving condition: - long time = System.currentTimeMillis(); - if(saveEveryUnit != null){ - if(saveEverySinceLast){ - //Consider last saved when deciding whether to save - long lastSaveTime = (lastCheckpoint != null ? lastCheckpoint.getTimestamp() : startTime); - if((time - lastSaveTime) >= saveEveryMs){ - saveCheckpoint(model); - return; - } - } else { - //Save periodically, regardless of when last model was saved - long lastSave = (lastSaveEveryMsNoSinceLast != null ? lastSaveEveryMsNoSinceLast : startTime); - if((time - lastSave) > saveEveryMs){ - saveCheckpoint(model); - lastSaveEveryMsNoSinceLast = time; - return; - } - } - } - } - - private void saveCheckpoint(Model model) { - try{ - saveCheckpointHelper(model); - } catch (Exception e){ - throw new RuntimeException("Error saving checkpoint", e); - } - } - - private void saveCheckpointHelper(Model model) throws Exception { - if(!checkpointRecordFile.exists()){ - checkpointRecordFile.createNewFile(); - write(Checkpoint.getFileHeader() + "\n", checkpointRecordFile); - } - - Checkpoint c = new Checkpoint(++lastCheckpointNum, System.currentTimeMillis(), getIter(model), getEpoch(model), - getModelType(model), null); - setFileName(c); - - ModelSerializer.writeModel(model, new File(rootDir, c.getFilename()), true); - - String s = c.toFileString(); - write(s + "\n", checkpointRecordFile); - - if(logSaving){ - log.info("Model checkpoint saved: epoch {}, iteration {}, path: {}", c.getEpoch(), c.getIteration(), - new File(rootDir, c.getFilename()).getPath() ); - } - this.lastCheckpoint = c; - - - //Finally: determine if we should delete some old models... - if(keepMode == null || keepMode == KeepMode.ALL){ - return; - } else if(keepMode == KeepMode.LAST){ - List checkpoints = availableCheckpoints(); - Iterator iter = checkpoints.iterator(); - while(checkpoints.size() > keepLast){ - Checkpoint toRemove = iter.next(); - File f = getFileForCheckpoint(toRemove); - f.delete(); - iter.remove(); - } - } else { - //Keep mode: last N and every M - for(Checkpoint cp : availableCheckpoints()){ - if(cp.getCheckpointNum() > 0 && (cp.getCheckpointNum()+1) % keepEvery == 0){ - //One of the "every M to keep" models - continue; - } else if(cp.getCheckpointNum() > lastCheckpointNum - keepLast ){ //Example: latest is 5, keep last 2 -> keep checkpoints 4 and 5 - //One of last N to keep - continue; - } - //Otherwise: delete file - File f = getFileForCheckpoint(cp); - f.delete(); - } - } - } - - private static void setFileName(Checkpoint c){ - String filename = getFileName(c.getCheckpointNum(), c.getModelType()); - c.setFilename(filename); - } - - private static String getFileName(int checkpointNum, String modelType){ - return "checkpoint_" + checkpointNum + "_" + modelType + ".zip"; - } - - private static String write(String str, File f){ - try { - if(!f.exists()){ - f.createNewFile(); - } - Files.append(str, f, Charset.defaultCharset()); - } catch (IOException e){ - throw new RuntimeException(e); - } - return str; - } - - protected static int getIter(Model model) { - if (model instanceof MultiLayerNetwork) { - return ((MultiLayerNetwork) model).getLayerWiseConfigurations().getIterationCount(); - } else if (model instanceof ComputationGraph) { - return ((ComputationGraph) model).getConfiguration().getIterationCount(); - } else { - return model.conf().getIterationCount(); - } - } - - protected static int getEpoch(Model model) { - if (model instanceof MultiLayerNetwork) { - return ((MultiLayerNetwork) model).getLayerWiseConfigurations().getEpochCount(); - } else if (model instanceof ComputationGraph) { - return ((ComputationGraph) model).getConfiguration().getEpochCount(); - } else { - return model.conf().getEpochCount(); - } - } - - protected static String getModelType(Model model){ - if(model.getClass() == MultiLayerNetwork.class){ - return "MultiLayerNetwork"; - } else if(model.getClass() == ComputationGraph.class){ - return "ComputationGraph"; - } else { - return "Model"; - } - } - - /** - * List all available checkpoints. A checkpoint is 'available' if the file can be loaded. Any checkpoint files that - * have been automatically deleted (given the configuration) will not be returned here. - * - * @return List of checkpoint files that can be loaded - */ - public List availableCheckpoints(){ - if(!checkpointRecordFile.exists()){ - return Collections.emptyList(); - } - - return availableCheckpoints(rootDir); - } - - /** - * List all available checkpoints. A checkpoint is 'available' if the file can be loaded. Any checkpoint files that - * have been automatically deleted (given the configuration) will not be returned here. - * Note that the checkpointInfo.txt file must exist, as this stores checkpoint information - * - * @return List of checkpoint files that can be loaded from the specified directory - */ - public static List availableCheckpoints(File directory){ - File checkpointRecordFile = new File(directory, "checkpointInfo.txt"); - Preconditions.checkState(checkpointRecordFile.exists(), "Could not find checkpoint record file at expected path %s", checkpointRecordFile.getAbsolutePath()); - - List lines; - try(InputStream is = new BufferedInputStream(new FileInputStream(checkpointRecordFile))){ - lines = IOUtils.readLines(is); - } catch (IOException e){ - throw new RuntimeException("Error loading checkpoint data from file: " + checkpointRecordFile.getAbsolutePath(), e); - } - - List out = new ArrayList<>(lines.size()-1); //Assume first line is header - for( int i=1; i all = availableCheckpoints(rootDir); - if(all.isEmpty()){ - return null; - } - return all.get(all.size()-1); - } - - /** - * Get the model file for the given checkpoint. Checkpoint model file must exist - * - * @param checkpoint Checkpoint to get the model file for - * @return Model file for the checkpoint - */ - public File getFileForCheckpoint(Checkpoint checkpoint){ - return getFileForCheckpoint(checkpoint.getCheckpointNum()); - } - - /** - * Get the model file for the given checkpoint number. Checkpoint model file must exist - * - * @param checkpointNum Checkpoint number to get the model file for - * @return Model file for the checkpoint - */ - public File getFileForCheckpoint(int checkpointNum) { - return getFileForCheckpoint(rootDir, checkpointNum); - } - - public static File getFileForCheckpoint(File rootDir, int checkpointNum){ - if(checkpointNum < 0){ - throw new IllegalArgumentException("Invalid checkpoint number: " + checkpointNum); - } - File f = null; - for(String s : MODEL_TYPES){ - f = new File(rootDir, getFileName(checkpointNum, s)); - if(f.exists()){ - return f; - } - } - throw new IllegalStateException("Model file for checkpoint " + checkpointNum + " does not exist"); - } - - /** - * Load a MultiLayerNetwork for the given checkpoint - * - * @param checkpoint Checkpoint model to load - * @return The loaded model - */ - public MultiLayerNetwork loadCheckpointMLN(Checkpoint checkpoint){ - return loadCheckpointMLN(checkpoint.getCheckpointNum()); - } - - /** - * Load a MultiLayerNetwork for the given checkpoint number - * - * @param checkpointNum Checkpoint model to load - * @return The loaded model - */ - public MultiLayerNetwork loadCheckpointMLN(int checkpointNum) { - return loadCheckpointMLN(rootDir, checkpointNum); - } - - /** - * Load a MultiLayerNetwork for the given checkpoint that resides in the specified root directory - * - * @param rootDir Root directory for the checkpoint - * @param checkpoint Checkpoint model to load - * @return The loaded model - */ - public static MultiLayerNetwork loadCheckpointMLN(File rootDir, Checkpoint checkpoint) { - return loadCheckpointMLN(rootDir, checkpoint.getCheckpointNum()); - } - - /** - * Load a MultiLayerNetwork for the given checkpoint number - * - * @param rootDir The directory that the checkpoint resides in - * @param checkpointNum Checkpoint model to load - * @return The loaded model - */ - public static MultiLayerNetwork loadCheckpointMLN(File rootDir, int checkpointNum){ - File f = getFileForCheckpoint(rootDir, checkpointNum); - try { - return ModelSerializer.restoreMultiLayerNetwork(f, true); - } catch (IOException e){ - throw new RuntimeException(e); - } - } - - /** - * Load the last (most recent) checkpoint from the specified root directory - * @param rootDir Root directory to load checpoint from - * @return MultiLayerNetwork for last checkpoint - */ - public static MultiLayerNetwork loadLastCheckpointMLN(File rootDir){ - Checkpoint last = lastCheckpoint(rootDir); - return loadCheckpointMLN(rootDir, last); - } - - /** - * Load a ComputationGraph for the given checkpoint - * - * @param checkpoint Checkpoint model to load - * @return The loaded model - */ - public ComputationGraph loadCheckpointCG(Checkpoint checkpoint){ - return loadCheckpointCG(checkpoint.getCheckpointNum()); - } - - /** - * Load a ComputationGraph for the given checkpoint from the specified root direcotry - * - * @param checkpoint Checkpoint model to load - * @return The loaded model - */ - public static ComputationGraph loadCheckpointCG(File rootDir, Checkpoint checkpoint){ - return loadCheckpointCG(rootDir, checkpoint.getCheckpointNum()); - } - - /** - * Load a ComputationGraph for the given checkpoint - * - * @param checkpointNum Checkpoint model number to load - * @return The loaded model - */ - public ComputationGraph loadCheckpointCG(int checkpointNum) { - return loadCheckpointCG(rootDir, checkpointNum); - } - - /** - * Load a ComputationGraph for the given checkpoint that resides in the specified root directory - * - * @param rootDir Directory that the checkpoint resides in - * @param checkpointNum Checkpoint model number to load - * @return The loaded model - */ - public static ComputationGraph loadCheckpointCG(File rootDir, int checkpointNum){ - File f = getFileForCheckpoint(rootDir, checkpointNum); - try { - return ModelSerializer.restoreComputationGraph(f, true); - } catch (IOException e){ - throw new RuntimeException(e); - } - } - - /** - * Load the last (most recent) checkpoint from the specified root directory - * @param rootDir Root directory to load checpoint from - * @return ComputationGraph for last checkpoint - */ - public static ComputationGraph loadLastCheckpointCG(File rootDir){ - Checkpoint last = lastCheckpoint(rootDir); - return loadCheckpointCG(rootDir, last); - } - - public static class Builder { - - private File rootDir; - private KeepMode keepMode; - private int keepLast; - private int keepEvery; - private boolean logSaving = true; - private boolean deleteExisting = false; - - private Integer saveEveryNEpochs; - private Integer saveEveryNIterations; - private boolean saveEveryNIterSinceLast; - private Long saveEveryAmount; - private TimeUnit saveEveryUnit; - private boolean saveEverySinceLast; - - /** - * @param rootDir Root directory to save models to - */ - public Builder(@NonNull String rootDir){ - this(new File(rootDir)); - } - - /** - * @param rootDir Root directory to save models to - */ - public Builder(@NonNull File rootDir){ - this.rootDir = rootDir; - } - - /** - * Save a model at the end of every epoch - */ - public Builder saveEveryEpoch(){ - return saveEveryNEpochs(1); - } - - /** - * Save a model at the end of every N epochs - */ - public Builder saveEveryNEpochs(int n){ - this.saveEveryNEpochs = n; - return this; - } - - /** - * Save a model every N iterations - */ - public Builder saveEveryNIterations(int n){ - return saveEveryNIterations(n, false); - } - - /** - * Save a model every N iterations (if sinceLast == false), or if N iterations have passed since - * the last model vas saved (if sinceLast == true) - */ - public Builder saveEveryNIterations(int n, boolean sinceLast){ - this.saveEveryNIterations = n; - this.saveEveryNIterSinceLast = sinceLast; - return this; - } - - /** - * Save a model periodically - * - * @param amount Quantity of the specified time unit - * @param timeUnit Time unit - */ - public Builder saveEvery(long amount, TimeUnit timeUnit){ - return saveEvery(amount, timeUnit, false); - } - - /** - * Save a model periodically (if sinceLast == false), or if the specified amount of time has elapsed since - * the last model was saved (if sinceLast == true) - * - * @param amount Quantity of the specified time unit - * @param timeUnit Time unit - */ - public Builder saveEvery(long amount, TimeUnit timeUnit, boolean sinceLast){ - this.saveEveryAmount = amount; - this.saveEveryUnit = timeUnit; - this.saveEverySinceLast = sinceLast; - return this; - } - - /** - * Keep all model checkpoints - i.e., don't delete any. Note that this is the default. - */ - public Builder keepAll(){ - this.keepMode = KeepMode.ALL; - return this; - } - - /** - * Keep only the last N most recent model checkpoint files. Older checkpoints will automatically be deleted. - * @param n Number of most recent checkpoints to keep - */ - public Builder keepLast(int n){ - if(n <= 0){ - throw new IllegalArgumentException("Number of model files to keep should be > 0 (got: " + n + ")"); - } - this.keepMode = KeepMode.LAST; - this.keepLast = n; - return this; - } - - /** - * Keep the last N most recent model checkpoint files, and every M checkpoint files.
- * For example: suppose you save every 100 iterations, for 2050 iteration, and use keepLastAndEvery(3,5). - * This means after 2050 iterations you would have saved 20 checkpoints - some of which will be deleted. - * Those remaining in this example: iterations 500, 1000, 1500, 1800, 1900, 2000. - * @param nLast Most recent checkpoints to keep - * @param everyN Every N checkpoints to keep (regardless of age) - */ - public Builder keepLastAndEvery(int nLast, int everyN){ - if(nLast <= 0){ - throw new IllegalArgumentException("Most recent number of model files to keep should be > 0 (got: " - + nLast + ")"); - } - if(everyN <= 0){ - throw new IllegalArgumentException("Every n model files to keep should be > 0 (got: " - + everyN + ")"); - } - - this.keepMode = KeepMode.LAST_AND_EVERY; - this.keepLast = nLast; - this.keepEvery = everyN; - return this; - } - - /** - * If true (the default) log a message every time a model is saved - * - * @param logSaving Whether checkpoint saves should be logged or not - */ - public Builder logSaving(boolean logSaving){ - this.logSaving = logSaving; - return this; - } - - /** - * If the checkpoint listener is set to save to a non-empty directory, should the CheckpointListener-related - * content be deleted?
- * This is disabled by default (and instead, an exception will be thrown if existing data is found)
- * WARNING: Be careful when enabling this, as it deletes all saved checkpoint models in the specified directory! - */ - public Builder deleteExisting(boolean deleteExisting){ - this.deleteExisting = deleteExisting; - return this; - } - - public CheckpointListener build(){ - if(saveEveryNEpochs == null && saveEveryAmount == null && saveEveryNIterations == null){ - throw new IllegalStateException("Cannot construct listener: no models will be saved (must use at least" + - " one of: save every N epochs, every N iterations, or every T time periods)"); - } - - return new CheckpointListener(this); - } - } -} diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/pom.xml b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/pom.xml new file mode 100644 index 000000000..e03ea6915 --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/pom.xml @@ -0,0 +1,104 @@ + + + + 4.0.0 + jar + + + net.brutex.ai + deeplearning4j-remote + 1.0.0-SNAPSHOT + + + deeplearning4j-json-server + 1.0.0-SNAPSHOT + deeplearning4j-json-server + + + + net.brutex.ai + nd4j-api + ${project.version} + + + + net.brutex.ai + nd4j-json-client + ${project.version} + + + + net.brutex.ai + nd4j-json-server + ${project.version} + + + + net.brutex.ai + deeplearning4j-parallel-wrapper + ${project.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + ch.qos.logback + logback-core + ${logback.version} + test + + + + ch.qos.logback + logback-classic + ${logback.version} + test + + + + net.brutex.ai + deeplearning4j-common-tests + ${project.version} + test + + + + + + + + test-nd4j-native + + false + + + + net.brutex.ai + nd4j-native + ${project.version} + test + + + + + + test-nd4j-cuda-11.2 + + false + + + + net.brutex.ai + nd4j-cuda-${cuda.version} + ${project.version} + test + + + + + diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/main/java/org/deeplearning4j/remote/DL4jServlet.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/main/java/org/deeplearning4j/remote/DL4jServlet.java new file mode 100644 index 000000000..66eaedb4c --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/main/java/org/deeplearning4j/remote/DL4jServlet.java @@ -0,0 +1,288 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.remote; + +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.parallelism.ParallelInference; +import org.nd4j.adapters.InferenceAdapter; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.remote.clients.serde.BinaryDeserializer; +import org.nd4j.remote.clients.serde.BinarySerializer; +import org.nd4j.remote.clients.serde.JsonDeserializer; +import org.nd4j.remote.clients.serde.JsonSerializer; +import org.nd4j.remote.serving.SameDiffServlet; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + + + +/** + * + * @author astoyakin + */ +@Slf4j +@NoArgsConstructor +public class DL4jServlet extends SameDiffServlet { + + protected ParallelInference parallelInference; + protected Model model; + protected boolean parallelEnabled = true; + + public DL4jServlet(@NonNull ParallelInference parallelInference, @NonNull InferenceAdapter inferenceAdapter, + JsonSerializer serializer, JsonDeserializer deserializer) { + super(inferenceAdapter, serializer, deserializer); + this.parallelInference = parallelInference; + this.model = null; + this.parallelEnabled = true; + } + + public DL4jServlet(@NonNull Model model, @NonNull InferenceAdapter inferenceAdapter, + JsonSerializer serializer, JsonDeserializer deserializer) { + super(inferenceAdapter, serializer, deserializer); + this.model = model; + this.parallelInference = null; + this.parallelEnabled = false; + } + + public DL4jServlet(@NonNull ParallelInference parallelInference, @NonNull InferenceAdapter inferenceAdapter, + BinarySerializer serializer, BinaryDeserializer deserializer) { + super(inferenceAdapter, serializer, deserializer); + this.parallelInference = parallelInference; + this.model = null; + this.parallelEnabled = true; + } + + public DL4jServlet(@NonNull Model model, @NonNull InferenceAdapter inferenceAdapter, + JsonSerializer jsonSerializer, JsonDeserializer jsonDeserializer, + BinarySerializer binarySerializer, BinaryDeserializer binaryDeserializer) { + super(inferenceAdapter, jsonSerializer, jsonDeserializer, binarySerializer, binaryDeserializer); + this.model = model; + this.parallelInference = null; + this.parallelEnabled = false; + } + + public DL4jServlet(@NonNull ParallelInference parallelInference, @NonNull InferenceAdapter inferenceAdapter, + JsonSerializer jsonSerializer, JsonDeserializer jsonDeserializer, + BinarySerializer binarySerializer, BinaryDeserializer binaryDeserializer) { + super(inferenceAdapter, jsonSerializer, jsonDeserializer, binarySerializer, binaryDeserializer); + this.parallelInference = parallelInference; + this.model = null; + this.parallelEnabled = true; + } + + private O process(MultiDataSet mds) { + O result = null; + if (parallelEnabled) { + // process result + result = inferenceAdapter.apply(parallelInference.output(mds.getFeatures(), mds.getFeaturesMaskArrays())); + } else { + synchronized (this) { + if (model instanceof ComputationGraph) + result = inferenceAdapter.apply(((ComputationGraph) model).output(false, mds.getFeatures(), mds.getFeaturesMaskArrays())); + else if (model instanceof MultiLayerNetwork) { + Preconditions.checkArgument(mds.getFeatures().length > 0 || (mds.getFeaturesMaskArrays() != null && mds.getFeaturesMaskArrays().length > 0), + "Input data for MultilayerNetwork is invalid!"); + result = inferenceAdapter.apply(((MultiLayerNetwork) model).output(mds.getFeatures()[0], false, + mds.getFeaturesMaskArrays() != null ? mds.getFeaturesMaskArrays()[0] : null, null)); + } + } + } + return result; + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException { + String processorReturned = ""; + MultiDataSet mds = null; + String path = request.getPathInfo(); + if (path.equals(SERVING_ENDPOINT)) { + val contentType = request.getContentType(); + if (contentType.equals(typeJson)) { + if (validateRequest(request, response)) { + val stream = request.getInputStream(); + val bufferedReader = new BufferedReader(new InputStreamReader(stream)); + char[] charBuffer = new char[128]; + int bytesRead = -1; + val buffer = new StringBuilder(); + while ((bytesRead = bufferedReader.read(charBuffer)) > 0) { + buffer.append(charBuffer, 0, bytesRead); + } + val requestString = buffer.toString(); + + mds = inferenceAdapter.apply(deserializer.deserialize(requestString)); + } + } + else if (contentType.equals(typeBinary)) { + val stream = request.getInputStream(); + int available = request.getContentLength(); + if (available <= 0) { + response.sendError(411, "Content length is unavailable"); + } + else { + byte[] data = new byte[available]; + stream.read(data, 0, available); + + mds = inferenceAdapter.apply(binaryDeserializer.deserialize(data)); + } + } + if (mds == null) + log.error("InferenceAdapter failed"); + else { + val result = process(mds); + if (binarySerializer != null) { + byte[] serialized = binarySerializer.serialize(result); + response.setContentType(typeBinary); + response.setContentLength(serialized.length); + val out = response.getOutputStream(); + out.write(serialized); + } + else { + processorReturned = serializer.serialize(result); + try { + val out = response.getWriter(); + out.write(processorReturned); + } catch (IOException e) { + log.error(e.getMessage()); + } + } + } + } else { + // we return error otherwise + sendError(request.getRequestURI(), response); + } + } + + /** + * Creates servlet to serve models + * + * @param type of Input class + * @param type of Output class + * + * @author raver119@gmail.com + * @author astoyakin + */ + public static class Builder { + + private ParallelInference pi; + private Model model; + + private InferenceAdapter inferenceAdapter; + private JsonSerializer serializer; + private JsonDeserializer deserializer; + private BinarySerializer binarySerializer; + private BinaryDeserializer binaryDeserializer; + private int port; + private boolean parallelEnabled = true; + + public Builder(@NonNull ParallelInference pi) { + this.pi = pi; + } + + public Builder(@NonNull Model model) { + this.model = model; + } + + public Builder inferenceAdapter(@NonNull InferenceAdapter inferenceAdapter) { + this.inferenceAdapter = inferenceAdapter; + return this; + } + + /** + * This method is required to specify serializer + * + * @param serializer + * @return + */ + public Builder serializer(JsonSerializer serializer) { + this.serializer = serializer; + return this; + } + + /** + * This method allows to specify deserializer + * + * @param deserializer + * @return + */ + public Builder deserializer(JsonDeserializer deserializer) { + this.deserializer = deserializer; + return this; + } + + /** + * This method is required to specify serializer + * + * @param serializer + * @return + */ + public Builder binarySerializer(BinarySerializer serializer) { + this.binarySerializer = serializer; + return this; + } + + /** + * This method allows to specify deserializer + * + * @param deserializer + * @return + */ + public Builder binaryDeserializer(BinaryDeserializer deserializer) { + this.binaryDeserializer = deserializer; + return this; + } + + /** + * This method allows to specify port + * + * @param port + * @return + */ + public Builder port(int port) { + this.port = port; + return this; + } + + /** + * This method activates parallel inference + * + * @param parallelEnabled + * @return + */ + public Builder parallelEnabled(boolean parallelEnabled) { + this.parallelEnabled = parallelEnabled; + return this; + } + + public DL4jServlet build() { + return parallelEnabled ? new DL4jServlet(pi, inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer) : + new DL4jServlet(model, inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer); + } + } +} + + + + diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/main/java/org/deeplearning4j/remote/JsonModelServer.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/main/java/org/deeplearning4j/remote/JsonModelServer.java new file mode 100644 index 000000000..1f0b93508 --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/main/java/org/deeplearning4j/remote/JsonModelServer.java @@ -0,0 +1,449 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.remote; + +import lombok.NonNull; +import lombok.val; +import org.deeplearning4j.nn.api.Model; +import org.deeplearning4j.nn.api.ModelAdapter; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.parallelism.ParallelInference; +import org.deeplearning4j.parallelism.inference.InferenceMode; +import org.deeplearning4j.parallelism.inference.LoadBalanceMode; +import org.nd4j.adapters.InferenceAdapter; +import org.nd4j.adapters.InputAdapter; +import org.nd4j.adapters.OutputAdapter; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.common.base.Preconditions; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.remote.SameDiffJsonModelServer; +import org.nd4j.remote.clients.serde.BinaryDeserializer; +import org.nd4j.remote.clients.serde.BinarySerializer; +import org.nd4j.remote.clients.serde.JsonDeserializer; +import org.nd4j.remote.clients.serde.JsonSerializer; + + +import java.util.List; + +/** + * This class provides JSON-based model serving ability for Deeplearning4j/SameDiff models + * + * Server url will be http://0.0.0.0:{port}>/v1/serving + * Server only accepts POST requests + * + * @param type of the input class, i.e. String + * @param type of the output class, i.e. Sentiment + * + * @author raver119@gmail.com + * @author astoyakin + */ +public class JsonModelServer extends SameDiffJsonModelServer { + + // all serving goes through ParallelInference + protected ParallelInference parallelInference; + + + protected ModelAdapter modelAdapter; + + // actual models + protected ComputationGraph cgModel; + protected MultiLayerNetwork mlnModel; + + // service stuff + protected InferenceMode inferenceMode; + protected int numWorkers; + + protected boolean enabledParallel = true; + + protected JsonModelServer(@NonNull SameDiff sdModel, InferenceAdapter inferenceAdapter, + JsonSerializer serializer, JsonDeserializer deserializer, + BinarySerializer binarySerializer, BinaryDeserializer binaryDeserializer, + int port, String[] orderedInputNodes, String[] orderedOutputNodes) { + super(sdModel, inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer, port, orderedInputNodes, orderedOutputNodes); + } + + protected JsonModelServer(@NonNull ComputationGraph cgModel, InferenceAdapter inferenceAdapter, + JsonSerializer serializer, JsonDeserializer deserializer, + BinarySerializer binarySerializer, BinaryDeserializer binaryDeserializer, + int port, @NonNull InferenceMode inferenceMode, int numWorkers) { + super(inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer, port); + + this.cgModel = cgModel; + this.inferenceMode = inferenceMode; + this.numWorkers = numWorkers; + } + + protected JsonModelServer(@NonNull MultiLayerNetwork mlnModel, InferenceAdapter inferenceAdapter, + JsonSerializer serializer, JsonDeserializer deserializer, + BinarySerializer binarySerializer, BinaryDeserializer binaryDeserializer, + int port, @NonNull InferenceMode inferenceMode, int numWorkers) { + super(inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer, port); + + this.mlnModel = mlnModel; + this.inferenceMode = inferenceMode; + this.numWorkers = numWorkers; + } + + protected JsonModelServer(@NonNull ParallelInference pi, InferenceAdapter inferenceAdapter, + JsonSerializer serializer, JsonDeserializer deserializer, + BinarySerializer binarySerializer, BinaryDeserializer binaryDeserializer, + int port) { + super(inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer, port); + + this.parallelInference = pi; + } + + /** + * This method stops server + * + * @throws Exception + */ + @Override + public void stop() throws Exception { + if (parallelInference != null) + parallelInference.shutdown(); + super.stop(); + } + + /** + * This method starts server + * @throws Exception + */ + @Override + public void start() throws Exception { + // if we're just serving sdModel - we'll just call super. no dl4j functionality required in this case + if (sdModel != null) { + super.start(); + return; + } + Preconditions.checkArgument(cgModel != null || mlnModel != null, "Model serving requires either MultilayerNetwork or ComputationGraph defined"); + + val model = cgModel != null ? (Model) cgModel : (Model) mlnModel; + // PI construction is optional, since we can have it defined + if (enabledParallel) { + if (parallelInference == null) { + Preconditions.checkArgument(numWorkers >= 1, "Number of workers should be >= 1, got " + numWorkers + " instead"); + + parallelInference = new ParallelInference.Builder(model) + .inferenceMode(inferenceMode) + .workers(numWorkers) + .loadBalanceMode(LoadBalanceMode.FIFO) + .batchLimit(16) + .queueLimit(128) + .build(); + } + servingServlet = new DL4jServlet.Builder(parallelInference) + .parallelEnabled(true) + .serializer(serializer) + .deserializer(deserializer) + .binarySerializer(binarySerializer) + .binaryDeserializer(binaryDeserializer) + .inferenceAdapter(inferenceAdapter) + .build(); + } + else { + servingServlet = new DL4jServlet.Builder(model) + .parallelEnabled(false) + .serializer(serializer) + .deserializer(deserializer) + .binarySerializer(binarySerializer) + .binaryDeserializer(binaryDeserializer) + .inferenceAdapter(inferenceAdapter) + .build(); + } + start(port, servingServlet); + } + + /** + * Creates servlet to serve different types of models + * + * @param type of Input class + * @param type of Output class + * + * @author raver119@gmail.com + * @author astoyakin + */ + public static class Builder { + + private SameDiff sdModel; + private ComputationGraph cgModel; + private MultiLayerNetwork mlnModel; + private ParallelInference pi; + + private String[] orderedInputNodes; + private String[] orderedOutputNodes; + + private InferenceAdapter inferenceAdapter; + private JsonSerializer serializer; + private JsonDeserializer deserializer; + private BinarySerializer binarySerializer; + private BinaryDeserializer binaryDeserializer; + + private InputAdapter inputAdapter; + private OutputAdapter outputAdapter; + + private int port; + + private boolean parallelMode = true; + + // these fields actually require defaults + private InferenceMode inferenceMode = InferenceMode.BATCHED; + private int numWorkers = Nd4j.getAffinityManager().getNumberOfDevices(); + + public Builder(@NonNull SameDiff sdModel) { + this.sdModel = sdModel; + } + + public Builder(@NonNull MultiLayerNetwork mlnModel) { + this.mlnModel = mlnModel; + } + + public Builder(@NonNull ComputationGraph cgModel) { + this.cgModel = cgModel; + } + + public Builder(@NonNull ParallelInference pi) { + this.pi = pi; + } + + /** + * This method defines InferenceAdapter implementation, which will be used to convert object of Input type to the set of INDArray(s), and for conversion of resulting INDArray(s) into object of Output type + * @param inferenceAdapter + * @return + */ + public Builder inferenceAdapter(@NonNull InferenceAdapter inferenceAdapter) { + this.inferenceAdapter = inferenceAdapter; + return this; + } + + /** + * This method allows you to specify InputAdapter to be used for inference + * + * PLEASE NOTE: This method is optional, and will require OutputAdapter defined + * @param inputAdapter + * @return + */ + public Builder inputAdapter(@NonNull InputAdapter inputAdapter) { + this.inputAdapter = inputAdapter; + return this; + } + + /** + * This method allows you to specify OutputtAdapter to be used for inference + * + * PLEASE NOTE: This method is optional, and will require InputAdapter defined + * @param outputAdapter + * @return + */ + public Builder outputAdapter(@NonNull OutputAdapter outputAdapter) { + this.outputAdapter = outputAdapter; + return this; + } + + /** + * This method allows you to specify JSON serializer. + * Incompatible with {@link #outputBinarySerializer(BinarySerializer)} + * Only one serializer - deserializer pair can be used by client and server. + * + * @param serializer + * @return + */ + public Builder outputSerializer(@NonNull JsonSerializer serializer) { + this.serializer = serializer; + return this; + } + + /** + * This method allows you to specify JSON deserializer. + * Incompatible with {@link #inputBinaryDeserializer(BinaryDeserializer)} + * Only one serializer - deserializer pair can be used by client and server. + * + * @param deserializer + * @return + */ + public Builder inputDeserializer(@NonNull JsonDeserializer deserializer) { + this.deserializer = deserializer; + return this; + } + + /** + * This method allows you to specify binary serializer. + * Incompatible with {@link #outputSerializer(JsonSerializer)} + * Only one serializer - deserializer pair can be used by client and server. + * + * @param serializer + * @return + */ + public Builder outputBinarySerializer(@NonNull BinarySerializer serializer) { + this.binarySerializer = serializer; + return this; + } + + /** + * This method allows you to specify binary deserializer + * Incompatible with {@link #inputDeserializer(JsonDeserializer)} + * Only one serializer - deserializer pair can be used by client and server. + * + * @param deserializer + * @return + */ + public Builder inputBinaryDeserializer(@NonNull BinaryDeserializer deserializer) { + this.binaryDeserializer = deserializer; + return this; + } + + /** + * This method allows you to specify inference mode for parallel mode. See {@link InferenceMode} for more details + * + * @param inferenceMode + * @return + */ + public Builder inferenceMode(@NonNull InferenceMode inferenceMode) { + this.inferenceMode = inferenceMode; + return this; + } + + /** + * This method allows you to specify number of worker threads for ParallelInference + * + * @param numWorkers + * @return + */ + public Builder numWorkers(int numWorkers) { + this.numWorkers = numWorkers; + return this; + } + + /** + * This method allows you to specify the order in which the inputs should be mapped to the model placeholder arrays. This is only required for {@link SameDiff} models, not {@link MultiLayerNetwork} or {@link ComputationGraph} models + * + * PLEASE NOTE: this argument only used for SameDiff models + * @param args + * @return + */ + public Builder orderedInputNodes(String... args) { + orderedInputNodes = args; + return this; + } + + /** + * This method allows you to specify the order in which the inputs should be mapped to the model placeholder arrays. This is only required for {@link SameDiff} models, not {@link MultiLayerNetwork} or {@link ComputationGraph} models + * + * PLEASE NOTE: this argument only used for SameDiff models + * @param args + * @return + */ + public Builder orderedInputNodes(@NonNull List args) { + orderedInputNodes = args.toArray(new String[args.size()]); + return this; + } + + /** + * This method allows you to specify output nodes + * + * PLEASE NOTE: this argument only used for SameDiff models + * @param args + * @return + */ + public Builder orderedOutputNodes(String... args) { + Preconditions.checkArgument(args != null && args.length > 0, "OutputNodes should contain at least 1 element"); + orderedOutputNodes = args; + return this; + } + + /** + * This method allows you to specify output nodes + * + * PLEASE NOTE: this argument only used for SameDiff models + * @param args + * @return + */ + public Builder orderedOutputNodes(@NonNull List args) { + Preconditions.checkArgument(args.size() > 0, "OutputNodes should contain at least 1 element"); + orderedOutputNodes = args.toArray(new String[args.size()]); + return this; + } + + /** + * This method allows you to specify http port + * + * PLEASE NOTE: port must be free and be in range regular TCP/IP ports range + * @param port + * @return + */ + public Builder port(int port) { + this.port = port; + return this; + } + + /** + * This method switches on ParallelInference usage + * @param - true - to use ParallelInference, false - to use ComputationGraph or + * MultiLayerNetwork directly + * + * PLEASE NOTE: this doesn't apply to SameDiff models + * + * @throws Exception + */ + public Builder parallelMode(boolean enable) { + this.parallelMode = enable; + return this; + } + + public JsonModelServer build() { + if (inferenceAdapter == null) { + if (inputAdapter != null && outputAdapter != null) { + inferenceAdapter = new InferenceAdapter() { + @Override + public MultiDataSet apply(I input) { + return inputAdapter.apply(input); + } + + @Override + public O apply(INDArray... outputs) { + return outputAdapter.apply(outputs); + } + }; + } else + throw new IllegalArgumentException("Either InferenceAdapter or InputAdapter + OutputAdapter should be configured"); + } + + JsonModelServer server = null; + if (sdModel != null) { + server = new JsonModelServer(sdModel, inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer, port, orderedInputNodes, orderedOutputNodes); + } + else if (cgModel != null) { + server = new JsonModelServer(cgModel, inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer, port, inferenceMode, numWorkers); + } + else if (mlnModel != null) { + server = new JsonModelServer(mlnModel, inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer, port, inferenceMode, numWorkers); + } + else if (pi != null) { + server = new JsonModelServer(pi, inferenceAdapter, serializer, deserializer, binarySerializer, binaryDeserializer, port); + } + else + throw new IllegalStateException("No models were defined for JsonModelServer"); + + server.enabledParallel = parallelMode; + return server; + } + } + +} diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/AssertTestsExtendBaseClass.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/AssertTestsExtendBaseClass.java new file mode 100644 index 000000000..6786a8249 --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/AssertTestsExtendBaseClass.java @@ -0,0 +1,48 @@ +/* ****************************************************************************** + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.deeplearning4j.remote; + +import lombok.extern.slf4j.Slf4j; +import java.util.*; + +import org.deeplearning4j.BaseDL4JTest; +import org.nd4j.common.tests.AbstractAssertTestsClass; + +/** + * This class checks that all test classes (i.e., anything with one or more methods annotated with @Test) + * extends BaseDl4JTest - either directly or indirectly. + * Other than a small set of exceptions, all tests must extend this + * + * @author Alexander Stoyakin + */ +@Slf4j +public class AssertTestsExtendBaseClass extends AbstractAssertTestsClass { + + @Override + protected Set> getExclusions() { + Set> exclusions = new HashSet<>(); + return exclusions; + } + + @Override + protected String getPackageName() { + return "org.deeplearning4j.remote"; + } + + @Override + protected Class getBaseClass() { return BaseDL4JTest.class; } +} + diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/BinaryModelServerTest.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/BinaryModelServerTest.java new file mode 100644 index 000000000..94a42e681 --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/BinaryModelServerTest.java @@ -0,0 +1,276 @@ +package org.deeplearning4j.remote; + +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.datavec.image.loader.Java2DNativeImageLoader; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.remote.helpers.ImageConversionUtils; +import org.deeplearning4j.util.ModelSerializer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.nd4j.adapters.InferenceAdapter; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.common.io.ClassPathResource; +import org.nd4j.remote.clients.JsonRemoteInference; +import org.nd4j.remote.clients.serde.BinaryDeserializer; +import org.nd4j.remote.clients.serde.BinarySerializer; +import org.nd4j.remote.clients.serde.impl.IntegerSerde; +import org.nd4j.common.resources.Resources; +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.*; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.deeplearning4j.parallelism.inference.InferenceMode.SEQUENTIAL; +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +public class BinaryModelServerTest extends BaseDL4JTest { + private final int PORT = 18080; + + @AfterEach + public void pause() throws Exception { + // TODO: the same port was used in previous test and not accessible immediately. Might be better solution. + TimeUnit.SECONDS.sleep(2); + } + + // Internal test for locally defined serializers + @Test + public void testBufferedImageSerde() { + BinarySerializer serde = new BinaryModelServerTest.BufferedImageSerde(); + BufferedImage image = ImageConversionUtils.makeRandomBufferedImage(28,28,1); + byte[] serialized = serde.serialize(image); + + BufferedImage deserialized = ((BufferedImageSerde) serde).deserialize(serialized); + int originalSize = image.getData().getDataBuffer().getSize(); + assertEquals(originalSize, deserialized.getData().getDataBuffer().getSize()); + for (int i = 0; i < originalSize; ++i) { + assertEquals(deserialized.getData().getDataBuffer().getElem(i), + image.getData().getDataBuffer().getElem(i)); + } + } + + @Test + public void testImageToINDArray() { + INDArray data = ImageConversionUtils.makeRandomImageAsINDArray(28,28,1); + assertNotNull(data); + } + + @Test + public void testMlnMnist_ImageInput() throws Exception { + + val modelFile = Resources.asFile("models/mnist/mnist-model.zip"); + MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(modelFile); + + val server = new JsonModelServer.Builder(net) + .outputSerializer(new IntegerSerde()) + .inputBinaryDeserializer(new BufferedImageSerde()) + .inferenceAdapter(new InferenceAdapter() { + @Override + public MultiDataSet apply(BufferedImage input) { + INDArray data = null; + try { + data = new Java2DNativeImageLoader().asMatrix(input); + data = data.reshape(1, 784); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return new MultiDataSet(data, null); + } + + @Override + public Integer apply(INDArray... nnOutput) { + return nnOutput[0].argMax().getInt(0); + } + }) + .port(PORT) + .inferenceMode(SEQUENTIAL) + .numWorkers(1) + .parallelMode(false) + .build(); + + val client = JsonRemoteInference.builder() + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .inputBinarySerializer(new BufferedImageSerde()) + .outputDeserializer(new IntegerSerde()) + .build(); + + try { + server.start(); + BufferedImage image = ImageConversionUtils.makeRandomBufferedImage(28,28,1); + Integer result = client.predict(image); + assertNotNull(result); + + File file = new ClassPathResource("datavec-local/imagetest/0/b.bmp").getFile(); + image = ImageIO.read(new FileInputStream(file)); + result = client.predict(image); + assertEquals(new Integer(0), result); + + file = new ClassPathResource("datavec-local/imagetest/1/a.bmp").getFile(); + image = ImageIO.read(new FileInputStream(file)); + result = client.predict(image); + assertEquals(new Integer(1), result); + + } catch (Exception e){ + log.error("",e); + throw e; + } finally { + server.stop(); + } + } + + @Test + public void testMlnMnist_ImageInput_Async() throws Exception { + + val modelFile = Resources.asFile("models/mnist/mnist-model.zip"); + MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(modelFile); + + val server = new JsonModelServer.Builder(net) + .outputSerializer(new IntegerSerde()) + .inputBinaryDeserializer(new BufferedImageSerde()) + .inferenceAdapter(new InferenceAdapter() { + @Override + public MultiDataSet apply(BufferedImage input) { + INDArray data = null; + try { + data = new Java2DNativeImageLoader().asMatrix(input); + data = data.reshape(1, 784); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return new MultiDataSet(data, null); + } + + @Override + public Integer apply(INDArray... nnOutput) { + return nnOutput[0].argMax().getInt(0); + } + }) + .port(PORT) + .inferenceMode(SEQUENTIAL) + .numWorkers(1) + .parallelMode(false) + .build(); + + val client = JsonRemoteInference.builder() + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .inputBinarySerializer(new BufferedImageSerde()) + .outputDeserializer(new IntegerSerde()) + .build(); + + try { + server.start(); + BufferedImage[] images = new BufferedImage[3]; + images[0] = ImageConversionUtils.makeRandomBufferedImage(28,28,1); + + File file = new ClassPathResource("datavec-local/imagetest/0/b.bmp").getFile(); + images[1] = ImageIO.read(new FileInputStream(file)); + + file = new ClassPathResource("datavec-local/imagetest/1/a.bmp").getFile(); + images[2] = ImageIO.read(new FileInputStream(file)); + + Future[] results = new Future[3]; + for (int i = 0; i < images.length; ++i) { + results[i] = client.predictAsync(images[i]); + assertNotNull(results[i]); + } + + assertNotNull(results[0].get()); + assertEquals(new Integer(0), results[1].get()); + assertEquals(new Integer(1), results[2].get()); + + } catch (Exception e){ + log.error("",e); + throw e; + } finally { + server.stop(); + } + } + + @Test + public void testBinaryIn_BinaryOut() throws Exception { + + val modelFile = Resources.asFile("models/mnist/mnist-model.zip"); + MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(modelFile); + + val server = new JsonModelServer.Builder(net) + .outputBinarySerializer(new BufferedImageSerde()) + .inputBinaryDeserializer(new BufferedImageSerde()) + .inferenceAdapter(new InferenceAdapter() { + @Override + public MultiDataSet apply(BufferedImage input) { + INDArray data = null; + try { + data = new Java2DNativeImageLoader().asMatrix(input); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return new MultiDataSet(data, null); + } + + @Override + public BufferedImage apply(INDArray... nnOutput) { + return ImageConversionUtils.makeRandomBufferedImage(28,28,3); + } + }) + .port(PORT) + .inferenceMode(SEQUENTIAL) + .numWorkers(1) + .parallelMode(false) + .build(); + + val client = JsonRemoteInference.builder() + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .inputBinarySerializer(new BufferedImageSerde()) + .outputBinaryDeserializer(new BufferedImageSerde()) + .build(); + + try { + server.start(); + BufferedImage image = ImageConversionUtils.makeRandomBufferedImage(28,28,1); + BufferedImage result = client.predict(image); + assertNotNull(result); + assertEquals(28, result.getHeight()); + assertEquals(28, result.getWidth()); + + } catch (Exception e){ + log.error("",e); + throw e; + } finally { + server.stop(); + } + } + + private static class BufferedImageSerde implements BinarySerializer, BinaryDeserializer { + + @Override + public BufferedImage deserialize(byte[] buffer) { + try { + BufferedImage img = ImageIO.read(new ByteArrayInputStream(buffer)); + return img; + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + @Override + public byte[] serialize(BufferedImage image) { + try{ + val baos = new ByteArrayOutputStream(); + ImageIO.write(image, "bmp", baos); + byte[] bytes = baos.toByteArray(); + return bytes; + } catch (IOException e){ + throw new RuntimeException(e); + } + } + } +} diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/JsonModelServerTest.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/JsonModelServerTest.java new file mode 100644 index 000000000..1de161c2e --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/JsonModelServerTest.java @@ -0,0 +1,761 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.remote; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.deeplearning4j.BaseDL4JTest; +import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; +import org.deeplearning4j.nn.conf.MultiLayerConfiguration; +import org.deeplearning4j.nn.conf.NeuralNetConfiguration; +import org.deeplearning4j.nn.conf.graph.MergeVertex; +import org.deeplearning4j.nn.conf.layers.*; +import org.deeplearning4j.nn.graph.ComputationGraph; +import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; +import org.deeplearning4j.nn.weights.WeightInit; +import org.deeplearning4j.parallelism.inference.InferenceMode; +import org.deeplearning4j.remote.helpers.House; +import org.deeplearning4j.remote.helpers.HouseToPredictedPriceAdapter; +import org.deeplearning4j.remote.helpers.PredictedPrice; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.adapters.InferenceAdapter; +import org.nd4j.autodiff.samediff.SDVariable; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.linalg.activations.Activation; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.learning.config.Adam; +import org.nd4j.linalg.learning.config.Sgd; +import org.nd4j.linalg.lossfunctions.LossFunctions; +import org.nd4j.remote.clients.JsonRemoteInference; +import org.nd4j.remote.clients.serde.JsonDeserializer; +import org.nd4j.remote.clients.serde.JsonSerializer; +import com.fasterxml.jackson.databind.ObjectMapper; + + +import java.io.IOException; +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.deeplearning4j.parallelism.inference.InferenceMode.INPLACE; +import static org.deeplearning4j.parallelism.inference.InferenceMode.SEQUENTIAL; +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +public class JsonModelServerTest extends BaseDL4JTest { + private static final MultiLayerNetwork model; + + static { + val conf = new NeuralNetConfiguration.Builder() + .seed(119) + .updater(new Adam(0.119f)) + .weightInit(WeightInit.XAVIER) + .list() + .layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(4).nOut(10).build()) + .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS).activation(Activation.SIGMOID).nIn(10).nOut(1).build()) + .build(); + + model = new MultiLayerNetwork(conf); + model.init(); + } + + @AfterEach + public void pause() throws Exception { + // Need to wait for server shutdown; without sleep, tests will fail if starting immediately after shutdown + TimeUnit.SECONDS.sleep(2); + } + + private AtomicInteger portCount = new AtomicInteger(18080); + private int PORT; + + @BeforeEach + public void setPort(){ + PORT = portCount.getAndIncrement(); + } + + + @Test + public void testStartStopParallel() throws Exception { + val sd = SameDiff.create(); + val sdVariable = sd.placeHolder("input", DataType.INT, 1,4); + val result = sdVariable.add(1.0); + val total = result.mean("total", Integer.MAX_VALUE); + + val serverDL = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .numWorkers(1) + .inferenceMode(SEQUENTIAL) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .port(PORT) + .build(); + + val serverSD = new JsonModelServer.Builder(sd) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .orderedInputNodes(new String[]{"input"}) + .orderedOutputNodes(new String[]{"total"}) + .port(PORT+1) + .build(); + try { + serverDL.start(); + serverSD.start(); + + val clientDL = JsonRemoteInference.builder() + .inputSerializer(new House.HouseSerializer()) + .outputDeserializer(new PredictedPrice.PredictedPriceDeserializer()) + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .build(); + + int district = 2; + House house = House.builder().area(100).bathrooms(2).bedrooms(3).district(district).build(); + PredictedPrice price = clientDL.predict(house); + long timeStart = System.currentTimeMillis(); + price = clientDL.predict(house); + long timeStop = System.currentTimeMillis(); + log.info("Time spent: {} ms", timeStop - timeStart); + assertNotNull(price); + assertEquals((float) 0.421444, price.getPrice(), 1e-5); + + val clientSD = JsonRemoteInference.builder() + .inputSerializer(new House.HouseSerializer()) + .outputDeserializer(new PredictedPrice.PredictedPriceDeserializer()) + .endpointAddress("http://localhost:" + (PORT+1) + "/v1/serving") + .build(); + + PredictedPrice price2 = clientSD.predict(house); + timeStart = System.currentTimeMillis(); + price = clientSD.predict(house); + timeStop = System.currentTimeMillis(); + log.info("Time spent: {} ms", timeStop - timeStart); + assertNotNull(price); + assertEquals((float) 3.0, price.getPrice(), 1e-5); + + } + finally { + serverSD.stop(); + serverDL.stop(); + } + } + + @Test + public void testStartStopSequential() throws Exception { + val sd = SameDiff.create(); + val sdVariable = sd.placeHolder("input", DataType.INT, 1,4); + val result = sdVariable.add(1.0); + val total = result.mean("total", Integer.MAX_VALUE); + + val serverDL = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .numWorkers(1) + .inferenceMode(SEQUENTIAL) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .port(PORT) + .build(); + + val serverSD = new JsonModelServer.Builder(sd) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .orderedInputNodes(new String[]{"input"}) + .orderedOutputNodes(new String[]{"total"}) + .port(PORT+1) + .build(); + + serverDL.start(); + serverDL.stop(); + + serverSD.start(); + serverSD.stop(); + } + + @Test + public void basicServingTestForSD() throws Exception { + val sd = SameDiff.create(); + val sdVariable = sd.placeHolder("input", DataType.INT, 1,4); + val result = sdVariable.add(1.0); + val total = result.mean("total", Integer.MAX_VALUE); + + val server = new JsonModelServer.Builder(sd) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .orderedInputNodes(new String[]{"input"}) + .orderedOutputNodes(new String[]{"total"}) + .port(PORT) + .build(); + + try { + server.start(); + + val client = JsonRemoteInference.builder() + .inputSerializer(new House.HouseSerializer()) + .outputDeserializer(new PredictedPrice.PredictedPriceDeserializer()) + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .build(); + + int district = 2; + House house = House.builder().area(100).bathrooms(2).bedrooms(3).district(district).build(); + + // warmup + PredictedPrice price = client.predict(house); + + val timeStart = System.currentTimeMillis(); + price = client.predict(house); + val timeStop = System.currentTimeMillis(); + + log.info("Time spent: {} ms", timeStop - timeStart); + + assertNotNull(price); + assertEquals((float) district + 1.0f, price.getPrice(), 1e-5); + } + finally { + server.stop(); + } + } + + @Test + public void basicServingTestForDLSynchronized() throws Exception { + val server = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .numWorkers(1) + .inferenceMode(INPLACE) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .port(PORT) + .build(); + + try { + server.start(); + + val client = JsonRemoteInference.builder() + .inputSerializer(new House.HouseSerializer()) + .outputDeserializer(new PredictedPrice.PredictedPriceDeserializer()) + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .build(); + + int district = 2; + House house1 = House.builder().area(100).bathrooms(2).bedrooms(3).district(district).build(); + House house2 = House.builder().area(50).bathrooms(1).bedrooms(2).district(district).build(); + House house3 = House.builder().area(80).bathrooms(1).bedrooms(3).district(district).build(); + + // warmup + PredictedPrice price = client.predict(house1); + + val timeStart = System.currentTimeMillis(); + PredictedPrice price1 = client.predict(house1); + PredictedPrice price2 = client.predict(house2); + PredictedPrice price3 = client.predict(house3); + val timeStop = System.currentTimeMillis(); + + log.info("Time spent: {} ms", timeStop - timeStart); + + assertNotNull(price); + assertEquals((float) 0.421444, price.getPrice(), 1e-5); + + } finally { + server.stop(); + } + } + + @Test + public void basicServingTestForDL() throws Exception { + + val server = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .numWorkers(1) + .inferenceMode(SEQUENTIAL) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .port(PORT) + .parallelMode(false) + .build(); + + try { + server.start(); + + val client = JsonRemoteInference.builder() + .inputSerializer(new House.HouseSerializer()) + .outputDeserializer(new PredictedPrice.PredictedPriceDeserializer()) + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .build(); + + int district = 2; + House house = House.builder().area(100).bathrooms(2).bedrooms(3).district(district).build(); + + // warmup + PredictedPrice price = client.predict(house); + + val timeStart = System.currentTimeMillis(); + price = client.predict(house); + val timeStop = System.currentTimeMillis(); + + log.info("Time spent: {} ms", timeStop - timeStart); + + assertNotNull(price); + assertEquals((float) 0.421444, price.getPrice(), 1e-5); + + } finally { + server.stop(); + } + } + + @Test + public void testDeserialization_1() { + String request = "{\"bedrooms\":3,\"area\":100,\"district\":2,\"bathrooms\":2}"; + val deserializer = new House.HouseDeserializer(); + val result = deserializer.deserialize(request); + assertEquals(2, result.getDistrict()); + assertEquals(100, result.getArea()); + assertEquals(2, result.getBathrooms()); + assertEquals(3, result.getBedrooms()); + + } + + @Test + public void testDeserialization_2() { + String request = "{\"price\":1}"; + val deserializer = new PredictedPrice.PredictedPriceDeserializer(); + val result = deserializer.deserialize(request); + assertEquals(1.0, result.getPrice(), 1e-4); + } + + @Test + public void negativeServingTest_1() throws Exception { + assertThrows(NullPointerException.class, () -> { + val server = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(null) + .port(PORT) + .build(); + }); + } + + @Test + public void negativeServingTest_2() throws Exception { + assertThrows(NullPointerException.class, () -> { + val server = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .port(PORT) + .build(); + }); + } + + @Test + public void negativeServingTest_3() throws Exception { + assertThrows(IOException.class, () -> { + val server = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .inferenceMode(SEQUENTIAL) + .numWorkers(1) + .port(PORT) + .build(); + + try { + server.start(); + + val client = JsonRemoteInference.builder() + .inputSerializer(new House.HouseSerializer()) + .outputDeserializer(new JsonDeserializer() { + @Override + public PredictedPrice deserialize(String json) { + return null; + } + }) + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .build(); + + int district = 2; + House house = House.builder().area(100).bathrooms(2).bedrooms(3).district(district).build(); + + // warmup + PredictedPrice price = client.predict(house); + } finally { + server.stop(); + } + }); + } + + @Test + public void asyncServingTest() throws Exception { + + val server = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .inferenceMode(SEQUENTIAL) + .numWorkers(1) + .port(PORT) + .build(); + + try { + server.start(); + + val client = JsonRemoteInference.builder() + .inputSerializer(new House.HouseSerializer()) + .outputDeserializer(new PredictedPrice.PredictedPriceDeserializer()) + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .build(); + + int district = 2; + House house = House.builder().area(100).bathrooms(2).bedrooms(3).district(district).build(); + + val timeStart = System.currentTimeMillis(); + Future price = client.predictAsync(house); + assertNotNull(price); + assertEquals((float) 0.421444, price.get().getPrice(), 1e-5); + val timeStop = System.currentTimeMillis(); + + log.info("Time spent: {} ms", timeStop - timeStart); + } + finally { + server.stop(); + } + } + + @Test + public void negativeAsyncTest() throws Exception { + + val server = new JsonModelServer.Builder(model) + .outputSerializer(new PredictedPrice.PredictedPriceSerializer()) + .inputDeserializer(new House.HouseDeserializer()) + .inferenceAdapter(new HouseToPredictedPriceAdapter()) + .inferenceMode(InferenceMode.BATCHED) + .numWorkers(1) + .port(PORT) + .build(); + + try { + server.start(); + + // Fake deserializer to test failure + val client = JsonRemoteInference.builder() + .inputSerializer(new House.HouseSerializer()) + .outputDeserializer(new JsonDeserializer() { + @Override + public PredictedPrice deserialize(String json) { + return null; + } + }) + .endpointAddress("http://localhost:" + PORT + "/v1/serving") + .build(); + + int district = 2; + House house = House.builder().area(100).bathrooms(2).bedrooms(3).district(district).build(); + + val timeStart = System.currentTimeMillis(); + try { + Future price = client.predictAsync(house); + assertNotNull(price); + assertEquals((float) district + 1.0f, price.get().getPrice(), 1e-5); + val timeStop = System.currentTimeMillis(); + + log.info("Time spent: {} ms", timeStop - timeStart); + } catch (ExecutionException e) { + assertTrue(e.getMessage().contains("Deserialization failed")); + } + } finally { + server.stop(); + } + } + + + @Test + public void testSameDiffMnist() throws Exception { + + SameDiff sd = SameDiff.create(); + SDVariable in = sd.placeHolder("in", DataType.FLOAT, -1, 28*28); + SDVariable w = sd.var("w", Nd4j.rand(DataType.FLOAT, 28*28, 10)); + SDVariable b = sd.var("b", Nd4j.rand(DataType.FLOAT, 1, 10)); + SDVariable sm = sd.nn.softmax("softmax", in.mmul(w).add(b), -1); + + val server = new JsonModelServer.Builder(sd) + .outputSerializer( new IntSerde()) + .inputDeserializer(new FloatSerde()) + .inferenceAdapter(new InferenceAdapter() { + @Override + public MultiDataSet apply(float[] input) { + return new MultiDataSet(Nd4j.create(input, 1, input.length), null); + } + + @Override + public Integer apply(INDArray... nnOutput) { + return nnOutput[0].argMax().getInt(0); + } + }) + .orderedInputNodes("in") + .orderedOutputNodes("softmax") + .port(PORT+1) + .build(); + + val client = JsonRemoteInference.builder() + .endpointAddress("http://localhost:" + (PORT+1) + "/v1/serving") + .outputDeserializer(new IntSerde()) + .inputSerializer( new FloatSerde()) + .build(); + + try{ + server.start(); + for( int i=0; i<10; i++ ){ + INDArray f = Nd4j.rand(DataType.FLOAT, 1, 28*28); + INDArray exp = sd.output(Collections.singletonMap("in", f), "softmax").get("softmax"); + float[] fArr = f.toFloatVector(); + int out = client.predict(fArr); + assertEquals(exp.argMax().getInt(0), out); + } + } finally { + server.stop(); + } + } + + @Test + public void testMlnMnist() throws Exception { + + MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .list() + .layer(new DenseLayer.Builder().nIn(784).nOut(10).build()) + .layer(new LossLayer.Builder().activation(Activation.SOFTMAX).build()) + .build(); + + MultiLayerNetwork net = new MultiLayerNetwork(conf); + net.init(); + + val server = new JsonModelServer.Builder(net) + .outputSerializer( new IntSerde()) + .inputDeserializer(new FloatSerde()) + .inferenceAdapter(new InferenceAdapter() { + @Override + public MultiDataSet apply(float[] input) { + return new MultiDataSet(Nd4j.create(input, 1, input.length), null); + } + + @Override + public Integer apply(INDArray... nnOutput) { + return nnOutput[0].argMax().getInt(0); + } + }) + .orderedInputNodes("in") + .orderedOutputNodes("softmax") + .port(PORT + 1) + .inferenceMode(SEQUENTIAL) + .numWorkers(2) + .build(); + + val client = JsonRemoteInference.builder() + .endpointAddress("http://localhost:" + (PORT + 1) + "/v1/serving") + .outputDeserializer(new IntSerde()) + .inputSerializer( new FloatSerde()) + .build(); + + try { + server.start(); + for (int i = 0; i < 10; i++) { + INDArray f = Nd4j.rand(DataType.FLOAT, 1, 28 * 28); + INDArray exp = net.output(f); + float[] fArr = f.toFloatVector(); + int out = client.predict(fArr); + assertEquals(exp.argMax().getInt(0), out); + } + } catch (Exception e){ + log.error("",e); + throw e; + } finally { + server.stop(); + } + } + + @Test + public void testCompGraph() throws Exception { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .graphBuilder() + .addInputs("input1", "input2") + .addLayer("L1", new DenseLayer.Builder().nIn(3).nOut(4).build(), "input1") + .addLayer("L2", new DenseLayer.Builder().nIn(3).nOut(4).build(), "input2") + .addVertex("merge", new MergeVertex(), "L1", "L2") + .addLayer("out", new OutputLayer.Builder().nIn(4+4).nOut(3).build(), "merge") + .setOutputs("out") + .build(); + + ComputationGraph net = new ComputationGraph(conf); + net.init(); + + val server = new JsonModelServer.Builder(net) + .outputSerializer( new IntSerde()) + .inputDeserializer(new FloatSerde()) + .inferenceAdapter(new InferenceAdapter() { + @Override + public MultiDataSet apply(float[] input) { + return new MultiDataSet(Nd4j.create(input, 1, input.length), null); + } + + @Override + public Integer apply(INDArray... nnOutput) { + return nnOutput[0].argMax().getInt(0); + } + }) + .orderedInputNodes("in") + .orderedOutputNodes("softmax") + .port(PORT + 1) + .inferenceMode(SEQUENTIAL) + .numWorkers(2) + .parallelMode(false) + .build(); + + val client = JsonRemoteInference.builder() + .endpointAddress("http://localhost:" + (PORT + 1) + "/v1/serving") + .outputDeserializer(new IntSerde()) + .inputSerializer( new FloatSerde()) + .build(); + + try { + server.start(); + //client.predict(new float[]{0.0f, 1.0f, 2.0f}); + } catch (Exception e){ + log.error("",e); + throw e; + } finally { + server.stop(); + } + } + + @Test + public void testCompGraph_1() throws Exception { + + ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() + .updater(new Sgd(0.01)) + .graphBuilder() + .addInputs("input") + .addLayer("L1", new DenseLayer.Builder().nIn(8).nOut(4).build(), "input") + .addLayer("out1", new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) + .nIn(4).nOut(3).build(), "L1") + .addLayer("out2", new OutputLayer.Builder() + .lossFunction(LossFunctions.LossFunction.MSE) + .nIn(4).nOut(2).build(), "L1") + .setOutputs("out1","out2") + .build(); + + final ComputationGraph net = new ComputationGraph(conf); + net.init(); + + val server = new JsonModelServer.Builder(net) + .outputSerializer( new IntSerde()) + .inputDeserializer(new FloatSerde()) + .inferenceAdapter(new InferenceAdapter() { + @Override + public MultiDataSet apply(float[] input) { + return new MultiDataSet(Nd4j.create(input, 1, input.length), null); + } + + @Override + public Integer apply(INDArray... nnOutput) { + return nnOutput[0].argMax().getInt(0); + } + }) + .orderedInputNodes("input") + .orderedOutputNodes("out") + .port(PORT + 1) + .inferenceMode(SEQUENTIAL) + .numWorkers(2) + .parallelMode(false) + .build(); + + val client = JsonRemoteInference.builder() + .endpointAddress("http://localhost:" + (PORT + 1) + "/v1/serving") + .outputDeserializer(new IntSerde()) + .inputSerializer( new FloatSerde()) + .build(); + + try { + server.start(); + val result = client.predict(new float[]{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}); + assertNotNull(result); + } catch (Exception e){ + log.error("",e); + throw e; + } finally { + server.stop(); + } + } + + private static class FloatSerde implements JsonSerializer, JsonDeserializer{ + private final ObjectMapper om = new ObjectMapper(); + + @Override + public float[] deserialize(String json) { + try { + return om.readValue(json, FloatHolder.class).getFloats(); + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + @Override + public String serialize(float[] o) { + try{ + return om.writeValueAsString(new FloatHolder(o)); + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + //Use float holder so Jackson does ser/de properly (no "{}" otherwise) + @AllArgsConstructor @NoArgsConstructor @Data + private static class FloatHolder { + private float[] floats; + } + } + + private static class IntSerde implements JsonSerializer, JsonDeserializer { + private final ObjectMapper om = new ObjectMapper(); + + @Override + public Integer deserialize(String json) { + try { + return om.readValue(json, Integer.class); + } catch (IOException e){ + throw new RuntimeException(e); + } + } + + @Override + public String serialize(Integer o) { + try{ + return om.writeValueAsString(o); + } catch (IOException e){ + throw new RuntimeException(e); + } + } + } +} \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/ServletTest.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/ServletTest.java new file mode 100644 index 000000000..17b289dbd --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/ServletTest.java @@ -0,0 +1,134 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.remote; + +import lombok.val; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.HttpClientBuilder; +import org.deeplearning4j.BaseDL4JTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.nd4j.adapters.InferenceAdapter; +import org.nd4j.autodiff.samediff.SameDiff; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.remote.clients.serde.JsonDeserializer; +import org.nd4j.remote.clients.serde.JsonSerializer; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ServletTest extends BaseDL4JTest { + + private JsonModelServer server; + + @BeforeEach + public void setUp() throws Exception { + val sd = SameDiff.create(); + server = new JsonModelServer.Builder(sd) + .port(8080) + .inferenceAdapter(new InferenceAdapter() { + @Override + public MultiDataSet apply(String input) { + return null; + } + + @Override + public String apply(INDArray... nnOutput) { + return null; + } + }) + .outputSerializer(new JsonSerializer() { + @Override + public String serialize(String o) { + return ""; + } + }) + .inputDeserializer(new JsonDeserializer() { + @Override + public String deserialize(String json) { + return ""; + } + }) + .orderedInputNodes("input") + .orderedOutputNodes("output") + .build(); + + server.start(); + //server.join(); + } + + @AfterEach + public void tearDown() throws Exception { + server.stop(); + } + + @Test + public void getEndpoints() throws IOException { + val request = new HttpGet( "http://localhost:8080/v1" ); + request.setHeader("Content-type", "application/json"); + + val response = HttpClientBuilder.create().build().execute( request ); + assertEquals(200, response.getStatusLine().getStatusCode()); + } + + @Test + public void testContentTypeGet() throws IOException { + val request = new HttpGet( "http://localhost:8080/v1" ); + request.setHeader("Content-type", "text/plain"); + + val response = HttpClientBuilder.create().build().execute( request ); + assertEquals(415, response.getStatusLine().getStatusCode()); + } + + @Test + public void testContentTypePost() throws Exception { + val request = new HttpPost("http://localhost:8080/v1/serving"); + request.setHeader("Content-type", "text/plain"); + val response = HttpClientBuilder.create().build().execute( request ); + assertEquals(415, response.getStatusLine().getStatusCode()); + } + + @Test + public void postForServing() throws Exception { + val request = new HttpPost("http://localhost:8080/v1/serving"); + request.setHeader("Content-type", "application/json"); + val response = HttpClientBuilder.create().build().execute( request ); + assertEquals(500, response.getStatusLine().getStatusCode()); + } + + @Test + public void testNotFoundPost() throws Exception { + val request = new HttpPost("http://localhost:8080/v1/serving/some"); + request.setHeader("Content-type", "application/json"); + val response = HttpClientBuilder.create().build().execute( request ); + assertEquals(404, response.getStatusLine().getStatusCode()); + } + + @Test + public void testNotFoundGet() throws Exception { + val requestGet = new HttpGet( "http://localhost:8080/v1/not_found" ); + requestGet.setHeader("Content-type", "application/json"); + + val responseGet = HttpClientBuilder.create().build().execute( requestGet ); + assertEquals(404, responseGet.getStatusLine().getStatusCode()); + } + +} diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/House.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/House.java new file mode 100644 index 000000000..d66c8bae5 --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/House.java @@ -0,0 +1,48 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.remote.helpers; + +import com.google.gson.Gson; +import lombok.*; +import org.nd4j.remote.clients.serde.JsonDeserializer; +import org.nd4j.remote.clients.serde.JsonSerializer; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class House { + private int district; + private int bedrooms; + private int bathrooms; + private int area; + + + public static class HouseSerializer implements JsonSerializer { + @Override + public String serialize(@NonNull House o) { + return new Gson().toJson(o); + } + } + + public static class HouseDeserializer implements JsonDeserializer { + @Override + public House deserialize(@NonNull String json) { + return new Gson().fromJson(json, House.class); + } + } +} diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/HouseToPredictedPriceAdapter.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/HouseToPredictedPriceAdapter.java new file mode 100644 index 000000000..82976a3da --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/HouseToPredictedPriceAdapter.java @@ -0,0 +1,40 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.remote.helpers; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.adapters.InferenceAdapter; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.dataset.MultiDataSet; +import org.nd4j.linalg.factory.Nd4j; + +@Slf4j +public class HouseToPredictedPriceAdapter implements InferenceAdapter { + + @Override + public MultiDataSet apply(@NonNull House input) { + // we just create vector array with shape[4] and assign it's value to the district value + return new MultiDataSet(Nd4j.create(DataType.FLOAT, 1, 4).assign(input.getDistrict()), null); + } + + @Override + public PredictedPrice apply(INDArray... nnOutput) { + return new PredictedPrice(nnOutput[0].getFloat(0)); + } +} diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/ImageConversionUtils.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/ImageConversionUtils.java new file mode 100644 index 000000000..bba9eb4a9 --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/ImageConversionUtils.java @@ -0,0 +1,82 @@ +package org.deeplearning4j.remote.helpers; + +import lombok.val; +import org.bytedeco.javacpp.indexer.UByteIndexer; +import org.bytedeco.javacv.Java2DFrameConverter; +import org.bytedeco.javacv.OpenCVFrameConverter; +import org.bytedeco.opencv.opencv_core.Mat; +import org.datavec.image.loader.Java2DNativeImageLoader; +import org.datavec.image.loader.NativeImageLoader; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.util.Random; + +import static org.bytedeco.opencv.global.opencv_core.CV_8UC; + +public class ImageConversionUtils { + + public static Mat makeRandomImage(int height, int width, int channels) { + if (height <= 0) { + + height = new Random().nextInt() % 100 + 100; + } + if (width <= 0) { + width = new Random().nextInt() % 100 + 100; + } + + Mat img = new Mat(height, width, CV_8UC(channels)); + UByteIndexer idx = img.createIndexer(); + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { + for (int k = 0; k < channels; k++) { + idx.put(i, j, k, new Random().nextInt()); + } + } + } + return img; + } + + public static BufferedImage makeRandomBufferedImage(int height, int width, int channels) { + Mat img = makeRandomImage(height, width, channels); + + OpenCVFrameConverter.ToMat c = new OpenCVFrameConverter.ToMat(); + Java2DFrameConverter c2 = new Java2DFrameConverter(); + + return c2.convert(c.convert(img)); + } + + public static INDArray convert(BufferedImage image) { + INDArray retVal = null; + try { + retVal = new Java2DNativeImageLoader(image.getHeight(), image.getWidth(), image.getRaster().getNumBands()). + asRowVector(image); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return retVal; + } + + public static INDArray convert(Mat image) { + INDArray retVal = null; + try { + new NativeImageLoader().asRowVector(image); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return retVal; + } + + public static BufferedImage convert(INDArray input) { + return new Java2DNativeImageLoader(input.rows(),input.columns()).asBufferedImage(input); + } + + public static INDArray makeRandomImageAsINDArray(int height, int width, int channels) { + val image = makeRandomBufferedImage(height, width, channels); + INDArray retVal = convert(image); + return retVal; + } +} diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/PredictedPrice.java b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/PredictedPrice.java new file mode 100644 index 000000000..c4024bb1d --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/java/org/deeplearning4j/remote/helpers/PredictedPrice.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.deeplearning4j.remote.helpers; + +import com.google.gson.Gson; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.remote.clients.serde.JsonDeserializer; +import org.nd4j.remote.clients.serde.JsonSerializer; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class PredictedPrice { + private float price; + + public static class PredictedPriceSerializer implements JsonSerializer { + @Override + public String serialize(@NonNull PredictedPrice o) { + return new Gson().toJson(o); + } + } + + public static class PredictedPriceDeserializer implements JsonDeserializer { + @Override + public PredictedPrice deserialize(@NonNull String json) { + return new Gson().fromJson(json, PredictedPrice.class); + } + } +} diff --git a/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/resources/logback.xml b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/resources/logback.xml new file mode 100644 index 000000000..cbcbed5d6 --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/deeplearning4j-json-server/src/test/resources/logback.xml @@ -0,0 +1,48 @@ + + + + + + + + logs/application.log + + %logger{15} - %message%n%xException{5} + + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-remote/pom.xml b/deeplearning4j/deeplearning4j-remote/pom.xml new file mode 100644 index 000000000..67aab8f3e --- /dev/null +++ b/deeplearning4j/deeplearning4j-remote/pom.xml @@ -0,0 +1,21 @@ + + + + 4.0.0 + pom + + + deeplearning4j-json-server + + + + net.brutex.ai + deeplearning4j-parent + 1.0.0-SNAPSHOT + + + deeplearning4j-remote + 1.0.0-SNAPSHOT + deeplearning4j-remote + diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/pom.xml b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/pom.xml deleted file mode 100644 index ed9625547..000000000 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper-parameter-server/pom.xml +++ /dev/null @@ -1,127 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-scaleout - 1.0.0-SNAPSHOT - - - deeplearning4j-scaleout-parallelwrapper-parameter-server - - deeplearning4j-scaleout-parallelwrapper-parameter-server - - - - 2.11.12 - 2.11 - 1.8 - 1.8 - 2.2.21 - - - - - io.reactivex.rxjava2 - rxjava - ${rxjava.version} - - - org.deeplearning4j - deeplearning4j-parallel-wrapper - ${project.version} - - - org.nd4j - nd4j-parameter-server-client - ${project.version} - - - org.deeplearning4j - deeplearning4j-core - ${project.version} - - - org.nd4j - nd4j-parameter-server-node_2.11 - ${nd4j.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - org.scala-lang - scala-library - ${scala.version} - - - ch.qos.logback - logback-classic - test - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - org.nd4j - nd4j-native - ${project.version} - test - - - - - nd4j-tests-cuda - - - org.nd4j - nd4j-cuda-11.0 - ${project.version} - test - - - - - diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/pom.xml b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/pom.xml deleted file mode 100644 index 09e9603c6..000000000 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/pom.xml +++ /dev/null @@ -1,108 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-scaleout - 1.0.0-SNAPSHOT - - - deeplearning4j-parallel-wrapper - - deeplearning4j-parallel-wrapper - - - 1.8 - 1.8 - - - - - com.beust - jcommander - ${jcommander.version} - - - - org.slf4j - slf4j-api - - - ch.qos.logback - logback-classic - test - - - org.nd4j - nd4j-parameter-server - ${nd4j.version} - - - org.nd4j - nd4j-parameter-server-client - ${nd4j.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - org.deeplearning4j - deeplearning4j-core - ${project.version} - - - org.deeplearning4j - deeplearning4j-ui - ${project.version} - test - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestListeners.java b/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestListeners.java deleted file mode 100644 index a5470fed4..000000000 --- a/deeplearning4j/deeplearning4j-scaleout/deeplearning4j-scaleout-parallelwrapper/src/test/java/org/deeplearning4j/parallelism/TestListeners.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.parallelism; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.core.storage.StatsStorage; -import org.deeplearning4j.core.storage.StatsStorageRouter; -import org.deeplearning4j.core.storage.listener.RoutingIterationListener; -import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator; -import org.deeplearning4j.nn.api.Model; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.optimize.api.BaseTrainingListener; -import org.deeplearning4j.optimize.api.TrainingListener; -import org.deeplearning4j.ui.model.stats.StatsListener; -import org.deeplearning4j.ui.model.storage.InMemoryStatsStorage; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) -public class TestListeners extends BaseDL4JTest { - - @Test - public void testListeners() { - TestListener.clearCounts(); - - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(0, - new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10) - .activation(Activation.TANH).build()); - - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - - testListenersForModel(model, Collections.singletonList(new TestListener())); - } - - @Test - public void testListenersGraph() { - TestListener.clearCounts(); - - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder() - .addInputs("in").addLayer("0", - new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10) - .activation(Activation.TANH).build(), - "in") - .setOutputs("0").build(); - - ComputationGraph model = new ComputationGraph(conf); - model.init(); - - testListenersForModel(model, Collections.singletonList(new TestListener())); - } - - @Test - public void testListenersViaModel() { - TestListener.clearCounts(); - - MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list().layer(0, - new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10) - .activation(Activation.TANH).build()); - - MultiLayerConfiguration conf = builder.build(); - MultiLayerNetwork model = new MultiLayerNetwork(conf); - model.init(); - - StatsStorage ss = new InMemoryStatsStorage(); - model.setListeners(new TestListener(), new StatsListener(ss)); - - testListenersForModel(model, null); - - assertEquals(1, ss.listSessionIDs().size()); - assertEquals(2, ss.listWorkerIDsForSession(ss.listSessionIDs().get(0)).size()); - } - - @Test - public void testListenersViaModelGraph() { - TestListener.clearCounts(); - - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder() - .addInputs("in").addLayer("0", - new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10) - .activation(Activation.TANH).build(), - "in") - .setOutputs("0").build(); - - ComputationGraph model = new ComputationGraph(conf); - model.init(); - - StatsStorage ss = new InMemoryStatsStorage(); - model.setListeners(new TestListener(), new StatsListener(ss)); - - testListenersForModel(model, null); - - assertEquals(1, ss.listSessionIDs().size()); - assertEquals(2, ss.listWorkerIDsForSession(ss.listSessionIDs().get(0)).size()); - } - - private static void testListenersForModel(Model model, List listeners) { - - int nWorkers = 2; - ParallelWrapper wrapper = new ParallelWrapper.Builder(model).workers(nWorkers).averagingFrequency(1) - .reportScoreAfterAveraging(true).build(); - - if (listeners != null) { - wrapper.setListeners(listeners); - } - - List data = new ArrayList<>(); - for (int i = 0; i < nWorkers; i++) { - data.add(new DataSet(Nd4j.rand(1, 10), Nd4j.rand(1, 10))); - } - - DataSetIterator iter = new ExistingDataSetIterator(data); - - TestListener.clearCounts(); - wrapper.fit(iter); - - assertEquals(2, TestListener.workerIDs.size()); - assertEquals(1, TestListener.sessionIDs.size()); - assertEquals(2, TestListener.forwardPassCount.get()); - assertEquals(2, TestListener.backwardPassCount.get()); - } - - - private static class TestListener extends BaseTrainingListener implements RoutingIterationListener { - - private static final AtomicInteger forwardPassCount = new AtomicInteger(); - private static final AtomicInteger backwardPassCount = new AtomicInteger(); - private static final AtomicInteger instanceCount = new AtomicInteger(); - private static final Set workerIDs = Collections.newSetFromMap(new ConcurrentHashMap()); - private static final Set sessionIDs = Collections.newSetFromMap(new ConcurrentHashMap()); - - public static void clearCounts() { - forwardPassCount.set(0); - backwardPassCount.set(0); - instanceCount.set(0); - workerIDs.clear(); - sessionIDs.clear(); - } - - public TestListener() { - instanceCount.incrementAndGet(); - } - - @Override - public void onEpochStart(Model model) {} - - @Override - public void onEpochEnd(Model model) {} - - @Override - public void onForwardPass(Model model, List activations) { - forwardPassCount.incrementAndGet(); - } - - @Override - public void onForwardPass(Model model, Map activations) { - forwardPassCount.incrementAndGet(); - } - - @Override - public void onGradientCalculation(Model model) {} - - @Override - public void onBackwardPass(Model model) { - backwardPassCount.getAndIncrement(); - } - - @Override - public void setStorageRouter(StatsStorageRouter router) {} - - @Override - public StatsStorageRouter getStorageRouter() { - return null; - } - - @Override - public void setWorkerID(String workerID) { - workerIDs.add(workerID); - } - - @Override - public String getWorkerID() { - return null; - } - - @Override - public void setSessionID(String sessionID) { - sessionIDs.add(sessionID); - } - - @Override - public String getSessionID() { - return "session_id"; - } - - @Override - public RoutingIterationListener clone() { - return new TestListener(); - } - - @Override - public void iterationDone(Model model, int iteration, int epoch) {} - } - -} diff --git a/deeplearning4j/deeplearning4j-scaleout/pom.xml b/deeplearning4j/deeplearning4j-scaleout/pom.xml index f0cb7bc0b..184415581 100644 --- a/deeplearning4j/deeplearning4j-scaleout/pom.xml +++ b/deeplearning4j/deeplearning4j-scaleout/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - org.deeplearning4j + net.brutex.ai deeplearning4j-parent 1.0.0-SNAPSHOT @@ -41,36 +41,4 @@ deeplearning4j-scaleout-parallelwrapper deeplearning4j-scaleout-parallelwrapper-parameter-server - - - - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - org.deeplearning4j - deeplearning4j-cuda-11.0 - ${nd4j.version} - - - - diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/pom.xml b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/pom.xml index 431ffe764..9ed470585 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/pom.xml +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/pom.xml @@ -26,12 +26,12 @@ 4.0.0 - org.deeplearning4j - spark_2.11 + net.brutex.ai + spark_2.12 1.0.0-SNAPSHOT - dl4j-spark-nlp-java8_2.11 + dl4j-spark-nlp-java8_2.12 dl4j-spark-nlp-java8 @@ -41,19 +41,19 @@ - org.deeplearning4j + net.brutex.ai deeplearning4j-nlp - ${deeplearning4j.version} - - - org.deeplearning4j - dl4j-spark_2.11 ${project.version} - org.nd4j - nd4j-parameter-server-node_2.11 - ${nd4j.version} + net.brutex.ai + dl4j-spark_2.12 + ${project.version} + + + net.brutex.ai + nd4j-parameter-server-node_2.12 + ${project.version} net.jpountz.lz4 @@ -61,38 +61,17 @@ - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - org.apache.spark - spark-core_2.11 + spark-core_2.12 ${spark.version} provided - org.deeplearning4j + net.brutex.ai deeplearning4j-common-tests ${project.version} test - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectors.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectors.java index 7769c7668..a9cc32461 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectors.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectors.java @@ -24,7 +24,7 @@ import lombok.Getter; import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.spark.Accumulator; +import org.apache.spark.util.CollectionAccumulator; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.broadcast.Broadcast; @@ -60,8 +60,8 @@ import java.util.Set; @Slf4j public class SparkSequenceVectors extends SequenceVectors { - protected Accumulator> elementsFreqAccum; - protected Accumulator> elementsFreqAccumExtra; + protected CollectionAccumulator> elementsFreqAccum; + protected CollectionAccumulator> elementsFreqAccumExtra; protected StorageLevel storageLevel = StorageLevel.MEMORY_ONLY(); @@ -198,8 +198,8 @@ public class SparkSequenceVectors extends SequenceVec if (isAutoDiscoveryMode) { log.info("Trying auto discovery mode..."); - elementsFreqAccumExtra = corpus.context().accumulator(new ExtraCounter(), - new ExtraElementsFrequenciesAccumulator()); + elementsFreqAccumExtra = corpus.context().collectionAccumulator("ExtraElementsFrequenciesAccumulator"); //.accumulator(new ExtraCounter(), + //new ExtraElementsFrequenciesAccumulator()); ExtraCountFunction elementsCounter = new ExtraCountFunction<>(elementsFreqAccumExtra, configuration.isTrainSequenceVectors()); @@ -209,7 +209,7 @@ public class SparkSequenceVectors extends SequenceVec // just to trigger map function, since we need huffman tree before proceeding numberOfSequences = countedCorpus.count(); - finalCounter = elementsFreqAccumExtra.value(); + finalCounter = elementsFreqAccumExtra.value().get(0); ExtraCounter spareReference = (ExtraCounter) finalCounter; @@ -260,7 +260,7 @@ public class SparkSequenceVectors extends SequenceVec // set up freqs accumulator - elementsFreqAccum = corpus.context().accumulator(new Counter(), new ElementsFrequenciesAccumulator()); + elementsFreqAccum = corpus.context().collectionAccumulator("ElementsFrequenciesAccumulator");//(new Counter(), new ElementsFrequenciesAccumulator()); CountFunction elementsCounter = new CountFunction<>(configurationBroadcast, paramServerConfigurationBroadcast, elementsFreqAccum, configuration.isTrainSequenceVectors()); @@ -272,7 +272,7 @@ public class SparkSequenceVectors extends SequenceVec numberOfSequences = countedCorpus.count(); // now we grab counter, which contains frequencies for all SequenceElements in corpus - finalCounter = elementsFreqAccum.value(); + finalCounter = elementsFreqAccum.value().get(0); } long numberOfElements = (long) finalCounter.totalCount(); @@ -362,9 +362,9 @@ public class SparkSequenceVectors extends SequenceVec protected Counter getCounter() { if (isAutoDiscoveryMode) - return elementsFreqAccumExtra.value(); + return elementsFreqAccumExtra.value().get(0); else - return elementsFreqAccum.value(); + return elementsFreqAccum.value().get(0); } diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/CountFunction.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/CountFunction.java index 72ed9a175..4694fae7e 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/CountFunction.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/CountFunction.java @@ -22,7 +22,7 @@ package org.deeplearning4j.spark.models.sequencevectors.functions; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.apache.spark.Accumulator; +import org.apache.spark.util.CollectionAccumulator; import org.apache.spark.api.java.function.Function; import org.apache.spark.broadcast.Broadcast; import org.deeplearning4j.common.config.DL4JClassLoading; @@ -40,7 +40,7 @@ import org.nd4j.parameterserver.distributed.transport.RoutedTransport; @Slf4j public class CountFunction implements Function, Pair, Long>> { - protected Accumulator> accumulator; + protected CollectionAccumulator> accumulator; protected boolean fetchLabels; protected Broadcast voidConfigurationBroadcast; protected Broadcast vectorsConfigurationBroadcast; @@ -50,7 +50,7 @@ public class CountFunction implements Function vectorsConfigurationBroadcast, @NonNull Broadcast voidConfigurationBroadcast, - @NonNull Accumulator> accumulator, boolean fetchLabels) { + @NonNull CollectionAccumulator> accumulator, boolean fetchLabels) { this.accumulator = accumulator; this.fetchLabels = fetchLabels; this.voidConfigurationBroadcast = voidConfigurationBroadcast; diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ElementsFrequenciesAccumulator.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ElementsFrequenciesAccumulator.java index f6411e9da..dcb411bca 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ElementsFrequenciesAccumulator.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ElementsFrequenciesAccumulator.java @@ -20,11 +20,10 @@ package org.deeplearning4j.spark.models.sequencevectors.functions; -import org.apache.spark.AccumulatorParam; +import org.apache.spark.util.CollectionAccumulator; import org.nd4j.common.primitives.Counter; -public class ElementsFrequenciesAccumulator implements AccumulatorParam> { - @Override +public class ElementsFrequenciesAccumulator extends CollectionAccumulator> { public Counter addAccumulator(Counter c1, Counter c2) { if (c1 == null) { return new Counter<>(); @@ -33,13 +32,13 @@ public class ElementsFrequenciesAccumulator implements AccumulatorParam addInPlace(Counter r1, Counter r2) { r1.incrementAll(r2); return r1; } - @Override + public Counter zero(Counter initialValue) { return new Counter<>(); } diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ExtraCountFunction.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ExtraCountFunction.java index ce7e8b738..01cf12ebe 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ExtraCountFunction.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ExtraCountFunction.java @@ -21,7 +21,7 @@ package org.deeplearning4j.spark.models.sequencevectors.functions; import lombok.NonNull; -import org.apache.spark.Accumulator; +import org.apache.spark.util.CollectionAccumulator; import org.apache.spark.api.java.function.Function; import org.deeplearning4j.models.sequencevectors.sequence.Sequence; import org.deeplearning4j.models.sequencevectors.sequence.SequenceElement; @@ -29,10 +29,10 @@ import org.deeplearning4j.spark.models.sequencevectors.primitives.ExtraCounter; import org.nd4j.common.primitives.Pair; public class ExtraCountFunction implements Function, Pair, Long>> { - protected Accumulator> accumulator; + protected CollectionAccumulator> accumulator; protected boolean fetchLabels; - public ExtraCountFunction(@NonNull Accumulator> accumulator, boolean fetchLabels) { + public ExtraCountFunction(@NonNull CollectionAccumulator> accumulator, boolean fetchLabels) { this.accumulator = accumulator; this.fetchLabels = fetchLabels; } diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ExtraElementsFrequenciesAccumulator.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ExtraElementsFrequenciesAccumulator.java index 9133c8532..e14c0a663 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ExtraElementsFrequenciesAccumulator.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/main/java/org/deeplearning4j/spark/models/sequencevectors/functions/ExtraElementsFrequenciesAccumulator.java @@ -20,11 +20,11 @@ package org.deeplearning4j.spark.models.sequencevectors.functions; -import org.apache.spark.AccumulatorParam; +import org.apache.spark.util.CollectionAccumulator; import org.deeplearning4j.spark.models.sequencevectors.primitives.ExtraCounter; -public class ExtraElementsFrequenciesAccumulator implements AccumulatorParam> { - @Override +public class ExtraElementsFrequenciesAccumulator extends CollectionAccumulator> { + public ExtraCounter addAccumulator(ExtraCounter c1, ExtraCounter c2) { if (c1 == null) { return new ExtraCounter<>(); @@ -33,13 +33,11 @@ public class ExtraElementsFrequenciesAccumulator implements AccumulatorParam addInPlace(ExtraCounter r1, ExtraCounter r2) { r1.incrementAll(r2); return r1; } - @Override public ExtraCounter zero(ExtraCounter initialValue) { return new ExtraCounter<>(); } diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectorsTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectorsTest.java index 5ba2a5dc1..2892b1653 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectorsTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/SparkSequenceVectorsTest.java @@ -21,8 +21,6 @@ package org.deeplearning4j.spark.models.sequencevectors; import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -35,24 +33,17 @@ import org.deeplearning4j.spark.models.sequencevectors.export.ExportContainer; import org.deeplearning4j.spark.models.sequencevectors.export.SparkModelExporter; import org.deeplearning4j.spark.models.word2vec.SparkWord2VecTest; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.nd4j.common.primitives.Counter; -import org.nd4j.common.resources.Downloader; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import java.io.File; -import java.net.URI; import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Slf4j + public class SparkSequenceVectorsTest extends BaseDL4JTest { @Override @@ -63,27 +54,6 @@ public class SparkSequenceVectorsTest extends BaseDL4JTest { protected static List> sequencesCyclic; private JavaSparkContext sc; - - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - @BeforeEach public void setUp() throws Exception { if (sequencesCyclic == null) { @@ -117,7 +87,6 @@ public class SparkSequenceVectorsTest extends BaseDL4JTest { } @Test - @Disabled("Timeout issue") public void testFrequenciesCount() throws Exception { if(Platform.isWindows()) { diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/export/ExportContainerTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/export/ExportContainerTest.java index eb4b790ed..3ca30f7d1 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/export/ExportContainerTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/sequencevectors/export/ExportContainerTest.java @@ -23,17 +23,11 @@ package org.deeplearning4j.spark.models.sequencevectors.export; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.models.word2vec.VocabWord; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.factory.Nd4j; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class ExportContainerTest extends BaseDL4JTest { @BeforeEach public void setUp() throws Exception { diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/word2vec/SparkWord2VecTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/word2vec/SparkWord2VecTest.java index ccadc9b51..d2d256005 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/word2vec/SparkWord2VecTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/java/org/deeplearning4j/spark/models/word2vec/SparkWord2VecTest.java @@ -20,9 +20,6 @@ package org.deeplearning4j.spark.models.word2vec; -import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -37,24 +34,18 @@ import org.deeplearning4j.spark.models.sequencevectors.export.ExportContainer; import org.deeplearning4j.spark.models.sequencevectors.export.SparkModelExporter; import org.deeplearning4j.spark.models.sequencevectors.learning.elements.SparkSkipGram; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; -import org.junit.jupiter.api.*; -import org.nd4j.common.resources.Downloader; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; import org.nd4j.parameterserver.distributed.conf.VoidConfiguration; -import java.io.File; import java.io.Serializable; -import java.net.URI; import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Slf4j + public class SparkWord2VecTest extends BaseDL4JTest { @Override @@ -65,27 +56,6 @@ public class SparkWord2VecTest extends BaseDL4JTest { private static List sentences; private JavaSparkContext sc; - - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - @BeforeEach public void setUp() throws Exception { if (sentences == null) { @@ -108,7 +78,7 @@ public class SparkWord2VecTest extends BaseDL4JTest { } @Test - @Disabled("AB 2019/05/21 - Failing - Issue #7657") + //@Ignore("AB 2019/05/21 - Failing - Issue #7657") public void testStringsTokenization1() throws Exception { JavaRDD rddSentences = sc.parallelize(sentences); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/resources/log4j.properties b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp-java8/src/test/resources/log4j.properties old mode 100755 new mode 100644 diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/pom.xml b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/pom.xml index ba96a4b88..e20fc1ea9 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/pom.xml +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/pom.xml @@ -26,12 +26,12 @@ 4.0.0 - org.deeplearning4j - spark_2.11 + net.brutex.ai + spark_2.12 1.0.0-SNAPSHOT - dl4j-spark-nlp_2.11 + dl4j-spark-nlp_2.12 dl4j-spark-nlp @@ -41,57 +41,42 @@ - org.deeplearning4j + net.brutex.ai deeplearning4j-nlp - ${deeplearning4j.version} - - - org.deeplearning4j - dl4j-spark_2.11 ${project.version} - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test + net.brutex.ai + dl4j-spark_2.12 + ${project.version} - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - org.datavec - datavec-spark_2.11 - ${datavec.version} + net.brutex.ai + datavec-spark_2.12 + ${project.version} org.apache.spark - spark-core_2.11 + spark-core_2.12 ${spark.version} provided com.fasterxml.jackson.module - jackson-module-scala_2.11 + jackson-module-scala_2.12 2.6.7.1 - org.deeplearning4j + net.brutex.ai deeplearning4j-common-tests ${project.version} test + + + com.google.guava + guava + ${guava.jre.version} + - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/MaxPerPartitionAccumulator.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/MaxPerPartitionAccumulator.java index 659fc1a41..36af95bd2 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/MaxPerPartitionAccumulator.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/MaxPerPartitionAccumulator.java @@ -20,26 +20,23 @@ package org.deeplearning4j.spark.text.accumulators; -import org.apache.spark.AccumulatorParam; +import org.apache.spark.util.CollectionAccumulator; import org.nd4j.common.primitives.Counter; /** * @author jeffreytang */ -public class MaxPerPartitionAccumulator implements AccumulatorParam> { +public class MaxPerPartitionAccumulator extends CollectionAccumulator> { - @Override public Counter addInPlace(Counter c1, Counter c2) { c1.incrementAll(c2); return c1; } - @Override public Counter zero(Counter initialCounter) { return new Counter<>(); } - @Override public Counter addAccumulator(Counter c1, Counter c2) { if (c1 == null) { return new Counter<>(); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/WordFreqAccumulator.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/WordFreqAccumulator.java index 4d5de1b61..6cd1e62cd 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/WordFreqAccumulator.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/accumulators/WordFreqAccumulator.java @@ -20,26 +20,23 @@ package org.deeplearning4j.spark.text.accumulators; -import org.apache.spark.AccumulatorParam; +import org.apache.spark.util.CollectionAccumulator; import org.nd4j.common.primitives.Counter; /** * @author jeffreytang */ -public class WordFreqAccumulator implements AccumulatorParam> { +public class WordFreqAccumulator extends CollectionAccumulator> { - @Override public Counter addInPlace(Counter c1, Counter c2) { c1.incrementAll(c2); return c1; } - @Override public Counter zero(Counter initialCounter) { return new Counter<>(); } - @Override public Counter addAccumulator(Counter c1, Counter c2) { if (c1 == null) { return new Counter<>(); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/CountCumSum.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/CountCumSum.java index 14f4c0899..4b757ec5f 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/CountCumSum.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/CountCumSum.java @@ -20,7 +20,7 @@ package org.deeplearning4j.spark.text.functions; -import org.apache.spark.Accumulator; +import org.apache.spark.util.CollectionAccumulator; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.broadcast.Broadcast; @@ -69,15 +69,15 @@ public class CountCumSum { public void cumSumWithinPartition() { // Accumulator to get the max of the cumulative sum in each partition - final Accumulator> maxPerPartitionAcc = - sc.accumulator(new Counter(), new MaxPerPartitionAccumulator()); + final CollectionAccumulator> maxPerPartitionAcc = + sc.sc().collectionAccumulator("MaxPerPartitionAccumulator"); // Partition mapping to fold within partition foldWithinPartitionRDD = sentenceCountRDD .mapPartitionsWithIndex(new FoldWithinPartitionFunction(maxPerPartitionAcc), true).cache(); actionForMapPartition(foldWithinPartitionRDD); // Broadcast the counter (partition index : sum of count) to all workers - broadcastedMaxPerPartitionCounter = sc.broadcast(maxPerPartitionAcc.value()); + broadcastedMaxPerPartitionCounter = sc.broadcast(maxPerPartitionAcc.value().get(0)); } public void cumSumBetweenPartition() { diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldWithinPartitionFunction.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldWithinPartitionFunction.java index 0730eb34b..38910c623 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldWithinPartitionFunction.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/FoldWithinPartitionFunction.java @@ -20,7 +20,7 @@ package org.deeplearning4j.spark.text.functions; -import org.apache.spark.Accumulator; +import org.apache.spark.util.CollectionAccumulator; import org.apache.spark.api.java.function.Function2; import org.nd4j.common.primitives.Counter; @@ -34,11 +34,11 @@ import java.util.concurrent.atomic.AtomicLong; */ public class FoldWithinPartitionFunction implements Function2, Iterator> { - public FoldWithinPartitionFunction(Accumulator> maxPartitionAcc) { + public FoldWithinPartitionFunction(CollectionAccumulator> maxPartitionAcc) { this.maxPerPartitionAcc = maxPartitionAcc; } - private Accumulator> maxPerPartitionAcc; + private CollectionAccumulator> maxPerPartitionAcc; @Override diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TextPipeline.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TextPipeline.java index 4d7a957a5..5fb7b0fbc 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TextPipeline.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/TextPipeline.java @@ -20,7 +20,7 @@ package org.deeplearning4j.spark.text.functions; -import org.apache.spark.Accumulator; +import org.apache.spark.util.CollectionAccumulator; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.broadcast.Broadcast; @@ -51,7 +51,7 @@ public class TextPipeline { private List stopWords = new ArrayList<>(); //Setup private JavaSparkContext sc; - private Accumulator> wordFreqAcc; + private CollectionAccumulator> wordFreqAcc; private Broadcast> stopWordBroadCast; // Return values private JavaRDD, AtomicLong>> sentenceWordsCountRDD; @@ -93,7 +93,7 @@ public class TextPipeline { private void setup() { // Set up accumulators and broadcast stopwords this.sc = new JavaSparkContext(corpusRDD.context()); - this.wordFreqAcc = sc.accumulator(new Counter(), new WordFreqAccumulator()); + this.wordFreqAcc = sc.sc().collectionAccumulator("WordFreqAccumulator"); //(new Counter(), new WordFreqAccumulator()); this.stopWordBroadCast = sc.broadcast(stopWords); } @@ -151,14 +151,14 @@ public class TextPipeline { for (Entry entry : wordFreq.entrySet()) { String stringToken = entry.getKey(); - Double tokenCount = entry.getValue().doubleValue(); + double tokenCount = entry.getValue().doubleValue(); // Turn words below min count to UNK stringToken = filterMinWord(stringToken, tokenCount); if (!useUnk && stringToken.equals("UNK")) { // Turn tokens to vocab and add to vocab cache } else - addTokenToVocabCache(stringToken, tokenCount.floatValue()); + addTokenToVocabCache(stringToken, entry.getValue().floatValue()); } } @@ -171,7 +171,7 @@ public class TextPipeline { sentenceWordsCountRDD = updateAndReturnAccumulatorVal(tokenizedRDD).cache(); // Get value from accumulator - Counter wordFreqCounter = wordFreqAcc.value(); + Counter wordFreqCounter = wordFreqAcc.value().get(0); // Filter out low count words and add to vocab cache object and feed into LookupCache filterMinWordAddVocab(wordFreqCounter); @@ -204,7 +204,7 @@ public class TextPipeline { } // Getters - public Accumulator> getWordFreqAcc() { + public CollectionAccumulator> getWordFreqAcc() { if (wordFreqAcc != null) { return wordFreqAcc; } else { diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/UpdateWordFreqAccumulatorFunction.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/UpdateWordFreqAccumulatorFunction.java index e8340803e..312677c98 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/UpdateWordFreqAccumulatorFunction.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/main/java/org/deeplearning4j/spark/text/functions/UpdateWordFreqAccumulatorFunction.java @@ -20,7 +20,7 @@ package org.deeplearning4j.spark.text.functions; -import org.apache.spark.Accumulator; +import org.apache.spark.util.CollectionAccumulator; import org.apache.spark.api.java.function.Function; import org.apache.spark.broadcast.Broadcast; import org.nd4j.common.primitives.Counter; @@ -35,10 +35,10 @@ import java.util.concurrent.atomic.AtomicLong; public class UpdateWordFreqAccumulatorFunction implements Function, Pair, AtomicLong>> { private Broadcast> stopWords; - private Accumulator> wordFreqAcc; + private CollectionAccumulator> wordFreqAcc; public UpdateWordFreqAccumulatorFunction(Broadcast> stopWords, - Accumulator> wordFreqAcc) { + CollectionAccumulator> wordFreqAcc) { this.wordFreqAcc = wordFreqAcc; this.stopWords = stopWords; } diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecTest.java index 9856fd9d1..4859b91a6 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/models/embeddings/word2vec/Word2VecTest.java @@ -21,16 +21,11 @@ package org.deeplearning4j.spark.models.embeddings.word2vec; import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.deeplearning4j.common.resources.DL4JResources; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.io.TempDir; import org.nd4j.common.io.ClassPathResource; import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; @@ -43,56 +38,28 @@ import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreproc import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.LowCasePreProcessor; import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Test; -import org.nd4j.common.resources.Downloader; -import org.nd4j.common.resources.strumpf.StrumpfResolver; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import java.io.File; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Slf4j -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) -@Disabled("Permissions issues on CI") -@Tag(TagNames.NEEDS_VERIFY) +//@Ignore public class Word2VecTest { - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } + @TempDir + public File testDir; @Test - public void testConcepts(@TempDir Path testDir) throws Exception { + public void testConcepts() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } // These are all default values for word2vec SparkConf sparkConf = new SparkConf().setMaster("local[8]") .set("spark.driver.host", "localhost") @@ -166,8 +133,7 @@ public class Word2VecTest { // test serialization - - File tempFile = Files.createTempFile(testDir,"temp" + System.currentTimeMillis(),"tmp").toFile(); + File tempFile = new File(testDir, "temp" + System.currentTimeMillis() + ".tmp"); int idx1 = word2Vec.vocab().wordFor("day").getIndex(); @@ -193,7 +159,7 @@ public class Word2VecTest { assertEquals(array1, array2); } - @Disabled + //@Ignore @Test public void testSparkW2VonBiggerCorpus() throws Exception { SparkConf sparkConf = new SparkConf().setMaster("local[8]").setAppName("sparktest") @@ -232,7 +198,7 @@ public class Word2VecTest { } @Test - @Disabled + //@Ignore public void testPortugeseW2V() throws Exception { WordVectors word2Vec = WordVectorSerializer.loadTxtVectors(new File("/ext/Temp/para.txt")); word2Vec.setModelUtils(new FlatModelUtils()); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java index 57c295f2c..d998ddde4 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/BaseSparkTest.java @@ -20,50 +20,21 @@ package org.deeplearning4j.spark.text; -import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.spark.models.embeddings.word2vec.Word2VecVariables; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.nd4j.common.resources.Downloader; -import java.io.File; import java.io.Serializable; import java.lang.reflect.Field; -import java.net.URI; import java.util.Collections; import java.util.Map; -@Slf4j + public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable { protected transient JavaSparkContext sc; - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - - - @Override public long getTimeoutMilliseconds() { return 120000L; diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TestFunction.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TestFunction.java index 3bfddf12e..618bf0ac7 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TestFunction.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TestFunction.java @@ -21,15 +21,9 @@ package org.deeplearning4j.spark.text; import org.apache.spark.api.java.function.Function; -import org.junit.jupiter.api.Tag; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.util.List; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestFunction implements Function { public TestFunction(List lst) { this.lst = lst; diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java index 4b71fb4b7..7e4a4944e 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/java/org/deeplearning4j/spark/text/TextPipelineTest.java @@ -21,7 +21,6 @@ package org.deeplearning4j.spark.text; import com.sun.jna.Platform; -import lombok.SneakyThrows; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; @@ -36,10 +35,9 @@ import org.deeplearning4j.spark.models.embeddings.word2vec.Word2Vec; import org.deeplearning4j.spark.text.functions.CountCumSum; import org.deeplearning4j.spark.text.functions.TextPipeline; import org.deeplearning4j.text.stopwords.StopWords; -import org.junit.jupiter.api.*; -import org.nd4j.common.resources.Downloader; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; +import org.junit.jupiter.api.BeforeEach; + +import org.junit.jupiter.api.Test; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.common.primitives.Counter; import org.nd4j.common.primitives.Pair; @@ -47,8 +45,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Tuple2; -import java.io.File; -import java.net.URI; import java.util.*; import java.util.concurrent.atomic.AtomicLong; @@ -58,10 +54,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Jeffrey Tang */ -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class TextPipelineTest extends BaseSparkTest { private List sentenceList; @@ -75,26 +67,6 @@ public class TextPipelineTest extends BaseSparkTest { return sc.parallelize(sentenceList, 2); } - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - @BeforeEach public void before() throws Exception { conf = new SparkConf().setMaster("local[4]").setAppName("sparktest").set("spark.driver.host", "localhost"); @@ -123,6 +95,10 @@ public class TextPipelineTest extends BaseSparkTest { @Test public void testTokenizer() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } JavaSparkContext sc = getContext(); JavaRDD corpusRDD = getCorpusRDD(sc); Broadcast> broadcastTokenizerVarMap = sc.broadcast(word2vec.getTokenizerVarMap()); @@ -148,7 +124,7 @@ public class TextPipelineTest extends BaseSparkTest { JavaRDD, AtomicLong>> sentenceWordsCountRDD = pipeline.updateAndReturnAccumulatorVal(tokenizedRDD); - Counter wordFreqCounter = pipeline.getWordFreqAcc().value(); + Counter wordFreqCounter = pipeline.getWordFreqAcc().value().get(0); assertEquals(wordFreqCounter.getCount("STOP"), 4, 0); assertEquals(wordFreqCounter.getCount("strange"), 2, 0); assertEquals(wordFreqCounter.getCount("flowers"), 1, 0); @@ -177,7 +153,7 @@ public class TextPipelineTest extends BaseSparkTest { JavaRDD> tokenizedRDD = pipeline.tokenize(); pipeline.updateAndReturnAccumulatorVal(tokenizedRDD); - Counter wordFreqCounter = pipeline.getWordFreqAcc().value(); + Counter wordFreqCounter = pipeline.getWordFreqAcc().value().get(0); assertEquals(wordFreqCounter.getCount("is"), 1, 0); assertEquals(wordFreqCounter.getCount("this"), 1, 0); assertEquals(wordFreqCounter.getCount("are"), 1, 0); @@ -202,7 +178,7 @@ public class TextPipelineTest extends BaseSparkTest { JavaRDD> tokenizedRDD = pipeline.tokenize(); pipeline.updateAndReturnAccumulatorVal(tokenizedRDD); - Counter wordFreqCounter = pipeline.getWordFreqAcc().value(); + Counter wordFreqCounter = pipeline.getWordFreqAcc().value().get(0); assertEquals(wordFreqCounter.getCount("is"), 0, 0); assertEquals(wordFreqCounter.getCount("this"), 0, 0); assertEquals(wordFreqCounter.getCount("are"), 0, 0); @@ -225,7 +201,7 @@ public class TextPipelineTest extends BaseSparkTest { TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap); JavaRDD> tokenizedRDD = pipeline.tokenize(); pipeline.updateAndReturnAccumulatorVal(tokenizedRDD); - Counter wordFreqCounter = pipeline.getWordFreqAcc().value(); + Counter wordFreqCounter = pipeline.getWordFreqAcc().value().get(0); pipeline.filterMinWordAddVocab(wordFreqCounter); VocabCache vocabCache = pipeline.getVocabCache(); @@ -359,7 +335,7 @@ public class TextPipelineTest extends BaseSparkTest { sc.stop(); } - @Test @Disabled //AB 2020/04/20 https://github.com/eclipse/deeplearning4j/issues/8849 + @Test //@Ignore //AB 2020/04/20 https://github.com/eclipse/deeplearning4j/issues/8849 public void testCountCumSum() throws Exception { JavaSparkContext sc = getContext(); JavaRDD corpusRDD = getCorpusRDD(sc); @@ -384,7 +360,7 @@ public class TextPipelineTest extends BaseSparkTest { * * @throws Exception */ - @Test @Disabled //AB 2020/04/19 https://github.com/eclipse/deeplearning4j/issues/8849 + @Test //@Ignore //AB 2020/04/19 https://github.com/eclipse/deeplearning4j/issues/8849 public void testZipFunction1() throws Exception { JavaSparkContext sc = getContext(); JavaRDD corpusRDD = getCorpusRDD(sc); @@ -422,7 +398,7 @@ public class TextPipelineTest extends BaseSparkTest { sc.stop(); } - @Test @Disabled //AB 2020/04/19 https://github.com/eclipse/deeplearning4j/issues/8849 + @Test //@Ignore //AB 2020/04/19 https://github.com/eclipse/deeplearning4j/issues/8849 public void testZipFunction2() throws Exception { JavaSparkContext sc = getContext(); JavaRDD corpusRDD = getCorpusRDD(sc); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/resources/log4j.properties b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-nlp/src/test/resources/log4j.properties old mode 100755 new mode 100644 diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/pom.xml b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/pom.xml index e60be88d2..a992c7f5c 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/pom.xml +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/pom.xml @@ -26,36 +26,30 @@ 4.0.0 - org.deeplearning4j - spark_2.11 + net.brutex.ai + spark_2.12 1.0.0-SNAPSHOT - dl4j-spark-parameterserver_2.11 + dl4j-spark-parameterserver_2.12 dl4j-spark-parameterserver - - 2.2.1 - 1.8 - 1.8 - - - org.nd4j + net.brutex.ai nd4j-aeron - ${nd4j.version} - - - org.deeplearning4j - dl4j-spark_2.11 ${project.version} - org.nd4j - nd4j-parameter-server-node_2.11 - ${nd4j.version} + net.brutex.ai + dl4j-spark_2.12 + ${project.version} + + + net.brutex.ai + nd4j-parameter-server-node_2.12 + ${project.version} net.jpountz.lz4 @@ -63,37 +57,38 @@ + - org.projectlombok - lombok - ${lombok.version} - provided - - - org.deeplearning4j + net.brutex.ai deeplearning4j-parallel-wrapper - ${nd4j.version} + ${project.version} org.apache.spark - spark-core_2.11 + spark-core_2.12 ${spark.version} provided - org.deeplearning4j + net.brutex.ai deeplearning4j-common-tests ${project.version} test + - nd4j-tests-cpu - - - nd4j-tests-cuda + enableNativeCPU + + + net.brutex.ai + nd4j-native + ${project.version} + windows-x86_64 + + diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java index 58d922ff4..2a17ab3e1 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/main/java/org/deeplearning4j/spark/parameterserver/training/SharedTrainingMaster.java @@ -79,8 +79,8 @@ import org.nd4j.parameterserver.distributed.util.NetworkOrganizer; import org.nd4j.parameterserver.distributed.v2.ModelParameterServer; import org.nd4j.parameterserver.distributed.v2.transport.Transport; import org.nd4j.parameterserver.distributed.v2.transport.impl.AeronUdpTransport; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.net.InetAddress; @@ -89,8 +89,18 @@ import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; + +/** + * SharedTrainingMaster implements distributed training of neural networks using a compressed quantized gradient + * (update) sharing implementation based on the Strom 2015 paper “Scalable Distributed DNN Training Using Commodity + * GPU Cloud Computing”: https://s3-us-west-2.amazonaws.com/amazon.jobs-public-documents/strom_interspeech2015.pdf. + * The Deeplearning4j implementation makes a number of modifications, such as having the option to use a + * parameter-server based implementation for fault tolerance and execution where multicast networking support + * is not available. + */ @Slf4j @Data + public class SharedTrainingMaster extends BaseTrainingMaster implements TrainingMaster { //Static counter/id fields used to determine which training master last set up the singleton param servers, etc diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java index ca3313a00..d110e41bd 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/BaseSparkTest.java @@ -20,9 +20,6 @@ package org.deeplearning4j.spark.parameterserver; -import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -32,9 +29,7 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.nd4j.common.resources.Downloader; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -42,14 +37,12 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.learning.config.Nesterovs; import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.io.File; import java.io.Serializable; -import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.Random; -@Slf4j + public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable { protected transient JavaSparkContext sc; protected transient INDArray labels; @@ -67,27 +60,6 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable return 120000L; } - - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - @BeforeEach public void before() { diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunctionTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunctionTest.java index 4d93e9329..5b62cc038 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunctionTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAccumulationFunctionTest.java @@ -20,26 +20,20 @@ package org.deeplearning4j.spark.parameterserver.accumulation; -import com.sun.jna.Platform; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; + +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; + import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; -import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + + public class SharedTrainingAccumulationFunctionTest { - @BeforeEach - public void setUp() throws Exception {} @Test public void testAccumulation1() throws Exception { + INDArray updates1 = Nd4j.create(1000).assign(1.0); INDArray updates2 = Nd4j.create(1000).assign(2.0); INDArray expUpdates = Nd4j.create(1000).assign(3.0); @@ -55,16 +49,16 @@ public class SharedTrainingAccumulationFunctionTest { SharedTrainingAccumulationTuple tupleE = accumulationFunction.call(null, tuple1); // testing null + tuple accumulation - assertEquals(1, tupleE.getAggregationsCount()); - assertEquals(1.0, tupleE.getScoreSum(), 0.01); - assertEquals(updates1, tupleE.getUpdaterStateArray()); + Assertions.assertEquals(1, tupleE.getAggregationsCount()); + Assertions.assertEquals(1.0, tupleE.getScoreSum(), 0.01); + Assertions.assertEquals(updates1, tupleE.getUpdaterStateArray()); // testing tuple + tuple accumulation SharedTrainingAccumulationTuple tupleResult = accumulationFunction.call(tuple1, tuple2); - assertEquals(2, tupleResult.getAggregationsCount()); - assertEquals(3.0, tupleResult.getScoreSum(), 0.01); - assertEquals(expUpdates, tupleResult.getUpdaterStateArray()); + Assertions.assertEquals(2, tupleResult.getAggregationsCount()); + Assertions.assertEquals(3.0, tupleResult.getScoreSum(), 0.01); + Assertions.assertEquals(expUpdates, tupleResult.getUpdaterStateArray()); } } diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunctionTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunctionTest.java index 86bbf9e6e..25ef434bd 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunctionTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/accumulation/SharedTrainingAggregateFunctionTest.java @@ -23,18 +23,12 @@ package org.deeplearning4j.spark.parameterserver.accumulation; import com.sun.jna.Platform; import org.deeplearning4j.spark.parameterserver.training.SharedTrainingResult; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class SharedTrainingAggregateFunctionTest { @BeforeEach public void setUp() throws Exception { @@ -43,6 +37,10 @@ public class SharedTrainingAggregateFunctionTest { @Test public void testAggregate1() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } INDArray updates1 = Nd4j.create(1000).assign(1.0); INDArray updates2 = Nd4j.create(1000).assign(2.0); INDArray expUpdates = Nd4j.create(1000).assign(3.0); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIteratorTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIteratorTest.java index 04b3dfd6f..b837efe5e 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIteratorTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualDataSetIteratorTest.java @@ -21,59 +21,29 @@ package org.deeplearning4j.spark.parameterserver.iterators; import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.resources.Downloader; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; -import java.io.File; -import java.net.URI; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Slf4j + public class VirtualDataSetIteratorTest { @BeforeEach public void setUp() throws Exception {} - - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - @Test public void testSimple1() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } List> iterators = new ArrayList<>(); List first = new ArrayList<>(); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIteratorTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIteratorTest.java index ffde49e96..4e56b575a 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIteratorTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/iterators/VirtualIteratorTest.java @@ -21,56 +21,26 @@ package org.deeplearning4j.spark.parameterserver.iterators; import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.resources.Downloader; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import java.io.File; -import java.net.URI; import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Slf4j + public class VirtualIteratorTest { @BeforeEach public void setUp() throws Exception { // } - - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - @Test public void testIteration1() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } List integers = new ArrayList<>(); for (int i = 0; i < 100; i++) { integers.add(i); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/TestElephasImport.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/TestElephasImport.java index 040fcafe3..16429f41e 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/TestElephasImport.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/modelimport/elephas/TestElephasImport.java @@ -21,59 +21,30 @@ package org.deeplearning4j.spark.parameterserver.modelimport.elephas; import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaSparkContext; import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; import org.deeplearning4j.spark.parameterserver.BaseSparkTest; import org.deeplearning4j.spark.parameterserver.training.SharedTrainingMaster; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.nd4j.common.io.ClassPathResource; -import org.nd4j.common.resources.Downloader; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import java.io.File; -import java.net.URI; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import static java.io.File.createTempFile; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Slf4j + public class TestElephasImport extends BaseSparkTest { - - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - @Test public void testElephasSequentialImport() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } String modelPath = "modelimport/elephas/elephas_sequential.h5"; SparkDl4jMultiLayer model = importElephasSequential(sc, modelPath); // System.out.println(model.getNetwork().summary()); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java index 79788a094..31cd119d7 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark-parameterserver/src/test/java/org/deeplearning4j/spark/parameterserver/train/GradientSharingTrainingTest.java @@ -44,15 +44,13 @@ import org.deeplearning4j.spark.api.RDDTrainingApproach; import org.deeplearning4j.spark.api.TrainingMaster; import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; +import org.deeplearning4j.spark.parameterserver.BaseSparkTest; import org.deeplearning4j.spark.parameterserver.training.SharedTrainingMaster; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -68,25 +66,17 @@ import org.nd4j.parameterserver.distributed.v2.enums.MeshBuildMode; import java.io.File; import java.io.Serializable; import java.net.Inet4Address; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import static org.junit.jupiter.api.Assertions.*; -import org.deeplearning4j.spark.parameterserver.BaseSparkTest; @Slf4j -//@Disabled("AB 2019/05/21 - Failing - Issue #7657") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) +////@Ignore("AB 2019/05/21 - Failing - Issue #7657") public class GradientSharingTrainingTest extends BaseSparkTest { - + @TempDir + public File testDir; @Override public long getTimeoutMilliseconds() { @@ -94,14 +84,12 @@ public class GradientSharingTrainingTest extends BaseSparkTest { } @Test - @Disabled - public void trainSanityCheck(@TempDir Path testDir) throws Exception { + public void trainSanityCheck() throws Exception { for(boolean mds : new boolean[]{false, true}) { INDArray last = null; - INDArray lastDup = null; - for (String s : new String[]{"paths", "direSparkSequenceVectorsTestct", "export"}) { + for (String s : new String[]{"paths", "direct", "export"}) { System.out.println("--------------------------------------------------------------------------------------------------------------"); log.info("Starting: {} - {}", s, (mds ? "MultiDataSet" : "DataSet")); boolean isPaths = "paths".equals(s); @@ -121,7 +109,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest { throw new RuntimeException(); } - File temp = testDir.toFile(); + File temp = testDir; //TODO this probably won't work everywhere... @@ -159,8 +147,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest { sparkNet.setCollectTrainingStats(tm.getIsCollectTrainingStats()); // System.out.println(Arrays.toString(sparkNet.getNetwork().params().get(NDArrayIndex.point(0), NDArrayIndex.interval(0, 256)).dup().data().asFloat())); - File f = new File(testDir.toFile(),"test-dir-1"); - f.mkdirs(); + File f = testDir; DataSetIterator iter = new MnistDataSetIterator(16, true, 12345); int count = 0; List paths = new ArrayList<>(); @@ -253,11 +240,11 @@ public class GradientSharingTrainingTest extends BaseSparkTest { } - @Test @Disabled //AB https://github.com/eclipse/deeplearning4j/issues/8985 - public void differentNetsTrainingTest(@TempDir Path testDir) throws Exception { + @Test //@Ignore //AB https://github.com/eclipse/deeplearning4j/issues/8985 + public void differentNetsTrainingTest() throws Exception { int batch = 3; - File temp = testDir.toFile(); + File temp = testDir; DataSet ds = new IrisDataSetIterator(150, 150).next(); List list = ds.asList(); Collections.shuffle(list, new Random(12345)); @@ -341,12 +328,11 @@ public class GradientSharingTrainingTest extends BaseSparkTest { } - @Test - public void testEpochUpdating(@TempDir Path testDir) throws Exception { + @Test //@Ignore + public void testEpochUpdating() throws Exception { //Ensure that epoch counter is incremented properly on the workers - File temp = testDir.resolve("new-dir-" + UUID.randomUUID().toString()).toFile(); - temp.mkdirs(); + File temp = testDir; //TODO this probably won't work everywhere... String controller = Inet4Address.getLocalHost().getHostAddress(); @@ -385,8 +371,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest { int count = 0; List paths = new ArrayList<>(); List ds = new ArrayList<>(); - File f = new File(testDir.toFile(),"test-dir-1"); - f.mkdirs(); + File f = testDir; while (iter.hasNext() && count++ < 8) { DataSet d = iter.next(); File out = new File(f, count + ".bin"); @@ -397,7 +382,7 @@ public class GradientSharingTrainingTest extends BaseSparkTest { } JavaRDD pathRdd = sc.parallelize(paths); - for( int i = 0; i < 3; i++) { + for( int i=0; i<3; i++ ) { ThresholdAlgorithm ta = tm.getThresholdAlgorithm(); sparkNet.fitPaths(pathRdd); //Check also that threshold algorithm was updated/averaged diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/pom.xml b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/pom.xml index 7a328ca52..2c99d6565 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/pom.xml +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/pom.xml @@ -26,44 +26,31 @@ 4.0.0 - org.deeplearning4j - spark_2.11 + net.brutex.ai + spark_2.12 1.0.0-SNAPSHOT - dl4j-spark_2.11 + dl4j-spark_2.12 dl4j-spark - org.deeplearning4j + net.brutex.ai deeplearning4j-core - ${deeplearning4j.version} + ${project.version} - org.datavec - datavec-spark_2.11 - ${datavec.version} + net.brutex.ai + datavec-spark_2.12 + ${project.version} - org.deeplearning4j + net.brutex.ai deeplearning4j-ui-components - ${deeplearning4j.version} - - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test + ${project.version} ch.qos.logback @@ -72,9 +59,9 @@ test - org.deeplearning4j + net.brutex.ai deeplearning4j-ui - ${deeplearning4j.version} + ${project.version} test @@ -84,31 +71,22 @@ - org.nd4j - nd4j-kryo_2.11 - ${nd4j.version} + net.brutex.ai + nd4j-kryo_2.12 + ${project.version} test org.apache.spark - spark-core_2.11 + spark-core_2.12 ${spark.version} provided - org.deeplearning4j + net.brutex.ai deeplearning4j-common-tests ${project.version} test - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitioner.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitioner.java index aba2e6202..e2f5814bd 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitioner.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitioner.java @@ -20,8 +20,8 @@ package org.deeplearning4j.spark.impl.common.repartition; -import org.nd4j.shade.guava.base.Predicate; -import org.nd4j.shade.guava.collect.Collections2; +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; import org.apache.spark.Partitioner; import scala.Tuple2; @@ -29,8 +29,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; -import static org.nd4j.shade.guava.base.Preconditions.checkArgument; -import static org.nd4j.shade.guava.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; public class HashingBalancedPartitioner extends Partitioner { private final int numClasses; // Total number of element classes diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingMaster.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingMaster.java index 84d89950c..dbe2cbb27 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingMaster.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/paramavg/BaseTrainingMaster.java @@ -43,16 +43,16 @@ import org.deeplearning4j.spark.util.serde.StorageLevelDeserializer; import org.deeplearning4j.spark.util.serde.StorageLevelSerializer; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.MultiDataSet; -import org.nd4j.shade.jackson.annotation.JsonAutoDetect; -import org.nd4j.shade.jackson.annotation.PropertyAccessor; -import org.nd4j.shade.jackson.core.JsonFactory; -import org.nd4j.shade.jackson.databind.DeserializationFeature; -import org.nd4j.shade.jackson.databind.MapperFeature; -import org.nd4j.shade.jackson.databind.ObjectMapper; -import org.nd4j.shade.jackson.databind.SerializationFeature; -import org.nd4j.shade.jackson.databind.annotation.JsonDeserialize; -import org.nd4j.shade.jackson.databind.annotation.JsonSerialize; -import org.nd4j.shade.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.IOException; import java.net.URI; diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java index e870d5ccb..8d8532e0b 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/impl/paramavg/ParameterAveragingTrainingMaster.java @@ -62,15 +62,15 @@ import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.MultiDataSet; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.OutputStream; import java.util.*; -import static org.nd4j.shade.guava.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkArgument; @Data @JsonIgnoreProperties({"stats", "listeners", "iterationCount", "rng", "lastExportedRDDId", "lastRDDExportPath", diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/iterator/SparkADSI.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/iterator/SparkADSI.java new file mode 100644 index 000000000..65f509566 --- /dev/null +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/iterator/SparkADSI.java @@ -0,0 +1,123 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.iterator; + +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.TaskContext; +import org.apache.spark.TaskContextHelper; +import org.nd4j.linalg.dataset.AsyncDataSetIterator; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.dataset.DataSet; +import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; +import org.nd4j.linalg.dataset.callbacks.DataSetCallback; +import org.nd4j.linalg.dataset.callbacks.DefaultCallback; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +@Slf4j +public class SparkADSI extends AsyncDataSetIterator { + protected TaskContext context; + + protected SparkADSI() { + super(); + } + + public SparkADSI(DataSetIterator baseIterator) { + this(baseIterator, 8); + } + + public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue queue) { + this(iterator, queueSize, queue, true); + } + + public SparkADSI(DataSetIterator baseIterator, int queueSize) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize)); + } + + public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace); + } + + public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, Integer deviceId) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace, new DefaultCallback(), + deviceId); + } + + public SparkADSI(DataSetIterator baseIterator, int queueSize, boolean useWorkspace, DataSetCallback callback) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace, callback); + } + + public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue queue, boolean useWorkspace) { + this(iterator, queueSize, queue, useWorkspace, new DefaultCallback()); + } + + public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue queue, boolean useWorkspace, + DataSetCallback callback) { + this(iterator, queueSize, queue, useWorkspace, callback, Nd4j.getAffinityManager().getDeviceForCurrentThread()); + } + + public SparkADSI(DataSetIterator iterator, int queueSize, BlockingQueue queue, boolean useWorkspace, + DataSetCallback callback, Integer deviceId) { + this(); + + if (queueSize < 2) + queueSize = 2; + + this.deviceId = deviceId; + this.callback = callback; + this.useWorkspace = useWorkspace; + this.buffer = queue; + this.prefetchSize = queueSize; + this.backedIterator = iterator; + this.workspaceId = "SADSI_ITER-" + java.util.UUID.randomUUID().toString(); + + if (iterator.resetSupported()) + this.backedIterator.reset(); + + context = TaskContext.get(); + + this.thread = new SparkPrefetchThread(buffer, iterator, terminator, null, Nd4j.getAffinityManager().getDeviceForCurrentThread()); + + /** + * We want to ensure, that background thread will have the same thread->device affinity, as master thread + */ + + thread.setDaemon(true); + thread.start(); + } + + @Override + protected void externalCall() { + TaskContextHelper.setTaskContext(context); + + } + + public class SparkPrefetchThread extends AsyncPrefetchThread { + + protected SparkPrefetchThread(BlockingQueue queue, DataSetIterator iterator, DataSet terminator, MemoryWorkspace workspace, int deviceId) { + super(queue, iterator, terminator, workspace, deviceId); + } + + + } +} diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/iterator/SparkAMDSI.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/iterator/SparkAMDSI.java new file mode 100644 index 000000000..712e62d28 --- /dev/null +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/iterator/SparkAMDSI.java @@ -0,0 +1,118 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.iterator; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.TaskContext; +import org.apache.spark.TaskContextHelper; +import org.nd4j.linalg.dataset.AsyncMultiDataSetIterator; +import org.nd4j.linalg.dataset.api.MultiDataSet; +import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; +import org.nd4j.linalg.dataset.callbacks.DataSetCallback; +import org.nd4j.linalg.dataset.callbacks.DefaultCallback; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +@Slf4j +public class SparkAMDSI extends AsyncMultiDataSetIterator { + protected TaskContext context; + + protected SparkAMDSI() { + super(); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator) { + this(baseIterator, 8); + } + + public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue queue) { + this(iterator, queueSize, queue, true); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize)); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace, Integer deviceId) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace, + new DefaultCallback(), deviceId); + } + + public SparkAMDSI(MultiDataSetIterator baseIterator, int queueSize, boolean useWorkspace, + DataSetCallback callback) { + this(baseIterator, queueSize, new LinkedBlockingQueue(queueSize), useWorkspace, callback); + } + + public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue queue, + boolean useWorkspace) { + this(iterator, queueSize, queue, useWorkspace, null); + } + + public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue queue, + boolean useWorkspace, DataSetCallback callback) { + this(iterator, queueSize, queue, useWorkspace, callback, Nd4j.getAffinityManager().getDeviceForCurrentThread()); + } + + public SparkAMDSI(MultiDataSetIterator iterator, int queueSize, BlockingQueue queue, + boolean useWorkspace, DataSetCallback callback, Integer deviceId) { + this(); + + if (queueSize < 2) + queueSize = 2; + + this.callback = callback; + this.buffer = queue; + this.backedIterator = iterator; + this.useWorkspaces = useWorkspace; + this.prefetchSize = queueSize; + this.workspaceId = "SAMDSI_ITER-" + java.util.UUID.randomUUID().toString(); + this.deviceId = deviceId; + + if (iterator.resetSupported()) + this.backedIterator.reset(); + + this.thread = new SparkPrefetchThread(buffer, iterator, terminator, Nd4j.getAffinityManager().getDeviceForCurrentThread()); + + context = TaskContext.get(); + + thread.setDaemon(true); + thread.start(); + } + + @Override + protected void externalCall() { + TaskContextHelper.setTaskContext(context); + } + + protected class SparkPrefetchThread extends AsyncPrefetchThread { + + protected SparkPrefetchThread(@NonNull BlockingQueue queue, @NonNull MultiDataSetIterator iterator, @NonNull MultiDataSet terminator, int deviceId) { + super(queue, iterator, terminator, deviceId); + } + } +} diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelDeserializer.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelDeserializer.java index 8d75898d5..cc9490a9a 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelDeserializer.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelDeserializer.java @@ -21,11 +21,11 @@ package org.deeplearning4j.spark.util.serde; import org.apache.spark.storage.StorageLevel; -import org.nd4j.shade.jackson.core.JsonParser; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.DeserializationContext; -import org.nd4j.shade.jackson.databind.JsonDeserializer; -import org.nd4j.shade.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelSerializer.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelSerializer.java index 1bdc55c7a..db02ea278 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelSerializer.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/main/java/org/deeplearning4j/spark/util/serde/StorageLevelSerializer.java @@ -21,10 +21,10 @@ package org.deeplearning4j.spark.util.serde; import org.apache.spark.storage.StorageLevel; -import org.nd4j.shade.jackson.core.JsonGenerator; -import org.nd4j.shade.jackson.core.JsonProcessingException; -import org.nd4j.shade.jackson.databind.JsonSerializer; -import org.nd4j.shade.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; import java.util.HashMap; diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/BaseSparkTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/BaseSparkTest.java index 12695656d..e00f8d6d3 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/BaseSparkTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/BaseSparkTest.java @@ -20,9 +20,6 @@ package org.deeplearning4j.spark; -import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; @@ -34,9 +31,7 @@ import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.nd4j.common.resources.Downloader; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -44,14 +39,12 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.learning.config.Nesterovs; import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.io.File; import java.io.Serializable; -import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.Random; -@Slf4j + public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable { protected transient JavaSparkContext sc; protected transient INDArray labels; @@ -67,25 +60,6 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable public long getTimeoutMilliseconds() { return 120000L; } - @BeforeAll - @SneakyThrows - public static void beforeAll() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } @BeforeEach public void before() { @@ -102,8 +76,6 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable labels.putScalar(new int[] {i, x1}, 1.0); } - - sparkData = getBasicSparkDataSet(nRows, input, labels); } @@ -150,7 +122,7 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable protected SparkDl4jMultiLayer getBasicNetwork() { return new SparkDl4jMultiLayer(sc, getBasicConf(), - new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0)); + new ParameterAveragingTrainingMaster(true, numExecutors(), 1, 10, 1, 0)); } protected int numExecutors() { @@ -160,12 +132,12 @@ public abstract class BaseSparkTest extends BaseDL4JTest implements Serializable protected MultiLayerConfiguration getBasicConf() { MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123) .updater(new Nesterovs(0.1, 0.9)).list() - .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(nIn).nOut(3) - .activation(Activation.TANH).build()) - .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( - LossFunctions.LossFunction.MCXENT).nIn(3).nOut(nOut) - .activation(Activation.SOFTMAX).build()) - .build(); + .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(nIn).nOut(3) + .activation(Activation.TANH).build()) + .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder( + LossFunctions.LossFunction.MCXENT).nIn(3).nOut(nOut) + .activation(Activation.SOFTMAX).build()) + .build(); return conf; } diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSpark.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSpark.java index e16bc135e..ed8de3623 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSpark.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSpark.java @@ -44,9 +44,7 @@ import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.deeplearning4j.spark.earlystopping.SparkDataSetLossCalculator; import org.deeplearning4j.spark.earlystopping.SparkEarlyStoppingTrainer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; @@ -62,14 +60,14 @@ import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.DIST_SYSTEMS) -@Tag(TagNames.SPARK) public class TestEarlyStoppingSpark extends BaseSparkTest { @Test public void testEarlyStoppingIris() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .updater(new Sgd()).weightInit(WeightInit.XAVIER).list() diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSparkCompGraph.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSparkCompGraph.java index 8dff45d31..3de17a742 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSparkCompGraph.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestEarlyStoppingSparkCompGraph.java @@ -46,9 +46,7 @@ import org.deeplearning4j.spark.earlystopping.SparkEarlyStoppingGraphTrainer; import org.deeplearning4j.spark.earlystopping.SparkLossCalculatorComputationGraph; import org.deeplearning4j.spark.impl.graph.dataset.DataSetToMultiDataSetFn; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; @@ -64,15 +62,15 @@ import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.LARGE_RESOURCES) -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.DIST_SYSTEMS) -@Tag(TagNames.SPARK) public class TestEarlyStoppingSparkCompGraph extends BaseSparkTest { @Test public void testEarlyStoppingIris() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT) .updater(new Sgd()).weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in") diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestKryo.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestKryo.java index 28b59d75f..33023d605 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestKryo.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/TestKryo.java @@ -30,10 +30,7 @@ import org.deeplearning4j.nn.conf.graph.rnn.DuplicateToTimeSeriesVertex; import org.deeplearning4j.nn.conf.graph.rnn.LastTimeStepVertex; import org.deeplearning4j.nn.conf.layers.*; import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.evaluation.IEvaluation; import org.nd4j.evaluation.classification.*; import org.nd4j.evaluation.regression.RegressionEvaluation; @@ -51,10 +48,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestKryo extends BaseSparkKryoTest { private void testSerialization(T in, SerializerInstance si) { diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/common/AddTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/common/AddTest.java index 36c84f6f6..f366de5b4 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/common/AddTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/common/AddTest.java @@ -23,10 +23,7 @@ package org.deeplearning4j.spark.common; import org.apache.spark.api.java.JavaRDD; import org.deeplearning4j.spark.BaseSparkTest; import org.deeplearning4j.spark.impl.common.Add; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; @@ -34,10 +31,7 @@ import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class AddTest extends BaseSparkTest { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/data/TestShuffleExamples.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/data/TestShuffleExamples.java index e01a01d13..f879cfd29 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/data/TestShuffleExamples.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/data/TestShuffleExamples.java @@ -20,26 +20,24 @@ package org.deeplearning4j.spark.data; +import org.apache.spark.HashPartitioner; +import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; import org.deeplearning4j.spark.BaseSparkTest; import org.deeplearning4j.spark.util.SparkUtils; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Random; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class TestShuffleExamples extends BaseSparkTest { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/data/TestSparkDataUtils.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/data/TestSparkDataUtils.java new file mode 100644 index 000000000..4e9f12dd9 --- /dev/null +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/data/TestSparkDataUtils.java @@ -0,0 +1,33 @@ +/* + * ****************************************************************************** + * * + * * + * * This program and the accompanying materials are made available under the + * * terms of the Apache License, Version 2.0 which is available at + * * https://www.apache.org/licenses/LICENSE-2.0. + * * + * * See the NOTICE file distributed with this work for additional + * * information regarding copyright ownership. + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * * License for the specific language governing permissions and limitations + * * under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * ***************************************************************************** + */ + +package org.deeplearning4j.spark.data; + +import org.deeplearning4j.spark.BaseSparkTest; +import org.junit.jupiter.api.Test; + +public class TestSparkDataUtils extends BaseSparkTest { + + @Test + public void testExport(){ + + } + +} diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/MiniBatchTests.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/MiniBatchTests.java index 44e15c4fc..43c50fdeb 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/MiniBatchTests.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/MiniBatchTests.java @@ -26,10 +26,7 @@ import org.datavec.api.conf.Configuration; import org.datavec.api.records.reader.RecordReader; import org.datavec.api.records.reader.impl.misc.SVMLightRecordReader; import org.deeplearning4j.spark.BaseSparkTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.common.io.ClassPathResource; import org.slf4j.Logger; @@ -39,10 +36,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class MiniBatchTests extends BaseSparkTest { private static final Logger log = LoggerFactory.getLogger(MiniBatchTests.class); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestDataVecDataSetFunctions.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestDataVecDataSetFunctions.java index 14f995772..fad1b4092 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestDataVecDataSetFunctions.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestDataVecDataSetFunctions.java @@ -46,13 +46,9 @@ import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator; import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; import org.deeplearning4j.spark.BaseSparkTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; @@ -66,19 +62,21 @@ import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestDataVecDataSetFunctions extends BaseSparkTest { - + @TempDir + public File testDir; @Test - public void testDataVecDataSetFunction(@TempDir Path testDir) throws Exception { + public void testDataVecDataSetFunction() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } JavaSparkContext sc = getContext(); - File f = testDir.toFile(); + File f = testDir; ClassPathResource cpr = new ClassPathResource("dl4j-spark/imagetest/"); cpr.copyDirectory(f); @@ -185,14 +183,14 @@ public class TestDataVecDataSetFunctions extends BaseSparkTest { } @Test - public void testDataVecSequenceDataSetFunction(@TempDir Path testDir) throws Exception { + public void testDataVecSequenceDataSetFunction() throws Exception { if(Platform.isWindows()) { //Spark tests don't run on windows return; } JavaSparkContext sc = getContext(); //Test Spark record reader functionality vs. local - File dir = testDir.toFile(); + File dir = testDir; ClassPathResource cpr = new ClassPathResource("dl4j-spark/csvsequence/"); cpr.copyDirectory(dir); @@ -247,15 +245,14 @@ public class TestDataVecDataSetFunctions extends BaseSparkTest { } @Test - @Disabled - public void testDataVecSequencePairDataSetFunction(@TempDir Path testDir) throws Exception { + public void testDataVecSequencePairDataSetFunction() throws Exception { if(Platform.isWindows()) { //Spark tests don't run on windows return; } JavaSparkContext sc = getContext(); - File f = new File(testDir.toFile(),"f"); + File f = testDir; ClassPathResource cpr = new ClassPathResource("dl4j-spark/csvsequence/"); cpr.copyDirectory(f); String path = f.getAbsolutePath() + "/*"; @@ -264,7 +261,7 @@ public class TestDataVecDataSetFunctions extends BaseSparkTest { JavaPairRDD toWrite = DataVecSparkUtil.combineFilesForSequenceFile(sc, path, path, pathConverter); - Path p = new File(testDir.toFile(),"dl4j_testSeqPairFn").toPath(); + Path p = new File(testDir,"dl4j_testSeqPairFn").toPath(); p.toFile().deleteOnExit(); String outPath = p.toString() + "/out"; new File(outPath).deleteOnExit(); @@ -347,18 +344,17 @@ public class TestDataVecDataSetFunctions extends BaseSparkTest { } @Test - @Disabled("Permissions issues") - public void testDataVecSequencePairDataSetFunctionVariableLength(@TempDir Path testDir) throws Exception { + public void testDataVecSequencePairDataSetFunctionVariableLength() throws Exception { //Same sort of test as testDataVecSequencePairDataSetFunction() but with variable length time series (labels shorter, align end) if(Platform.isWindows()) { //Spark tests don't run on windows return; } - File dirFeatures = new File(testDir.toFile(),"dirFeatures"); + File dirFeatures = testDir; ClassPathResource cpr = new ClassPathResource("dl4j-spark/csvsequence/"); cpr.copyDirectory(dirFeatures); - File dirLabels = new File(testDir.toFile(),"dirLables"); + File dirLabels = testDir; ClassPathResource cpr2 = new ClassPathResource("dl4j-spark/csvsequencelabels/"); cpr2.copyDirectory(dirLabels); @@ -367,7 +363,7 @@ public class TestDataVecDataSetFunctions extends BaseSparkTest { JavaPairRDD toWrite = DataVecSparkUtil.combineFilesForSequenceFile(sc, dirFeatures.getAbsolutePath(), dirLabels.getAbsolutePath(), pathConverter); - Path p = new File(testDir.toFile(),"dl4j_testSeqPairFnVarLength").toPath(); + Path p = new File(testDir, "dl4j_testSeqPairFnVarLength").toPath(); p.toFile().deleteOnExit(); String outPath = p.toFile().getAbsolutePath() + "/out"; new File(outPath).deleteOnExit(); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestExport.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestExport.java index 42282c53d..b9eef9113 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestExport.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestExport.java @@ -27,10 +27,7 @@ import org.apache.spark.api.java.JavaRDD; import org.deeplearning4j.spark.BaseSparkTest; import org.deeplearning4j.spark.data.BatchAndExportDataSetsFunction; import org.deeplearning4j.spark.data.BatchAndExportMultiDataSetsFunction; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.MultiDataSet; import org.nd4j.linalg.factory.Nd4j; @@ -43,14 +40,15 @@ import java.util.Random; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestExport extends BaseSparkTest { @Test public void testBatchAndExportDataSetsFunction() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } String baseDir = System.getProperty("java.io.tmpdir"); baseDir = FilenameUtils.concat(baseDir, "dl4j_spark_testBatchAndExport/"); baseDir = baseDir.replaceAll("\\\\", "/"); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestPreProcessedData.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestPreProcessedData.java index 554e48332..714c3ffb6 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestPreProcessedData.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/datavec/TestPreProcessedData.java @@ -41,10 +41,7 @@ import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; import org.deeplearning4j.spark.iterator.PortableDataStreamDataSetIterator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -61,15 +58,16 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestPreProcessedData extends BaseSparkTest { @Test public void testPreprocessedData() { //Test _loading_ of preprocessed data + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } int dataSetObjSize = 5; int batchSizePerExecutor = 10; diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/TestKryoWarning.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/TestKryoWarning.java index 33a7d86da..ec2195081 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/TestKryoWarning.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/TestKryoWarning.java @@ -30,16 +30,9 @@ import org.deeplearning4j.spark.api.TrainingMaster; import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +import org.junit.jupiter.api.Test; + public class TestKryoWarning { private static void doTestMLN(SparkConf sparkConf) { @@ -77,7 +70,7 @@ public class TestKryoWarning { } @Test - @Disabled + //@Ignore public void testKryoMessageMLNIncorrectConfig() { //Should print warning message SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest") @@ -88,7 +81,7 @@ public class TestKryoWarning { } @Test - @Disabled + //@Ignore public void testKryoMessageMLNCorrectConfigKryo() { //Should NOT print warning message SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest") @@ -100,7 +93,7 @@ public class TestKryoWarning { } @Test - @Disabled + //@Ignore public void testKryoMessageMLNCorrectConfigNoKryo() { //Should NOT print warning message SparkConf sparkConf = new SparkConf().setMaster("local[*]") @@ -113,7 +106,7 @@ public class TestKryoWarning { @Test - @Disabled + //@Ignore public void testKryoMessageCGIncorrectConfig() { //Should print warning message SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest") @@ -124,7 +117,7 @@ public class TestKryoWarning { } @Test - @Disabled + //@Ignore public void testKryoMessageCGCorrectConfigKryo() { //Should NOT print warning message SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparktest") @@ -136,7 +129,7 @@ public class TestKryoWarning { } @Test - @Disabled + //@Ignore public void testKryoMessageCGCorrectConfigNoKryo() { //Should NOT print warning message SparkConf sparkConf = new SparkConf().setMaster("local[*]") diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitionerTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitionerTest.java index 0d8dbc078..8559d5330 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitionerTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/common/repartition/BalancedPartitionerTest.java @@ -20,17 +20,11 @@ package org.deeplearning4j.spark.impl.common.repartition; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class BalancedPartitionerTest { @@ -40,7 +34,7 @@ public class BalancedPartitionerTest { // the 10 first elements should go in the 1st partition for (int i = 0; i < 10; i++) { int p = bp.getPartition(i); - assertEquals(0, p,"Found wrong partition output " + p + ", not 0"); + assertEquals( 0, p, "Found wrong partition output " + p + ", not 0"); } } @@ -50,7 +44,7 @@ public class BalancedPartitionerTest { // the 10 first elements should go in the 1st partition for (int i = 0; i < 10; i++) { int p = bp.getPartition(i); - assertEquals( 0, p,"Found wrong partition output " + p + ", not 0"); + assertEquals( 0, p, "Found wrong partition output " + p + ", not 0"); } } diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitionerTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitionerTest.java index f4aeff17c..74e8f03be 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitionerTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/common/repartition/HashingBalancedPartitionerTest.java @@ -27,20 +27,14 @@ import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import org.deeplearning4j.spark.BaseSparkTest; import org.deeplearning4j.spark.impl.common.repartition.HashingBalancedPartitioner.LinearCongruentialGenerator; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import scala.Tuple2; import java.util.*; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class HashingBalancedPartitionerTest extends BaseSparkTest { // e.g. we have 3 partitions, with red and blue elements, red is indexed by 0, blue by 1: diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/TestCustomLayer.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/TestCustomLayer.java index fa9656b1b..b3c96333d 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/TestCustomLayer.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/TestCustomLayer.java @@ -30,10 +30,7 @@ import org.deeplearning4j.spark.BaseSparkTest; import org.deeplearning4j.spark.impl.customlayer.layer.CustomLayer; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; @@ -43,15 +40,15 @@ import org.nd4j.linalg.lossfunctions.LossFunctions; import java.util.ArrayList; import java.util.List; import java.util.Random; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Tag(TagNames.CUSTOM_FUNCTIONALITY) + public class TestCustomLayer extends BaseSparkTest { @Test public void testSparkWithCustomLayer() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } //Basic test - checks whether exceptions etc are thrown with custom layers + spark //Custom layers are tested more extensively in dl4j core MultiLayerConfiguration conf = diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayer.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayer.java index 15dda016b..189e1f529 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayer.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/customlayer/layer/CustomLayer.java @@ -32,7 +32,7 @@ import org.deeplearning4j.nn.params.DefaultParamInitializer; import org.deeplearning4j.optimize.api.TrainingListener; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.shade.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Collection; import java.util.Map; diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/graph/TestSparkComputationGraph.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/graph/TestSparkComputationGraph.java index ea37895ab..cc6e5f9ec 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/graph/TestSparkComputationGraph.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/graph/TestSparkComputationGraph.java @@ -47,12 +47,9 @@ import org.deeplearning4j.spark.api.RDDTrainingApproach; import org.deeplearning4j.spark.api.Repartition; import org.deeplearning4j.spark.api.TrainingMaster; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.evaluation.IEvaluation; import org.nd4j.evaluation.classification.Evaluation; import org.nd4j.evaluation.classification.ROC; @@ -75,11 +72,7 @@ import java.util.*; import static org.junit.jupiter.api.Assertions.*; -@Disabled("AB 2019/05/24 - Rarely getting stuck on CI - see issue #7657") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +//@Ignore("AB 2019/05/24 - Rarely getting stuck on CI - see issue #7657") public class TestSparkComputationGraph extends BaseSparkTest { public static ComputationGraph getBasicNetIris2Class() { @@ -221,7 +214,7 @@ public class TestSparkComputationGraph extends BaseSparkTest { } } - @Disabled("AB 2019/05/23 - Failing on CI only - passing locally. Possible precision or threading issue") + //@Ignore("AB 2019/05/23 - Failing on CI only - passing locally. Possible precision or threading issue") public void testSeedRepeatability() throws Exception { ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(Updater.RMSPROP) @@ -294,8 +287,7 @@ public class TestSparkComputationGraph extends BaseSparkTest { } - @Test() - @Timeout(60000L) + @Test @Timeout(60) public void testEvaluationAndRoc() { for( int evalWorkers : new int[]{1, 4, 8}) { DataSetIterator iter = new IrisDataSetIterator(5, 150); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/misc/TestFrozenLayers.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/misc/TestFrozenLayers.java index 48b8535c5..887696af3 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/misc/TestFrozenLayers.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/misc/TestFrozenLayers.java @@ -33,10 +33,7 @@ import org.deeplearning4j.spark.api.RDDTrainingApproach; import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -50,10 +47,7 @@ import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestFrozenLayers extends BaseSparkTest { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestMiscFunctions.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestMiscFunctions.java index 6989ab25d..550ccc9b2 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestMiscFunctions.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestMiscFunctions.java @@ -36,10 +36,7 @@ import org.deeplearning4j.spark.BaseSparkTest; import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; import org.deeplearning4j.spark.impl.multilayer.scoring.VaeReconstructionErrorWithKeyFunction; import org.deeplearning4j.spark.impl.multilayer.scoring.VaeReconstructionProbWithKeyFunction; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.api.DataSet; @@ -54,10 +51,7 @@ import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestMiscFunctions extends BaseSparkTest { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestSparkDl4jMultiLayer.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestSparkDl4jMultiLayer.java index c53302069..c64618557 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestSparkDl4jMultiLayer.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/multilayer/TestSparkDl4jMultiLayer.java @@ -33,10 +33,7 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.spark.BaseSparkTest; import org.deeplearning4j.spark.api.TrainingMaster; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.evaluation.classification.Evaluation; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.buffer.DataType; @@ -54,10 +51,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag public class TestSparkDl4jMultiLayer extends BaseSparkTest { @Override @@ -77,6 +70,10 @@ public class TestSparkDl4jMultiLayer extends BaseSparkTest { @Test public void testEvaluationSimple() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } Nd4j.getRandom().setSeed(12345); for( int evalWorkers : new int[]{1, 4, 8}) { diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestCompareParameterAveragingSparkVsSingleMachine.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestCompareParameterAveragingSparkVsSingleMachine.java index 5f67a22a3..cbe7247bd 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestCompareParameterAveragingSparkVsSingleMachine.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestCompareParameterAveragingSparkVsSingleMachine.java @@ -21,8 +21,6 @@ package org.deeplearning4j.spark.impl.paramavg; import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -42,11 +40,7 @@ import org.deeplearning4j.spark.api.TrainingMaster; import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.resources.Downloader; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -56,42 +50,17 @@ import org.nd4j.linalg.learning.config.RmsProp; import org.nd4j.linalg.learning.config.Sgd; import org.nd4j.linalg.lossfunctions.LossFunctions; -import java.io.File; -import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Slf4j + public class TestCompareParameterAveragingSparkVsSingleMachine { @BeforeEach public void setUp() { //CudaEnvironment.getInstance().getConfiguration().allowMultiGPU(false); } - @SneakyThrows - @BeforeEach - void before() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } private static MultiLayerConfiguration getConf(int seed, IUpdater updater) { diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestJsonYaml.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestJsonYaml.java index 97233b74f..64c984ad7 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestJsonYaml.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestJsonYaml.java @@ -22,17 +22,10 @@ package org.deeplearning4j.spark.impl.paramavg; import org.apache.spark.storage.StorageLevel; import org.deeplearning4j.spark.api.TrainingMaster; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Tag(TagNames.JACKSON_SERDE) + public class TestJsonYaml { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java index 0295144c4..bc1ced484 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/TestSparkMultiLayerParameterAveraging.java @@ -55,15 +55,12 @@ import org.deeplearning4j.spark.impl.graph.SparkComputationGraph; import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.stats.EventStats; import org.deeplearning4j.spark.stats.ExampleCountEventStats; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.evaluation.classification.Evaluation; import org.nd4j.evaluation.classification.ROC; import org.nd4j.evaluation.classification.ROCMultiClass; @@ -88,10 +85,7 @@ import java.util.*; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { public static class TestFn implements Function{ @@ -101,7 +95,8 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { } } - + @TempDir + public File testDir; @Override @@ -434,13 +429,12 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { @Test - @Disabled("Permissions issues on CI") - public void testFitViaStringPaths(@TempDir Path testDir) throws Exception { + public void testFitViaStringPaths() throws Exception { if(Platform.isWindows()) { //Spark tests don't run on windows return; } - Path tempDir = new File(testDir.toFile(),"DL4J-testFitViaStringPaths").toPath(); + Path tempDir = new File(testDir, "DL4J-testFitViaStringPaths").toPath(); File tempDirF = tempDir.toFile(); tempDirF.deleteOnExit(); @@ -502,13 +496,12 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { } @Test - @Disabled("Permissions issues on CI") - public void testFitViaStringPathsSize1(@TempDir Path testDir) throws Exception { + public void testFitViaStringPathsSize1() throws Exception { if(Platform.isWindows()) { //Spark tests don't run on windows return; } - Path tempDir = new File(testDir.toFile(),"DL4J-testFitViaStringPathsSize1").toPath(); + Path tempDir = new File(testDir, "DL4J-testFitViaStringPathsSize1").toPath(); File tempDirF = tempDir.toFile(); tempDirF.deleteOnExit(); @@ -587,14 +580,13 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { @Test - @Disabled("Permissions issues on CI") - public void testFitViaStringPathsCompGraph(@TempDir Path testDir) throws Exception { + public void testFitViaStringPathsCompGraph() throws Exception { if(Platform.isWindows()) { //Spark tests don't run on windows return; } - Path tempDir = new File(testDir.toFile(),"DL4J-testFitViaStringPathsCG").toPath(); - Path tempDir2 = new File(testDir.toFile(),"DL4J-testFitViaStringPathsCG-MDS").toPath(); + Path tempDir = new File(testDir, "DL4J-testFitViaStringPathsCG").toPath(); + Path tempDir2 = new File(testDir, "DL4J-testFitViaStringPathsCG-MDS").toPath(); File tempDirF = tempDir.toFile(); File tempDirF2 = tempDir2.toFile(); tempDirF.deleteOnExit(); @@ -686,7 +678,7 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { @Test - @Disabled("AB 2019/05/23 - Failing on CI only - passing locally. Possible precision or threading issue") + //@Ignore("AB 2019/05/23 - Failing on CI only - passing locally. Possible precision or threading issue") public void testSeedRepeatability() throws Exception { if(Platform.isWindows()) { //Spark tests don't run on windows @@ -862,7 +854,7 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { @Test - @Disabled //Ignored 2019/04/09 - low priority: https://github.com/eclipse/deeplearning4j/issues/6656 + //@Ignore //Ignored 2019/04/09 - low priority: https://github.com/eclipse/deeplearning4j/issues/6656 public void testVaePretrainSimple() { //Simple sanity check on pretraining int nIn = 8; @@ -898,7 +890,7 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { } @Test - @Disabled //Ignored 2019/04/09 - low priority: https://github.com/eclipse/deeplearning4j/issues/6656 + //@Ignore //Ignored 2019/04/09 - low priority: https://github.com/eclipse/deeplearning4j/issues/6656 public void testVaePretrainSimpleCG() { //Simple sanity check on pretraining int nIn = 8; @@ -1046,8 +1038,8 @@ public class TestSparkMultiLayerParameterAveraging extends BaseSparkTest { } - @Test() - @Timeout(120000) + @Test + @Timeout(120) public void testEpochCounter() throws Exception { if(Platform.isWindows()) { //Spark tests don't run on windows diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupportTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupportTest.java index bd993d362..0fdeaaabf 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupportTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/paramavg/util/ExportSupportTest.java @@ -20,18 +20,12 @@ package org.deeplearning4j.spark.impl.paramavg.util; -import com.sun.jna.Platform; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.nd4j.common.resources.Downloader; -import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -42,30 +36,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Ede Meijer */ -@Slf4j public class ExportSupportTest { private static final String FS_CONF = "spark.hadoop.fs.defaultFS"; - @SneakyThrows - @BeforeEach - void before() { - if(Platform.isWindows()) { - File hadoopHome = new File(System.getProperty("java.io.tmpdir"),"hadoop-tmp"); - File binDir = new File(hadoopHome,"bin"); - if(!binDir.exists()) - binDir.mkdirs(); - File outputFile = new File(binDir,"winutils.exe"); - if(!outputFile.exists()) { - log.info("Fixing spark for windows"); - Downloader.download("winutils.exe", - URI.create("https://github.com/cdarlint/winutils/blob/master/hadoop-2.6.5/bin/winutils.exe?raw=true").toURL(), - outputFile,"db24b404d2331a1bec7443336a5171f1",3); - } - - System.setProperty("hadoop.home.dir", hadoopHome.getAbsolutePath()); - } - } - @Test public void testLocalSupported() throws IOException { assertSupported(new SparkConf().setMaster("local").set(FS_CONF, "file:///")); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/stats/TestTrainingStatsCollection.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/stats/TestTrainingStatsCollection.java index 9b5e40709..f4939e369 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/stats/TestTrainingStatsCollection.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/impl/stats/TestTrainingStatsCollection.java @@ -40,10 +40,7 @@ import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingMa import org.deeplearning4j.spark.impl.paramavg.stats.ParameterAveragingTrainingWorkerStats; import org.deeplearning4j.spark.stats.EventStats; import org.deeplearning4j.spark.stats.StatsUtils; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; @@ -52,15 +49,18 @@ import java.io.ByteArrayOutputStream; import java.lang.reflect.Field; import java.util.*; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestTrainingStatsCollection extends BaseSparkTest { @Test public void testStatsCollection() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } int nWorkers = numExecutors(); JavaSparkContext sc = getContext(); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/time/TestTimeSource.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/time/TestTimeSource.java index 7eb17c944..85a73aab4 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/time/TestTimeSource.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/time/TestTimeSource.java @@ -20,17 +20,11 @@ package org.deeplearning4j.spark.time; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestTimeSource { @Test diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/ui/TestListeners.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/ui/TestListeners.java index a66b67cab..6f79d7595 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/ui/TestListeners.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/ui/TestListeners.java @@ -38,10 +38,7 @@ import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer; import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster; import org.deeplearning4j.ui.model.stats.StatsListener; import org.deeplearning4j.ui.model.storage.mapdb.MapDBStatsStorage; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.lossfunctions.LossFunctions; @@ -51,15 +48,15 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag -@Tag(TagNames.UI) + public class TestListeners extends BaseSparkTest { @Test public void testStatsCollection() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } JavaSparkContext sc = getContext(); int nExecutors = numExecutors(); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/MLLIbUtilTest.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/MLLIbUtilTest.java index 23674df13..ef7c0788f 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/MLLIbUtilTest.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/MLLIbUtilTest.java @@ -27,10 +27,7 @@ import org.apache.spark.mllib.linalg.Matrix; import org.apache.spark.mllib.regression.LabeledPoint; import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator; import org.deeplearning4j.spark.BaseSparkTest; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; @@ -42,10 +39,7 @@ import java.util.List; import java.util.Random; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class MLLIbUtilTest extends BaseSparkTest { private static final Logger log = LoggerFactory.getLogger(MLLIbUtilTest.class); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestRepartitioning.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestRepartitioning.java index c51f6c15f..c83282547 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestRepartitioning.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestRepartitioning.java @@ -29,10 +29,8 @@ import org.deeplearning4j.spark.api.Repartition; import org.deeplearning4j.spark.api.RepartitionStrategy; import org.deeplearning4j.spark.impl.common.CountPartitionsFunction; import org.deeplearning4j.spark.impl.repartitioner.DefaultRepartitioner; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import scala.Tuple2; import java.util.ArrayList; @@ -40,11 +38,10 @@ import java.util.Arrays; import java.util.List; import java.util.Random; -import static org.junit.jupiter.api.Assertions.*; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class TestRepartitioning extends BaseSparkTest { @Override @@ -54,6 +51,10 @@ public class TestRepartitioning extends BaseSparkTest { @Test public void testRepartitioning() { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } List list = new ArrayList<>(); for (int i = 0; i < 1000; i++) { list.add(String.valueOf(i)); @@ -191,7 +192,7 @@ public class TestRepartitioning extends BaseSparkTest { new Tuple2<>(4,34), new Tuple2<>(5,35), new Tuple2<>(6,34)); - assertEquals(initialExpected, partitionCounts); + Assertions.assertEquals(initialExpected, partitionCounts); JavaRDD afterRepartition = SparkUtils.repartitionBalanceIfRequired(initial.values(), Repartition.Always, 2, 112); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestValidation.java b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestValidation.java index 04201aec8..21ba9fc23 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestValidation.java +++ b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/java/org/deeplearning4j/spark/util/TestValidation.java @@ -26,33 +26,33 @@ import org.deeplearning4j.spark.BaseSparkTest; import org.deeplearning4j.spark.util.data.SparkDataValidation; import org.deeplearning4j.spark.util.data.ValidationResult; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.MultiDataSet; import org.nd4j.linalg.factory.Nd4j; import java.io.File; -import java.nio.file.Path; import java.util.Arrays; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SPARK) -@Tag(TagNames.DIST_SYSTEMS) -@NativeTag + public class TestValidation extends BaseSparkTest { + @TempDir + public File folder; + @Test - public void testDataSetValidation(@TempDir Path folder) throws Exception { - File f = folder.toFile(); + public void testDataSetValidation() throws Exception { + if(Platform.isWindows()) { + //Spark tests don't run on windows + return; + } + File f = folder; for( int i = 0; i < 3; i++ ) { DataSet ds = new DataSet(Nd4j.create(1,10), Nd4j.create(1,10)); @@ -114,12 +114,12 @@ public class TestValidation extends BaseSparkTest { } @Test - public void testMultiDataSetValidation(@TempDir Path folder) throws Exception { + public void testMultiDataSetValidation() throws Exception { if(Platform.isWindows()) { //Spark tests don't run on windows return; } - File f = folder.toFile(); + File f = folder; for( int i = 0; i < 3; i++ ) { MultiDataSet ds = new MultiDataSet(Nd4j.create(1,10), Nd4j.create(1,10)); diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/resources/log4j.properties b/deeplearning4j/deeplearning4j-scaleout/spark/dl4j-spark/src/test/resources/log4j.properties old mode 100755 new mode 100644 diff --git a/deeplearning4j/deeplearning4j-scaleout/spark/pom.xml b/deeplearning4j/deeplearning4j-scaleout/spark/pom.xml index 0ad28bf14..f95f4d817 100644 --- a/deeplearning4j/deeplearning4j-scaleout/spark/pom.xml +++ b/deeplearning4j/deeplearning4j-scaleout/spark/pom.xml @@ -26,12 +26,12 @@ 4.0.0 - org.deeplearning4j + net.brutex.ai deeplearning4j-scaleout 1.0.0-SNAPSHOT - spark_2.11 + spark_2.12 pom Spark parent @@ -45,32 +45,29 @@ 2.1.0 - - 2.11.12 - 2.11 - org.nd4j - jackson - ${nd4j.version} + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} org.apache.spark - spark-mllib_2.11 + spark-mllib_2.12 ${spark.version} org.scala-lang scala-library - ${scala.version} + 2.12.14 org.scala-lang scala-reflect - ${scala.version} + 2.12.14 com.typesafe @@ -79,7 +76,7 @@ org.apache.spark - spark-core_2.11 + spark-core_2.12 ${spark.version} @@ -129,15 +126,6 @@ - - get-cpu-count - - cpu-count - - - system.numCores - - @@ -167,7 +155,7 @@ - ${scala.version} + 2.12.14 -deprecation -explaintypes @@ -180,7 +168,7 @@ org.scalamacros - paradise_${scala.version} + paradise_2.12 ${scala.macros.version} @@ -188,31 +176,4 @@ - - - - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - - diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/pom.xml b/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/pom.xml deleted file mode 100644 index e5b5254d0..000000000 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-components/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-ui-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-ui-components - - - - - org.nd4j - jackson - ${nd4j.version} - - - org.freemarker - freemarker - ${freemarker.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - commons-io - commons-io - ${commonsio.version} - - - org.nd4j - nd4j-common - ${nd4j.version} - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/pom.xml b/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/pom.xml deleted file mode 100644 index 040011ab8..000000000 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-model/pom.xml +++ /dev/null @@ -1,113 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-ui-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-ui-model - - deeplearning4j-ui-model - - - - ch.qos.logback - logback-classic - test - - - org.deeplearning4j - deeplearning4j-core - ${project.version} - - - - org.nd4j - nd4j-api - ${nd4j.version} - - - org.nd4j - nd4j-native-api - ${nd4j.version} - - - - org.agrona - Agrona - ${agrona.version} - - - - org.mapdb - mapdb - ${mapdb.version} - - - - org.xerial - sqlite-jdbc - ${sqlite.version} - - - - javax.annotation - javax.annotation-api - ${javax.annotation-api.version} - provided - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-standalone/pom.xml b/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-standalone/pom.xml deleted file mode 100644 index b02387920..000000000 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui-standalone/pom.xml +++ /dev/null @@ -1,150 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-ui-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-ui-standalone - - - - org.deeplearning4j - deeplearning4j-ui - ${deeplearning4j.version} - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - - - reference.conf - - - - org.deeplearning4j.ui.play.PlayUIServer - - - - - - - - - false - true - true - - - - *:* - - org/datanucleus/** - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - empty-javadoc-jar - package - - jar - - - javadoc - ${basedir}/javadoc - - - - empty-sources-jar - package - - jar - - - sources - ${basedir}/src - - - - - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/pom.xml b/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/pom.xml deleted file mode 100644 index aa0271686..000000000 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-ui/pom.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-ui-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-ui - - deeplearning4j-ui - - - 1.8 - 1.8 - - - - - org.deeplearning4j - deeplearning4j-vertx - ${project.version} - - - commons-io - commons-io - ${commonsio.version} - - - org.deeplearning4j - deeplearning4j-nlp - ${project.version} - - - org.junit.jupiter - junit-jupiter-api - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - - - org.deeplearning4j - deeplearning4j-ui-model - ${project.version} - - - ch.qos.logback - logback-classic - test - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - diff --git a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/pom.xml b/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/pom.xml deleted file mode 100644 index a9df8ea56..000000000 --- a/deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/pom.xml +++ /dev/null @@ -1,435 +0,0 @@ - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-ui-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-vertx - - - 1.8 - 1.8 - - - - - io.vertx - vertx-core - ${vertx.version} - - - io.vertx - vertx-web - ${vertx.version} - - - org.deeplearning4j - deeplearning4j-core - ${project.version} - - - org.deeplearning4j - deeplearning4j-ui-model - ${project.version} - - - ch.qos.logback - logback-classic - test - - - org.freemarker - freemarker - ${freemarker.version} - - - com.beust - jcommander - ${jcommander.version} - - - jakarta.xml.bind - jakarta.xml.bind-api - 2.3.2 - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - - org.webjars.npm - babel__polyfill - 7.4.4 - - - org.webjars.npm - coreui__coreui - 2.1.9 - - - org.webjars.npm - coreui__coreui-plugin-npm-postinstall - - - - - org.webjars.npm - coreui__icons - 0.3.0 - - - org.webjars.npm - jquery - 3.4.1 - - - org.webjars.bower - popper.js - 1.12.9 - - - org.webjars.npm - bootstrap - - - org.webjars - jquery - 2.2.0 - - - org.webjars - jquery-migrate - 1.2.1 - - - org.webjars - jquery-ui - 1.10.2 - - - org.webjars - modernizr - - 2.8.3-1 - - - org.webjars - jquery-cookie - 1.4.1-1 - - - org.webjars - fullcalendar - 1.6.4 - - - org.webjars - excanvas - 3 - - - org.webjars.npm - cytoscape - 3.3.3 - - - org.webjars.bower - cytoscape-dagre - 2.1.0 - - - org.webjars.npm - dagre - 0.8.4 - - - org.webjars.npm - cytoscape-cola - 2.3.0 - - - org.webjars.npm - cytoscape-cose-bilkent - 4.0.0 - - - org.webjars.npm - cytoscape-euler - 1.2.1 - - - org.webjars.npm - cytoscape-klay - 3.1.2 - - - org.webjars.npm - klayjs - 0.4.1 - - - org.webjars.npm - cytoscape-spread - 3.0.0 - - - org.webjars.npm - weaverjs - 1.2.0 - - - org.webjars - retinajs - 0.0.2 - - - org.webjars - flot - 0.8.3 - - - org.webjars - chosen - 0.9.8 - - - org.webjars - uniform - 2.1.2-1 - - - org.webjars - noty - 2.2.2 - - - org.webjars - jquery-raty - 2.5.2 - - - org.webjars - imagesloaded - 2.1.1 - - - org.webjars - masonry - 3.1.5 - - - org.webjars - jquery.sparkline - 2.1.2 - - - org.webjars - jquery-knob - 1.2.2 - - - org.webjars - datatables - 1.9.4 - - - org.webjars - jquery-ui-touch-punch - 0.2.2 - - - org.webjars - d3js - 3.3.5 - - - org.webjars - bootstrap-notify - 3.1.3-1 - - - org.webjars.npm - github-com-jboesch-Gritter - 1.7.4 - - - - org.webjars.bowergithub.stenin-nikita - open-sans - 0.1.3 - - - org.webjars - font-awesome - 3.0.2 - - - org.webjars - bootstrap-glyphicons - bdd2cbfba0 - - - - org.webjars.npm - flatbuffers - 1.9.0 - - - - - - - - - - org.webjars.npm - core-js - 2.6.5 - - - org.webjars.npm - regenerator-runtime - 0.13.2 - - - - org.webjars.npm - bootstrap - 4.3.1 - - - - org.webjars.npm - heap - 0.2.6 - - - org.webjars.npm - lodash.debounce - 4.0.8 - - - - org.webjars.npm - graphlib - 2.1.7 - - - org.webjars.bower - cytoscape - 3.2.5 - - - org.webjars.bower - dagre - 0.7.4 - - - org.webjars.bower - graphlib - 1.0.7 - - - org.webjars.bower - lodash - 3.10.1 - - - org.webjars.npm - lodash - 4.17.11 - - - org.webjars.npm - webcola - 3.3.8 - - - org.webjars.npm - d3-dispatch - 1.0.5 - - - org.webjars.npm - d3-drag - 1.2.3 - - - org.webjars.npm - d3-selection - 1.4.0 - - - org.webjars.npm - d3-timer - 1.0.9 - - - org.webjars.npm - klayjs - 0.4.1 - - - org.webjars.npm - weaverjs - 1.2.0 - - - org.webjars - explorercanvas - r3-1 - - - org.webjars - bootstrap - 2.2.2-1 - - - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - - \ No newline at end of file diff --git a/deeplearning4j/deeplearning4j-ui-parent/pom.xml b/deeplearning4j/deeplearning4j-ui-parent/pom.xml deleted file mode 100644 index de20ba184..000000000 --- a/deeplearning4j/deeplearning4j-ui-parent/pom.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-ui-parent - pom - - - deeplearning4j-ui - deeplearning4j-ui-components - deeplearning4j-ui-model - deeplearning4j-vertx - - - - - ui-jar - - deeplearning4j-ui-standalone - - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - false - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - - - diff --git a/deeplearning4j/deeplearning4j-zoo/nd4j-native.properties b/deeplearning4j/deeplearning4j-zoo/nd4j-native.properties deleted file mode 100644 index 5a5f8fb3c..000000000 --- a/deeplearning4j/deeplearning4j-zoo/nd4j-native.properties +++ /dev/null @@ -1,38 +0,0 @@ -# -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ -# - -real.class.double = org.nd4j.linalg.cpu.NDArray -shapeinfoprovider = org.nd4j.linalg.cpu.nativecpu.DirectShapeInfoProvider -constantsprovider = org.nd4j.linalg.cpu.nativecpu.cache.ConstantBuffersCache -affinitymanager = org.nd4j.linalg.cpu.nativecpu.CpuAffinityManager -memorymanager = org.nd4j.linalg.cpu.nativecpu.CpuMemoryManager -dtype = float -blas.ops = org.nd4j.linalg.cpu.nativecpu.BlasWrapper - -native.ops= org.nd4j.nativeblas.Nd4jCpu -ndarrayfactory.class = org.nd4j.linalg.cpu.nativecpu.CpuNDArrayFactory -ndarray.order = c -resourcemanager_state = false -databufferfactory = org.nd4j.linalg.cpu.nativecpu.buffer.DefaultDataBufferFactory -workspacemanager = org.nd4j.linalg.cpu.nativecpu.workspace.CpuWorkspaceManager -alloc = javacpp -opexec= org.nd4j.linalg.cpu.nativecpu.ops.NativeOpExecutioner -opexec.mode= native -random=org.nd4j.linalg.cpu.nativecpu.rng.CpuNativeRandom diff --git a/deeplearning4j/deeplearning4j-zoo/pom.xml b/deeplearning4j/deeplearning4j-zoo/pom.xml deleted file mode 100644 index 28627284f..000000000 --- a/deeplearning4j/deeplearning4j-zoo/pom.xml +++ /dev/null @@ -1,119 +0,0 @@ - - - - - - 4.0.0 - - - org.deeplearning4j - deeplearning4j-parent - 1.0.0-SNAPSHOT - - - deeplearning4j-zoo - - - - org.slf4j - slf4j-api - - - org.nd4j - nd4j-api - ${nd4j.version} - - - org.deeplearning4j - deeplearning4j-nn - ${project.version} - - - org.deeplearning4j - deeplearning4j-common - ${project.version} - - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - ch.qos.logback - logback-classic - test - - - org.deeplearning4j - deeplearning4j-core - ${deeplearning4j.version} - test - - - org.deeplearning4j - deeplearning4j-common-tests - ${project.version} - test - - - - - - nd4j-tests-cpu - - - - nd4j-tests-cuda - - false - - - - org.deeplearning4j - dl4j-test-resources - ${dl4j-test-resources.version} - test - - - org.nd4j - nd4j-cuda-11.0 - ${nd4j.version} - test - - - org.deeplearning4j - deeplearning4j-cuda-11.0 - ${nd4j.version} - - - - - diff --git a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/MiscTests.java b/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/MiscTests.java deleted file mode 100644 index 7e7decac8..000000000 --- a/deeplearning4j/deeplearning4j-zoo/src/test/java/org/deeplearning4j/zoo/MiscTests.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * ****************************************************************************** - * * - * * - * * This program and the accompanying materials are made available under the - * * terms of the Apache License, Version 2.0 which is available at - * * https://www.apache.org/licenses/LICENSE-2.0. - * * - * * See the NOTICE file distributed with this work for additional - * * information regarding copyright ownership. - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * * License for the specific language governing permissions and limitations - * * under the License. - * * - * * SPDX-License-Identifier: Apache-2.0 - * ***************************************************************************** - */ - -package org.deeplearning4j.zoo; - -import org.deeplearning4j.BaseDL4JTest; -import org.deeplearning4j.nn.conf.layers.OutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.transferlearning.TransferLearning; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.zoo.model.VGG16; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.io.File; -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag -@Tag(TagNames.LONG_TEST) -@Tag(TagNames.LARGE_RESOURCES) -public class MiscTests extends BaseDL4JTest { - - @Override - public long getTimeoutMilliseconds() { - return Long.MAX_VALUE; - } - - @Test - public void testTransferVGG() throws Exception { - DataSet ds = new DataSet(); - ds.setFeatures(Nd4j.create(1, 3, 224, 224)); - ds.setLabels(Nd4j.create(1, 2)); - - ComputationGraph model = (ComputationGraph)( - VGG16.builder().build() - .initPretrained(PretrainedType.IMAGENET)); -// System.out.println(model.summary()); - - ComputationGraph transferModel = new TransferLearning.GraphBuilder(model) - .setFeatureExtractor("fc2") - .removeVertexKeepConnections("predictions") - .addLayer("predictions", - new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD) - .nIn(4096).nOut(2) - .weightInit(WeightInit.XAVIER) - .activation(Activation.SOFTMAX).build(), "fc2") - .build(); - -// System.out.println(transferModel.summary()); -// System.out.println("Fitting"); - transferModel.fit(ds); - - ComputationGraph g2 = TestUtils.testModelSerialization(transferModel); - g2.fit(ds); - } - -} diff --git a/deeplearning4j/dl4j-integration-tests/pom.xml b/deeplearning4j/dl4j-integration-tests/pom.xml index a491f38a7..bc595b5e7 100644 --- a/deeplearning4j/dl4j-integration-tests/pom.xml +++ b/deeplearning4j/dl4j-integration-tests/pom.xml @@ -24,7 +24,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - org.deeplearning4j + net.brutex.ai deeplearning4j-parent 1.0.0-SNAPSHOT @@ -33,10 +33,6 @@ dl4j-integration-tests - - 1.8 - 1.8 - @@ -44,49 +40,43 @@ slf4j-api - org.nd4j + net.brutex.ai nd4j-api - ${nd4j.version} + ${project.version} - org.deeplearning4j + net.brutex.ai deeplearning4j-core - ${deeplearning4j.version} + ${project.version} - org.deeplearning4j + net.brutex.ai deeplearning4j-zoo ${project.version} - org.deeplearning4j + net.brutex.ai deeplearning4j-parallel-wrapper ${project.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - ch.qos.logback logback-classic test - org.deeplearning4j + net.brutex.ai deeplearning4j-common-tests ${project.version} test + + net.brutex.ai + nd4j-common + ${project.version} + test + + @@ -107,20 +97,10 @@ org.apache.maven.plugins maven-deploy-plugin - ${maven-deploy-plugin.version} true - - - - nd4j-tests-cpu - - - nd4j-tests-cuda - - \ No newline at end of file diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java index 3c6d81e9a..7c4bcc9ac 100644 --- a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java +++ b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestBaselineGenerator.java @@ -45,7 +45,7 @@ import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.common.primitives.Pair; -import org.nd4j.shade.guava.io.Files; +import com.google.common.io.Files; import java.io.*; import java.nio.charset.StandardCharsets; diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java index 4b5df6ead..43c112d0a 100644 --- a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java +++ b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestRunner.java @@ -68,13 +68,12 @@ import org.nd4j.linalg.indexing.conditions.Conditions; import org.nd4j.linalg.ops.transforms.Transforms; import org.nd4j.common.primitives.Pair; import org.nd4j.common.resources.Resources; -import org.nd4j.shade.guava.collect.ImmutableSet; -import org.nd4j.shade.guava.reflect.ClassPath; +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.ClassPath; import java.io.*; import java.lang.reflect.Modifier; import java.nio.charset.StandardCharsets; -import java.nio.file.Path; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; @@ -156,7 +155,7 @@ public class IntegrationTestRunner { evaluationClassesSeen = new HashMap<>(); } - public static void runTest(TestCase tc, Path testDir) throws Exception { + public static void runTest(TestCase tc, File testDir) throws Exception { BaseDL4JTest.skipUnlessIntegrationTests(); //Tests will ONLY be run if integration test profile is enabled. //This could alternatively be done via maven surefire configuration @@ -164,10 +163,10 @@ public class IntegrationTestRunner { log.info("Starting test case: {} - type = {}", tc.getTestName(), modelType); long start = System.currentTimeMillis(); - File workingDir = new File(testDir.toFile(),"workingDir"); + File workingDir = testDir; tc.initialize(workingDir); - File testBaseDir = new File(testDir.toFile(),"baseDir"); + File testBaseDir = testDir; // new ClassPathResource("dl4j-integration-tests/" + tc.getTestName()).copyDirectory(testBaseDir); Resources.copyDirectory((modelType == ModelType.SAMEDIFF ? "samediff-integration-tests/" : "dl4j-integration-tests/") + tc.getTestName(), testBaseDir); @@ -189,8 +188,8 @@ public class IntegrationTestRunner { MultiLayerNetwork loaded = MultiLayerNetwork.load(savedModel, true); assertEquals(loaded.getLayerWiseConfigurations(), mln.getLayerWiseConfigurations(), "Configs not equal"); - assertEquals(loaded.params(), mln.params(), "Params not equal"); - assertEquals(loaded.paramTable(), mln.paramTable(), "Param table not equal"); + assertEquals( loaded.params(), mln.params(), "Params not equal"); + assertEquals( loaded.paramTable(), mln.paramTable(), "Param table not equal"); } else if(config instanceof ComputationGraphConfiguration ){ ComputationGraphConfiguration cgc = (ComputationGraphConfiguration) config; cg = new ComputationGraph(cgc); @@ -198,8 +197,8 @@ public class IntegrationTestRunner { m = cg; ComputationGraph loaded = ComputationGraph.load(savedModel, true); - assertEquals(loaded.getConfiguration(), cg.getConfiguration(), "Configs not equal"); - assertEquals(loaded.params(), cg.params(), "Params not equal"); + assertEquals(loaded.getConfiguration(), cg.getConfiguration(), "Configs not equal" ); + assertEquals( loaded.params(), cg.params(), "Params not equal"); assertEquals(loaded.paramTable(), cg.paramTable(), "Param table not equal"); } else if(config instanceof SameDiff){ sd = (SameDiff)config; @@ -257,7 +256,7 @@ public class IntegrationTestRunner { INDArray predictionExceedsRE = exceedsRelError(outSaved, out, tc.getMaxRelativeErrorOutput(), tc.getMinAbsErrorOutput()); int countExceeds = predictionExceedsRE.sumNumber().intValue(); - assertEquals(0, countExceeds,"Predictions do not match saved predictions - output"); + assertEquals( 0, countExceeds, "Predictions do not match saved predictions - output"); } } else if(modelType == ModelType.CG){ for (Pair p : inputs) { @@ -275,7 +274,7 @@ public class IntegrationTestRunner { for( int i=0; i 0) { logFailedParams(20, "Gradient", layers, gradExceedsRE, gradientFlatSaved, gradientFlat); } - assertEquals( 0, count,"Saved flattened gradients: not equal (using relative error)"); + assertEquals( 0, count, "Saved flattened gradients: not equal (using relative error)"); } //Load the gradient table: @@ -368,7 +367,7 @@ public class IntegrationTestRunner { INDArray gradExceedsRE = exceedsRelError(loaded, now, tc.getMaxRelativeErrorGradients(), tc.getMinAbsErrorGradients()); int count = gradExceedsRE.sumNumber().intValue(); - assertEquals(0, count,"Gradients: not equal (using relative error) for parameter: " + key); + assertEquals( 0, count, "Gradients: not equal (using relative error) for parameter: " + key); } } @@ -411,7 +410,7 @@ public class IntegrationTestRunner { if(count > 0){ logFailedParams(20, "Parameter", layers, exceedsRelError, expParams, paramsPostTraining); } - assertEquals(0, count,"Number of parameters exceeding relative error"); + assertEquals( 0, count, "Number of parameters exceeding relative error"); //Set params to saved ones - to avoid accumulation of roundoff errors causing later failures... m.setParams(expParams); @@ -497,7 +496,7 @@ public class IntegrationTestRunner { String[] s = FileUtils.readFileToString(f, StandardCharsets.UTF_8).split(","); if(tc.isTestTrainingCurves()) { - assertEquals(s.length, scores.length,"Different number of scores"); + assertEquals( s.length, scores.length, "Different number of scores"); boolean pass = true; for (int i = 0; i < s.length; i++) { @@ -522,7 +521,7 @@ public class IntegrationTestRunner { if (count > 0) { logFailedParams(20, "Parameter", layers, z, paramsExp, m.params()); } - assertEquals( 0, count,"Number of params exceeded max relative error"); + assertEquals( 0, count, "Number of params exceeded max relative error"); } else { File dir = new File(testBaseDir, IntegrationTestRunner.PARAMS_POST_TRAIN_SAMEDIFF_DIR); for(SDVariable v : sd.variables()){ @@ -536,7 +535,7 @@ public class IntegrationTestRunner { if (count > 0) { logFailedParams(20, "Parameter: " + v.name(), layers, z, exp, paramNow); } - assertEquals(0, count,"Number of params exceeded max relative error for parameter: \"" + v.name() + "\""); + assertEquals( 0, count, "Number of params exceeded max relative error for parameter: \"" + v.name() + "\""); } } } @@ -583,7 +582,7 @@ public class IntegrationTestRunner { } - assertEquals(e, evals[i], "Evaluation not equal: " + evals[i].getClass()); + assertEquals( e, evals[i], "Evaluation not equal: " + evals[i].getClass()); //Evaluation coverage information: evaluationClassesSeen.put(evals[i].getClass(), evaluationClassesSeen.getOrDefault(evals[i].getClass(), 0) + 1); @@ -598,8 +597,8 @@ public class IntegrationTestRunner { { log.info("Testing model serialization"); - File f = new File(testDir.toFile(),"test-file"); - f.deleteOnExit(); + File f = new File(testDir, UUID.randomUUID().toString()); + f.delete(); if (modelType == ModelType.MLN) { ModelSerializer.writeModel(m, f, true); @@ -809,8 +808,8 @@ public class IntegrationTestRunner { } for(org.deeplearning4j.nn.api.Layer l : layers){ - assertEquals(expEpoch, l.getEpochCount(),"Epoch count"); - assertEquals(expIter, l.getIterationCount(),"Iteration count"); + assertEquals( expEpoch, l.getEpochCount(), "Epoch count"); + assertEquals( expIter, l.getIterationCount(), "Iteration count"); } } @@ -866,7 +865,7 @@ public class IntegrationTestRunner { } } - public static void printCoverageInformation() { + public static void printCoverageInformation(){ log.info("||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||"); diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java index acb1f060c..ebf4a9442 100644 --- a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java +++ b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsDL4J.java @@ -23,31 +23,25 @@ package org.deeplearning4j.integration; import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.integration.testcases.dl4j.*; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import java.nio.file.Path; +import java.io.File; -//@Disabled("AB - 2019/05/27 - Integration tests need to be updated") -@Tag(TagNames.FILE_IO) -@Tag(TagNames.DL4J_OLD_API) -@NativeTag +////@Ignore("AB - 2019/05/27 - Integration tests need to be updated") public class IntegrationTestsDL4J extends BaseDL4JTest { - @TempDir - static Path testDir; @Override public long getTimeoutMilliseconds() { return 300_000L; } + @TempDir + public File testDir; - @AfterEach + @AfterAll public static void afterClass(){ IntegrationTestRunner.printCoverageInformation(); } diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java index 6dd41c698..cc86b5cb3 100644 --- a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java +++ b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/IntegrationTestsSameDiff.java @@ -23,28 +23,21 @@ import org.deeplearning4j.BaseDL4JTest; import org.deeplearning4j.integration.testcases.samediff.SameDiffCNNCases; import org.deeplearning4j.integration.testcases.samediff.SameDiffMLPTestCases; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.nd4j.common.tests.tags.NativeTag; -import org.nd4j.common.tests.tags.TagNames; -import java.nio.file.Path; +import java.io.File; + -@Tag(TagNames.FILE_IO) -@Tag(TagNames.SAMEDIFF) -@NativeTag public class IntegrationTestsSameDiff extends BaseDL4JTest { - @TempDir - static Path testDir; - @Override public long getTimeoutMilliseconds() { return 300_000L; } - + @TempDir + public File testDir; @Test diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java index c67546a23..025f1ab54 100644 --- a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java +++ b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/dl4j/RNNTestCases.java @@ -29,7 +29,7 @@ import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.dataset.adapter.MultiDataSetIteratorAdapter; import org.nd4j.linalg.dataset.api.preprocessor.CompositeMultiDataSetPreProcessor; import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.shade.guava.io.Files; +import com.google.common.io.Files; import org.deeplearning4j.integration.TestCase; import org.deeplearning4j.integration.testcases.dl4j.misc.CharacterIterator; import org.datavec.api.records.reader.SequenceRecordReader; diff --git a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java index b163acd3d..e3daa2126 100644 --- a/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java +++ b/deeplearning4j/dl4j-integration-tests/src/test/java/org/deeplearning4j/integration/testcases/samediff/SameDiffRNNTestCases.java @@ -52,7 +52,7 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.indexing.NDArrayIndex; import org.nd4j.linalg.learning.config.Adam; import org.nd4j.common.resources.Resources; -import org.nd4j.shade.guava.io.Files; +import com.google.common.io.Files; import java.io.File; import java.util.ArrayList; diff --git a/deeplearning4j/dl4j-integration-tests/src/test/resources/junit-platform.properties b/deeplearning4j/dl4j-integration-tests/src/test/resources/junit-platform.properties deleted file mode 100644 index 8ec0fbcee..000000000 --- a/deeplearning4j/dl4j-integration-tests/src/test/resources/junit-platform.properties +++ /dev/null @@ -1,25 +0,0 @@ -# -# /* -# * ****************************************************************************** -# * * -# * * -# * * This program and the accompanying materials are made available under the -# * * terms of the Apache License, Version 2.0 which is available at -# * * https://www.apache.org/licenses/LICENSE-2.0. -# * * -# * * See the NOTICE file distributed with this work for additional -# * * information regarding copyright ownership. -# * * Unless required by applicable law or agreed to in writing, software -# * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * * License for the specific language governing permissions and limitations -# * * under the License. -# * * -# * * SPDX-License-Identifier: Apache-2.0 -# * ***************************************************************************** -# */ -# -# - -junit.jupiter.execution.parallel.enabled = true -junit.jupiter.execution.parallel.mode.default = concurrent \ No newline at end of file diff --git a/deeplearning4j/pom.xml b/deeplearning4j/pom.xml index cd886ef69..d4f2ead30 100644 --- a/deeplearning4j/pom.xml +++ b/deeplearning4j/pom.xml @@ -26,26 +26,20 @@ 4.0.0 - org.deeplearning4j + net.brutex.ai deeplearning4j 1.0.0-SNAPSHOT - org.deeplearning4j + net.brutex.ai deeplearning4j-parent pom - DeepLearning4j + deeplearning4j-parent DeepLearning for java - - scm:git://github.com:eclipse/deeplearning4j.git - scm:git:git@github.com:eclipse/deeplearning4j.git - - git@github.com:eclipse/deeplearning4j.git - HEAD - + deeplearning4j-core @@ -55,13 +49,16 @@ deeplearning4j-nlp-parent deeplearning4j-nn deeplearning4j-dataimport-solrj + deeplearning4j-manifold deeplearning4j-modelimport deeplearning4j-modelexport-solr + deeplearning4j-nearestneighbors-parent deeplearning4j-zoo deeplearning4j-data dl4j-integration-tests deeplearning4j-common deeplearning4j-common-tests + deeplearning4j-remote @@ -91,102 +88,15 @@ slf4j-api ${slf4j.version} - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.vintage - junit-vintage-engine - ${junit.version} - test - - - - org.junit.jupiter - junit-jupiter-api - - - org.junit.jupiter - junit-jupiter-engine - - - org.junit.jupiter - junit-jupiter-params - - - org.projectlombok - lombok - ${lombok.version} - provided - - - org.nd4j - nd4j-common-tests - 1.0.0-SNAPSHOT - test - - - com.google.android - android - 4.1.1.4 - test - + - - - org.apache.maven.wagon - wagon-http - 2.9 - - - org.kuali.maven.wagons - maven-s3-wagon - 1.2.1 - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.apache.maven.plugins - maven-enforcer-plugin - ${maven-enforcer-plugin.version} - - - test - enforce-choice-of-nd4j-test-backend - - enforce - - - ${skipBackendChoice} - - - nd4j-tests-cpu,nd4j-tests-cuda - false - - - true - - - - - - org.apache.maven.plugins - maven-compiler-plugin - com.lewisd lint-maven-plugin @@ -228,19 +138,11 @@ deeplearning4j-modelimport deeplearning4j-modelexport-solr deeplearning4j-zoo + deeplearning4j-nearestneighbors-parent - - - pl.project13.maven - git-commit-id-plugin - - - - org.codehaus.mojo - build-helper-maven-plugin - + @@ -269,28 +171,136 @@ deeplearning4j-cuda - - nd4j-tests-cpu + test-nd4j-native false - org.deeplearning4j + net.brutex.ai dl4j-test-resources ${dl4j-test-resources.version} test - org.nd4j + net.brutex.ai nd4j-native - ${nd4j.version} + ${project.version} test - + + + + org.apache.maven.plugins + maven-surefire-plugin + true + + + net.brutex.ai + nd4j-native + ${project.version} + + + + true + + + src/test/java + + *.java + **/*.java + **/Test*.java + **/*Test.java + **/*TestCase.java + + junit:junit + + + org.org.nd4j.linalg.cpu.nativecpu.CpuBackend + + + org.org.nd4j.linalg.cpu.nativecpu.CpuBackend + + + + " + + + + + + + + test-nd4j-cuda-11.2 + + false + + + + net.brutex.ai + dl4j-test-resources + ${dl4j-test-resources.version} + test + + + net.brutex.ai + nd4j-cuda-${cuda.version} + ${project.version} + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.surefire + surefire-junit47 + 2.19.1 + + + + + + src/test/java + + *.java + **/*.java + **/Test*.java + **/*Test.java + **/*TestCase.java + + junit:junit + + + org.org.nd4j.linalg.jcublas.JCublasBackend + + + org.org.nd4j.linalg.jcublas.JCublasBackend + + + + -Djava.library.path="${nd4j.basedir}/nd4j-backends/nd4j-backend-impls/nd4j-cuda/target/classes" + + + + + diff --git a/diff.txt b/diff.txt deleted file mode 100644 index 557262399..000000000 --- a/diff.txt +++ /dev/null @@ -1,782 +0,0 @@ -2874,2876d2873 -< deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/target/classes/templates/TrainingModel.html.ftl -< deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/target/classes/templates/TrainingOverview.html.ftl -< deeplearning4j/deeplearning4j-ui-parent/deeplearning4j-vertx/target/classes/templates/TrainingSystem.html.ftl -3171,3185d3167 -< libnd4j/blasbuild/cpu/cpu_features-src/include/cpu_features_cache_info.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/cpu_features_macros.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/cpuinfo_aarch64.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/cpuinfo_arm.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/cpuinfo_mips.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/cpuinfo_ppc.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/cpuinfo_x86.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/internal/bit_utils.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/internal/cpuid_x86.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/internal/filesystem.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/internal/hwcaps.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/internal/stack_line_reader.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/internal/string_view.h -< libnd4j/blasbuild/cpu/cpu_features-src/include/internal/unix_features_aggregator.h -< libnd4j/blasbuild/cpu/cpu_features-src/LICENSE -3188,3211d3169 -< libnd4j/blasbuild/cpu/cpu_features-src/src/cpuinfo_aarch64.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/cpuinfo_arm.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/cpuinfo_mips.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/cpuinfo_ppc.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/cpuinfo_x86.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/filesystem.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/hwcaps.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/stack_line_reader.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/string_view.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/unix_features_aggregator.c -< libnd4j/blasbuild/cpu/cpu_features-src/src/utils/list_cpu_features.c -< libnd4j/blasbuild/cpu/cpu_features-src/test/bit_utils_test.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/cpuinfo_aarch64_test.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/cpuinfo_arm_test.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/cpuinfo_mips_test.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/cpuinfo_ppc_test.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/cpuinfo_x86_test.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/filesystem_for_testing.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/filesystem_for_testing.h -< libnd4j/blasbuild/cpu/cpu_features-src/test/hwcaps_for_testing.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/hwcaps_for_testing.h -< libnd4j/blasbuild/cpu/cpu_features-src/test/stack_line_reader_test.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/string_view_test.cc -< libnd4j/blasbuild/cpu/cpu_features-src/test/unix_features_aggregator_test.cc -3215,3216d3172 -< libnd4j/blasbuild/cpu/flatbuffers-src/android/jni/include.mk -< libnd4j/blasbuild/cpu/flatbuffers-src/android/jni/main.cpp -3220,3221d3175 -< libnd4j/blasbuild/cpu/flatbuffers-src/CMake/BuildFlatBuffers.cmake -< libnd4j/blasbuild/cpu/flatbuffers-src/CMake/FindFlatBuffers.cmake -3226d3179 -< libnd4j/blasbuild/cpu/flatbuffers-src/conan/test_package/test_package.cpp -3230,3231d3182 -< libnd4j/blasbuild/cpu/flatbuffers-src/dart/example/example.dart -< libnd4j/blasbuild/cpu/flatbuffers-src/dart/LICENSE -3239d3189 -< libnd4j/blasbuild/cpu/flatbuffers-src/grpc/flatbuffers-java-grpc/src/main/java/com/google/flatbuffers/grpc/FlatbuffersUtils.java -3241,3244d3190 -< libnd4j/blasbuild/cpu/flatbuffers-src/grpc/src/compiler/java_generator.cc -< libnd4j/blasbuild/cpu/flatbuffers-src/grpc/src/compiler/java_generator.h -< libnd4j/blasbuild/cpu/flatbuffers-src/grpc/tests/grpctest.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/grpc/tests/JavaGrpcTest.java -3247,3263d3192 -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/code_generators.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/flatbuffers.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/flatc.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/flexbuffers.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/grpc.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/hash.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/idl.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/minireflect.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/reflection.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/registry.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/stl_emulation.h -< libnd4j/blasbuild/cpu/flatbuffers-src/include/flatbuffers/util.h -< libnd4j/blasbuild/cpu/flatbuffers-src/java/com/google/flatbuffers/ByteBufferUtil.java -< libnd4j/blasbuild/cpu/flatbuffers-src/java/com/google/flatbuffers/Constants.java -< libnd4j/blasbuild/cpu/flatbuffers-src/java/com/google/flatbuffers/FlatBufferBuilder.java -< libnd4j/blasbuild/cpu/flatbuffers-src/java/com/google/flatbuffers/Struct.java -< libnd4j/blasbuild/cpu/flatbuffers-src/java/com/google/flatbuffers/Table.java -3265,3279d3193 -< libnd4j/blasbuild/cpu/flatbuffers-src/LICENSE.txt -< libnd4j/blasbuild/cpu/flatbuffers-src/lobster/flatbuffers.lobster -< libnd4j/blasbuild/cpu/flatbuffers-src/net/FlatBuffers/ByteBuffer.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/net/FlatBuffers/ByteBufferUtil.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/net/FlatBuffers/FlatBufferBuilder.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/net/FlatBuffers/FlatBufferConstants.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/net/FlatBuffers/IFlatbufferObject.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/net/FlatBuffers/Offset.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/net/FlatBuffers/Struct.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/net/FlatBuffers/Table.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/php/ByteBuffer.php -< libnd4j/blasbuild/cpu/flatbuffers-src/php/Constants.php -< libnd4j/blasbuild/cpu/flatbuffers-src/php/FlatbufferBuilder.php -< libnd4j/blasbuild/cpu/flatbuffers-src/php/Struct.php -< libnd4j/blasbuild/cpu/flatbuffers-src/php/Table.php -3291d3204 -< libnd4j/blasbuild/cpu/flatbuffers-src/reflection/generate_code.bat -3305,3306d3217 -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/android/jni/main.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/android/jni/schemas/animal.fbs -3317,3319d3227 -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/sample_binary.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/sample_binary.go -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/sample_binary.lobster -3322,3325d3229 -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/sample_text.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/sample_text.lobster -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/SampleBinary.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/SampleBinary.java -3327d3230 -< libnd4j/blasbuild/cpu/flatbuffers-src/samples/SampleBinary.php -3329,3349d3231 -< libnd4j/blasbuild/cpu/flatbuffers-src/src/code_generators.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/flatc.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/flatc_main.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/flathash.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_cpp.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_dart.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_fbs.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_general.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_go.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_grpc.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_js.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_json_schema.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_lobster.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_lua.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_php.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_python.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_rust.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_gen_text.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/idl_parser.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/reflection.cpp -< libnd4j/blasbuild/cpu/flatbuffers-src/src/util.cpp -3351,3359d3232 -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/Assert.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/ByteBufferTests.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/FlatBufferBuilderTests.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersTestClassAttribute.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersTestMethodAttribute.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/FuzzTestData.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/Lcg.cs -3361,3363d3233 -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/Program.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/Properties/AssemblyInfo.cs -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/FlatBuffers.Test/TestTable.cs -3367d3236 -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/generate_code.bat -3369d3237 -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/go_test.go -3374,3375d3241 -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/JavaTest.bat -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/JavaTest.java -3377d3242 -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/lobstertest.lobster -3415d3279 -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/RustTest.bat -3417d3280 -< libnd4j/blasbuild/cpu/flatbuffers-src/tests/test.cpp -3491,3492d3353 -< libnd4j/blasbuild/cuda/flatbuffers-src/android/jni/include.mk -< libnd4j/blasbuild/cuda/flatbuffers-src/android/jni/main.cpp -3496,3497d3356 -< libnd4j/blasbuild/cuda/flatbuffers-src/CMake/BuildFlatBuffers.cmake -< libnd4j/blasbuild/cuda/flatbuffers-src/CMake/FindFlatBuffers.cmake -3502d3360 -< libnd4j/blasbuild/cuda/flatbuffers-src/conan/test_package/test_package.cpp -3506,3507d3363 -< libnd4j/blasbuild/cuda/flatbuffers-src/dart/example/example.dart -< libnd4j/blasbuild/cuda/flatbuffers-src/dart/LICENSE -3515d3370 -< libnd4j/blasbuild/cuda/flatbuffers-src/grpc/flatbuffers-java-grpc/src/main/java/com/google/flatbuffers/grpc/FlatbuffersUtils.java -3517,3520d3371 -< libnd4j/blasbuild/cuda/flatbuffers-src/grpc/src/compiler/java_generator.cc -< libnd4j/blasbuild/cuda/flatbuffers-src/grpc/src/compiler/java_generator.h -< libnd4j/blasbuild/cuda/flatbuffers-src/grpc/tests/grpctest.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/grpc/tests/JavaGrpcTest.java -3523,3539d3373 -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/code_generators.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/flatbuffers.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/flatc.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/flexbuffers.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/grpc.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/hash.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/idl.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/minireflect.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/reflection.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/registry.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/stl_emulation.h -< libnd4j/blasbuild/cuda/flatbuffers-src/include/flatbuffers/util.h -< libnd4j/blasbuild/cuda/flatbuffers-src/java/com/google/flatbuffers/ByteBufferUtil.java -< libnd4j/blasbuild/cuda/flatbuffers-src/java/com/google/flatbuffers/Constants.java -< libnd4j/blasbuild/cuda/flatbuffers-src/java/com/google/flatbuffers/FlatBufferBuilder.java -< libnd4j/blasbuild/cuda/flatbuffers-src/java/com/google/flatbuffers/Struct.java -< libnd4j/blasbuild/cuda/flatbuffers-src/java/com/google/flatbuffers/Table.java -3541,3555d3374 -< libnd4j/blasbuild/cuda/flatbuffers-src/LICENSE.txt -< libnd4j/blasbuild/cuda/flatbuffers-src/lobster/flatbuffers.lobster -< libnd4j/blasbuild/cuda/flatbuffers-src/net/FlatBuffers/ByteBuffer.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/net/FlatBuffers/ByteBufferUtil.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/net/FlatBuffers/FlatBufferBuilder.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/net/FlatBuffers/FlatBufferConstants.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/net/FlatBuffers/IFlatbufferObject.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/net/FlatBuffers/Offset.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/net/FlatBuffers/Struct.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/net/FlatBuffers/Table.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/php/ByteBuffer.php -< libnd4j/blasbuild/cuda/flatbuffers-src/php/Constants.php -< libnd4j/blasbuild/cuda/flatbuffers-src/php/FlatbufferBuilder.php -< libnd4j/blasbuild/cuda/flatbuffers-src/php/Struct.php -< libnd4j/blasbuild/cuda/flatbuffers-src/php/Table.php -3567d3385 -< libnd4j/blasbuild/cuda/flatbuffers-src/reflection/generate_code.bat -3581,3582d3398 -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/android/jni/main.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/android/jni/schemas/animal.fbs -3593,3595d3408 -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/sample_binary.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/sample_binary.go -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/sample_binary.lobster -3598,3601d3410 -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/sample_text.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/sample_text.lobster -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/SampleBinary.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/SampleBinary.java -3603d3411 -< libnd4j/blasbuild/cuda/flatbuffers-src/samples/SampleBinary.php -3605,3625d3412 -< libnd4j/blasbuild/cuda/flatbuffers-src/src/code_generators.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/flatc.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/flatc_main.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/flathash.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_cpp.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_dart.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_fbs.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_general.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_go.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_grpc.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_js.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_json_schema.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_lobster.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_lua.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_php.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_python.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_rust.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_gen_text.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/idl_parser.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/reflection.cpp -< libnd4j/blasbuild/cuda/flatbuffers-src/src/util.cpp -3627,3635d3413 -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/Assert.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/ByteBufferTests.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/FlatBufferBuilderTests.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersTestClassAttribute.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersTestMethodAttribute.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/FuzzTestData.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/Lcg.cs -3637,3639d3414 -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/Program.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/Properties/AssemblyInfo.cs -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/FlatBuffers.Test/TestTable.cs -3643d3417 -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/generate_code.bat -3645d3418 -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/go_test.go -3650,3651d3422 -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/JavaTest.bat -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/JavaTest.java -3653d3423 -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/lobstertest.lobster -3694d3463 -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/RustTest.bat -3696d3464 -< libnd4j/blasbuild/cuda/flatbuffers-src/tests/test.cpp -3941,3955d3708 -< libnd4j/cmake-build-debug/cpu_features-src/include/cpu_features_cache_info.h -< libnd4j/cmake-build-debug/cpu_features-src/include/cpu_features_macros.h -< libnd4j/cmake-build-debug/cpu_features-src/include/cpuinfo_aarch64.h -< libnd4j/cmake-build-debug/cpu_features-src/include/cpuinfo_arm.h -< libnd4j/cmake-build-debug/cpu_features-src/include/cpuinfo_mips.h -< libnd4j/cmake-build-debug/cpu_features-src/include/cpuinfo_ppc.h -< libnd4j/cmake-build-debug/cpu_features-src/include/cpuinfo_x86.h -< libnd4j/cmake-build-debug/cpu_features-src/include/internal/bit_utils.h -< libnd4j/cmake-build-debug/cpu_features-src/include/internal/cpuid_x86.h -< libnd4j/cmake-build-debug/cpu_features-src/include/internal/filesystem.h -< libnd4j/cmake-build-debug/cpu_features-src/include/internal/hwcaps.h -< libnd4j/cmake-build-debug/cpu_features-src/include/internal/stack_line_reader.h -< libnd4j/cmake-build-debug/cpu_features-src/include/internal/string_view.h -< libnd4j/cmake-build-debug/cpu_features-src/include/internal/unix_features_aggregator.h -< libnd4j/cmake-build-debug/cpu_features-src/LICENSE -3958,3981d3710 -< libnd4j/cmake-build-debug/cpu_features-src/src/cpuinfo_aarch64.c -< libnd4j/cmake-build-debug/cpu_features-src/src/cpuinfo_arm.c -< libnd4j/cmake-build-debug/cpu_features-src/src/cpuinfo_mips.c -< libnd4j/cmake-build-debug/cpu_features-src/src/cpuinfo_ppc.c -< libnd4j/cmake-build-debug/cpu_features-src/src/cpuinfo_x86.c -< libnd4j/cmake-build-debug/cpu_features-src/src/filesystem.c -< libnd4j/cmake-build-debug/cpu_features-src/src/hwcaps.c -< libnd4j/cmake-build-debug/cpu_features-src/src/stack_line_reader.c -< libnd4j/cmake-build-debug/cpu_features-src/src/string_view.c -< libnd4j/cmake-build-debug/cpu_features-src/src/unix_features_aggregator.c -< libnd4j/cmake-build-debug/cpu_features-src/src/utils/list_cpu_features.c -< libnd4j/cmake-build-debug/cpu_features-src/test/bit_utils_test.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/cpuinfo_aarch64_test.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/cpuinfo_arm_test.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/cpuinfo_mips_test.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/cpuinfo_ppc_test.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/cpuinfo_x86_test.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/filesystem_for_testing.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/filesystem_for_testing.h -< libnd4j/cmake-build-debug/cpu_features-src/test/hwcaps_for_testing.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/hwcaps_for_testing.h -< libnd4j/cmake-build-debug/cpu_features-src/test/stack_line_reader_test.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/string_view_test.cc -< libnd4j/cmake-build-debug/cpu_features-src/test/unix_features_aggregator_test.cc -3986,3987d3714 -< libnd4j/cmake-build-debug/flatbuffers-src/android/jni/include.mk -< libnd4j/cmake-build-debug/flatbuffers-src/android/jni/main.cpp -3991,3992d3717 -< libnd4j/cmake-build-debug/flatbuffers-src/CMake/BuildFlatBuffers.cmake -< libnd4j/cmake-build-debug/flatbuffers-src/CMake/FindFlatBuffers.cmake -3997d3721 -< libnd4j/cmake-build-debug/flatbuffers-src/conan/test_package/test_package.cpp -4001,4002d3724 -< libnd4j/cmake-build-debug/flatbuffers-src/dart/example/example.dart -< libnd4j/cmake-build-debug/flatbuffers-src/dart/LICENSE -4010d3731 -< libnd4j/cmake-build-debug/flatbuffers-src/grpc/flatbuffers-java-grpc/src/main/java/com/google/flatbuffers/grpc/FlatbuffersUtils.java -4012,4015d3732 -< libnd4j/cmake-build-debug/flatbuffers-src/grpc/src/compiler/java_generator.cc -< libnd4j/cmake-build-debug/flatbuffers-src/grpc/src/compiler/java_generator.h -< libnd4j/cmake-build-debug/flatbuffers-src/grpc/tests/grpctest.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/grpc/tests/JavaGrpcTest.java -4018,4034d3734 -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/code_generators.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/flatbuffers.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/flatc.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/flexbuffers.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/grpc.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/hash.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/idl.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/minireflect.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/reflection.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/registry.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/stl_emulation.h -< libnd4j/cmake-build-debug/flatbuffers-src/include/flatbuffers/util.h -< libnd4j/cmake-build-debug/flatbuffers-src/java/com/google/flatbuffers/ByteBufferUtil.java -< libnd4j/cmake-build-debug/flatbuffers-src/java/com/google/flatbuffers/Constants.java -< libnd4j/cmake-build-debug/flatbuffers-src/java/com/google/flatbuffers/FlatBufferBuilder.java -< libnd4j/cmake-build-debug/flatbuffers-src/java/com/google/flatbuffers/Struct.java -< libnd4j/cmake-build-debug/flatbuffers-src/java/com/google/flatbuffers/Table.java -4036,4050d3735 -< libnd4j/cmake-build-debug/flatbuffers-src/LICENSE.txt -< libnd4j/cmake-build-debug/flatbuffers-src/lobster/flatbuffers.lobster -< libnd4j/cmake-build-debug/flatbuffers-src/net/FlatBuffers/ByteBuffer.cs -< libnd4j/cmake-build-debug/flatbuffers-src/net/FlatBuffers/ByteBufferUtil.cs -< libnd4j/cmake-build-debug/flatbuffers-src/net/FlatBuffers/FlatBufferBuilder.cs -< libnd4j/cmake-build-debug/flatbuffers-src/net/FlatBuffers/FlatBufferConstants.cs -< libnd4j/cmake-build-debug/flatbuffers-src/net/FlatBuffers/IFlatbufferObject.cs -< libnd4j/cmake-build-debug/flatbuffers-src/net/FlatBuffers/Offset.cs -< libnd4j/cmake-build-debug/flatbuffers-src/net/FlatBuffers/Struct.cs -< libnd4j/cmake-build-debug/flatbuffers-src/net/FlatBuffers/Table.cs -< libnd4j/cmake-build-debug/flatbuffers-src/php/ByteBuffer.php -< libnd4j/cmake-build-debug/flatbuffers-src/php/Constants.php -< libnd4j/cmake-build-debug/flatbuffers-src/php/FlatbufferBuilder.php -< libnd4j/cmake-build-debug/flatbuffers-src/php/Struct.php -< libnd4j/cmake-build-debug/flatbuffers-src/php/Table.php -4062d3746 -< libnd4j/cmake-build-debug/flatbuffers-src/reflection/generate_code.bat -4077,4078d3760 -< libnd4j/cmake-build-debug/flatbuffers-src/samples/android/jni/main.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/samples/android/jni/schemas/animal.fbs -4089,4091d3770 -< libnd4j/cmake-build-debug/flatbuffers-src/samples/sample_binary.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/samples/sample_binary.go -< libnd4j/cmake-build-debug/flatbuffers-src/samples/sample_binary.lobster -4094,4097d3772 -< libnd4j/cmake-build-debug/flatbuffers-src/samples/sample_text.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/samples/sample_text.lobster -< libnd4j/cmake-build-debug/flatbuffers-src/samples/SampleBinary.cs -< libnd4j/cmake-build-debug/flatbuffers-src/samples/SampleBinary.java -4099d3773 -< libnd4j/cmake-build-debug/flatbuffers-src/samples/SampleBinary.php -4101,4121d3774 -< libnd4j/cmake-build-debug/flatbuffers-src/src/code_generators.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/flatc.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/flatc_main.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/flathash.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_cpp.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_dart.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_fbs.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_general.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_go.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_grpc.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_js.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_json_schema.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_lobster.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_lua.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_php.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_python.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_rust.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_gen_text.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/idl_parser.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/reflection.cpp -< libnd4j/cmake-build-debug/flatbuffers-src/src/util.cpp -4123,4131d3775 -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/Assert.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/ByteBufferTests.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/FlatBufferBuilderTests.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersTestClassAttribute.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersTestMethodAttribute.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/FuzzTestData.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/Lcg.cs -4133,4135d3776 -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/Program.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/Properties/AssemblyInfo.cs -< libnd4j/cmake-build-debug/flatbuffers-src/tests/FlatBuffers.Test/TestTable.cs -4139d3779 -< libnd4j/cmake-build-debug/flatbuffers-src/tests/generate_code.bat -4141d3780 -< libnd4j/cmake-build-debug/flatbuffers-src/tests/go_test.go -4146,4147d3784 -< libnd4j/cmake-build-debug/flatbuffers-src/tests/JavaTest.bat -< libnd4j/cmake-build-debug/flatbuffers-src/tests/JavaTest.java -4149d3785 -< libnd4j/cmake-build-debug/flatbuffers-src/tests/lobstertest.lobster -4190d3825 -< libnd4j/cmake-build-debug/flatbuffers-src/tests/RustTest.bat -4192d3826 -< libnd4j/cmake-build-debug/flatbuffers-src/tests/test.cpp -4420,4434d4053 -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/cpu_features_cache_info.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/cpu_features_macros.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/cpuinfo_aarch64.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/cpuinfo_arm.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/cpuinfo_mips.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/cpuinfo_ppc.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/cpuinfo_x86.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/internal/bit_utils.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/internal/cpuid_x86.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/internal/filesystem.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/internal/hwcaps.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/internal/stack_line_reader.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/internal/string_view.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/include/internal/unix_features_aggregator.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/LICENSE -4437,4460d4055 -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/cpuinfo_aarch64.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/cpuinfo_arm.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/cpuinfo_mips.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/cpuinfo_ppc.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/cpuinfo_x86.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/filesystem.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/hwcaps.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/stack_line_reader.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/string_view.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/unix_features_aggregator.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/src/utils/list_cpu_features.c -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/bit_utils_test.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/cpuinfo_aarch64_test.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/cpuinfo_arm_test.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/cpuinfo_mips_test.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/cpuinfo_ppc_test.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/cpuinfo_x86_test.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/filesystem_for_testing.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/filesystem_for_testing.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/hwcaps_for_testing.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/hwcaps_for_testing.h -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/stack_line_reader_test.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/string_view_test.cc -< libnd4j/cmake-build-debug-mingw/cpu_features-src/test/unix_features_aggregator_test.cc -4465,4466d4059 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/android/jni/include.mk -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/android/jni/main.cpp -4470,4471d4062 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/CMake/BuildFlatBuffers.cmake -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/CMake/FindFlatBuffers.cmake -4476d4066 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/conan/test_package/test_package.cpp -4480,4481d4069 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/dart/example/example.dart -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/dart/LICENSE -4489d4076 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/grpc/flatbuffers-java-grpc/src/main/java/com/google/flatbuffers/grpc/FlatbuffersUtils.java -4491,4494d4077 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/grpc/src/compiler/java_generator.cc -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/grpc/src/compiler/java_generator.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/grpc/tests/grpctest.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/grpc/tests/JavaGrpcTest.java -4497,4513d4079 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/code_generators.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/flatbuffers.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/flatc.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/flexbuffers.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/grpc.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/hash.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/idl.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/minireflect.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/reflection.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/registry.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/stl_emulation.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/include/flatbuffers/util.h -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/java/com/google/flatbuffers/ByteBufferUtil.java -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/java/com/google/flatbuffers/Constants.java -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/java/com/google/flatbuffers/FlatBufferBuilder.java -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/java/com/google/flatbuffers/Struct.java -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/java/com/google/flatbuffers/Table.java -4515,4529d4080 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/LICENSE.txt -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/lobster/flatbuffers.lobster -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/net/FlatBuffers/ByteBuffer.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/net/FlatBuffers/ByteBufferUtil.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/net/FlatBuffers/FlatBufferBuilder.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/net/FlatBuffers/FlatBufferConstants.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/net/FlatBuffers/IFlatbufferObject.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/net/FlatBuffers/Offset.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/net/FlatBuffers/Struct.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/net/FlatBuffers/Table.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/php/ByteBuffer.php -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/php/Constants.php -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/php/FlatbufferBuilder.php -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/php/Struct.php -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/php/Table.php -4541d4091 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/reflection/generate_code.bat -4556,4557d4105 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/android/jni/main.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/android/jni/schemas/animal.fbs -4568,4570d4115 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/sample_binary.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/sample_binary.go -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/sample_binary.lobster -4573,4576d4117 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/sample_text.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/sample_text.lobster -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/SampleBinary.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/SampleBinary.java -4578d4118 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/samples/SampleBinary.php -4580,4600d4119 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/code_generators.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/flatc.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/flatc_main.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/flathash.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_cpp.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_dart.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_fbs.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_general.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_go.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_grpc.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_js.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_json_schema.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_lobster.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_lua.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_php.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_python.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_rust.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_gen_text.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/idl_parser.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/reflection.cpp -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/src/util.cpp -4602,4610d4120 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/Assert.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/ByteBufferTests.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/FlatBufferBuilderTests.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersTestClassAttribute.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/FlatBuffersTestMethodAttribute.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/FuzzTestData.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/Lcg.cs -4612,4614d4121 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/Program.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/Properties/AssemblyInfo.cs -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/FlatBuffers.Test/TestTable.cs -4618d4124 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/generate_code.bat -4620d4125 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/go_test.go -4625,4626d4129 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/JavaTest.bat -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/JavaTest.java -4628d4130 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/lobstertest.lobster -4669d4170 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/RustTest.bat -4671d4171 -< libnd4j/cmake-build-debug-mingw/flatbuffers-src/tests/test.cpp -4695d4194 -< libnd4j/cmake-build-debug-mingw/tests_cpu/googletest-src/googlemock/scripts/generator/LICENSE -6338d5836 -< LICENSE -7728,7906d7225 -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/bfloat16/bfloat16.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/snapfn.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/db/sqlite_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gif/gif_io.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_internal.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/array_slice_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/cleanup_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/compactptrset_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/edit_distance_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatmap_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatrep.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/flatset_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/inlined_vector_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/int_type_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/iterator_range_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/manual_constructor_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/map_util_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/optional_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/priority_queue_util.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/stl_util.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/gtl/top_n_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_accelerate.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/crc32c_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/hash/hash_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/histogram/histogram_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/block_builder.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/buffered_inputstream_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/compression.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/format.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputbuffer_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/inputstream_interface_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/iterator.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/path_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/proto_encode_helper.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/random_inputstream_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_reader_writer_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/record_writer.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/recordio_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_inputbuffer.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/snappy/snappy_outputbuffer.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_builder.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_options.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/table_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/two_level_iterator.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_buffers_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_compression_options.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_inputstream.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/io/zlib_outputbuffer.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_handle.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/math/math_util_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collected_metrics.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/collection_registry_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/counter_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/gauge_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/metric_def_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_counter.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_gauge.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/mobile_sampler.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/monitoring/sampler_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/png/png_io.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/distribution_sampler_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/exact_uniform_int.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/philox_random_test_utils.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_distributions_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/random_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/simple_philox_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/random/weighted_picker_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/base64_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/numbers_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/ordered_code_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_serialization.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/proto_text_util.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/scanner_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/str_util_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/strcat_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/strings/stringprintf_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io.h -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/lib/wav/wav_io_test.cc -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/cluster.proto -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/device_properties.proto -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/master.proto -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/master_service.proto -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/tensorflow_server.proto -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker.proto -< nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/protobuf/worker_service.proto -8757,8758d8075 -< nd4j/nd4j-shade/guava/target/classes/META-INF/maven/com.google.errorprone/error_prone_annotations/pom.xml -< nd4j/nd4j-shade/guava/target/classes/META-INF/maven/com.google.j2objc/j2objc-annotations/pom.xml diff --git a/eclipse_deeplearning4j.png b/eclipse_deeplearning4j.png deleted file mode 100644 index 1768fa5e5..000000000 Binary files a/eclipse_deeplearning4j.png and /dev/null differ diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 000000000..5af1ca6f2 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,39 @@ +# +# +# ****************************************************************************** +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ***************************************************************************** +# +# +systemProp.org.gradle.internal.publish.checksums.insecure=true + +# Project-wide Gradle settings. + +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. + +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html + +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx8192m + +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..7454180f2 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..49fc93b14 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,9 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists + +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx8128m diff --git a/gradlew b/gradlew new file mode 100644 index 000000000..744e882ed --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MSYS* | MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/libnd4j/.gitignore b/libnd4j/.gitignore deleted file mode 100644 index c174d0d2f..000000000 --- a/libnd4j/.gitignore +++ /dev/null @@ -1,38 +0,0 @@ -CMakeCache.txt -CMakeFiles -CMakeScripts/ -.vscode/ -.vs/ -libnd4j.xcodeproj/ -runtests*/ -.csettings/ -.project -*.so -*.a -*.dll -*.dylib -Makefile -cmake_install.cmake -install_manifest.txt -cmake-build-debug/ -libnd4j.build/ -build -*.ptx -*.cubin -cubinbuild -pitxbuild -blasbuild -testbuild -.idea/ -libnd4jtests -runtests -.DS_Store -*.cbp -CTestTestfile.cmake -compile_commands.json -CPackConfig.cmake -CPackSourceConfig.cmake -target -minifier -tests_cpu/layers_tests/minifier -tests_cpu/layers_tests/minifier.dSYM/ diff --git a/libnd4j/AddingNewOps.md b/libnd4j/AddingNewOps.md deleted file mode 100644 index 1d8309157..000000000 --- a/libnd4j/AddingNewOps.md +++ /dev/null @@ -1,187 +0,0 @@ -There's multiple different Ops designs supported in libND4j, and in this guide we'll try to explain how to build your very own operation. - -## XYZ operations - -This kind of operations is actually split into multiple subtypes, based on element-access and result type: -- Transform operations: These operations typically take some NDArray in, and change each element independent of others. -- Reduction operations: These operations take some NDArray and dimensions, and return reduced NDArray (or scalar) back. I.e. sum along dimension(s). -- Scalar operations: These operations are similar to transforms, but they only do arithmetic operations, and second operand is scalar. I.e. each element in given NDArray will add given scalar value. -- Pairwise operations: These operations are between regular transform opeartions and scalar operations. I.e. element-wise addition of two NDArrays. -- Random operations: Most of these operations related to random numbers distributions: Uniform, Gauss, Bernoulli etc. - -Despite differences between these operations, they are all using XZ/XYZ three-operand design, where X and Y are inputs, and Z is output. -Data access in these operations is usually trivial, and loop based. I.e. most trivial loop for scalar transform will look like this: -```c++ -for (Nd4jLong i = start; i < end; i++) { - result[i] = OpType::op(x[i], scalar, extraParams); -} -``` - -Operation used in this loop will be template-driven, and compiled statically. There are another loops implementation, depending on op group or strides within NDArrays, but idea will be the same all the time: each element of the NDArray will be accessed within loop. - -Now, let's take a look into typical XYZ op implementation. Here's how `Add` operation will look like: - -```c++ - -template -class Add { -public: - op_def static T op(T d1, T d2) { - return d1 + d2; - } - - // this signature will be used in Scalar loops - op_def static T op(T d1, T d2, T *params) { - return d1 + d2; - } - - // this signature will be used in reductions - op_def static T op(T d1) { - return d1; - } - - // op for MetaOps - op_def static T op(T d1, T *params) { - return d1 + params[0]; - } -}; -``` - -This particular operation is used in different XYZ op groups, but you see the idea: element-wise operation, which is invoked on each element in given NDArray. -So, if you want to add new XYZ operation to libnd4j, you should just add operation implementation to file `includes/ops/ops.h`, and assign it to specific ops group in file `includes/loops/legacy_ops.h` together with some number unique to this ops group, i.e.: `(21, simdOps::Add)` - -After libnd4j is recompiled, this op will become available for legacy execution mechanism, NDArray wrappers, and `LegacyOp` wrappers (those are made to map legacy operations to CustomOps design for Graph). - - -## Custom operations - -Custom operations is a new concept, added recently and mostly suits SameDiff/Graph needs. -For CustomOps we defined universal signature, with variable number of input/output NDArrays, and variable number of floating-point and integer arguments. -However, there are some minor difference between various CustomOp declarations: -- **DECLARE_OP**(string, int, int, bool): these operations take no fp/int arguments, and output shape equals to input shape. -- **DECLARE_CONFIGURABLE_OP**(string, int, int, bool, int, int): these operations do take fp/int output arguments, and output shape equals to input shape. -- **DECLARE_REDUCTION_OP**(string, int, int, bool, int, int): these operations do take fp/int output arguments, and output shape is calculated as Reduction. -- **DECLARE_CUSTOM_OP**(string, int, int, bool, int, int): these operations return NDArray with custom shape, that usually depends on input and arguments. -- **DECLARE_BOOLEAN_OP**(string, int, bool): these operations take some NDArrays and return scalar, where 0 is **False**, and other values are treated as **True**. - -Let's take a look at example CustomOp: - -```c++ - -CUSTOM_OP_IMPL(tear, 1, -1, false, 0, -1) { - auto input = INPUT_VARIABLE(0); - - REQUIRE_TRUE(!block.getIArguments()->empty(), 0, "At least 1 dimension should be specified for Tear"); - - std::vector dims(*block.getIArguments()); - - for (auto &v: dims) - REQUIRE_TRUE(v >= 0 && v < input->rankOf(), 0, "Tear dimensions should be non-negative values, and lower then input rank. Got %i instead", v); - - auto tads = input->allTensorsAlongDimension(dims); - for (int e = 0; e < tads->size(); e++) { - auto outE = OUTPUT_VARIABLE(e); - outE->assign(tads->at(e)); - - this->storeResult(block, e, *outE); - } - - delete tads; - - return ND4J_STATUS_OK; -} - -DECLARE_SHAPE_FN(tear) { - auto inShape = inputShape->at(0); - - std::vector dims(*block.getIArguments()); - - if (dims.size() > 1) - std::sort(dims.begin(), dims.end()); - - shape::TAD tad(inShape, dims.data(), (int) dims.size()); - tad.createTadOnlyShapeInfo(); - Nd4jLong numTads = shape::tadLength(inShape, dims.data(), (int) dims.size()); - - auto result = SHAPELIST(); - for (int e = 0; e < numTads; e++) { - int *newShape; - COPY_SHAPE(tad.tadOnlyShapeInfo, newShape); - result->push_back(newShape); - } - - return result; -} -``` - -In the example above, we declare `tear` CustomOp implementation, and shape function for this op. -So, at the moment of op execution, we assume that we will either have output array(s) provided by end-user, or they will be generated with shape function. - -You can also see number of macros used, we'll cover those later as well. Beyond that - op execution logic is fairly simple & linear: -Each new op implements protected member function `DeclarableOp::validateAndExecute(Block& block)`, and this method is eventually called either from GraphExecutioner, or via direct call, like `DeclarableOp::execute(Block& block)`. - -Important part of op declaration is input/output description for the op. I.e. as shown above: `CUSTOM_OP_IMPL(tear, 1, -1, false, 0, -1)`. -This declaration means: -- Op name: `tear` -- Op expects at least 1 NDArray as input -- Op returns unknown positive number of NDArrays as output -- Op can't be run in-place, so under any circumstances original NDArray will stay intact -- Op doesn't expect any T (aka floating point) arguments -- Op expects unknown positive number of integer arguments. In case of this op it's dimensions to split input NDArray. - -Here's another example: `DECLARE_CUSTOM_OP(permute, 1, 1, true, 0, -2);` -This declaration means: -- Op name: `permute` -- Op expects at least 1 NDArray as input -- Op returns 1 NDArray as output -- Op can be run in-place if needed (it means: input == output, and input is modified and returned as output) -- Op doesn't expect any T arguments -- Op expects unknown number of integer arguments OR no integer arguments at all. - -## c++11 syntactic sugar - -In ops you can easily use c++11 features, including lambdas. In some cases it might be easiest way to build your custom op (or some part of it) via `NDArray::applyLambda` or `NDArray::applyPairwiseLambda`: -```c++ -auto lambda = LAMBDA_TT(_x, _y) { - return (_x + _y) * 2; -}; - -x.applyPairwiseLambda(&y, lambda); -``` - -In this simple example, each element of NDArray `x` will get values set to `x[e] = (x[e] + y[e]) * 2`. - -## Tests - -For tests libnd4j uses Google Tests suit. All tests are located at `tests_cpu/layers_tests` folder. Here's simple way to run those from command line: -``` -cd tests_cpu -cmake -G "Unix Makefiles" -make -j 4 -./layers_tests/runtests -``` - -You can also use your IDE (i.e. Jetbrains CLion) to run tests via GUI. - -**PLEASE NOTE:** if you're considering submitting your new op to libnd4j repository via pull request - consider adding tests for it. Ops without tests won't be approved. - -## Backend-specific operation - -GPU/MPI/whatever to be added soon. - - -## Utility macros -We have number of utility macros, suitable for custom ops. Here they are: -- **INPUT_VARIABLE**(int): this macro returns you NDArray at specified input index. -- **OUTPUT_VARIABLE**(int): this macro returns you NDArray at specified output index. -- **STORE_RESULT**(NDArray): this macro stores result to VariableSpace. -- **STORE_2_RESULTS**(NDArray, NDArray): this macro stores results accordingly to VariableSpace. -- **INT_ARG**(int): this macro returns you specific Integer argument passed to the given op. -- **T_ARG**(int): this macro returns you specific T argument passed to the given op. -- **ALLOCATE**(...): this macro check if Workspace is available, and either uses Workspace or direct memory allocation if Workspace isn't available. -- **RELEASE**(...): this macro is made to release memory allocated with **ALLOCATE()** macro. -- **REQUIRE_TRUE**(...): this macro takes condition, and evaluates it. If evaluation doesn't end up as True - exception is raised, and specified message is printed out. -- **LAMBDA_T**(X) and **LAMBDA_TT**(X, Y): lambda declaration for `NDArray::applyLambda` and `NDArray::applyPairwiseLambda` -- **COPY_SHAPE**(SRC, TGT): this macro allocates memory for TGT pointer and copies shape from SRC pointer -- **ILAMBDA_T**(X) and **ILAMBDA_TT**(X, Y): lambda declaration for indexed lambdas, index argument is passed in as Nd4jLong (aka **long long**) -- **FORCEINLINE**: platform-specific definition for functions inlining diff --git a/libnd4j/CMakeLists.txt b/libnd4j/CMakeLists.txt deleted file mode 100755 index 1e2633e07..000000000 --- a/libnd4j/CMakeLists.txt +++ /dev/null @@ -1,428 +0,0 @@ -cmake_minimum_required(VERSION 3.15) -project(libnd4j) -set(CMAKE_VERBOSE_MAKEFILE ON) - - -set (CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -message("CMAKE MODULE PATH ${CMAKE_MODULE_PATH}") - -#ensure we create lib files -set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS OFF) - -option(SD_NATIVE "Optimize for build machine (might not work on others)" OFF) -option(SD_CHECK_VECTORIZATION "checks for vectorization" OFF) -option(SD_BUILD_TESTS "Build tests" OFF) -option(SD_STATIC_LIB "Build static library" OFF) -option(SD_SHARED_LIB "Build shared library" ON) -option(SD_SANITIZE "Enable Address Sanitizer" ON) - -option(FLATBUFFERS_BUILD_FLATC "Enable the build of the flatbuffers compiler" OFF) -set(FLATBUFFERS_BUILD_FLATC "OFF" CACHE STRING "Hack to disable flatc build" FORCE) - -set(CMAKE_CXX_STANDARD 11) - -#/////////////////////////////////////////////////////////////////////////////// -# genCompilation: Generates cpp, cu files -# INPUT: -# $FILE_ITEM template-configuration that utilizes libnd4j type, macros helpers -# defined inside { include/types/types.h, include/system/type_boilerplate.h} -# OUTPUT: -# $CUSTOMOPS_GENERIC_SOURCES generated files will be added into this List -#//////////////////////////////////////////////////////////////////////////////// -# A simple template-configuration file example: -# // hints and defines what types will be generated -# #cmakedefine LIBND4J_TYPE_GEN -# #cmakedefine FLOAT_TYPE_GEN -# // below if defines blocks are needed for correctly handling multiple types -# #if defined(LIBND4J_TYPE_GEN) -# BUILD_DOUBLE_TEMPLATE(template void someFunc, (arg_list,..), -# LIBND4J_TYPES_@FL_TYPE_INDEX@, INDEXING_TYPES); -# #endif -# #if defined(FLOAT_TYPE_GEN) -# BUILD_SINGLE_TEMPLATE(template class SomeClass,, FLOAT_TYPES_@FL_TYPE_INDEX@); -# #endif -#//////////////////////////////////////////////////////////////////////////////// - -set_property(GLOBAL PROPERTY JOB_POOLS one_jobs=1 two_jobs=2) - - - - -function(genCompilation FILE_ITEM) - get_filename_component(FILE_ITEM_WE ${FL_ITEM} NAME_WE) - - set(EXTENSION "cpp") - - if(FL_ITEM MATCHES "cu.in$") - set(EXTENSION "cu") - endif() - - file(READ ${FL_ITEM} CONTENT_FL) - #check content for types - - #set all to false - set (FLOAT_TYPE_GEN 0) - set (INT_TYPE_GEN 0) - set (LIBND4J_TYPE_GEN 0) - set (PAIRWISE_TYPE_GEN 0) - set (RANGE_STOP -1) - - string(REGEX MATCHALL "#cmakedefine[ \t]+[^_]+_TYPE_GEN" TYPE_MATCHES ${CONTENT_FL}) - - foreach(TYPEX ${TYPE_MATCHES}) - set(STOP -1) - if(TYPEX MATCHES "INT_TYPE_GEN$") - set (INT_TYPE_GEN 1) - set(STOP 7) - endif() - if(TYPEX MATCHES "LIBND4J_TYPE_GEN$") - set (LIBND4J_TYPE_GEN 1) - set(STOP 9) - endif() - if(TYPEX MATCHES "FLOAT_TYPE_GEN$") - set (FLOAT_TYPE_GEN 1) - set(STOP 3) - endif() - if(TYPEX MATCHES "PAIRWISE_TYPE_GEN$") - set (PAIRWISE_TYPE_GEN 1) - set(STOP 12) - endif() - if(STOP GREATER RANGE_STOP) - set(RANGE_STOP ${STOP}) - endif() - - endforeach() - - if(RANGE_STOP GREATER -1) - foreach(FL_TYPE_INDEX RANGE 0 ${RANGE_STOP}) - # set OFF if the index is above - if(FL_TYPE_INDEX GREATER 3) - set (FLOAT_TYPE_GEN 0) - endif() - if(FL_TYPE_INDEX GREATER 7) - set (INT_TYPE_GEN 0) - endif() - if(FL_TYPE_INDEX GREATER 9) - set (LIBND4J_TYPE_GEN 0) - endif() - set(GENERATED_SOURCE "${CMAKE_BINARY_DIR}/compilation_units/${FILE_ITEM_WE}_${FL_TYPE_INDEX}.${EXTENSION}") - configure_file( "${FL_ITEM}" "${GENERATED_SOURCE}" @ONLY) - LIST(APPEND CUSTOMOPS_GENERIC_SOURCES ${GENERATED_SOURCE} ) - endforeach() - endif() - - set(CUSTOMOPS_GENERIC_SOURCES ${CUSTOMOPS_GENERIC_SOURCES} PARENT_SCOPE) -endfunction() - - -if (SD_CUDA) - enable_language(CUDA) - set(CMAKE_CUDA_STANDARD 11) - - set(DEFAULT_ENGINE "samediff::ENGINE_CUDA") -else() - set(DEFAULT_ENGINE "samediff::ENGINE_CPU") -endif() - -# MSVC runtime lib can be either "MultiThreaded" or "MultiThreadedDLL", /MT and /MD respectively -set(MSVC_RT_LIB "MultiThreadedDLL") - -set(SD_X86_BUILD false) - -if (NOT SD_IOS_BUILD AND NOT SD_ANDROID_BUILD AND NOT ${SD_ARCH} MATCHES "power*" AND NOT ${SD_ARCH} MATCHES "arm*") - set(SD_X86_BUILD true) -endif() - -# -fsanitize=address -# -fsanitize=leak -if (SD_ANDROID_BUILD) - set_property(GLOBAL PROPERTY JOB_POOLS one_job=1 two_jobs=2) - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else -D_RELEASE=true") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else") -elseif (APPLE) - set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else -D__APPLE_OS__=true -D_RELEASE=true") - set(CMAKE_CXX_FLAGS_DEBUG " -O0 -g -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else -D__APPLE_OS__=true") -elseif(WIN32) - set(SD_X86_BUILD true) - if (SD_CUDA) - set(CMAKE_CXX_FLAGS_RELEASE "-D_RELEASE=true") - set(CMAKE_CXX_FLAGS_DEBUG " /FS /EHsc") - set(CMAKE_CUDA_STANDARD 14) - else() - set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fPIC -D_RELEASE=true") - set(CMAKE_CXX_FLAGS_DEBUG " -g -O2 -fPIC") - set(CMAKE_CUDA_STANDARD 14) - endif() -else() - set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fPIC -D_RELEASE=true") - set(CMAKE_CXX_FLAGS_DEBUG " -g -O0 -fPIC") - - if (SD_CPU AND SD_SANITIZE) - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address") - endif() -endif() - -if(SD_NATIVE) - IF(${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64*") - set(SD_X86_BUILD false) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=native") - ELSE() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") - ENDIF() -endif() - - -if(NOT SD_CUDA) - # we need this definition to avoid global memory use within mkldnn - add_definitions(-DDNNL_ENABLE_CONCURRENT_EXEC=true) - - # there's a chance, we have no BLAS provided externally - if ("${OPENBLAS_PATH}" STREQUAL "") - #we don't want OpenBLAS on Apple - if (NOT APPLE) - # note: this is not a typo - set(BLA_VENDOR "OpenBLAS") - endif() - - # look around for system blas instead, see: https://cmake.org/cmake/help/latest/module/FindBLAS.html - find_package(BLAS REQUIRED) - if (BLAS_FOUND) - message("Found external BLAS implementation: ${BLAS_LIBRARIES} ") - add_definitions(-D__EXTERNAL_BLAS__=true) - endif() - else() - # if we have externally provided OPENBLAS_PATH - let's use it - set(HAVE_OPENBLAS 1) - message("Setting openblas") - include_directories(${OPENBLAS_PATH}/include/) - link_directories(${OPENBLAS_PATH} ${OPENBLAS_PATH}/lib/) - set(OPENBLAS_LIBRARIES openblas) - endif() - - # building cpu_features - if (SD_X86_BUILD) - add_definitions(-DCPU_FEATURES=true) - set(BUILD_PIC "ON" CACHE STRING "Hack to enforce fPIC mode" FORCE) - configure_file(./CMakeLists.txt.cpu_features.in cpu_features-download/CMakeLists.txt) - message("CMAKE_COMMAND: ${CMAKE_COMMAND}") - execute_process(COMMAND ${CMAKE_COMMAND} -DBUILD_PIC=ON -G "${CMAKE_GENERATOR}" . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/cpu_features-download ) - - if(result) - message(FATAL_ERROR "CMake step for cpu_features failed: ${result}") - endif() - execute_process(COMMAND ${CMAKE_COMMAND} --build . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/cpu_features-download ) - if(result) - message(FATAL_ERROR "Build step for cpu_features failed: ${result}") - endif() - - add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/cpu_features-src - ${CMAKE_CURRENT_BINARY_DIR}/cpu_features-build - EXCLUDE_FROM_ALL) - set(CPUF_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cpu_features-src) - include_directories(${CPUF_SOURCE_DIR}/include) - set(CPU_FEATURES cpu_features) - endif() -endif() - - -#arm-compute entry -if(${HELPERS_armcompute}) - find_package(ARMCOMPUTE REQUIRED) - execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE ld_version) - if ("${ld_version}" MATCHES "GNU gold") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold ") - if (CMAKE_BUILD_TYPE STREQUAL "Debug") - add_link_options("-Wl,--long-plt") - endif() - endif() - - if(ARMCOMPUTE_FOUND) - message("Found ARMCOMPUTE: ${ARMCOMPUTE_LIBRARIES}") - set(HAVE_ARMCOMPUTE 1) - # Add preprocessor definition for ARM Compute NEON - add_definitions(-DARMCOMPUTENEON_ENABLED) - include_directories(${ARMCOMPUTE_INCLUDE}) - message("----${ARMCOMPUTE_INCLUDE}---") - endif() - - -endif() - - - -# new mkl-dnn entry -if (${HELPERS_mkldnn}) - message("Going to pull & build mkldnn") - set(HAVE_MKLDNN 1) - set(DNNL_LIBRARY_TYPE "STATIC" CACHE STRING "Hack to enforce static mode" FORCE) - - configure_file(./CMakeLists.txt.mkldnn.in mkldnn-download/CMakeLists.txt) - execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-download ) - if(result) - message(FATAL_ERROR "CMake step for mkldnn failed: ${result}") - endif() - execute_process(COMMAND ${CMAKE_COMMAND} --build . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-download ) - if(result) - message(FATAL_ERROR "Build step for mkldnn failed: ${result}") - endif() - - add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/mkldnn-src - ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-build - EXCLUDE_FROM_ALL) - - set(mkldnn_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-build) - set(mkldnn_EXT_DIR ${CMAKE_CURRENT_BINARY_DIR}/mkldnn-src) - set(MKLDNN_PATH "${mkldnn_SOURCE_DIR}") - include_directories(${mkldnn_SOURCE_DIR}/include ${mkldnn_EXT_DIR}/include ${mkldnn_SOURCE_DIR}) - set(MKLDNN dnnl) -endif() - - -if (${HELPERS_cudnn}) - if (NOT SD_CUDA) - message(FATAL_ERROR "Can't build cuDNN on non-CUDA platform") - endif() - - set(CUDNN_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA cuDNN") - - SET(CUDNN_LIBNAME "cudnn") - find_path(CUDNN_INCLUDE_DIR cudnn.h - HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} - PATH_SUFFIXES cuda/include include) - - find_library(CUDNN_LIBRARY ${CUDNN_LIBNAME} - HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} - PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) - - #find_library(CULIBOS_LIBRARY ${CULIBOS_LIBNAME} - # HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} - # PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) - - - if (CUDNN_LIBRARY) - set(HAVE_CUDNN true) - set(CUDNN ${CUDNN_LIBRARY}) - else() - message(FATAL_ERROR "Unable to find cuDNN") - endif() -endif() - -# Download and unpack flatbuffers at configure time -configure_file(CMakeLists.txt.in flatbuffers-download/CMakeLists.txt) -execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-download ) -if(result) - message(FATAL_ERROR "CMake step for flatbuffers failed: ${result}") -endif() -execute_process(COMMAND ${CMAKE_COMMAND} --build . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-download ) -if(result) - message(FATAL_ERROR "Build step for flatbuffers failed: ${result}") -endif() - -# Add flatbuffers directly to our build. -add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-src - ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-build - EXCLUDE_FROM_ALL) - -set(HAVE_FLATBUFFERS 1) -set(FLATBUFFERS_PATH ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-src) -include_directories(${FLATBUFFERS_PATH}/include) - - - -configure_file(include/config.h.in include/config.h) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/include) - - -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) -add_subdirectory(blas) -if(SD_BUILD_TESTS) - # tests are always compiled with all ops included - set(SD_ALL_OPS true) - set(SD_BUILD_MINIFIER true) - add_subdirectory(tests_cpu) -endif() - - -if (MSVC_DEV) - set(SD_BUILD_MINIFIER false) -endif () - -set (CMAKE_INSTALL_PREFIX $ENV{ND4J_HOME}/nd4j-native-parent/nd4j-native/src/main/resources) - -# Set package information -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Native operations for nd4j.") -set(CPACK_PACKAGE_RELEASE 1) -set(CPACK_PACKAGE_CONTACT "agibsonccc ") -set(CPACK_PACKAGE_VENDOR "Eclipse") -set(CPACK_SETDESTDIR "false") -set(CPACK_PACKAGING_INSTALL_PREFIX "/usr/local/lib") -set(CPACK_PACKAGE_NAME "libnd4j") -set(CPACK_PACKAGE_VERSION_MAJOR "0") -set(CPACK_PACKAGE_VERSION_MINOR "8") -set(CPACK_PACKAGE_VERSION_PATCH "0") -set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") -set(CPACK_PACKAGE_INSTALL_DIRECTORY "libnd4j") -set(CPACK_RESOURCE_FILE_README "${CMAKE_CURRENT_SOURCE_DIR}/README.md") - -# Determine distribution and release — may require redhat-lsb-core installed on CentOS / RH -execute_process(COMMAND lsb_release -si OUTPUT_VARIABLE DISTRIBUTION OUTPUT_STRIP_TRAILING_WHITESPACE) -execute_process(COMMAND lsb_release -sc OUTPUT_VARIABLE RELEASE OUTPUT_STRIP_TRAILING_WHITESPACE) -execute_process(COMMAND uname -i OUTPUT_VARIABLE ARCHITECTURE) - -# Set package name and type (deb vs rpm) -if(DISTRIBUTION STREQUAL "Ubuntu") - - # Set Ubuntu-specific information (see http://www.cmake.org/Wiki/CMake:CPackPackageGenerators) - if(ARCHITECTURE MATCHES ".*x86_64.*") - set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64") - else() - set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "i386") - endif() - set(CPACK_DEBIAN_PACKAGE_MAINTAINER "raver119") - set(CPACK_DEBIAN_PACKAGE_SECTION "devel") - set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "cuda") - # For Ubuntu <= 12, libatlas3gf-base, liblapack3gf - # Build deps: libatlas3-base liblapack3 libopenblas-dev libatlas-dev liblapack-dev gcc-5 g++-5 - set(CPACK_DEBIAN_PACKAGE_DEPENDS "") - set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/eclipse/deeplearning4j") - set(CPACK_GENERATOR "DEB") - set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION}-${RELEASE}_${CPACK_DEBIAN_PACKAGE_ARCHITECTURE}) - set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/cmake/postinst;${CMAKE_CURRENT_SOURCE_DIR}/cmake/postrm;" ) - -elseif(DISTRIBUTION STREQUAL "CentOS") - - # Set Fedora-specific information (see http://www.cmake.org/Wiki/CMake:CPackPackageGenerators) - execute_process(COMMAND lsb_release -sr OUTPUT_VARIABLE RELEASE OUTPUT_STRIP_TRAILING_WHITESPACE) - if(ARCHITECTURE MATCHES ".*x86_64.*") - set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64") - else() - set(CPACK_RPM_PACKAGE_ARCHITECTURE "i686") - endif() - set(CPACK_PACKAGE_CONTACT "agibsonccc") - set(CPACK_RPM_PACKAGE_GROUP "Development/Tools") - set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") - set(CPACK_RPM_PACKAGE_SUGGESTS "cuda") - # Build deps: atlas blas lapack cmake3 devtoolset-4-gcc devtoolset-4-gcc-c++ - set(CPACK_RPM_PACKAGE_REQUIRES "") - set(CPACK_RPM_PACKAGE_URL "https://github.com/eclipse/deeplearning4j/libnd4j") - set(CPACK_GENERATOR "RPM") - set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}.fc${RELEASE}.${CPACK_RPM_PACKAGE_ARCHITECTURE}) - set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/postinst") - set(CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/postrm") - set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/usr/local/lib") - -endif() - -include(CPack) diff --git a/libnd4j/CMakeLists.txt.cpu_features.in b/libnd4j/CMakeLists.txt.cpu_features.in deleted file mode 100644 index da1d6ebda..000000000 --- a/libnd4j/CMakeLists.txt.cpu_features.in +++ /dev/null @@ -1,16 +0,0 @@ -cmake_minimum_required(VERSION 2.8.2) - -project(mkldnn-download NONE) - -include(ExternalProject) -ExternalProject_Add(mkldnn - GIT_REPOSITORY https://github.com/google/cpu_features.git - GIT_TAG v0.4.1 - SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/cpu_features-src" - BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/cpu_features-build" - CONFIGURE_COMMAND "" - CMAKE_ARGS "-DBUILD_PIC=ON" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" -) diff --git a/libnd4j/CMakeLists.txt.in b/libnd4j/CMakeLists.txt.in deleted file mode 100644 index 8e8741c86..000000000 --- a/libnd4j/CMakeLists.txt.in +++ /dev/null @@ -1,16 +0,0 @@ -cmake_minimum_required(VERSION 2.8.2) - -project(flatbuffers-download NONE) - -include(ExternalProject) -ExternalProject_Add(flatbuffers - GIT_REPOSITORY https://github.com/google/flatbuffers.git - GIT_TAG v1.10.0 - SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-src" - BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-build" - CONFIGURE_COMMAND "" - CMAKE_ARGS "-DFLATBUFFERS_BUILD_FLATC=OFF" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" -) diff --git a/libnd4j/CMakeSettings.json b/libnd4j/CMakeSettings.json deleted file mode 100644 index fe7790fa0..000000000 --- a/libnd4j/CMakeSettings.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "configurations": [ - { - "name": "x64-Debug", - "generator": "Ninja", - "configurationType": "Debug", - "inheritEnvironments": [ - "msvc_x64_x64" - ], - "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", - "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\install\\${name}", - "cmakeCommandArgs": " -DSD_CUDA=true -DLIBND4J_NAME=nd4jcuda -DMSVC_DEV=true -DCOMPUTE=61 -DBUILD_TESTS=true", - "buildCommandArgs": "-v", - "ctestCommandArgs": "" - }, - { - "name": "WSL-GCC-Debug", - "generator": "Unix Makefiles", - "configurationType": "Debug", - "buildRoot": "${projectDir}\\out\\build\\${name}", - "installRoot": "${projectDir}\\out\\install\\${name}", - "cmakeExecutable": "/usr/bin/cmake", - "cmakeCommandArgs": "-DSD_ALL_OPS=true -DCMAKE_BUILD_TYPE=Debug -DSD_CPU=true -DLIBND4J_NAME=nd4jcpu -DBUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DOPENBLAS_PATH=/usr/lib/openblas-base/ -DEXTENSION=avx2 ", - "buildCommandArgs": "-j 4", - "ctestCommandArgs": "", - "inheritEnvironments": [ "linux_x64" ], - "wslPath": "${defaultWSLPath}", - "addressSanitizerRuntimeFlags": "detect_leaks=0", - "variables": [] - } - ] -} \ No newline at end of file diff --git a/libnd4j/RaspberryPi.md b/libnd4j/RaspberryPi.md deleted file mode 100644 index 07c0574da..000000000 --- a/libnd4j/RaspberryPi.md +++ /dev/null @@ -1,60 +0,0 @@ - -### Cross compiling for rapsberry pi and android on linux - -`bash pi_build.sh` using this helper script one can cross build libnd4j and dl4j with **arm COMPUTE LIBRARY** . it will download cross compiler and arm compute library. - - -|options | value | description -|--|--|--| -| -a or --arch | arm32 | cross compiles for pi/linux 32bit -| -a or --arch | arm64 | cross compiles for pi/linux 64bit -| -a or --arch | android-arm | cross compiles for android 32bit -| -a or --arch | android-arm64 | cross compiles for android 64bit -|-m or --mvn | | if provided will build dl4j using maven - -example: -`bash pi_build.sh --arch android-arm64 --mvn` - -to change version of the **arm COMPUTE LIBRARY** modify this line in the script - ``` - ARMCOMPUTE_TAG=v20.05 - ``` - - -##### old one - -Please follow following instructions to build nd4j for raspberry PI: - -1. download cross compilation tools for Raspberry PI - - ``` - $ apt-get/yum install git cmake - (You may substitute any path you prefer instead of $HOME/raspberrypi in the following two steps) - $ mkdir $HOME/raspberrypi - $ export RPI_HOME=$HOME/raspberrypi - $ cd $RPI_HOME - $ git clone git://github.com/raspberrypi/tools.git - $ export PATH=$PATH:$RPI_HOME/tools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/bin - ``` - -2. download deeplearning4j: - - ``` - $ cd $HOME - $ git clone https://github.com/eclipse/deeplearning4j.git - ``` - -3. build libnd4j: - - ``` - $ cd deeplearning4j/libnd4j - $ ./buildnativeoperations.sh -o linux-armhf - ``` - -4. build nd4j - - ``` - $ export LIBND4J_HOME= - $ cd $HOME/deeplearning4j/nd4j - $ mvn clean install -Djavacpp.platform=linux-armhf -Djavacpp.platform.compiler=$HOME/raspberrypi/tools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/bin/arm-linux-gnueabihf-g++ -DskipTests -Dmaven.javadoc.skip=true -pl '!:nd4j-cuda-9.1,!:nd4j-cuda-9.1-platform,!:nd4j-tests' - ``` diff --git a/libnd4j/UnderstandingGraph.md b/libnd4j/UnderstandingGraph.md deleted file mode 100644 index d1c51b428..000000000 --- a/libnd4j/UnderstandingGraph.md +++ /dev/null @@ -1,173 +0,0 @@ -# Graph - -### Basic idea -libnd4j contains Directed Acyclic Graph execution engine, suited for both local and remote execution. However, main goal here is execution of externally originated graphs, serialized into FlatBuffers and provided either via pointer, or file. - - -This basic example shows execution of graph loaded from file: -```c++ -auto graph = GraphExecutioner::importFromFlatBuffers("./some_file.fb"); -GraphExecutioner::execute(graph); -// ... do something with results ... -delete graph; -``` - -### FlatBuffers schemas -You can find scheme files [here](https://github.com/eclipse/deeplearning4j/tree/master/libnd4j/include/graph/scheme). - -At this moment libnd4j repo contains compiled definitions for C++, Python, Java, and JSON, but FlatBuffers can be compiled for PHP, C#, JavaScript, TypeScript and Go as well. Please refer to `flatc` instructions to do that. - -Such bindings allow you to build FlatBuffers files/buffers suitable for remote execution of your graph and obtaining results back. I.e. you can use JavaScript to build graph (or just update variables/placeholders), send them to remote RPC server powered by libnd4j, and get results back. - -### Graph execution logic -No matter how graph is represented on the front-end, on backend it's rather simple: topologically sorted list of operations executed sequentially if there's shared dependencies, or (optionally) in parallel, if there's no shared dependencies for current graph nodes. - -Each node in the graph represents single linear algebra operation applied to input(s) of the node. For example: `z = Add(x, y)` is operation that takes 2 NDArrays as input, and produes 1 NDArray as output. So, graph is built of such primitive operations, which are executed sequentially. - -### Memory management within graph -Everything that happens within graph during execution, stays within VariableSpace. It acts as storage for Variables and NDArrays produced during graph execution. On top of that, there's an option to use pre-allocated Workspaces for allocation of NDArrays. - - -### Current graph limitations -There are some limitations. Some of them will be lifted eventually, others won't be. Here's the list: -- Graph has single data type. I.e. Graph<float> or Graph<float16> or Graph<double> _This limitation will be lifted soon._ -- On some platforms, like Java, single Variable/Placeholder size is limited to 2GB buffer size. However, on libnd4j side there's no such limitation. -- Variable size/dimensionality has limitations: max NDArray rank is limited to 32 at this moment, and any single dimension is limited to MAX_INT size. -- Recursion isn't directly supported at this moment. -- CUDA isn't supported at this moment. _This limitation will be lifted soon._ -- When used from C++, Graph only supports FeedForward mode. _This limitation will be lifted soon._ - -### Minified Graph binaries -There's an option to build minified binaries suited for execution of ***specific graphs***. Idea is quite simple: you feed your existing Graph(s) in FlatBuffers format into special app, which extracts operations used in your Graph(s) and excludes all other operations from target binary. -```bash -# building full libnd4j copy AND minfier app -./buildnativeoperations.sh -a native -m -... -# building libnd4j for 2 specific graphs -./minifier -l -a native -o libnd4j_special ../some_path/some_graph1.fb ../some_path/some_graph2.fb -Option 'l': Build library -Option 'a': Target arch: native -Option 'o': Output file name is libnd4j_special -Total available operations: 423 - -Retrieving ops from the Graph and collect them... - -Collecting out Scopes... -Operations found so far: -rank -range -subtract -transpose -matmul -biasadd -TRANSFORM{15} - -Building minified library... -``` - -Once `minifier` finishes - you'll have `libnd4j_special.so` and `libnd4j_special.h` files ready, and they'll contain only those operations used in 2 graphs provided at compilation time + basic primitives used to work with Graph. Things like NDArray, GraphExecutioner etc will be included as well. - -This library can be used in your application as any other shared libray out there: you'll include headers file and you'll be able to call for things you need. - -### Documentation -Documentation for individual operations, and basic classes (like NDArray, Graph etc) is available as part of Nd4j javadoc: https://nd4j.org/doc/ - -### Embedded profiling -If you're adding new ops, and want to make sure they run ok on your specific device - you might want to give a shot to embedded Graph profiling helper. -Despite being simple - it still provides you with time spent in various parts of Graph. - -```c++ -Environment::getInstance().setProfiling(true); -auto graph = GraphExecutioner::importFromFlatBuffers("./resources/ae_00.fb"); - -auto profile = GraphProfilingHelper::profile(graph, 1000); -profile->printOut(); - -delete graph; -``` - -1000 iterations laterm you'll get statistics printed out. Statistics basically includes time spent in various parts of code and memory allocation details. - -Here's how it'll look like: -``` -Printing out Graph... -8. matmul; Inputs: [{1:0}, {2:0}]; -9. biasadd; Inputs: [{8:0}, {3:0}]; -10. TRANSFORM:{15}; Inputs: [{9:0}]; -11. rank; Inputs: [{2:0}]; -12. subtract; Inputs: [{11:0}, {4:0}]; -13. range; Inputs: [{5:0}, {11:0}, {6:0}]; -14. subtract; Inputs: [{12:0}, {13:0}]; -15. transpose; Inputs: [{2:0}, {14:0}]; -16. matmul; Inputs: [{10:0}, {15:0}]; -17. biasadd; Inputs: [{16:0}, {7:0}]; -18. TRANSFORM:{15}; Inputs: [{17:0}]; - -Printing out Scopes... -Graph profile: 1000 executions - -Memory: -ACT: 0; TMP: 0; OBJ: 0; TTL: 1788; - -Time: -Construction time: 2135 ns; -Execution time: 41820 ns; - -Per-node reports: -Node: <8:MatMul> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 200; - Time: PREP: 1160 ns; EXEC: 3167 ns; TTL: 5929 ns; - PREP: INPUT: 251 ns; SHAPE: 382 ns; ARRAY: 217 ns; -Node: <9:BiasAdd> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 104; - Time: PREP: 917 ns; EXEC: 3580 ns; TTL: 5957 ns; - PREP: INPUT: 220 ns; SHAPE: 213 ns; ARRAY: 217 ns; -Node: <10:Tanh> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 104; - Time: PREP: 756 ns; EXEC: 241 ns; TTL: 1927 ns; - PREP: INPUT: 140 ns; SHAPE: 195 ns; ARRAY: 205 ns; -Node: <11:transpose/Rank> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 36; - Time: PREP: 522 ns; EXEC: 119 ns; TTL: 1403 ns; - PREP: INPUT: 109 ns; SHAPE: 69 ns; ARRAY: 171 ns; -Node: <12:transpose/sub> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 36; - Time: PREP: 666 ns; EXEC: 185 ns; TTL: 1684 ns; - PREP: INPUT: 192 ns; SHAPE: 94 ns; ARRAY: 168 ns; -Node: <13:transpose/Range> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 556; - Time: PREP: 808 ns; EXEC: 647 ns; TTL: 2416 ns; - PREP: INPUT: 297 ns; SHAPE: 228 ns; ARRAY: 181 ns; -Node: <14:transpose/sub_1> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 56; - Time: PREP: 721 ns; EXEC: 541 ns; TTL: 2205 ns; - PREP: INPUT: 23 ns; SHAPE: 92 ns; ARRAY: 165 ns; -Node: <15:transpose> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 96; - Time: PREP: 3936 ns; EXEC: 602 ns; TTL: 5811 ns; - PREP: INPUT: 194 ns; SHAPE: 3241 ns; ARRAY: 257 ns; -Node: <16:MatMul_1> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 312; - Time: PREP: 970 ns; EXEC: 3565 ns; TTL: 6066 ns; - PREP: INPUT: 203 ns; SHAPE: 320 ns; ARRAY: 193 ns; -Node: <17:BiasAdd_1> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 144; - Time: PREP: 914 ns; EXEC: 3528 ns; TTL: 5870 ns; - PREP: INPUT: 231 ns; SHAPE: 191 ns; ARRAY: 223 ns; -Node: <18:output> - Memory: ACT: 0; TMP: 0; OBJ: 0; TTL: 144; - Time: PREP: 805 ns; EXEC: 285 ns; TTL: 1928 ns; - PREP: INPUT: 157 ns; SHAPE: 192 ns; ARRAY: 232 ns; - -Special timers: -No special timers were set -``` - - -### Roadmap -In short-to-medium term following improvements are expected: -- CUDA support for all new ops -- Additional data types support: int, long long, q types, bool -- Sparse tensors support - - diff --git a/libnd4j/assembly-cuda.xml b/libnd4j/assembly-cuda.xml deleted file mode 100644 index c1f6d89ae..000000000 --- a/libnd4j/assembly-cuda.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - ${libnd4j.platform}-cuda-${cuda.version} - - zip - - libnd4j - - - ${project.basedir}/ - - true - - **/target/** - **/CMakeFiles/** - **/CMakeCache.txt - %regex[(?!.*cuda/).*blasbuild.*] - %regex[.*/lib/googletest.*] - - - - - diff --git a/libnd4j/assembly.xml b/libnd4j/assembly.xml deleted file mode 100644 index 42f429fb6..000000000 --- a/libnd4j/assembly.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - ${libnd4j.classifier} - - zip - - libnd4j - - - ${project.basedir}/ - - true - - **/target/** - **/CMakeFiles/** - **/CMakeCache.txt - %regex[(?!.*${libnd4j.chip}/).*blasbuild.*] - %regex[.*/lib/googletest.*] - - - - - diff --git a/libnd4j/auto_vectorization/AutoVectorization.md b/libnd4j/auto_vectorization/AutoVectorization.md deleted file mode 100644 index 44da56665..000000000 --- a/libnd4j/auto_vectorization/AutoVectorization.md +++ /dev/null @@ -1,77 +0,0 @@ - -# Auto-vectorization Report - -This report tool is used to get a human-friendly compiler output of the auto-vectorization process. It is intended for developers to help them to investigate the obstacles that compiler faced during auto-vectorization. - -## Usage -```--check-vectorization``` option should be added to the **release** build to be able to get the auto-vectorization report -```./buildnativeoperations.sh -a native -j 28 --check-vectorization``` -it will output ```vecmiss.html``` inside blasbuild/cpu folder. - -For the direct usage: -`compile command | python3 auto_vect.py` -Also please note that to use it with `parallel make` one should add `--output-sync=target` - -## Report Format -Each filename contains info about optimization attempts for the source code lines. -Each line number is also expandable (⇲) and contains distinct failure notes. -It is possible to click on the line number to see source code - -| file name | total successful attempts | total failed attempts | ⇲ | -|---|---|---|--| -| line number | successful attempts | failed attempts | ⇲ | -|- failure reasons | -| line number | successful attempts | failed attempts |⇲ | - -##### Requirements -- GCC (Currently, only GCC is supported) -- python3 - -##### Adding new compiler support for the stdin message parsing -To add new compiler for the stdin processing one should add entry in `STDIN_COMPILER_ENTRY` for that compiler with the following syntax - - { 'compiler_name' : [('comparision', 'version with dot delimiter', 'entry_name') , other version and etc] } - example: STDIN_COMPILER_ENTRY = { 'gcc' : [('<','9','gcc_old'),...] ,...} - - The next step to add a parser for the entry in `STDIN_PARSERS` - ` STDIN_PARSERS = { 'gcc_old' : parser_method }` - the signature of the parser function is: - `Parse_info parser_method(line, helper_storage)` -- the line is a compiler output that needs to be parsed. -- helper_storage is a dict and can be used as a state storage to parse multi-line and et cetera, as parser called for each line. -- Please note that Parse_info members should be the same with those which were defined in `general_stdin_parser local_parser` - -to simplify adding compiler, especially, for those which outputs message details in one line, there is the helper method `general_stdin_parser("succes hint in the message", "failure hint in the message", (file, line, message) extractor regex pattern)`: - - example: general_stdin_parser("vectorized loop", "unvectorized loop", r'[^/]+([^,]+)\,\s*line\s*(\d+)\:(.*)') - - -### Detailed report with `-fsave-optimization-record` option: -If you want to get more detailed information (for now it reports the functions of failures) you should use new version of the toolchain (GCC > 9). As the new version of GCC compilers have `-fsave-optimization-record` option. -`buildnativeoperations.sh` using CMake will detect it and switch to the more detailed version. -Please, note that this option is still experimental and so the compiler can fail to output some json.gz file with error. -On that case try to exclude those files from the build. -And also the internal structure of the `-fsave-optimization-record` json.gz can be changed in future. - -It outputs two files **vecmiss_fsave.html** and **vecmiss_fsave.html.js**. So to see report details you need to enable javascript on browser if it was disabled. - -There is also `--inverted-file` option to generate inverted index for optimization messages in json format **vecmiss_fsave_inverted_index.json**. -`inverted_index.py` script contains methods to work with those generated json outputs. For now one can get postings for optimization messages and filter those message based on file index and function index. File and function index can be obtained using the methods with a predicate filter . - - message : [ file_index, line_position, [ compressed list of function index] ] -#### Requirements for the Detailed report -- GCC version > 9 -- python3 -- Cython (python3) -- json (python3) -- gzip (python3) -- c++filt - -##### Some internal notes for `-fsave-optimization-record` output format handling -Internally, we are using Cython to speed up json.gz file processing (bigGzipJson.pyx). Because json.gz files can take big memory in raw when loaded in whole. - -If you want to use bigGzipJson outside `buildnativeoperations.sh` and CMake then you should compile it manually using this command in auto_vectorization folder: -`python3 cython_setup.py build_ext --inplace` - -json.gz files could be processed outside of `buildnativeoperations.sh`. -You need to call `python3 auto_vect.py --fsave` inside base source folder and where json.gz files exist. diff --git a/libnd4j/auto_vectorization/auto_vect.py b/libnd4j/auto_vectorization/auto_vect.py deleted file mode 100644 index 743c8f270..000000000 --- a/libnd4j/auto_vectorization/auto_vect.py +++ /dev/null @@ -1,732 +0,0 @@ -''' -@author : Abdelrauf rauf@konduit.ai -''' -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ - -import argparse -import sys -import re -import os -import subprocess -import fnmatch -import json -import gzip -import argparse - -try: - from bigGzipJson import json_gzip_extract_objects -except ImportError: - pass -from pathlib import Path -from multiprocessing import Pool, Manager ,cpu_count -import traceback -import html - -# compiler_name :[ (check_operation, version, name_entry), ..] -# positions playes role as checking will stop if it finds non empty entry -STDIN_COMPILER_ENTRY = { 'gcc' : [('<','9','gcc_old')], 'g++' : [('<','9','gcc_old'), ('t','_','')],'nc++' :[('t','_', 'ncxx')] } - -# FSAVE_SUPPORT compiler_name : (check_operation, version ) True or False -# if you want to make it false for all just put 'f' and 't' for true case -FSAVE_SUPPORT = { 'gcc' : ('>=','9'), 'g++' : ('>=','9'), 'nc++' : ('f','_')} - -stdin_parser = None -HAS_FSAVE = False - -FALLBACK_TO_FSAVE_FILES = True -FSAVE_INVERTED_INDEX = False - -number_replace = re.compile(r"(\d+)?\.?(\d+)?_?\d+\.?(\d+)?") -cmake_build_progress = re.compile(r"\s{0,4}\[\s{0,2}\d+\%\]") - -internal_match = 'deeplearning4j'+os.path.sep+'libnd4j'+os.path.sep -internal_match_replace = "./" - -BASE_URL = '' - -FSAVE_IGNORE_EXTERNALS = True -FSAVE_SHOW_SUCCESSFULS = True - -def general_stdin_parser(std_success_msg, std_fail_msg , std_line_regex_str): - ''' - General Parser from success and error message and line regex extractor - Parameters: - std_line_regex_str: it should match group(1) to file, group(2) to line_number and group(3) to message - ''' - matcher = re.compile(std_line_regex_str) - def local_parser(line, helper_storage): - #for generic we will parsing stdin input line by line - #so we dont need any storage - parse_info = ParseInfo() - x = matcher.match(line) - parse_info.external_source = True - if x: - #print(line) - file_name =x.group(1).strip() - ppos = file_name.find(internal_match) - if ppos>=0: - file_name = internal_match_replace + file_name[ppos+len(internal_match):] - parse_info.external_source = False - parse_info.line_pos = int(x.group(2)) - msg = x.group(3).lower().strip() - parse_info.file_name = file_name - if std_fail_msg in msg: - msg = number_replace.sub("_numb",msg.replace(std_fail_msg,"fail:")) - parse_info.msg = msg.strip() - parse_info.miss = 1 - parse_info.success = 0 - #print(parse_info.__dict__) - return parse_info - elif std_success_msg in msg: - parse_info.msg = msg.strip() - parse_info.miss = 0 - parse_info.success = 1 - #print(parse_info.__dict__) - return parse_info - return None - - return local_parser - - -# entry: parser list for compilers that can parse compilers output and return Parse_info -# the signature of the parser function is `Parse_info parser_function_name_(line, helper_storage)` -# Please note that Parse_info members should be the same as we defined in `general_stdin_parser local_parser` -# the line is a compiler output. helper_storage is a dict and can be used as a state storage -# to parse multi-line and et cetera, as parser called for each line. -STDIN_PARSERS = { 'gcc_old' : general_stdin_parser('loop vectorized', 'note: not vectorized:', r"[^/]*([^:]+)\:(\d+)\:\d+\:(.*)" ), - 'ncxx' : general_stdin_parser("vectorized loop", "unvectorized loop", r'[^/]+([^,]+)\,\s*line\s*(\d+)\:(.*)') -} - - - -def version_check( version1, version2, op='>='): - op_list = {"<": (lambda x,y: x": (lambda x,y: x>y), ">=": (lambda x,y: x>=y), - 'f': (lambda x,y: False),'t': (lambda x,y: True) - - } - return op_list[op](version1.split('.'),version2.split('.')) - - -def init_global_options(args): - global stdin_parser - global HAS_FSAVE - global BASE_URL - global FSAVE_INVERTED_INDEX - - FSAVE_INVERTED_INDEX = args.inverted_index - BASE_URL = args.base_url - if BASE_URL.endswith("/")==False: - BASE_URL = BASE_URL + "/" - - entry_name = '' - - if args.compiler in STDIN_COMPILER_ENTRY: - for x in STDIN_COMPILER_ENTRY[args.compiler]: - ret = version_check(args.compiler_version,x[1],x[0]) - if ret == True: - entry_name = x[2] - break - - if len(entry_name)>0: - stdin_parser = STDIN_PARSERS[entry_name] - if args.compiler in FSAVE_SUPPORT: - x = FSAVE_SUPPORT[args.compiler] - HAS_FSAVE = version_check(args.compiler_version,x[1],x[0]) - -class info: - def __repr__(self): - return str(self.__dict__) - - - -def get_cxx_filt_result(strx): - if len(strx)<1: - return "" - res = subprocess.Popen(["c++filt","-i", strx], stdout=subprocess.PIPE).communicate()[0] - res =res.decode('utf-8') - #replace some long names to reduce size - res = res.replace("unsigned long long", "uLL") - res = res.replace("unsigned long int","uL") - res = res.replace("unsigned long", "uL") - res = res.replace("unsigned int", "ui") - res = res.replace("unsigned char", "uchar") - res = res.replace("unsigned short", "ushort") - res = res.replace("long long", "LL") - res = res.replace(", ",",") - return res.strip() - - -def internal_glob(dir, match): - listx = [] - for root, dirnames, filenames in os.walk(dir): - for filename in fnmatch.filter(filenames, match): - listx.append(os.path.join(root, filename)) - return listx - -def get_obj_json_gz(filename): - with gzip.GzipFile(filename, 'r') as f: - return json.loads(f.read().decode('utf-8'))[-1] - - -class ParseInfo: - pass - - -class File_Info: - ''' - Holds information about vectorized and miss vectorized lines for one file - ''' - - def __init__(self): - self.infos = {} - self.total_opted =0 - self.total_missed = 0 - self.external = False - - - def add_line(self, line_pos): - if line_pos not in self.infos: - v = info() - v.optimized = 0 - v.missed = 0 - v.miss_details = set() - self.infos[line_pos] = v - return v - else: - return self.infos[line_pos] - - - def add_line_fsave(self, line_pos): - if line_pos not in self.infos: - v = info() - v.optimized = 0 - v.missed = 0 - v.miss_details2 = dict() - self.infos[line_pos] = v - return v - else: - return self.infos[line_pos] - - - - def add_fsave(self, line_pos,success, msg, function ,inline_fns=''): - v = self.add_line_fsave(line_pos) - if success and "loop vectorized" in msg: - v.optimized +=1 - self.total_opted +=1 - if FSAVE_SHOW_SUCCESSFULS==True: - if "success" in v.miss_details2: - ls = v.miss_details2.get("success") - ls.add(function) - else: - ls =set() - v.miss_details2["success"]=ls - ls.add(function) - elif success==False and "not vectorized:" in msg: - #reduce this msg - msg = msg.replace("not vectorized:","").strip() - v.missed +=1 - self.total_missed +=1 - msg = sys.intern(msg) - if msg in v.miss_details2: - ls = v.miss_details2.get(msg) - ls.add(function) - else: - ls =set() - v.miss_details2[msg]=ls - ls.add(function) - return self - - def add(self, line_pos, msg, success, missed): - v = self.add_line(line_pos) - if msg is not None: - v.optimized += success - v.missed += missed - self.total_opted += success - self.total_missed += missed - if msg is not None: - v.miss_details.add(msg) - return self - - - def __repr__(self): - return str(self.__dict__) - - - - -def process_gzip_json_mp(args): - process_gzip_json_new(*args) - -def process_gzip_json_new(json_gz_fname,list_Queue): - gz_name = Path(json_gz_fname).stem - #print("::--open and process {0}".format(gz_name)) - queue_count = len(list_Queue) - #print(queue_count) - q = list_Queue[0] - old_fname = '' - total_c = 0 - for x in json_gzip_extract_objects(json_gz_fname,'message','vectorized'): - external_source = True - if len(x['message'])>0 and 'location' in x: - line = int(x['location']['line']) - file_name = x['location']['file'].strip() - ppos = file_name.find(internal_match) - if ppos>=0: - file_name = internal_match_replace + file_name[ppos+len(internal_match):] - external_source = False - msg = x['message'][0] - success = x['kind'] == 'success' - func = '' if 'function' not in x else x['function'] - - if file_name!=old_fname: - #send our info to the right consumer - queue_ind = hash(file_name) % queue_count - #print("quen index {0}".format(queue_ind)) - q =list_Queue[queue_ind] - old_fname = file_name - total_c +=1 - #print("pp {0} {1}".format(q,(file_name,line,success, msg, func,external_source ))) - if FSAVE_IGNORE_EXTERNALS==True and external_source == True: - continue - q.put((file_name,line,success, msg, func,external_source )) - print("::finished {0:60s} :{1:8d}".format(gz_name,total_c)) - -def consume_processed_mp(args): - return consume_processed_new(*args) - - - -def consume_processed_new(list_Queue , c_index): - - info_ = dict() - func_list = dict() - last_func_index = 0 - q = list_Queue[c_index] - print("::consumer {0}".format(c_index)) - total_c = 0 - r_c = 0 - while True: - #print("try to get new from {0}".format(index)) - obj = q.get() - #print("cc {0} {1}".format(q,obj)) - if obj==None: - break #we received the end - file_name,line,success, msg, func, external_source = obj - try: - #get function index - func_index = -1 - if func in func_list: - func_index = func_list[func] - else: - func_list[func] = last_func_index - func_index = last_func_index - last_func_index +=1 - - if file_name in info_: - info_[file_name].add_fsave(line, success, msg, func_index) - else: - info_[file_name] = File_Info().add_fsave(line, success, msg, func_index) - info_[file_name].external = external_source - total_c +=1 - if total_c - r_c >10000: - r_c = total_c - print("::consumer {0:2d} :{1:10d}".format(c_index,total_c)) - except Exception as e: - print(traceback.format_exc()) - break - - print("::consumer {0:2d} :{1:10d}".format(c_index,total_c)) - #write to temp file - wr_fname= "vecmiss_fsave{0}.html".format(str(c_index) if len(list_Queue)>1 else '') - print("generate report for consumer {0} {1}".format(c_index,len(info_))) - try: - uniq_ind = str(c_index)+'_' if len(list_Queue)>1 else '' - wr = generate_report(wr_fname,info_ ,only_body = False, unique_id_prefix = uniq_ind,fsave_format = True, function_list= func_list) - print(" consumer {0} saved output into {1}".format(c_index, wr)) - except Exception as e: - print(traceback.format_exc()) - - - -def obtain_info_from(input_): - info_ = dict() - parser_storage = dict() #can be used for parsing multi-lines - if HAS_FSAVE ==True or stdin_parser is None: - #just print progress - for line in input_: - if cmake_build_progress.match(line): - #actually we redirect only, stderr so this should not happen - print("__"+line.strip()) - elif "error" in line or "Error" in line: - print("****"+line.strip()) - return info_ - for line in input_: - x = stdin_parser(line, parser_storage) - if x is not None: - if x.file_name in info_: - #ignore col_number - info_[x.file_name].add(x.line_pos, x.msg, x.success, x.miss) - info_[x.file_name].external = x.external_source - else: - info_[x.file_name] = File_Info().add(x.line_pos, x.msg, x.success, x.miss) - info_[x.file_name].external = x.external_source - elif cmake_build_progress.match(line): - #actually we redirect only, stderr so this should not happen - print("__"+line.strip()) - elif "error" in line or "Error" in line: - print("****"+line.strip()) - return info_ - - - -def custom_style(fsave): - st = '''''' - -def header(fsave=False): - strx ='\n\n\n\nAuto-Vectorization\n' - strx +=''.format(BASE_URL) - strx +=custom_style(fsave) - strx +='\n\n\n' - return strx - -def footer(): - return '\n' - - - -def get_compressed_indices_list(set_a): - new_list = sorted(list(set_a)) - for i in range(len(new_list)-1,0,-1): - new_list[i] = new_list[i] - new_list[i-1] - return new_list - -def get_compressed_indices(set_a): - a_len = len(set_a) - if a_len<=1: - if a_len<1: - return '' - return str(set_a)[1:-1] - #we sorted and only saved difference - # 1,14,15,19 --> 1,13,1,4 10bytes=>8bytes - list_sorted = sorted(list(set_a)) - last = list_sorted[0] - str_x = str(list_sorted[0]) - for i in range(1,a_len): - str_x += ','+str(list_sorted[i]-last) - last = list_sorted[i] - return str_x - - - - - -def get_content(k, v, unique_id_prefix = '', fsave_format=False): - inner_str='' - content = '' - inc_id = 0 - for fk,fv in sorted(v.infos.items()): - if fsave_format==True: - inner_str+='
{1}
{2}